From 283d441c19d5bafa01a7df24db277a6b08a86c00 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Sun, 30 Aug 2020 10:35:18 +1000 Subject: [PATCH 001/210] zig fmt: fix #3978, fix #2748 --- lib/std/zig/parser_test.zig | 53 +++++++++++++++++++++++++++++++++++++ lib/std/zig/render.zig | 15 ++++++++--- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 36ceb400dc..1aec1c3567 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3321,6 +3321,59 @@ test "zig fmt: Don't add extra newline after if" { ); } +test "zig fmt: comments in ternary ifs" { + try testCanonical( + \\const x = if (true) { + \\ 1; + \\} else if (false) + \\ // Comment + \\ 0; + \\const y = if (true) + \\ // Comment + \\ 1 + \\else + \\ 0; + \\ + \\pub extern "c" fn printf(format: [*:0]const u8, ...) c_int; + \\ + ); +} + +test "zig fmt: test comments in field access chain" { + try testCanonical( + \\pub const str = struct { + \\ pub const Thing = more.more // + \\ .more() // + \\ .more().more() // + \\ .more() // + \\ // .more() // + \\ .more() // + \\ .more(); + \\ data: Data, + \\}; + \\ + \\pub const str = struct { + \\ pub const Thing = more.more // + \\ .more() // + \\ // .more() // + \\ // .more() // + \\ // .more() // + \\ .more() // + \\ .more(); + \\ data: Data, + \\}; + \\ + \\pub const str = struct { + \\ pub const Thing = more // + \\ .more // + \\ .more() // + \\ .more(); + \\ data: Data, + \\}; + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 237ca07d2b..4432d08787 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -522,8 +522,12 @@ fn renderExpression( break :blk if (loc.line == 0) op_space else Space.Newline; }; - try renderToken(tree, ais, infix_op_node.op_token, after_op_space); - ais.pushIndentOneShot(); + { + try ais.pushIndent(); + defer ais.popIndent(); + try renderToken(tree, ais, infix_op_node.op_token, after_op_space); + } + try ais.pushIndentOneShot(); return renderExpression(allocator, ais, tree, infix_op_node.rhs, space); }, @@ -1873,7 +1877,12 @@ fn renderExpression( if (src_has_newline) { const after_rparen_space = if (if_node.payload == null) Space.Newline else Space.Space; - try renderToken(tree, ais, rparen, after_rparen_space); // ) + + { + try ais.pushIndent(); + defer ais.popIndent(); + try renderToken(tree, ais, rparen, after_rparen_space); // ) + } if (if_node.payload) |payload| { try renderExpression(allocator, ais, tree, payload, Space.Newline); From 601331833a148ff3a1ab5cb4bba8bc63f4850e13 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Sun, 30 Aug 2020 10:34:44 +1000 Subject: [PATCH 002/210] Add passing test. close #5343 --- lib/std/zig/parser_test.zig | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 1aec1c3567..6b8734c9d4 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3374,6 +3374,24 @@ test "zig fmt: test comments in field access chain" { ); } +test "zig fmt: Indent comma correctly after multiline string literals in arg list (trailing comma)" { + try testCanonical( + \\fn foo() void { + \\ z.display_message_dialog( + \\ *const [323:0]u8, + \\ \\Message Text + \\ \\------------ + \\ \\xxxxxxxxxxxx + \\ \\xxxxxxxxxxxx + \\ , + \\ g.GtkMessageType.GTK_MESSAGE_WARNING, + \\ null, + \\ ); + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; From 503ba7b27c6e8e248d271aec936623853cd8fcd1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 3 Sep 2020 20:23:00 -0700 Subject: [PATCH 003/210] start moving `zig cc` to stage2 * build.zig: repair the ability to link against llvm, clang, and lld * move the zig cc arg parsing logic to stage2 - the preprocessor flag is still TODO - the clang arg iterator code is improved to use slices instead of raw pointers because it no longer has to deal with an extern struct. * clean up error printing with a `fatal` function and use log API for messages rather than std.debug.print * add support for more CLI options to stage2 & update usage text - hooking up most of these new options is TODO * clean up the way libc and libc++ are detected via command line options. target information is used to determine if any of the libc candidate names are chosen. * add native library directory detection * implement the ability to invoke clang from stage2 * introduce a build_options.have_llvm so we can comptime branch on whether LLVM is linked in or not. --- build.zig | 91 +-- src-self-hosted/clang_options.zig | 4 +- src-self-hosted/main.zig | 942 +++++++++++++++++++++++++++--- src-self-hosted/stage2.zig | 280 +-------- src/config.zig.in | 2 + src/main.cpp | 412 +------------ src/stage2.cpp | 17 +- src/stage2.h | 57 +- 8 files changed, 938 insertions(+), 867 deletions(-) diff --git a/build.zig b/build.zig index 3f7f1a9038..43b3b7eebf 100644 --- a/build.zig +++ b/build.zig @@ -9,6 +9,7 @@ const ArrayList = std.ArrayList; const io = std.io; const fs = std.fs; const InstallDirectoryOptions = std.build.InstallDirectoryOptions; +const assert = std.debug.assert; const zig_version = std.builtin.Version{ .major = 0, .minor = 6, .patch = 0 }; @@ -57,11 +58,13 @@ pub fn build(b: *Builder) !void { if (!only_install_lib_files) { var exe = b.addExecutable("zig", "src-self-hosted/main.zig"); + exe.install(); exe.setBuildMode(mode); exe.setTarget(target); test_step.dependOn(&exe.step); b.default_step.dependOn(&exe.step); + exe.addBuildOption(bool, "have_llvm", enable_llvm); if (enable_llvm) { const config_h_text = if (config_h_path_option) |config_h_path| try std.fs.cwd().readFileAlloc(b.allocator, toNativePathSep(b, config_h_path), max_config_h_bytes) @@ -73,11 +76,8 @@ pub fn build(b: *Builder) !void { try configureStage2(b, exe, ctx); } - if (!only_install_lib_files) { - exe.install(); - } const tracy = b.option([]const u8, "tracy", "Enable Tracy integration. Supply path to Tracy source"); - const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse false; + const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse enable_llvm; if (link_libc) { exe.linkLibC(); test_stage2.linkLibC(); @@ -323,17 +323,13 @@ fn configureStage2(b: *Builder, exe: anytype, ctx: Context) !void { exe.addIncludeDir("src"); exe.addIncludeDir(ctx.cmake_binary_dir); addCppLib(b, exe, ctx.cmake_binary_dir, "zig_cpp"); - if (ctx.lld_include_dir.len != 0) { - exe.addIncludeDir(ctx.lld_include_dir); + assert(ctx.lld_include_dir.len != 0); + exe.addIncludeDir(ctx.lld_include_dir); + { var it = mem.tokenize(ctx.lld_libraries, ";"); while (it.next()) |lib| { exe.addObjectFile(lib); } - } else { - addCppLib(b, exe, ctx.cmake_binary_dir, "embedded_lld_wasm"); - addCppLib(b, exe, ctx.cmake_binary_dir, "embedded_lld_elf"); - addCppLib(b, exe, ctx.cmake_binary_dir, "embedded_lld_coff"); - addCppLib(b, exe, ctx.cmake_binary_dir, "embedded_lld_lib"); } { var it = mem.tokenize(ctx.clang_libraries, ";"); @@ -343,42 +339,51 @@ fn configureStage2(b: *Builder, exe: anytype, ctx: Context) !void { } dependOnLib(b, exe, ctx.llvm); - if (exe.target.getOsTag() == .linux) { - // First we try to static link against gcc libstdc++. If that doesn't work, - // we fall back to -lc++ and cross our fingers. - addCxxKnownPath(b, ctx, exe, "libstdc++.a", "") catch |err| switch (err) { - error.RequiredLibraryNotFound => { - exe.linkSystemLibrary("c++"); - }, - else => |e| return e, - }; + // Boy, it sure would be nice to simply linkSystemLibrary("c++") and rely on zig's + // ability to provide libc++ right? Well thanks to C++ not having a stable ABI this + // will cause linker errors. It would work in the situation when `zig cc` is used to + // build LLVM, Clang, and LLD, however when depending on them as system libraries, system + // libc++ must be used. + const cross_compile = false; // TODO + if (cross_compile) { + // In this case we assume that zig cc was used to build the LLVM, Clang, LLD dependencies. + exe.linkSystemLibrary("c++"); + } else { + if (exe.target.getOsTag() == .linux) { + // First we try to static link against gcc libstdc++. If that doesn't work, + // we fall back to -lc++ and cross our fingers. + addCxxKnownPath(b, ctx, exe, "libstdc++.a", "") catch |err| switch (err) { + error.RequiredLibraryNotFound => { + exe.linkSystemLibrary("c++"); + }, + else => |e| return e, + }; - exe.linkSystemLibrary("pthread"); - } else if (exe.target.isFreeBSD()) { - try addCxxKnownPath(b, ctx, exe, "libc++.a", null); - exe.linkSystemLibrary("pthread"); - } else if (exe.target.isDarwin()) { - if (addCxxKnownPath(b, ctx, exe, "libgcc_eh.a", "")) { - // Compiler is GCC. - try addCxxKnownPath(b, ctx, exe, "libstdc++.a", null); exe.linkSystemLibrary("pthread"); - // TODO LLD cannot perform this link. - // See https://github.com/ziglang/zig/issues/1535 - exe.enableSystemLinkerHack(); - } else |err| switch (err) { - error.RequiredLibraryNotFound => { - // System compiler, not gcc. - exe.linkSystemLibrary("c++"); - }, - else => |e| return e, + } else if (exe.target.isFreeBSD()) { + try addCxxKnownPath(b, ctx, exe, "libc++.a", null); + exe.linkSystemLibrary("pthread"); + } else if (exe.target.isDarwin()) { + if (addCxxKnownPath(b, ctx, exe, "libgcc_eh.a", "")) { + // Compiler is GCC. + try addCxxKnownPath(b, ctx, exe, "libstdc++.a", null); + exe.linkSystemLibrary("pthread"); + // TODO LLD cannot perform this link. + // See https://github.com/ziglang/zig/issues/1535 + exe.enableSystemLinkerHack(); + } else |err| switch (err) { + error.RequiredLibraryNotFound => { + // System compiler, not gcc. + exe.linkSystemLibrary("c++"); + }, + else => |e| return e, + } + } + + if (ctx.dia_guids_lib.len != 0) { + exe.addObjectFile(ctx.dia_guids_lib); } } - - if (ctx.dia_guids_lib.len != 0) { - exe.addObjectFile(ctx.dia_guids_lib); - } - - exe.linkSystemLibrary("c"); } fn addCxxKnownPath( diff --git a/src-self-hosted/clang_options.zig b/src-self-hosted/clang_options.zig index 1b70c71dac..42bfecb746 100644 --- a/src-self-hosted/clang_options.zig +++ b/src-self-hosted/clang_options.zig @@ -7,9 +7,7 @@ pub const CliArg = struct { name: []const u8, syntax: Syntax, - /// TODO we're going to want to change this when we start shipping self-hosted because this causes - /// all the functions in stage2.zig to get exported. - zig_equivalent: @import("stage2.zig").ClangArgIterator.ZigEquivalent, + zig_equivalent: @import("main.zig").ClangArgIterator.ZigEquivalent, /// Prefixed by "-" pd1: bool = false, diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index fb20a09f1d..330c0468f1 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const assert = std.debug.assert; const io = std.io; const fs = std.fs; const mem = std.mem; @@ -11,6 +12,13 @@ const link = @import("link.zig"); const Package = @import("Package.zig"); const zir = @import("zir.zig"); const build_options = @import("build_options"); +const warn = std.log.warn; +const info = std.log.info; + +fn fatal(comptime format: []const u8, args: anytype) noreturn { + std.log.emerg(format, args); + process.exit(1); +} pub const max_src_size = 2 * 1024 * 1024 * 1024; // 2 GiB @@ -28,9 +36,11 @@ const usage = \\ build-exe [source] Create executable from source or object files \\ build-lib [source] Create library from source or object files \\ build-obj [source] Create object from source or assembly + \\ cc Use Zig as a drop-in C compiler + \\ c++ Use Zig as a drop-in C++ compiler + \\ env Print lib path, std path, compiler id and version \\ fmt [source] Parse file and render in canonical zig format \\ targets List available compilation targets - \\ env Print lib path, std path, compiler id and version \\ version Print version number and exit \\ zen Print zen of zig and exit \\ @@ -84,11 +94,19 @@ pub fn main() !void { const cmd = args[1]; const cmd_args = args[2..]; if (mem.eql(u8, cmd, "build-exe")) { - return buildOutputType(gpa, arena, cmd_args, .Exe); + return buildOutputType(gpa, arena, args, .{ .build = .Exe }); } else if (mem.eql(u8, cmd, "build-lib")) { - return buildOutputType(gpa, arena, cmd_args, .Lib); + return buildOutputType(gpa, arena, args, .{ .build = .Lib }); } else if (mem.eql(u8, cmd, "build-obj")) { - return buildOutputType(gpa, arena, cmd_args, .Obj); + return buildOutputType(gpa, arena, args, .{ .build = .Obj }); + } else if (mem.eql(u8, cmd, "cc")) { + return buildOutputType(gpa, arena, args, .cc); + } else if (mem.eql(u8, cmd, "c++")) { + return buildOutputType(gpa, arena, args, .cpp); + } else if (mem.eql(u8, cmd, "clang") or + mem.eql(u8, cmd, "-cc1") or mem.eql(u8, cmd, "-cc1as")) + { + return punt_to_clang(arena, args); } else if (mem.eql(u8, cmd, "fmt")) { return cmdFmt(gpa, cmd_args); } else if (mem.eql(u8, cmd, "targets")) { @@ -147,6 +165,8 @@ const usage_build_generic = \\ ReleaseFast Optimizations on, safety off \\ ReleaseSafe Optimizations on, safety on \\ ReleaseSmall Optimize for small binary, safety off + \\ -fPIC Force-enable Position Independent Code + \\ -fno-PIC Force-disable Position Independent Code \\ --dynamic Force output to be dynamically linked \\ --strip Exclude debug symbols \\ -ofmt=[mode] Override target object format @@ -158,11 +178,19 @@ const usage_build_generic = \\ macho (planned) macOS relocatables \\ hex (planned) Intel IHEX \\ raw (planned) Dump machine code directly + \\ -dirafter [dir] Add directory to AFTER include search path + \\ -isystem [dir] Add directory to SYSTEM include search path + \\ -I[dir] Add directory to include search path + \\ -D[macro]=[value] Define C [macro] to [value] (1 if [value] omitted) \\ \\Link Options: \\ -l[lib], --library [lib] Link against system library + \\ -L[d], --library-directory [d] Add a directory to the library search path + \\ -T[script] Use a custom linker script \\ --dynamic-linker [path] Set the dynamic interpreter path (usually ld.so) \\ --version [ver] Dynamic library semver + \\ -rdynamic Add all symbols to the dynamic symbol table + \\ -rpath [path] Add directory to the runtime library search path \\ \\Debug Options (Zig Compiler Development): \\ -ftime-report Print timing diagnostics @@ -181,11 +209,15 @@ const Emit = union(enum) { yes: []const u8, }; -fn buildOutputType( +pub fn buildOutputType( gpa: *Allocator, arena: *Allocator, - args: []const []const u8, - output_mode: std.builtin.OutputMode, + all_args: []const []const u8, + arg_mode: union(enum) { + build: std.builtin.OutputMode, + cc, + cpp, + }, ) !void { var color: Color = .Auto; var build_mode: std.builtin.Mode = .Debug; @@ -194,6 +226,7 @@ fn buildOutputType( var root_src_file: ?[]const u8 = null; var version: std.builtin.Version = .{ .major = 0, .minor = 0, .patch = 0 }; var strip = false; + var emit_h = true; var watch = false; var debug_tokenize = false; var debug_ast_tree = false; @@ -201,6 +234,7 @@ fn buildOutputType( var debug_link = false; var debug_ir = false; var debug_codegen = false; + var debug_cc = false; var time_report = false; var emit_bin: Emit = .yes_default_path; var emit_zir: Emit = .no; @@ -208,11 +242,57 @@ fn buildOutputType( var target_mcpu: ?[]const u8 = null; var target_dynamic_linker: ?[]const u8 = null; var target_ofmt: ?[]const u8 = null; + var output_mode: std.builtin.OutputMode = undefined; + var ensure_libc_on_non_freestanding = false; + var ensure_libcpp_on_non_freestanding = false; + var have_libc = false; + var have_libcpp = false; + var want_native_include_dirs = false; + var enable_cache: ?bool = null; + var want_pic: ?bool = null; + var want_sanitize_c: ?bool = null; + var rdynamic: bool = false; + var only_pp_or_asm = false; + var linker_script: ?[]const u8 = null; + var version_script: ?[]const u8 = null; + var disable_c_depfile = false; + var override_soname: ?[]const u8 = null; + var linker_optimization: ?[]const u8 = null; + var linker_gc_sections: ?bool = null; + var linker_allow_shlib_undefined: ?bool = null; + var linker_bind_global_refs_locally: ?bool = null; + var linker_z_nodelete = false; + var linker_z_defs = false; + var stack_size_override: u64 = 0; var system_libs = std.ArrayList([]const u8).init(gpa); defer system_libs.deinit(); - { + var clang_argv = std.ArrayList([]const u8).init(gpa); + defer clang_argv.deinit(); + + var lib_dirs = std.ArrayList([]const u8).init(gpa); + defer lib_dirs.deinit(); + + var rpath_list = std.ArrayList([]const u8).init(gpa); + defer rpath_list.deinit(); + + var c_source_files = std.ArrayList([]const u8).init(gpa); + defer c_source_files.deinit(); + + var link_objects = std.ArrayList([]const u8).init(gpa); + defer link_objects.deinit(); + + var framework_dirs = std.ArrayList([]const u8).init(gpa); + defer framework_dirs.deinit(); + + var frameworks = std.ArrayList([]const u8).init(gpa); + defer frameworks.deinit(); + + if (arg_mode == .build) { + output_mode = arg_mode.build; + + const args = all_args[2..]; var i: usize = 0; while (i < args.len) : (i += 1) { const arg = args[i]; @@ -222,8 +302,7 @@ fn buildOutputType( process.exit(0); } else if (mem.eql(u8, arg, "--color")) { if (i + 1 >= args.len) { - std.debug.print("expected [auto|on|off] after --color\n", .{}); - process.exit(1); + fatal("expected [auto|on|off] after --color", .{}); } i += 1; const next_arg = args[i]; @@ -234,13 +313,11 @@ fn buildOutputType( } else if (mem.eql(u8, next_arg, "off")) { color = .Off; } else { - std.debug.print("expected [auto|on|off] after --color, found '{}'\n", .{next_arg}); - process.exit(1); + fatal("expected [auto|on|off] after --color, found '{}'", .{next_arg}); } } else if (mem.eql(u8, arg, "--mode")) { if (i + 1 >= args.len) { - std.debug.print("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode\n", .{}); - process.exit(1); + fatal("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode", .{}); } i += 1; const next_arg = args[i]; @@ -253,44 +330,66 @@ fn buildOutputType( } else if (mem.eql(u8, next_arg, "ReleaseSmall")) { build_mode = .ReleaseSmall; } else { - std.debug.print("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode, found '{}'\n", .{next_arg}); - process.exit(1); + fatal("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode, found '{}'", .{next_arg}); } + } else if (mem.eql(u8, arg, "--stack")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + stack_size_override = std.fmt.parseInt(u64, args[i], 10) catch |err| { + fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); + }; } else if (mem.eql(u8, arg, "--name")) { - if (i + 1 >= args.len) { - std.debug.print("expected parameter after --name\n", .{}); - process.exit(1); - } + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; provided_name = args[i]; - } else if (mem.eql(u8, arg, "--library")) { - if (i + 1 >= args.len) { - std.debug.print("expected parameter after --library\n", .{}); - process.exit(1); - } + } else if (mem.eql(u8, arg, "-rpath")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + try rpath_list.append(args[i]); + } else if (mem.eql(u8, arg, "--library-directory") or mem.eql(u8, arg, "-L")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + try lib_dirs.append(args[i]); + } else if (mem.eql(u8, arg, "-T")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + linker_script = args[i]; + } else if (mem.eql(u8, arg, "--version-script")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + version_script = args[i]; + } else if (mem.eql(u8, arg, "--library") or mem.eql(u8, arg, "-l")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + // We don't know whether this library is part of libc or libc++ until we resolve the target. + // So we simply append to the list for now. i += 1; try system_libs.append(args[i]); + } else if (mem.eql(u8, arg, "-D") or + mem.eql(u8, arg, "-isystem") or + mem.eql(u8, arg, "-I") or + mem.eql(u8, arg, "-dirafter")) + { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + try clang_argv.append(arg); + try clang_argv.append(args[i]); } else if (mem.eql(u8, arg, "--version")) { if (i + 1 >= args.len) { - std.debug.print("expected parameter after --version\n", .{}); - process.exit(1); + fatal("expected parameter after --version", .{}); } i += 1; version = std.builtin.Version.parse(args[i]) catch |err| { - std.debug.print("unable to parse --version '{}': {}\n", .{ args[i], @errorName(err) }); - process.exit(1); + fatal("unable to parse --version '{}': {}", .{ args[i], @errorName(err) }); }; } else if (mem.eql(u8, arg, "-target")) { if (i + 1 >= args.len) { - std.debug.print("expected parameter after -target\n", .{}); - process.exit(1); + fatal("expected parameter after -target", .{}); } i += 1; target_arch_os_abi = args[i]; } else if (mem.eql(u8, arg, "-mcpu")) { if (i + 1 >= args.len) { - std.debug.print("expected parameter after -mcpu\n", .{}); - process.exit(1); + fatal("expected parameter after -mcpu", .{}); } i += 1; target_mcpu = args[i]; @@ -300,8 +399,7 @@ fn buildOutputType( target_mcpu = arg["-mcpu=".len..]; } else if (mem.eql(u8, arg, "--dynamic-linker")) { if (i + 1 >= args.len) { - std.debug.print("expected parameter after --dynamic-linker\n", .{}); - process.exit(1); + fatal("expected parameter after --dynamic-linker", .{}); } i += 1; target_dynamic_linker = args[i]; @@ -309,6 +407,12 @@ fn buildOutputType( watch = true; } else if (mem.eql(u8, arg, "-ftime-report")) { time_report = true; + } else if (mem.eql(u8, arg, "-fPIC")) { + want_pic = true; + } else if (mem.eql(u8, arg, "-fno-PIC")) { + want_pic = false; + } else if (mem.eql(u8, arg, "-rdynamic")) { + rdynamic = true; } else if (mem.eql(u8, arg, "-femit-bin")) { emit_bin = .yes_default_path; } else if (mem.startsWith(u8, arg, "-femit-bin=")) { @@ -327,6 +431,8 @@ fn buildOutputType( link_mode = .Static; } else if (mem.eql(u8, arg, "--strip")) { strip = true; + } else if (mem.eql(u8, arg, "-Bsymbolic")) { + linker_bind_global_refs_locally = true; } else if (mem.eql(u8, arg, "--debug-tokenize")) { debug_tokenize = true; } else if (mem.eql(u8, arg, "--debug-ast-tree")) { @@ -339,44 +445,321 @@ fn buildOutputType( debug_ir = true; } else if (mem.eql(u8, arg, "--debug-codegen")) { debug_codegen = true; + } else if (mem.eql(u8, arg, "--debug-cc")) { + debug_cc = true; + } else if (mem.startsWith(u8, arg, "-T")) { + linker_script = arg[2..]; + } else if (mem.startsWith(u8, arg, "-L")) { + try lib_dirs.append(arg[2..]); } else if (mem.startsWith(u8, arg, "-l")) { + // We don't know whether this library is part of libc or libc++ until we resolve the target. + // So we simply append to the list for now. try system_libs.append(arg[2..]); + } else if (mem.startsWith(u8, arg, "-D") or + mem.startsWith(u8, arg, "-I")) + { + try clang_argv.append(arg); } else { - std.debug.print("unrecognized parameter: '{}'\n", .{arg}); - process.exit(1); + fatal("unrecognized parameter: '{}'", .{arg}); } - } else if (mem.endsWith(u8, arg, ".s") or mem.endsWith(u8, arg, ".S")) { - std.debug.print("assembly files not supported yet\n", .{}); - process.exit(1); } else if (mem.endsWith(u8, arg, ".o") or mem.endsWith(u8, arg, ".obj") or mem.endsWith(u8, arg, ".a") or mem.endsWith(u8, arg, ".lib")) { - std.debug.print("object files and static libraries not supported yet\n", .{}); - process.exit(1); - } else if (mem.endsWith(u8, arg, ".c") or - mem.endsWith(u8, arg, ".cpp")) - { - std.debug.print("compilation of C and C++ source code requires LLVM extensions which are not implemented yet\n", .{}); - process.exit(1); + try link_objects.append(arg); + } else if (hasAsmExt(arg) or hasCExt(arg) or hasCppExt(arg)) { + try c_source_files.append(arg); } else if (mem.endsWith(u8, arg, ".so") or mem.endsWith(u8, arg, ".dylib") or mem.endsWith(u8, arg, ".dll")) { - std.debug.print("linking against dynamic libraries not yet supported\n", .{}); - process.exit(1); + fatal("linking against dynamic libraries not yet supported", .{}); } else if (mem.endsWith(u8, arg, ".zig") or mem.endsWith(u8, arg, ".zir")) { if (root_src_file) |other| { - std.debug.print("found another zig file '{}' after root source file '{}'\n", .{ arg, other }); - process.exit(1); + fatal("found another zig file '{}' after root source file '{}'", .{ arg, other }); } else { root_src_file = arg; } } else { - std.debug.print("unrecognized file extension of parameter '{}'\n", .{arg}); + fatal("unrecognized file extension of parameter '{}'", .{arg}); } } + } else { + if (!build_options.have_llvm) + fatal("`zig cc` and `zig c++` unavailable: compiler not built with LLVM extensions enabled", .{}); + emit_h = false; + strip = true; + ensure_libc_on_non_freestanding = true; + ensure_libcpp_on_non_freestanding = arg_mode == .cpp; + want_native_include_dirs = true; + + var c_arg = false; + var is_shared_lib = false; + var linker_args = std.ArrayList([]const u8).init(arena); + var it = ClangArgIterator.init(arena, all_args); + while (it.has_next) { + it.next() catch |err| { + fatal("unable to parse command line parameters: {}", .{@errorName(err)}); + }; + switch (it.zig_equivalent) { + .target => target_arch_os_abi = it.only_arg, // example: -target riscv64-linux-unknown + .o => { + // -o + emit_bin = .{ .yes = it.only_arg }; + enable_cache = true; + }, + .c => c_arg = true, // -c + .other => { + try clang_argv.appendSlice(it.other_args); + }, + .positional => { + const file_ext = classify_file_ext(mem.spanZ(it.only_arg)); + switch (file_ext) { + .assembly, .c, .cpp, .ll, .bc, .h => try c_source_files.append(it.only_arg), + .unknown => try link_objects.append(it.only_arg), + } + }, + .l => { + // -l + // We don't know whether this library is part of libc or libc++ until we resolve the target. + // So we simply append to the list for now. + try system_libs.append(it.only_arg); + }, + .ignore => {}, + .driver_punt => { + // Never mind what we're doing, just pass the args directly. For example --help. + return punt_to_clang(arena, all_args); + }, + .pic => want_pic = true, + .no_pic => want_pic = false, + .nostdlib => ensure_libc_on_non_freestanding = false, + .nostdlib_cpp => ensure_libcpp_on_non_freestanding = false, + .shared => { + link_mode = .Dynamic; + is_shared_lib = true; + }, + .rdynamic => rdynamic = true, + .wl => { + var split_it = mem.split(it.only_arg, ","); + @breakpoint(); // TODO the first arg is empty string right? skip past that. + while (split_it.next()) |linker_arg| { + try linker_args.append(linker_arg); + } + }, + .pp_or_asm => { + // This handles both -E and -S. + only_pp_or_asm = true; + try clang_argv.appendSlice(it.other_args); + }, + .optimize => { + // Alright, what release mode do they want? + if (mem.eql(u8, it.only_arg, "Os")) { + build_mode = .ReleaseSmall; + } else if (mem.eql(u8, it.only_arg, "O2") or + mem.eql(u8, it.only_arg, "O3") or + mem.eql(u8, it.only_arg, "O4")) + { + build_mode = .ReleaseFast; + } else if (mem.eql(u8, it.only_arg, "Og") or + mem.eql(u8, it.only_arg, "O0")) + { + build_mode = .Debug; + } else { + try clang_argv.appendSlice(it.other_args); + } + }, + .debug => { + strip = false; + if (mem.eql(u8, it.only_arg, "-g")) { + // We handled with strip = false above. + } else { + try clang_argv.appendSlice(it.other_args); + } + }, + .sanitize => { + if (mem.eql(u8, it.only_arg, "undefined")) { + want_sanitize_c = true; + } else { + try clang_argv.appendSlice(it.other_args); + } + }, + .linker_script => linker_script = it.only_arg, + .verbose_cmds => { + debug_cc = true; + debug_link = true; + }, + .for_linker => try linker_args.append(it.only_arg), + .linker_input_z => { + try linker_args.append("-z"); + try linker_args.append(it.only_arg); + }, + .lib_dir => try lib_dirs.append(it.only_arg), + .mcpu => target_mcpu = it.only_arg, + .dep_file => { + disable_c_depfile = true; + try clang_argv.appendSlice(it.other_args); + }, + .framework_dir => try framework_dirs.append(it.only_arg), + .framework => try frameworks.append(it.only_arg), + .nostdlibinc => want_native_include_dirs = false, + } + } + // Parse linker args. + var i: usize = 0; + while (i < linker_args.items.len) : (i += 1) { + const arg = linker_args.items[i]; + if (mem.eql(u8, arg, "-soname")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + const soname = linker_args.items[i]; + override_soname = soname; + // Use it as --name. + // Example: libsoundio.so.2 + var prefix: usize = 0; + if (mem.startsWith(u8, soname, "lib")) { + prefix = 3; + } + var end: usize = soname.len; + if (mem.endsWith(u8, soname, ".so")) { + end -= 3; + } else { + var found_digit = false; + while (end > 0 and std.ascii.isDigit(soname[end - 1])) { + found_digit = true; + end -= 1; + } + if (found_digit and end > 0 and soname[end - 1] == '.') { + end -= 1; + } else { + end = soname.len; + } + if (mem.endsWith(u8, soname[prefix..end], ".so")) { + end -= 3; + } + } + provided_name = soname[prefix..end]; + } else if (mem.eql(u8, arg, "-rpath")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + try rpath_list.append(linker_args.items[i]); + } else if (mem.eql(u8, arg, "-I") or + mem.eql(u8, arg, "--dynamic-linker") or + mem.eql(u8, arg, "-dynamic-linker")) + { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + target_dynamic_linker = linker_args.items[i]; + } else if (mem.eql(u8, arg, "-E") or + mem.eql(u8, arg, "--export-dynamic") or + mem.eql(u8, arg, "-export-dynamic")) + { + rdynamic = true; + } else if (mem.eql(u8, arg, "--version-script")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + version_script = linker_args.items[i]; + } else if (mem.startsWith(u8, arg, "-O")) { + linker_optimization = arg; + } else if (mem.eql(u8, arg, "--gc-sections")) { + linker_gc_sections = true; + } else if (mem.eql(u8, arg, "--no-gc-sections")) { + linker_gc_sections = false; + } else if (mem.eql(u8, arg, "--allow-shlib-undefined") or + mem.eql(u8, arg, "-allow-shlib-undefined")) + { + linker_allow_shlib_undefined = true; + } else if (mem.eql(u8, arg, "--no-allow-shlib-undefined") or + mem.eql(u8, arg, "-no-allow-shlib-undefined")) + { + linker_allow_shlib_undefined = false; + } else if (mem.eql(u8, arg, "-Bsymbolic")) { + linker_bind_global_refs_locally = true; + } else if (mem.eql(u8, arg, "-z")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + const z_arg = linker_args.items[i]; + if (mem.eql(u8, z_arg, "nodelete")) { + linker_z_nodelete = true; + } else if (mem.eql(u8, z_arg, "defs")) { + linker_z_defs = true; + } else { + warn("unsupported linker arg: -z {}", .{z_arg}); + } + } else if (mem.eql(u8, arg, "--major-image-version")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + version.major = std.fmt.parseInt(u32, linker_args.items[i], 10) catch |err| { + fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); + }; + } else if (mem.eql(u8, arg, "--minor-image-version")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + version.minor = std.fmt.parseInt(u32, linker_args.items[i], 10) catch |err| { + fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); + }; + } else if (mem.eql(u8, arg, "--stack")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + stack_size_override = std.fmt.parseInt(u64, linker_args.items[i], 10) catch |err| { + fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); + }; + } else { + warn("unsupported linker arg: {}", .{arg}); + } + } + + if (want_sanitize_c == true and build_mode == .ReleaseFast) { + build_mode = .ReleaseSafe; + } + + if (only_pp_or_asm) { + output_mode = .Obj; + fatal("TODO implement using zig cc as a preprocessor", .{}); + //// Transfer "link_objects" into c_source_files so that all those + //// args make it onto the command line. + //try c_source_files.appendSlice(link_objects.items); + //for (c_source_files.items) |c_source_file| { + // const src_path = switch (emit_bin) { + // .yes => |p| p, + // else => c_source_file.source_path, + // }; + // const basename = std.fs.path.basename(src_path); + // c_source_file.preprocessor_only_basename = basename; + //} + //emit_bin = .no; + } else if (!c_arg) { + output_mode = if (is_shared_lib) .Lib else .Exe; + switch (emit_bin) { + .no, .yes_default_path => { + emit_bin = .{ .yes = "a.out" }; + enable_cache = true; + }, + .yes => {}, + } + } else { + output_mode = .Obj; + } + if (c_source_files.items.len == 0 and link_objects.items.len == 0) { + // For example `zig cc` and no args should print the "no input files" message. + return punt_to_clang(arena, all_args); + } } const root_name = if (provided_name) |n| n else blk: { @@ -385,16 +768,10 @@ fn buildOutputType( var it = mem.split(basename, "."); break :blk it.next() orelse basename; } else { - std.debug.print("--name [name] not provided and unable to infer\n", .{}); - process.exit(1); + fatal("--name [name] not provided and unable to infer", .{}); } }; - if (system_libs.items.len != 0) { - std.debug.print("linking against system libraries not yet supported\n", .{}); - process.exit(1); - } - var diags: std.zig.CrossTarget.ParseOptions.Diagnostics = .{}; const cross_target = std.zig.CrossTarget.parse(.{ .arch_os_abi = target_arch_os_abi, @@ -429,17 +806,67 @@ fn buildOutputType( else => |e| return e, }; - var target_info = try std.zig.system.NativeTargetInfo.detect(gpa, cross_target); + const target_info = try std.zig.system.NativeTargetInfo.detect(gpa, cross_target); if (target_info.cpu_detection_unimplemented) { // TODO We want to just use detected_info.target but implementing // CPU model & feature detection is todo so here we rely on LLVM. - std.debug.print("CPU features detection is not yet available for this system without LLVM extensions\n", .{}); - process.exit(1); + fatal("CPU features detection is not yet available for this system without LLVM extensions", .{}); + } + + if (target_info.target.os.tag != .freestanding) { + if (ensure_libc_on_non_freestanding) + have_libc = true; + if (ensure_libcpp_on_non_freestanding) + have_libcpp = true; + } + + // Now that we have target info, we can find out if any of the system libraries + // are part of libc or libc++. We remove them from the list and communicate their + // existence via flags instead. + { + var i: usize = 0; + while (i < system_libs.items.len) { + const lib_name = system_libs.items[i]; + if (is_libc_lib_name(target_info.target, lib_name)) { + have_libc = true; + _ = system_libs.orderedRemove(i); + continue; + } + if (is_libcpp_lib_name(target_info.target, lib_name)) { + have_libcpp = true; + _ = system_libs.orderedRemove(i); + continue; + } + i += 1; + } + } + + if (cross_target.isNativeOs() and (system_libs.items.len != 0 or want_native_include_dirs)) { + const paths = std.zig.system.NativePaths.detect(arena) catch |err| { + fatal("unable to detect native system paths: {}", .{@errorName(err)}); + }; + for (paths.warnings.items) |warning| { + warn("{}", .{warning}); + } + try clang_argv.ensureCapacity(clang_argv.items.len + paths.include_dirs.items.len * 2); + for (paths.include_dirs.items) |include_dir| { + clang_argv.appendAssumeCapacity("-isystem"); + clang_argv.appendAssumeCapacity(include_dir); + } + for (paths.lib_dirs.items) |lib_dir| { + try lib_dirs.append(lib_dir); + } + for (paths.rpaths.items) |rpath| { + try rpath_list.append(rpath); + } + } + + if (system_libs.items.len != 0) { + fatal("linking against system libraries not yet supported", .{}); } const src_path = root_src_file orelse { - std.debug.print("expected at least one file argument", .{}); - process.exit(1); + fatal("expected at least one file argument", .{}); }; const object_format: ?std.Target.ObjectFormat = blk: { @@ -461,15 +888,13 @@ fn buildOutputType( } else if (mem.eql(u8, ofmt, "raw")) { break :blk .raw; } else { - std.debug.print("unsupported object format: {}", .{ofmt}); - process.exit(1); + fatal("unsupported object format: {}", .{ofmt}); } }; const bin_path = switch (emit_bin) { .no => { - std.debug.print("-fno-emit-bin not supported yet", .{}); - process.exit(1); + fatal("-fno-emit-bin not supported yet", .{}); }, .yes_default_path => if (object_format != null and object_format.? == .c) try std.fmt.allocPrint(arena, "{}.c", .{root_name}) @@ -515,6 +940,11 @@ fn buildOutputType( try updateModule(gpa, &module, zir_out_path); + if (build_options.have_llvm and only_pp_or_asm) { + // this may include dumping the output to stdout + fatal("TODO: implement `zig cc` when using it as a preprocessor", .{}); + } + while (watch) { try stderr.print("🦎 ", .{}); if (output_mode == .Exe) { @@ -562,7 +992,7 @@ fn updateModule(gpa: *Allocator, module: *Module, zir_out_path: ?[]const u8) !vo }); } } else { - std.log.scoped(.compiler).info("Update completed in {} ms\n", .{update_nanos / std.time.ns_per_ms}); + info("Update completed in {} ms", .{update_nanos / std.time.ns_per_ms}); } if (zir_out_path) |zop| { @@ -631,8 +1061,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { process.exit(0); } else if (mem.eql(u8, arg, "--color")) { if (i + 1 >= args.len) { - std.debug.print("expected [auto|on|off] after --color\n", .{}); - process.exit(1); + fatal("expected [auto|on|off] after --color", .{}); } i += 1; const next_arg = args[i]; @@ -643,16 +1072,14 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { } else if (mem.eql(u8, next_arg, "off")) { color = .Off; } else { - std.debug.print("expected [auto|on|off] after --color, found '{}'\n", .{next_arg}); - process.exit(1); + fatal("expected [auto|on|off] after --color, found '{}'", .{next_arg}); } } else if (mem.eql(u8, arg, "--stdin")) { stdin_flag = true; } else if (mem.eql(u8, arg, "--check")) { check_flag = true; } else { - std.debug.print("unrecognized parameter: '{}'", .{arg}); - process.exit(1); + fatal("unrecognized parameter: '{}'", .{arg}); } } else { try input_files.append(arg); @@ -662,8 +1089,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { if (stdin_flag) { if (input_files.items.len != 0) { - std.debug.print("cannot use --stdin with positional arguments\n", .{}); - process.exit(1); + fatal("cannot use --stdin with positional arguments", .{}); } const stdin = io.getStdIn().inStream(); @@ -672,8 +1098,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { defer gpa.free(source_code); const tree = std.zig.parse(gpa, source_code) catch |err| { - std.debug.print("error parsing stdin: {}\n", .{err}); - process.exit(1); + fatal("error parsing stdin: {}", .{err}); }; defer tree.deinit(); @@ -695,8 +1120,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { } if (input_files.items.len == 0) { - std.debug.print("expected at least one source file argument\n", .{}); - process.exit(1); + fatal("expected at least one source file argument", .{}); } var fmt = Fmt{ @@ -712,8 +1136,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { for (input_files.span()) |file_path| { // Get the real path here to avoid Windows failing on relative file paths with . or .. in them. const real_path = fs.realpathAlloc(gpa, file_path) catch |err| { - std.debug.print("unable to open '{}': {}\n", .{ file_path, err }); - process.exit(1); + fatal("unable to open '{}': {}", .{ file_path, err }); }; defer gpa.free(real_path); @@ -752,7 +1175,7 @@ fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool, dir: fs.Dir, sub_ fmtPathFile(fmt, file_path, check_mode, dir, sub_path) catch |err| switch (err) { error.IsDir, error.AccessDenied => return fmtPathDir(fmt, file_path, check_mode, dir, sub_path), else => { - std.debug.print("unable to format '{}': {}\n", .{ file_path, err }); + warn("unable to format '{}': {}", .{ file_path, err }); fmt.any_error = true; return; }, @@ -783,7 +1206,7 @@ fn fmtPathDir( try fmtPathDir(fmt, full_path, check_mode, dir, entry.name); } else { fmtPathFile(fmt, full_path, check_mode, dir, entry.name) catch |err| { - std.debug.print("unable to format '{}': {}\n", .{ full_path, err }); + warn("unable to format '{}': {}", .{ full_path, err }); fmt.any_error = true; return; }; @@ -841,6 +1264,7 @@ fn fmtPathFile( if (check_mode) { const anything_changed = try std.zig.render(fmt.gpa, io.null_out_stream, tree); if (anything_changed) { + // TODO this should output to stdout instead of stderr. std.debug.print("{}\n", .{file_path}); fmt.any_error = true; } @@ -858,6 +1282,7 @@ fn fmtPathFile( try af.file.writeAll(fmt.out_buffer.items); try af.finish(); + // TODO this should output to stdout instead of stderr. std.debug.print("{}\n", .{file_path}); } } @@ -925,3 +1350,350 @@ pub const info_zen = \\ \\ ; + +const FileExt = enum { + c, + cpp, + h, + ll, + bc, + assembly, + unknown, +}; + +fn hasCExt(filename: []const u8) bool { + return mem.endsWith(u8, filename, ".c"); +} + +fn hasCppExt(filename: []const u8) bool { + return mem.endsWith(u8, filename, ".C") or + mem.endsWith(u8, filename, ".cc") or + mem.endsWith(u8, filename, ".cpp") or + mem.endsWith(u8, filename, ".cxx"); +} + +fn hasAsmExt(filename: []const u8) bool { + return mem.endsWith(u8, filename, ".s") or mem.endsWith(u8, filename, ".S"); +} + +fn classify_file_ext(filename: []const u8) FileExt { + if (hasCExt(filename)) { + return .c; + } else if (hasCppExt(filename)) { + return .cpp; + } else if (mem.endsWith(u8, filename, ".ll")) { + return .ll; + } else if (mem.endsWith(u8, filename, ".bc")) { + return .bc; + } else if (hasAsmExt(filename)) { + return .assembly; + } else if (mem.endsWith(u8, filename, ".h")) { + return .h; + } else { + // TODO look for .so, .so.X, .so.X.Y, .so.X.Y.Z + return .unknown; + } +} + +extern "c" fn ZigClang_main(argc: c_int, argv: [*:null]?[*:0]u8) c_int; + +/// TODO make it so the return value can be !noreturn +fn punt_to_clang(arena: *Allocator, args: []const []const u8) error{OutOfMemory} { + // Convert the args to the format Clang expects. + const argv = try arena.alloc(?[*:0]u8, args.len + 1); + for (args) |arg, i| { + argv[i] = try arena.dupeZ(u8, arg); // TODO If there was an argsAllocZ we could avoid this allocation. + } + argv[args.len] = null; + const exit_code = ZigClang_main(@intCast(c_int, args.len), argv[0..args.len :null].ptr); + process.exit(@bitCast(u8, @truncate(i8, exit_code))); +} + +const clang_args = @import("clang_options.zig").list; + +pub const ClangArgIterator = struct { + has_next: bool, + zig_equivalent: ZigEquivalent, + only_arg: []const u8, + second_arg: []const u8, + other_args: []const []const u8, + argv: []const []const u8, + next_index: usize, + root_args: ?*Args, + allocator: *Allocator, + + pub const ZigEquivalent = enum { + target, + o, + c, + other, + positional, + l, + ignore, + driver_punt, + pic, + no_pic, + nostdlib, + nostdlib_cpp, + shared, + rdynamic, + wl, + pp_or_asm, + optimize, + debug, + sanitize, + linker_script, + verbose_cmds, + for_linker, + linker_input_z, + lib_dir, + mcpu, + dep_file, + framework_dir, + framework, + nostdlibinc, + }; + + const Args = struct { + next_index: usize, + argv: []const []const u8, + }; + + fn init(allocator: *Allocator, argv: []const []const u8) ClangArgIterator { + return .{ + .next_index = 2, // `zig cc foo` this points to `foo` + .has_next = argv.len > 2, + .zig_equivalent = undefined, + .only_arg = undefined, + .second_arg = undefined, + .other_args = undefined, + .argv = argv, + .root_args = null, + .allocator = allocator, + }; + } + + fn next(self: *ClangArgIterator) !void { + assert(self.has_next); + assert(self.next_index < self.argv.len); + // In this state we know that the parameter we are looking at is a root parameter + // rather than an argument to a parameter. + // We adjust the len below when necessary. + self.other_args = (self.argv.ptr + self.next_index)[0..1]; + var arg = mem.span(self.argv[self.next_index]); + self.incrementArgIndex(); + + if (mem.startsWith(u8, arg, "@")) { + if (self.root_args != null) return error.NestedResponseFile; + + // This is a "compiler response file". We must parse the file and treat its + // contents as command line parameters. + const allocator = self.allocator; + const max_bytes = 10 * 1024 * 1024; // 10 MiB of command line arguments is a reasonable limit + const resp_file_path = arg[1..]; + const resp_contents = fs.cwd().readFileAlloc(allocator, resp_file_path, max_bytes) catch |err| { + fatal("unable to read response file '{}': {}", .{ resp_file_path, @errorName(err) }); + }; + defer allocator.free(resp_contents); + // TODO is there a specification for this file format? Let's find it and make this parsing more robust + // at the very least I'm guessing this needs to handle quotes and `#` comments. + var it = mem.tokenize(resp_contents, " \t\r\n"); + var resp_arg_list = std.ArrayList([]const u8).init(allocator); + defer resp_arg_list.deinit(); + { + errdefer { + for (resp_arg_list.span()) |item| { + allocator.free(mem.span(item)); + } + } + while (it.next()) |token| { + const dupe_token = try mem.dupeZ(allocator, u8, token); + errdefer allocator.free(dupe_token); + try resp_arg_list.append(dupe_token); + } + const args = try allocator.create(Args); + errdefer allocator.destroy(args); + args.* = .{ + .next_index = self.next_index, + .argv = self.argv, + }; + self.root_args = args; + } + const resp_arg_slice = resp_arg_list.toOwnedSlice(); + self.next_index = 0; + self.argv = resp_arg_slice; + + if (resp_arg_slice.len == 0) { + self.resolveRespFileArgs(); + return; + } + + self.has_next = true; + self.other_args = (self.argv.ptr + self.next_index)[0..1]; // We adjust len below when necessary. + arg = mem.span(self.argv[self.next_index]); + self.incrementArgIndex(); + } + if (!mem.startsWith(u8, arg, "-")) { + self.zig_equivalent = .positional; + self.only_arg = arg; + return; + } + + find_clang_arg: for (clang_args) |clang_arg| switch (clang_arg.syntax) { + .flag => { + const prefix_len = clang_arg.matchEql(arg); + if (prefix_len > 0) { + self.zig_equivalent = clang_arg.zig_equivalent; + self.only_arg = arg[prefix_len..]; + + break :find_clang_arg; + } + }, + .joined, .comma_joined => { + // joined example: --target=foo + // comma_joined example: -Wl,-soname,libsoundio.so.2 + const prefix_len = clang_arg.matchStartsWith(arg); + if (prefix_len != 0) { + self.zig_equivalent = clang_arg.zig_equivalent; + self.only_arg = arg[prefix_len..]; // This will skip over the "--target=" part. + + break :find_clang_arg; + } + }, + .joined_or_separate => { + // Examples: `-lfoo`, `-l foo` + const prefix_len = clang_arg.matchStartsWith(arg); + if (prefix_len == arg.len) { + if (self.next_index >= self.argv.len) { + fatal("Expected parameter after '{}'", .{arg}); + } + self.only_arg = self.argv[self.next_index]; + self.incrementArgIndex(); + self.other_args.len += 1; + self.zig_equivalent = clang_arg.zig_equivalent; + + break :find_clang_arg; + } else if (prefix_len != 0) { + self.zig_equivalent = clang_arg.zig_equivalent; + self.only_arg = arg[prefix_len..]; + + break :find_clang_arg; + } + }, + .joined_and_separate => { + // Example: `-Xopenmp-target=riscv64-linux-unknown foo` + const prefix_len = clang_arg.matchStartsWith(arg); + if (prefix_len != 0) { + self.only_arg = arg[prefix_len..]; + if (self.next_index >= self.argv.len) { + fatal("Expected parameter after '{}'", .{arg}); + } + self.second_arg = self.argv[self.next_index]; + self.incrementArgIndex(); + self.other_args.len += 1; + self.zig_equivalent = clang_arg.zig_equivalent; + break :find_clang_arg; + } + }, + .separate => if (clang_arg.matchEql(arg) > 0) { + if (self.next_index >= self.argv.len) { + fatal("Expected parameter after '{}'", .{arg}); + } + self.only_arg = self.argv[self.next_index]; + self.incrementArgIndex(); + self.other_args.len += 1; + self.zig_equivalent = clang_arg.zig_equivalent; + break :find_clang_arg; + }, + .remaining_args_joined => { + const prefix_len = clang_arg.matchStartsWith(arg); + if (prefix_len != 0) { + @panic("TODO"); + } + }, + .multi_arg => if (clang_arg.matchEql(arg) > 0) { + @panic("TODO"); + }, + } + else { + fatal("Unknown Clang option: '{}'", .{arg}); + } + } + + fn incrementArgIndex(self: *ClangArgIterator) void { + self.next_index += 1; + self.resolveRespFileArgs(); + } + + fn resolveRespFileArgs(self: *ClangArgIterator) void { + const allocator = self.allocator; + if (self.next_index >= self.argv.len) { + if (self.root_args) |root_args| { + self.next_index = root_args.next_index; + self.argv = root_args.argv; + + allocator.destroy(root_args); + self.root_args = null; + } + if (self.next_index >= self.argv.len) { + self.has_next = false; + } + } + } +}; + +fn eqlIgnoreCase(ignore_case: bool, a: []const u8, b: []const u8) bool { + if (ignore_case) { + return std.ascii.eqlIgnoreCase(a, b); + } else { + return mem.eql(u8, a, b); + } +} + +fn is_libc_lib_name(target: std.Target, name: []const u8) bool { + const ignore_case = target.os.tag.isDarwin() or target.os.tag == .windows; + + if (eqlIgnoreCase(ignore_case, name, "c")) + return true; + + if (target.isMinGW()) { + if (eqlIgnoreCase(ignore_case, name, "m")) + return true; + + return false; + } + + if (target.abi.isGnu() or target.abi.isMusl() or target.os.tag.isDarwin()) { + if (eqlIgnoreCase(ignore_case, name, "m")) + return true; + if (eqlIgnoreCase(ignore_case, name, "rt")) + return true; + if (eqlIgnoreCase(ignore_case, name, "pthread")) + return true; + if (eqlIgnoreCase(ignore_case, name, "crypt")) + return true; + if (eqlIgnoreCase(ignore_case, name, "util")) + return true; + if (eqlIgnoreCase(ignore_case, name, "xnet")) + return true; + if (eqlIgnoreCase(ignore_case, name, "resolv")) + return true; + if (eqlIgnoreCase(ignore_case, name, "dl")) + return true; + if (eqlIgnoreCase(ignore_case, name, "util")) + return true; + } + + if (target.os.tag.isDarwin() and eqlIgnoreCase(ignore_case, name, "System")) + return true; + + return false; +} + +fn is_libcpp_lib_name(target: std.Target, name: []const u8) bool { + const ignore_case = target.os.tag.isDarwin() or target.os.tag == .windows; + + return eqlIgnoreCase(ignore_case, name, "c++") or + eqlIgnoreCase(ignore_case, name, "stdc++") or + eqlIgnoreCase(ignore_case, name, "c++abi"); +} diff --git a/src-self-hosted/stage2.zig b/src-self-hosted/stage2.zig index 45b8ad3073..a9a9496b94 100644 --- a/src-self-hosted/stage2.zig +++ b/src-self-hosted/stage2.zig @@ -414,6 +414,23 @@ export fn stage2_env(argc: c_int, argv: [*]const [*:0]const u8) c_int { return 0; } +export fn stage2_cc(argc: c_int, argv: [*]const [*:0]const u8, is_cpp: bool) c_int { + const allocator = std.heap.c_allocator; + + var args_list = argvToArrayList(allocator, argc, argv) catch |err| { + std.debug.print("unable to parse arguments: {}\n", .{@errorName(err)}); + return -1; + }; + defer args_list.deinit(); + + self_hosted_main.buildOutputType(allocator, allocator, args_list.items, if (is_cpp) .cpp else .cc) catch |err| { + std.debug.print("zig cc failure: {}\n", .{@errorName(err)}); + return -1; + }; + + return 0; +} + // ABI warning export fn stage2_cmd_targets( zig_triple: ?[*:0]const u8, @@ -1038,267 +1055,4 @@ fn convertSlice(slice: [][:0]u8, ptr: *[*][*:0]u8, len: *usize) !void { ptr.* = new_slice.ptr; } -const clang_args = @import("clang_options.zig").list; - -// ABI warning -pub const ClangArgIterator = extern struct { - has_next: bool, - zig_equivalent: ZigEquivalent, - only_arg: [*:0]const u8, - second_arg: [*:0]const u8, - other_args_ptr: [*]const [*:0]const u8, - other_args_len: usize, - argv_ptr: [*]const [*:0]const u8, - argv_len: usize, - next_index: usize, - root_args: ?*Args, - - // ABI warning - pub const ZigEquivalent = extern enum { - target, - o, - c, - other, - positional, - l, - ignore, - driver_punt, - pic, - no_pic, - nostdlib, - nostdlib_cpp, - shared, - rdynamic, - wl, - pp_or_asm, - optimize, - debug, - sanitize, - linker_script, - verbose_cmds, - for_linker, - linker_input_z, - lib_dir, - mcpu, - dep_file, - framework_dir, - framework, - nostdlibinc, - }; - - const Args = struct { - next_index: usize, - argv_ptr: [*]const [*:0]const u8, - argv_len: usize, - }; - - pub fn init(argv: []const [*:0]const u8) ClangArgIterator { - return .{ - .next_index = 2, // `zig cc foo` this points to `foo` - .has_next = argv.len > 2, - .zig_equivalent = undefined, - .only_arg = undefined, - .second_arg = undefined, - .other_args_ptr = undefined, - .other_args_len = undefined, - .argv_ptr = argv.ptr, - .argv_len = argv.len, - .root_args = null, - }; - } - - pub fn next(self: *ClangArgIterator) !void { - assert(self.has_next); - assert(self.next_index < self.argv_len); - // In this state we know that the parameter we are looking at is a root parameter - // rather than an argument to a parameter. - self.other_args_ptr = self.argv_ptr + self.next_index; - self.other_args_len = 1; // We adjust this value below when necessary. - var arg = mem.span(self.argv_ptr[self.next_index]); - self.incrementArgIndex(); - - if (mem.startsWith(u8, arg, "@")) { - if (self.root_args != null) return error.NestedResponseFile; - - // This is a "compiler response file". We must parse the file and treat its - // contents as command line parameters. - const allocator = std.heap.c_allocator; - const max_bytes = 10 * 1024 * 1024; // 10 MiB of command line arguments is a reasonable limit - const resp_file_path = arg[1..]; - const resp_contents = fs.cwd().readFileAlloc(allocator, resp_file_path, max_bytes) catch |err| { - std.debug.warn("unable to read response file '{}': {}\n", .{ resp_file_path, @errorName(err) }); - process.exit(1); - }; - defer allocator.free(resp_contents); - // TODO is there a specification for this file format? Let's find it and make this parsing more robust - // at the very least I'm guessing this needs to handle quotes and `#` comments. - var it = mem.tokenize(resp_contents, " \t\r\n"); - var resp_arg_list = std.ArrayList([*:0]const u8).init(allocator); - defer resp_arg_list.deinit(); - { - errdefer { - for (resp_arg_list.span()) |item| { - allocator.free(mem.span(item)); - } - } - while (it.next()) |token| { - const dupe_token = try mem.dupeZ(allocator, u8, token); - errdefer allocator.free(dupe_token); - try resp_arg_list.append(dupe_token); - } - const args = try allocator.create(Args); - errdefer allocator.destroy(args); - args.* = .{ - .next_index = self.next_index, - .argv_ptr = self.argv_ptr, - .argv_len = self.argv_len, - }; - self.root_args = args; - } - const resp_arg_slice = resp_arg_list.toOwnedSlice(); - self.next_index = 0; - self.argv_ptr = resp_arg_slice.ptr; - self.argv_len = resp_arg_slice.len; - - if (resp_arg_slice.len == 0) { - self.resolveRespFileArgs(); - return; - } - - self.has_next = true; - self.other_args_ptr = self.argv_ptr + self.next_index; - self.other_args_len = 1; // We adjust this value below when necessary. - arg = mem.span(self.argv_ptr[self.next_index]); - self.incrementArgIndex(); - } - if (!mem.startsWith(u8, arg, "-")) { - self.zig_equivalent = .positional; - self.only_arg = arg.ptr; - return; - } - - find_clang_arg: for (clang_args) |clang_arg| switch (clang_arg.syntax) { - .flag => { - const prefix_len = clang_arg.matchEql(arg); - if (prefix_len > 0) { - self.zig_equivalent = clang_arg.zig_equivalent; - self.only_arg = arg.ptr + prefix_len; - - break :find_clang_arg; - } - }, - .joined, .comma_joined => { - // joined example: --target=foo - // comma_joined example: -Wl,-soname,libsoundio.so.2 - const prefix_len = clang_arg.matchStartsWith(arg); - if (prefix_len != 0) { - self.zig_equivalent = clang_arg.zig_equivalent; - self.only_arg = arg.ptr + prefix_len; // This will skip over the "--target=" part. - - break :find_clang_arg; - } - }, - .joined_or_separate => { - // Examples: `-lfoo`, `-l foo` - const prefix_len = clang_arg.matchStartsWith(arg); - if (prefix_len == arg.len) { - if (self.next_index >= self.argv_len) { - std.debug.warn("Expected parameter after '{}'\n", .{arg}); - process.exit(1); - } - self.only_arg = self.argv_ptr[self.next_index]; - self.incrementArgIndex(); - self.other_args_len += 1; - self.zig_equivalent = clang_arg.zig_equivalent; - - break :find_clang_arg; - } else if (prefix_len != 0) { - self.zig_equivalent = clang_arg.zig_equivalent; - self.only_arg = arg.ptr + prefix_len; - - break :find_clang_arg; - } - }, - .joined_and_separate => { - // Example: `-Xopenmp-target=riscv64-linux-unknown foo` - const prefix_len = clang_arg.matchStartsWith(arg); - if (prefix_len != 0) { - self.only_arg = arg.ptr + prefix_len; - if (self.next_index >= self.argv_len) { - std.debug.warn("Expected parameter after '{}'\n", .{arg}); - process.exit(1); - } - self.second_arg = self.argv_ptr[self.next_index]; - self.incrementArgIndex(); - self.other_args_len += 1; - self.zig_equivalent = clang_arg.zig_equivalent; - break :find_clang_arg; - } - }, - .separate => if (clang_arg.matchEql(arg) > 0) { - if (self.next_index >= self.argv_len) { - std.debug.warn("Expected parameter after '{}'\n", .{arg}); - process.exit(1); - } - self.only_arg = self.argv_ptr[self.next_index]; - self.incrementArgIndex(); - self.other_args_len += 1; - self.zig_equivalent = clang_arg.zig_equivalent; - break :find_clang_arg; - }, - .remaining_args_joined => { - const prefix_len = clang_arg.matchStartsWith(arg); - if (prefix_len != 0) { - @panic("TODO"); - } - }, - .multi_arg => if (clang_arg.matchEql(arg) > 0) { - @panic("TODO"); - }, - } - else { - std.debug.warn("Unknown Clang option: '{}'\n", .{arg}); - process.exit(1); - } - } - - fn incrementArgIndex(self: *ClangArgIterator) void { - self.next_index += 1; - self.resolveRespFileArgs(); - } - - fn resolveRespFileArgs(self: *ClangArgIterator) void { - const allocator = std.heap.c_allocator; - if (self.next_index >= self.argv_len) { - if (self.root_args) |root_args| { - self.next_index = root_args.next_index; - self.argv_ptr = root_args.argv_ptr; - self.argv_len = root_args.argv_len; - - allocator.destroy(root_args); - self.root_args = null; - } - if (self.next_index >= self.argv_len) { - self.has_next = false; - } - } - } -}; - -export fn stage2_clang_arg_iterator( - result: *ClangArgIterator, - argc: usize, - argv: [*]const [*:0]const u8, -) void { - result.* = ClangArgIterator.init(argv[0..argc]); -} - -export fn stage2_clang_arg_next(it: *ClangArgIterator) Error { - it.next() catch |err| switch (err) { - error.NestedResponseFile => return .NestedResponseFile, - error.OutOfMemory => return .OutOfMemory, - }; - return .None; -} - export const stage2_is_zig0 = false; diff --git a/src/config.zig.in b/src/config.zig.in index ccb618df2d..a44149111e 100644 --- a/src/config.zig.in +++ b/src/config.zig.in @@ -1,3 +1,5 @@ +pub const have_llvm = true; pub const version: []const u8 = "@ZIG_VERSION@"; pub const log_scopes: []const []const u8 = &[_][]const u8{}; +pub const zir_dumps: []const []const u8 = &[_][]const u8{}; pub const enable_tracy = false; diff --git a/src/main.cpp b/src/main.cpp index e2f6a82a12..af3bc4b2a1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -404,7 +404,6 @@ static int main0(int argc, char **argv) { ZigList framework_dirs = {0}; ZigList frameworks = {0}; bool have_libc = false; - bool have_libcpp = false; const char *target_string = nullptr; bool rdynamic = false; const char *linker_script = nullptr; @@ -446,18 +445,8 @@ static int main0(int argc, char **argv) { bool function_sections = false; const char *mcpu = nullptr; CodeModel code_model = CodeModelDefault; - const char *override_soname = nullptr; - bool only_pp_or_asm = false; - bool ensure_libc_on_non_freestanding = false; - bool ensure_libcpp_on_non_freestanding = false; - bool disable_c_depfile = false; bool want_native_include_dirs = false; - Buf *linker_optimization = nullptr; - OptionalBool linker_gc_sections = OptionalBoolNull; - OptionalBool linker_allow_shlib_undefined = OptionalBoolNull; OptionalBool linker_bind_global_refs_locally = OptionalBoolNull; - bool linker_z_nodelete = false; - bool linker_z_defs = false; size_t stack_size_override = 0; ZigList llvm_argv = {0}; @@ -585,355 +574,10 @@ static int main0(int argc, char **argv) { return stage2_fmt(argc, argv); } else if (argc >= 2 && strcmp(argv[1], "env") == 0) { return stage2_env(argc, argv); - } else if (argc >= 2 && (strcmp(argv[1], "cc") == 0 || strcmp(argv[1], "c++") == 0)) { - emit_h = false; - strip = true; - ensure_libc_on_non_freestanding = true; - ensure_libcpp_on_non_freestanding = (strcmp(argv[1], "c++") == 0); - want_native_include_dirs = true; - - bool c_arg = false; - Stage2ClangArgIterator it; - stage2_clang_arg_iterator(&it, argc, argv); - bool is_shared_lib = false; - ZigList linker_args = {}; - while (it.has_next) { - if ((err = stage2_clang_arg_next(&it))) { - fprintf(stderr, "unable to parse command line parameters: %s\n", err_str(err)); - return EXIT_FAILURE; - } - switch (it.kind) { - case Stage2ClangArgTarget: // example: -target riscv64-linux-unknown - target_string = it.only_arg; - break; - case Stage2ClangArgO: // -o - emit_bin_override_path = it.only_arg; - enable_cache = CacheOptOn; - break; - case Stage2ClangArgC: // -c - c_arg = true; - break; - case Stage2ClangArgOther: - for (size_t i = 0; i < it.other_args_len; i += 1) { - clang_argv.append(it.other_args_ptr[i]); - } - break; - case Stage2ClangArgPositional: { - FileExt file_ext = classify_file_ext(it.only_arg, strlen(it.only_arg)); - switch (file_ext) { - case FileExtAsm: - case FileExtC: - case FileExtCpp: - case FileExtLLVMIr: - case FileExtLLVMBitCode: - case FileExtHeader: { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = it.only_arg; - c_source_files.append(c_file); - break; - } - case FileExtUnknown: - objects.append(it.only_arg); - break; - } - break; - } - case Stage2ClangArgL: // -l - if (strcmp(it.only_arg, "c") == 0) { - have_libc = true; - link_libs.append("c"); - } else if (strcmp(it.only_arg, "c++") == 0 || - strcmp(it.only_arg, "stdc++") == 0) - { - have_libcpp = true; - link_libs.append("c++"); - } else { - link_libs.append(it.only_arg); - } - break; - case Stage2ClangArgIgnore: - break; - case Stage2ClangArgDriverPunt: - // Never mind what we're doing, just pass the args directly. For example --help. - return ZigClang_main(argc, argv); - case Stage2ClangArgPIC: - want_pic = WantPICEnabled; - break; - case Stage2ClangArgNoPIC: - want_pic = WantPICDisabled; - break; - case Stage2ClangArgNoStdLib: - ensure_libc_on_non_freestanding = false; - break; - case Stage2ClangArgNoStdLibCpp: - ensure_libcpp_on_non_freestanding = false; - break; - case Stage2ClangArgShared: - is_dynamic = true; - is_shared_lib = true; - break; - case Stage2ClangArgRDynamic: - rdynamic = true; - break; - case Stage2ClangArgWL: { - const char *arg = it.only_arg; - for (;;) { - size_t pos = 0; - while (arg[pos] != ',' && arg[pos] != 0) pos += 1; - linker_args.append(buf_create_from_mem(arg, pos)); - if (arg[pos] == 0) break; - arg += pos + 1; - } - break; - } - case Stage2ClangArgPreprocessOrAsm: - // this handles both -E and -S - only_pp_or_asm = true; - for (size_t i = 0; i < it.other_args_len; i += 1) { - clang_argv.append(it.other_args_ptr[i]); - } - break; - case Stage2ClangArgOptimize: - // alright what release mode do they want? - if (strcmp(it.only_arg, "Os") == 0) { - build_mode = BuildModeSmallRelease; - } else if (strcmp(it.only_arg, "O2") == 0 || - strcmp(it.only_arg, "O3") == 0 || - strcmp(it.only_arg, "O4") == 0) - { - build_mode = BuildModeFastRelease; - } else if (strcmp(it.only_arg, "Og") == 0 || - strcmp(it.only_arg, "O0") == 0) - { - build_mode = BuildModeDebug; - } else { - for (size_t i = 0; i < it.other_args_len; i += 1) { - clang_argv.append(it.other_args_ptr[i]); - } - } - break; - case Stage2ClangArgDebug: - strip = false; - if (strcmp(it.only_arg, "-g") == 0) { - // we handled with strip = false above - } else { - for (size_t i = 0; i < it.other_args_len; i += 1) { - clang_argv.append(it.other_args_ptr[i]); - } - } - break; - case Stage2ClangArgSanitize: - if (strcmp(it.only_arg, "undefined") == 0) { - want_sanitize_c = WantCSanitizeEnabled; - } else { - for (size_t i = 0; i < it.other_args_len; i += 1) { - clang_argv.append(it.other_args_ptr[i]); - } - } - break; - case Stage2ClangArgLinkerScript: - linker_script = it.only_arg; - break; - case Stage2ClangArgVerboseCmds: - verbose_cc = true; - verbose_link = true; - break; - case Stage2ClangArgForLinker: - linker_args.append(buf_create_from_str(it.only_arg)); - break; - case Stage2ClangArgLinkerInputZ: - linker_args.append(buf_create_from_str("-z")); - linker_args.append(buf_create_from_str(it.only_arg)); - break; - case Stage2ClangArgLibDir: - lib_dirs.append(it.only_arg); - break; - case Stage2ClangArgMCpu: - mcpu = it.only_arg; - break; - case Stage2ClangArgDepFile: - disable_c_depfile = true; - for (size_t i = 0; i < it.other_args_len; i += 1) { - clang_argv.append(it.other_args_ptr[i]); - } - break; - case Stage2ClangArgFrameworkDir: - framework_dirs.append(it.only_arg); - break; - case Stage2ClangArgFramework: - frameworks.append(it.only_arg); - break; - case Stage2ClangArgNoStdLibInc: - want_native_include_dirs = false; - break; - } - } - // Parse linker args - for (size_t i = 0; i < linker_args.length; i += 1) { - Buf *arg = linker_args.at(i); - if (buf_eql_str(arg, "-soname")) { - i += 1; - if (i >= linker_args.length) { - fprintf(stderr, "expected linker arg after '%s'\n", buf_ptr(arg)); - return EXIT_FAILURE; - } - Buf *soname_buf = linker_args.at(i); - override_soname = buf_ptr(soname_buf); - // use it as --name - // example: libsoundio.so.2 - size_t prefix = 0; - if (buf_starts_with_str(soname_buf, "lib")) { - prefix = 3; - } - size_t end = buf_len(soname_buf); - if (buf_ends_with_str(soname_buf, ".so")) { - end -= 3; - } else { - bool found_digit = false; - while (end > 0 && isdigit(buf_ptr(soname_buf)[end - 1])) { - found_digit = true; - end -= 1; - } - if (found_digit && end > 0 && buf_ptr(soname_buf)[end - 1] == '.') { - end -= 1; - } else { - end = buf_len(soname_buf); - } - if (buf_ends_with_str(buf_slice(soname_buf, prefix, end), ".so")) { - end -= 3; - } - } - out_name = buf_ptr(buf_slice(soname_buf, prefix, end)); - } else if (buf_eql_str(arg, "-rpath")) { - i += 1; - if (i >= linker_args.length) { - fprintf(stderr, "expected linker arg after '%s'\n", buf_ptr(arg)); - return EXIT_FAILURE; - } - Buf *rpath = linker_args.at(i); - rpath_list.append(buf_ptr(rpath)); - } else if (buf_eql_str(arg, "-I") || - buf_eql_str(arg, "--dynamic-linker") || - buf_eql_str(arg, "-dynamic-linker")) - { - i += 1; - if (i >= linker_args.length) { - fprintf(stderr, "expected linker arg after '%s'\n", buf_ptr(arg)); - return EXIT_FAILURE; - } - dynamic_linker = buf_ptr(linker_args.at(i)); - } else if (buf_eql_str(arg, "-E") || - buf_eql_str(arg, "--export-dynamic") || - buf_eql_str(arg, "-export-dynamic")) - { - rdynamic = true; - } else if (buf_eql_str(arg, "--version-script")) { - i += 1; - if (i >= linker_args.length) { - fprintf(stderr, "expected linker arg after '%s'\n", buf_ptr(arg)); - return EXIT_FAILURE; - } - version_script = linker_args.at(i); - } else if (buf_starts_with_str(arg, "-O")) { - linker_optimization = arg; - } else if (buf_eql_str(arg, "--gc-sections")) { - linker_gc_sections = OptionalBoolTrue; - } else if (buf_eql_str(arg, "--no-gc-sections")) { - linker_gc_sections = OptionalBoolFalse; - } else if (buf_eql_str(arg, "--allow-shlib-undefined") || - buf_eql_str(arg, "-allow-shlib-undefined")) - { - linker_allow_shlib_undefined = OptionalBoolTrue; - } else if (buf_eql_str(arg, "--no-allow-shlib-undefined") || - buf_eql_str(arg, "-no-allow-shlib-undefined")) - { - linker_allow_shlib_undefined = OptionalBoolFalse; - } else if (buf_eql_str(arg, "-Bsymbolic")) { - linker_bind_global_refs_locally = OptionalBoolTrue; - } else if (buf_eql_str(arg, "-z")) { - i += 1; - if (i >= linker_args.length) { - fprintf(stderr, "expected linker arg after '%s'\n", buf_ptr(arg)); - return EXIT_FAILURE; - } - Buf *z_arg = linker_args.at(i); - if (buf_eql_str(z_arg, "nodelete")) { - linker_z_nodelete = true; - } else if (buf_eql_str(z_arg, "defs")) { - linker_z_defs = true; - } else { - fprintf(stderr, "warning: unsupported linker arg: -z %s\n", buf_ptr(z_arg)); - } - } else if (buf_eql_str(arg, "--major-image-version")) { - i += 1; - if (i >= linker_args.length) { - fprintf(stderr, "expected linker arg after '%s'\n", buf_ptr(arg)); - return EXIT_FAILURE; - } - ver_major = atoi(buf_ptr(linker_args.at(i))); - } else if (buf_eql_str(arg, "--minor-image-version")) { - i += 1; - if (i >= linker_args.length) { - fprintf(stderr, "expected linker arg after '%s'\n", buf_ptr(arg)); - return EXIT_FAILURE; - } - ver_minor = atoi(buf_ptr(linker_args.at(i))); - } else if (buf_eql_str(arg, "--stack")) { - i += 1; - if (i >= linker_args.length) { - fprintf(stderr, "expected linker arg after '%s'\n", buf_ptr(arg)); - return EXIT_FAILURE; - } - stack_size_override = atoi(buf_ptr(linker_args.at(i))); - } else { - fprintf(stderr, "warning: unsupported linker arg: %s\n", buf_ptr(arg)); - } - } - - if (want_sanitize_c == WantCSanitizeEnabled && build_mode == BuildModeFastRelease) { - build_mode = BuildModeSafeRelease; - } - - if (only_pp_or_asm) { - cmd = CmdBuild; - out_type = OutTypeObj; - emit_bin = false; - // Transfer "objects" into c_source_files - for (size_t i = 0; i < objects.length; i += 1) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = objects.at(i); - c_source_files.append(c_file); - } - for (size_t i = 0; i < c_source_files.length; i += 1) { - Buf *src_path; - if (emit_bin_override_path != nullptr) { - src_path = buf_create_from_str(emit_bin_override_path); - } else { - src_path = buf_create_from_str(c_source_files.at(i)->source_path); - } - Buf basename = BUF_INIT; - os_path_split(src_path, nullptr, &basename); - c_source_files.at(i)->preprocessor_only_basename = buf_ptr(&basename); - } - } else if (!c_arg) { - cmd = CmdBuild; - if (is_shared_lib) { - out_type = OutTypeLib; - } else { - out_type = OutTypeExe; - } - if (emit_bin_override_path == nullptr) { - emit_bin_override_path = "a.out"; - enable_cache = CacheOptOn; - } - } else { - cmd = CmdBuild; - out_type = OutTypeObj; - } - if (c_source_files.length == 0 && objects.length == 0) { - // For example `zig cc` and no args should print the "no input files" message. - return ZigClang_main(argc, argv); - } + } else if (argc >= 2 && strcmp(argv[1], "cc") == 0) { + return stage2_cc(argc, argv, false); + } else if (argc >= 2 && strcmp(argv[1], "c++") == 0) { + return stage2_cc(argc, argv, true); } else for (int i = 1; i < argc; i += 1) { char *arg = argv[i]; @@ -1038,7 +682,6 @@ static int main0(int argc, char **argv) { have_libc = true; link_libs.append("c"); } else if (strcmp(l, "c++") == 0 || strcmp(l, "stdc++") == 0) { - have_libcpp = true; link_libs.append("c++"); } else { link_libs.append(l); @@ -1185,7 +828,6 @@ static int main0(int argc, char **argv) { have_libc = true; link_libs.append("c"); } else if (strcmp(argv[i], "c++") == 0 || strcmp(argv[i], "stdc++") == 0) { - have_libcpp = true; link_libs.append("c++"); } else { link_libs.append(argv[i]); @@ -1351,15 +993,6 @@ static int main0(int argc, char **argv) { return print_error_usage(arg0); } - if (!have_libc && ensure_libc_on_non_freestanding && target.os != OsFreestanding) { - have_libc = true; - link_libs.append("c"); - } - if (!have_libcpp && ensure_libcpp_on_non_freestanding && target.os != OsFreestanding) { - have_libcpp = true; - link_libs.append("c++"); - } - Buf zig_triple_buf = BUF_INIT; target_triple_zig(&zig_triple_buf, &target); @@ -1616,20 +1249,10 @@ static int main0(int argc, char **argv) { g->system_linker_hack = system_linker_hack; g->function_sections = function_sections; g->code_model = code_model; - g->disable_c_depfile = disable_c_depfile; - g->linker_optimization = linker_optimization; - g->linker_gc_sections = linker_gc_sections; - g->linker_allow_shlib_undefined = linker_allow_shlib_undefined; g->linker_bind_global_refs_locally = linker_bind_global_refs_locally; - g->linker_z_nodelete = linker_z_nodelete; - g->linker_z_defs = linker_z_defs; g->stack_size_override = stack_size_override; - if (override_soname) { - g->override_soname = buf_create_from_str(override_soname); - } - for (size_t i = 0; i < lib_dirs.length; i += 1) { codegen_add_lib_dir(g, lib_dirs.at(i)); } @@ -1713,37 +1336,12 @@ static int main0(int argc, char **argv) { buf_replace(g->output_dir, '/', '\\'); #endif Buf *dest_path = buf_create_from_str(emit_bin_override_path); - Buf *source_path; - if (only_pp_or_asm) { - source_path = buf_alloc(); - Buf *pp_only_basename = buf_create_from_str( - c_source_files.at(0)->preprocessor_only_basename); - os_path_join(g->output_dir, pp_only_basename, source_path); - - } else { - source_path = &g->bin_file_output_path; - } + Buf *source_path = &g->bin_file_output_path; if ((err = os_update_file(source_path, dest_path))) { fprintf(stderr, "unable to copy %s to %s: %s\n", buf_ptr(source_path), buf_ptr(dest_path), err_str(err)); return main_exit(root_progress_node, EXIT_FAILURE); } - } else if (only_pp_or_asm) { -#if defined(ZIG_OS_WINDOWS) - buf_replace(g->c_artifact_dir, '/', '\\'); -#endif - // dump the preprocessed output to stdout - for (size_t i = 0; i < c_source_files.length; i += 1) { - Buf *source_path = buf_alloc(); - Buf *pp_only_basename = buf_create_from_str( - c_source_files.at(i)->preprocessor_only_basename); - os_path_join(g->c_artifact_dir, pp_only_basename, source_path); - if ((err = os_dump_file(source_path, stdout))) { - fprintf(stderr, "unable to read %s: %s\n", buf_ptr(source_path), - err_str(err)); - return main_exit(root_progress_node, EXIT_FAILURE); - } - } } else if (g->enable_cache) { #if defined(ZIG_OS_WINDOWS) buf_replace(&g->bin_file_output_path, '/', '\\'); diff --git a/src/stage2.cpp b/src/stage2.cpp index 6c010de84f..ad6a9f9d97 100644 --- a/src/stage2.cpp +++ b/src/stage2.cpp @@ -32,6 +32,11 @@ int stage2_env(int argc, char** argv) { stage2_panic(msg, strlen(msg)); } +int stage2_cc(int argc, char** argv, bool is_cpp) { + const char *msg = "stage0 called stage2_cc"; + stage2_panic(msg, strlen(msg)); +} + void stage2_attach_segfault_handler(void) { } void stage2_panic(const char *ptr, size_t len) { @@ -316,16 +321,4 @@ enum Error stage2_detect_native_paths(struct Stage2NativePaths *native_paths) { return ErrorNone; } -void stage2_clang_arg_iterator(struct Stage2ClangArgIterator *it, - size_t argc, char **argv) -{ - const char *msg = "stage0 called stage2_clang_arg_iterator"; - stage2_panic(msg, strlen(msg)); -} - -enum Error stage2_clang_arg_next(struct Stage2ClangArgIterator *it) { - const char *msg = "stage0 called stage2_clang_arg_next"; - stage2_panic(msg, strlen(msg)); -} - const bool stage2_is_zig0 = true; diff --git a/src/stage2.h b/src/stage2.h index 38a1f77d46..2e64189450 100644 --- a/src/stage2.h +++ b/src/stage2.h @@ -144,6 +144,9 @@ ZIG_EXTERN_C void stage2_zen(const char **ptr, size_t *len); // ABI warning ZIG_EXTERN_C int stage2_env(int argc, char **argv); +// ABI warning +ZIG_EXTERN_C int stage2_cc(int argc, char **argv, bool is_cpp); + // ABI warning ZIG_EXTERN_C void stage2_attach_segfault_handler(void); @@ -328,60 +331,6 @@ struct Stage2NativePaths { // ABI warning ZIG_EXTERN_C enum Error stage2_detect_native_paths(struct Stage2NativePaths *native_paths); -// ABI warning -enum Stage2ClangArg { - Stage2ClangArgTarget, - Stage2ClangArgO, - Stage2ClangArgC, - Stage2ClangArgOther, - Stage2ClangArgPositional, - Stage2ClangArgL, - Stage2ClangArgIgnore, - Stage2ClangArgDriverPunt, - Stage2ClangArgPIC, - Stage2ClangArgNoPIC, - Stage2ClangArgNoStdLib, - Stage2ClangArgNoStdLibCpp, - Stage2ClangArgShared, - Stage2ClangArgRDynamic, - Stage2ClangArgWL, - Stage2ClangArgPreprocessOrAsm, - Stage2ClangArgOptimize, - Stage2ClangArgDebug, - Stage2ClangArgSanitize, - Stage2ClangArgLinkerScript, - Stage2ClangArgVerboseCmds, - Stage2ClangArgForLinker, - Stage2ClangArgLinkerInputZ, - Stage2ClangArgLibDir, - Stage2ClangArgMCpu, - Stage2ClangArgDepFile, - Stage2ClangArgFrameworkDir, - Stage2ClangArgFramework, - Stage2ClangArgNoStdLibInc, -}; - -// ABI warning -struct Stage2ClangArgIterator { - bool has_next; - enum Stage2ClangArg kind; - const char *only_arg; - const char *second_arg; - const char **other_args_ptr; - size_t other_args_len; - const char **argv_ptr; - size_t argv_len; - size_t next_index; - size_t root_args; -}; - -// ABI warning -ZIG_EXTERN_C void stage2_clang_arg_iterator(struct Stage2ClangArgIterator *it, - size_t argc, char **argv); - -// ABI warning -ZIG_EXTERN_C enum Error stage2_clang_arg_next(struct Stage2ClangArgIterator *it); - // ABI warning ZIG_EXTERN_C const bool stage2_is_zig0; From 71687b30a24dc4468944620a5e1514077e6d5325 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 4 Sep 2020 12:42:17 -0700 Subject: [PATCH 004/210] work around stage1 invalid LLVM IR --- src-self-hosted/main.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 330c0468f1..0386d801a2 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -725,8 +725,10 @@ pub fn buildOutputType( } } - if (want_sanitize_c == true and build_mode == .ReleaseFast) { - build_mode = .ReleaseSafe; + if (want_sanitize_c) |wsc| { + if (wsc and build_mode == .ReleaseFast) { + build_mode = .ReleaseSafe; + } } if (only_pp_or_asm) { From f064f0564fcdd3a163f137f72ecb7a997fdebfd1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 4 Sep 2020 13:23:57 -0700 Subject: [PATCH 005/210] stage2: improve log message format --- src-self-hosted/main.zig | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 0386d801a2..9e5a5b73f7 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -13,7 +13,6 @@ const Package = @import("Package.zig"); const zir = @import("zir.zig"); const build_options = @import("build_options"); const warn = std.log.warn; -const info = std.log.info; fn fatal(comptime format: []const u8, args: anytype) noreturn { std.log.emerg(format, args); @@ -67,10 +66,16 @@ pub fn log( return; } - const prefix = "[" ++ @tagName(level) ++ "] " ++ "(" ++ @tagName(scope) ++ "): "; + const level_txt = switch (level) { + .emerg => "error", + .warn => "warning", + else => @tagName(level), + }; + const prefix1 = level_txt ++ ": "; + const prefix2 = if (scope == .default) "" else "(" ++ @tagName(scope) ++ "): "; // Print the message to stderr, silently ignoring any errors - std.debug.print(prefix ++ format ++ "\n", args); + std.debug.print(prefix1 ++ prefix2 ++ format ++ "\n", args); } var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){}; @@ -486,8 +491,6 @@ pub fn buildOutputType( } } } else { - if (!build_options.have_llvm) - fatal("`zig cc` and `zig c++` unavailable: compiler not built with LLVM extensions enabled", .{}); emit_h = false; strip = true; ensure_libc_on_non_freestanding = true; @@ -994,7 +997,7 @@ fn updateModule(gpa: *Allocator, module: *Module, zir_out_path: ?[]const u8) !vo }); } } else { - info("Update completed in {} ms", .{update_nanos / std.time.ns_per_ms}); + std.log.info("Update completed in {} ms", .{update_nanos / std.time.ns_per_ms}); } if (zir_out_path) |zop| { @@ -1401,6 +1404,8 @@ extern "c" fn ZigClang_main(argc: c_int, argv: [*:null]?[*:0]u8) c_int; /// TODO make it so the return value can be !noreturn fn punt_to_clang(arena: *Allocator, args: []const []const u8) error{OutOfMemory} { + if (!build_options.have_llvm) + fatal("`zig cc` and `zig c++` unavailable: compiler not built with LLVM extensions enabled", .{}); // Convert the args to the format Clang expects. const argv = try arena.alloc(?[*:0]u8, args.len + 1); for (args) |arg, i| { From 472ee184862415d1f2651d81d248f3000032932d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 4 Sep 2020 13:46:14 -0700 Subject: [PATCH 006/210] stage2: infer --name in more ways --- src-self-hosted/main.zig | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 9e5a5b73f7..3ee91d54a2 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -770,8 +770,16 @@ pub fn buildOutputType( const root_name = if (provided_name) |n| n else blk: { if (root_src_file) |file| { const basename = fs.path.basename(file); - var it = mem.split(basename, "."); - break :blk it.next() orelse basename; + break :blk mem.split(basename, ".").next().?; + } else if (c_source_files.items.len == 1) { + const basename = fs.path.basename(c_source_files.items[0]); + break :blk mem.split(basename, ".").next().?; + } else if (link_objects.items.len == 1) { + const basename = fs.path.basename(link_objects.items[0]); + break :blk mem.split(basename, ".").next().?; + } else if (emit_bin == .yes) { + const basename = fs.path.basename(emit_bin.yes); + break :blk mem.split(basename, ".").next().?; } else { fatal("--name [name] not provided and unable to infer", .{}); } From 4056bb92e6d6ca3d2ab8f49b4ac83c01fb25bd11 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 8 Sep 2020 01:11:10 -0700 Subject: [PATCH 007/210] stage2: more progress moving `zig cc` to stage2 * std.cache_hash exposes Hasher type * std.cache_hash makes hasher_init a global const * std.cache_hash supports cloning so that clones can share the same open manifest dir handle as well as fork from shared hasher state * start to populate the cache_hash for stage2 builds * remove a footgun from std.cache_hash add function * get rid of std.Target.ObjectFormat.unknown * rework stage2 logic for resolving output artifact names by adding object_format as an optional parameter to std.zig.binNameAlloc * support -Denable-llvm in stage2 tests * Module supports the use case when there are no .zig files * introduce c_object_table and failed_c_objects to Module * propagate many new kinds of data from CLI into Module and into linker.Options * introduce -fLLVM, -fLLD, -fClang and their -fno- counterparts. closes #6251. - add logic for choosing when to use LLD or zig's self-hosted linker * stub code for implementing invoking Clang to build C objects * add -femit-h, -femit-h=foo, and -fno-emit-h CLI options --- build.zig | 1 + lib/std/cache_hash.zig | 108 ++++++---- lib/std/target.zig | 2 - lib/std/zig.zig | 53 ++++- src-self-hosted/Module.zig | 353 +++++++++++++++++++++++++++++---- src-self-hosted/link.zig | 31 ++- src-self-hosted/link/C.zig | 3 + src-self-hosted/link/Coff.zig | 3 + src-self-hosted/link/Elf.zig | 21 +- src-self-hosted/link/MachO.zig | 3 + src-self-hosted/link/Wasm.zig | 3 + src-self-hosted/main.zig | 106 ++++++++-- src-self-hosted/test.zig | 37 ++-- 13 files changed, 595 insertions(+), 129 deletions(-) diff --git a/build.zig b/build.zig index 43b3b7eebf..d3846fbdf8 100644 --- a/build.zig +++ b/build.zig @@ -139,6 +139,7 @@ pub fn build(b: *Builder) !void { const is_wasmtime_enabled = b.option(bool, "enable-wasmtime", "Use Wasmtime to enable and run WASI libstd tests") orelse false; const glibc_multi_dir = b.option([]const u8, "enable-foreign-glibc", "Provide directory with glibc installations to run cross compiled tests that link glibc"); + test_stage2.addBuildOption(bool, "have_llvm", enable_llvm); test_stage2.addBuildOption(bool, "enable_qemu", is_qemu_enabled); test_stage2.addBuildOption(bool, "enable_wine", is_wine_enabled); test_stage2.addBuildOption(bool, "enable_wasmtime", is_wasmtime_enabled); diff --git a/lib/std/cache_hash.zig b/lib/std/cache_hash.zig index 5cd8194e21..f98ec8d7f1 100644 --- a/lib/std/cache_hash.zig +++ b/lib/std/cache_hash.zig @@ -5,7 +5,6 @@ // and substantial portions of the software. const std = @import("std.zig"); const crypto = std.crypto; -const Hasher = crypto.auth.siphash.SipHash128(1, 3); // provides enough collision resistance for the CacheHash use cases, while being one of our fastest options right now const fs = std.fs; const base64 = std.base64; const ArrayList = std.ArrayList; @@ -23,6 +22,14 @@ const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN); const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024; +/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it +/// provides enough collision resistance for the CacheHash use cases, while being one of our +/// fastest options right now. +pub const Hasher = crypto.auth.siphash.SipHash128(1, 3); + +/// Initial state, that can be copied. +pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.minimum_key_length); + pub const File = struct { path: ?[]const u8, max_file_size: ?usize, @@ -45,52 +52,82 @@ pub const File = struct { /// CacheHash manages project-local `zig-cache` directories. /// This is not a general-purpose cache. -/// It was designed to be fast and simple, not to withstand attacks using specially-crafted input. +/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input. pub const CacheHash = struct { allocator: *Allocator, - hasher_init: Hasher, // initial state, that can be copied - hasher: Hasher, // current state for incremental hashing + /// Current state for incremental hashing. + hasher: Hasher, manifest_dir: fs.Dir, manifest_file: ?fs.File, manifest_dirty: bool, + owns_manifest_dir: bool, files: ArrayList(File), b64_digest: [BASE64_DIGEST_LEN]u8, /// Be sure to call release after successful initialization. pub fn init(allocator: *Allocator, dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash { - const hasher_init = Hasher.init(&[_]u8{0} ** Hasher.minimum_key_length); return CacheHash{ .allocator = allocator, - .hasher_init = hasher_init, .hasher = hasher_init, .manifest_dir = try dir.makeOpenPath(manifest_dir_path, .{}), .manifest_file = null, .manifest_dirty = false, + .owns_manifest_dir = true, + .files = ArrayList(File).init(allocator), + .b64_digest = undefined, + }; + } + + /// Allows one to fork a CacheHash instance into another one, which does not require an additional + /// directory handle to be opened. The new instance inherits the hash state. + pub fn clone(self: CacheHash) CacheHash { + assert(self.manifest_file == null); + assert(files.items.len == 0); + return .{ + .allocator = self.allocator, + .hasher = self.hasher, + .manifest_dir = self.manifest_dir, + .manifest_file = null, + .manifest_dirty = false, + .owns_manifest_dir = false, .files = ArrayList(File).init(allocator), .b64_digest = undefined, }; } /// Record a slice of bytes as an dependency of the process being cached - pub fn addSlice(self: *CacheHash, val: []const u8) void { + pub fn addBytes(self: *CacheHash, bytes: []const u8) void { assert(self.manifest_file == null); - self.hasher.update(val); - self.hasher.update(&[_]u8{0}); + self.hasher.update(mem.asBytes(&bytes.len)); + self.hasher.update(bytes); } - /// Convert the input value into bytes and record it as a dependency of the - /// process being cached - pub fn add(self: *CacheHash, val: anytype) void { + pub fn addListOfBytes(self: *CacheHash, list_of_bytes: []const []const u8) void { assert(self.manifest_file == null); - const valPtr = switch (@typeInfo(@TypeOf(val))) { - .Int => &val, - .Pointer => val, - else => &val, - }; + self.add(list_of_bytes.items.len); + for (list_of_bytes) |bytes| self.addBytes(bytes); + } - self.addSlice(mem.asBytes(valPtr)); + /// Convert the input value into bytes and record it as a dependency of the process being cached. + pub fn add(self: *CacheHash, x: anytype) void { + assert(self.manifest_file == null); + + switch (@TypeOf(x)) { + std.builtin.Version => { + self.add(x.major); + self.add(x.minor); + self.add(x.patch); + return; + }, + else => {}, + } + + switch (@typeInfo(@TypeOf(x))) { + .Bool, .Int, .Enum, .Array => self.addBytes(mem.asBytes(&x)), + else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))), + } } /// Add a file as a dependency of process being cached. When `CacheHash.hit` is @@ -122,7 +159,7 @@ pub const CacheHash = struct { .bin_digest = undefined, }; - self.addSlice(resolved_path); + self.addBytes(resolved_path); return idx; } @@ -143,7 +180,7 @@ pub const CacheHash = struct { base64_encoder.encode(self.b64_digest[0..], &bin_digest); - self.hasher = self.hasher_init; + self.hasher = hasher_init; self.hasher.update(&bin_digest); const manifest_file_path = try fmt.allocPrint(self.allocator, "{}.txt", .{self.b64_digest}); @@ -244,7 +281,7 @@ pub const CacheHash = struct { } var actual_digest: [BIN_DIGEST_LEN]u8 = undefined; - try hashFile(this_file, &actual_digest, self.hasher_init); + try hashFile(this_file, &actual_digest); if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) { cache_hash_file.bin_digest = actual_digest; @@ -262,7 +299,7 @@ pub const CacheHash = struct { // cache miss // keep the manifest file open // reset the hash - self.hasher = self.hasher_init; + self.hasher = hasher_init; self.hasher.update(&bin_digest); // Remove files not in the initial hash @@ -310,7 +347,7 @@ pub const CacheHash = struct { // Hash while reading from disk, to keep the contents in the cpu cache while // doing hashing. - var hasher = self.hasher_init; + var hasher = hasher_init; var off: usize = 0; while (true) { // give me everything you've got, captain @@ -323,7 +360,7 @@ pub const CacheHash = struct { ch_file.contents = contents; } else { - try hashFile(file, &ch_file.bin_digest, self.hasher_init); + try hashFile(file, &ch_file.bin_digest); } self.hasher.update(&ch_file.bin_digest); @@ -435,11 +472,12 @@ pub const CacheHash = struct { file.deinit(self.allocator); } self.files.deinit(); - self.manifest_dir.close(); + if (self.owns_manifest_dir) + self.manifest_dir.close(); } }; -fn hashFile(file: fs.File, bin_digest: []u8, hasher_init: anytype) !void { +fn hashFile(file: fs.File, bin_digest: []u8) !void { var buf: [1024]u8 = undefined; var hasher = hasher_init; @@ -509,7 +547,7 @@ test "cache file and then recall it" { ch.add(true); ch.add(@as(u16, 1234)); - ch.add("1234"); + ch.addBytes("1234"); _ = try ch.addFile(temp_file, null); // There should be nothing in the cache @@ -523,7 +561,7 @@ test "cache file and then recall it" { ch.add(true); ch.add(@as(u16, 1234)); - ch.add("1234"); + ch.addBytes("1234"); _ = try ch.addFile(temp_file, null); // Cache hit! We just "built" the same file @@ -577,7 +615,7 @@ test "check that changing a file makes cache fail" { var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); defer ch.release(); - ch.add("1234"); + ch.addBytes("1234"); const temp_file_idx = try ch.addFile(temp_file, 100); // There should be nothing in the cache @@ -594,7 +632,7 @@ test "check that changing a file makes cache fail" { var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); defer ch.release(); - ch.add("1234"); + ch.addBytes("1234"); const temp_file_idx = try ch.addFile(temp_file, 100); // A file that we depend on has been updated, so the cache should not contain an entry for it @@ -628,7 +666,7 @@ test "no file inputs" { var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); defer ch.release(); - ch.add("1234"); + ch.addBytes("1234"); // There should be nothing in the cache testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit()); @@ -639,7 +677,7 @@ test "no file inputs" { var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); defer ch.release(); - ch.add("1234"); + ch.addBytes("1234"); digest2 = (try ch.hit()).?; } @@ -674,7 +712,7 @@ test "CacheHashes with files added after initial hash work" { var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); defer ch.release(); - ch.add("1234"); + ch.addBytes("1234"); _ = try ch.addFile(temp_file1, null); // There should be nothing in the cache @@ -688,7 +726,7 @@ test "CacheHashes with files added after initial hash work" { var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); defer ch.release(); - ch.add("1234"); + ch.addBytes("1234"); _ = try ch.addFile(temp_file1, null); digest2 = (try ch.hit()).?; @@ -707,7 +745,7 @@ test "CacheHashes with files added after initial hash work" { var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); defer ch.release(); - ch.add("1234"); + ch.addBytes("1234"); _ = try ch.addFile(temp_file1, null); // A file that we depend on has been updated, so the cache should not contain an entry for it diff --git a/lib/std/target.zig b/lib/std/target.zig index 37425a9a29..a245868dc0 100644 --- a/lib/std/target.zig +++ b/lib/std/target.zig @@ -465,8 +465,6 @@ pub const Target = struct { }; pub const ObjectFormat = enum { - /// TODO Get rid of this one. - unknown, coff, pe, elf, diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 1dedce4067..b6dfada6cc 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -71,17 +71,52 @@ pub fn binNameAlloc( target: std.Target, output_mode: std.builtin.OutputMode, link_mode: ?std.builtin.LinkMode, + object_format: ?std.Target.ObjectFormat, ) error{OutOfMemory}![]u8 { - switch (output_mode) { - .Exe => return std.fmt.allocPrint(allocator, "{}{}", .{ root_name, target.exeFileExt() }), - .Lib => { - const suffix = switch (link_mode orelse .Static) { - .Static => target.staticLibSuffix(), - .Dynamic => target.dynamicLibSuffix(), - }; - return std.fmt.allocPrint(allocator, "{}{}{}", .{ target.libPrefix(), root_name, suffix }); + switch (object_format orelse target.getObjectFormat()) { + .coff, .pe => switch (output_mode) { + .Exe => { + const suffix = switch (target.os.tag) { + .uefi => ".efi", + else => ".exe", + }; + return std.fmt.allocPrint(allocator, "{}{}", .{ root_name, suffix }); + }, + .Lib => { + const suffix = switch (link_mode orelse .Static) { + .Static => ".lib", + .Dynamic => ".dll", + }; + return std.fmt.allocPrint(allocator, "{}{}{}", .{ target.libPrefix(), root_name, suffix }); + }, + .Obj => return std.fmt.allocPrint(allocator, "{}.obj", .{root_name}), }, - .Obj => return std.fmt.allocPrint(allocator, "{}{}", .{ root_name, target.oFileExt() }), + .elf => switch (output_mode) { + .Exe => return allocator.dupe(u8, root_name), + .Lib => { + const suffix = switch (link_mode orelse .Static) { + .Static => ".a", + .Dynamic => ".so", + }; + return std.fmt.allocPrint(allocator, "{}{}{}", .{ target.libPrefix(), root_name, suffix }); + }, + .Obj => return std.fmt.allocPrint(allocator, "{}.o", .{root_name}), + }, + .macho => switch (output_mode) { + .Exe => return allocator.dupe(u8, root_name), + .Lib => { + const suffix = switch (link_mode orelse .Static) { + .Static => ".a", + .Dynamic => ".dylib", + }; + return std.fmt.allocPrint(allocator, "{}{}{}", .{ target.libPrefix(), root_name, suffix }); + }, + .Obj => return std.fmt.allocPrint(allocator, "{}.o", .{root_name}), + }, + .wasm => return std.fmt.allocPrint(allocator, "{}.wasm", .{root_name}), + .c => return std.fmt.allocPrint(allocator, "{}.c", .{root_name}), + .hex => return std.fmt.allocPrint(allocator, "{}.ihex", .{root_name}), + .raw => return std.fmt.allocPrint(allocator, "{}.bin", .{root_name}), } } diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index dc48ae23e7..1ce7a92ebf 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -22,11 +22,12 @@ const trace = @import("tracy.zig").trace; const liveness = @import("liveness.zig"); const astgen = @import("astgen.zig"); const zir_sema = @import("zir_sema.zig"); +const build_options = @import("build_options"); /// General-purpose allocator. Used for both temporary and long-term storage. gpa: *Allocator, -/// Pointer to externally managed resource. -root_pkg: *Package, +/// Pointer to externally managed resource. `null` if there is no zig file being compiled. +root_pkg: ?*Package, /// Module owns this resource. /// The `Scope` is either a `Scope.ZIRModule` or `Scope.File`. root_scope: *Scope, @@ -48,22 +49,26 @@ export_owners: std.AutoArrayHashMapUnmanaged(*Decl, []*Export) = .{}, /// Maps fully qualified namespaced names to the Decl struct for them. decl_table: std.ArrayHashMapUnmanaged(Scope.NameHash, *Decl, Scope.name_hash_hash, Scope.name_hash_eql, false) = .{}, +c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{}, + link_error_flags: link.File.ErrorFlags = .{}, work_queue: std.fifo.LinearFifo(WorkItem, .Dynamic), /// We optimize memory usage for a compilation with no compile errors by storing the /// error messages and mapping outside of `Decl`. -/// The ErrorMsg memory is owned by the decl, using Module's allocator. +/// The ErrorMsg memory is owned by the decl, using Module's general purpose allocator. /// Note that a Decl can succeed but the Fn it represents can fail. In this case, /// a Decl can have a failed_decls entry but have analysis status of success. failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *ErrorMsg) = .{}, /// Using a map here for consistency with the other fields here. -/// The ErrorMsg memory is owned by the `Scope`, using Module's allocator. +/// The ErrorMsg memory is owned by the `Scope`, using Module's general purpose allocator. failed_files: std.AutoArrayHashMapUnmanaged(*Scope, *ErrorMsg) = .{}, /// Using a map here for consistency with the other fields here. -/// The ErrorMsg memory is owned by the `Export`, using Module's allocator. +/// The ErrorMsg memory is owned by the `Export`, using Module's general purpose allocator. failed_exports: std.AutoArrayHashMapUnmanaged(*Export, *ErrorMsg) = .{}, +/// The ErrorMsg memory is owned by the `CObject`, using Module's general purpose allocator. +failed_c_objects: std.AutoArrayHashMapUnmanaged(*CObject, *ErrorMsg) = .{}, /// Incrementing integer used to compare against the corresponding Decl /// field to determine whether a Decl's status applies to an ongoing update, or a @@ -79,10 +84,15 @@ deletion_set: std.ArrayListUnmanaged(*Decl) = .{}, /// Owned by Module. root_name: []u8, keep_source_files_loaded: bool, +use_clang: bool, /// Error tags and their values, tag names are duped with mod.gpa. global_error_set: std.StringHashMapUnmanaged(u16) = .{}, +c_source_files: []const []const u8, +clang_argv: []const []const u8, +cache: std.cache_hash.CacheHash, + pub const InnerError = error{ OutOfMemory, AnalysisFail }; const WorkItem = union(enum) { @@ -95,6 +105,9 @@ const WorkItem = union(enum) { /// The source file containing the Decl has been updated, and so the /// Decl may need its line number information updated in the debug info. update_line_number: *Decl, + /// Invoke the Clang compiler to create an object file, which gets linked + /// with the Module. + c_object: *CObject, }; pub const Export = struct { @@ -230,6 +243,7 @@ pub const Decl = struct { const src_decl = module.decls[self.src_index]; return src_decl.inst.src; }, + .none => unreachable, .file, .block => unreachable, .gen_zir => unreachable, .local_val => unreachable, @@ -282,6 +296,30 @@ pub const Decl = struct { } }; +pub const CObject = struct { + /// Relative to cwd. Owned by arena. + src_path: []const u8, + /// Owned by arena. + extra_flags: []const []const u8, + arena: std.heap.ArenaAllocator.State, + status: union(enum) { + new, + /// This is the output object path. Owned by gpa. + success: []u8, + /// There will be a corresponding ErrorMsg in Module.failed_c_objects. + /// This is the C source file contents (used for printing error messages). Owned by gpa. + failure: []u8, + }, + + pub fn destroy(self: *CObject, gpa: *Allocator) void { + switch (self.status) { + .new => {}, + .failure, .success => |data| gpa.free(data), + } + self.arena.promote(gpa).deinit(); + } +}; + /// Fn struct memory is owned by the Decl's TypedValue.Managed arena allocator. pub const Fn = struct { /// This memory owned by the Decl's TypedValue.Managed arena allocator. @@ -361,6 +399,7 @@ pub const Scope = struct { .zir_module => return &self.cast(ZIRModule).?.contents.module.arena.allocator, .file => unreachable, .container => unreachable, + .none => unreachable, } } @@ -376,6 +415,7 @@ pub const Scope = struct { .zir_module => null, .file => null, .container => null, + .none => unreachable, }; } @@ -390,6 +430,7 @@ pub const Scope = struct { .decl => return self.cast(DeclAnalysis).?.decl.scope, .file => return &self.cast(File).?.root_container.base, .zir_module, .container => return self, + .none => unreachable, } } @@ -406,6 +447,7 @@ pub const Scope = struct { .file => unreachable, .zir_module => return self.cast(ZIRModule).?.fullyQualifiedNameHash(name), .container => return self.cast(Container).?.fullyQualifiedNameHash(name), + .none => unreachable, } } @@ -414,6 +456,7 @@ pub const Scope = struct { switch (self.tag) { .file => return self.cast(File).?.contents.tree, .zir_module => unreachable, + .none => unreachable, .decl => return self.cast(DeclAnalysis).?.decl.scope.cast(Container).?.file_scope.contents.tree, .block => return self.cast(Block).?.decl.scope.cast(Container).?.file_scope.contents.tree, .gen_zir => return self.cast(GenZIR).?.decl.scope.cast(Container).?.file_scope.contents.tree, @@ -434,6 +477,7 @@ pub const Scope = struct { .zir_module => unreachable, .file => unreachable, .container => unreachable, + .none => unreachable, }; } @@ -444,6 +488,7 @@ pub const Scope = struct { .container => return @fieldParentPtr(Container, "base", base).file_scope.sub_file_path, .file => return @fieldParentPtr(File, "base", base).sub_file_path, .zir_module => return @fieldParentPtr(ZIRModule, "base", base).sub_file_path, + .none => unreachable, .block => unreachable, .gen_zir => unreachable, .local_val => unreachable, @@ -456,6 +501,7 @@ pub const Scope = struct { switch (base.tag) { .file => return @fieldParentPtr(File, "base", base).unload(gpa), .zir_module => return @fieldParentPtr(ZIRModule, "base", base).unload(gpa), + .none => {}, .block => unreachable, .gen_zir => unreachable, .local_val => unreachable, @@ -470,6 +516,7 @@ pub const Scope = struct { .container => return @fieldParentPtr(Container, "base", base).file_scope.getSource(module), .file => return @fieldParentPtr(File, "base", base).getSource(module), .zir_module => return @fieldParentPtr(ZIRModule, "base", base).getSource(module), + .none => unreachable, .gen_zir => unreachable, .local_val => unreachable, .local_ptr => unreachable, @@ -483,6 +530,7 @@ pub const Scope = struct { switch (base.tag) { .container => return @fieldParentPtr(Container, "base", base).removeDecl(child), .zir_module => return @fieldParentPtr(ZIRModule, "base", base).removeDecl(child), + .none => unreachable, .file => unreachable, .block => unreachable, .gen_zir => unreachable, @@ -505,6 +553,10 @@ pub const Scope = struct { scope_zir_module.deinit(gpa); gpa.destroy(scope_zir_module); }, + .none => { + const scope_none = @fieldParentPtr(None, "base", base); + gpa.destroy(scope_none); + }, .block => unreachable, .gen_zir => unreachable, .local_val => unreachable, @@ -527,6 +579,8 @@ pub const Scope = struct { zir_module, /// .zig source code. file, + /// There is no .zig or .zir source code being compiled in this Module. + none, /// struct, enum or union, every .file contains one of these. container, block, @@ -622,7 +676,7 @@ pub const Scope = struct { pub fn getSource(self: *File, module: *Module) ![:0]const u8 { switch (self.source) { .unloaded => { - const source = try module.root_pkg.root_src_dir.readFileAllocOptions( + const source = try module.root_pkg.?.root_src_dir.readFileAllocOptions( module.gpa, self.sub_file_path, std.math.maxInt(u32), @@ -638,6 +692,12 @@ pub const Scope = struct { } }; + /// For when there is no top level scope because there are no .zig files being compiled. + pub const None = struct { + pub const base_tag: Tag = .none; + base: Scope = Scope{ .tag = base_tag }, + }; + pub const ZIRModule = struct { pub const base_tag: Tag = .zir_module; base: Scope = Scope{ .tag = base_tag }, @@ -720,7 +780,7 @@ pub const Scope = struct { pub fn getSource(self: *ZIRModule, module: *Module) ![:0]const u8 { switch (self.source) { .unloaded => { - const source = try module.root_pkg.root_src_dir.readFileAllocOptions( + const source = try module.root_pkg.?.root_src_dir.readFileAllocOptions( module.gpa, self.sub_file_path, std.math.maxInt(u32), @@ -855,20 +915,81 @@ pub const AllErrors = struct { pub const InitOptions = struct { target: std.Target, root_name: []const u8, - root_pkg: *Package, + root_pkg: ?*Package, output_mode: std.builtin.OutputMode, bin_file_dir: ?std.fs.Dir = null, bin_file_path: []const u8, + emit_h: ?[]const u8 = null, link_mode: ?std.builtin.LinkMode = null, object_format: ?std.builtin.ObjectFormat = null, optimize_mode: std.builtin.Mode = .Debug, keep_source_files_loaded: bool = false, + clang_argv: []const []const u8 = &[0][]const u8{}, + lib_dirs: []const []const u8 = &[0][]const u8{}, + rpath_list: []const []const u8 = &[0][]const u8{}, + c_source_files: []const []const u8 = &[0][]const u8{}, + link_objects: []const []const u8 = &[0][]const u8{}, + framework_dirs: []const []const u8 = &[0][]const u8{}, + frameworks: []const []const u8 = &[0][]const u8{}, + system_libs: []const []const u8 = &[0][]const u8{}, + have_libc: bool = false, + have_libcpp: bool = false, + want_pic: ?bool = null, + want_sanitize_c: ?bool = null, + use_llvm: ?bool = null, + use_lld: ?bool = null, + use_clang: ?bool = null, + rdynamic: bool = false, + strip: bool = false, + linker_script: ?[]const u8 = null, + version_script: ?[]const u8 = null, + disable_c_depfile: bool = false, + override_soname: ?[]const u8 = null, + linker_optimization: ?[]const u8 = null, + linker_gc_sections: ?bool = null, + linker_allow_shlib_undefined: ?bool = null, + linker_bind_global_refs_locally: ?bool = null, + linker_z_nodelete: bool = false, + linker_z_defs: bool = false, + stack_size_override: u64 = 0, + compiler_id: [16]u8, }; pub fn init(gpa: *Allocator, options: InitOptions) !Module { const root_name = try gpa.dupe(u8, options.root_name); errdefer gpa.free(root_name); + const ofmt = options.object_format orelse options.target.getObjectFormat(); + + // Make a decision on whether to use LLD or our own linker. + const use_lld = if (options.use_lld) |explicit| explicit else blk: { + if (!build_options.have_llvm) + break :blk false; + + if (ofmt == .c) + break :blk false; + + // Our linker can't handle objects or most advanced options yet. + if (options.link_objects.len != 0 or + options.c_source_files.len != 0 or + options.frameworks.len != 0 or + options.system_libs.len != 0 or + options.have_libc or options.have_libcpp or + options.linker_script != null or options.version_script != null) + { + break :blk true; + } + break :blk false; + }; + + // Make a decision on whether to use LLVM or our own backend. + const use_llvm = if (options.use_llvm) |explicit| explicit else blk: { + // We would want to prefer LLVM for release builds when it is available, however + // we don't have an LLVM backend yet :) + // We would also want to prefer LLVM for architectures that we don't have self-hosted support for too. + break :blk false; + }; + const bin_file_dir = options.bin_file_dir orelse std.fs.cwd(); const bin_file = try link.File.openPath(gpa, bin_file_dir, options.bin_file_path, .{ .root_name = root_name, @@ -876,40 +997,131 @@ pub fn init(gpa: *Allocator, options: InitOptions) !Module { .target = options.target, .output_mode = options.output_mode, .link_mode = options.link_mode orelse .Static, - .object_format = options.object_format orelse options.target.getObjectFormat(), + .object_format = ofmt, .optimize_mode = options.optimize_mode, + .use_lld = use_lld, + .use_llvm = use_llvm, + .objects = options.link_objects, + .frameworks = options.frameworks, + .framework_dirs = options.framework_dirs, + .system_libs = options.system_libs, + .lib_dirs = options.lib_dirs, + .rpath_list = options.rpath_list, + .strip = options.strip, }); errdefer bin_file.destroy(); const root_scope = blk: { - if (mem.endsWith(u8, options.root_pkg.root_src_path, ".zig")) { - const root_scope = try gpa.create(Scope.File); - root_scope.* = .{ - .sub_file_path = options.root_pkg.root_src_path, - .source = .{ .unloaded = {} }, - .contents = .{ .not_available = {} }, - .status = .never_loaded, - .root_container = .{ - .file_scope = root_scope, + if (options.root_pkg) |root_pkg| { + if (mem.endsWith(u8, root_pkg.root_src_path, ".zig")) { + const root_scope = try gpa.create(Scope.File); + root_scope.* = .{ + .sub_file_path = root_pkg.root_src_path, + .source = .{ .unloaded = {} }, + .contents = .{ .not_available = {} }, + .status = .never_loaded, + .root_container = .{ + .file_scope = root_scope, + .decls = .{}, + }, + }; + break :blk &root_scope.base; + } else if (mem.endsWith(u8, root_pkg.root_src_path, ".zir")) { + const root_scope = try gpa.create(Scope.ZIRModule); + root_scope.* = .{ + .sub_file_path = root_pkg.root_src_path, + .source = .{ .unloaded = {} }, + .contents = .{ .not_available = {} }, + .status = .never_loaded, .decls = .{}, - }, - }; - break :blk &root_scope.base; - } else if (mem.endsWith(u8, options.root_pkg.root_src_path, ".zir")) { - const root_scope = try gpa.create(Scope.ZIRModule); - root_scope.* = .{ - .sub_file_path = options.root_pkg.root_src_path, - .source = .{ .unloaded = {} }, - .contents = .{ .not_available = {} }, - .status = .never_loaded, - .decls = .{}, - }; - break :blk &root_scope.base; + }; + break :blk &root_scope.base; + } else { + unreachable; + } } else { - unreachable; + const root_scope = try gpa.create(Scope.None); + root_scope.* = .{}; + break :blk &root_scope.base; } }; + // We put everything into the cache hash except for the root source file, because we want to + // find the same binary and incrementally update it even if the file contents changed. + const cache_dir = if (options.root_pkg) |root_pkg| root_pkg.root_src_dir else std.fs.cwd(); + var cache = try std.cache_hash.CacheHash.init(gpa, cache_dir, "zig-cache"); + errdefer cache.release(); + + // Now we will prepare hash state initializations to avoid redundantly computing hashes. + // First we add common things between things that apply to zig source and all c source files. + cache.add(options.compiler_id); + cache.add(options.optimize_mode); + cache.add(options.target.cpu.arch); + cache.addBytes(options.target.cpu.model.name); + cache.add(options.target.cpu.features.ints); + cache.add(options.target.os.tag); + switch (options.target.os.tag) { + .linux => { + cache.add(options.target.os.version_range.linux.range.min); + cache.add(options.target.os.version_range.linux.range.max); + cache.add(options.target.os.version_range.linux.glibc); + }, + .windows => { + cache.add(options.target.os.version_range.windows.min); + cache.add(options.target.os.version_range.windows.max); + }, + .freebsd, + .macosx, + .ios, + .tvos, + .watchos, + .netbsd, + .openbsd, + .dragonfly, + => { + cache.add(options.target.os.version_range.semver.min); + cache.add(options.target.os.version_range.semver.max); + }, + else => {}, + } + cache.add(options.target.abi); + cache.add(ofmt); + // TODO PIC (see detect_pic from codegen.cpp) + cache.add(bin_file.options.link_mode); + cache.add(options.strip); + + // Make a decision on whether to use Clang for translate-c and compiling C files. + const use_clang = if (options.use_clang) |explicit| explicit else blk: { + if (build_options.have_llvm) { + // Can't use it if we don't have it! + break :blk false; + } + // It's not planned to do our own translate-c or C compilation. + break :blk true; + }; + var c_object_table = std.AutoArrayHashMapUnmanaged(*CObject, void){}; + errdefer { + for (c_object_table.items()) |entry| entry.key.destroy(gpa); + c_object_table.deinit(gpa); + } + // Add a `CObject` for each `c_source_files`. + try c_object_table.ensureCapacity(gpa, options.c_source_files.len); + for (options.c_source_files) |c_source_file| { + var local_arena = std.heap.ArenaAllocator.init(gpa); + errdefer local_arena.deinit(); + + const c_object = try local_arena.allocator.create(CObject); + const src_path = try local_arena.allocator.dupe(u8, c_source_file); + + c_object.* = .{ + .status = .{ .new = {} }, + .src_path = src_path, + .extra_flags = &[0][]const u8{}, + .arena = local_arena.state, + }; + c_object_table.putAssumeCapacityNoClobber(c_object, {}); + } + return Module{ .gpa = gpa, .root_name = root_name, @@ -920,6 +1132,11 @@ pub fn init(gpa: *Allocator, options: InitOptions) !Module { .bin_file = bin_file, .work_queue = std.fifo.LinearFifo(WorkItem, .Dynamic).init(gpa), .keep_source_files_loaded = options.keep_source_files_loaded, + .use_clang = use_clang, + .clang_argv = options.clang_argv, + .c_source_files = options.c_source_files, + .cache = cache, + .c_object_table = c_object_table, }; } @@ -935,11 +1152,21 @@ pub fn deinit(self: *Module) void { } self.decl_table.deinit(gpa); + for (self.c_object_table.items()) |entry| { + entry.key.destroy(gpa); + } + self.c_object_table.deinit(gpa); + for (self.failed_decls.items()) |entry| { entry.value.destroy(gpa); } self.failed_decls.deinit(gpa); + for (self.failed_c_objects.items()) |entry| { + entry.value.destroy(gpa); + } + self.failed_c_objects.deinit(gpa); + for (self.failed_files.items()) |entry| { entry.value.destroy(gpa); } @@ -969,6 +1196,7 @@ pub fn deinit(self: *Module) void { gpa.free(entry.key); } self.global_error_set.deinit(gpa); + self.cache.release(); self.* = undefined; } @@ -995,7 +1223,15 @@ pub fn update(self: *Module) !void { self.generation += 1; - // TODO Use the cache hash file system to detect which source files changed. + // For compiling C objects, we rely on the cache hash system to avoid duplicating work. + // TODO Look into caching this data in memory to improve performance. + // Add a WorkItem for each C object. + try self.work_queue.ensureUnusedCapacity(self.c_object_table.items().len); + for (self.c_object_table.items()) |entry| { + self.work_queue.writeItemAssumeCapacity(.{ .c_object = entry.key }); + } + + // TODO Detect which source files changed. // Until then we simulate a full cache miss. Source files could have been loaded for any reason; // to force a refresh we unload now. if (self.root_scope.cast(Scope.File)) |zig_file| { @@ -1053,6 +1289,7 @@ pub fn makeBinFileWritable(self: *Module) !void { pub fn totalErrorCount(self: *Module) usize { const total = self.failed_decls.items().len + + self.failed_c_objects.items().len + self.failed_files.items().len + self.failed_exports.items().len; return if (total == 0) @boolToInt(self.link_error_flags.no_entry_point_found) else total; @@ -1065,6 +1302,12 @@ pub fn getAllErrorsAlloc(self: *Module) !AllErrors { var errors = std.ArrayList(AllErrors.Message).init(self.gpa); defer errors.deinit(); + for (self.failed_c_objects.items()) |entry| { + const c_object = entry.key; + const err_msg = entry.value; + const source = c_object.status.failure; + try AllErrors.add(&arena, &errors, c_object.src_path, source, err_msg.*); + } for (self.failed_files.items()) |entry| { const scope = entry.key; const err_msg = entry.value; @@ -1085,8 +1328,14 @@ pub fn getAllErrorsAlloc(self: *Module) !AllErrors { } if (errors.items.len == 0 and self.link_error_flags.no_entry_point_found) { + const global_err_src_path = blk: { + if (self.root_pkg) |root_pkg| break :blk root_pkg.root_src_path; + if (self.c_source_files.len != 0) break :blk self.c_source_files[0]; + if (self.bin_file.options.objects.len != 0) break :blk self.bin_file.options.objects[0]; + break :blk "(no file)"; + }; try errors.append(.{ - .src_path = self.root_pkg.root_src_path, + .src_path = global_err_src_path, .line = 0, .column = 0, .byte_offset = 0, @@ -1175,6 +1424,41 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void { decl.analysis = .codegen_failure_retryable; }; }, + .c_object => |c_object| { + // Free the previous attempt. + switch (c_object.status) { + .new => {}, + .success => |o_file_path| { + self.gpa.free(o_file_path); + c_object.status = .{ .new = {} }; + }, + .failure => |source| { + self.failed_c_objects.removeAssertDiscard(c_object); + self.gpa.free(source); + + c_object.status = .{ .new = {} }; + }, + } + if (!build_options.have_llvm) { + try self.failed_c_objects.ensureCapacity(self.gpa, self.failed_c_objects.items().len + 1); + self.failed_c_objects.putAssumeCapacityNoClobber(c_object, try ErrorMsg.create( + self.gpa, + 0, + "clang not available: compiler not built with LLVM extensions enabled", + .{}, + )); + c_object.status = .{ .failure = "" }; + continue; + } + try self.failed_c_objects.ensureCapacity(self.gpa, self.failed_c_objects.items().len + 1); + self.failed_c_objects.putAssumeCapacityNoClobber(c_object, try ErrorMsg.create( + self.gpa, + 0, + "TODO: implement invoking clang to compile C source files", + .{}, + )); + c_object.status = .{ .failure = "" }; + }, }; } @@ -3161,6 +3445,7 @@ fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Err zir_module.status = .loaded_sema_failure; self.failed_files.putAssumeCapacityNoClobber(scope, err_msg); }, + .none => unreachable, .file => unreachable, .container => unreachable, } diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index fff69a6bbd..14166e7e30 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -16,14 +16,36 @@ pub const Options = struct { object_format: std.builtin.ObjectFormat, optimize_mode: std.builtin.Mode, root_name: []const u8, - root_pkg: *const Package, + root_pkg: ?*const Package, /// Used for calculating how much space to reserve for symbols in case the binary file /// does not already have a symbol table. symbol_count_hint: u64 = 32, /// Used for calculating how much space to reserve for executable program code in case - /// the binary file deos not already have such a section. + /// the binary file does not already have such a section. program_code_size_hint: u64 = 256 * 1024, entry_addr: ?u64 = null, + /// Set to `true` to omit debug info. + strip: bool = false, + /// If this is true then this link code is responsible for outputting an object + /// file and then using LLD to link it together with the link options and other objects. + /// Otherwise (depending on `use_llvm`) this link code directly outputs and updates the final binary. + use_lld: bool = false, + /// If this is true then this link code is responsible for making an LLVM IR Module, + /// outputting it to an object file, and then linking that together with link options and + /// other objects. + /// Otherwise (depending on `use_lld`) this link code directly outputs and updates the final binary. + use_llvm: bool = false, + + objects: []const []const u8 = &[0][]const u8{}, + framework_dirs: []const []const u8 = &[0][]const u8{}, + frameworks: []const []const u8 = &[0][]const u8{}, + system_libs: []const []const u8 = &[0][]const u8{}, + lib_dirs: []const []const u8 = &[0][]const u8{}, + rpath_list: []const []const u8 = &[0][]const u8{}, + + pub fn effectiveOutputMode(options: Options) std.builtin.OutputMode { + return if (options.use_lld) .Obj else options.output_mode; + } }; pub const File = struct { @@ -67,14 +89,13 @@ pub const File = struct { /// and does not cause Illegal Behavior. This operation is not atomic. pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: Options) !*File { switch (options.object_format) { - .unknown => unreachable, .coff, .pe => return Coff.openPath(allocator, dir, sub_path, options), .elf => return Elf.openPath(allocator, dir, sub_path, options), .macho => return MachO.openPath(allocator, dir, sub_path, options), .wasm => return Wasm.openPath(allocator, dir, sub_path, options), .c => return C.openPath(allocator, dir, sub_path, options), - .hex => return error.TODOImplementHex, - .raw => return error.TODOImplementRaw, + .hex => return error.HexObjectFormatUnimplemented, + .raw => return error.RawObjectFormatUnimplemented, } } diff --git a/src-self-hosted/link/C.zig b/src-self-hosted/link/C.zig index 69eabd1f8b..ae0516dae0 100644 --- a/src-self-hosted/link/C.zig +++ b/src-self-hosted/link/C.zig @@ -25,6 +25,9 @@ error_msg: *Module.ErrorMsg = undefined, pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: link.Options) !*File { assert(options.object_format == .c); + if (options.use_llvm) return error.LLVM_HasNoCBackend; + if (options.use_lld) return error.LLD_HasNoCBackend; + const file = try dir.createFile(sub_path, .{ .truncate = true, .read = true, .mode = link.determineMode(options) }); errdefer file.close(); diff --git a/src-self-hosted/link/Coff.zig b/src-self-hosted/link/Coff.zig index 4d1f95e567..4cab7a75fd 100644 --- a/src-self-hosted/link/Coff.zig +++ b/src-self-hosted/link/Coff.zig @@ -113,6 +113,9 @@ pub const SrcFn = void; pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: link.Options) !*link.File { assert(options.object_format == .coff); + if (options.use_llvm) return error.LLVM_BackendIsTODO_ForCoff; // TODO + if (options.use_lld) return error.LLD_LinkingIsTODO_ForCoff; // TODO + const file = try dir.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options) }); errdefer file.close(); diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index e5acde947c..8b75662d35 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -219,6 +219,9 @@ pub const SrcFn = struct { pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: link.Options) !*File { assert(options.object_format == .elf); + if (options.use_llvm) return error.LLVM_BackendIsTODO_ForELF; // TODO + if (options.use_lld) return error.LLD_LinkingIsTODOForELF; // TODO + const file = try dir.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options) }); errdefer file.close(); @@ -235,7 +238,7 @@ pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, option /// Returns error.IncrFailed if incremental update could not be performed. fn openFile(allocator: *Allocator, file: fs.File, options: link.Options) !Elf { - switch (options.output_mode) { + switch (options.effectiveOutputMode()) { .Exe => {}, .Obj => {}, .Lib => return error.IncrFailed, @@ -264,7 +267,7 @@ fn openFile(allocator: *Allocator, file: fs.File, options: link.Options) !Elf { /// Truncates the existing file contents and overwrites the contents. /// Returns an error if `file` is not already open with +read +write +seek abilities. fn createFile(allocator: *Allocator, file: fs.File, options: link.Options) !Elf { - switch (options.output_mode) { + switch (options.effectiveOutputMode()) { .Exe => {}, .Obj => {}, .Lib => return error.TODOImplementWritingLibFiles, @@ -861,8 +864,8 @@ pub fn flush(self: *Elf, module: *Module) !void { }, } // Write the form for the compile unit, which must match the abbrev table above. - const name_strp = try self.makeDebugString(self.base.options.root_pkg.root_src_path); - const comp_dir_strp = try self.makeDebugString(self.base.options.root_pkg.root_src_dir_path); + const name_strp = try self.makeDebugString(self.base.options.root_pkg.?.root_src_path); + const comp_dir_strp = try self.makeDebugString(self.base.options.root_pkg.?.root_src_dir_path); const producer_strp = try self.makeDebugString(link.producer_string); // Currently only one compilation unit is supported, so the address range is simply // identical to the main program header virtual address and memory size. @@ -1031,7 +1034,7 @@ pub fn flush(self: *Elf, module: *Module) !void { 0, // include_directories (none except the compilation unit cwd) }); // file_names[0] - di_buf.appendSliceAssumeCapacity(self.base.options.root_pkg.root_src_path); // relative path name + di_buf.appendSliceAssumeCapacity(self.base.options.root_pkg.?.root_src_path); // relative path name di_buf.appendSliceAssumeCapacity(&[_]u8{ 0, // null byte for the relative path name 0, // directory_index @@ -1195,7 +1198,7 @@ pub fn flush(self: *Elf, module: *Module) !void { } self.shdr_table_dirty = false; } - if (self.entry_addr == null and self.base.options.output_mode == .Exe) { + if (self.entry_addr == null and self.base.options.effectiveOutputMode() == .Exe) { log.debug("flushing. no_entry_point_found = true\n", .{}); self.error_flags.no_entry_point_found = true; } else { @@ -1255,7 +1258,7 @@ fn writeElfHeader(self: *Elf) !void { assert(index == 16); - const elf_type = switch (self.base.options.output_mode) { + const elf_type = switch (self.base.options.effectiveOutputMode()) { .Exe => elf.ET.EXEC, .Obj => elf.ET.REL, .Lib => switch (self.base.options.link_mode) { @@ -2430,8 +2433,8 @@ fn dbgLineNeededHeaderBytes(self: Elf) u32 { directory_count * 8 + file_name_count * 8 + // These are encoded as DW.FORM_string rather than DW.FORM_strp as we would like // because of a workaround for readelf and gdb failing to understand DWARFv5 correctly. - self.base.options.root_pkg.root_src_dir_path.len + - self.base.options.root_pkg.root_src_path.len); + self.base.options.root_pkg.?.root_src_dir_path.len + + self.base.options.root_pkg.?.root_src_path.len); } fn dbgInfoNeededHeaderBytes(self: Elf) u32 { diff --git a/src-self-hosted/link/MachO.zig b/src-self-hosted/link/MachO.zig index 13932e514a..95f0d26f28 100644 --- a/src-self-hosted/link/MachO.zig +++ b/src-self-hosted/link/MachO.zig @@ -137,6 +137,9 @@ pub const SrcFn = struct { pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: link.Options) !*File { assert(options.object_format == .macho); + if (options.use_llvm) return error.LLVM_BackendIsTODO_ForMachO; // TODO + if (options.use_lld) return error.LLD_LinkingIsTODO_ForMachO; // TODO + const file = try dir.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options) }); errdefer file.close(); diff --git a/src-self-hosted/link/Wasm.zig b/src-self-hosted/link/Wasm.zig index d8f172f584..cc5cb427cd 100644 --- a/src-self-hosted/link/Wasm.zig +++ b/src-self-hosted/link/Wasm.zig @@ -52,6 +52,9 @@ funcs: std.ArrayListUnmanaged(*Module.Decl) = .{}, pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: link.Options) !*link.File { assert(options.object_format == .wasm); + if (options.use_llvm) return error.LLVM_BackendIsTODO_ForWasm; // TODO + if (options.use_lld) return error.LLD_LinkingIsTODO_ForWasm; // TODO + // TODO: read the file and keep vaild parts instead of truncating const file = try dir.createFile(sub_path, .{ .truncate = true, .read = true }); errdefer file.close(); diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 3ee91d54a2..ddcd02be9e 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -13,6 +13,7 @@ const Package = @import("Package.zig"); const zir = @import("zir.zig"); const build_options = @import("build_options"); const warn = std.log.warn; +const introspect = @import("introspect.zig"); fn fatal(comptime format: []const u8, args: anytype) noreturn { std.log.emerg(format, args); @@ -231,7 +232,6 @@ pub fn buildOutputType( var root_src_file: ?[]const u8 = null; var version: std.builtin.Version = .{ .major = 0, .minor = 0, .patch = 0 }; var strip = false; - var emit_h = true; var watch = false; var debug_tokenize = false; var debug_ast_tree = false; @@ -248,6 +248,7 @@ pub fn buildOutputType( var target_dynamic_linker: ?[]const u8 = null; var target_ofmt: ?[]const u8 = null; var output_mode: std.builtin.OutputMode = undefined; + var emit_h: Emit = undefined; var ensure_libc_on_non_freestanding = false; var ensure_libcpp_on_non_freestanding = false; var have_libc = false; @@ -269,6 +270,9 @@ pub fn buildOutputType( var linker_z_nodelete = false; var linker_z_defs = false; var stack_size_override: u64 = 0; + var use_llvm: ?bool = null; + var use_lld: ?bool = null; + var use_clang: ?bool = null; var system_libs = std.ArrayList([]const u8).init(gpa); defer system_libs.deinit(); @@ -296,6 +300,10 @@ pub fn buildOutputType( if (arg_mode == .build) { output_mode = arg_mode.build; + emit_h = switch (output_mode) { + .Exe => .no, + .Obj, .Lib => .yes_default_path, + }; const args = all_args[2..]; var i: usize = 0; @@ -416,6 +424,18 @@ pub fn buildOutputType( want_pic = true; } else if (mem.eql(u8, arg, "-fno-PIC")) { want_pic = false; + } else if (mem.eql(u8, arg, "-fLLVM")) { + use_llvm = true; + } else if (mem.eql(u8, arg, "-fno-LLVM")) { + use_llvm = false; + } else if (mem.eql(u8, arg, "-fLLD")) { + use_lld = true; + } else if (mem.eql(u8, arg, "-fno-LLD")) { + use_lld = false; + } else if (mem.eql(u8, arg, "-fClang")) { + use_clang = true; + } else if (mem.eql(u8, arg, "-fno-Clang")) { + use_clang = false; } else if (mem.eql(u8, arg, "-rdynamic")) { rdynamic = true; } else if (mem.eql(u8, arg, "-femit-bin")) { @@ -430,6 +450,12 @@ pub fn buildOutputType( emit_zir = .{ .yes = arg["-femit-zir=".len..] }; } else if (mem.eql(u8, arg, "-fno-emit-zir")) { emit_zir = .no; + } else if (mem.eql(u8, arg, "-femit-h")) { + emit_h = .yes_default_path; + } else if (mem.startsWith(u8, arg, "-femit-h=")) { + emit_h = .{ .yes = arg["-femit-h=".len..] }; + } else if (mem.eql(u8, arg, "-fno-emit-h")) { + emit_h = .no; } else if (mem.eql(u8, arg, "-dynamic")) { link_mode = .Dynamic; } else if (mem.eql(u8, arg, "-static")) { @@ -491,7 +517,7 @@ pub fn buildOutputType( } } } else { - emit_h = false; + emit_h = .no; strip = true; ensure_libc_on_non_freestanding = true; ensure_libcpp_on_non_freestanding = arg_mode == .cpp; @@ -874,14 +900,6 @@ pub fn buildOutputType( } } - if (system_libs.items.len != 0) { - fatal("linking against system libraries not yet supported", .{}); - } - - const src_path = root_src_file orelse { - fatal("expected at least one file argument", .{}); - }; - const object_format: ?std.Target.ObjectFormat = blk: { const ofmt = target_ofmt orelse break :blk null; if (mem.eql(u8, ofmt, "elf")) { @@ -909,11 +927,14 @@ pub fn buildOutputType( .no => { fatal("-fno-emit-bin not supported yet", .{}); }, - .yes_default_path => if (object_format != null and object_format.? == .c) - try std.fmt.allocPrint(arena, "{}.c", .{root_name}) - else - try std.zig.binNameAlloc(arena, root_name, target_info.target, output_mode, link_mode), - + .yes_default_path => try std.zig.binNameAlloc( + arena, + root_name, + target_info.target, + output_mode, + link_mode, + object_format, + ), .yes => |p| p, }; @@ -930,10 +951,25 @@ pub fn buildOutputType( .yes => |p| p, }; - const root_pkg = try Package.create(gpa, fs.cwd(), ".", src_path); - defer root_pkg.destroy(); + const root_pkg = if (root_src_file) |src_path| try Package.create(gpa, fs.cwd(), ".", src_path) else null; + defer if (root_pkg) |pkg| pkg.destroy(); - var module = try Module.init(gpa, .{ + const emit_h_path: ?[]const u8 = switch (emit_h) { + .yes => |p| p, + .no => null, + .yes_default_path => try std.fmt.allocPrint(arena, "{}.h", .{root_name}), + }; + + // TODO Remove this, we'll have this error emitted lazily only if the features would end + // up actually getting used. + //if (!build_options.have_llvm) { + // if ((use_llvm orelse false) or (use_lld orelse false) or (use_clang orelse false)) + // fatal("-fLLVM, -fLLD, and -fClang unavailable: compiler not built with LLVM extensions enabled", .{}); + //} + + const compiler_id = try introspect.resolveCompilerId(gpa); + + var module = Module.init(gpa, .{ .root_name = root_name, .target = target_info.target, .output_mode = output_mode, @@ -944,7 +980,39 @@ pub fn buildOutputType( .object_format = object_format, .optimize_mode = build_mode, .keep_source_files_loaded = zir_out_path != null, - }); + .clang_argv = clang_argv.items, + .lib_dirs = lib_dirs.items, + .rpath_list = rpath_list.items, + .c_source_files = c_source_files.items, + .link_objects = link_objects.items, + .framework_dirs = framework_dirs.items, + .frameworks = frameworks.items, + .system_libs = system_libs.items, + .emit_h = emit_h_path, + .have_libc = have_libc, + .have_libcpp = have_libcpp, + .want_pic = want_pic, + .want_sanitize_c = want_sanitize_c, + .use_llvm = use_llvm, + .use_lld = use_lld, + .use_clang = use_clang, + .rdynamic = rdynamic, + .linker_script = linker_script, + .version_script = version_script, + .disable_c_depfile = disable_c_depfile, + .override_soname = override_soname, + .linker_optimization = linker_optimization, + .linker_gc_sections = linker_gc_sections, + .linker_allow_shlib_undefined = linker_allow_shlib_undefined, + .linker_bind_global_refs_locally = linker_bind_global_refs_locally, + .linker_z_nodelete = linker_z_nodelete, + .linker_z_defs = linker_z_defs, + .stack_size_override = stack_size_override, + .compiler_id = compiler_id, + .strip = strip, + }) catch |err| { + fatal("unable to initialize module: {}", .{@errorName(err)}); + }; defer module.deinit(); const stdin = std.io.getStdIn().inStream(); diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index aef48e198b..519cbb46a3 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -9,6 +9,7 @@ const enable_qemu: bool = build_options.enable_qemu; const enable_wine: bool = build_options.enable_wine; const enable_wasmtime: bool = build_options.enable_wasmtime; const glibc_multi_install_dir: ?[]const u8 = build_options.glibc_multi_install_dir; +const introspect = @import("introspect.zig"); const cheader = @embedFile("link/cbe.h"); @@ -435,7 +436,10 @@ pub const TestContext = struct { const root_pkg = try Package.create(allocator, tmp.dir, ".", tmp_src_path); defer root_pkg.destroy(); - const bin_name = try std.zig.binNameAlloc(arena, "test_case", target, case.output_mode, null); + const ofmt: ?std.builtin.ObjectFormat = if (case.cbe) .c else null; + const bin_name = try std.zig.binNameAlloc(arena, "test_case", target, case.output_mode, null, ofmt); + + const compiler_id = try introspect.resolveCompilerId(arena); var module = try Module.init(allocator, .{ .root_name = "test_case", @@ -450,7 +454,8 @@ pub const TestContext = struct { .bin_file_path = bin_name, .root_pkg = root_pkg, .keep_source_files_loaded = true, - .object_format = if (case.cbe) .c else null, + .object_format = ofmt, + .compiler_id = compiler_id, }); defer module.deinit(); @@ -693,23 +698,23 @@ pub const TestContext = struct { } var interpreter = spu.Interpreter(struct { - RAM: [0x10000]u8 = undefined, + RAM: [0x10000]u8 = undefined, - pub fn read8(bus: @This(), addr: u16) u8 { - return bus.RAM[addr]; - } - pub fn read16(bus: @This(), addr: u16) u16 { - return std.mem.readIntLittle(u16, bus.RAM[addr..][0..2]); - } + pub fn read8(bus: @This(), addr: u16) u8 { + return bus.RAM[addr]; + } + pub fn read16(bus: @This(), addr: u16) u16 { + return std.mem.readIntLittle(u16, bus.RAM[addr..][0..2]); + } - pub fn write8(bus: *@This(), addr: u16, val: u8) void { - bus.RAM[addr] = val; - } + pub fn write8(bus: *@This(), addr: u16, val: u8) void { + bus.RAM[addr] = val; + } - pub fn write16(bus: *@This(), addr: u16, val: u16) void { - std.mem.writeIntLittle(u16, bus.RAM[addr..][0..2], val); - } - }){ + pub fn write16(bus: *@This(), addr: u16, val: u16) void { + std.mem.writeIntLittle(u16, bus.RAM[addr..][0..2], val); + } + }){ .bus = .{}, }; From 35f334ae0fe0f7eef7f8610103e860527df37f42 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 8 Sep 2020 01:30:56 -0700 Subject: [PATCH 008/210] organize some TODO comments --- src-self-hosted/Module.zig | 2 ++ src-self-hosted/main.zig | 8 +------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 1ce7a92ebf..9e2e7d2cc3 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -1048,6 +1048,8 @@ pub fn init(gpa: *Allocator, options: InitOptions) !Module { // We put everything into the cache hash except for the root source file, because we want to // find the same binary and incrementally update it even if the file contents changed. + // TODO Look into storing this information in memory rather than on disk and solving + // serialization/deserialization of *all* incremental compilation state in a more generic way. const cache_dir = if (options.root_pkg) |root_pkg| root_pkg.root_src_dir else std.fs.cwd(); var cache = try std.cache_hash.CacheHash.init(gpa, cache_dir, "zig-cache"); errdefer cache.release(); diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index ddcd02be9e..73e29c6365 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -960,13 +960,7 @@ pub fn buildOutputType( .yes_default_path => try std.fmt.allocPrint(arena, "{}.h", .{root_name}), }; - // TODO Remove this, we'll have this error emitted lazily only if the features would end - // up actually getting used. - //if (!build_options.have_llvm) { - // if ((use_llvm orelse false) or (use_lld orelse false) or (use_clang orelse false)) - // fatal("-fLLVM, -fLLD, and -fClang unavailable: compiler not built with LLVM extensions enabled", .{}); - //} - + // TODO look into implementing compiler_id at build time so we don't have to compute it at runtime. const compiler_id = try introspect.resolveCompilerId(gpa); var module = Module.init(gpa, .{ From c99e34a00e1e839effbc8b257a400eb3b643fa12 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 8 Sep 2020 11:15:32 -0700 Subject: [PATCH 009/210] stage2: eliminate the "compiler id" concept Instead, append a "dirty suffix" to the version string when there are dirty git changes and use the version string as the compiler id. This avoids a dependency on the cache hash system, and saves time on first invocation of the compiler since it does not have to compute its compiler id. It also saves time by not having to check the cache for a saved compiler id. --- build.zig | 149 ++++++++++++++++++++-------------- src-self-hosted/Module.zig | 3 +- src-self-hosted/main.zig | 4 - src-self-hosted/print_env.zig | 7 -- src-self-hosted/test.zig | 4 - 5 files changed, 87 insertions(+), 80 deletions(-) diff --git a/build.zig b/build.zig index d3846fbdf8..1666d92274 100644 --- a/build.zig +++ b/build.zig @@ -56,69 +56,6 @@ pub fn build(b: *Builder) !void { const enable_llvm = b.option(bool, "enable-llvm", "Build self-hosted compiler with LLVM backend enabled") orelse false; const config_h_path_option = b.option([]const u8, "config_h", "Path to the generated config.h"); - if (!only_install_lib_files) { - var exe = b.addExecutable("zig", "src-self-hosted/main.zig"); - exe.install(); - exe.setBuildMode(mode); - exe.setTarget(target); - test_step.dependOn(&exe.step); - b.default_step.dependOn(&exe.step); - - exe.addBuildOption(bool, "have_llvm", enable_llvm); - if (enable_llvm) { - const config_h_text = if (config_h_path_option) |config_h_path| - try std.fs.cwd().readFileAlloc(b.allocator, toNativePathSep(b, config_h_path), max_config_h_bytes) - else - try findAndReadConfigH(b); - - var ctx = parseConfigH(b, config_h_text); - ctx.llvm = try findLLVM(b, ctx.llvm_config_exe); - - try configureStage2(b, exe, ctx); - } - const tracy = b.option([]const u8, "tracy", "Enable Tracy integration. Supply path to Tracy source"); - const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse enable_llvm; - if (link_libc) { - exe.linkLibC(); - test_stage2.linkLibC(); - } - - const log_scopes = b.option([]const []const u8, "log", "Which log scopes to enable") orelse &[0][]const u8{}; - const zir_dumps = b.option([]const []const u8, "dump-zir", "Which functions to dump ZIR for before codegen") orelse &[0][]const u8{}; - - const opt_version_string = b.option([]const u8, "version-string", "Override Zig version string. Default is to find out with git."); - const version = if (opt_version_string) |version| version else v: { - var code: u8 = undefined; - const version_untrimmed = b.execAllowFail(&[_][]const u8{ - "git", "-C", b.build_root, "name-rev", "HEAD", - "--tags", "--name-only", "--no-undefined", "--always", - }, &code, .Ignore) catch |err| { - std.debug.print( - \\Unable to determine zig version string: {} - \\Provide the zig version string explicitly using the `version-string` build option. - , .{err}); - std.process.exit(1); - }; - const trimmed = mem.trim(u8, version_untrimmed, " \n\r"); - break :v b.fmt("{}.{}.{}+{}", .{ zig_version.major, zig_version.minor, zig_version.patch, trimmed }); - }; - exe.addBuildOption([]const u8, "version", version); - - exe.addBuildOption([]const []const u8, "log_scopes", log_scopes); - exe.addBuildOption([]const []const u8, "zir_dumps", zir_dumps); - exe.addBuildOption(bool, "enable_tracy", tracy != null); - if (tracy) |tracy_path| { - const client_cpp = fs.path.join( - b.allocator, - &[_][]const u8{ tracy_path, "TracyClient.cpp" }, - ) catch unreachable; - exe.addIncludeDir(tracy_path); - exe.addCSourceFile(client_cpp, &[_][]const u8{ "-DTRACY_ENABLE=1", "-fno-sanitize=undefined" }); - exe.linkSystemLibraryName("c++"); - exe.linkLibC(); - } - } - b.installDirectory(InstallDirectoryOptions{ .source_dir = "lib", .install_dir = .Lib, @@ -132,6 +69,91 @@ pub fn build(b: *Builder) !void { }, }); + if (only_install_lib_files) + return; + + var exe = b.addExecutable("zig", "src-self-hosted/main.zig"); + exe.install(); + exe.setBuildMode(mode); + exe.setTarget(target); + test_step.dependOn(&exe.step); + b.default_step.dependOn(&exe.step); + + exe.addBuildOption(bool, "have_llvm", enable_llvm); + if (enable_llvm) { + const config_h_text = if (config_h_path_option) |config_h_path| + try std.fs.cwd().readFileAlloc(b.allocator, toNativePathSep(b, config_h_path), max_config_h_bytes) + else + try findAndReadConfigH(b); + + var ctx = parseConfigH(b, config_h_text); + ctx.llvm = try findLLVM(b, ctx.llvm_config_exe); + + try configureStage2(b, exe, ctx); + } + const tracy = b.option([]const u8, "tracy", "Enable Tracy integration. Supply path to Tracy source"); + const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse enable_llvm; + if (link_libc) { + exe.linkLibC(); + test_stage2.linkLibC(); + } + + const log_scopes = b.option([]const []const u8, "log", "Which log scopes to enable") orelse &[0][]const u8{}; + const zir_dumps = b.option([]const []const u8, "dump-zir", "Which functions to dump ZIR for before codegen") orelse &[0][]const u8{}; + + const opt_version_string = b.option([]const u8, "version-string", "Override Zig version string. Default is to find out with git."); + const version = if (opt_version_string) |version| version else v: { + const version_string = b.fmt("{}.{}.{}", .{ zig_version.major, zig_version.minor, zig_version.patch }); + + var code: u8 = undefined; + const git_sha_untrimmed = b.execAllowFail(&[_][]const u8{ + "git", "-C", b.build_root, "name-rev", "HEAD", + "--tags", "--name-only", "--no-undefined", "--always", + }, &code, .Ignore) catch { + break :v version_string; + }; + const git_sha_trimmed = mem.trim(u8, git_sha_untrimmed, " \n\r"); + // Detect dirty changes. + const diff_untrimmed = b.execAllowFail(&[_][]const u8{ + "git", "-C", b.build_root, "diff", "HEAD", + }, &code, .Ignore) catch |err| { + std.debug.print("Error executing git diff: {}", .{err}); + std.process.exit(1); + }; + const trimmed_diff = mem.trim(u8, diff_untrimmed, " \n\r"); + const dirty_suffix = if (trimmed_diff.len == 0) "" else s: { + const dirty_hash = std.hash.Wyhash.hash(0, trimmed_diff); + break :s b.fmt("dirty{x}", .{@truncate(u32, dirty_hash)}); + }; + + // This will look like e.g. "0.6.0^0" for a tag commit. + if (mem.endsWith(u8, git_sha_trimmed, "^0")) { + const git_ver_string = git_sha_trimmed[0 .. git_sha_trimmed.len - 2]; + if (!mem.eql(u8, git_ver_string, version_string)) { + std.debug.print("Expected git tag '{}', found '{}'", .{ version_string, git_ver_string }); + std.process.exit(1); + } + break :v b.fmt("{}{}", .{ version_string, dirty_suffix }); + } else { + break :v b.fmt("{}+{}{}", .{ version_string, git_sha_trimmed, dirty_suffix }); + } + }; + exe.addBuildOption([]const u8, "version", version); + + exe.addBuildOption([]const []const u8, "log_scopes", log_scopes); + exe.addBuildOption([]const []const u8, "zir_dumps", zir_dumps); + exe.addBuildOption(bool, "enable_tracy", tracy != null); + if (tracy) |tracy_path| { + const client_cpp = fs.path.join( + b.allocator, + &[_][]const u8{ tracy_path, "TracyClient.cpp" }, + ) catch unreachable; + exe.addIncludeDir(tracy_path); + exe.addCSourceFile(client_cpp, &[_][]const u8{ "-DTRACY_ENABLE=1", "-fno-sanitize=undefined" }); + exe.linkSystemLibraryName("c++"); + exe.linkLibC(); + } + const test_filter = b.option([]const u8, "test-filter", "Skip tests that do not match filter"); const is_wine_enabled = b.option(bool, "enable-wine", "Use Wine to run cross compiled Windows tests") orelse false; @@ -144,6 +166,7 @@ pub fn build(b: *Builder) !void { test_stage2.addBuildOption(bool, "enable_wine", is_wine_enabled); test_stage2.addBuildOption(bool, "enable_wasmtime", is_wasmtime_enabled); test_stage2.addBuildOption(?[]const u8, "glibc_multi_install_dir", glibc_multi_dir); + test_stage2.addBuildOption([]const u8, "version", version); const test_stage2_step = b.step("test-stage2", "Run the stage2 compiler tests"); test_stage2_step.dependOn(&test_stage2.step); diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 9e2e7d2cc3..04b6cc5ca7 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -952,7 +952,6 @@ pub const InitOptions = struct { linker_z_nodelete: bool = false, linker_z_defs: bool = false, stack_size_override: u64 = 0, - compiler_id: [16]u8, }; pub fn init(gpa: *Allocator, options: InitOptions) !Module { @@ -1056,7 +1055,7 @@ pub fn init(gpa: *Allocator, options: InitOptions) !Module { // Now we will prepare hash state initializations to avoid redundantly computing hashes. // First we add common things between things that apply to zig source and all c source files. - cache.add(options.compiler_id); + cache.addBytes(build_options.version); cache.add(options.optimize_mode); cache.add(options.target.cpu.arch); cache.addBytes(options.target.cpu.model.name); diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 73e29c6365..a398a6a5d1 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -960,9 +960,6 @@ pub fn buildOutputType( .yes_default_path => try std.fmt.allocPrint(arena, "{}.h", .{root_name}), }; - // TODO look into implementing compiler_id at build time so we don't have to compute it at runtime. - const compiler_id = try introspect.resolveCompilerId(gpa); - var module = Module.init(gpa, .{ .root_name = root_name, .target = target_info.target, @@ -1002,7 +999,6 @@ pub fn buildOutputType( .linker_z_nodelete = linker_z_nodelete, .linker_z_defs = linker_z_defs, .stack_size_override = stack_size_override, - .compiler_id = compiler_id, .strip = strip, }) catch |err| { fatal("unable to initialize module: {}", .{@errorName(err)}); diff --git a/src-self-hosted/print_env.zig b/src-self-hosted/print_env.zig index 9b68633d3e..907e9e234d 100644 --- a/src-self-hosted/print_env.zig +++ b/src-self-hosted/print_env.zig @@ -16,10 +16,6 @@ pub fn cmdEnv(gpa: *Allocator, args: []const []const u8, stdout: anytype) !void const global_cache_dir = try introspect.resolveGlobalCacheDir(gpa); defer gpa.free(global_cache_dir); - const compiler_id_digest = try introspect.resolveCompilerId(gpa); - var compiler_id_buf: [compiler_id_digest.len * 2]u8 = undefined; - const compiler_id = std.fmt.bufPrint(&compiler_id_buf, "{x}", .{compiler_id_digest}) catch unreachable; - var bos = std.io.bufferedOutStream(stdout); const bos_stream = bos.outStream(); @@ -32,9 +28,6 @@ pub fn cmdEnv(gpa: *Allocator, args: []const []const u8, stdout: anytype) !void try jws.objectField("std_dir"); try jws.emitString(zig_std_dir); - try jws.objectField("id"); - try jws.emitString(compiler_id); - try jws.objectField("global_cache_dir"); try jws.emitString(global_cache_dir); diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 519cbb46a3..cd41ff0d78 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -9,7 +9,6 @@ const enable_qemu: bool = build_options.enable_qemu; const enable_wine: bool = build_options.enable_wine; const enable_wasmtime: bool = build_options.enable_wasmtime; const glibc_multi_install_dir: ?[]const u8 = build_options.glibc_multi_install_dir; -const introspect = @import("introspect.zig"); const cheader = @embedFile("link/cbe.h"); @@ -439,8 +438,6 @@ pub const TestContext = struct { const ofmt: ?std.builtin.ObjectFormat = if (case.cbe) .c else null; const bin_name = try std.zig.binNameAlloc(arena, "test_case", target, case.output_mode, null, ofmt); - const compiler_id = try introspect.resolveCompilerId(arena); - var module = try Module.init(allocator, .{ .root_name = "test_case", .target = target, @@ -455,7 +452,6 @@ pub const TestContext = struct { .root_pkg = root_pkg, .keep_source_files_loaded = true, .object_format = ofmt, - .compiler_id = compiler_id, }); defer module.deinit(); From 193ad413f03322b047bbfe17c4b2b368ba6bc097 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 9 Sep 2020 00:05:38 -0700 Subject: [PATCH 010/210] stage2: compiling C objects with clang * add target_util.zig which has ported code from src/target.cpp * Module gains an arena that owns memory used during initialization that has the same lifetime as the Module. Useful for constructing file paths and lists of strings that have mixed lifetimes. - The Module memory itself is allocated in this arena. init/deinit are modified to be create/destroy. - root_name moves to the arena and no longer needs manual free * implement the ability to invoke `zig clang` as a subprocess - there are lots of TODOs that should be solved before merging * Module now requires a Random object and zig_lib_dir * Module now requires a path to its own executable or any zig executable that can do `zig clang`. * Wire up more CLI options. * Module creates "zig-cache" directory and "tmp" and "o" subdirectories ("h" is created by the cache_hash) * stubbed out some of the things linker code needs to do with TODO prints * delete dead code for computing compiler id. the previous commit eliminated the need for it. * add `zig translate-c` CLI option but it's not fully hooked up yet. It should be possible for this to be fully wired up before merging this branch. * `zig targets` now uses canonical data for available_libcs --- src-self-hosted/Module.zig | 890 +++++++++++++++++++++++------- src-self-hosted/codegen/llvm.zig | 125 +++++ src-self-hosted/introspect.zig | 43 -- src-self-hosted/link.zig | 3 + src-self-hosted/link/Elf.zig | 22 +- src-self-hosted/main.zig | 194 ++++--- src-self-hosted/print_targets.zig | 59 +- src-self-hosted/target.zig | 111 ++++ src-self-hosted/test.zig | 28 +- src-self-hosted/zir.zig | 6 +- src/main.cpp | 2 +- 11 files changed, 1076 insertions(+), 407 deletions(-) create mode 100644 src-self-hosted/codegen/llvm.zig create mode 100644 src-self-hosted/target.zig diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 04b6cc5ca7..10a26a4aa8 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -10,6 +10,7 @@ const log = std.log.scoped(.module); const BigIntConst = std.math.big.int.Const; const BigIntMutable = std.math.big.int.Mutable; const Target = std.Target; +const target_util = @import("target.zig"); const Package = @import("Package.zig"); const link = @import("link.zig"); const ir = @import("ir.zig"); @@ -26,6 +27,8 @@ const build_options = @import("build_options"); /// General-purpose allocator. Used for both temporary and long-term storage. gpa: *Allocator, +/// Arena-allocated memory used during initialization. Should be untouched until deinit. +arena_state: std.heap.ArenaAllocator.State, /// Pointer to externally managed resource. `null` if there is no zig file being compiled. root_pkg: ?*Package, /// Module owns this resource. @@ -85,6 +88,12 @@ deletion_set: std.ArrayListUnmanaged(*Decl) = .{}, root_name: []u8, keep_source_files_loaded: bool, use_clang: bool, +sanitize_c: bool, +/// When this is `true` it means invoking clang as a sub-process is expected to inherit +/// stdin, stdout, stderr, and if it returns non success, to forward the exit code. +/// Otherwise we attempt to parse the error messages and expose them via the Module API. +/// This is `true` for `zig cc`, `zig c++`, and `zig translate-c`. +clang_passthrough_mode: bool, /// Error tags and their values, tag names are duped with mod.gpa. global_error_set: std.StringHashMapUnmanaged(u16) = .{}, @@ -92,6 +101,12 @@ global_error_set: std.StringHashMapUnmanaged(u16) = .{}, c_source_files: []const []const u8, clang_argv: []const []const u8, cache: std.cache_hash.CacheHash, +/// Path to own executable for invoking `zig clang`. +self_exe_path: ?[]const u8, +zig_lib_dir: []const u8, +zig_cache_dir_path: []const u8, +libc_include_dir_list: []const []const u8, +rand: *std.rand.Random, pub const InnerError = error{ OutOfMemory, AnalysisFail }; @@ -913,10 +928,12 @@ pub const AllErrors = struct { }; pub const InitOptions = struct { - target: std.Target, + zig_lib_dir: []const u8, + target: Target, root_name: []const u8, root_pkg: ?*Package, output_mode: std.builtin.OutputMode, + rand: *std.rand.Random, bin_file_dir: ?std.fs.Dir = null, bin_file_path: []const u8, emit_h: ?[]const u8 = null, @@ -932,8 +949,8 @@ pub const InitOptions = struct { framework_dirs: []const []const u8 = &[0][]const u8{}, frameworks: []const []const u8 = &[0][]const u8{}, system_libs: []const []const u8 = &[0][]const u8{}, - have_libc: bool = false, - have_libcpp: bool = false, + link_libc: bool = false, + link_libcpp: bool = false, want_pic: ?bool = null, want_sanitize_c: ?bool = null, use_llvm: ?bool = null, @@ -943,170 +960,232 @@ pub const InitOptions = struct { strip: bool = false, linker_script: ?[]const u8 = null, version_script: ?[]const u8 = null, - disable_c_depfile: bool = false, override_soname: ?[]const u8 = null, linker_optimization: ?[]const u8 = null, linker_gc_sections: ?bool = null, + function_sections: ?bool = null, linker_allow_shlib_undefined: ?bool = null, linker_bind_global_refs_locally: ?bool = null, + disable_c_depfile: bool = false, linker_z_nodelete: bool = false, linker_z_defs: bool = false, + clang_passthrough_mode: bool = false, stack_size_override: u64 = 0, + self_exe_path: ?[]const u8 = null, }; -pub fn init(gpa: *Allocator, options: InitOptions) !Module { - const root_name = try gpa.dupe(u8, options.root_name); - errdefer gpa.free(root_name); +pub fn create(gpa: *Allocator, options: InitOptions) !*Module { + const mod: *Module = mod: { + // For allocations that have the same lifetime as Module. This arena is used only during this + // initialization and then is freed in deinit(). + var arena_allocator = std.heap.ArenaAllocator.init(gpa); + errdefer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; - const ofmt = options.object_format orelse options.target.getObjectFormat(); + // We put the `Module` itself in the arena. Freeing the arena will free the module. + // It's initialized later after we prepare the initialization options. + const mod = try arena.create(Module); + const root_name = try arena.dupe(u8, options.root_name); - // Make a decision on whether to use LLD or our own linker. - const use_lld = if (options.use_lld) |explicit| explicit else blk: { - if (!build_options.have_llvm) - break :blk false; + const ofmt = options.object_format orelse options.target.getObjectFormat(); - if (ofmt == .c) - break :blk false; + // Make a decision on whether to use LLD or our own linker. + const use_lld = if (options.use_lld) |explicit| explicit else blk: { + if (!build_options.have_llvm) + break :blk false; - // Our linker can't handle objects or most advanced options yet. - if (options.link_objects.len != 0 or - options.c_source_files.len != 0 or - options.frameworks.len != 0 or - options.system_libs.len != 0 or - options.have_libc or options.have_libcpp or - options.linker_script != null or options.version_script != null) - { - break :blk true; - } - break :blk false; - }; + if (ofmt == .c) + break :blk false; - // Make a decision on whether to use LLVM or our own backend. - const use_llvm = if (options.use_llvm) |explicit| explicit else blk: { - // We would want to prefer LLVM for release builds when it is available, however - // we don't have an LLVM backend yet :) - // We would also want to prefer LLVM for architectures that we don't have self-hosted support for too. - break :blk false; - }; - - const bin_file_dir = options.bin_file_dir orelse std.fs.cwd(); - const bin_file = try link.File.openPath(gpa, bin_file_dir, options.bin_file_path, .{ - .root_name = root_name, - .root_pkg = options.root_pkg, - .target = options.target, - .output_mode = options.output_mode, - .link_mode = options.link_mode orelse .Static, - .object_format = ofmt, - .optimize_mode = options.optimize_mode, - .use_lld = use_lld, - .use_llvm = use_llvm, - .objects = options.link_objects, - .frameworks = options.frameworks, - .framework_dirs = options.framework_dirs, - .system_libs = options.system_libs, - .lib_dirs = options.lib_dirs, - .rpath_list = options.rpath_list, - .strip = options.strip, - }); - errdefer bin_file.destroy(); - - const root_scope = blk: { - if (options.root_pkg) |root_pkg| { - if (mem.endsWith(u8, root_pkg.root_src_path, ".zig")) { - const root_scope = try gpa.create(Scope.File); - root_scope.* = .{ - .sub_file_path = root_pkg.root_src_path, - .source = .{ .unloaded = {} }, - .contents = .{ .not_available = {} }, - .status = .never_loaded, - .root_container = .{ - .file_scope = root_scope, - .decls = .{}, - }, - }; - break :blk &root_scope.base; - } else if (mem.endsWith(u8, root_pkg.root_src_path, ".zir")) { - const root_scope = try gpa.create(Scope.ZIRModule); - root_scope.* = .{ - .sub_file_path = root_pkg.root_src_path, - .source = .{ .unloaded = {} }, - .contents = .{ .not_available = {} }, - .status = .never_loaded, - .decls = .{}, - }; - break :blk &root_scope.base; - } else { - unreachable; + // Our linker can't handle objects or most advanced options yet. + if (options.link_objects.len != 0 or + options.c_source_files.len != 0 or + options.frameworks.len != 0 or + options.system_libs.len != 0 or + options.link_libc or options.link_libcpp or + options.linker_script != null or options.version_script != null) + { + break :blk true; } - } else { - const root_scope = try gpa.create(Scope.None); - root_scope.* = .{}; - break :blk &root_scope.base; - } - }; - - // We put everything into the cache hash except for the root source file, because we want to - // find the same binary and incrementally update it even if the file contents changed. - // TODO Look into storing this information in memory rather than on disk and solving - // serialization/deserialization of *all* incremental compilation state in a more generic way. - const cache_dir = if (options.root_pkg) |root_pkg| root_pkg.root_src_dir else std.fs.cwd(); - var cache = try std.cache_hash.CacheHash.init(gpa, cache_dir, "zig-cache"); - errdefer cache.release(); - - // Now we will prepare hash state initializations to avoid redundantly computing hashes. - // First we add common things between things that apply to zig source and all c source files. - cache.addBytes(build_options.version); - cache.add(options.optimize_mode); - cache.add(options.target.cpu.arch); - cache.addBytes(options.target.cpu.model.name); - cache.add(options.target.cpu.features.ints); - cache.add(options.target.os.tag); - switch (options.target.os.tag) { - .linux => { - cache.add(options.target.os.version_range.linux.range.min); - cache.add(options.target.os.version_range.linux.range.max); - cache.add(options.target.os.version_range.linux.glibc); - }, - .windows => { - cache.add(options.target.os.version_range.windows.min); - cache.add(options.target.os.version_range.windows.max); - }, - .freebsd, - .macosx, - .ios, - .tvos, - .watchos, - .netbsd, - .openbsd, - .dragonfly, - => { - cache.add(options.target.os.version_range.semver.min); - cache.add(options.target.os.version_range.semver.max); - }, - else => {}, - } - cache.add(options.target.abi); - cache.add(ofmt); - // TODO PIC (see detect_pic from codegen.cpp) - cache.add(bin_file.options.link_mode); - cache.add(options.strip); - - // Make a decision on whether to use Clang for translate-c and compiling C files. - const use_clang = if (options.use_clang) |explicit| explicit else blk: { - if (build_options.have_llvm) { - // Can't use it if we don't have it! break :blk false; + }; + + // Make a decision on whether to use LLVM or our own backend. + const use_llvm = if (options.use_llvm) |explicit| explicit else blk: { + // We would want to prefer LLVM for release builds when it is available, however + // we don't have an LLVM backend yet :) + // We would also want to prefer LLVM for architectures that we don't have self-hosted support for too. + break :blk false; + }; + + const bin_file_dir = options.bin_file_dir orelse std.fs.cwd(); + const bin_file = try link.File.openPath(gpa, bin_file_dir, options.bin_file_path, .{ + .root_name = root_name, + .root_pkg = options.root_pkg, + .target = options.target, + .output_mode = options.output_mode, + .link_mode = options.link_mode orelse .Static, + .object_format = ofmt, + .optimize_mode = options.optimize_mode, + .use_lld = use_lld, + .use_llvm = use_llvm, + .link_libc = options.link_libc, + .link_libcpp = options.link_libcpp, + .objects = options.link_objects, + .frameworks = options.frameworks, + .framework_dirs = options.framework_dirs, + .system_libs = options.system_libs, + .lib_dirs = options.lib_dirs, + .rpath_list = options.rpath_list, + .strip = options.strip, + .function_sections = options.function_sections orelse false, + }); + errdefer bin_file.destroy(); + + // We arena-allocate the root scope so there is no free needed. + const root_scope = blk: { + if (options.root_pkg) |root_pkg| { + if (mem.endsWith(u8, root_pkg.root_src_path, ".zig")) { + const root_scope = try gpa.create(Scope.File); + root_scope.* = .{ + .sub_file_path = root_pkg.root_src_path, + .source = .{ .unloaded = {} }, + .contents = .{ .not_available = {} }, + .status = .never_loaded, + .root_container = .{ + .file_scope = root_scope, + .decls = .{}, + }, + }; + break :blk &root_scope.base; + } else if (mem.endsWith(u8, root_pkg.root_src_path, ".zir")) { + const root_scope = try gpa.create(Scope.ZIRModule); + root_scope.* = .{ + .sub_file_path = root_pkg.root_src_path, + .source = .{ .unloaded = {} }, + .contents = .{ .not_available = {} }, + .status = .never_loaded, + .decls = .{}, + }; + break :blk &root_scope.base; + } else { + unreachable; + } + } else { + const root_scope = try gpa.create(Scope.None); + root_scope.* = .{}; + break :blk &root_scope.base; + } + }; + + // We put everything into the cache hash except for the root source file, because we want to + // find the same binary and incrementally update it even if the file contents changed. + // TODO Look into storing this information in memory rather than on disk and solving + // serialization/deserialization of *all* incremental compilation state in a more generic way. + const cache_parent_dir = if (options.root_pkg) |root_pkg| root_pkg.root_src_dir else std.fs.cwd(); + var cache_dir = try cache_parent_dir.makeOpenPath("zig-cache", .{}); + defer cache_dir.close(); + + try cache_dir.makePath("tmp"); + try cache_dir.makePath("o"); + // We need this string because of sending paths to clang as a child process. + const zig_cache_dir_path = if (options.root_pkg) |root_pkg| + try std.fmt.allocPrint(arena, "{}" ++ std.fs.path.sep_str ++ "zig-cache", .{root_pkg.root_src_dir_path}) + else + "zig-cache"; + + var cache = try std.cache_hash.CacheHash.init(gpa, cache_dir, "h"); + errdefer cache.release(); + + // Now we will prepare hash state initializations to avoid redundantly computing hashes. + // First we add common things between things that apply to zig source and all c source files. + cache.addBytes(build_options.version); + cache.add(options.optimize_mode); + cache.add(options.target.cpu.arch); + cache.addBytes(options.target.cpu.model.name); + cache.add(options.target.cpu.features.ints); + cache.add(options.target.os.tag); + switch (options.target.os.tag) { + .linux => { + cache.add(options.target.os.version_range.linux.range.min); + cache.add(options.target.os.version_range.linux.range.max); + cache.add(options.target.os.version_range.linux.glibc); + }, + .windows => { + cache.add(options.target.os.version_range.windows.min); + cache.add(options.target.os.version_range.windows.max); + }, + .freebsd, + .macosx, + .ios, + .tvos, + .watchos, + .netbsd, + .openbsd, + .dragonfly, + => { + cache.add(options.target.os.version_range.semver.min); + cache.add(options.target.os.version_range.semver.max); + }, + else => {}, } - // It's not planned to do our own translate-c or C compilation. - break :blk true; + cache.add(options.target.abi); + cache.add(ofmt); + // TODO PIC (see detect_pic from codegen.cpp) + cache.add(bin_file.options.link_mode); + cache.add(options.strip); + + // Make a decision on whether to use Clang for translate-c and compiling C files. + const use_clang = if (options.use_clang) |explicit| explicit else blk: { + if (build_options.have_llvm) { + // Can't use it if we don't have it! + break :blk false; + } + // It's not planned to do our own translate-c or C compilation. + break :blk true; + }; + + const libc_include_dir_list = try detectLibCIncludeDirs( + arena, + options.zig_lib_dir, + options.target, + options.link_libc, + ); + + const sanitize_c: bool = options.want_sanitize_c orelse switch (options.optimize_mode) { + .Debug, .ReleaseSafe => true, + .ReleaseSmall, .ReleaseFast => false, + }; + + mod.* = .{ + .gpa = gpa, + .arena_state = arena_allocator.state, + .zig_lib_dir = options.zig_lib_dir, + .zig_cache_dir_path = zig_cache_dir_path, + .root_name = root_name, + .root_pkg = options.root_pkg, + .root_scope = root_scope, + .bin_file_dir = bin_file_dir, + .bin_file_path = options.bin_file_path, + .bin_file = bin_file, + .work_queue = std.fifo.LinearFifo(WorkItem, .Dynamic).init(gpa), + .keep_source_files_loaded = options.keep_source_files_loaded, + .use_clang = use_clang, + .clang_argv = options.clang_argv, + .c_source_files = options.c_source_files, + .cache = cache, + .self_exe_path = options.self_exe_path, + .libc_include_dir_list = libc_include_dir_list, + .sanitize_c = sanitize_c, + .rand = options.rand, + .clang_passthrough_mode = options.clang_passthrough_mode, + }; + break :mod mod; }; - var c_object_table = std.AutoArrayHashMapUnmanaged(*CObject, void){}; - errdefer { - for (c_object_table.items()) |entry| entry.key.destroy(gpa); - c_object_table.deinit(gpa); - } + errdefer mod.destroy(); + // Add a `CObject` for each `c_source_files`. - try c_object_table.ensureCapacity(gpa, options.c_source_files.len); + try mod.c_object_table.ensureCapacity(gpa, options.c_source_files.len); for (options.c_source_files) |c_source_file| { var local_arena = std.heap.ArenaAllocator.init(gpa); errdefer local_arena.deinit(); @@ -1120,31 +1199,15 @@ pub fn init(gpa: *Allocator, options: InitOptions) !Module { .extra_flags = &[0][]const u8{}, .arena = local_arena.state, }; - c_object_table.putAssumeCapacityNoClobber(c_object, {}); + mod.c_object_table.putAssumeCapacityNoClobber(c_object, {}); } - return Module{ - .gpa = gpa, - .root_name = root_name, - .root_pkg = options.root_pkg, - .root_scope = root_scope, - .bin_file_dir = bin_file_dir, - .bin_file_path = options.bin_file_path, - .bin_file = bin_file, - .work_queue = std.fifo.LinearFifo(WorkItem, .Dynamic).init(gpa), - .keep_source_files_loaded = options.keep_source_files_loaded, - .use_clang = use_clang, - .clang_argv = options.clang_argv, - .c_source_files = options.c_source_files, - .cache = cache, - .c_object_table = c_object_table, - }; + return mod; } -pub fn deinit(self: *Module) void { +pub fn destroy(self: *Module) void { self.bin_file.destroy(); const gpa = self.gpa; - self.gpa.free(self.root_name); self.deletion_set.deinit(gpa); self.work_queue.deinit(); @@ -1198,7 +1261,9 @@ pub fn deinit(self: *Module) void { } self.global_error_set.deinit(gpa); self.cache.release(); - self.* = undefined; + + // This destroys `self`. + self.arena_state.promote(gpa).deinit(); } fn freeExportList(gpa: *Allocator, export_list: []*Export) void { @@ -1209,7 +1274,7 @@ fn freeExportList(gpa: *Allocator, export_list: []*Export) void { gpa.free(export_list); } -pub fn target(self: Module) std.Target { +pub fn getTarget(self: Module) Target { return self.bin_file.options.target; } @@ -1440,29 +1505,335 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void { c_object.status = .{ .new = {} }; }, } - if (!build_options.have_llvm) { - try self.failed_c_objects.ensureCapacity(self.gpa, self.failed_c_objects.items().len + 1); - self.failed_c_objects.putAssumeCapacityNoClobber(c_object, try ErrorMsg.create( - self.gpa, - 0, - "clang not available: compiler not built with LLVM extensions enabled", - .{}, - )); - c_object.status = .{ .failure = "" }; - continue; - } - try self.failed_c_objects.ensureCapacity(self.gpa, self.failed_c_objects.items().len + 1); - self.failed_c_objects.putAssumeCapacityNoClobber(c_object, try ErrorMsg.create( - self.gpa, - 0, - "TODO: implement invoking clang to compile C source files", - .{}, - )); - c_object.status = .{ .failure = "" }; + self.buildCObject(c_object) catch |err| switch (err) { + error.AnalysisFail => continue, + else => { + try self.failed_c_objects.ensureCapacity(self.gpa, self.failed_c_objects.items().len + 1); + self.failed_c_objects.putAssumeCapacityNoClobber(c_object, try ErrorMsg.create( + self.gpa, + 0, + "unable to build C object: {}", + .{@errorName(err)}, + )); + c_object.status = .{ .failure = "" }; + }, + }; }, }; } +fn buildCObject(mod: *Module, c_object: *CObject) !void { + const tracy = trace(@src()); + defer tracy.end(); + + if (!build_options.have_llvm) { + return mod.failCObj(c_object, "clang not available: compiler not built with LLVM extensions enabled", .{}); + } + const self_exe_path = mod.self_exe_path orelse + return mod.failCObj(c_object, "clang compilation disabled", .{}); + + var arena_allocator = std.heap.ArenaAllocator.init(mod.gpa); + defer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + var argv = std.ArrayList([]const u8).init(mod.gpa); + defer argv.deinit(); + + const c_source_basename = std.fs.path.basename(c_object.src_path); + // Special case when doing build-obj for just one C file. When there are more than one object + // file and building an object we need to link them together, but with just one it should go + // directly to the output file. + const direct_o = mod.c_source_files.len == 1 and mod.root_pkg == null and + mod.bin_file.options.output_mode == .Obj and mod.bin_file.options.objects.len == 0; + const o_basename_noext = if (direct_o) mod.root_name else mem.split(c_source_basename, ".").next().?; + const o_basename = try std.fmt.allocPrint(arena, "{}{}", .{ o_basename_noext, mod.getTarget().oFileExt() }); + + // We can't know the digest until we do the C compiler invocation, so we need a temporary filename. + const out_obj_path = try mod.tmpFilePath(arena, o_basename); + + try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang", "-c" }); + + const ext = classifyFileExt(c_object.src_path); + // TODO capture the .d file and deal with caching stuff + try mod.addCCArgs(arena, &argv, ext, false, null); + + try argv.append("-o"); + try argv.append(out_obj_path); + + try argv.append(c_object.src_path); + try argv.appendSlice(c_object.extra_flags); + + //for (argv.items) |arg| { + // std.debug.print("{} ", .{arg}); + //} + + const child = try std.ChildProcess.init(argv.items, arena); + defer child.deinit(); + + if (mod.clang_passthrough_mode) { + child.stdin_behavior = .Inherit; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; + + const term = child.spawnAndWait() catch |err| { + return mod.failCObj(c_object, "unable to spawn {}: {}", .{ argv.items[0], @errorName(err) }); + }; + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO make std.process.exit and std.ChildProcess exit code have the same type + // and forward it here. Currently it is u32 vs u8. + std.process.exit(1); + } + }, + else => std.process.exit(1), + } + } else { + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; + + try child.spawn(); + + const stdout_reader = child.stdout.?.reader(); + const stderr_reader = child.stderr.?.reader(); + + // TODO Need to poll to read these streams to prevent a deadlock (or rely on evented I/O). + const stdout = try stdout_reader.readAllAlloc(arena, std.math.maxInt(u32)); + const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); + + const term = child.wait() catch |err| { + return mod.failCObj(c_object, "unable to spawn {}: {}", .{ argv.items[0], @errorName(err) }); + }; + + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO parse clang stderr and turn it into an error message + // and then call failCObjWithOwnedErrorMsg + std.log.err("clang failed with stderr: {}", .{stderr}); + return mod.failCObj(c_object, "clang exited with code {}", .{code}); + } + }, + else => { + std.log.err("clang terminated with stderr: {}", .{stderr}); + return mod.failCObj(c_object, "clang terminated unexpectedly", .{}); + }, + } + } + + // TODO handle .d files + + // TODO rename into place + std.debug.print("TODO rename {} into cache dir\n", .{out_obj_path}); + + // TODO use the cache file name instead of tmp file name + const success_file_path = try mod.gpa.dupe(u8, out_obj_path); + c_object.status = .{ .success = success_file_path }; +} + +fn tmpFilePath(mod: *Module, arena: *Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 { + const s = std.fs.path.sep_str; + return std.fmt.allocPrint( + arena, + "{}" ++ s ++ "tmp" ++ s ++ "{x}-{}", + .{ mod.zig_cache_dir_path, mod.rand.int(u64), suffix }, + ); +} + +/// Add common C compiler args between translate-c and C object compilation. +fn addCCArgs( + mod: *Module, + arena: *Allocator, + argv: *std.ArrayList([]const u8), + ext: FileExt, + translate_c: bool, + out_dep_path: ?[]const u8, +) !void { + const target = mod.getTarget(); + + if (translate_c) { + try argv.appendSlice(&[_][]const u8{ "-x", "c" }); + } + + if (ext == .cpp) { + try argv.append("-nostdinc++"); + } + try argv.appendSlice(&[_][]const u8{ + "-nostdinc", + "-fno-spell-checking", + }); + + // We don't ever put `-fcolor-diagnostics` or `-fno-color-diagnostics` because in passthrough mode + // we want Clang to infer it, and in normal mode we always want it off, which will be true since + // clang will detect stderr as a pipe rather than a terminal. + if (!mod.clang_passthrough_mode) { + // Make stderr more easily parseable. + try argv.append("-fno-caret-diagnostics"); + } + + if (mod.bin_file.options.function_sections) { + try argv.append("-ffunction-sections"); + } + + try argv.ensureCapacity(argv.items.len + mod.bin_file.options.framework_dirs.len * 2); + for (mod.bin_file.options.framework_dirs) |framework_dir| { + argv.appendAssumeCapacity("-iframework"); + argv.appendAssumeCapacity(framework_dir); + } + + if (mod.bin_file.options.link_libcpp) { + const libcxx_include_path = try std.fs.path.join(arena, &[_][]const u8{ + mod.zig_lib_dir, "libcxx", "include", + }); + const libcxxabi_include_path = try std.fs.path.join(arena, &[_][]const u8{ + mod.zig_lib_dir, "libcxxabi", "include", + }); + + try argv.append("-isystem"); + try argv.append(libcxx_include_path); + + try argv.append("-isystem"); + try argv.append(libcxxabi_include_path); + + if (target.abi.isMusl()) { + try argv.append("-D_LIBCPP_HAS_MUSL_LIBC"); + } + try argv.append("-D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS"); + try argv.append("-D_LIBCXXABI_DISABLE_VISIBILITY_ANNOTATIONS"); + } + + const llvm_triple = try @import("codegen/llvm.zig").targetTriple(arena, target); + try argv.appendSlice(&[_][]const u8{ "-target", llvm_triple }); + + switch (ext) { + .c, .cpp, .h => { + // According to Rich Felker libc headers are supposed to go before C language headers. + // However as noted by @dimenus, appending libc headers before c_headers breaks intrinsics + // and other compiler specific items. + const c_headers_dir = try std.fs.path.join(arena, &[_][]const u8{ mod.zig_lib_dir, "include" }); + try argv.append("-isystem"); + try argv.append(c_headers_dir); + + for (mod.libc_include_dir_list) |include_dir| { + try argv.append("-isystem"); + try argv.append(include_dir); + } + + if (target.cpu.model.llvm_name) |llvm_name| { + try argv.appendSlice(&[_][]const u8{ + "-Xclang", "-target-cpu", "-Xclang", llvm_name, + }); + } + // TODO CLI args for target features + //if (g->zig_target->llvm_cpu_features != nullptr) { + // // https://github.com/ziglang/zig/issues/5017 + // SplitIterator it = memSplit(str(g->zig_target->llvm_cpu_features), str(",")); + // Optional> flag = SplitIterator_next(&it); + // while (flag.is_some) { + // try argv.append("-Xclang"); + // try argv.append("-target-feature"); + // try argv.append("-Xclang"); + // try argv.append(buf_ptr(buf_create_from_slice(flag.value))); + // flag = SplitIterator_next(&it); + // } + //} + if (translate_c) { + // This gives us access to preprocessing entities, presumably at the cost of performance. + try argv.append("-Xclang"); + try argv.append("-detailed-preprocessing-record"); + } + if (out_dep_path) |p| { + try argv.append("-MD"); + try argv.append("-MV"); + try argv.append("-MF"); + try argv.append(p); + } + }, + .assembly, .ll, .bc, .unknown => {}, + } + // TODO CLI args for cpu features when compiling assembly + //for (size_t i = 0; i < g->zig_target->llvm_cpu_features_asm_len; i += 1) { + // try argv.append(g->zig_target->llvm_cpu_features_asm_ptr[i]); + //} + + if (target.os.tag == .freestanding) { + try argv.append("-ffreestanding"); + } + + // windows.h has files such as pshpack1.h which do #pragma packing, triggering a clang warning. + // So for this target, we disable this warning. + if (target.os.tag == .windows and target.abi.isGnu()) { + try argv.append("-Wno-pragma-pack"); + } + + if (!mod.bin_file.options.strip) { + try argv.append("-g"); + } + + if (mod.haveFramePointer()) { + try argv.append("-fno-omit-frame-pointer"); + } else { + try argv.append("-fomit-frame-pointer"); + } + + if (mod.sanitize_c) { + try argv.append("-fsanitize=undefined"); + try argv.append("-fsanitize-trap=undefined"); + } + + switch (mod.bin_file.options.optimize_mode) { + .Debug => { + // windows c runtime requires -D_DEBUG if using debug libraries + try argv.append("-D_DEBUG"); + try argv.append("-Og"); + + if (mod.bin_file.options.link_libc) { + try argv.append("-fstack-protector-strong"); + try argv.append("--param"); + try argv.append("ssp-buffer-size=4"); + } else { + try argv.append("-fno-stack-protector"); + } + }, + .ReleaseSafe => { + // See the comment in the BuildModeFastRelease case for why we pass -O2 rather + // than -O3 here. + try argv.append("-O2"); + if (mod.bin_file.options.link_libc) { + try argv.append("-D_FORTIFY_SOURCE=2"); + try argv.append("-fstack-protector-strong"); + try argv.append("--param"); + try argv.append("ssp-buffer-size=4"); + } else { + try argv.append("-fno-stack-protector"); + } + }, + .ReleaseFast => { + try argv.append("-DNDEBUG"); + // Here we pass -O2 rather than -O3 because, although we do the equivalent of + // -O3 in Zig code, the justification for the difference here is that Zig + // has better detection and prevention of undefined behavior, so -O3 is safer for + // Zig code than it is for C code. Also, C programmers are used to their code + // running in -O2 and thus the -O3 path has been tested less. + try argv.append("-O2"); + try argv.append("-fno-stack-protector"); + }, + .ReleaseSmall => { + try argv.append("-DNDEBUG"); + try argv.append("-Os"); + try argv.append("-fno-stack-protector"); + }, + } + + // TODO add CLI args for PIC + //if (target_supports_fpic(g->zig_target) and g->have_pic) { + // try argv.append("-fPIC"); + //} + + try argv.appendSlice(mod.clang_argv); +} + pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void { const tracy = trace(@src()); defer tracy.end(); @@ -3041,7 +3412,7 @@ pub fn cmpNumeric( } else if (rhs_ty_tag == .ComptimeFloat) { break :x lhs.ty; } - if (lhs.ty.floatBits(self.target()) >= rhs.ty.floatBits(self.target())) { + if (lhs.ty.floatBits(self.getTarget()) >= rhs.ty.floatBits(self.getTarget())) { break :x lhs.ty; } else { break :x rhs.ty; @@ -3100,7 +3471,7 @@ pub fn cmpNumeric( } else if (lhs_is_float) { dest_float_type = lhs.ty; } else { - const int_info = lhs.ty.intInfo(self.target()); + const int_info = lhs.ty.intInfo(self.getTarget()); lhs_bits = int_info.bits + @boolToInt(!int_info.signed and dest_int_is_signed); } @@ -3135,7 +3506,7 @@ pub fn cmpNumeric( } else if (rhs_is_float) { dest_float_type = rhs.ty; } else { - const int_info = rhs.ty.intInfo(self.target()); + const int_info = rhs.ty.intInfo(self.getTarget()); rhs_bits = int_info.bits + @boolToInt(!int_info.signed and dest_int_is_signed); } @@ -3200,13 +3571,13 @@ pub fn resolvePeerTypes(self: *Module, scope: *Scope, instructions: []*Inst) !Ty next_inst.ty.isInt() and prev_inst.ty.isSignedInt() == next_inst.ty.isSignedInt()) { - if (prev_inst.ty.intInfo(self.target()).bits < next_inst.ty.intInfo(self.target()).bits) { + if (prev_inst.ty.intInfo(self.getTarget()).bits < next_inst.ty.intInfo(self.getTarget()).bits) { prev_inst = next_inst; } continue; } if (prev_inst.ty.isFloat() and next_inst.ty.isFloat()) { - if (prev_inst.ty.floatBits(self.target()) < next_inst.ty.floatBits(self.target())) { + if (prev_inst.ty.floatBits(self.getTarget()) < next_inst.ty.floatBits(self.getTarget())) { prev_inst = next_inst; } continue; @@ -3274,8 +3645,8 @@ pub fn coerce(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst if (inst.ty.zigTypeTag() == .Int and dest_type.zigTypeTag() == .Int) { assert(inst.value() == null); // handled above - const src_info = inst.ty.intInfo(self.target()); - const dst_info = dest_type.intInfo(self.target()); + const src_info = inst.ty.intInfo(self.getTarget()); + const dst_info = dest_type.intInfo(self.getTarget()); if ((src_info.signed == dst_info.signed and dst_info.bits >= src_info.bits) or // small enough unsigned ints can get casted to large enough signed ints (src_info.signed and !dst_info.signed and dst_info.bits > src_info.bits)) @@ -3289,8 +3660,8 @@ pub fn coerce(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst if (inst.ty.zigTypeTag() == .Float and dest_type.zigTypeTag() == .Float) { assert(inst.value() == null); // handled above - const src_bits = inst.ty.floatBits(self.target()); - const dst_bits = dest_type.floatBits(self.target()); + const src_bits = inst.ty.floatBits(self.getTarget()); + const dst_bits = dest_type.floatBits(self.getTarget()); if (dst_bits >= src_bits) { const b = try self.requireRuntimeBlock(scope, inst.src); return self.addUnOp(b, inst.src, dest_type, .floatcast, inst); @@ -3312,14 +3683,14 @@ pub fn coerceNum(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !?* } return self.fail(scope, inst.src, "TODO float to int", .{}); } else if (src_zig_tag == .Int or src_zig_tag == .ComptimeInt) { - if (!val.intFitsInType(dest_type, self.target())) { + if (!val.intFitsInType(dest_type, self.getTarget())) { return self.fail(scope, inst.src, "type {} cannot represent integer value {}", .{ inst.ty, val }); } return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = val }); } } else if (dst_zig_tag == .ComptimeFloat or dst_zig_tag == .Float) { if (src_zig_tag == .Float or src_zig_tag == .ComptimeFloat) { - const res = val.floatCast(scope.arena(), dest_type, self.target()) catch |err| switch (err) { + const res = val.floatCast(scope.arena(), dest_type, self.getTarget()) catch |err| switch (err) { error.Overflow => return self.fail( scope, inst.src, @@ -3370,6 +3741,22 @@ fn coerceArrayPtrToSlice(self: *Module, scope: *Scope, dest_type: Type, inst: *I return self.fail(scope, inst.src, "TODO implement coerceArrayPtrToSlice runtime instruction", .{}); } +fn failCObj(mod: *Module, c_object: *CObject, comptime format: []const u8, args: anytype) InnerError { + @setCold(true); + const err_msg = try ErrorMsg.create(mod.gpa, 0, "unable to build C object: " ++ format, args); + return mod.failCObjWithOwnedErrorMsg(c_object, err_msg); +} + +fn failCObjWithOwnedErrorMsg(mod: *Module, c_object: *CObject, err_msg: *ErrorMsg) InnerError { + { + errdefer err_msg.destroy(mod.gpa); + try mod.failed_c_objects.ensureCapacity(mod.gpa, mod.failed_c_objects.items().len + 1); + } + mod.failed_c_objects.putAssumeCapacityNoClobber(c_object, err_msg); + c_object.status = .{ .failure = "" }; + return error.AnalysisFail; +} + pub fn fail(self: *Module, scope: *Scope, src: usize, comptime format: []const u8, args: anytype) InnerError { @setCold(true); const err_msg = try ErrorMsg.create(self.gpa, src, format, args); @@ -3560,7 +3947,7 @@ pub fn intSub(allocator: *Allocator, lhs: Value, rhs: Value) !Value { pub fn floatAdd(self: *Module, scope: *Scope, float_type: Type, src: usize, lhs: Value, rhs: Value) !Value { var bit_count = switch (float_type.tag()) { .comptime_float => 128, - else => float_type.floatBits(self.target()), + else => float_type.floatBits(self.getTarget()), }; const allocator = scope.arena(); @@ -3594,7 +3981,7 @@ pub fn floatAdd(self: *Module, scope: *Scope, float_type: Type, src: usize, lhs: pub fn floatSub(self: *Module, scope: *Scope, float_type: Type, src: usize, lhs: Value, rhs: Value) !Value { var bit_count = switch (float_type.tag()) { .comptime_float => 128, - else => float_type.floatBits(self.target()), + else => float_type.floatBits(self.getTarget()), }; const allocator = scope.arena(); @@ -3865,3 +4252,106 @@ pub fn safetyPanic(mod: *Module, block: *Scope.Block, src: usize, panic_id: Pani _ = try mod.addNoOp(block, src, Type.initTag(.void), .breakpoint); return mod.addNoOp(block, src, Type.initTag(.noreturn), .unreach); } + +pub const FileExt = enum { + c, + cpp, + h, + ll, + bc, + assembly, + unknown, +}; + +pub fn hasCExt(filename: []const u8) bool { + return mem.endsWith(u8, filename, ".c"); +} + +pub fn hasCppExt(filename: []const u8) bool { + return mem.endsWith(u8, filename, ".C") or + mem.endsWith(u8, filename, ".cc") or + mem.endsWith(u8, filename, ".cpp") or + mem.endsWith(u8, filename, ".cxx"); +} + +pub fn hasAsmExt(filename: []const u8) bool { + return mem.endsWith(u8, filename, ".s") or mem.endsWith(u8, filename, ".S"); +} + +pub fn classifyFileExt(filename: []const u8) FileExt { + if (hasCExt(filename)) { + return .c; + } else if (hasCppExt(filename)) { + return .cpp; + } else if (mem.endsWith(u8, filename, ".ll")) { + return .ll; + } else if (mem.endsWith(u8, filename, ".bc")) { + return .bc; + } else if (hasAsmExt(filename)) { + return .assembly; + } else if (mem.endsWith(u8, filename, ".h")) { + return .h; + } else { + // TODO look for .so, .so.X, .so.X.Y, .so.X.Y.Z + return .unknown; + } +} + +fn haveFramePointer(mod: *Module) bool { + return switch (mod.bin_file.options.optimize_mode) { + .Debug, .ReleaseSafe => !mod.bin_file.options.strip, + .ReleaseSmall, .ReleaseFast => false, + }; +} + +fn detectLibCIncludeDirs( + arena: *Allocator, + zig_lib_dir: []const u8, + target: Target, + link_libc: bool, +) ![]const []const u8 { + if (!link_libc) return &[0][]u8{}; + + // TODO Support --libc file explicitly providing libc paths. Or not? Maybe we are better off + // deleting that feature. + + if (target_util.canBuildLibC(target)) { + const generic_name = target_util.libCGenericName(target); + // Some architectures are handled by the same set of headers. + const arch_name = if (target.abi.isMusl()) target_util.archMuslName(target.cpu.arch) else @tagName(target.cpu.arch); + const os_name = @tagName(target.os.tag); + // Musl's headers are ABI-agnostic and so they all have the "musl" ABI name. + const abi_name = if (target.abi.isMusl()) "musl" else @tagName(target.abi); + const s = std.fs.path.sep_str; + const arch_include_dir = try std.fmt.allocPrint( + arena, + "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{}-{}-{}", + .{ zig_lib_dir, arch_name, os_name, abi_name }, + ); + const generic_include_dir = try std.fmt.allocPrint( + arena, + "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "generic-{}", + .{ zig_lib_dir, generic_name }, + ); + const arch_os_include_dir = try std.fmt.allocPrint( + arena, + "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{}-{}-any", + .{ zig_lib_dir, @tagName(target.cpu.arch), os_name }, + ); + const generic_os_include_dir = try std.fmt.allocPrint( + arena, + "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-{}-any", + .{ zig_lib_dir, os_name }, + ); + + const list = try arena.alloc([]const u8, 4); + list[0] = arch_include_dir; + list[1] = generic_include_dir; + list[2] = arch_os_include_dir; + list[3] = generic_os_include_dir; + return list; + } + + // TODO finish porting detect_libc from codegen.cpp + return error.LibCDetectionUnimplemented; +} diff --git a/src-self-hosted/codegen/llvm.zig b/src-self-hosted/codegen/llvm.zig new file mode 100644 index 0000000000..01fa0baf02 --- /dev/null +++ b/src-self-hosted/codegen/llvm.zig @@ -0,0 +1,125 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; + +pub fn targetTriple(allocator: *Allocator, target: std.Target) ![]u8 { + const llvm_arch = switch (target.cpu.arch) { + .arm => "arm", + .armeb => "armeb", + .aarch64 => "aarch64", + .aarch64_be => "aarch64_be", + .aarch64_32 => "aarch64_32", + .arc => "arc", + .avr => "avr", + .bpfel => "bpfel", + .bpfeb => "bpfeb", + .hexagon => "hexagon", + .mips => "mips", + .mipsel => "mipsel", + .mips64 => "mips64", + .mips64el => "mips64el", + .msp430 => "msp430", + .powerpc => "powerpc", + .powerpc64 => "powerpc64", + .powerpc64le => "powerpc64le", + .r600 => "r600", + .amdgcn => "amdgcn", + .riscv32 => "riscv32", + .riscv64 => "riscv64", + .sparc => "sparc", + .sparcv9 => "sparcv9", + .sparcel => "sparcel", + .s390x => "s390x", + .tce => "tce", + .tcele => "tcele", + .thumb => "thumb", + .thumbeb => "thumbeb", + .i386 => "i386", + .x86_64 => "x86_64", + .xcore => "xcore", + .nvptx => "nvptx", + .nvptx64 => "nvptx64", + .le32 => "le32", + .le64 => "le64", + .amdil => "amdil", + .amdil64 => "amdil64", + .hsail => "hsail", + .hsail64 => "hsail64", + .spir => "spir", + .spir64 => "spir64", + .kalimba => "kalimba", + .shave => "shave", + .lanai => "lanai", + .wasm32 => "wasm32", + .wasm64 => "wasm64", + .renderscript32 => "renderscript32", + .renderscript64 => "renderscript64", + .ve => "ve", + .spu_2 => return error.LLVMBackendDoesNotSupportSPUMarkII, + }; + // TODO Add a sub-arch for some architectures depending on CPU features. + + const llvm_os = switch (target.os.tag) { + .freestanding => "unknown", + .ananas => "ananas", + .cloudabi => "cloudabi", + .dragonfly => "dragonfly", + .freebsd => "freebsd", + .fuchsia => "fuchsia", + .ios => "ios", + .kfreebsd => "kfreebsd", + .linux => "linux", + .lv2 => "lv2", + .macosx => "macosx", + .netbsd => "netbsd", + .openbsd => "openbsd", + .solaris => "solaris", + .windows => "windows", + .haiku => "haiku", + .minix => "minix", + .rtems => "rtems", + .nacl => "nacl", + .cnk => "cnk", + .aix => "aix", + .cuda => "cuda", + .nvcl => "nvcl", + .amdhsa => "amdhsa", + .ps4 => "ps4", + .elfiamcu => "elfiamcu", + .tvos => "tvos", + .watchos => "watchos", + .mesa3d => "mesa3d", + .contiki => "contiki", + .amdpal => "amdpal", + .hermit => "hermit", + .hurd => "hurd", + .wasi => "wasi", + .emscripten => "emscripten", + .uefi => "windows", + .other => "unknown", + }; + + const llvm_abi = switch (target.abi) { + .none => "unknown", + .gnu => "gnu", + .gnuabin32 => "gnuabin32", + .gnuabi64 => "gnuabi64", + .gnueabi => "gnueabi", + .gnueabihf => "gnueabihf", + .gnux32 => "gnux32", + .code16 => "code16", + .eabi => "eabi", + .eabihf => "eabihf", + .android => "android", + .musl => "musl", + .musleabi => "musleabi", + .musleabihf => "musleabihf", + .msvc => "msvc", + .itanium => "itanium", + .cygnus => "cygnus", + .coreclr => "coreclr", + .simulator => "simulator", + .macabi => "macabi", + }; + + return std.fmt.allocPrint(allocator, "{}-unknown-{}-{}", .{ llvm_arch, llvm_os, llvm_abi }); +} diff --git a/src-self-hosted/introspect.zig b/src-self-hosted/introspect.zig index 80f10c8656..019da4a39e 100644 --- a/src-self-hosted/introspect.zig +++ b/src-self-hosted/introspect.zig @@ -93,46 +93,3 @@ pub fn openGlobalCacheDir() !fs.Dir { const path_name = try resolveGlobalCacheDir(&fba.allocator); return fs.cwd().makeOpenPath(path_name, .{}); } - -var compiler_id_mutex = std.Mutex{}; -var compiler_id: [16]u8 = undefined; -var compiler_id_computed = false; - -pub fn resolveCompilerId(gpa: *mem.Allocator) ![16]u8 { - const held = compiler_id_mutex.acquire(); - defer held.release(); - - if (compiler_id_computed) - return compiler_id; - compiler_id_computed = true; - - var cache_dir = try openGlobalCacheDir(); - defer cache_dir.close(); - - var ch = try CacheHash.init(gpa, cache_dir, "exe"); - defer ch.release(); - - const self_exe_path = try fs.selfExePathAlloc(gpa); - defer gpa.free(self_exe_path); - - _ = try ch.addFile(self_exe_path, null); - - if (try ch.hit()) |digest| { - compiler_id = digest[0..16].*; - return compiler_id; - } - - const libs = try std.process.getSelfExeSharedLibPaths(gpa); - defer { - for (libs) |lib| gpa.free(lib); - gpa.free(libs); - } - - for (libs) |lib| { - try ch.addFilePost(lib); - } - - const digest = ch.final(); - compiler_id = digest[0..16].*; - return compiler_id; -} diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 14166e7e30..3b07899aec 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -35,6 +35,9 @@ pub const Options = struct { /// other objects. /// Otherwise (depending on `use_lld`) this link code directly outputs and updates the final binary. use_llvm: bool = false, + link_libc: bool = false, + link_libcpp: bool = false, + function_sections: bool = false, objects: []const []const u8 = &[0][]const u8{}, framework_dirs: []const []const u8 = &[0][]const u8{}, diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 8b75662d35..4bb2a4e940 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -219,8 +219,11 @@ pub const SrcFn = struct { pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: link.Options) !*File { assert(options.object_format == .elf); - if (options.use_llvm) return error.LLVM_BackendIsTODO_ForELF; // TODO - if (options.use_lld) return error.LLD_LinkingIsTODOForELF; // TODO + if (options.use_llvm) return error.LLVMBackendUnimplementedForELF; // TODO + + if (build_options.have_llvm and options.use_lld) { + std.debug.print("TODO open a temporary object file, not the final output file because we want to link with LLD\n", .{}); + } const file = try dir.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options) }); errdefer file.close(); @@ -741,8 +744,21 @@ pub const abbrev_base_type = 4; pub const abbrev_pad1 = 5; pub const abbrev_parameter = 6; -/// Commit pending changes and write headers. pub fn flush(self: *Elf, module: *Module) !void { + if (build_options.have_llvm and self.base.options.use_lld) { + // If there is no Zig code to compile, then we should skip flushing the output file because it + // will not be part of the linker line anyway. + if (module.root_pkg != null) { + try self.flushInner(module); + } + std.debug.print("TODO create an LLD command line and invoke it\n", .{}); + } else { + return self.flushInner(module); + } +} + +/// Commit pending changes and write headers. +fn flushInner(self: *Elf, module: *Module) !void { const target_endian = self.base.options.target.cpu.arch.endian(); const foreign_endian = target_endian != std.Target.current.cpu.arch.endian(); const ptr_width_bytes: u8 = self.ptrWidthBytes(); diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index a398a6a5d1..84bb6e9351 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -33,13 +33,14 @@ const usage = \\ \\Commands: \\ - \\ build-exe [source] Create executable from source or object files - \\ build-lib [source] Create library from source or object files - \\ build-obj [source] Create object from source or assembly + \\ build-exe [source] Create executable from source or object files + \\ build-lib [source] Create library from source or object files + \\ build-obj [source] Create object from source or assembly \\ cc Use Zig as a drop-in C compiler \\ c++ Use Zig as a drop-in C++ compiler \\ env Print lib path, std path, compiler id and version - \\ fmt [source] Parse file and render in canonical zig format + \\ fmt [source] Parse file and render in canonical zig format + \\ translate-c [source] Convert C code to Zig code \\ targets List available compilation targets \\ version Print version number and exit \\ zen Print zen of zig and exit @@ -47,15 +48,21 @@ const usage = \\ ; +pub const log_level: std.log.Level = switch (std.builtin.mode) { + .Debug => .debug, + .ReleaseSafe, .ReleaseFast => .info, + .ReleaseSmall => .crit, +}; + pub fn log( comptime level: std.log.Level, comptime scope: @TypeOf(.EnumLiteral), comptime format: []const u8, args: anytype, ) void { - // Hide anything more verbose than warn unless it was added with `-Dlog=foo`. + // Hide debug messages unless added with `-Dlog=foo`. if (@enumToInt(level) > @enumToInt(std.log.level) or - @enumToInt(level) > @enumToInt(std.log.Level.warn)) + @enumToInt(level) > @enumToInt(std.log.Level.info)) { const scope_name = @tagName(scope); const ok = comptime for (build_options.log_scopes) |log_scope| { @@ -67,13 +74,15 @@ pub fn log( return; } + // We only recognize 4 log levels in this application. const level_txt = switch (level) { - .emerg => "error", - .warn => "warning", - else => @tagName(level), + .emerg, .alert, .crit => "error", + .err, .warn => "warning", + .notice, .info => "info", + .debug => "debug", }; - const prefix1 = level_txt ++ ": "; - const prefix2 = if (scope == .default) "" else "(" ++ @tagName(scope) ++ "): "; + const prefix1 = level_txt; + const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; // Print the message to stderr, silently ignoring any errors std.debug.print(prefix1 ++ prefix2 ++ format ++ "\n", args); @@ -93,8 +102,8 @@ pub fn main() !void { const args = try process.argsAlloc(arena); if (args.len <= 1) { - std.debug.print("expected command argument\n\n{}", .{usage}); - process.exit(1); + std.log.info("{}", .{usage}); + fatal("expected command argument", .{}); } const cmd = args[1]; @@ -109,6 +118,8 @@ pub fn main() !void { return buildOutputType(gpa, arena, args, .cc); } else if (mem.eql(u8, cmd, "c++")) { return buildOutputType(gpa, arena, args, .cpp); + } else if (mem.eql(u8, cmd, "translate-c")) { + return buildOutputType(gpa, arena, args, .translate_c); } else if (mem.eql(u8, cmd, "clang") or mem.eql(u8, cmd, "-cc1") or mem.eql(u8, cmd, "-cc1as")) { @@ -128,8 +139,8 @@ pub fn main() !void { } else if (mem.eql(u8, cmd, "help")) { try io.getStdOut().writeAll(usage); } else { - std.debug.print("unknown command: {}\n\n{}", .{ args[1], usage }); - process.exit(1); + std.log.info("{}", .{usage}); + fatal("unknown command: {}", .{args[1]}); } } @@ -223,6 +234,7 @@ pub fn buildOutputType( build: std.builtin.OutputMode, cc, cpp, + translate_c, }, ) !void { var color: Color = .Auto; @@ -251,8 +263,8 @@ pub fn buildOutputType( var emit_h: Emit = undefined; var ensure_libc_on_non_freestanding = false; var ensure_libcpp_on_non_freestanding = false; - var have_libc = false; - var have_libcpp = false; + var link_libc = false; + var link_libcpp = false; var want_native_include_dirs = false; var enable_cache: ?bool = null; var want_pic: ?bool = null; @@ -298,13 +310,20 @@ pub fn buildOutputType( var frameworks = std.ArrayList([]const u8).init(gpa); defer frameworks.deinit(); - if (arg_mode == .build) { - output_mode = arg_mode.build; - emit_h = switch (output_mode) { - .Exe => .no, - .Obj, .Lib => .yes_default_path, + if (arg_mode == .build or arg_mode == .translate_c) { + output_mode = switch (arg_mode) { + .build => |m| m, + .translate_c => .Obj, + else => unreachable, }; - + switch (arg_mode) { + .build => switch (output_mode) { + .Exe => emit_h = .no, + .Obj, .Lib => emit_h = .yes_default_path, + }, + .translate_c => emit_h = .no, + else => unreachable, + } const args = all_args[2..]; var i: usize = 0; while (i < args.len) : (i += 1) { @@ -499,7 +518,7 @@ pub fn buildOutputType( mem.endsWith(u8, arg, ".lib")) { try link_objects.append(arg); - } else if (hasAsmExt(arg) or hasCExt(arg) or hasCppExt(arg)) { + } else if (Module.hasAsmExt(arg) or Module.hasCExt(arg) or Module.hasCppExt(arg)) { try c_source_files.append(arg); } else if (mem.endsWith(u8, arg, ".so") or mem.endsWith(u8, arg, ".dylib") or @@ -543,7 +562,7 @@ pub fn buildOutputType( try clang_argv.appendSlice(it.other_args); }, .positional => { - const file_ext = classify_file_ext(mem.spanZ(it.only_arg)); + const file_ext = Module.classifyFileExt(mem.spanZ(it.only_arg)); switch (file_ext) { .assembly, .c, .cpp, .ll, .bc, .h => try c_source_files.append(it.only_arg), .unknown => try link_objects.append(it.only_arg), @@ -819,28 +838,28 @@ pub fn buildOutputType( .diagnostics = &diags, }) catch |err| switch (err) { error.UnknownCpuModel => { - std.debug.print("Unknown CPU: '{}'\nAvailable CPUs for architecture '{}':\n", .{ - diags.cpu_name.?, - @tagName(diags.arch.?), - }); - for (diags.arch.?.allCpuModels()) |cpu| { - std.debug.print(" {}\n", .{cpu.name}); + help: { + var help_text = std.ArrayList(u8).init(arena); + for (diags.arch.?.allCpuModels()) |cpu| { + help_text.writer().print(" {}\n", .{cpu.name}) catch break :help; + } + std.log.info("Available CPUs for architecture '{}': {}", .{ + @tagName(diags.arch.?), help_text.items, + }); } - process.exit(1); + fatal("Unknown CPU: '{}'", .{diags.cpu_name.?}); }, error.UnknownCpuFeature => { - std.debug.print( - \\Unknown CPU feature: '{}' - \\Available CPU features for architecture '{}': - \\ - , .{ - diags.unknown_feature_name, - @tagName(diags.arch.?), - }); - for (diags.arch.?.allFeaturesList()) |feature| { - std.debug.print(" {}: {}\n", .{ feature.name, feature.description }); + help: { + var help_text = std.ArrayList(u8).init(arena); + for (diags.arch.?.allFeaturesList()) |feature| { + help_text.writer().print(" {}: {}\n", .{ feature.name, feature.description }) catch break :help; + } + std.log.info("Available CPU features for architecture '{}': {}", .{ + @tagName(diags.arch.?), help_text.items, + }); } - process.exit(1); + fatal("Unknown CPU feature: '{}'", .{diags.unknown_feature_name}); }, else => |e| return e, }; @@ -849,14 +868,16 @@ pub fn buildOutputType( if (target_info.cpu_detection_unimplemented) { // TODO We want to just use detected_info.target but implementing // CPU model & feature detection is todo so here we rely on LLVM. + // TODO The workaround to use LLVM to detect features needs to be used for + // `zig targets` as well. fatal("CPU features detection is not yet available for this system without LLVM extensions", .{}); } if (target_info.target.os.tag != .freestanding) { if (ensure_libc_on_non_freestanding) - have_libc = true; + link_libc = true; if (ensure_libcpp_on_non_freestanding) - have_libcpp = true; + link_libcpp = true; } // Now that we have target info, we can find out if any of the system libraries @@ -867,12 +888,12 @@ pub fn buildOutputType( while (i < system_libs.items.len) { const lib_name = system_libs.items[i]; if (is_libc_lib_name(target_info.target, lib_name)) { - have_libc = true; + link_libc = true; _ = system_libs.orderedRemove(i); continue; } if (is_libcpp_lib_name(target_info.target, lib_name)) { - have_libcpp = true; + link_libcpp = true; _ = system_libs.orderedRemove(i); continue; } @@ -960,7 +981,21 @@ pub fn buildOutputType( .yes_default_path => try std.fmt.allocPrint(arena, "{}.h", .{root_name}), }; - var module = Module.init(gpa, .{ + const self_exe_path = try fs.selfExePathAlloc(arena); + const zig_lib_dir = introspect.resolveZigLibDir(gpa) catch |err| { + fatal("unable to find zig installation directory: {}\n", .{@errorName(err)}); + }; + defer gpa.free(zig_lib_dir); + + const random_seed = blk: { + var random_seed: u64 = undefined; + try std.crypto.randomBytes(mem.asBytes(&random_seed)); + break :blk random_seed; + }; + var default_prng = std.rand.DefaultPrng.init(random_seed); + + const module = Module.create(gpa, .{ + .zig_lib_dir = zig_lib_dir, .root_name = root_name, .target = target_info.target, .output_mode = output_mode, @@ -980,8 +1015,8 @@ pub fn buildOutputType( .frameworks = frameworks.items, .system_libs = system_libs.items, .emit_h = emit_h_path, - .have_libc = have_libc, - .have_libcpp = have_libcpp, + .link_libc = link_libc, + .link_libcpp = link_libcpp, .want_pic = want_pic, .want_sanitize_c = want_sanitize_c, .use_llvm = use_llvm, @@ -1000,16 +1035,19 @@ pub fn buildOutputType( .linker_z_defs = linker_z_defs, .stack_size_override = stack_size_override, .strip = strip, + .self_exe_path = self_exe_path, + .rand = &default_prng.random, + .clang_passthrough_mode = arg_mode != .build, }) catch |err| { - fatal("unable to initialize module: {}", .{@errorName(err)}); + fatal("unable to create module: {}", .{@errorName(err)}); }; - defer module.deinit(); + defer module.destroy(); const stdin = std.io.getStdIn().inStream(); const stderr = std.io.getStdErr().outStream(); var repl_buf: [1024]u8 = undefined; - try updateModule(gpa, &module, zir_out_path); + try updateModule(gpa, module, zir_out_path); if (build_options.have_llvm and only_pp_or_asm) { // this may include dumping the output to stdout @@ -1031,7 +1069,7 @@ pub fn buildOutputType( if (output_mode == .Exe) { try module.makeBinFileWritable(); } - try updateModule(gpa, &module, zir_out_path); + try updateModule(gpa, module, zir_out_path); } else if (mem.eql(u8, actual_line, "exit")) { break; } else if (mem.eql(u8, actual_line, "help")) { @@ -1062,12 +1100,10 @@ fn updateModule(gpa: *Allocator, module: *Module, zir_out_path: ?[]const u8) !vo full_err_msg.msg, }); } - } else { - std.log.info("Update completed in {} ms", .{update_nanos / std.time.ns_per_ms}); } if (zir_out_path) |zop| { - var new_zir_module = try zir.emit(gpa, module.*); + var new_zir_module = try zir.emit(gpa, module); defer new_zir_module.deinit(gpa); const baf = try io.BufferedAtomicFile.create(gpa, fs.cwd(), zop, .{}); @@ -1422,50 +1458,6 @@ pub const info_zen = \\ ; -const FileExt = enum { - c, - cpp, - h, - ll, - bc, - assembly, - unknown, -}; - -fn hasCExt(filename: []const u8) bool { - return mem.endsWith(u8, filename, ".c"); -} - -fn hasCppExt(filename: []const u8) bool { - return mem.endsWith(u8, filename, ".C") or - mem.endsWith(u8, filename, ".cc") or - mem.endsWith(u8, filename, ".cpp") or - mem.endsWith(u8, filename, ".cxx"); -} - -fn hasAsmExt(filename: []const u8) bool { - return mem.endsWith(u8, filename, ".s") or mem.endsWith(u8, filename, ".S"); -} - -fn classify_file_ext(filename: []const u8) FileExt { - if (hasCExt(filename)) { - return .c; - } else if (hasCppExt(filename)) { - return .cpp; - } else if (mem.endsWith(u8, filename, ".ll")) { - return .ll; - } else if (mem.endsWith(u8, filename, ".bc")) { - return .bc; - } else if (hasAsmExt(filename)) { - return .assembly; - } else if (mem.endsWith(u8, filename, ".h")) { - return .h; - } else { - // TODO look for .so, .so.X, .so.X.Y, .so.X.Y.Z - return .unknown; - } -} - extern "c" fn ZigClang_main(argc: c_int, argv: [*:null]?[*:0]u8) c_int; /// TODO make it so the return value can be !noreturn diff --git a/src-self-hosted/print_targets.zig b/src-self-hosted/print_targets.zig index 0fe755ffb4..33a513efd7 100644 --- a/src-self-hosted/print_targets.zig +++ b/src-self-hosted/print_targets.zig @@ -4,60 +4,11 @@ const io = std.io; const mem = std.mem; const Allocator = mem.Allocator; const Target = std.Target; +const target = @import("target.zig"); const assert = std.debug.assert; const introspect = @import("introspect.zig"); -// TODO this is hard-coded until self-hosted gains this information canonically -const available_libcs = [_][]const u8{ - "aarch64_be-linux-gnu", - "aarch64_be-linux-musl", - "aarch64_be-windows-gnu", - "aarch64-linux-gnu", - "aarch64-linux-musl", - "aarch64-windows-gnu", - "armeb-linux-gnueabi", - "armeb-linux-gnueabihf", - "armeb-linux-musleabi", - "armeb-linux-musleabihf", - "armeb-windows-gnu", - "arm-linux-gnueabi", - "arm-linux-gnueabihf", - "arm-linux-musleabi", - "arm-linux-musleabihf", - "arm-windows-gnu", - "i386-linux-gnu", - "i386-linux-musl", - "i386-windows-gnu", - "mips64el-linux-gnuabi64", - "mips64el-linux-gnuabin32", - "mips64el-linux-musl", - "mips64-linux-gnuabi64", - "mips64-linux-gnuabin32", - "mips64-linux-musl", - "mipsel-linux-gnu", - "mipsel-linux-musl", - "mips-linux-gnu", - "mips-linux-musl", - "powerpc64le-linux-gnu", - "powerpc64le-linux-musl", - "powerpc64-linux-gnu", - "powerpc64-linux-musl", - "powerpc-linux-gnu", - "powerpc-linux-musl", - "riscv64-linux-gnu", - "riscv64-linux-musl", - "s390x-linux-gnu", - "s390x-linux-musl", - "sparc-linux-gnu", - "sparcv9-linux-gnu", - "wasm32-freestanding-musl", - "x86_64-linux-gnu", - "x86_64-linux-gnux32", - "x86_64-linux-musl", - "x86_64-windows-gnu", -}; - pub fn cmdTargets( allocator: *Allocator, args: []const []const u8, @@ -127,9 +78,13 @@ pub fn cmdTargets( try jws.objectField("libc"); try jws.beginArray(); - for (available_libcs) |libc| { + for (target.available_libcs) |libc| { + const tmp = try std.fmt.allocPrint(allocator, "{}-{}-{}", .{ + @tagName(libc.arch), @tagName(libc.os), @tagName(libc.abi), + }); + defer allocator.free(tmp); try jws.arrayElem(); - try jws.emitString(libc); + try jws.emitString(tmp); } try jws.endArray(); diff --git a/src-self-hosted/target.zig b/src-self-hosted/target.zig new file mode 100644 index 0000000000..cb09779d74 --- /dev/null +++ b/src-self-hosted/target.zig @@ -0,0 +1,111 @@ +const std = @import("std"); + +pub const ArchOsAbi = struct { + arch: std.Target.Cpu.Arch, + os: std.Target.Os.Tag, + abi: std.Target.Abi, +}; + +pub const available_libcs = [_]ArchOsAbi{ + .{ .arch = .aarch64_be, .os = .linux, .abi = .gnu }, + .{ .arch = .aarch64_be, .os = .linux, .abi = .musl }, + .{ .arch = .aarch64_be, .os = .windows, .abi = .gnu }, + .{ .arch = .aarch64, .os = .linux, .abi = .gnu }, + .{ .arch = .aarch64, .os = .linux, .abi = .musl }, + .{ .arch = .aarch64, .os = .windows, .abi = .gnu }, + .{ .arch = .armeb, .os = .linux, .abi = .gnueabi }, + .{ .arch = .armeb, .os = .linux, .abi = .gnueabihf }, + .{ .arch = .armeb, .os = .linux, .abi = .musleabi }, + .{ .arch = .armeb, .os = .linux, .abi = .musleabihf }, + .{ .arch = .armeb, .os = .windows, .abi = .gnu }, + .{ .arch = .arm, .os = .linux, .abi = .gnueabi }, + .{ .arch = .arm, .os = .linux, .abi = .gnueabihf }, + .{ .arch = .arm, .os = .linux, .abi = .musleabi }, + .{ .arch = .arm, .os = .linux, .abi = .musleabihf }, + .{ .arch = .arm, .os = .windows, .abi = .gnu }, + .{ .arch = .i386, .os = .linux, .abi = .gnu }, + .{ .arch = .i386, .os = .linux, .abi = .musl }, + .{ .arch = .i386, .os = .windows, .abi = .gnu }, + .{ .arch = .mips64el, .os = .linux, .abi = .gnuabi64 }, + .{ .arch = .mips64el, .os = .linux, .abi = .gnuabin32 }, + .{ .arch = .mips64el, .os = .linux, .abi = .musl }, + .{ .arch = .mips64, .os = .linux, .abi = .gnuabi64 }, + .{ .arch = .mips64, .os = .linux, .abi = .gnuabin32 }, + .{ .arch = .mips64, .os = .linux, .abi = .musl }, + .{ .arch = .mipsel, .os = .linux, .abi = .gnu }, + .{ .arch = .mipsel, .os = .linux, .abi = .musl }, + .{ .arch = .mips, .os = .linux, .abi = .gnu }, + .{ .arch = .mips, .os = .linux, .abi = .musl }, + .{ .arch = .powerpc64le, .os = .linux, .abi = .gnu }, + .{ .arch = .powerpc64le, .os = .linux, .abi = .musl }, + .{ .arch = .powerpc64, .os = .linux, .abi = .gnu }, + .{ .arch = .powerpc64, .os = .linux, .abi = .musl }, + .{ .arch = .powerpc, .os = .linux, .abi = .gnu }, + .{ .arch = .powerpc, .os = .linux, .abi = .musl }, + .{ .arch = .riscv64, .os = .linux, .abi = .gnu }, + .{ .arch = .riscv64, .os = .linux, .abi = .musl }, + .{ .arch = .s390x, .os = .linux, .abi = .gnu }, + .{ .arch = .s390x, .os = .linux, .abi = .musl }, + .{ .arch = .sparc, .os = .linux, .abi = .gnu }, + .{ .arch = .sparcv9, .os = .linux, .abi = .gnu }, + .{ .arch = .wasm32, .os = .freestanding, .abi = .musl }, + .{ .arch = .x86_64, .os = .linux, .abi = .gnu }, + .{ .arch = .x86_64, .os = .linux, .abi = .gnux32 }, + .{ .arch = .x86_64, .os = .linux, .abi = .musl }, + .{ .arch = .x86_64, .os = .windows, .abi = .gnu }, +}; + +pub fn libCGenericName(target: std.Target) [:0]const u8 { + if (target.os.tag == .windows) + return "mingw"; + switch (target.abi) { + .gnu, + .gnuabin32, + .gnuabi64, + .gnueabi, + .gnueabihf, + .gnux32, + => return "glibc", + .musl, + .musleabi, + .musleabihf, + .none, + => return "musl", + .code16, + .eabi, + .eabihf, + .android, + .msvc, + .itanium, + .cygnus, + .coreclr, + .simulator, + .macabi, + => unreachable, + } +} + +pub fn archMuslName(arch: std.Target.Cpu.Arch) [:0]const u8 { + switch (arch) { + .aarch64, .aarch64_be => return "aarch64", + .arm, .armeb => return "arm", + .mips, .mipsel => return "mips", + .mips64el, .mips64 => return "mips64", + .powerpc => return "powerpc", + .powerpc64, .powerpc64le => return "powerpc64", + .s390x => return "s390x", + .i386 => return "i386", + .x86_64 => return "x86_64", + .riscv64 => return "riscv64", + else => unreachable, + } +} + +pub fn canBuildLibC(target: std.Target) bool { + for (available_libcs) |libc| { + if (target.cpu.arch == libc.arch and target.os.tag == libc.os and target.abi == libc.abi) { + return true; + } + } + return false; +} diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index cd41ff0d78..004b56b029 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -4,6 +4,7 @@ const Module = @import("Module.zig"); const Allocator = std.mem.Allocator; const zir = @import("zir.zig"); const Package = @import("Package.zig"); +const introspect = @import("introspect.zig"); const build_options = @import("build_options"); const enable_qemu: bool = build_options.enable_qemu; const enable_wine: bool = build_options.enable_wine; @@ -406,6 +407,16 @@ pub const TestContext = struct { const root_node = try progress.start("tests", self.cases.items.len); defer root_node.end(); + const zig_lib_dir = try introspect.resolveZigLibDir(std.testing.allocator); + defer std.testing.allocator.free(zig_lib_dir); + + const random_seed = blk: { + var random_seed: u64 = undefined; + try std.crypto.randomBytes(std.mem.asBytes(&random_seed)); + break :blk random_seed; + }; + var default_prng = std.rand.DefaultPrng.init(random_seed); + for (self.cases.items) |case| { var prg_node = root_node.start(case.name, case.updates.items.len); prg_node.activate(); @@ -416,11 +427,18 @@ pub const TestContext = struct { progress.initial_delay_ns = 0; progress.refresh_rate_ns = 0; - try self.runOneCase(std.testing.allocator, &prg_node, case); + try self.runOneCase(std.testing.allocator, &prg_node, case, zig_lib_dir, &default_prng.random); } } - fn runOneCase(self: *TestContext, allocator: *Allocator, root_node: *std.Progress.Node, case: Case) !void { + fn runOneCase( + self: *TestContext, + allocator: *Allocator, + root_node: *std.Progress.Node, + case: Case, + zig_lib_dir: []const u8, + rand: *std.rand.Random, + ) !void { const target_info = try std.zig.system.NativeTargetInfo.detect(allocator, case.target); const target = target_info.target; @@ -438,7 +456,9 @@ pub const TestContext = struct { const ofmt: ?std.builtin.ObjectFormat = if (case.cbe) .c else null; const bin_name = try std.zig.binNameAlloc(arena, "test_case", target, case.output_mode, null, ofmt); - var module = try Module.init(allocator, .{ + const module = try Module.create(allocator, .{ + .zig_lib_dir = zig_lib_dir, + .rand = rand, .root_name = "test_case", .target = target, // TODO: support tests for object file building, and library builds @@ -453,7 +473,7 @@ pub const TestContext = struct { .keep_source_files_loaded = true, .object_format = ofmt, }); - defer module.deinit(); + defer module.destroy(); for (case.updates.items) |update, update_index| { var update_node = root_node.start("update", 3); diff --git a/src-self-hosted/zir.zig b/src-self-hosted/zir.zig index b6d7fab4c5..7e723fc674 100644 --- a/src-self-hosted/zir.zig +++ b/src-self-hosted/zir.zig @@ -1700,12 +1700,12 @@ const Parser = struct { } }; -pub fn emit(allocator: *Allocator, old_module: IrModule) !Module { +pub fn emit(allocator: *Allocator, old_module: *IrModule) !Module { var ctx: EmitZIR = .{ .allocator = allocator, .decls = .{}, .arena = std.heap.ArenaAllocator.init(allocator), - .old_module = &old_module, + .old_module = old_module, .next_auto_name = 0, .names = std.StringArrayHashMap(void).init(allocator), .primitive_table = std.AutoHashMap(Inst.Primitive.Builtin, *Decl).init(allocator), @@ -2539,7 +2539,7 @@ const EmitZIR = struct { return self.emitUnnamedDecl(&fntype_inst.base); }, .Int => { - const info = ty.intInfo(self.old_module.target()); + const info = ty.intInfo(self.old_module.getTarget()); const signed = try self.emitPrimitive(src, if (info.signed) .@"true" else .@"false"); const bits_payload = try self.arena.allocator.create(Value.Payload.Int_u64); bits_payload.* = .{ .int = info.bits }; diff --git a/src/main.cpp b/src/main.cpp index af3bc4b2a1..9f6d2b3d7b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -78,7 +78,7 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) { " -fno-emit-asm (default) do not output .s (assembly code)\n" " -femit-llvm-ir produce a .ll file with LLVM IR\n" " -fno-emit-llvm-ir (default) do not produce a .ll file with LLVM IR\n" - " -femit-h generate a C header file (.h)\n" + " -femit-h generate a C header file (.h)\n" " -fno-emit-h (default) do not generate a C header file (.h)\n" " --libc [file] Provide a file which specifies libc paths\n" " --name [name] override output name\n" From b37955f2739dc20f65cc8d352cab98dc07f44b5f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 9 Sep 2020 10:54:40 -0700 Subject: [PATCH 011/210] stage2 linker code supports opening an intermediate object file For when linking with LLD, we always create an object rather than going straight to the executable. Next step is putting this object on the LLD linker line. --- src-self-hosted/Module.zig | 13 ++++---- src-self-hosted/link.zig | 55 +++++++++++++++++++++++++--------- src-self-hosted/link/C.zig | 8 ++--- src-self-hosted/link/Coff.zig | 10 +++++-- src-self-hosted/link/Elf.zig | 55 +++++++--------------------------- src-self-hosted/link/MachO.zig | 8 +++-- src-self-hosted/link/Wasm.zig | 4 +-- src-self-hosted/main.zig | 1 + 8 files changed, 78 insertions(+), 76 deletions(-) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 10a26a4aa8..441ce15dc0 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -35,8 +35,6 @@ root_pkg: ?*Package, /// The `Scope` is either a `Scope.ZIRModule` or `Scope.File`. root_scope: *Scope, bin_file: *link.File, -bin_file_dir: std.fs.Dir, -bin_file_path: []const u8, /// It's rare for a decl to be exported, so we save memory by having a sparse map of /// Decl pointers to details about them being exported. /// The Export memory is owned by the `export_owners` table; the slice itself is owned by this table. @@ -934,6 +932,7 @@ pub const InitOptions = struct { root_pkg: ?*Package, output_mode: std.builtin.OutputMode, rand: *std.rand.Random, + bin_file_dir_path: ?[]const u8 = null, bin_file_dir: ?std.fs.Dir = null, bin_file_path: []const u8, emit_h: ?[]const u8 = null, @@ -1018,8 +1017,10 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { break :blk false; }; - const bin_file_dir = options.bin_file_dir orelse std.fs.cwd(); - const bin_file = try link.File.openPath(gpa, bin_file_dir, options.bin_file_path, .{ + const bin_file = try link.File.openPath(gpa, .{ + .dir = options.bin_file_dir orelse std.fs.cwd(), + .dir_path = options.bin_file_dir_path, + .sub_path = options.bin_file_path, .root_name = root_name, .root_pkg = options.root_pkg, .target = options.target, @@ -1165,8 +1166,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { .root_name = root_name, .root_pkg = options.root_pkg, .root_scope = root_scope, - .bin_file_dir = bin_file_dir, - .bin_file_path = options.bin_file_path, .bin_file = bin_file, .work_queue = std.fifo.LinearFifo(WorkItem, .Dynamic).init(gpa), .keep_source_files_loaded = options.keep_source_files_loaded, @@ -1350,7 +1349,7 @@ pub fn makeBinFileExecutable(self: *Module) !void { } pub fn makeBinFileWritable(self: *Module) !void { - return self.bin_file.makeWritable(self.bin_file_dir, self.bin_file_path); + return self.bin_file.makeWritable(); } pub fn totalErrorCount(self: *Module) usize { diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 3b07899aec..22c8a6c8e3 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -10,6 +10,12 @@ const build_options = @import("build_options"); pub const producer_string = if (std.builtin.is_test) "zig test" else "zig " ++ build_options.version; pub const Options = struct { + dir: fs.Dir, + /// Redundant with dir. Needed when linking with LLD because we have to pass paths rather + /// than file descriptors. `null` means cwd. OK to pass `null` when `use_lld` is `false`. + dir_path: ?[]const u8, + /// Path to the output file, relative to dir. + sub_path: []const u8, target: std.Target, output_mode: std.builtin.OutputMode, link_mode: std.builtin.LinkMode, @@ -56,6 +62,9 @@ pub const File = struct { options: Options, file: ?fs.File, allocator: *Allocator, + /// When linking with LLD, this linker code will output an object file only at + /// this location, and then this path can be placed on the LLD linker line. + intermediary_basename: ?[]const u8 = null, pub const LinkBlock = union { elf: Elf.TextBlock, @@ -90,16 +99,29 @@ pub const File = struct { /// incremental linking fails, falls back to truncating the file and /// rewriting it. A malicious file is detected as incremental link failure /// and does not cause Illegal Behavior. This operation is not atomic. - pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: Options) !*File { - switch (options.object_format) { - .coff, .pe => return Coff.openPath(allocator, dir, sub_path, options), - .elf => return Elf.openPath(allocator, dir, sub_path, options), - .macho => return MachO.openPath(allocator, dir, sub_path, options), - .wasm => return Wasm.openPath(allocator, dir, sub_path, options), - .c => return C.openPath(allocator, dir, sub_path, options), + pub fn openPath(allocator: *Allocator, options: Options) !*File { + const use_lld = build_options.have_llvm and options.use_lld; // comptime known false when !have_llvm + const sub_path = if (use_lld) blk: { + // Open a temporary object file, not the final output file because we want to link with LLD. + break :blk try std.fmt.allocPrint(allocator, "{}{}", .{ options.sub_path, options.target.oFileExt() }); + } else options.sub_path; + errdefer if (use_lld) allocator.free(sub_path); + + const file: *File = switch (options.object_format) { + .coff, .pe => try Coff.openPath(allocator, sub_path, options), + .elf => try Elf.openPath(allocator, sub_path, options), + .macho => try MachO.openPath(allocator, sub_path, options), + .wasm => try Wasm.openPath(allocator, sub_path, options), + .c => try C.openPath(allocator, sub_path, options), .hex => return error.HexObjectFormatUnimplemented, .raw => return error.RawObjectFormatUnimplemented, + }; + + if (use_lld) { + file.intermediary_basename = sub_path; } + + return file; } pub fn cast(base: *File, comptime T: type) ?*T { @@ -109,11 +131,11 @@ pub const File = struct { return @fieldParentPtr(T, "base", base); } - pub fn makeWritable(base: *File, dir: fs.Dir, sub_path: []const u8) !void { + pub fn makeWritable(base: *File) !void { switch (base.tag) { .coff, .elf, .macho => { if (base.file != null) return; - base.file = try dir.createFile(sub_path, .{ + base.file = try base.options.dir.createFile(base.options.sub_path, .{ .truncate = false, .read = true, .mode = determineMode(base.options), @@ -125,12 +147,16 @@ pub const File = struct { pub fn makeExecutable(base: *File) !void { switch (base.tag) { - .c => unreachable, - .wasm => {}, - else => if (base.file) |f| { + .coff, .elf, .macho => if (base.file) |f| { + if (base.intermediary_basename != null) { + // The file we have open is not the final file that we want to + // make executable, so we don't have to close it. + return; + } f.close(); base.file = null; }, + .c, .wasm => {}, } } @@ -167,7 +193,6 @@ pub const File = struct { } pub fn deinit(base: *File) void { - if (base.file) |f| f.close(); switch (base.tag) { .coff => @fieldParentPtr(Coff, "base", base).deinit(), .elf => @fieldParentPtr(Elf, "base", base).deinit(), @@ -175,6 +200,8 @@ pub const File = struct { .c => @fieldParentPtr(C, "base", base).deinit(), .wasm => @fieldParentPtr(Wasm, "base", base).deinit(), } + if (base.file) |f| f.close(); + if (base.intermediary_basename) |sub_path| base.allocator.free(sub_path); } pub fn destroy(base: *File) void { @@ -292,7 +319,7 @@ pub fn determineMode(options: Options) fs.File.Mode { // more leniently. As another data point, C's fopen seems to open files with the // 666 mode. const executable_mode = if (std.Target.current.os.tag == .windows) 0 else 0o777; - switch (options.output_mode) { + switch (options.effectiveOutputMode()) { .Lib => return switch (options.link_mode) { .Dynamic => executable_mode, .Static => fs.File.default_mode, diff --git a/src-self-hosted/link/C.zig b/src-self-hosted/link/C.zig index ae0516dae0..8ec6f91959 100644 --- a/src-self-hosted/link/C.zig +++ b/src-self-hosted/link/C.zig @@ -22,13 +22,13 @@ need_stddef: bool = false, need_stdint: bool = false, error_msg: *Module.ErrorMsg = undefined, -pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: link.Options) !*File { +pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*File { assert(options.object_format == .c); - if (options.use_llvm) return error.LLVM_HasNoCBackend; - if (options.use_lld) return error.LLD_HasNoCBackend; + if (options.use_llvm) return error.LLVMHasNoCBackend; + if (options.use_lld) return error.LLDHasNoCBackend; - const file = try dir.createFile(sub_path, .{ .truncate = true, .read = true, .mode = link.determineMode(options) }); + const file = try options.dir.createFile(sub_path, .{ .truncate = true, .read = true, .mode = link.determineMode(options) }); errdefer file.close(); var c_file = try allocator.create(C); diff --git a/src-self-hosted/link/Coff.zig b/src-self-hosted/link/Coff.zig index 4cab7a75fd..950e3537d9 100644 --- a/src-self-hosted/link/Coff.zig +++ b/src-self-hosted/link/Coff.zig @@ -19,7 +19,7 @@ const file_alignment = 512; const image_base = 0x400_000; const section_table_size = 2 * 40; comptime { - std.debug.assert(std.mem.isAligned(image_base, section_alignment)); + assert(std.mem.isAligned(image_base, section_alignment)); } pub const base_tag: link.File.Tag = .coff; @@ -110,13 +110,17 @@ pub const TextBlock = struct { pub const SrcFn = void; -pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: link.Options) !*link.File { +pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*link.File { assert(options.object_format == .coff); if (options.use_llvm) return error.LLVM_BackendIsTODO_ForCoff; // TODO if (options.use_lld) return error.LLD_LinkingIsTODO_ForCoff; // TODO - const file = try dir.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options) }); + const file = try options.dir.createFile(sub_path, .{ + .truncate = false, + .read = true, + .mode = link.determineMode(options), + }); errdefer file.close(); var coff_file = try allocator.create(Coff); diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 4bb2a4e940..22ed0446c4 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -216,65 +216,28 @@ pub const SrcFn = struct { }; }; -pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: link.Options) !*File { +pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*File { assert(options.object_format == .elf); if (options.use_llvm) return error.LLVMBackendUnimplementedForELF; // TODO - if (build_options.have_llvm and options.use_lld) { - std.debug.print("TODO open a temporary object file, not the final output file because we want to link with LLD\n", .{}); - } - - const file = try dir.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options) }); + const file = try options.dir.createFile(sub_path, .{ + .truncate = false, + .read = true, + .mode = link.determineMode(options), + }); errdefer file.close(); var elf_file = try allocator.create(Elf); errdefer allocator.destroy(elf_file); - elf_file.* = openFile(allocator, file, options) catch |err| switch (err) { - error.IncrFailed => try createFile(allocator, file, options), - else => |e| return e, - }; - + elf_file.* = try createFile(allocator, file, options); return &elf_file.base; } -/// Returns error.IncrFailed if incremental update could not be performed. -fn openFile(allocator: *Allocator, file: fs.File, options: link.Options) !Elf { - switch (options.effectiveOutputMode()) { - .Exe => {}, - .Obj => {}, - .Lib => return error.IncrFailed, - } - var self: Elf = .{ - .base = .{ - .file = file, - .tag = .elf, - .options = options, - .allocator = allocator, - }, - .ptr_width = switch (options.target.cpu.arch.ptrBitWidth()) { - 0 ... 32 => .p32, - 33 ... 64 => .p64, - else => return error.UnsupportedELFArchitecture, - }, - }; - errdefer self.deinit(); - - // TODO implement reading the elf file - return error.IncrFailed; - //try self.populateMissingMetadata(); - //return self; -} - /// Truncates the existing file contents and overwrites the contents. /// Returns an error if `file` is not already open with +read +write +seek abilities. fn createFile(allocator: *Allocator, file: fs.File, options: link.Options) !Elf { - switch (options.effectiveOutputMode()) { - .Exe => {}, - .Obj => {}, - .Lib => return error.TODOImplementWritingLibFiles, - } var self: Elf = .{ .base = .{ .tag = .elf, @@ -753,6 +716,10 @@ pub fn flush(self: *Elf, module: *Module) !void { } std.debug.print("TODO create an LLD command line and invoke it\n", .{}); } else { + switch (self.base.options.effectiveOutputMode()) { + .Exe, .Obj => {}, + .Lib => return error.TODOImplementWritingLibFiles, + } return self.flushInner(module); } } diff --git a/src-self-hosted/link/MachO.zig b/src-self-hosted/link/MachO.zig index 95f0d26f28..3ffa606d04 100644 --- a/src-self-hosted/link/MachO.zig +++ b/src-self-hosted/link/MachO.zig @@ -134,13 +134,17 @@ pub const SrcFn = struct { pub const empty = SrcFn{}; }; -pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: link.Options) !*File { +pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*File { assert(options.object_format == .macho); if (options.use_llvm) return error.LLVM_BackendIsTODO_ForMachO; // TODO if (options.use_lld) return error.LLD_LinkingIsTODO_ForMachO; // TODO - const file = try dir.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options) }); + const file = try options.dir.createFile(sub_path, .{ + .truncate = false, + .read = true, + .mode = link.determineMode(options), + }); errdefer file.close(); var macho_file = try allocator.create(MachO); diff --git a/src-self-hosted/link/Wasm.zig b/src-self-hosted/link/Wasm.zig index cc5cb427cd..46bafaaf10 100644 --- a/src-self-hosted/link/Wasm.zig +++ b/src-self-hosted/link/Wasm.zig @@ -49,14 +49,14 @@ base: link.File, /// TODO: can/should we access some data structure in Module directly? funcs: std.ArrayListUnmanaged(*Module.Decl) = .{}, -pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: link.Options) !*link.File { +pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*link.File { assert(options.object_format == .wasm); if (options.use_llvm) return error.LLVM_BackendIsTODO_ForWasm; // TODO if (options.use_lld) return error.LLD_LinkingIsTODO_ForWasm; // TODO // TODO: read the file and keep vaild parts instead of truncating - const file = try dir.createFile(sub_path, .{ .truncate = true, .read = true }); + const file = try options.dir.createFile(sub_path, .{ .truncate = true, .read = true }); errdefer file.close(); const wasm = try allocator.create(Wasm); diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 84bb6e9351..81a3126c7f 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -1000,6 +1000,7 @@ pub fn buildOutputType( .target = target_info.target, .output_mode = output_mode, .root_pkg = root_pkg, + .bin_file_dir_path = null, .bin_file_dir = fs.cwd(), .bin_file_path = bin_path, .link_mode = link_mode, From 5746a8658ea52dee4bf310c1290d76b1255cb5ec Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 9 Sep 2020 22:23:59 -0700 Subject: [PATCH 012/210] stage1: link: fix incorrect LDM option for mips64 --- src/link.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/link.cpp b/src/link.cpp index b54fdf6b93..3983b48b42 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -1777,6 +1777,7 @@ static const char *getLDMOption(const ZigTarget *t) { return "elf32btsmip"; case ZigLLVM_mipsel: return "elf32ltsmip"; + case ZigLLVM_mips64: return "elf64btsmip"; case ZigLLVM_mips64el: return "elf64ltsmip"; From e05ecbf165931d14440e6e5d089b64788b82d14f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 9 Sep 2020 22:24:17 -0700 Subject: [PATCH 013/210] stage2: progress towards LLD linking * add `zig libc` command * add `--libc` CLI and integrate it with Module and linker code * implement libc detection and paths resolution * port LLD ELF linker line construction to stage2 * integrate dynamic linker option into Module and linker code * implement default link_mode detection and error handling if user requests static when it cannot be fulfilled * integrate more linker options * implement detection of .so.X.Y.Z file extension as a shared object file. nice try, you can't fool me. * correct usage text for -dynamic and -static --- src-self-hosted/Module.zig | 193 +++++++++++++--- src-self-hosted/libc_installation.zig | 2 + src-self-hosted/link.zig | 19 ++ src-self-hosted/link/Elf.zig | 310 +++++++++++++++++++++++++- src-self-hosted/main.zig | 148 +++++++++--- src-self-hosted/target.zig | 25 +++ src-self-hosted/test.zig | 1 + 7 files changed, 634 insertions(+), 64 deletions(-) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 441ce15dc0..482a1d9a49 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -24,6 +24,7 @@ const liveness = @import("liveness.zig"); const astgen = @import("astgen.zig"); const zir_sema = @import("zir_sema.zig"); const build_options = @import("build_options"); +const LibCInstallation = @import("libc_installation.zig").LibCInstallation; /// General-purpose allocator. Used for both temporary and long-term storage. gpa: *Allocator, @@ -82,8 +83,6 @@ next_anon_name_index: usize = 0, /// contains Decls that need to be deleted if they end up having no references to them. deletion_set: std.ArrayListUnmanaged(*Decl) = .{}, -/// Owned by Module. -root_name: []u8, keep_source_files_loaded: bool, use_clang: bool, sanitize_c: bool, @@ -106,6 +105,19 @@ zig_cache_dir_path: []const u8, libc_include_dir_list: []const []const u8, rand: *std.rand.Random, +/// Populated when we build libc++.a. A WorkItem to build this is placed in the queue +/// and resolved before calling linker.flush(). +libcxx_static_lib: ?[]const u8 = null, +/// Populated when we build libc++abi.a. A WorkItem to build this is placed in the queue +/// and resolved before calling linker.flush(). +libcxxabi_static_lib: ?[]const u8 = null, +/// Populated when we build libunwind.a. A WorkItem to build this is placed in the queue +/// and resolved before calling linker.flush(). +libunwind_static_lib: ?[]const u8 = null, +/// Populated when we build c.a. A WorkItem to build this is placed in the queue +/// and resolved before calling linker.flush(). +libc_static_lib: ?[]const u8 = null, + pub const InnerError = error{ OutOfMemory, AnalysisFail }; const WorkItem = union(enum) { @@ -932,6 +944,7 @@ pub const InitOptions = struct { root_pkg: ?*Package, output_mode: std.builtin.OutputMode, rand: *std.rand.Random, + dynamic_linker: ?[]const u8 = null, bin_file_dir_path: ?[]const u8 = null, bin_file_dir: ?std.fs.Dir = null, bin_file_path: []const u8, @@ -941,6 +954,7 @@ pub const InitOptions = struct { optimize_mode: std.builtin.Mode = .Debug, keep_source_files_loaded: bool = false, clang_argv: []const []const u8 = &[0][]const u8{}, + lld_argv: []const []const u8 = &[0][]const u8{}, lib_dirs: []const []const u8 = &[0][]const u8{}, rpath_list: []const []const u8 = &[0][]const u8{}, c_source_files: []const []const u8 = &[0][]const u8{}, @@ -957,10 +971,11 @@ pub const InitOptions = struct { use_clang: ?bool = null, rdynamic: bool = false, strip: bool = false, + is_native_os: bool, + link_eh_frame_hdr: bool = false, linker_script: ?[]const u8 = null, version_script: ?[]const u8 = null, override_soname: ?[]const u8 = null, - linker_optimization: ?[]const u8 = null, linker_gc_sections: ?bool = null, function_sections: ?bool = null, linker_allow_shlib_undefined: ?bool = null, @@ -969,8 +984,10 @@ pub const InitOptions = struct { linker_z_nodelete: bool = false, linker_z_defs: bool = false, clang_passthrough_mode: bool = false, - stack_size_override: u64 = 0, + stack_size_override: ?u64 = null, self_exe_path: ?[]const u8 = null, + version: std.builtin.Version = .{ .major = 0, .minor = 0, .patch = 0 }, + libc_installation: ?*const LibCInstallation = null, }; pub fn create(gpa: *Allocator, options: InitOptions) !*Module { @@ -1002,6 +1019,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { options.frameworks.len != 0 or options.system_libs.len != 0 or options.link_libc or options.link_libcpp or + options.link_eh_frame_hdr or options.linker_script != null or options.version_script != null) { break :blk true; @@ -1017,6 +1035,35 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { break :blk false; }; + const must_dynamic_link = dl: { + if (target_util.cannotDynamicLink(options.target)) + break :dl false; + if (target_util.osRequiresLibC(options.target)) + break :dl true; + if (options.link_libc and options.target.isGnuLibC()) + break :dl true; + if (options.system_libs.len != 0) + break :dl true; + + break :dl false; + }; + const default_link_mode: std.builtin.LinkMode = if (must_dynamic_link) .Dynamic else .Static; + const link_mode: std.builtin.LinkMode = if (options.link_mode) |lm| blk: { + if (lm == .Static and must_dynamic_link) { + return error.UnableToStaticLink; + } + break :blk lm; + } else default_link_mode; + + const libc_dirs = try detectLibCIncludeDirs( + arena, + options.zig_lib_dir, + options.target, + options.is_native_os, + options.link_libc, + options.libc_installation, + ); + const bin_file = try link.File.openPath(gpa, .{ .dir = options.bin_file_dir orelse std.fs.cwd(), .dir_path = options.bin_file_dir_path, @@ -1024,8 +1071,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { .root_name = root_name, .root_pkg = options.root_pkg, .target = options.target, + .dynamic_linker = options.dynamic_linker, .output_mode = options.output_mode, - .link_mode = options.link_mode orelse .Static, + .link_mode = link_mode, .object_format = ofmt, .optimize_mode = options.optimize_mode, .use_lld = use_lld, @@ -1039,7 +1087,22 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { .lib_dirs = options.lib_dirs, .rpath_list = options.rpath_list, .strip = options.strip, + .is_native_os = options.is_native_os, .function_sections = options.function_sections orelse false, + .allow_shlib_undefined = options.linker_allow_shlib_undefined, + .bind_global_refs_locally = options.linker_bind_global_refs_locally orelse false, + .z_nodelete = options.linker_z_nodelete, + .z_defs = options.linker_z_defs, + .stack_size_override = options.stack_size_override, + .linker_script = options.linker_script, + .version_script = options.version_script, + .gc_sections = options.linker_gc_sections, + .eh_frame_hdr = options.link_eh_frame_hdr, + .rdynamic = options.rdynamic, + .extra_lld_args = options.lld_argv, + .override_soname = options.override_soname, + .version = options.version, + .libc_installation = libc_dirs.libc_installation, }); errdefer bin_file.destroy(); @@ -1146,13 +1209,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { break :blk true; }; - const libc_include_dir_list = try detectLibCIncludeDirs( - arena, - options.zig_lib_dir, - options.target, - options.link_libc, - ); - const sanitize_c: bool = options.want_sanitize_c orelse switch (options.optimize_mode) { .Debug, .ReleaseSafe => true, .ReleaseSmall, .ReleaseFast => false, @@ -1163,7 +1219,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { .arena_state = arena_allocator.state, .zig_lib_dir = options.zig_lib_dir, .zig_cache_dir_path = zig_cache_dir_path, - .root_name = root_name, .root_pkg = options.root_pkg, .root_scope = root_scope, .bin_file = bin_file, @@ -1174,7 +1229,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { .c_source_files = options.c_source_files, .cache = cache, .self_exe_path = options.self_exe_path, - .libc_include_dir_list = libc_include_dir_list, + .libc_include_dir_list = libc_dirs.libc_include_dir_list, .sanitize_c = sanitize_c, .rand = options.rand, .clang_passthrough_mode = options.clang_passthrough_mode, @@ -1544,7 +1599,10 @@ fn buildCObject(mod: *Module, c_object: *CObject) !void { // directly to the output file. const direct_o = mod.c_source_files.len == 1 and mod.root_pkg == null and mod.bin_file.options.output_mode == .Obj and mod.bin_file.options.objects.len == 0; - const o_basename_noext = if (direct_o) mod.root_name else mem.split(c_source_basename, ".").next().?; + const o_basename_noext = if (direct_o) + mod.bin_file.options.root_name + else + mem.split(c_source_basename, ".").next().?; const o_basename = try std.fmt.allocPrint(arena, "{}{}", .{ o_basename_noext, mod.getTarget().oFileExt() }); // We can't know the digest until we do the C compiler invocation, so we need a temporary filename. @@ -1749,7 +1807,7 @@ fn addCCArgs( try argv.append(p); } }, - .assembly, .ll, .bc, .unknown => {}, + .so, .assembly, .ll, .bc, .unknown => {}, } // TODO CLI args for cpu features when compiling assembly //for (size_t i = 0; i < g->zig_target->llvm_cpu_features_asm_len; i += 1) { @@ -4259,6 +4317,7 @@ pub const FileExt = enum { ll, bc, assembly, + so, unknown, }; @@ -4290,10 +4349,36 @@ pub fn classifyFileExt(filename: []const u8) FileExt { return .assembly; } else if (mem.endsWith(u8, filename, ".h")) { return .h; - } else { - // TODO look for .so, .so.X, .so.X.Y, .so.X.Y.Z - return .unknown; + } else if (mem.endsWith(u8, filename, ".so")) { + return .so; } + // Look for .so.X, .so.X.Y, .so.X.Y.Z + var it = mem.split(filename, "."); + _ = it.next().?; + var so_txt = it.next() orelse return .unknown; + while (!mem.eql(u8, so_txt, "so")) { + so_txt = it.next() orelse return .unknown; + } + const n1 = it.next() orelse return .unknown; + const n2 = it.next(); + const n3 = it.next(); + + _ = std.fmt.parseInt(u32, n1, 10) catch return .unknown; + if (n2) |x| _ = std.fmt.parseInt(u32, x, 10) catch return .unknown; + if (n3) |x| _ = std.fmt.parseInt(u32, x, 10) catch return .unknown; + if (it.next() != null) return .unknown; + + return .so; +} + +test "classifyFileExt" { + std.testing.expectEqual(FileExt.cpp, classifyFileExt("foo.cc")); + std.testing.expectEqual(FileExt.unknown, classifyFileExt("foo.nim")); + std.testing.expectEqual(FileExt.so, classifyFileExt("foo.so")); + std.testing.expectEqual(FileExt.so, classifyFileExt("foo.so.1")); + std.testing.expectEqual(FileExt.so, classifyFileExt("foo.so.1.2")); + std.testing.expectEqual(FileExt.so, classifyFileExt("foo.so.1.2.3")); + std.testing.expectEqual(FileExt.unknown, classifyFileExt("foo.so.1.2.3~")); } fn haveFramePointer(mod: *Module) bool { @@ -4303,16 +4388,29 @@ fn haveFramePointer(mod: *Module) bool { }; } +const LibCDirs = struct { + libc_include_dir_list: []const []const u8, + libc_installation: ?*const LibCInstallation, +}; + fn detectLibCIncludeDirs( arena: *Allocator, zig_lib_dir: []const u8, target: Target, + is_native_os: bool, link_libc: bool, -) ![]const []const u8 { - if (!link_libc) return &[0][]u8{}; + libc_installation: ?*const LibCInstallation, +) !LibCDirs { + if (!link_libc) { + return LibCDirs{ + .libc_include_dir_list = &[0][]u8{}, + .libc_installation = null, + }; + } - // TODO Support --libc file explicitly providing libc paths. Or not? Maybe we are better off - // deleting that feature. + if (libc_installation) |lci| { + return detectLibCFromLibCInstallation(arena, target, lci); + } if (target_util.canBuildLibC(target)) { const generic_name = target_util.libCGenericName(target); @@ -4348,9 +4446,52 @@ fn detectLibCIncludeDirs( list[1] = generic_include_dir; list[2] = arch_os_include_dir; list[3] = generic_os_include_dir; - return list; + return LibCDirs{ + .libc_include_dir_list = list, + .libc_installation = null, + }; } - // TODO finish porting detect_libc from codegen.cpp - return error.LibCDetectionUnimplemented; + if (is_native_os) { + const libc = try arena.create(LibCInstallation); + libc.* = try LibCInstallation.findNative(.{ .allocator = arena }); + return detectLibCFromLibCInstallation(arena, target, libc); + } + + return LibCDirs{ + .libc_include_dir_list = &[0][]u8{}, + .libc_installation = null, + }; +} + +fn detectLibCFromLibCInstallation(arena: *Allocator, target: Target, lci: *const LibCInstallation) !LibCDirs { + var list = std.ArrayList([]const u8).init(arena); + try list.ensureCapacity(4); + + list.appendAssumeCapacity(lci.include_dir.?); + + const is_redundant = mem.eql(u8, lci.sys_include_dir.?, lci.include_dir.?); + if (!is_redundant) list.appendAssumeCapacity(lci.sys_include_dir.?); + + if (target.os.tag == .windows) { + if (std.fs.path.dirname(lci.include_dir.?)) |include_dir_parent| { + const um_dir = try std.fs.path.join(arena, &[_][]const u8{ include_dir_parent, "um" }); + list.appendAssumeCapacity(um_dir); + + const shared_dir = try std.fs.path.join(arena, &[_][]const u8{ include_dir_parent, "shared" }); + list.appendAssumeCapacity(shared_dir); + } + } + return LibCDirs{ + .libc_include_dir_list = list.items, + .libc_installation = lci, + }; +} + +pub fn get_libc_crt_file(mod: *Module, arena: *Allocator, basename: []const u8) ![]const u8 { + // TODO port support for building crt files from stage1 + const lci = mod.bin_file.options.libc_installation orelse return error.LibCInstallationNotAvailable; + const crt_dir_path = lci.crt_dir orelse return error.LibCInstallationMissingCRTDir; + const full_path = try std.fs.path.join(arena, &[_][]const u8{ crt_dir_path, basename }); + return full_path; } diff --git a/src-self-hosted/libc_installation.zig b/src-self-hosted/libc_installation.zig index 65c6c8c16d..125f514060 100644 --- a/src-self-hosted/libc_installation.zig +++ b/src-self-hosted/libc_installation.zig @@ -11,6 +11,8 @@ const is_gnu = Target.current.isGnu(); usingnamespace @import("windows_sdk.zig"); +// TODO Rework this abstraction to use std.log instead of taking a stderr stream. + /// See the render function implementation for documentation of the fields. pub const LibCInstallation = struct { include_dir: ?[]const u8 = null, diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 22c8a6c8e3..68292fbfe6 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -6,6 +6,7 @@ const trace = @import("tracy.zig").trace; const Package = @import("Package.zig"); const Type = @import("type.zig").Type; const build_options = @import("build_options"); +const LibCInstallation = @import("libc_installation.zig").LibCInstallation; pub const producer_string = if (std.builtin.is_test) "zig test" else "zig " ++ build_options.version; @@ -23,6 +24,7 @@ pub const Options = struct { optimize_mode: std.builtin.Mode, root_name: []const u8, root_pkg: ?*const Package, + dynamic_linker: ?[]const u8 = null, /// Used for calculating how much space to reserve for symbols in case the binary file /// does not already have a symbol table. symbol_count_hint: u64 = 32, @@ -30,6 +32,7 @@ pub const Options = struct { /// the binary file does not already have such a section. program_code_size_hint: u64 = 256 * 1024, entry_addr: ?u64 = null, + stack_size_override: ?u64 = null, /// Set to `true` to omit debug info. strip: bool = false, /// If this is true then this link code is responsible for outputting an object @@ -44,6 +47,19 @@ pub const Options = struct { link_libc: bool = false, link_libcpp: bool = false, function_sections: bool = false, + eh_frame_hdr: bool = false, + rdynamic: bool = false, + z_nodelete: bool = false, + z_defs: bool = false, + bind_global_refs_locally: bool, + is_native_os: bool, + gc_sections: ?bool = null, + allow_shlib_undefined: ?bool = null, + linker_script: ?[]const u8 = null, + version_script: ?[]const u8 = null, + override_soname: ?[]const u8 = null, + /// Extra args passed directly to LLD. Ignored when not linking with LLD. + extra_lld_args: []const []const u8 = &[0][]const u8, objects: []const []const u8 = &[0][]const u8{}, framework_dirs: []const []const u8 = &[0][]const u8{}, @@ -52,6 +68,9 @@ pub const Options = struct { lib_dirs: []const []const u8 = &[0][]const u8{}, rpath_list: []const []const u8 = &[0][]const u8{}, + version: std.builtin.Version, + libc_installation: ?*const LibCInstallation, + pub fn effectiveOutputMode(options: Options) std.builtin.OutputMode { return if (options.use_lld) .Obj else options.output_mode; } diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 22ed0446c4..835ab12cfa 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -18,6 +18,7 @@ const link = @import("../link.zig"); const File = link.File; const Elf = @This(); const build_options = @import("build_options"); +const target_util = @import("../target.zig"); const default_entry_addr = 0x8000000; @@ -709,12 +710,7 @@ pub const abbrev_parameter = 6; pub fn flush(self: *Elf, module: *Module) !void { if (build_options.have_llvm and self.base.options.use_lld) { - // If there is no Zig code to compile, then we should skip flushing the output file because it - // will not be part of the linker line anyway. - if (module.root_pkg != null) { - try self.flushInner(module); - } - std.debug.print("TODO create an LLD command line and invoke it\n", .{}); + return self.linkWithLLD(module); } else { switch (self.base.options.effectiveOutputMode()) { .Exe, .Obj => {}, @@ -1202,6 +1198,275 @@ fn flushInner(self: *Elf, module: *Module) !void { assert(!self.debug_strtab_dirty); } +fn linkWithLLD(self: *Elf, module: *Module) !void { + // If there is no Zig code to compile, then we should skip flushing the output file because it + // will not be part of the linker line anyway. + if (module.root_pkg != null) { + try self.flushInner(module); + } + var arena_allocator = std.heap.ArenaAllocator.init(self.base.allocator); + defer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + const target = self.base.options.target; + const is_obj = self.base.options.output_mode == .Obj; + + // Create an LLD command line and invoke it. + var argv = std.ArrayList([]const u8).init(self.base.allocator); + defer argv.deinit(); + // Even though we're calling LLD as a library it thinks the first argument is its own exe name. + try argv.append("lld"); + if (is_obj) { + try argv.append("-r"); + } + if (self.base.options.output_mode == .Lib and + self.base.options.link_mode == .Static and + !target.isWasm()) + { + // TODO port the code from link.cpp + return error.TODOMakeArchive; + } + const link_in_crt = self.base.options.link_libc and self.base.options.output_mode == .Exe; + + try argv.append("-error-limit=0"); + + if (self.base.options.output_mode == .Exe) { + try argv.append("-z"); + const stack_size = self.base.options.stack_size_override orelse 16777216; + const arg = try std.fmt.allocPrint(arena, "stack-size={}", .{stack_size}); + try argv.append(arg); + } + + if (self.base.options.linker_script) |linker_script| { + try argv.append("-T"); + try argv.append(linker_script); + } + + const gc_sections = self.base.options.gc_sections orelse !is_obj; + if (gc_sections) { + try argv.append("--gc-sections"); + } + + if (self.base.options.eh_frame_hdr) { + try argv.append("--eh-frame-hdr"); + } + + if (self.base.options.rdynamic) { + try argv.append("--export-dynamic"); + } + + try argv.appendSlice(self.base.options.extra_lld_args); + + if (self.base.options.z_nodelete) { + try argv.append("-z"); + try argv.append("nodelete"); + } + if (self.base.options.z_defs) { + try argv.append("-z"); + try argv.append("defs"); + } + + if (getLDMOption(target)) |ldm| { + // Any target ELF will use the freebsd osabi if suffixed with "_fbsd". + const arg = if (target.os.tag == .freebsd) + try std.fmt.allocPrint(arena, "{}_fbsd", .{ldm}) + else + ldm; + try argv.append("-m"); + try argv.append(arg); + } + + const is_lib = self.base.options.output_mode == .Lib; + const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib; + if (self.base.options.link_mode == .Static) { + if (target.cpu.arch.isARM() or target.cpu.arch.isThumb()) { + try argv.append("-Bstatic"); + } else { + try argv.append("-static"); + } + } else if (is_dyn_lib) { + try argv.append("-shared"); + } + + if (target_util.requiresPIE(target) and self.base.options.output_mode == .Exe) { + try argv.append("-pie"); + } + + const full_out_path = if (self.base.options.dir_path) |dir_path| + try std.fs.path.join(arena, &[_][]const u8{dir_path, self.base.options.sub_path}) + else + self.base.options.sub_path; + try argv.append("-o"); + try argv.append(full_out_path); + + if (link_in_crt) { + const crt1o: []const u8 = o: { + if (target.os.tag == .netbsd) { + break :o "crt0.o"; + } else if (target.isAndroid()) { + if (self.base.options.link_mode == .Dynamic) { + break :o "crtbegin_dynamic.o"; + } else { + break :o "crtbegin_static.o"; + } + } else if (self.base.options.link_mode == .Static) { + break :o "crt1.o"; + } else { + break :o "Scrt1.o"; + } + }; + try argv.append(try module.get_libc_crt_file(arena, crt1o)); + if (target_util.libc_needs_crti_crtn(target)) { + try argv.append(try module.get_libc_crt_file(arena, "crti.o")); + } + } + + // TODO rpaths + //for (size_t i = 0; i < g->rpath_list.length; i += 1) { + // Buf *rpath = g->rpath_list.at(i); + // add_rpath(lj, rpath); + //} + //if (g->each_lib_rpath) { + // for (size_t i = 0; i < g->lib_dirs.length; i += 1) { + // const char *lib_dir = g->lib_dirs.at(i); + // for (size_t i = 0; i < g->link_libs_list.length; i += 1) { + // LinkLib *link_lib = g->link_libs_list.at(i); + // if (buf_eql_str(link_lib->name, "c")) { + // continue; + // } + // bool does_exist; + // Buf *test_path = buf_sprintf("%s/lib%s.so", lib_dir, buf_ptr(link_lib->name)); + // if (os_file_exists(test_path, &does_exist) != ErrorNone) { + // zig_panic("link: unable to check if file exists: %s", buf_ptr(test_path)); + // } + // if (does_exist) { + // add_rpath(lj, buf_create_from_str(lib_dir)); + // break; + // } + // } + // } + //} + + for (self.base.options.lib_dirs) |lib_dir| { + try argv.append("-L"); + try argv.append(lib_dir); + } + + if (self.base.options.link_libc) { + if (self.base.options.libc_installation) |libc_installation| { + try argv.append("-L"); + try argv.append(libc_installation.crt_dir.?); + } + + if (self.base.options.link_mode == .Dynamic and (is_dyn_lib or self.base.options.output_mode == .Exe)) { + if (self.base.options.dynamic_linker) |dynamic_linker| { + try argv.append("-dynamic-linker"); + try argv.append(dynamic_linker); + } + } + } + + if (is_dyn_lib) { + const soname = self.base.options.override_soname orelse + try std.fmt.allocPrint(arena, "lib{}.so.{}", .{self.base.options.root_name, + self.base.options.version.major,}); + try argv.append("-soname"); + try argv.append(soname); + + if (self.base.options.version_script) |version_script| { + try argv.append("-version-script"); + try argv.append(version_script); + } + } + + // Positional arguments to the linker such as object files. + try argv.appendSlice(self.base.options.objects); + + // TODO compiler-rt and libc + //if (!g->is_dummy_so && (g->out_type == OutTypeExe || is_dyn_lib)) { + // if (g->libc_link_lib == nullptr) { + // Buf *libc_a_path = build_c(g, OutTypeLib, lj->build_dep_prog_node); + // try argv.append(buf_ptr(libc_a_path)); + // } + + // Buf *compiler_rt_o_path = build_compiler_rt(g, OutTypeLib, lj->build_dep_prog_node); + // try argv.append(buf_ptr(compiler_rt_o_path)); + //} + + // Shared libraries. + try argv.ensureCapacity(argv.items.len + self.base.options.system_libs.len); + for (self.base.options.system_libs) |link_lib| { + // By this time, we depend on these libs being dynamically linked libraries and not static libraries + // (the check for that needs to be earlier), but they could be full paths to .so files, in which + // case we want to avoid prepending "-l". + const ext = Module.classifyFileExt(link_lib); + const arg = if (ext == .so) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib}); + argv.appendAssumeCapacity(arg); + } + + if (!is_obj) { + // libc++ dep + if (self.base.options.link_libcpp) { + try argv.append(module.libcxxabi_static_lib.?); + try argv.append(module.libcxx_static_lib.?); + } + + // libc dep + if (self.base.options.link_libc) { + if (self.base.options.libc_installation != null) { + if (self.base.options.link_mode == .Static) { + try argv.append("--start-group"); + try argv.append("-lc"); + try argv.append("-lm"); + try argv.append("--end-group"); + } else { + try argv.append("-lc"); + try argv.append("-lm"); + } + + if (target.os.tag == .freebsd or target.os.tag == .netbsd) { + try argv.append("-lpthread"); + } + } else if (target.isGnuLibC()) { + try argv.append(module.libunwind_static_lib.?); + // TODO here we need to iterate over the glibc libs and add the .so files to the linker line. + std.log.warn("TODO port add_glibc_libs to stage2", .{}); + try argv.append(try module.get_libc_crt_file(arena, "libc_nonshared.a")); + } else if (target.isMusl()) { + try argv.append(module.libunwind_static_lib.?); + try argv.append(module.libc_static_lib.?); + } else if (self.base.options.link_libcpp) { + try argv.append(module.libunwind_static_lib.?); + } else { + unreachable; // Compiler was supposed to emit an error for not being able to provide libc. + } + } + } + + // crt end + if (link_in_crt) { + if (target.isAndroid()) { + try argv.append(try module.get_libc_crt_file(arena, "crtend_android.o")); + } else if (target_util.libc_needs_crti_crtn(target)) { + try argv.append(try module.get_libc_crt_file(arena, "crtn.o")); + } + } + + const allow_shlib_undefined = self.base.options.allow_shlib_undefined orelse !self.base.options.is_native_os; + if (allow_shlib_undefined) { + try argv.append("--allow-shlib-undefined"); + } + + if (self.base.options.bind_global_refs_locally) { + try argv.append("-Bsymbolic"); + } + + for (argv.items) |arg| { + std.debug.print("{} ", .{arg}); + } + @panic("invoke LLD"); +} + fn writeDwarfAddrAssumeCapacity(self: *Elf, buf: *std.ArrayList(u8), addr: u64) void { const target_endian = self.base.options.target.cpu.arch.endian(); switch (self.ptr_width) { @@ -2616,3 +2881,36 @@ fn sectHeaderTo32(shdr: elf.Elf64_Shdr) elf.Elf32_Shdr { .sh_entsize = @intCast(u32, shdr.sh_entsize), }; } + +fn getLDMOption(target: std.Target) ?[]const u8 { + switch (target.cpu.arch) { + .i386 => return "elf_i386", + .aarch64 => return "aarch64linux", + .aarch64_be => return "aarch64_be_linux", + .arm, .thumb => return "armelf_linux_eabi", + .armeb, .thumbeb => return "armebelf_linux_eabi", + .powerpc => return "elf32ppclinux", + .powerpc64 => return "elf64ppc", + .powerpc64le => return "elf64lppc", + .sparc, .sparcel => return "elf32_sparc", + .sparcv9 => return "elf64_sparc", + .mips => return "elf32btsmip", + .mipsel => return "elf32ltsmip", + .mips64 => return "elf64btsmip", + .mips64el => return "elf64ltsmip", + .s390x => return "elf64_s390", + .x86_64 => { + if (target.abi == .gnux32) { + return "elf32_x86_64"; + } + // Any target elf will use the freebsd osabi if suffixed with "_fbsd". + if (target.os.tag == .freebsd) { + return "elf_x86_64_fbsd"; + } + return "elf_x86_64"; + }, + .riscv32 => return "elf32lriscv", + .riscv64 => return "elf64lriscv", + else => return null, + } +} diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 81a3126c7f..1de025769d 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -14,6 +14,7 @@ const zir = @import("zir.zig"); const build_options = @import("build_options"); const warn = std.log.warn; const introspect = @import("introspect.zig"); +const LibCInstallation = @import("libc_installation.zig").LibCInstallation; fn fatal(comptime format: []const u8, args: anytype) noreturn { std.log.emerg(format, args); @@ -33,18 +34,22 @@ const usage = \\ \\Commands: \\ - \\ build-exe [source] Create executable from source or object files - \\ build-lib [source] Create library from source or object files - \\ build-obj [source] Create object from source or assembly - \\ cc Use Zig as a drop-in C compiler - \\ c++ Use Zig as a drop-in C++ compiler - \\ env Print lib path, std path, compiler id and version - \\ fmt [source] Parse file and render in canonical zig format - \\ translate-c [source] Convert C code to Zig code - \\ targets List available compilation targets - \\ version Print version number and exit - \\ zen Print zen of zig and exit + \\ build-exe Create executable from source or object files + \\ build-lib Create library from source or object files + \\ build-obj Create object from source or assembly + \\ cc Use Zig as a drop-in C compiler + \\ c++ Use Zig as a drop-in C++ compiler + \\ env Print lib path, std path, compiler id and version + \\ fmt Parse file and render in canonical zig format + \\ libc Display native libc paths file or validate one + \\ translate-c Convert C code to Zig code + \\ targets List available compilation targets + \\ version Print version number and exit + \\ zen Print zen of zig and exit \\ + \\General Options: + \\ + \\ --help Print command-specific usage \\ ; @@ -126,6 +131,8 @@ pub fn main() !void { return punt_to_clang(arena, args); } else if (mem.eql(u8, cmd, "fmt")) { return cmdFmt(gpa, cmd_args); + } else if (mem.eql(u8, cmd, "libc")) { + return cmdLibC(gpa, cmd_args); } else if (mem.eql(u8, cmd, "targets")) { const info = try std.zig.system.NativeTargetInfo.detect(arena, .{}); const stdout = io.getStdOut().outStream(); @@ -184,7 +191,6 @@ const usage_build_generic = \\ ReleaseSmall Optimize for small binary, safety off \\ -fPIC Force-enable Position Independent Code \\ -fno-PIC Force-disable Position Independent Code - \\ --dynamic Force output to be dynamically linked \\ --strip Exclude debug symbols \\ -ofmt=[mode] Override target object format \\ elf Executable and Linking Format @@ -199,6 +205,7 @@ const usage_build_generic = \\ -isystem [dir] Add directory to SYSTEM include search path \\ -I[dir] Add directory to include search path \\ -D[macro]=[value] Define C [macro] to [value] (1 if [value] omitted) + \\ --libc [file] Provide a file which specifies libc paths \\ \\Link Options: \\ -l[lib], --library [lib] Link against system library @@ -208,6 +215,9 @@ const usage_build_generic = \\ --version [ver] Dynamic library semver \\ -rdynamic Add all symbols to the dynamic symbol table \\ -rpath [path] Add directory to the runtime library search path + \\ --eh-frame-hdr Enable C++ exception handling by passing --eh-frame-hdr to linker + \\ -dynamic Force output to be dynamically linked + \\ -static Force output to be statically linked \\ \\Debug Options (Zig Compiler Development): \\ -ftime-report Print timing diagnostics @@ -220,6 +230,14 @@ const usage_build_generic = \\ ; +const repl_help = + \\Commands: + \\ update Detect changes to source files and update output files. + \\ help Print this text + \\ exit Quit this repl + \\ +; + const Emit = union(enum) { no, yes_default_path, @@ -275,16 +293,17 @@ pub fn buildOutputType( var version_script: ?[]const u8 = null; var disable_c_depfile = false; var override_soname: ?[]const u8 = null; - var linker_optimization: ?[]const u8 = null; var linker_gc_sections: ?bool = null; var linker_allow_shlib_undefined: ?bool = null; var linker_bind_global_refs_locally: ?bool = null; var linker_z_nodelete = false; var linker_z_defs = false; - var stack_size_override: u64 = 0; + var stack_size_override: ?u64 = null; var use_llvm: ?bool = null; var use_lld: ?bool = null; var use_clang: ?bool = null; + var link_eh_frame_hdr = false; + var libc_paths_file: ?[]const u8 = null; var system_libs = std.ArrayList([]const u8).init(gpa); defer system_libs.deinit(); @@ -292,6 +311,9 @@ pub fn buildOutputType( var clang_argv = std.ArrayList([]const u8).init(gpa); defer clang_argv.deinit(); + var lld_argv = std.ArrayList([]const u8).init(gpa); + defer lld_argv.deinit(); + var lib_dirs = std.ArrayList([]const u8).init(gpa); defer lib_dirs.deinit(); @@ -414,15 +436,11 @@ pub fn buildOutputType( fatal("unable to parse --version '{}': {}", .{ args[i], @errorName(err) }); }; } else if (mem.eql(u8, arg, "-target")) { - if (i + 1 >= args.len) { - fatal("expected parameter after -target", .{}); - } + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; target_arch_os_abi = args[i]; } else if (mem.eql(u8, arg, "-mcpu")) { - if (i + 1 >= args.len) { - fatal("expected parameter after -mcpu", .{}); - } + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; target_mcpu = args[i]; } else if (mem.startsWith(u8, arg, "-ofmt=")) { @@ -430,11 +448,13 @@ pub fn buildOutputType( } else if (mem.startsWith(u8, arg, "-mcpu=")) { target_mcpu = arg["-mcpu=".len..]; } else if (mem.eql(u8, arg, "--dynamic-linker")) { - if (i + 1 >= args.len) { - fatal("expected parameter after --dynamic-linker", .{}); - } + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; target_dynamic_linker = args[i]; + } else if (mem.eql(u8, arg, "--libc")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + libc_paths_file = args[i]; } else if (mem.eql(u8, arg, "--watch")) { watch = true; } else if (mem.eql(u8, arg, "-ftime-report")) { @@ -481,6 +501,8 @@ pub fn buildOutputType( link_mode = .Static; } else if (mem.eql(u8, arg, "--strip")) { strip = true; + } else if (mem.eql(u8, arg, "--eh-frame-hdr")) { + link_eh_frame_hdr = true; } else if (mem.eql(u8, arg, "-Bsymbolic")) { linker_bind_global_refs_locally = true; } else if (mem.eql(u8, arg, "--debug-tokenize")) { @@ -565,7 +587,7 @@ pub fn buildOutputType( const file_ext = Module.classifyFileExt(mem.spanZ(it.only_arg)); switch (file_ext) { .assembly, .c, .cpp, .ll, .bc, .h => try c_source_files.append(it.only_arg), - .unknown => try link_objects.append(it.only_arg), + .unknown, .so => try link_objects.append(it.only_arg), } }, .l => { @@ -716,7 +738,7 @@ pub fn buildOutputType( } version_script = linker_args.items[i]; } else if (mem.startsWith(u8, arg, "-O")) { - linker_optimization = arg; + try lld_argv.append(arg); } else if (mem.eql(u8, arg, "--gc-sections")) { linker_gc_sections = true; } else if (mem.eql(u8, arg, "--no-gc-sections")) { @@ -994,10 +1016,21 @@ pub fn buildOutputType( }; var default_prng = std.rand.DefaultPrng.init(random_seed); + var libc_installation: ?LibCInstallation = null; + defer if (libc_installation) |*l| l.deinit(gpa); + + if (libc_paths_file) |paths_file| { + libc_installation = LibCInstallation.parse(gpa, paths_file, io.getStdErr().writer()) catch |err| { + fatal("unable to parse libc paths file: {}", .{@errorName(err)}); + }; + } + const module = Module.create(gpa, .{ .zig_lib_dir = zig_lib_dir, .root_name = root_name, .target = target_info.target, + .is_native_os = cross_target.isNativeOs(), + .dynamic_linker = target_info.dynamic_linker.get(), .output_mode = output_mode, .root_pkg = root_pkg, .bin_file_dir_path = null, @@ -1008,6 +1041,7 @@ pub fn buildOutputType( .optimize_mode = build_mode, .keep_source_files_loaded = zir_out_path != null, .clang_argv = clang_argv.items, + .lld_argv = lld_argv.items, .lib_dirs = lib_dirs.items, .rpath_list = rpath_list.items, .c_source_files = c_source_files.items, @@ -1028,17 +1062,19 @@ pub fn buildOutputType( .version_script = version_script, .disable_c_depfile = disable_c_depfile, .override_soname = override_soname, - .linker_optimization = linker_optimization, .linker_gc_sections = linker_gc_sections, .linker_allow_shlib_undefined = linker_allow_shlib_undefined, .linker_bind_global_refs_locally = linker_bind_global_refs_locally, .linker_z_nodelete = linker_z_nodelete, .linker_z_defs = linker_z_defs, + .link_eh_frame_hdr = link_eh_frame_hdr, .stack_size_override = stack_size_override, .strip = strip, .self_exe_path = self_exe_path, .rand = &default_prng.random, .clang_passthrough_mode = arg_mode != .build, + .version = version, + .libc_installation = if (libc_installation) |*lci| lci else null, }) catch |err| { fatal("unable to create module: {}", .{@errorName(err)}); }; @@ -1116,16 +1152,64 @@ fn updateModule(gpa: *Allocator, module: *Module, zir_out_path: ?[]const u8) !vo } } -const repl_help = - \\Commands: - \\ update Detect changes to source files and update output files. - \\ help Print this text - \\ exit Quit this repl +pub const usage_libc = + \\Usage: zig libc + \\ + \\ Detect the native libc installation and print the resulting + \\ paths to stdout. You can save this into a file and then edit + \\ the paths to create a cross compilation libc kit. Then you + \\ can pass `--libc [file]` for Zig to use it. + \\ + \\Usage: zig libc [paths_file] + \\ + \\ Parse a libc installation text file and validate it. \\ ; +pub fn cmdLibC(gpa: *Allocator, args: []const []const u8) !void { + var input_file: ?[]const u8 = null; + { + var i: usize = 0; + while (i < args.len) : (i += 1) { + const arg = args[i]; + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "--help")) { + const stdout = io.getStdOut().writer(); + try stdout.writeAll(usage_libc); + process.exit(0); + } else { + fatal("unrecognized parameter: '{}'", .{arg}); + } + } else if (input_file != null) { + fatal("unexpected extra parameter: '{}'", .{arg}); + } else { + input_file = arg; + } + } + } + if (input_file) |libc_file| { + const stderr = std.io.getStdErr().writer(); + var libc = LibCInstallation.parse(gpa, libc_file, stderr) catch |err| { + fatal("unable to parse libc file: {}", .{@errorName(err)}); + }; + defer libc.deinit(gpa); + } else { + var libc = LibCInstallation.findNative(.{ + .allocator = gpa, + .verbose = true, + }) catch |err| { + fatal("unable to detect native libc: {}", .{@errorName(err)}); + }; + defer libc.deinit(gpa); + + var bos = io.bufferedOutStream(io.getStdOut().writer()); + try libc.render(bos.writer()); + try bos.flush(); + } +} + pub const usage_fmt = - \\usage: zig fmt [file]... + \\Usage: zig fmt [file]... \\ \\ Formats the input files and modifies them in-place. \\ Arguments can be files or directories, which are searched diff --git a/src-self-hosted/target.zig b/src-self-hosted/target.zig index cb09779d74..127abeeadc 100644 --- a/src-self-hosted/target.zig +++ b/src-self-hosted/target.zig @@ -109,3 +109,28 @@ pub fn canBuildLibC(target: std.Target) bool { } return false; } + +pub fn cannotDynamicLink(target: std.Target) bool { + return switch (target.os.tag) { + .freestanding, .other => true, + else => false, + }; +} + +pub fn osRequiresLibC(target: std.Target) bool { + // On Darwin, we always link libSystem which contains libc. + // Similarly on FreeBSD and NetBSD we always link system libc + // since this is the stable syscall interface. + return switch (target.os.tag) { + .freebsd, .netbsd, .dragonfly, .macosx, .ios, .watchos, .tvos => true, + else => false, + }; +} + +pub fn requiresPIE(target: std.Target) bool { + return target.isAndroid(); +} + +pub fn libc_needs_crti_crtn(target: std.Target) bool { + return !(target.cpu.arch.isRISCV() or target.isAndroid()); +} diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 004b56b029..f23695257b 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -472,6 +472,7 @@ pub const TestContext = struct { .root_pkg = root_pkg, .keep_source_files_loaded = true, .object_format = ofmt, + .is_native_os = case.target.isNativeOs(), }); defer module.destroy(); From 98583be6e110406a37d0904c50c04d68efbffcd0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 10 Sep 2020 22:22:42 -0700 Subject: [PATCH 014/210] stage1: fix crash with slice type --- src/analyze.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/analyze.cpp b/src/analyze.cpp index b1d362f6e9..0e036087f3 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -825,6 +825,7 @@ ZigType *get_array_type(CodeGen *g, ZigType *child_type, uint64_t array_size, Zi } ZigType *get_slice_type(CodeGen *g, ZigType *ptr_type) { + Error err; assert(ptr_type->id == ZigTypeIdPointer); assert(ptr_type->data.pointer.ptr_len == PtrLenUnknown); @@ -833,6 +834,11 @@ ZigType *get_slice_type(CodeGen *g, ZigType *ptr_type) { return *parent_pointer; } + // We use the pointer type's abi size below, so we have to resolve it now. + if ((err = type_resolve(g, ptr_type, ResolveStatusSizeKnown))) { + codegen_report_errors_and_exit(g); + } + ZigType *entry = new_type_table_entry(ZigTypeIdStruct); buf_resize(&entry->name, 0); From 8cf40f3445b6548836be3db2a8ef5317af6545ac Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 10 Sep 2020 22:24:27 -0700 Subject: [PATCH 015/210] stage2: loading glibc metadata --- src-self-hosted/Module.zig | 13 ++ src-self-hosted/glibc.zig | 231 ++++++++++++++++++++++++++++++ src-self-hosted/main.zig | 2 +- src-self-hosted/print_targets.zig | 43 ++---- 4 files changed, 258 insertions(+), 31 deletions(-) create mode 100644 src-self-hosted/glibc.zig diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 482a1d9a49..715979bf11 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -1256,6 +1256,14 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { mod.c_object_table.putAssumeCapacityNoClobber(c_object, {}); } + // If we need to build glibc for the target, add work items for it. + if (mod.bin_file.options.link_libc and + mod.bin_file.options.libc_installation == null and + mod.bin_file.options.target.isGnuLibC()) + { + try mod.addBuildingGLibCWorkItems(); + } + return mod; } @@ -4495,3 +4503,8 @@ pub fn get_libc_crt_file(mod: *Module, arena: *Allocator, basename: []const u8) const full_path = try std.fs.path.join(arena, &[_][]const u8{ crt_dir_path, basename }); return full_path; } + +fn addBuildingGLibCWorkItems(mod: *Module) !void { + // crti.o, crtn.o, start.os, abi-note.o, Scrt1.o, libc_nonshared.a + try mod.work_queue.ensureUnusedCapacity(6); +} diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig new file mode 100644 index 0000000000..ea429a69a3 --- /dev/null +++ b/src-self-hosted/glibc.zig @@ -0,0 +1,231 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const target_util = @import("target.zig"); +const mem = std.mem; + +pub const Lib = struct { + name: []const u8, + sover: u8, +}; + +pub const Fn = struct { + name: []const u8, + lib: *const Lib, +}; + +pub const VerList = struct { + /// 7 is just the max number, we know statically it's big enough. + versions: [7]u8, + len: u8, +}; + +pub const ABI = struct { + all_versions: []const std.builtin.Version, + all_functions: []const Fn, + /// The value is a pointer to all_functions.len items and each item is an index into all_functions. + version_table: std.AutoHashMapUnmanaged(target_util.ArchOsAbi, [*]VerList), + arena_state: std.heap.ArenaAllocator.State, + + pub fn destroy(abi: *ABI, gpa: *Allocator) void { + abi.version_table.deinit(gpa); + abi.arena_state.promote(gpa).deinit(); // Frees the ABI memory too. + } +}; + +pub const libs = [_]Lib{ + .{ .name = "c", .sover = 6 }, + .{ .name = "m", .sover = 6 }, + .{ .name = "pthread", .sover = 0 }, + .{ .name = "dl", .sover = 2 }, + .{ .name = "rt", .sover = 1 }, + .{ .name = "ld", .sover = 2 }, + .{ .name = "util", .sover = 1 }, +}; + +pub const LoadMetaDataError = error{ + /// The files that ship with the Zig compiler were unable to be read, or otherwise had malformed data. + ZigInstallationCorrupt, + OutOfMemory, +}; + +/// This function will emit a log error when there is a problem with the zig installation and then return +/// `error.ZigInstallationCorrupt`. +pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!*ABI { + var arena_allocator = std.heap.ArenaAllocator.init(gpa); + errdefer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + var all_versions = std.ArrayListUnmanaged(std.builtin.Version){}; + var all_functions = std.ArrayListUnmanaged(Fn){}; + var version_table = std.AutoHashMapUnmanaged(target_util.ArchOsAbi, [*]VerList){}; + errdefer version_table.deinit(gpa); + + var glibc_dir = zig_lib_dir.openDir("libc" ++ std.fs.path.sep_str ++ "glibc", .{}) catch |err| { + std.log.err("unable to open glibc dir: {}", .{@errorName(err)}); + return error.ZigInstallationCorrupt; + }; + defer glibc_dir.close(); + + const max_txt_size = 500 * 1024; // Bigger than this and something is definitely borked. + const vers_txt_contents = glibc_dir.readFileAlloc(gpa, "vers.txt", max_txt_size) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => { + std.log.err("unable to read vers.txt: {}", .{@errorName(err)}); + return error.ZigInstallationCorrupt; + }, + }; + defer gpa.free(vers_txt_contents); + + const fns_txt_contents = glibc_dir.readFileAlloc(gpa, "fns.txt", max_txt_size) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => { + std.log.err("unable to read fns.txt: {}", .{@errorName(err)}); + return error.ZigInstallationCorrupt; + }, + }; + defer gpa.free(fns_txt_contents); + + const abi_txt_contents = glibc_dir.readFileAlloc(gpa, "abi.txt", max_txt_size) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => { + std.log.err("unable to read abi.txt: {}", .{@errorName(err)}); + return error.ZigInstallationCorrupt; + }, + }; + defer gpa.free(abi_txt_contents); + + { + var it = mem.tokenize(vers_txt_contents, "\r\n"); + var line_i: usize = 1; + while (it.next()) |line| : (line_i += 1) { + const prefix = "GLIBC_"; + if (!mem.startsWith(u8, line, prefix)) { + std.log.err("vers.txt:{}: expected 'GLIBC_' prefix", .{line_i}); + return error.ZigInstallationCorrupt; + } + const adjusted_line = line[prefix.len..]; + const ver = std.builtin.Version.parse(adjusted_line) catch |err| { + std.log.err("vers.txt:{}: unable to parse glibc version '{}': {}", .{ line_i, line, @errorName(err) }); + return error.ZigInstallationCorrupt; + }; + try all_versions.append(arena, ver); + } + } + { + var file_it = mem.tokenize(fns_txt_contents, "\r\n"); + var line_i: usize = 1; + while (file_it.next()) |line| : (line_i += 1) { + var line_it = mem.tokenize(line, " "); + const fn_name = line_it.next() orelse { + std.log.err("fns.txt:{}: expected function name", .{line_i}); + return error.ZigInstallationCorrupt; + }; + const lib_name = line_it.next() orelse { + std.log.err("fns.txt:{}: expected library name", .{line_i}); + return error.ZigInstallationCorrupt; + }; + const lib = findLib(lib_name) orelse { + std.log.err("fns.txt:{}: unknown library name: {}", .{ line_i, lib_name }); + return error.ZigInstallationCorrupt; + }; + try all_functions.append(arena, .{ + .name = fn_name, + .lib = lib, + }); + } + } + { + var file_it = mem.split(abi_txt_contents, "\n"); + var line_i: usize = 0; + while (true) { + const ver_list_base: []VerList = blk: { + const line = file_it.next() orelse break; + if (line.len == 0) break; + line_i += 1; + const ver_list_base = try arena.alloc(VerList, all_functions.items.len); + var line_it = mem.tokenize(line, " "); + while (line_it.next()) |target_string| { + var component_it = mem.tokenize(target_string, "-"); + const arch_name = component_it.next() orelse { + std.log.err("abi.txt:{}: expected arch name", .{line_i}); + return error.ZigInstallationCorrupt; + }; + const os_name = component_it.next() orelse { + std.log.err("abi.txt:{}: expected OS name", .{line_i}); + return error.ZigInstallationCorrupt; + }; + const abi_name = component_it.next() orelse { + std.log.err("abi.txt:{}: expected ABI name", .{line_i}); + return error.ZigInstallationCorrupt; + }; + const arch_tag = std.meta.stringToEnum(std.Target.Cpu.Arch, arch_name) orelse { + std.log.err("abi.txt:{}: unrecognized arch: '{}'", .{ line_i, arch_name }); + return error.ZigInstallationCorrupt; + }; + if (!mem.eql(u8, os_name, "linux")) { + std.log.err("abi.txt:{}: expected OS 'linux', found '{}'", .{ line_i, os_name }); + return error.ZigInstallationCorrupt; + } + const abi_tag = std.meta.stringToEnum(std.Target.Abi, abi_name) orelse { + std.log.err("abi.txt:{}: unrecognized ABI: '{}'", .{ line_i, abi_name }); + return error.ZigInstallationCorrupt; + }; + + const triple = target_util.ArchOsAbi{ + .arch = arch_tag, + .os = .linux, + .abi = abi_tag, + }; + try version_table.put(arena, triple, ver_list_base.ptr); + } + break :blk ver_list_base; + }; + for (ver_list_base) |*ver_list| { + const line = file_it.next() orelse { + std.log.err("abi.txt:{}: missing version number line", .{line_i}); + return error.ZigInstallationCorrupt; + }; + line_i += 1; + + ver_list.* = .{ + .versions = undefined, + .len = 0, + }; + var line_it = mem.tokenize(line, " "); + while (line_it.next()) |version_index_string| { + if (ver_list.len >= ver_list.versions.len) { + // If this happens with legit data, increase the array len in the type. + std.log.err("abi.txt:{}: too many versions", .{line_i}); + return error.ZigInstallationCorrupt; + } + const version_index = std.fmt.parseInt(u8, version_index_string, 10) catch |err| { + // If this happens with legit data, increase the size of the integer type in the struct. + std.log.err("abi.txt:{}: unable to parse version: {}", .{ line_i, @errorName(err) }); + return error.ZigInstallationCorrupt; + }; + + ver_list.versions[ver_list.len] = version_index; + ver_list.len += 1; + } + } + } + } + + const abi = try arena.create(ABI); + abi.* = .{ + .all_versions = all_versions.items, + .all_functions = all_functions.items, + .version_table = version_table, + .arena_state = arena_allocator.state, + }; + return abi; +} + +fn findLib(name: []const u8) ?*const Lib { + for (libs) |*lib| { + if (mem.eql(u8, lib.name, name)) { + return lib; + } + } + return null; +} diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 1de025769d..bc3e5a8e6c 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -16,7 +16,7 @@ const warn = std.log.warn; const introspect = @import("introspect.zig"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; -fn fatal(comptime format: []const u8, args: anytype) noreturn { +pub fn fatal(comptime format: []const u8, args: anytype) noreturn { std.log.emerg(format, args); process.exit(1); } diff --git a/src-self-hosted/print_targets.zig b/src-self-hosted/print_targets.zig index 33a513efd7..e6ed33b80c 100644 --- a/src-self-hosted/print_targets.zig +++ b/src-self-hosted/print_targets.zig @@ -6,8 +6,9 @@ const Allocator = mem.Allocator; const Target = std.Target; const target = @import("target.zig"); const assert = std.debug.assert; - +const glibc = @import("glibc.zig"); const introspect = @import("introspect.zig"); +const fatal = @import("main.zig").fatal; pub fn cmdTargets( allocator: *Allocator, @@ -16,33 +17,16 @@ pub fn cmdTargets( stdout: anytype, native_target: Target, ) !void { - const available_glibcs = blk: { - const zig_lib_dir = introspect.resolveZigLibDir(allocator) catch |err| { - std.debug.print("unable to find zig installation directory: {}\n", .{@errorName(err)}); - std.process.exit(1); - }; - defer allocator.free(zig_lib_dir); - - var dir = try std.fs.cwd().openDir(zig_lib_dir, .{}); - defer dir.close(); - - const vers_txt = try dir.readFileAlloc(allocator, "libc" ++ std.fs.path.sep_str ++ "glibc" ++ std.fs.path.sep_str ++ "vers.txt", 10 * 1024); - defer allocator.free(vers_txt); - - var list = std.ArrayList(std.builtin.Version).init(allocator); - defer list.deinit(); - - var it = mem.tokenize(vers_txt, "\r\n"); - while (it.next()) |line| { - const prefix = "GLIBC_"; - assert(mem.startsWith(u8, line, prefix)); - const adjusted_line = line[prefix.len..]; - const ver = try std.builtin.Version.parse(adjusted_line); - try list.append(ver); - } - break :blk list.toOwnedSlice(); + const zig_lib_dir_path = introspect.resolveZigLibDir(allocator) catch |err| { + fatal("unable to find zig installation directory: {}\n", .{@errorName(err)}); }; - defer allocator.free(available_glibcs); + defer allocator.free(zig_lib_dir_path); + + var zig_lib_dir = try fs.cwd().openDir(zig_lib_dir_path, .{}); + defer zig_lib_dir.close(); + + const glibc_abi = try glibc.loadMetaData(allocator, zig_lib_dir); + errdefer glibc_abi.destroy(allocator); var bos = io.bufferedOutStream(stdout); const bos_stream = bos.outStream(); @@ -90,10 +74,10 @@ pub fn cmdTargets( try jws.objectField("glibc"); try jws.beginArray(); - for (available_glibcs) |glibc| { + for (glibc_abi.all_versions) |ver| { try jws.arrayElem(); - const tmp = try std.fmt.allocPrint(allocator, "{}", .{glibc}); + const tmp = try std.fmt.allocPrint(allocator, "{}", .{ver}); defer allocator.free(tmp); try jws.emitString(tmp); } @@ -170,7 +154,6 @@ pub fn cmdTargets( try jws.emitString(@tagName(native_target.os.tag)); try jws.objectField("abi"); try jws.emitString(@tagName(native_target.abi)); - // TODO implement native glibc version detection in self-hosted try jws.endObject(); try jws.endObject(); From c903dad915cd21697692b00c69c8424438d0da15 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 12 Sep 2020 00:50:17 -0700 Subject: [PATCH 016/210] fix zig_llvm.h not complying with C --- src/zig_llvm.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zig_llvm.h b/src/zig_llvm.h index f07684f2a4..13c4de0535 100644 --- a/src/zig_llvm.h +++ b/src/zig_llvm.h @@ -59,8 +59,8 @@ enum ZigLLVMABIType { }; ZIG_EXTERN_C LLVMTargetMachineRef ZigLLVMCreateTargetMachine(LLVMTargetRef T, const char *Triple, - const char *CPU, const char *Features, LLVMCodeGenOptLevel Level, LLVMRelocMode Reloc, - LLVMCodeModel CodeModel, bool function_sections, ZigLLVMABIType float_abi, const char *abi_name); + const char *CPU, const char *Features, enum LLVMCodeGenOptLevel Level, enum LLVMRelocMode Reloc, + enum LLVMCodeModel CodeModel, bool function_sections, enum ZigLLVMABIType float_abi, const char *abi_name); ZIG_EXTERN_C LLVMTypeRef ZigLLVMTokenTypeInContext(LLVMContextRef context_ref); From 8374be1a1c6cba12ec01d7a6301a8da0a00907f4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 12 Sep 2020 00:50:37 -0700 Subject: [PATCH 017/210] std.cache_hash: bump up the base64 digest len to avoid '==' --- lib/std/cache_hash.zig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/std/cache_hash.zig b/lib/std/cache_hash.zig index f98ec8d7f1..4727169407 100644 --- a/lib/std/cache_hash.zig +++ b/lib/std/cache_hash.zig @@ -16,8 +16,9 @@ const Allocator = std.mem.Allocator; const base64_encoder = fs.base64_encoder; const base64_decoder = fs.base64_decoder; -/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 -const BIN_DIGEST_LEN = 16; +/// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 +/// We round up to 18 to avoid the `==` padding after base64 encoding. +const BIN_DIGEST_LEN = 18; const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN); const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024; From 03a23418ff13e6ff64cdeed3ef4b54f99c533d88 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 12 Sep 2020 00:51:06 -0700 Subject: [PATCH 018/210] stage2: linking with LLD and building glibc static CRT files * implement --debug-cc and --debug-link * implement C source files having extra flags - TODO a way to pass them on the CLI * introduce the Directory abstraction which contains both an open file descriptor and a file path name. The former is preferred but the latter is needed when communicating paths over a command line (e.g. to Clang or LLD). * use the cache hash to choose an artifact directory - TODO: use separate cache hash instances for the zig module and each C object * Module: introduce the crt_files table for keeping track of built libc artifacts for linking. * Add the ability to build 4/6 of the glibc static CRT lib files. * The zig-cache directory is now passed as a parameter to Module. * Implement the CLI logic of -femit-bin and -femit-h - TODO: respect -fno-emit-bin - TODO: the emit .h feature * Add the -fvalgrind, -fstack-check, and --single-threaded CLI options. * Implement the logic for auto detecting whether to enable PIC, sanitize-C, stack-check, valgrind, and single-threaded. * Properly add PIC args (or not) to clang argv. * Implement renaming clang-compiled object files into their proper place within the cache artifact directory. - TODO: std lib needs a proper higher level abstraction for std.os.renameat. * Package is cleaned up to use the "Unmanaged" StringHashMap and use the new Directory abstraction. * Clean up zig lib directory detection to make proper use of directory handles. * Linker code invokes LLD. - TODO properly deal with the stdout and stderr that we get from it and expose diagnostics from the Module API that match the expected error message format. * Delete the bitrotted LLVM C ABI bindings. We'll resurrect just the functions we need as we introduce dependencies on them. So far it only has ZigLLDLink in it. * Remove dead timer code. * `zig env` now prints the path to the zig executable as well. --- src-self-hosted/Module.zig | 366 ++++++++++++++++++--------- src-self-hosted/Package.zig | 58 ++--- src-self-hosted/glibc.zig | 396 +++++++++++++++++++++++++++++- src-self-hosted/introspect.zig | 98 ++++---- src-self-hosted/link.zig | 15 +- src-self-hosted/link/C.zig | 2 +- src-self-hosted/link/Coff.zig | 2 +- src-self-hosted/link/Elf.zig | 65 ++++- src-self-hosted/link/MachO.zig | 2 +- src-self-hosted/link/Wasm.zig | 2 +- src-self-hosted/llvm.zig | 307 ++--------------------- src-self-hosted/main.zig | 154 +++++++++--- src-self-hosted/print_env.zig | 21 +- src-self-hosted/print_targets.zig | 12 +- src-self-hosted/target.zig | 40 ++- src-self-hosted/test.zig | 33 ++- 16 files changed, 1003 insertions(+), 570 deletions(-) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 715979bf11..4c48837d9e 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -25,6 +25,8 @@ const astgen = @import("astgen.zig"); const zir_sema = @import("zir_sema.zig"); const build_options = @import("build_options"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; +const glibc = @import("glibc.zig"); +const fatal = @import("main.zig").fatal; /// General-purpose allocator. Used for both temporary and long-term storage. gpa: *Allocator, @@ -91,17 +93,20 @@ sanitize_c: bool, /// Otherwise we attempt to parse the error messages and expose them via the Module API. /// This is `true` for `zig cc`, `zig c++`, and `zig translate-c`. clang_passthrough_mode: bool, +/// Whether to print clang argvs to stdout. +debug_cc: bool, /// Error tags and their values, tag names are duped with mod.gpa. global_error_set: std.StringHashMapUnmanaged(u16) = .{}, -c_source_files: []const []const u8, +c_source_files: []const CSourceFile, clang_argv: []const []const u8, cache: std.cache_hash.CacheHash, /// Path to own executable for invoking `zig clang`. self_exe_path: ?[]const u8, -zig_lib_dir: []const u8, -zig_cache_dir_path: []const u8, +zig_lib_directory: Directory, +zig_cache_directory: Directory, +zig_cache_artifact_directory: Directory, libc_include_dir_list: []const []const u8, rand: *std.rand.Random, @@ -118,8 +123,19 @@ libunwind_static_lib: ?[]const u8 = null, /// and resolved before calling linker.flush(). libc_static_lib: ?[]const u8 = null, +/// For example `Scrt1.o` and `libc.so.6`. These are populated after building libc from source, +/// The set of needed CRT (C runtime) files differs depending on the target and compilation settings. +/// The key is the basename, and the value is the absolute path to the completed build artifact. +crt_files: std.StringHashMapUnmanaged([]const u8) = .{}, + pub const InnerError = error{ OutOfMemory, AnalysisFail }; +/// For passing to a C compiler. +pub const CSourceFile = struct { + src_path: []const u8, + extra_flags: []const []const u8 = &[0][]const u8{}, +}; + const WorkItem = union(enum) { /// Write the machine code for a Decl to the output file. codegen_decl: *Decl, @@ -133,6 +149,11 @@ const WorkItem = union(enum) { /// Invoke the Clang compiler to create an object file, which gets linked /// with the Module. c_object: *CObject, + + /// one of the glibc static objects + glibc_crt_file: glibc.CRTFile, + /// one of the glibc shared objects + glibc_so: *const glibc.Lib, }; pub const Export = struct { @@ -701,7 +722,7 @@ pub const Scope = struct { pub fn getSource(self: *File, module: *Module) ![:0]const u8 { switch (self.source) { .unloaded => { - const source = try module.root_pkg.?.root_src_dir.readFileAllocOptions( + const source = try module.root_pkg.?.root_src_directory.handle.readFileAllocOptions( module.gpa, self.sub_file_path, std.math.maxInt(u32), @@ -805,7 +826,7 @@ pub const Scope = struct { pub fn getSource(self: *ZIRModule, module: *Module) ![:0]const u8 { switch (self.source) { .unloaded => { - const source = try module.root_pkg.?.root_src_dir.readFileAllocOptions( + const source = try module.root_pkg.?.root_src_directory.handle.readFileAllocOptions( module.gpa, self.sub_file_path, std.math.maxInt(u32), @@ -937,18 +958,35 @@ pub const AllErrors = struct { } }; +pub const Directory = struct { + /// This field is redundant for operations that can act on the open directory handle + /// directly, but it is needed when passing the directory to a child process. + /// `null` means cwd. + path: ?[]const u8, + handle: std.fs.Dir, +}; + +pub const EmitLoc = struct { + /// If this is `null` it means the file will be output to the cache directory. + /// When provided, both the open file handle and the path name must outlive the `Module`. + directory: ?Module.Directory, + /// This may not have sub-directories in it. + basename: []const u8, +}; + pub const InitOptions = struct { - zig_lib_dir: []const u8, + zig_lib_directory: Directory, + zig_cache_directory: Directory, target: Target, root_name: []const u8, root_pkg: ?*Package, output_mode: std.builtin.OutputMode, rand: *std.rand.Random, dynamic_linker: ?[]const u8 = null, - bin_file_dir_path: ?[]const u8 = null, - bin_file_dir: ?std.fs.Dir = null, - bin_file_path: []const u8, - emit_h: ?[]const u8 = null, + /// `null` means to not emit a binary file. + emit_bin: ?EmitLoc, + /// `null` means to not emit a C header file. + emit_h: ?EmitLoc = null, link_mode: ?std.builtin.LinkMode = null, object_format: ?std.builtin.ObjectFormat = null, optimize_mode: std.builtin.Mode = .Debug, @@ -957,7 +995,7 @@ pub const InitOptions = struct { lld_argv: []const []const u8 = &[0][]const u8{}, lib_dirs: []const []const u8 = &[0][]const u8{}, rpath_list: []const []const u8 = &[0][]const u8{}, - c_source_files: []const []const u8 = &[0][]const u8{}, + c_source_files: []const CSourceFile = &[0]CSourceFile{}, link_objects: []const []const u8 = &[0][]const u8{}, framework_dirs: []const []const u8 = &[0][]const u8{}, frameworks: []const []const u8 = &[0][]const u8{}, @@ -966,11 +1004,14 @@ pub const InitOptions = struct { link_libcpp: bool = false, want_pic: ?bool = null, want_sanitize_c: ?bool = null, + want_stack_check: ?bool = null, + want_valgrind: ?bool = null, use_llvm: ?bool = null, use_lld: ?bool = null, use_clang: ?bool = null, rdynamic: bool = false, strip: bool = false, + single_threaded: bool = false, is_native_os: bool, link_eh_frame_hdr: bool = false, linker_script: ?[]const u8 = null, @@ -984,6 +1025,8 @@ pub const InitOptions = struct { linker_z_nodelete: bool = false, linker_z_defs: bool = false, clang_passthrough_mode: bool = false, + debug_cc: bool = false, + debug_link: bool = false, stack_size_override: ?u64 = null, self_exe_path: ?[]const u8 = null, version: std.builtin.Version = .{ .major = 0, .minor = 0, .patch = 0 }, @@ -1057,17 +1100,126 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { const libc_dirs = try detectLibCIncludeDirs( arena, - options.zig_lib_dir, + options.zig_lib_directory.path.?, options.target, options.is_native_os, options.link_libc, options.libc_installation, ); + const must_pic: bool = b: { + if (target_util.requiresPIC(options.target, options.link_libc)) + break :b true; + break :b link_mode == .Dynamic; + }; + const pic = options.want_pic orelse must_pic; + + if (options.emit_h != null) fatal("-femit-h not supported yet", .{}); // TODO + + const emit_bin = options.emit_bin orelse fatal("-fno-emit-bin not supported yet", .{}); // TODO + + // Make a decision on whether to use Clang for translate-c and compiling C files. + const use_clang = if (options.use_clang) |explicit| explicit else blk: { + if (build_options.have_llvm) { + // Can't use it if we don't have it! + break :blk false; + } + // It's not planned to do our own translate-c or C compilation. + break :blk true; + }; + + const is_safe_mode = switch (options.optimize_mode) { + .Debug, .ReleaseSafe => true, + .ReleaseFast, .ReleaseSmall => false, + }; + + const sanitize_c = options.want_sanitize_c orelse is_safe_mode; + + const stack_check: bool = b: { + if (!target_util.supportsStackProbing(options.target)) + break :b false; + break :b options.want_stack_check orelse is_safe_mode; + }; + + const valgrind: bool = b: { + if (!target_util.hasValgrindSupport(options.target)) + break :b false; + break :b options.want_valgrind orelse (options.optimize_mode == .Debug); + }; + + const single_threaded = options.single_threaded or target_util.isSingleThreaded(options.target); + + // We put everything into the cache hash that *cannot be modified during an incremental update*. + // For example, one cannot change the target between updates, but one can change source files, + // so the target goes into the cache hash, but source files do not. This is so that we can + // find the same binary and incrementally update it even if there are modified source files. + // We do this even if outputting to the current directory because (1) this cache_hash instance + // will be the "parent" of other cache_hash instances such as for C objects, (2) we need + // a place for intermediate build artifacts, such as a .o file to be linked with LLD, and (3) + // we need somewhere to store serialization of incremental compilation metadata. + var cache = try std.cache_hash.CacheHash.init(gpa, options.zig_cache_directory.handle, "h"); + errdefer cache.release(); + + // Now we will prepare hash state initializations to avoid redundantly computing hashes. + // First we add common things between things that apply to zig source and all c source files. + cache.addBytes(build_options.version); + cache.add(options.optimize_mode); + cache.add(options.target.cpu.arch); + cache.addBytes(options.target.cpu.model.name); + cache.add(options.target.cpu.features.ints); + cache.add(options.target.os.tag); + switch (options.target.os.tag) { + .linux => { + cache.add(options.target.os.version_range.linux.range.min); + cache.add(options.target.os.version_range.linux.range.max); + cache.add(options.target.os.version_range.linux.glibc); + }, + .windows => { + cache.add(options.target.os.version_range.windows.min); + cache.add(options.target.os.version_range.windows.max); + }, + .freebsd, + .macosx, + .ios, + .tvos, + .watchos, + .netbsd, + .openbsd, + .dragonfly, + => { + cache.add(options.target.os.version_range.semver.min); + cache.add(options.target.os.version_range.semver.max); + }, + else => {}, + } + cache.add(options.target.abi); + cache.add(ofmt); + cache.add(pic); + cache.add(stack_check); + cache.add(sanitize_c); + cache.add(valgrind); + cache.add(link_mode); + cache.add(options.strip); + cache.add(single_threaded); + // TODO audit this and make sure everything is in it + + // We don't care whether we find something there, just show us the digest. + const digest = (try cache.hit()) orelse cache.final(); + + const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); + var artifact_dir = try options.zig_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); + errdefer artifact_dir.close(); + const zig_cache_artifact_directory: Directory = .{ + .handle = artifact_dir, + .path = if (options.zig_cache_directory.path) |p| + try std.fs.path.join(arena, &[_][]const u8{ p, artifact_sub_dir }) + else + artifact_sub_dir, + }; + const bin_file = try link.File.openPath(gpa, .{ - .dir = options.bin_file_dir orelse std.fs.cwd(), - .dir_path = options.bin_file_dir_path, - .sub_path = options.bin_file_path, + .directory = emit_bin.directory orelse zig_cache_artifact_directory, + .sub_path = emit_bin.basename, .root_name = root_name, .root_pkg = options.root_pkg, .target = options.target, @@ -1103,6 +1255,11 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { .override_soname = options.override_soname, .version = options.version, .libc_installation = libc_dirs.libc_installation, + .pic = pic, + .valgrind = valgrind, + .stack_check = stack_check, + .single_threaded = single_threaded, + .debug_link = options.debug_link, }); errdefer bin_file.destroy(); @@ -1142,83 +1299,12 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { } }; - // We put everything into the cache hash except for the root source file, because we want to - // find the same binary and incrementally update it even if the file contents changed. - // TODO Look into storing this information in memory rather than on disk and solving - // serialization/deserialization of *all* incremental compilation state in a more generic way. - const cache_parent_dir = if (options.root_pkg) |root_pkg| root_pkg.root_src_dir else std.fs.cwd(); - var cache_dir = try cache_parent_dir.makeOpenPath("zig-cache", .{}); - defer cache_dir.close(); - - try cache_dir.makePath("tmp"); - try cache_dir.makePath("o"); - // We need this string because of sending paths to clang as a child process. - const zig_cache_dir_path = if (options.root_pkg) |root_pkg| - try std.fmt.allocPrint(arena, "{}" ++ std.fs.path.sep_str ++ "zig-cache", .{root_pkg.root_src_dir_path}) - else - "zig-cache"; - - var cache = try std.cache_hash.CacheHash.init(gpa, cache_dir, "h"); - errdefer cache.release(); - - // Now we will prepare hash state initializations to avoid redundantly computing hashes. - // First we add common things between things that apply to zig source and all c source files. - cache.addBytes(build_options.version); - cache.add(options.optimize_mode); - cache.add(options.target.cpu.arch); - cache.addBytes(options.target.cpu.model.name); - cache.add(options.target.cpu.features.ints); - cache.add(options.target.os.tag); - switch (options.target.os.tag) { - .linux => { - cache.add(options.target.os.version_range.linux.range.min); - cache.add(options.target.os.version_range.linux.range.max); - cache.add(options.target.os.version_range.linux.glibc); - }, - .windows => { - cache.add(options.target.os.version_range.windows.min); - cache.add(options.target.os.version_range.windows.max); - }, - .freebsd, - .macosx, - .ios, - .tvos, - .watchos, - .netbsd, - .openbsd, - .dragonfly, - => { - cache.add(options.target.os.version_range.semver.min); - cache.add(options.target.os.version_range.semver.max); - }, - else => {}, - } - cache.add(options.target.abi); - cache.add(ofmt); - // TODO PIC (see detect_pic from codegen.cpp) - cache.add(bin_file.options.link_mode); - cache.add(options.strip); - - // Make a decision on whether to use Clang for translate-c and compiling C files. - const use_clang = if (options.use_clang) |explicit| explicit else blk: { - if (build_options.have_llvm) { - // Can't use it if we don't have it! - break :blk false; - } - // It's not planned to do our own translate-c or C compilation. - break :blk true; - }; - - const sanitize_c: bool = options.want_sanitize_c orelse switch (options.optimize_mode) { - .Debug, .ReleaseSafe => true, - .ReleaseSmall, .ReleaseFast => false, - }; - mod.* = .{ .gpa = gpa, .arena_state = arena_allocator.state, - .zig_lib_dir = options.zig_lib_dir, - .zig_cache_dir_path = zig_cache_dir_path, + .zig_lib_directory = options.zig_lib_directory, + .zig_cache_directory = options.zig_cache_directory, + .zig_cache_artifact_directory = zig_cache_artifact_directory, .root_pkg = options.root_pkg, .root_scope = root_scope, .bin_file = bin_file, @@ -1233,6 +1319,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { .sanitize_c = sanitize_c, .rand = options.rand, .clang_passthrough_mode = options.clang_passthrough_mode, + .debug_cc = options.debug_cc, }; break :mod mod; }; @@ -1245,22 +1332,21 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { errdefer local_arena.deinit(); const c_object = try local_arena.allocator.create(CObject); - const src_path = try local_arena.allocator.dupe(u8, c_source_file); c_object.* = .{ .status = .{ .new = {} }, - .src_path = src_path, - .extra_flags = &[0][]const u8{}, + // TODO why are we duplicating this memory? do we need to? + // look into refactoring to turn these 2 fields simply into a CSourceFile + .src_path = try local_arena.allocator.dupe(u8, c_source_file.src_path), + .extra_flags = try local_arena.allocator.dupe([]const u8, c_source_file.extra_flags), .arena = local_arena.state, }; mod.c_object_table.putAssumeCapacityNoClobber(c_object, {}); } // If we need to build glibc for the target, add work items for it. - if (mod.bin_file.options.link_libc and - mod.bin_file.options.libc_installation == null and - mod.bin_file.options.target.isGnuLibC()) - { + // We go through the work queue so that building can be done in parallel. + if (mod.wantBuildGLibCFromSource()) { try mod.addBuildingGLibCWorkItems(); } @@ -1273,6 +1359,15 @@ pub fn destroy(self: *Module) void { self.deletion_set.deinit(gpa); self.work_queue.deinit(); + { + var it = self.crt_files.iterator(); + while (it.next()) |entry| { + gpa.free(entry.key); + gpa.free(entry.value); + } + self.crt_files.deinit(gpa); + } + for (self.decl_table.items()) |entry| { entry.value.destroy(gpa); } @@ -1322,6 +1417,8 @@ pub fn destroy(self: *Module) void { gpa.free(entry.key); } self.global_error_set.deinit(gpa); + + self.zig_cache_artifact_directory.handle.close(); self.cache.release(); // This destroys `self`. @@ -1458,7 +1555,7 @@ pub fn getAllErrorsAlloc(self: *Module) !AllErrors { if (errors.items.len == 0 and self.link_error_flags.no_entry_point_found) { const global_err_src_path = blk: { if (self.root_pkg) |root_pkg| break :blk root_pkg.root_src_path; - if (self.c_source_files.len != 0) break :blk self.c_source_files[0]; + if (self.c_source_files.len != 0) break :blk self.c_source_files[0].src_path; if (self.bin_file.options.objects.len != 0) break :blk self.bin_file.options.objects[0]; break :blk "(no file)"; }; @@ -1581,6 +1678,17 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void { }, }; }, + .glibc_crt_file => |crt_file| { + glibc.buildCRTFile(self, crt_file) catch |err| { + // This is a problem with the Zig installation. It's mostly OK to crash here, + // but TODO because it would be even better if we could recover gracefully + // from temporary problems such as out-of-disk-space. + fatal("unable to build glibc CRT file: {}", .{@errorName(err)}); + }; + }, + .glibc_so => |glibc_lib| { + fatal("TODO build glibc shared object '{}.so.{}'", .{ glibc_lib.name, glibc_lib.sover }); + }, }; } @@ -1588,6 +1696,8 @@ fn buildCObject(mod: *Module, c_object: *CObject) !void { const tracy = trace(@src()); defer tracy.end(); + // TODO this C source file needs its own cache hash instance + if (!build_options.have_llvm) { return mod.failCObj(c_object, "clang not available: compiler not built with LLVM extensions enabled", .{}); } @@ -1616,6 +1726,9 @@ fn buildCObject(mod: *Module, c_object: *CObject) !void { // We can't know the digest until we do the C compiler invocation, so we need a temporary filename. const out_obj_path = try mod.tmpFilePath(arena, o_basename); + var zig_cache_tmp_dir = try mod.zig_cache_directory.handle.makeOpenPath("tmp", .{}); + defer zig_cache_tmp_dir.close(); + try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang", "-c" }); const ext = classifyFileExt(c_object.src_path); @@ -1628,9 +1741,12 @@ fn buildCObject(mod: *Module, c_object: *CObject) !void { try argv.append(c_object.src_path); try argv.appendSlice(c_object.extra_flags); - //for (argv.items) |arg| { - // std.debug.print("{} ", .{arg}); - //} + if (mod.debug_cc) { + for (argv.items[0 .. argv.items.len - 1]) |arg| { + std.debug.print("{} ", .{arg}); + } + std.debug.print("{}\n", .{argv.items[argv.items.len - 1]}); + } const child = try std.ChildProcess.init(argv.items, arena); defer child.deinit(); @@ -1689,11 +1805,13 @@ fn buildCObject(mod: *Module, c_object: *CObject) !void { // TODO handle .d files - // TODO rename into place - std.debug.print("TODO rename {} into cache dir\n", .{out_obj_path}); + // TODO Add renameat capabilities to the std lib in a higher layer than the posix layer. + const tmp_basename = std.fs.path.basename(out_obj_path); + try std.os.renameat(zig_cache_tmp_dir.fd, tmp_basename, mod.zig_cache_artifact_directory.handle.fd, o_basename); - // TODO use the cache file name instead of tmp file name - const success_file_path = try mod.gpa.dupe(u8, out_obj_path); + const success_file_path = try std.fs.path.join(mod.gpa, &[_][]const u8{ + mod.zig_cache_artifact_directory.path.?, o_basename, + }); c_object.status = .{ .success = success_file_path }; } @@ -1702,7 +1820,7 @@ fn tmpFilePath(mod: *Module, arena: *Allocator, suffix: []const u8) error{OutOfM return std.fmt.allocPrint( arena, "{}" ++ s ++ "tmp" ++ s ++ "{x}-{}", - .{ mod.zig_cache_dir_path, mod.rand.int(u64), suffix }, + .{ mod.zig_cache_directory.path.?, mod.rand.int(u64), suffix }, ); } @@ -1749,10 +1867,10 @@ fn addCCArgs( if (mod.bin_file.options.link_libcpp) { const libcxx_include_path = try std.fs.path.join(arena, &[_][]const u8{ - mod.zig_lib_dir, "libcxx", "include", + mod.zig_lib_directory.path.?, "libcxx", "include", }); const libcxxabi_include_path = try std.fs.path.join(arena, &[_][]const u8{ - mod.zig_lib_dir, "libcxxabi", "include", + mod.zig_lib_directory.path.?, "libcxxabi", "include", }); try argv.append("-isystem"); @@ -1776,7 +1894,7 @@ fn addCCArgs( // According to Rich Felker libc headers are supposed to go before C language headers. // However as noted by @dimenus, appending libc headers before c_headers breaks intrinsics // and other compiler specific items. - const c_headers_dir = try std.fs.path.join(arena, &[_][]const u8{ mod.zig_lib_dir, "include" }); + const c_headers_dir = try std.fs.path.join(arena, &[_][]const u8{ mod.zig_lib_directory.path.?, "include" }); try argv.append("-isystem"); try argv.append(c_headers_dir); @@ -1891,10 +2009,9 @@ fn addCCArgs( }, } - // TODO add CLI args for PIC - //if (target_supports_fpic(g->zig_target) and g->have_pic) { - // try argv.append("-fPIC"); - //} + if (target_util.supports_fpic(target) and mod.bin_file.options.pic) { + try argv.append("-fPIC"); + } try argv.appendSlice(mod.clang_argv); } @@ -4497,7 +4614,9 @@ fn detectLibCFromLibCInstallation(arena: *Allocator, target: Target, lci: *const } pub fn get_libc_crt_file(mod: *Module, arena: *Allocator, basename: []const u8) ![]const u8 { - // TODO port support for building crt files from stage1 + if (mod.wantBuildGLibCFromSource()) { + return mod.crt_files.get(basename).?; + } const lci = mod.bin_file.options.libc_installation orelse return error.LibCInstallationNotAvailable; const crt_dir_path = lci.crt_dir orelse return error.LibCInstallationMissingCRTDir; const full_path = try std.fs.path.join(arena, &[_][]const u8{ crt_dir_path, basename }); @@ -4505,6 +4624,23 @@ pub fn get_libc_crt_file(mod: *Module, arena: *Allocator, basename: []const u8) } fn addBuildingGLibCWorkItems(mod: *Module) !void { - // crti.o, crtn.o, start.os, abi-note.o, Scrt1.o, libc_nonshared.a - try mod.work_queue.ensureUnusedCapacity(6); + const static_file_work_items = [_]WorkItem{ + .{ .glibc_crt_file = .crti_o }, + .{ .glibc_crt_file = .crtn_o }, + .{ .glibc_crt_file = .start_os }, + .{ .glibc_crt_file = .abi_note_o }, + .{ .glibc_crt_file = .scrt1_o }, + .{ .glibc_crt_file = .libc_nonshared_a }, + }; + try mod.work_queue.ensureUnusedCapacity(static_file_work_items.len + glibc.libs.len); + mod.work_queue.writeAssumeCapacity(&static_file_work_items); + for (glibc.libs) |*glibc_so| { + mod.work_queue.writeItemAssumeCapacity(.{ .glibc_so = glibc_so }); + } +} + +fn wantBuildGLibCFromSource(mod: *Module) bool { + return mod.bin_file.options.link_libc and + mod.bin_file.options.libc_installation == null and + mod.bin_file.options.target.isGnuLibC(); } diff --git a/src-self-hosted/Package.zig b/src-self-hosted/Package.zig index 4bf4defbc8..11b1436293 100644 --- a/src-self-hosted/Package.zig +++ b/src-self-hosted/Package.zig @@ -1,59 +1,59 @@ -pub const Table = std.StringHashMap(*Package); +pub const Table = std.StringHashMapUnmanaged(*Package); -/// This should be used for file operations. -root_src_dir: std.fs.Dir, -/// This is for metadata purposes, for example putting into debug information. -root_src_dir_path: []u8, -/// Relative to `root_src_dir` and `root_src_dir_path`. +root_src_directory: Module.Directory, +/// Relative to `root_src_directory`. root_src_path: []u8, table: Table, /// No references to `root_src_dir` and `root_src_path` are kept. pub fn create( - allocator: *mem.Allocator, + gpa: *Allocator, base_dir: std.fs.Dir, /// Relative to `base_dir`. root_src_dir: []const u8, /// Relative to `root_src_dir`. root_src_path: []const u8, ) !*Package { - const ptr = try allocator.create(Package); - errdefer allocator.destroy(ptr); - const root_src_path_dupe = try mem.dupe(allocator, u8, root_src_path); - errdefer allocator.free(root_src_path_dupe); - const root_src_dir_path = try mem.dupe(allocator, u8, root_src_dir); - errdefer allocator.free(root_src_dir_path); + const ptr = try gpa.create(Package); + errdefer gpa.destroy(ptr); + const root_src_path_dupe = try mem.dupe(gpa, u8, root_src_path); + errdefer gpa.free(root_src_path_dupe); + const root_src_dir_path = try mem.dupe(gpa, u8, root_src_dir); + errdefer gpa.free(root_src_dir_path); ptr.* = .{ - .root_src_dir = try base_dir.openDir(root_src_dir, .{}), - .root_src_dir_path = root_src_dir_path, + .root_src_directory = .{ + .path = root_src_dir_path, + .handle = try base_dir.openDir(root_src_dir, .{}), + }, .root_src_path = root_src_path_dupe, - .table = Table.init(allocator), + .table = .{}, }; return ptr; } -pub fn destroy(self: *Package) void { - const allocator = self.table.allocator; - self.root_src_dir.close(); - allocator.free(self.root_src_path); - allocator.free(self.root_src_dir_path); +pub fn destroy(pkg: *Package, gpa: *Allocator) void { + pkg.root_src_directory.handle.close(); + gpa.free(pkg.root_src_path); + if (pkg.root_src_directory.path) |p| gpa.free(p); { - var it = self.table.iterator(); + var it = pkg.table.iterator(); while (it.next()) |kv| { - allocator.free(kv.key); + gpa.free(kv.key); } } - self.table.deinit(); - allocator.destroy(self); + pkg.table.deinit(gpa); + gpa.destroy(pkg); } -pub fn add(self: *Package, name: []const u8, package: *Package) !void { - try self.table.ensureCapacity(self.table.items().len + 1); - const name_dupe = try mem.dupe(self.table.allocator, u8, name); - self.table.putAssumeCapacityNoClobber(name_dupe, package); +pub fn add(pkg: *Package, gpa: *Allocator, name: []const u8, package: *Package) !void { + try pkg.table.ensureCapacity(gpa, pkg.table.items().len + 1); + const name_dupe = try mem.dupe(gpa, u8, name); + pkg.table.putAssumeCapacityNoClobber(name_dupe, package); } const std = @import("std"); const mem = std.mem; +const Allocator = std.mem.Allocator; const assert = std.debug.assert; const Package = @This(); +const Module = @import("Module.zig"); diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index ea429a69a3..9d0a0d45fd 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -2,6 +2,9 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const target_util = @import("target.zig"); const mem = std.mem; +const Module = @import("Module.zig"); +const path = std.fs.path; +const build_options = @import("build_options"); pub const Lib = struct { name: []const u8, @@ -60,7 +63,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError! var version_table = std.AutoHashMapUnmanaged(target_util.ArchOsAbi, [*]VerList){}; errdefer version_table.deinit(gpa); - var glibc_dir = zig_lib_dir.openDir("libc" ++ std.fs.path.sep_str ++ "glibc", .{}) catch |err| { + var glibc_dir = zig_lib_dir.openDir("libc" ++ path.sep_str ++ "glibc", .{}) catch |err| { std.log.err("unable to open glibc dir: {}", .{@errorName(err)}); return error.ZigInstallationCorrupt; }; @@ -229,3 +232,394 @@ fn findLib(name: []const u8) ?*const Lib { } return null; } + +pub const CRTFile = enum { + crti_o, + crtn_o, + start_os, + abi_note_o, + scrt1_o, + libc_nonshared_a, +}; + +pub fn buildCRTFile(mod: *Module, crt_file: CRTFile) !void { + if (!build_options.have_llvm) { + return error.ZigCompilerNotBuiltWithLLVMExtensions; + } + const gpa = mod.gpa; + var arena_allocator = std.heap.ArenaAllocator.init(gpa); + errdefer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + switch (crt_file) { + .crti_o => { + var args = std.ArrayList([]const u8).init(arena); + try add_include_dirs(mod, arena, &args); + try args.appendSlice(&[_][]const u8{ + "-D_LIBC_REENTRANT", + "-include", + try lib_path(mod, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-modules.h"), + "-DMODULE_NAME=libc", + "-Wno-nonportable-include-path", + "-include", + try lib_path(mod, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-symbols.h"), + "-DTOP_NAMESPACE=glibc", + "-DASSEMBLER", + "-g", + "-Wa,--noexecstack", + }); + const c_source_file: Module.CSourceFile = .{ + .src_path = try start_asm_path(mod, arena, "crti.S"), + .extra_flags = args.items, + }; + return build_libc_object(mod, "crti.o", c_source_file); + }, + .crtn_o => { + var args = std.ArrayList([]const u8).init(arena); + try add_include_dirs(mod, arena, &args); + try args.appendSlice(&[_][]const u8{ + "-D_LIBC_REENTRANT", + "-DMODULE_NAME=libc", + "-DTOP_NAMESPACE=glibc", + "-DASSEMBLER", + "-g", + "-Wa,--noexecstack", + }); + const c_source_file: Module.CSourceFile = .{ + .src_path = try start_asm_path(mod, arena, "crtn.S"), + .extra_flags = args.items, + }; + return build_libc_object(mod, "crtn.o", c_source_file); + }, + .start_os => { + var args = std.ArrayList([]const u8).init(arena); + try add_include_dirs(mod, arena, &args); + try args.appendSlice(&[_][]const u8{ + "-D_LIBC_REENTRANT", + "-include", + try lib_path(mod, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-modules.h"), + "-DMODULE_NAME=libc", + "-Wno-nonportable-include-path", + "-include", + try lib_path(mod, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-symbols.h"), + "-DPIC", + "-DSHARED", + "-DTOP_NAMESPACE=glibc", + "-DASSEMBLER", + "-g", + "-Wa,--noexecstack", + }); + const c_source_file: Module.CSourceFile = .{ + .src_path = try start_asm_path(mod, arena, "start.S"), + .extra_flags = args.items, + }; + return build_libc_object(mod, "start.os", c_source_file); + }, + .abi_note_o => { + var args = std.ArrayList([]const u8).init(arena); + try args.appendSlice(&[_][]const u8{ + "-I", + try lib_path(mod, arena, lib_libc_glibc ++ "glibc" ++ path.sep_str ++ "csu"), + }); + try add_include_dirs(mod, arena, &args); + try args.appendSlice(&[_][]const u8{ + "-D_LIBC_REENTRANT", + "-DMODULE_NAME=libc", + "-DTOP_NAMESPACE=glibc", + "-DASSEMBLER", + "-g", + "-Wa,--noexecstack", + }); + const c_source_file: Module.CSourceFile = .{ + .src_path = try lib_path(mod, arena, lib_libc_glibc ++ "csu" ++ path.sep_str ++ "abi-note.S"), + .extra_flags = args.items, + }; + return build_libc_object(mod, "abi-note.o", c_source_file); + }, + .scrt1_o => { + return error.Unimplemented; // TODO + }, + .libc_nonshared_a => { + return error.Unimplemented; // TODO + }, + } +} + +fn start_asm_path(mod: *Module, arena: *Allocator, basename: []const u8) ![]const u8 { + const arch = mod.getTarget().cpu.arch; + const is_ppc = arch == .powerpc or arch == .powerpc64 or arch == .powerpc64le; + const is_aarch64 = arch == .aarch64 or arch == .aarch64_be; + const is_sparc = arch == .sparc or arch == .sparcel or arch == .sparcv9; + const is_64 = arch.ptrBitWidth() == 64; + + const s = path.sep_str; + + var result = std.ArrayList(u8).init(arena); + try result.appendSlice(mod.zig_lib_directory.path.?); + try result.appendSlice(s ++ "libc" ++ s ++ "glibc" ++ s ++ "sysdeps" ++ s); + if (is_sparc) { + if (is_64) { + try result.appendSlice("sparc" ++ s ++ "sparc64"); + } else { + try result.appendSlice("sparc" ++ s ++ "sparc32"); + } + } else if (arch.isARM()) { + try result.appendSlice("arm"); + } else if (arch.isMIPS()) { + try result.appendSlice("mips"); + } else if (arch == .x86_64) { + try result.appendSlice("x86_64"); + } else if (arch == .i386) { + try result.appendSlice("i386"); + } else if (is_aarch64) { + try result.appendSlice("aarch64"); + } else if (arch.isRISCV()) { + try result.appendSlice("riscv"); + } else if (is_ppc) { + if (is_64) { + try result.appendSlice("powerpc" ++ s ++ "powerpc64"); + } else { + try result.appendSlice("powerpc" ++ s ++ "powerpc32"); + } + } + + try result.appendSlice(s); + try result.appendSlice(basename); + return result.items; +} + +fn add_include_dirs(mod: *Module, arena: *Allocator, args: *std.ArrayList([]const u8)) error{OutOfMemory}!void { + const target = mod.getTarget(); + const arch = target.cpu.arch; + const opt_nptl: ?[]const u8 = if (target.os.tag == .linux) "nptl" else "htl"; + const glibc = try lib_path(mod, arena, lib_libc ++ "glibc"); + + const s = path.sep_str; + + try args.append("-I"); + try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "include")); + + if (target.os.tag == .linux) { + try add_include_dirs_arch(arena, args, arch, null, try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix" ++ s ++ "sysv" ++ s ++ "linux")); + } + + if (opt_nptl) |nptl| { + try add_include_dirs_arch(arena, args, arch, nptl, try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps")); + } + + if (target.os.tag == .linux) { + try args.append("-I"); + try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ + "unix" ++ s ++ "sysv" ++ s ++ "linux" ++ s ++ "generic")); + + try args.append("-I"); + try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ + "unix" ++ s ++ "sysv" ++ s ++ "linux" ++ s ++ "include")); + try args.append("-I"); + try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ + "unix" ++ s ++ "sysv" ++ s ++ "linux")); + } + if (opt_nptl) |nptl| { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ mod.zig_lib_directory.path.?, lib_libc_glibc ++ "sysdeps", nptl })); + } + + try args.append("-I"); + try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "pthread")); + + try args.append("-I"); + try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix" ++ s ++ "sysv")); + + try add_include_dirs_arch(arena, args, arch, null, try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix")); + + try args.append("-I"); + try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix")); + + try add_include_dirs_arch(arena, args, arch, null, try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps")); + + try args.append("-I"); + try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "generic")); + + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ mod.zig_lib_directory.path.?, lib_libc ++ "glibc" })); + + try args.append("-I"); + try args.append(try std.fmt.allocPrint(arena, "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{}-{}-{}", .{ + mod.zig_lib_directory.path.?, @tagName(arch), @tagName(target.os.tag), @tagName(target.abi), + })); + + try args.append("-I"); + try args.append(try lib_path(mod, arena, lib_libc ++ "include" ++ s ++ "generic-glibc")); + + try args.append("-I"); + try args.append(try std.fmt.allocPrint(arena, "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{}-linux-any", .{ + mod.zig_lib_directory.path.?, @tagName(arch), + })); + + try args.append("-I"); + try args.append(try lib_path(mod, arena, lib_libc ++ "include" ++ s ++ "any-linux-any")); +} + +fn add_include_dirs_arch( + arena: *Allocator, + args: *std.ArrayList([]const u8), + arch: std.Target.Cpu.Arch, + opt_nptl: ?[]const u8, + dir: []const u8, +) error{OutOfMemory}!void { + const is_x86 = arch == .i386 or arch == .x86_64; + const is_aarch64 = arch == .aarch64 or arch == .aarch64_be; + const is_ppc = arch == .powerpc or arch == .powerpc64 or arch == .powerpc64le; + const is_sparc = arch == .sparc or arch == .sparcel or arch == .sparcv9; + const is_64 = arch.ptrBitWidth() == 64; + + const s = path.sep_str; + + if (is_x86) { + if (arch == .x86_64) { + if (opt_nptl) |nptl| { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "x86_64", nptl })); + } else { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "x86_64" })); + } + } else if (arch == .i386) { + if (opt_nptl) |nptl| { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "i386", nptl })); + } else { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "i386" })); + } + } + if (opt_nptl) |nptl| { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "x86", nptl })); + } else { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "x86" })); + } + } else if (arch.isARM()) { + if (opt_nptl) |nptl| { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "arm", nptl })); + } else { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "arm" })); + } + } else if (arch.isMIPS()) { + if (opt_nptl) |nptl| { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "mips", nptl })); + } else { + if (is_64) { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "mips" ++ s ++ "mips64" })); + } else { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "mips" ++ s ++ "mips32" })); + } + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "mips" })); + } + } else if (is_sparc) { + if (opt_nptl) |nptl| { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "sparc", nptl })); + } else { + if (is_64) { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "sparc" ++ s ++ "sparc64" })); + } else { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "sparc" ++ s ++ "sparc32" })); + } + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "sparc" })); + } + } else if (is_aarch64) { + if (opt_nptl) |nptl| { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "aarch64", nptl })); + } else { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "aarch64" })); + } + } else if (is_ppc) { + if (opt_nptl) |nptl| { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "powerpc", nptl })); + } else { + if (is_64) { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "powerpc" ++ s ++ "powerpc64" })); + } else { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "powerpc" ++ s ++ "powerpc32" })); + } + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "powerpc" })); + } + } else if (arch.isRISCV()) { + if (opt_nptl) |nptl| { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "riscv", nptl })); + } else { + try args.append("-I"); + try args.append(try path.join(arena, &[_][]const u8{ dir, "riscv" })); + } + } +} + +fn path_from_lib(mod: *Module, arena: *Allocator, sub_path: []const u8) ![]const u8 { + return path.join(arena, &[_][]const u8{ mod.zig_lib_directory.path.?, sub_path }); +} + +const lib_libc = "libc" ++ path.sep_str; +const lib_libc_glibc = lib_libc ++ "glibc" ++ path.sep_str; + +fn lib_path(mod: *Module, arena: *Allocator, sub_path: []const u8) ![]const u8 { + return path.join(arena, &[_][]const u8{ mod.zig_lib_directory.path.?, sub_path }); +} + +fn build_libc_object(mod: *Module, basename: []const u8, c_source_file: Module.CSourceFile) !void { + // TODO: This is extracted into a local variable to work around a stage1 miscompilation. + const emit_bin = Module.EmitLoc{ + .directory = null, // Put it in the cache directory. + .basename = basename, + }; + const sub_module = try Module.create(mod.gpa, .{ + // TODO use the global cache directory here + .zig_cache_directory = mod.zig_cache_directory, + .zig_lib_directory = mod.zig_lib_directory, + .target = mod.getTarget(), + .root_name = mem.split(basename, ".").next().?, + .root_pkg = null, + .output_mode = .Obj, + .rand = mod.rand, + .libc_installation = mod.bin_file.options.libc_installation, + .emit_bin = emit_bin, + .optimize_mode = mod.bin_file.options.optimize_mode, + .want_sanitize_c = false, + .want_stack_check = false, + .want_valgrind = false, + .want_pic = mod.bin_file.options.pic, + .emit_h = null, + .strip = mod.bin_file.options.strip, + .is_native_os = mod.bin_file.options.is_native_os, + .self_exe_path = mod.self_exe_path, + .c_source_files = &[1]Module.CSourceFile{c_source_file}, + .debug_cc = mod.debug_cc, + .debug_link = mod.bin_file.options.debug_link, + }); + defer sub_module.destroy(); + + try sub_module.update(); + + try mod.crt_files.ensureCapacity(mod.gpa, mod.crt_files.count() + 1); + const artifact_path = try std.fs.path.join(mod.gpa, &[_][]const u8{ + sub_module.zig_cache_artifact_directory.path.?, basename, + }); + mod.crt_files.putAssumeCapacityNoClobber(basename, artifact_path); +} diff --git a/src-self-hosted/introspect.zig b/src-self-hosted/introspect.zig index 019da4a39e..3dd040881c 100644 --- a/src-self-hosted/introspect.zig +++ b/src-self-hosted/introspect.zig @@ -1,77 +1,65 @@ -//! Introspection and determination of system libraries needed by zig. - const std = @import("std"); const mem = std.mem; const fs = std.fs; const CacheHash = std.cache_hash.CacheHash; +const Module = @import("Module.zig"); -/// Caller must free result -pub fn testZigInstallPrefix(allocator: *mem.Allocator, test_path: []const u8) ![]u8 { - { - const test_zig_dir = try fs.path.join(allocator, &[_][]const u8{ test_path, "lib", "zig" }); - errdefer allocator.free(test_zig_dir); +/// Returns the sub_path that worked, or `null` if none did. +/// The path of the returned Directory is relative to `base`. +/// The handle of the returned Directory is open. +fn testZigInstallPrefix(base_dir: fs.Dir) ?Module.Directory { + const test_index_file = "std" ++ fs.path.sep_str ++ "std.zig"; - const test_index_file = try fs.path.join(allocator, &[_][]const u8{ test_zig_dir, "std", "std.zig" }); - defer allocator.free(test_index_file); - - if (fs.cwd().openFile(test_index_file, .{})) |file| { - file.close(); - return test_zig_dir; - } else |err| switch (err) { - error.FileNotFound => { - allocator.free(test_zig_dir); - }, - else => |e| return e, - } + zig_dir: { + // Try lib/zig/std/std.zig + const lib_zig = "lib" ++ fs.path.sep_str ++ "zig"; + var test_zig_dir = base_dir.openDir(lib_zig, .{}) catch break :zig_dir; + const file = test_zig_dir.openFile(test_index_file, .{}) catch { + test_zig_dir.close(); + break :zig_dir; + }; + file.close(); + return Module.Directory{ .handle = test_zig_dir, .path = lib_zig }; } - // Also try without "zig" - const test_zig_dir = try fs.path.join(allocator, &[_][]const u8{ test_path, "lib" }); - errdefer allocator.free(test_zig_dir); - - const test_index_file = try fs.path.join(allocator, &[_][]const u8{ test_zig_dir, "std", "std.zig" }); - defer allocator.free(test_index_file); - - const file = try fs.cwd().openFile(test_index_file, .{}); + // Try lib/std/std.zig + var test_zig_dir = base_dir.openDir("lib", .{}) catch return null; + const file = test_zig_dir.openFile(test_index_file, .{}) catch { + test_zig_dir.close(); + return null; + }; file.close(); - - return test_zig_dir; + return Module.Directory{ .handle = test_zig_dir, .path = "lib" }; } -/// Caller must free result -pub fn findZigLibDir(allocator: *mem.Allocator) ![]u8 { - const self_exe_path = try fs.selfExePathAlloc(allocator); - defer allocator.free(self_exe_path); +/// Both the directory handle and the path are newly allocated resources which the caller now owns. +pub fn findZigLibDir(gpa: *mem.Allocator) !Module.Directory { + const self_exe_path = try fs.selfExePathAlloc(gpa); + defer gpa.free(self_exe_path); + return findZigLibDirFromSelfExe(gpa, self_exe_path); +} + +/// Both the directory handle and the path are newly allocated resources which the caller now owns. +pub fn findZigLibDirFromSelfExe( + allocator: *mem.Allocator, + self_exe_path: []const u8, +) error{ OutOfMemory, FileNotFound }!Module.Directory { + const cwd = fs.cwd(); var cur_path: []const u8 = self_exe_path; - while (true) { - const test_dir = fs.path.dirname(cur_path) orelse "."; + while (fs.path.dirname(cur_path)) |dirname| : (cur_path = dirname) { + var base_dir = cwd.openDir(dirname, .{}) catch continue; + defer base_dir.close(); - if (mem.eql(u8, test_dir, cur_path)) { - break; - } - - return testZigInstallPrefix(allocator, test_dir) catch |err| { - cur_path = test_dir; - continue; + const sub_directory = testZigInstallPrefix(base_dir) orelse continue; + return Module.Directory{ + .handle = sub_directory.handle, + .path = try fs.path.join(allocator, &[_][]const u8{ dirname, sub_directory.path.? }), }; } - return error.FileNotFound; } -pub fn resolveZigLibDir(allocator: *mem.Allocator) ![]u8 { - return findZigLibDir(allocator) catch |err| { - std.debug.print( - \\Unable to find zig lib directory: {}. - \\Reinstall Zig or use --zig-install-prefix. - \\ - , .{@errorName(err)}); - - return error.ZigLibDirNotFound; - }; -} - /// Caller owns returned memory. pub fn resolveGlobalCacheDir(allocator: *mem.Allocator) ![]u8 { const appname = "zig"; diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 68292fbfe6..626ec6cca8 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -11,11 +11,9 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation; pub const producer_string = if (std.builtin.is_test) "zig test" else "zig " ++ build_options.version; pub const Options = struct { - dir: fs.Dir, - /// Redundant with dir. Needed when linking with LLD because we have to pass paths rather - /// than file descriptors. `null` means cwd. OK to pass `null` when `use_lld` is `false`. - dir_path: ?[]const u8, - /// Path to the output file, relative to dir. + /// Where the output will go. + directory: Module.Directory, + /// Path to the output file, relative to `directory`. sub_path: []const u8, target: std.Target, output_mode: std.builtin.OutputMode, @@ -53,6 +51,11 @@ pub const Options = struct { z_defs: bool = false, bind_global_refs_locally: bool, is_native_os: bool, + pic: bool, + valgrind: bool, + stack_check: bool, + single_threaded: bool, + debug_link: bool = false, gc_sections: ?bool = null, allow_shlib_undefined: ?bool = null, linker_script: ?[]const u8 = null, @@ -154,7 +157,7 @@ pub const File = struct { switch (base.tag) { .coff, .elf, .macho => { if (base.file != null) return; - base.file = try base.options.dir.createFile(base.options.sub_path, .{ + base.file = try base.options.directory.handle.createFile(base.options.sub_path, .{ .truncate = false, .read = true, .mode = determineMode(base.options), diff --git a/src-self-hosted/link/C.zig b/src-self-hosted/link/C.zig index 8ec6f91959..277fea7080 100644 --- a/src-self-hosted/link/C.zig +++ b/src-self-hosted/link/C.zig @@ -28,7 +28,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio if (options.use_llvm) return error.LLVMHasNoCBackend; if (options.use_lld) return error.LLDHasNoCBackend; - const file = try options.dir.createFile(sub_path, .{ .truncate = true, .read = true, .mode = link.determineMode(options) }); + const file = try options.directory.handle.createFile(sub_path, .{ .truncate = true, .read = true, .mode = link.determineMode(options) }); errdefer file.close(); var c_file = try allocator.create(C); diff --git a/src-self-hosted/link/Coff.zig b/src-self-hosted/link/Coff.zig index 950e3537d9..6d7e87e898 100644 --- a/src-self-hosted/link/Coff.zig +++ b/src-self-hosted/link/Coff.zig @@ -116,7 +116,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio if (options.use_llvm) return error.LLVM_BackendIsTODO_ForCoff; // TODO if (options.use_lld) return error.LLD_LinkingIsTODO_ForCoff; // TODO - const file = try options.dir.createFile(sub_path, .{ + const file = try options.directory.handle.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options), diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 835ab12cfa..d46053493a 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -19,6 +19,7 @@ const File = link.File; const Elf = @This(); const build_options = @import("build_options"); const target_util = @import("../target.zig"); +const fatal = @import("main.zig").fatal; const default_entry_addr = 0x8000000; @@ -222,7 +223,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio if (options.use_llvm) return error.LLVMBackendUnimplementedForELF; // TODO - const file = try options.dir.createFile(sub_path, .{ + const file = try options.directory.handle.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options), @@ -844,7 +845,7 @@ fn flushInner(self: *Elf, module: *Module) !void { } // Write the form for the compile unit, which must match the abbrev table above. const name_strp = try self.makeDebugString(self.base.options.root_pkg.?.root_src_path); - const comp_dir_strp = try self.makeDebugString(self.base.options.root_pkg.?.root_src_dir_path); + const comp_dir_strp = try self.makeDebugString(self.base.options.root_pkg.?.root_src_directory.path.?); const producer_strp = try self.makeDebugString(link.producer_string); // Currently only one compilation unit is supported, so the address range is simply // identical to the main program header virtual address and memory size. @@ -1199,11 +1200,6 @@ fn flushInner(self: *Elf, module: *Module) !void { } fn linkWithLLD(self: *Elf, module: *Module) !void { - // If there is no Zig code to compile, then we should skip flushing the output file because it - // will not be part of the linker line anyway. - if (module.root_pkg != null) { - try self.flushInner(module); - } var arena_allocator = std.heap.ArenaAllocator.init(self.base.allocator); defer arena_allocator.deinit(); const arena = &arena_allocator.allocator; @@ -1292,7 +1288,7 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { try argv.append("-pie"); } - const full_out_path = if (self.base.options.dir_path) |dir_path| + const full_out_path = if (self.base.options.directory.path) |dir_path| try std.fs.path.join(arena, &[_][]const u8{dir_path, self.base.options.sub_path}) else self.base.options.sub_path; @@ -1382,6 +1378,30 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { // Positional arguments to the linker such as object files. try argv.appendSlice(self.base.options.objects); + for (module.c_object_table.items()) |entry| { + const c_object = entry.key; + switch (c_object.status) { + .new => unreachable, + .failure => return error.NotAllCSourceFilesAvailableToLink, + .success => |full_obj_path| { + try argv.append(full_obj_path); + }, + } + } + + // If there is no Zig code to compile, then we should skip flushing the output file because it + // will not be part of the linker line anyway. + if (module.root_pkg != null) { + try self.flushInner(module); + + const obj_basename = self.base.intermediary_basename.?; + const full_obj_path = if (self.base.options.directory.path) |dir_path| + try std.fs.path.join(arena, &[_][]const u8{dir_path, obj_basename}) + else + obj_basename; + try argv.append(full_obj_path); + } + // TODO compiler-rt and libc //if (!g->is_dummy_so && (g->out_type == OutTypeExe || is_dyn_lib)) { // if (g->libc_link_lib == nullptr) { @@ -1461,10 +1481,31 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { try argv.append("-Bsymbolic"); } - for (argv.items) |arg| { - std.debug.print("{} ", .{arg}); + if (self.base.options.debug_link) { + for (argv.items[0 .. argv.items.len - 1]) |arg| { + std.debug.print("{} ", .{arg}); + } + std.debug.print("{}\n", .{argv.items[argv.items.len - 1]}); } - @panic("invoke LLD"); + + // Oh, snapplesauce! We need null terminated argv. + // TODO allocSentinel crashed stage1 so this is working around it. + const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1); + new_argv_with_sentinel[argv.items.len] = null; + const new_argv = new_argv_with_sentinel[0..argv.items.len: null]; + for (argv.items) |arg, i| { + new_argv[i] = try arena.dupeZ(u8, arg); + } + + const ZigLLDLink = @import("../llvm.zig").ZigLLDLink; + const ok = ZigLLDLink(.ELF, new_argv.ptr, new_argv.len, append_diagnostic, 0, 0); + if (!ok) return error.LLDReportedFailure; +} + +fn append_diagnostic(context: usize, ptr: [*]const u8, len: usize) callconv(.C) void { + // TODO collect diagnostics and handle cleanly + const msg = ptr[0..len]; + std.log.err("LLD: {}", .{msg}); } fn writeDwarfAddrAssumeCapacity(self: *Elf, buf: *std.ArrayList(u8), addr: u64) void { @@ -2681,7 +2722,7 @@ fn dbgLineNeededHeaderBytes(self: Elf) u32 { directory_count * 8 + file_name_count * 8 + // These are encoded as DW.FORM_string rather than DW.FORM_strp as we would like // because of a workaround for readelf and gdb failing to understand DWARFv5 correctly. - self.base.options.root_pkg.?.root_src_dir_path.len + + self.base.options.root_pkg.?.root_src_directory.path.?.len + self.base.options.root_pkg.?.root_src_path.len); } diff --git a/src-self-hosted/link/MachO.zig b/src-self-hosted/link/MachO.zig index 3ffa606d04..b57f3d303a 100644 --- a/src-self-hosted/link/MachO.zig +++ b/src-self-hosted/link/MachO.zig @@ -140,7 +140,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio if (options.use_llvm) return error.LLVM_BackendIsTODO_ForMachO; // TODO if (options.use_lld) return error.LLD_LinkingIsTODO_ForMachO; // TODO - const file = try options.dir.createFile(sub_path, .{ + const file = try options.directory.handle.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options), diff --git a/src-self-hosted/link/Wasm.zig b/src-self-hosted/link/Wasm.zig index 46bafaaf10..bea36723ff 100644 --- a/src-self-hosted/link/Wasm.zig +++ b/src-self-hosted/link/Wasm.zig @@ -56,7 +56,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio if (options.use_lld) return error.LLD_LinkingIsTODO_ForWasm; // TODO // TODO: read the file and keep vaild parts instead of truncating - const file = try options.dir.createFile(sub_path, .{ .truncate = true, .read = true }); + const file = try options.directory.handle.createFile(sub_path, .{ .truncate = true, .read = true }); errdefer file.close(); const wasm = try allocator.create(Wasm); diff --git a/src-self-hosted/llvm.zig b/src-self-hosted/llvm.zig index 4159968252..d65e046169 100644 --- a/src-self-hosted/llvm.zig +++ b/src-self-hosted/llvm.zig @@ -1,293 +1,20 @@ -const c = @import("c.zig"); -const assert = @import("std").debug.assert; +//! We do this instead of @cImport because the self-hosted compiler is easier +//! to bootstrap if it does not depend on translate-c. -// we wrap the c module for 3 reasons: -// 1. to avoid accidentally calling the non-thread-safe functions -// 2. patch up some of the types to remove nullability -// 3. some functions have been augmented by zig_llvm.cpp to be more powerful, -// such as ZigLLVMTargetMachineEmitToFile - -pub const AttributeIndex = c_uint; -pub const Bool = c_int; - -pub const Builder = c.LLVMBuilderRef.Child.Child; -pub const Context = c.LLVMContextRef.Child.Child; -pub const Module = c.LLVMModuleRef.Child.Child; -pub const Value = c.LLVMValueRef.Child.Child; -pub const Type = c.LLVMTypeRef.Child.Child; -pub const BasicBlock = c.LLVMBasicBlockRef.Child.Child; -pub const Attribute = c.LLVMAttributeRef.Child.Child; -pub const Target = c.LLVMTargetRef.Child.Child; -pub const TargetMachine = c.LLVMTargetMachineRef.Child.Child; -pub const TargetData = c.LLVMTargetDataRef.Child.Child; -pub const DIBuilder = c.ZigLLVMDIBuilder; -pub const DIFile = c.ZigLLVMDIFile; -pub const DICompileUnit = c.ZigLLVMDICompileUnit; - -pub const ABIAlignmentOfType = c.LLVMABIAlignmentOfType; -pub const AddAttributeAtIndex = c.LLVMAddAttributeAtIndex; -pub const AddModuleCodeViewFlag = c.ZigLLVMAddModuleCodeViewFlag; -pub const AddModuleDebugInfoFlag = c.ZigLLVMAddModuleDebugInfoFlag; -pub const ClearCurrentDebugLocation = c.ZigLLVMClearCurrentDebugLocation; -pub const ConstAllOnes = c.LLVMConstAllOnes; -pub const ConstArray = c.LLVMConstArray; -pub const ConstBitCast = c.LLVMConstBitCast; -pub const ConstIntOfArbitraryPrecision = c.LLVMConstIntOfArbitraryPrecision; -pub const ConstNeg = c.LLVMConstNeg; -pub const ConstStructInContext = c.LLVMConstStructInContext; -pub const DIBuilderFinalize = c.ZigLLVMDIBuilderFinalize; -pub const DisposeBuilder = c.LLVMDisposeBuilder; -pub const DisposeDIBuilder = c.ZigLLVMDisposeDIBuilder; -pub const DisposeMessage = c.LLVMDisposeMessage; -pub const DisposeModule = c.LLVMDisposeModule; -pub const DisposeTargetData = c.LLVMDisposeTargetData; -pub const DisposeTargetMachine = c.LLVMDisposeTargetMachine; -pub const DoubleTypeInContext = c.LLVMDoubleTypeInContext; -pub const DumpModule = c.LLVMDumpModule; -pub const FP128TypeInContext = c.LLVMFP128TypeInContext; -pub const FloatTypeInContext = c.LLVMFloatTypeInContext; -pub const GetEnumAttributeKindForName = c.LLVMGetEnumAttributeKindForName; -pub const GetMDKindIDInContext = c.LLVMGetMDKindIDInContext; -pub const GetUndef = c.LLVMGetUndef; -pub const HalfTypeInContext = c.LLVMHalfTypeInContext; -pub const InitializeAllAsmParsers = c.LLVMInitializeAllAsmParsers; -pub const InitializeAllAsmPrinters = c.LLVMInitializeAllAsmPrinters; -pub const InitializeAllTargetInfos = c.LLVMInitializeAllTargetInfos; -pub const InitializeAllTargetMCs = c.LLVMInitializeAllTargetMCs; -pub const InitializeAllTargets = c.LLVMInitializeAllTargets; -pub const InsertBasicBlockInContext = c.LLVMInsertBasicBlockInContext; -pub const Int128TypeInContext = c.LLVMInt128TypeInContext; -pub const Int16TypeInContext = c.LLVMInt16TypeInContext; -pub const Int1TypeInContext = c.LLVMInt1TypeInContext; -pub const Int32TypeInContext = c.LLVMInt32TypeInContext; -pub const Int64TypeInContext = c.LLVMInt64TypeInContext; -pub const Int8TypeInContext = c.LLVMInt8TypeInContext; -pub const IntPtrTypeForASInContext = c.LLVMIntPtrTypeForASInContext; -pub const IntPtrTypeInContext = c.LLVMIntPtrTypeInContext; -pub const LabelTypeInContext = c.LLVMLabelTypeInContext; -pub const MDNodeInContext = c.LLVMMDNodeInContext; -pub const MDStringInContext = c.LLVMMDStringInContext; -pub const MetadataTypeInContext = c.LLVMMetadataTypeInContext; -pub const PPCFP128TypeInContext = c.LLVMPPCFP128TypeInContext; -pub const SetAlignment = c.LLVMSetAlignment; -pub const SetDataLayout = c.LLVMSetDataLayout; -pub const SetGlobalConstant = c.LLVMSetGlobalConstant; -pub const SetInitializer = c.LLVMSetInitializer; -pub const SetLinkage = c.LLVMSetLinkage; -pub const SetTarget = c.LLVMSetTarget; -pub const SetUnnamedAddr = c.LLVMSetUnnamedAddr; -pub const SetVolatile = c.LLVMSetVolatile; -pub const StructTypeInContext = c.LLVMStructTypeInContext; -pub const TokenTypeInContext = c.LLVMTokenTypeInContext; -pub const X86FP80TypeInContext = c.LLVMX86FP80TypeInContext; -pub const X86MMXTypeInContext = c.LLVMX86MMXTypeInContext; - -pub const AddGlobal = LLVMAddGlobal; -extern fn LLVMAddGlobal(M: *Module, Ty: *Type, Name: [*:0]const u8) ?*Value; - -pub const ConstStringInContext = LLVMConstStringInContext; -extern fn LLVMConstStringInContext(C: *Context, Str: [*]const u8, Length: c_uint, DontNullTerminate: Bool) ?*Value; - -pub const ConstInt = LLVMConstInt; -extern fn LLVMConstInt(IntTy: *Type, N: c_ulonglong, SignExtend: Bool) ?*Value; - -pub const BuildLoad = LLVMBuildLoad; -extern fn LLVMBuildLoad(arg0: *Builder, PointerVal: *Value, Name: [*:0]const u8) ?*Value; - -pub const ConstNull = LLVMConstNull; -extern fn LLVMConstNull(Ty: *Type) ?*Value; - -pub const CreateStringAttribute = LLVMCreateStringAttribute; -extern fn LLVMCreateStringAttribute( - C: *Context, - K: [*]const u8, - KLength: c_uint, - V: [*]const u8, - VLength: c_uint, -) ?*Attribute; - -pub const CreateEnumAttribute = LLVMCreateEnumAttribute; -extern fn LLVMCreateEnumAttribute(C: *Context, KindID: c_uint, Val: u64) ?*Attribute; - -pub const AddFunction = LLVMAddFunction; -extern fn LLVMAddFunction(M: *Module, Name: [*:0]const u8, FunctionTy: *Type) ?*Value; - -pub const CreateCompileUnit = ZigLLVMCreateCompileUnit; -extern fn ZigLLVMCreateCompileUnit( - dibuilder: *DIBuilder, - lang: c_uint, - difile: *DIFile, - producer: [*:0]const u8, - is_optimized: bool, - flags: [*:0]const u8, - runtime_version: c_uint, - split_name: [*:0]const u8, - dwo_id: u64, - emit_debug_info: bool, -) ?*DICompileUnit; - -pub const CreateFile = ZigLLVMCreateFile; -extern fn ZigLLVMCreateFile(dibuilder: *DIBuilder, filename: [*:0]const u8, directory: [*:0]const u8) ?*DIFile; - -pub const ArrayType = LLVMArrayType; -extern fn LLVMArrayType(ElementType: *Type, ElementCount: c_uint) ?*Type; - -pub const CreateDIBuilder = ZigLLVMCreateDIBuilder; -extern fn ZigLLVMCreateDIBuilder(module: *Module, allow_unresolved: bool) ?*DIBuilder; - -pub const PointerType = LLVMPointerType; -extern fn LLVMPointerType(ElementType: *Type, AddressSpace: c_uint) ?*Type; - -pub const CreateBuilderInContext = LLVMCreateBuilderInContext; -extern fn LLVMCreateBuilderInContext(C: *Context) ?*Builder; - -pub const IntTypeInContext = LLVMIntTypeInContext; -extern fn LLVMIntTypeInContext(C: *Context, NumBits: c_uint) ?*Type; - -pub const ModuleCreateWithNameInContext = LLVMModuleCreateWithNameInContext; -extern fn LLVMModuleCreateWithNameInContext(ModuleID: [*:0]const u8, C: *Context) ?*Module; - -pub const VoidTypeInContext = LLVMVoidTypeInContext; -extern fn LLVMVoidTypeInContext(C: *Context) ?*Type; - -pub const ContextCreate = LLVMContextCreate; -extern fn LLVMContextCreate() ?*Context; - -pub const ContextDispose = LLVMContextDispose; -extern fn LLVMContextDispose(C: *Context) void; - -pub const CopyStringRepOfTargetData = LLVMCopyStringRepOfTargetData; -extern fn LLVMCopyStringRepOfTargetData(TD: *TargetData) ?[*:0]u8; - -pub const CreateTargetDataLayout = LLVMCreateTargetDataLayout; -extern fn LLVMCreateTargetDataLayout(T: *TargetMachine) ?*TargetData; - -pub const CreateTargetMachine = ZigLLVMCreateTargetMachine; -extern fn ZigLLVMCreateTargetMachine( - T: *Target, - Triple: [*:0]const u8, - CPU: [*:0]const u8, - Features: [*:0]const u8, - Level: CodeGenOptLevel, - Reloc: RelocMode, - CodeModel: CodeModel, - function_sections: bool, -) ?*TargetMachine; - -pub const GetHostCPUName = LLVMGetHostCPUName; -extern fn LLVMGetHostCPUName() ?[*:0]u8; - -pub const GetNativeFeatures = ZigLLVMGetNativeFeatures; -extern fn ZigLLVMGetNativeFeatures() ?[*:0]u8; - -pub const GetElementType = LLVMGetElementType; -extern fn LLVMGetElementType(Ty: *Type) *Type; - -pub const TypeOf = LLVMTypeOf; -extern fn LLVMTypeOf(Val: *Value) *Type; - -pub const BuildStore = LLVMBuildStore; -extern fn LLVMBuildStore(arg0: *Builder, Val: *Value, Ptr: *Value) ?*Value; - -pub const BuildAlloca = LLVMBuildAlloca; -extern fn LLVMBuildAlloca(arg0: *Builder, Ty: *Type, Name: ?[*:0]const u8) ?*Value; - -pub const ConstInBoundsGEP = LLVMConstInBoundsGEP; -pub extern fn LLVMConstInBoundsGEP(ConstantVal: *Value, ConstantIndices: [*]*Value, NumIndices: c_uint) ?*Value; - -pub const GetTargetFromTriple = LLVMGetTargetFromTriple; -extern fn LLVMGetTargetFromTriple(Triple: [*:0]const u8, T: **Target, ErrorMessage: ?*[*:0]u8) Bool; - -pub const VerifyModule = LLVMVerifyModule; -extern fn LLVMVerifyModule(M: *Module, Action: VerifierFailureAction, OutMessage: *?[*:0]u8) Bool; - -pub const GetInsertBlock = LLVMGetInsertBlock; -extern fn LLVMGetInsertBlock(Builder: *Builder) *BasicBlock; - -pub const FunctionType = LLVMFunctionType; -extern fn LLVMFunctionType( - ReturnType: *Type, - ParamTypes: [*]*Type, - ParamCount: c_uint, - IsVarArg: Bool, -) ?*Type; - -pub const GetParam = LLVMGetParam; -extern fn LLVMGetParam(Fn: *Value, Index: c_uint) *Value; - -pub const AppendBasicBlockInContext = LLVMAppendBasicBlockInContext; -extern fn LLVMAppendBasicBlockInContext(C: *Context, Fn: *Value, Name: [*:0]const u8) ?*BasicBlock; - -pub const PositionBuilderAtEnd = LLVMPositionBuilderAtEnd; -extern fn LLVMPositionBuilderAtEnd(Builder: *Builder, Block: *BasicBlock) void; - -pub const AbortProcessAction = VerifierFailureAction.LLVMAbortProcessAction; -pub const PrintMessageAction = VerifierFailureAction.LLVMPrintMessageAction; -pub const ReturnStatusAction = VerifierFailureAction.LLVMReturnStatusAction; -pub const VerifierFailureAction = c.LLVMVerifierFailureAction; - -pub const CodeGenLevelNone = CodeGenOptLevel.LLVMCodeGenLevelNone; -pub const CodeGenLevelLess = CodeGenOptLevel.LLVMCodeGenLevelLess; -pub const CodeGenLevelDefault = CodeGenOptLevel.LLVMCodeGenLevelDefault; -pub const CodeGenLevelAggressive = CodeGenOptLevel.LLVMCodeGenLevelAggressive; -pub const CodeGenOptLevel = c.LLVMCodeGenOptLevel; - -pub const RelocDefault = RelocMode.LLVMRelocDefault; -pub const RelocStatic = RelocMode.LLVMRelocStatic; -pub const RelocPIC = RelocMode.LLVMRelocPIC; -pub const RelocDynamicNoPic = RelocMode.LLVMRelocDynamicNoPic; -pub const RelocMode = c.LLVMRelocMode; - -pub const CodeModelDefault = CodeModel.LLVMCodeModelDefault; -pub const CodeModelJITDefault = CodeModel.LLVMCodeModelJITDefault; -pub const CodeModelSmall = CodeModel.LLVMCodeModelSmall; -pub const CodeModelKernel = CodeModel.LLVMCodeModelKernel; -pub const CodeModelMedium = CodeModel.LLVMCodeModelMedium; -pub const CodeModelLarge = CodeModel.LLVMCodeModelLarge; -pub const CodeModel = c.LLVMCodeModel; - -pub const EmitAssembly = EmitOutputType.ZigLLVM_EmitAssembly; -pub const EmitBinary = EmitOutputType.ZigLLVM_EmitBinary; -pub const EmitLLVMIr = EmitOutputType.ZigLLVM_EmitLLVMIr; -pub const EmitOutputType = c.ZigLLVM_EmitOutputType; - -pub const CCallConv = CallConv.LLVMCCallConv; -pub const FastCallConv = CallConv.LLVMFastCallConv; -pub const ColdCallConv = CallConv.LLVMColdCallConv; -pub const WebKitJSCallConv = CallConv.LLVMWebKitJSCallConv; -pub const AnyRegCallConv = CallConv.LLVMAnyRegCallConv; -pub const X86StdcallCallConv = CallConv.LLVMX86StdcallCallConv; -pub const X86FastcallCallConv = CallConv.LLVMX86FastcallCallConv; -pub const CallConv = c.LLVMCallConv; - -pub const CallAttr = extern enum { - Auto, - NeverTail, - NeverInline, - AlwaysTail, - AlwaysInline, -}; - -fn removeNullability(comptime T: type) type { - comptime assert(@typeInfo(T).Pointer.size == .C); - return *T.Child; -} - -pub const BuildRet = LLVMBuildRet; -extern fn LLVMBuildRet(arg0: *Builder, V: ?*Value) ?*Value; - -pub const TargetMachineEmitToFile = ZigLLVMTargetMachineEmitToFile; -extern fn ZigLLVMTargetMachineEmitToFile( - targ_machine_ref: *TargetMachine, - module_ref: *Module, - filename: [*:0]const u8, - output_type: EmitOutputType, - error_message: *[*:0]u8, - is_debug: bool, - is_small: bool, +pub extern fn ZigLLDLink( + oformat: ZigLLVM_ObjectFormatType, + args: [*:null]const ?[*:0]const u8, + arg_count: usize, + append_diagnostic: fn (context: usize, ptr: [*]const u8, len: usize) callconv(.C) void, + context_stdout: usize, + context_stderr: usize, ) bool; -pub const BuildCall = ZigLLVMBuildCall; -extern fn ZigLLVMBuildCall(B: *Builder, Fn: *Value, Args: [*]*Value, NumArgs: c_uint, CC: CallConv, fn_inline: CallAttr, Name: [*:0]const u8) ?*Value; - -pub const PrivateLinkage = c.LLVMLinkage.LLVMPrivateLinkage; +pub const ZigLLVM_ObjectFormatType = extern enum(c_int) { + Unknown, + COFF, + ELF, + MachO, + Wasm, + XCOFF, +}; diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index bc3e5a8e6c..154e09bc68 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -191,7 +191,14 @@ const usage_build_generic = \\ ReleaseSmall Optimize for small binary, safety off \\ -fPIC Force-enable Position Independent Code \\ -fno-PIC Force-disable Position Independent Code + \\ -fstack-check Enable stack probing in unsafe builds + \\ -fno-stack-check Disable stack probing in safe builds + \\ -fsanitize-c Enable C undefined behavior detection in unsafe builds + \\ -fno-sanitize-c Disable C undefined behavior detection in safe builds + \\ -fvalgrind Include valgrind client requests in release builds + \\ -fno-valgrind Omit valgrind client requests in debug builds \\ --strip Exclude debug symbols + \\ --single-threaded Code assumes it is only used single-threaded \\ -ofmt=[mode] Override target object format \\ elf Executable and Linking Format \\ c Compile to C source code @@ -262,6 +269,7 @@ pub fn buildOutputType( var root_src_file: ?[]const u8 = null; var version: std.builtin.Version = .{ .major = 0, .minor = 0, .patch = 0 }; var strip = false; + var single_threaded = false; var watch = false; var debug_tokenize = false; var debug_ast_tree = false; @@ -287,6 +295,8 @@ pub fn buildOutputType( var enable_cache: ?bool = null; var want_pic: ?bool = null; var want_sanitize_c: ?bool = null; + var want_stack_check: ?bool = null; + var want_valgrind: ?bool = null; var rdynamic: bool = false; var only_pp_or_asm = false; var linker_script: ?[]const u8 = null; @@ -320,7 +330,7 @@ pub fn buildOutputType( var rpath_list = std.ArrayList([]const u8).init(gpa); defer rpath_list.deinit(); - var c_source_files = std.ArrayList([]const u8).init(gpa); + var c_source_files = std.ArrayList(Module.CSourceFile).init(gpa); defer c_source_files.deinit(); var link_objects = std.ArrayList([]const u8).init(gpa); @@ -463,6 +473,18 @@ pub fn buildOutputType( want_pic = true; } else if (mem.eql(u8, arg, "-fno-PIC")) { want_pic = false; + } else if (mem.eql(u8, arg, "-fstack-check")) { + want_stack_check = true; + } else if (mem.eql(u8, arg, "-fno-stack-check")) { + want_stack_check = false; + } else if (mem.eql(u8, arg, "-fsanitize-c")) { + want_sanitize_c = true; + } else if (mem.eql(u8, arg, "-fno-sanitize-c")) { + want_sanitize_c = false; + } else if (mem.eql(u8, arg, "-fvalgrind")) { + want_valgrind = true; + } else if (mem.eql(u8, arg, "-fno-valgrind")) { + want_valgrind = false; } else if (mem.eql(u8, arg, "-fLLVM")) { use_llvm = true; } else if (mem.eql(u8, arg, "-fno-LLVM")) { @@ -501,6 +523,8 @@ pub fn buildOutputType( link_mode = .Static; } else if (mem.eql(u8, arg, "--strip")) { strip = true; + } else if (mem.eql(u8, arg, "--single-threaded")) { + single_threaded = true; } else if (mem.eql(u8, arg, "--eh-frame-hdr")) { link_eh_frame_hdr = true; } else if (mem.eql(u8, arg, "-Bsymbolic")) { @@ -541,7 +565,8 @@ pub fn buildOutputType( { try link_objects.append(arg); } else if (Module.hasAsmExt(arg) or Module.hasCExt(arg) or Module.hasCppExt(arg)) { - try c_source_files.append(arg); + // TODO a way to pass extra flags on the CLI + try c_source_files.append(.{ .src_path = arg }); } else if (mem.endsWith(u8, arg, ".so") or mem.endsWith(u8, arg, ".dylib") or mem.endsWith(u8, arg, ".dll")) @@ -586,7 +611,7 @@ pub fn buildOutputType( .positional => { const file_ext = Module.classifyFileExt(mem.spanZ(it.only_arg)); switch (file_ext) { - .assembly, .c, .cpp, .ll, .bc, .h => try c_source_files.append(it.only_arg), + .assembly, .c, .cpp, .ll, .bc, .h => try c_source_files.append(.{ .src_path = it.only_arg }), .unknown, .so => try link_objects.append(it.only_arg), } }, @@ -812,7 +837,7 @@ pub fn buildOutputType( // .yes => |p| p, // else => c_source_file.source_path, // }; - // const basename = std.fs.path.basename(src_path); + // const basename = fs.path.basename(src_path); // c_source_file.preprocessor_only_basename = basename; //} //emit_bin = .no; @@ -839,7 +864,7 @@ pub fn buildOutputType( const basename = fs.path.basename(file); break :blk mem.split(basename, ".").next().?; } else if (c_source_files.items.len == 1) { - const basename = fs.path.basename(c_source_files.items[0]); + const basename = fs.path.basename(c_source_files.items[0].src_path); break :blk mem.split(basename, ".").next().?; } else if (link_objects.items.len == 1) { const basename = fs.path.basename(link_objects.items[0]); @@ -966,19 +991,71 @@ pub fn buildOutputType( } }; - const bin_path = switch (emit_bin) { - .no => { - fatal("-fno-emit-bin not supported yet", .{}); + var cleanup_emit_bin_dir: ?fs.Dir = null; + defer if (cleanup_emit_bin_dir) |*dir| dir.close(); + + const emit_bin_loc: ?Module.EmitLoc = switch (emit_bin) { + .no => null, + .yes_default_path => Module.EmitLoc{ + .directory = .{ .path = null, .handle = fs.cwd() }, + .basename = try std.zig.binNameAlloc( + arena, + root_name, + target_info.target, + output_mode, + link_mode, + object_format, + ), + }, + .yes => |full_path| b: { + const basename = fs.path.basename(full_path); + if (fs.path.dirname(full_path)) |dirname| { + const handle = try fs.cwd().openDir(dirname, .{}); + cleanup_emit_bin_dir = handle; + break :b Module.EmitLoc{ + .basename = basename, + .directory = .{ + .path = dirname, + .handle = handle, + }, + }; + } else { + break :b Module.EmitLoc{ + .basename = basename, + .directory = .{ .path = null, .handle = fs.cwd() }, + }; + } + }, + }; + + var cleanup_emit_h_dir: ?fs.Dir = null; + defer if (cleanup_emit_h_dir) |*dir| dir.close(); + + const emit_h_loc: ?Module.EmitLoc = switch (emit_h) { + .no => null, + .yes_default_path => Module.EmitLoc{ + .directory = .{ .path = null, .handle = fs.cwd() }, + .basename = try std.fmt.allocPrint(arena, "{}.h", .{root_name}), + }, + .yes => |full_path| b: { + const basename = fs.path.basename(full_path); + if (fs.path.dirname(full_path)) |dirname| { + const handle = try fs.cwd().openDir(dirname, .{}); + cleanup_emit_h_dir = handle; + break :b Module.EmitLoc{ + .basename = basename, + .directory = .{ + .path = dirname, + .handle = handle, + }, + }; + } else { + break :b Module.EmitLoc{ + .basename = basename, + .directory = .{ .path = null, .handle = fs.cwd() }, + }; + } }, - .yes_default_path => try std.zig.binNameAlloc( - arena, - root_name, - target_info.target, - output_mode, - link_mode, - object_format, - ), - .yes => |p| p, }; const zir_out_path: ?[]const u8 = switch (emit_zir) { @@ -995,19 +1072,13 @@ pub fn buildOutputType( }; const root_pkg = if (root_src_file) |src_path| try Package.create(gpa, fs.cwd(), ".", src_path) else null; - defer if (root_pkg) |pkg| pkg.destroy(); - - const emit_h_path: ?[]const u8 = switch (emit_h) { - .yes => |p| p, - .no => null, - .yes_default_path => try std.fmt.allocPrint(arena, "{}.h", .{root_name}), - }; + defer if (root_pkg) |pkg| pkg.destroy(gpa); const self_exe_path = try fs.selfExePathAlloc(arena); - const zig_lib_dir = introspect.resolveZigLibDir(gpa) catch |err| { + var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| { fatal("unable to find zig installation directory: {}\n", .{@errorName(err)}); }; - defer gpa.free(zig_lib_dir); + defer zig_lib_directory.handle.close(); const random_seed = blk: { var random_seed: u64 = undefined; @@ -1025,17 +1096,32 @@ pub fn buildOutputType( }; } + const cache_parent_dir = if (root_pkg) |pkg| pkg.root_src_directory.handle else fs.cwd(); + var cache_dir = try cache_parent_dir.makeOpenPath("zig-cache", .{}); + defer cache_dir.close(); + const zig_cache_directory: Module.Directory = .{ + .handle = cache_dir, + .path = blk: { + if (root_pkg) |pkg| { + if (pkg.root_src_directory.path) |p| { + break :blk try fs.path.join(arena, &[_][]const u8{ p, "zig-cache" }); + } + } + break :blk "zig-cache"; + }, + }; + const module = Module.create(gpa, .{ - .zig_lib_dir = zig_lib_dir, + .zig_lib_directory = zig_lib_directory, + .zig_cache_directory = zig_cache_directory, .root_name = root_name, .target = target_info.target, .is_native_os = cross_target.isNativeOs(), .dynamic_linker = target_info.dynamic_linker.get(), .output_mode = output_mode, .root_pkg = root_pkg, - .bin_file_dir_path = null, - .bin_file_dir = fs.cwd(), - .bin_file_path = bin_path, + .emit_bin = emit_bin_loc, + .emit_h = emit_h_loc, .link_mode = link_mode, .object_format = object_format, .optimize_mode = build_mode, @@ -1049,11 +1135,12 @@ pub fn buildOutputType( .framework_dirs = framework_dirs.items, .frameworks = frameworks.items, .system_libs = system_libs.items, - .emit_h = emit_h_path, .link_libc = link_libc, .link_libcpp = link_libcpp, .want_pic = want_pic, .want_sanitize_c = want_sanitize_c, + .want_stack_check = want_stack_check, + .want_valgrind = want_valgrind, .use_llvm = use_llvm, .use_lld = use_lld, .use_clang = use_clang, @@ -1070,11 +1157,14 @@ pub fn buildOutputType( .link_eh_frame_hdr = link_eh_frame_hdr, .stack_size_override = stack_size_override, .strip = strip, + .single_threaded = single_threaded, .self_exe_path = self_exe_path, .rand = &default_prng.random, .clang_passthrough_mode = arg_mode != .build, .version = version, .libc_installation = if (libc_installation) |*lci| lci else null, + .debug_cc = debug_cc, + .debug_link = debug_link, }) catch |err| { fatal("unable to create module: {}", .{@errorName(err)}); }; @@ -1121,9 +1211,7 @@ pub fn buildOutputType( } fn updateModule(gpa: *Allocator, module: *Module, zir_out_path: ?[]const u8) !void { - var timer = try std.time.Timer.start(); try module.update(); - const update_nanos = timer.read(); var errors = try module.getAllErrorsAlloc(); defer errors.deinit(module.gpa); diff --git a/src-self-hosted/print_env.zig b/src-self-hosted/print_env.zig index 907e9e234d..d1956911e9 100644 --- a/src-self-hosted/print_env.zig +++ b/src-self-hosted/print_env.zig @@ -2,15 +2,19 @@ const std = @import("std"); const build_options = @import("build_options"); const introspect = @import("introspect.zig"); const Allocator = std.mem.Allocator; +const fatal = @import("main.zig").fatal; pub fn cmdEnv(gpa: *Allocator, args: []const []const u8, stdout: anytype) !void { - const zig_lib_dir = introspect.resolveZigLibDir(gpa) catch |err| { - std.debug.print("unable to find zig installation directory: {}\n", .{@errorName(err)}); - std.process.exit(1); - }; - defer gpa.free(zig_lib_dir); + const self_exe_path = try std.fs.selfExePathAlloc(gpa); + defer gpa.free(self_exe_path); - const zig_std_dir = try std.fs.path.join(gpa, &[_][]const u8{ zig_lib_dir, "std" }); + var zig_lib_directory = introspect.findZigLibDirFromSelfExe(gpa, self_exe_path) catch |err| { + fatal("unable to find zig installation directory: {}\n", .{@errorName(err)}); + }; + defer gpa.free(zig_lib_directory.path.?); + defer zig_lib_directory.handle.close(); + + const zig_std_dir = try std.fs.path.join(gpa, &[_][]const u8{ zig_lib_directory.path.?, "std" }); defer gpa.free(zig_std_dir); const global_cache_dir = try introspect.resolveGlobalCacheDir(gpa); @@ -22,8 +26,11 @@ pub fn cmdEnv(gpa: *Allocator, args: []const []const u8, stdout: anytype) !void var jws = std.json.WriteStream(@TypeOf(bos_stream), 6).init(bos_stream); try jws.beginObject(); + try jws.objectField("zig_exe"); + try jws.emitString(self_exe_path); + try jws.objectField("lib_dir"); - try jws.emitString(zig_lib_dir); + try jws.emitString(zig_lib_directory.path.?); try jws.objectField("std_dir"); try jws.emitString(zig_std_dir); diff --git a/src-self-hosted/print_targets.zig b/src-self-hosted/print_targets.zig index e6ed33b80c..724cb7a9ac 100644 --- a/src-self-hosted/print_targets.zig +++ b/src-self-hosted/print_targets.zig @@ -17,16 +17,14 @@ pub fn cmdTargets( stdout: anytype, native_target: Target, ) !void { - const zig_lib_dir_path = introspect.resolveZigLibDir(allocator) catch |err| { + var zig_lib_directory = introspect.findZigLibDir(allocator) catch |err| { fatal("unable to find zig installation directory: {}\n", .{@errorName(err)}); }; - defer allocator.free(zig_lib_dir_path); + defer zig_lib_directory.handle.close(); + defer allocator.free(zig_lib_directory.path.?); - var zig_lib_dir = try fs.cwd().openDir(zig_lib_dir_path, .{}); - defer zig_lib_dir.close(); - - const glibc_abi = try glibc.loadMetaData(allocator, zig_lib_dir); - errdefer glibc_abi.destroy(allocator); + const glibc_abi = try glibc.loadMetaData(allocator, zig_lib_directory.handle); + defer glibc_abi.destroy(allocator); var bos = io.bufferedOutStream(stdout); const bos_stream = bos.outStream(); diff --git a/src-self-hosted/target.zig b/src-self-hosted/target.zig index 127abeeadc..b09c363d8a 100644 --- a/src-self-hosted/target.zig +++ b/src-self-hosted/target.zig @@ -117,10 +117,10 @@ pub fn cannotDynamicLink(target: std.Target) bool { }; } +/// On Darwin, we always link libSystem which contains libc. +/// Similarly on FreeBSD and NetBSD we always link system libc +/// since this is the stable syscall interface. pub fn osRequiresLibC(target: std.Target) bool { - // On Darwin, we always link libSystem which contains libc. - // Similarly on FreeBSD and NetBSD we always link system libc - // since this is the stable syscall interface. return switch (target.os.tag) { .freebsd, .netbsd, .dragonfly, .macosx, .ios, .watchos, .tvos => true, else => false, @@ -131,6 +131,40 @@ pub fn requiresPIE(target: std.Target) bool { return target.isAndroid(); } +/// This function returns whether non-pic code is completely invalid on the given target. +pub fn requiresPIC(target: std.Target, linking_libc: bool) bool { + return target.isAndroid() or + target.os.tag == .windows or target.os.tag == .uefi or + osRequiresLibC(target) or + (linking_libc and target.isGnuLibC()); +} + +/// This is not whether the target supports Position Independent Code, but whether the -fPIC +/// C compiler argument is valid to Clang. +pub fn supports_fpic(target: std.Target) bool { + return target.os.tag != .windows; +} + pub fn libc_needs_crti_crtn(target: std.Target) bool { return !(target.cpu.arch.isRISCV() or target.isAndroid()); } + +pub fn isSingleThreaded(target: std.Target) bool { + return target.isWasm(); +} + +/// Valgrind supports more, but Zig does not support them yet. +pub fn hasValgrindSupport(target: std.Target) bool { + switch (target.cpu.arch) { + .x86_64 => { + return target.os.tag == .linux or target.isDarwin() or target.os.tag == .solaris or + (target.os.tag == .windows and target.abi != .msvc); + }, + else => return false, + } +} + +pub fn supportsStackProbing(target: std.Target) bool { + return target.os.tag != .windows and target.os.tag != .uefi and + (target.cpu.arch == .i386 or target.cpu.arch == .x86_64); +} diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index f23695257b..3fd6aa3bab 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -407,8 +407,9 @@ pub const TestContext = struct { const root_node = try progress.start("tests", self.cases.items.len); defer root_node.end(); - const zig_lib_dir = try introspect.resolveZigLibDir(std.testing.allocator); - defer std.testing.allocator.free(zig_lib_dir); + var zig_lib_directory = try introspect.findZigLibDir(std.testing.allocator); + defer zig_lib_directory.handle.close(); + defer std.testing.allocator.free(zig_lib_directory.path.?); const random_seed = blk: { var random_seed: u64 = undefined; @@ -427,7 +428,7 @@ pub const TestContext = struct { progress.initial_delay_ns = 0; progress.refresh_rate_ns = 0; - try self.runOneCase(std.testing.allocator, &prg_node, case, zig_lib_dir, &default_prng.random); + try self.runOneCase(std.testing.allocator, &prg_node, case, zig_lib_directory, &default_prng.random); } } @@ -436,7 +437,7 @@ pub const TestContext = struct { allocator: *Allocator, root_node: *std.Progress.Node, case: Case, - zig_lib_dir: []const u8, + zig_lib_directory: Module.Directory, rand: *std.rand.Random, ) !void { const target_info = try std.zig.system.NativeTargetInfo.detect(allocator, case.target); @@ -449,15 +450,32 @@ pub const TestContext = struct { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); + var cache_dir = try tmp.dir.makeOpenPath("zig-cache", .{}); + defer cache_dir.close(); + const bogus_path = "bogus"; // TODO this will need to be fixed before we can test LLVM extensions + const zig_cache_directory: Module.Directory = .{ + .handle = cache_dir, + .path = try std.fs.path.join(arena, &[_][]const u8{ bogus_path, "zig-cache" }), + }; + const tmp_src_path = if (case.extension == .Zig) "test_case.zig" else if (case.extension == .ZIR) "test_case.zir" else unreachable; const root_pkg = try Package.create(allocator, tmp.dir, ".", tmp_src_path); - defer root_pkg.destroy(); + defer root_pkg.destroy(allocator); const ofmt: ?std.builtin.ObjectFormat = if (case.cbe) .c else null; const bin_name = try std.zig.binNameAlloc(arena, "test_case", target, case.output_mode, null, ofmt); + const emit_directory: Module.Directory = .{ + .path = bogus_path, + .handle = tmp.dir, + }; + const emit_bin: Module.EmitLoc = .{ + .directory = emit_directory, + .basename = bin_name, + }; const module = try Module.create(allocator, .{ - .zig_lib_dir = zig_lib_dir, + .zig_cache_directory = zig_cache_directory, + .zig_lib_directory = zig_lib_directory, .rand = rand, .root_name = "test_case", .target = target, @@ -467,8 +485,7 @@ pub const TestContext = struct { .output_mode = case.output_mode, // TODO: support testing optimizations .optimize_mode = .Debug, - .bin_file_dir = tmp.dir, - .bin_file_path = bin_name, + .emit_bin = emit_bin, .root_pkg = root_pkg, .keep_source_files_loaded = true, .object_format = ofmt, From 1baa56a25f7b4cf7b3e0a78ded90c432a40c4efa Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2020 18:04:17 -0700 Subject: [PATCH 019/210] std.cache_hash: break up the API and improve implementation into smaller exposed components and expose all of them. This makes it more flexible. `*const Cache` is now passed in with an open manifest dir handle which the caller is responsible for managing. Expose some of the base64 stuff. Extract the hash helper functions into `HashHelper` and add some more methods such as addOptional and addListOfFiles. Add `CacheHash.toOwnedLock` so that you can deinitialize everything except the open file handle which represents the file system lock on the build artifacts. Use ArrayListUnmanaged, saving space per allocated CacheHash. Avoid 1 memory allocation in hit() with a static buffer. hit() returns a bool; caller code is responsible for calling final() in either case. This is a simpler and easier to use API. writeManifest() is no longer called from deinit() with errors ignored. --- lib/std/cache_hash.zig | 408 ++++++++++++++++++++++++----------------- 1 file changed, 244 insertions(+), 164 deletions(-) diff --git a/lib/std/cache_hash.zig b/lib/std/cache_hash.zig index 4727169407..19c82210c0 100644 --- a/lib/std/cache_hash.zig +++ b/lib/std/cache_hash.zig @@ -7,19 +7,18 @@ const std = @import("std.zig"); const crypto = std.crypto; const fs = std.fs; const base64 = std.base64; -const ArrayList = std.ArrayList; const assert = std.debug.assert; const testing = std.testing; const mem = std.mem; const fmt = std.fmt; const Allocator = std.mem.Allocator; -const base64_encoder = fs.base64_encoder; -const base64_decoder = fs.base64_decoder; +pub const base64_encoder = fs.base64_encoder; +pub const base64_decoder = fs.base64_decoder; /// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 /// We round up to 18 to avoid the `==` padding after base64 encoding. -const BIN_DIGEST_LEN = 18; -const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN); +pub const BIN_DIGEST_LEN = 18; +pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN); const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024; @@ -51,87 +50,105 @@ pub const File = struct { } }; -/// CacheHash manages project-local `zig-cache` directories. -/// This is not a general-purpose cache. -/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input. -pub const CacheHash = struct { - allocator: *Allocator, - /// Current state for incremental hashing. - hasher: Hasher, +pub const Cache = struct { + gpa: *Allocator, manifest_dir: fs.Dir, - manifest_file: ?fs.File, - manifest_dirty: bool, - owns_manifest_dir: bool, - files: ArrayList(File), - b64_digest: [BASE64_DIGEST_LEN]u8, + hash: HashHelper = .{}, - /// Be sure to call release after successful initialization. - pub fn init(allocator: *Allocator, dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash { + /// Be sure to call `CacheHash.deinit` after successful initialization. + pub fn obtain(cache: *const Cache) CacheHash { return CacheHash{ - .allocator = allocator, - .hasher = hasher_init, - .manifest_dir = try dir.makeOpenPath(manifest_dir_path, .{}), + .cache = cache, + .hash = cache.hash, .manifest_file = null, .manifest_dirty = false, - .owns_manifest_dir = true, - .files = ArrayList(File).init(allocator), .b64_digest = undefined, }; } +}; - /// Allows one to fork a CacheHash instance into another one, which does not require an additional - /// directory handle to be opened. The new instance inherits the hash state. - pub fn clone(self: CacheHash) CacheHash { - assert(self.manifest_file == null); - assert(files.items.len == 0); - return .{ - .allocator = self.allocator, - .hasher = self.hasher, - .manifest_dir = self.manifest_dir, - .manifest_file = null, - .manifest_dirty = false, - .owns_manifest_dir = false, - .files = ArrayList(File).init(allocator), - .b64_digest = undefined, - }; - } +pub const HashHelper = struct { + hasher: Hasher = hasher_init, /// Record a slice of bytes as an dependency of the process being cached - pub fn addBytes(self: *CacheHash, bytes: []const u8) void { - assert(self.manifest_file == null); - - self.hasher.update(mem.asBytes(&bytes.len)); - self.hasher.update(bytes); + pub fn addBytes(hh: *HashHelper, bytes: []const u8) void { + hh.hasher.update(mem.asBytes(&bytes.len)); + hh.hasher.update(bytes); } - pub fn addListOfBytes(self: *CacheHash, list_of_bytes: []const []const u8) void { - assert(self.manifest_file == null); + pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void { + hh.add(optional_bytes != null); + hh.addBytes(optional_bytes orelse return); + } - self.add(list_of_bytes.items.len); - for (list_of_bytes) |bytes| self.addBytes(bytes); + pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void { + hh.add(list_of_bytes.items.len); + for (list_of_bytes) |bytes| hh.addBytes(bytes); } /// Convert the input value into bytes and record it as a dependency of the process being cached. - pub fn add(self: *CacheHash, x: anytype) void { - assert(self.manifest_file == null); - + pub fn add(hh: *HashHelper, x: anytype) void { switch (@TypeOf(x)) { std.builtin.Version => { - self.add(x.major); - self.add(x.minor); - self.add(x.patch); + hh.add(x.major); + hh.add(x.minor); + hh.add(x.patch); return; }, else => {}, } switch (@typeInfo(@TypeOf(x))) { - .Bool, .Int, .Enum, .Array => self.addBytes(mem.asBytes(&x)), + .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)), else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))), } } - /// Add a file as a dependency of process being cached. When `CacheHash.hit` is + pub fn addOptional(hh: *HashHelper, optional: anytype) void { + hh.add(optional != null); + hh.add(optional orelse return); + } + + /// Returns a base64 encoded hash of the inputs, without modifying state. + pub fn peek(hh: HashHelper) [BASE64_DIGEST_LEN]u8 { + var copy = hh; + return copy.final(); + } + + /// Returns a base64 encoded hash of the inputs, mutating the state of the hasher. + pub fn final(hh: *HashHelper) [BASE64_DIGEST_LEN]u8 { + var bin_digest: [BIN_DIGEST_LEN]u8 = undefined; + hh.hasher.final(&bin_digest); + + var out_digest: [BASE64_DIGEST_LEN]u8 = undefined; + base64_encoder.encode(&out_digest, &bin_digest); + + return out_digest; + } +}; + +pub const Lock = struct { + manifest_file: fs.File, + + pub fn release(lock: *Lock) void { + lock.manifest_file.close(); + lock.* = undefined; + } +}; + +/// CacheHash manages project-local `zig-cache` directories. +/// This is not a general-purpose cache. +/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input. +pub const CacheHash = struct { + cache: *const Cache, + /// Current state for incremental hashing. + hash: HashHelper, + manifest_file: ?fs.File, + manifest_dirty: bool, + files: std.ArrayListUnmanaged(File) = .{}, + b64_digest: [BASE64_DIGEST_LEN]u8, + + /// Add a file as a dependency of process being cached. When `hit` is /// called, the file's contents will be checked to ensure that it matches /// the contents from previous times. /// @@ -139,8 +156,8 @@ pub const CacheHash = struct { /// are allowed to take up in memory. If max_file_size is null, then the contents /// will not be loaded into memory. /// - /// Returns the index of the entry in the `CacheHash.files` ArrayList. You can use it - /// to access the contents of the file after calling `CacheHash.hit()` like so: + /// Returns the index of the entry in the `files` array list. You can use it + /// to access the contents of the file after calling `hit()` like so: /// /// ``` /// var file_contents = cache_hash.files.items[file_index].contents.?; @@ -148,8 +165,8 @@ pub const CacheHash = struct { pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize { assert(self.manifest_file == null); - try self.files.ensureCapacity(self.files.items.len + 1); - const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path}); + try self.files.ensureCapacity(self.cache.gpa, self.files.items.len + 1); + const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path}); const idx = self.files.items.len; self.files.addOneAssumeCapacity().* = .{ @@ -160,35 +177,53 @@ pub const CacheHash = struct { .bin_digest = undefined, }; - self.addBytes(resolved_path); + self.hash.addBytes(resolved_path); return idx; } - /// Check the cache to see if the input exists in it. If it exists, a base64 encoding - /// of it's hash will be returned; otherwise, null will be returned. + pub fn addOptionalFile(self: *CacheHash, optional_file_path: ?[]const u8) !void { + self.hash.add(optional_file_path != null); + const file_path = optional_file_path orelse return; + _ = try self.addFile(file_path, null); + } + + pub fn addListOfFiles(self: *CacheHash, list_of_files: []const []const u8) !void { + self.hash.add(list_of_files.len); + for (list_of_files) |file_path| { + _ = try self.addFile(file_path, null); + } + } + + /// Check the cache to see if the input exists in it. If it exists, returns `true`. + /// A base64 encoding of its hash is available by calling `final`. /// /// This function will also acquire an exclusive lock to the manifest file. This means /// that a process holding a CacheHash will block any other process attempting to /// acquire the lock. /// - /// The lock on the manifest file is released when `CacheHash.release` is called. - pub fn hit(self: *CacheHash) !?[BASE64_DIGEST_LEN]u8 { + /// The lock on the manifest file is released when `deinit` is called. As another + /// option, one may call `toOwnedLock` to obtain a smaller object which can represent + /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called. + pub fn hit(self: *CacheHash) !bool { assert(self.manifest_file == null); + const ext = ".txt"; + var manifest_file_path: [self.b64_digest.len + ext.len]u8 = undefined; + var bin_digest: [BIN_DIGEST_LEN]u8 = undefined; - self.hasher.final(&bin_digest); + self.hash.hasher.final(&bin_digest); base64_encoder.encode(self.b64_digest[0..], &bin_digest); - self.hasher = hasher_init; - self.hasher.update(&bin_digest); + self.hash.hasher = hasher_init; + self.hash.hasher.update(&bin_digest); - const manifest_file_path = try fmt.allocPrint(self.allocator, "{}.txt", .{self.b64_digest}); - defer self.allocator.free(manifest_file_path); + mem.copy(u8, &manifest_file_path, &self.b64_digest); + manifest_file_path[self.b64_digest.len..][0..ext.len].* = ext.*; if (self.files.items.len != 0) { - self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{ + self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ .read = true, .truncate = false, .lock = .Exclusive, @@ -196,26 +231,26 @@ pub const CacheHash = struct { } else { // If there are no file inputs, we check if the manifest file exists instead of // comparing the hashes on the files used for the cached item - self.manifest_file = self.manifest_dir.openFile(manifest_file_path, .{ + self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{ .read = true, .write = true, .lock = .Exclusive, }) catch |err| switch (err) { error.FileNotFound => { self.manifest_dirty = true; - self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{ + self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ .read = true, .truncate = false, .lock = .Exclusive, }); - return null; + return false; }, else => |e| return e, }; } - const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.allocator, MANIFEST_FILE_SIZE_MAX); - defer self.allocator.free(file_contents); + const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, MANIFEST_FILE_SIZE_MAX); + defer self.cache.gpa.free(file_contents); const input_file_count = self.files.items.len; var any_file_changed = false; @@ -225,7 +260,7 @@ pub const CacheHash = struct { defer idx += 1; const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: { - const new = try self.files.addOne(); + const new = try self.files.addOne(self.cache.gpa); new.* = .{ .path = null, .contents = null, @@ -258,7 +293,7 @@ pub const CacheHash = struct { } if (cache_hash_file.path == null) { - cache_hash_file.path = try self.allocator.dupe(u8, file_path); + cache_hash_file.path = try self.cache.gpa.dupe(u8, file_path); } const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch { @@ -292,7 +327,7 @@ pub const CacheHash = struct { } if (!any_file_changed) { - self.hasher.update(&cache_hash_file.bin_digest); + self.hash.hasher.update(&cache_hash_file.bin_digest); } } @@ -300,19 +335,19 @@ pub const CacheHash = struct { // cache miss // keep the manifest file open // reset the hash - self.hasher = hasher_init; - self.hasher.update(&bin_digest); + self.hash.hasher = hasher_init; + self.hash.hasher.update(&bin_digest); // Remove files not in the initial hash for (self.files.items[input_file_count..]) |*file| { - file.deinit(self.allocator); + file.deinit(self.cache.gpa); } - self.files.shrink(input_file_count); + self.files.shrinkRetainingCapacity(input_file_count); for (self.files.items) |file| { - self.hasher.update(&file.bin_digest); + self.hash.hasher.update(&file.bin_digest); } - return null; + return false; } if (idx < input_file_count) { @@ -321,10 +356,10 @@ pub const CacheHash = struct { const ch_file = &self.files.items[idx]; try self.populateFileHash(ch_file); } - return null; + return false; } - return self.final(); + return true; } fn populateFileHash(self: *CacheHash, ch_file: *File) !void { @@ -343,8 +378,8 @@ pub const CacheHash = struct { return error.FileTooBig; } - const contents = try self.allocator.alloc(u8, @intCast(usize, ch_file.stat.size)); - errdefer self.allocator.free(contents); + const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size)); + errdefer self.cache.gpa.free(contents); // Hash while reading from disk, to keep the contents in the cpu cache while // doing hashing. @@ -364,7 +399,7 @@ pub const CacheHash = struct { try hashFile(file, &ch_file.bin_digest); } - self.hasher.update(&ch_file.bin_digest); + self.hash.hasher.update(&ch_file.bin_digest); } /// Add a file as a dependency of process being cached, after the initial hash has been @@ -374,10 +409,10 @@ pub const CacheHash = struct { pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 { assert(self.manifest_file != null); - const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path}); - errdefer self.allocator.free(resolved_path); + const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path}); + errdefer self.cache.gpa.free(resolved_path); - const new_ch_file = try self.files.addOne(); + const new_ch_file = try self.files.addOne(self.cache.gpa); new_ch_file.* = .{ .path = resolved_path, .max_file_size = max_file_size, @@ -385,7 +420,7 @@ pub const CacheHash = struct { .bin_digest = undefined, .contents = null, }; - errdefer self.files.shrink(self.files.items.len - 1); + errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); try self.populateFileHash(new_ch_file); @@ -399,10 +434,10 @@ pub const CacheHash = struct { pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void { assert(self.manifest_file != null); - const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path}); - errdefer self.allocator.free(resolved_path); + const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path}); + errdefer self.cache.gpa.free(resolved_path); - const new_ch_file = try self.files.addOne(); + const new_ch_file = try self.files.addOne(self.cache.gpa); new_ch_file.* = .{ .path = resolved_path, .max_file_size = null, @@ -410,7 +445,7 @@ pub const CacheHash = struct { .bin_digest = undefined, .contents = null, }; - errdefer self.files.shrink(self.files.items.len - 1); + errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); try self.populateFileHash(new_ch_file); } @@ -426,7 +461,7 @@ pub const CacheHash = struct { // the artifacts to cache. var bin_digest: [BIN_DIGEST_LEN]u8 = undefined; - self.hasher.final(&bin_digest); + self.hash.hasher.final(&bin_digest); var out_digest: [BASE64_DIGEST_LEN]u8 = undefined; base64_encoder.encode(&out_digest, &bin_digest); @@ -436,45 +471,48 @@ pub const CacheHash = struct { pub fn writeManifest(self: *CacheHash) !void { assert(self.manifest_file != null); + if (!self.manifest_dirty) return; var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined; - var contents = ArrayList(u8).init(self.allocator); - var outStream = contents.outStream(); + var contents = std.ArrayList(u8).init(self.cache.gpa); + var writer = contents.writer(); defer contents.deinit(); for (self.files.items) |file| { base64_encoder.encode(encoded_digest[0..], &file.bin_digest); - try outStream.print("{} {} {} {} {}\n", .{ file.stat.size, file.stat.inode, file.stat.mtime, encoded_digest[0..], file.path }); + try writer.print("{} {} {} {} {}\n", .{ + file.stat.size, + file.stat.inode, + file.stat.mtime, + encoded_digest[0..], + file.path, + }); } try self.manifest_file.?.pwriteAll(contents.items, 0); self.manifest_dirty = false; } + /// Obtain only the data needed to maintain a lock on the manifest file. + /// The `CacheHash` remains safe to deinit. + /// Don't forget to call `writeManifest` before this! + pub fn toOwnedLock(self: *CacheHash) Lock { + const manifest_file = self.manifest_file.?; + self.manifest_file = null; + return Lock{ .manifest_file = manifest_file }; + } + /// Releases the manifest file and frees any memory the CacheHash was using. /// `CacheHash.hit` must be called first. - /// - /// Will also attempt to write to the manifest file if the manifest is dirty. - /// Writing to the manifest file can fail, but this function ignores those errors. - /// To detect failures from writing the manifest, one may explicitly call - /// `writeManifest` before `release`. - pub fn release(self: *CacheHash) void { + /// Don't forget to call `writeManifest` before this! + pub fn deinit(self: *CacheHash) void { if (self.manifest_file) |file| { - if (self.manifest_dirty) { - // To handle these errors, API users should call - // writeManifest before release(). - self.writeManifest() catch {}; - } - file.close(); } - for (self.files.items) |*file| { - file.deinit(self.allocator); + file.deinit(self.cache.gpa); } - self.files.deinit(); - if (self.owns_manifest_dir) - self.manifest_dir.close(); + self.files.deinit(self.cache.gpa); } }; @@ -542,31 +580,41 @@ test "cache file and then recall it" { var digest1: [BASE64_DIGEST_LEN]u8 = undefined; var digest2: [BASE64_DIGEST_LEN]u8 = undefined; - { - var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); - defer ch.release(); + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + defer cache.manifest_dir.close(); - ch.add(true); - ch.add(@as(u16, 1234)); - ch.addBytes("1234"); + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.add(true); + ch.hash.add(@as(u16, 1234)); + ch.hash.addBytes("1234"); _ = try ch.addFile(temp_file, null); // There should be nothing in the cache - testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit()); + testing.expectEqual(false, try ch.hit()); digest1 = ch.final(); + try ch.writeManifest(); } { - var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); - defer ch.release(); + var ch = cache.obtain(); + defer ch.deinit(); - ch.add(true); - ch.add(@as(u16, 1234)); - ch.addBytes("1234"); + ch.hash.add(true); + ch.hash.add(@as(u16, 1234)); + ch.hash.addBytes("1234"); _ = try ch.addFile(temp_file, null); // Cache hit! We just "built" the same file - digest2 = (try ch.hit()).?; + testing.expect(try ch.hit()); + digest2 = ch.final(); + + try ch.writeManifest(); } testing.expectEqual(digest1, digest2); @@ -612,37 +660,47 @@ test "check that changing a file makes cache fail" { var digest1: [BASE64_DIGEST_LEN]u8 = undefined; var digest2: [BASE64_DIGEST_LEN]u8 = undefined; - { - var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); - defer ch.release(); + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + defer cache.manifest_dir.close(); - ch.addBytes("1234"); + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); const temp_file_idx = try ch.addFile(temp_file, 100); // There should be nothing in the cache - testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit()); + testing.expectEqual(false, try ch.hit()); testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?)); digest1 = ch.final(); + + try ch.writeManifest(); } try cwd.writeFile(temp_file, updated_temp_file_contents); { - var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); - defer ch.release(); + var ch = cache.obtain(); + defer ch.deinit(); - ch.addBytes("1234"); + ch.hash.addBytes("1234"); const temp_file_idx = try ch.addFile(temp_file, 100); // A file that we depend on has been updated, so the cache should not contain an entry for it - testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit()); + testing.expectEqual(false, try ch.hit()); // The cache system does not keep the contents of re-hashed input files. testing.expect(ch.files.items[temp_file_idx].contents == null); digest2 = ch.final(); + + try ch.writeManifest(); } testing.expect(!mem.eql(u8, digest1[0..], digest2[0..])); @@ -663,24 +721,34 @@ test "no file inputs" { var digest1: [BASE64_DIGEST_LEN]u8 = undefined; var digest2: [BASE64_DIGEST_LEN]u8 = undefined; - { - var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); - defer ch.release(); + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + defer cache.manifest_dir.close(); - ch.addBytes("1234"); + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); // There should be nothing in the cache - testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit()); + testing.expectEqual(false, try ch.hit()); digest1 = ch.final(); + + try ch.writeManifest(); } { - var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); - defer ch.release(); + var ch = cache.obtain(); + defer ch.deinit(); - ch.addBytes("1234"); + ch.hash.addBytes("1234"); - digest2 = (try ch.hit()).?; + testing.expect(try ch.hit()); + digest2 = ch.final(); + try ch.writeManifest(); } testing.expectEqual(digest1, digest2); @@ -709,28 +777,38 @@ test "CacheHashes with files added after initial hash work" { var digest2: [BASE64_DIGEST_LEN]u8 = undefined; var digest3: [BASE64_DIGEST_LEN]u8 = undefined; - { - var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); - defer ch.release(); + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + defer cache.manifest_dir.close(); - ch.addBytes("1234"); + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); _ = try ch.addFile(temp_file1, null); // There should be nothing in the cache - testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit()); + testing.expectEqual(false, try ch.hit()); _ = try ch.addFilePost(temp_file2); digest1 = ch.final(); + try ch.writeManifest(); } { - var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); - defer ch.release(); + var ch = cache.obtain(); + defer ch.deinit(); - ch.addBytes("1234"); + ch.hash.addBytes("1234"); _ = try ch.addFile(temp_file1, null); - digest2 = (try ch.hit()).?; + testing.expect(try ch.hit()); + digest2 = ch.final(); + + try ch.writeManifest(); } testing.expect(mem.eql(u8, &digest1, &digest2)); @@ -743,18 +821,20 @@ test "CacheHashes with files added after initial hash work" { } { - var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir); - defer ch.release(); + var ch = cache.obtain(); + defer ch.deinit(); - ch.addBytes("1234"); + ch.hash.addBytes("1234"); _ = try ch.addFile(temp_file1, null); // A file that we depend on has been updated, so the cache should not contain an entry for it - testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit()); + testing.expectEqual(false, try ch.hit()); _ = try ch.addFilePost(temp_file2); digest3 = ch.final(); + + try ch.writeManifest(); } testing.expect(!mem.eql(u8, &digest1, &digest3)); From 2a8fc1a18e7d9017262f5c7ee9669ca7d80ebaa6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2020 19:17:58 -0700 Subject: [PATCH 020/210] stage2: caching system integration & Module/Compilation splitting * update to the new cache hash API * std.Target defaultVersionRange moves to std.Target.Os.Tag * std.Target.Os gains getVersionRange which returns a tagged union * start the process of splitting Module into Compilation and "zig module". - The parts of Module having to do with only compiling zig code are extracted into ZigModule.zig. - Next step is to rename Module to Compilation. - After that rename ZigModule back to Module. * implement proper cache hash usage when compiling C objects, and properly manage the file lock of the build artifacts. * make versions optional to match recent changes to master branch. * proper cache hash integration for compiling zig code * proper cache hash integration for linking even when not compiling zig code. * ELF LLD linking integrates with the caching system. A comment from the source code: Here we want to determine whether we can save time by not invoking LLD when the output is unchanged. None of the linker options or the object files that are being linked are in the hash that namespaces the directory we are outputting to. Therefore, we must hash those now, and the resulting digest will form the "id" of the linking job we are about to perform. After a successful link, we store the id in the metadata of a symlink named "id.txt" in the artifact directory. So, now, we check if this symlink exists, and if it matches our digest. If so, we can skip linking. Otherwise, we proceed with invoking LLD. * implement disable_c_depfile option * add tracy to a few more functions --- lib/std/cache_hash.zig | 2 +- lib/std/target.zig | 38 +- lib/std/zig/cross_target.zig | 2 +- lib/std/zig/system.zig | 2 +- src-self-hosted/Module.zig | 3951 ++++-------------------------- src-self-hosted/ZigModule.zig | 3238 ++++++++++++++++++++++++ src-self-hosted/astgen.zig | 2 +- src-self-hosted/codegen.zig | 7 +- src-self-hosted/codegen/c.zig | 2 +- src-self-hosted/codegen/wasm.zig | 3 +- src-self-hosted/glibc.zig | 16 +- src-self-hosted/ir.zig | 2 +- src-self-hosted/link.zig | 39 +- src-self-hosted/link/C.zig | 9 +- src-self-hosted/link/Coff.zig | 9 +- src-self-hosted/link/Elf.zig | 221 +- src-self-hosted/link/MachO.zig | 5 +- src-self-hosted/link/Wasm.zig | 7 +- src-self-hosted/main.zig | 10 +- src-self-hosted/test.zig | 2 +- src-self-hosted/type.zig | 2 +- src-self-hosted/value.zig | 2 +- src-self-hosted/zir.zig | 2 +- src-self-hosted/zir_sema.zig | 8 +- test/stage2/test.zig | 7 +- 25 files changed, 3940 insertions(+), 3648 deletions(-) create mode 100644 src-self-hosted/ZigModule.zig diff --git a/lib/std/cache_hash.zig b/lib/std/cache_hash.zig index 19c82210c0..2075bedc22 100644 --- a/lib/std/cache_hash.zig +++ b/lib/std/cache_hash.zig @@ -82,7 +82,7 @@ pub const HashHelper = struct { } pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void { - hh.add(list_of_bytes.items.len); + hh.add(list_of_bytes.len); for (list_of_bytes) |bytes| hh.addBytes(bytes); } diff --git a/lib/std/target.zig b/lib/std/target.zig index a245868dc0..1b8e7c1519 100644 --- a/lib/std/target.zig +++ b/lib/std/target.zig @@ -75,6 +75,13 @@ pub const Target = struct { else => return ".so", } } + + pub fn defaultVersionRange(tag: Tag) Os { + return .{ + .tag = tag, + .version_range = VersionRange.default(tag), + }; + } }; /// Based on NTDDI version constants from @@ -290,11 +297,32 @@ pub const Target = struct { } }; - pub fn defaultVersionRange(tag: Tag) Os { - return .{ - .tag = tag, - .version_range = VersionRange.default(tag), - }; + pub const TaggedVersionRange = union(enum) { + none: void, + semver: Version.Range, + linux: LinuxVersionRange, + windows: WindowsVersion.Range, + }; + + /// Provides a tagged union. `Target` does not store the tag because it is + /// redundant with the OS tag; this function abstracts that part away. + pub fn getVersionRange(self: Os) TaggedVersionRange { + switch (self.tag) { + .linux => return TaggedVersionRange{ .linux = self.version_range.linux }, + .windows => return TaggedVersionRange{ .windows = self.version_range.windows }, + + .freebsd, + .macosx, + .ios, + .tvos, + .watchos, + .netbsd, + .openbsd, + .dragonfly, + => return TaggedVersionRange{ .semver = self.version_range.semver }, + + else => return .none, + } } /// Checks if system is guaranteed to be at least `version` or older than `version`. diff --git a/lib/std/zig/cross_target.zig b/lib/std/zig/cross_target.zig index 75fc5969a5..f94598c30c 100644 --- a/lib/std/zig/cross_target.zig +++ b/lib/std/zig/cross_target.zig @@ -375,7 +375,7 @@ pub const CrossTarget = struct { // `Target.current.os` works when doing `zig build` because Zig generates a build executable using // native OS version range. However this will not be accurate otherwise, and // will need to be integrated with `std.zig.system.NativeTargetInfo.detect`. - var adjusted_os = if (self.os_tag) |os_tag| Target.Os.defaultVersionRange(os_tag) else Target.current.os; + var adjusted_os = if (self.os_tag) |os_tag| os_tag.defaultVersionRange() else Target.current.os; if (self.os_version_min) |min| switch (min) { .none => {}, diff --git a/lib/std/zig/system.zig b/lib/std/zig/system.zig index 37eea36d47..57d0449379 100644 --- a/lib/std/zig/system.zig +++ b/lib/std/zig/system.zig @@ -203,7 +203,7 @@ pub const NativeTargetInfo = struct { /// deinitialization method. /// TODO Remove the Allocator requirement from this function. pub fn detect(allocator: *Allocator, cross_target: CrossTarget) DetectError!NativeTargetInfo { - var os = Target.Os.defaultVersionRange(cross_target.getOsTag()); + var os = cross_target.getOsTag().defaultVersionRange(); if (cross_target.os_tag == null) { switch (Target.current.os.tag) { .linux => { diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 4c48837d9e..557b5dcfcd 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -1,90 +1,39 @@ +//! TODO This is going to get renamed from Module to Compilation. +const Module = @This(); +const Compilation = @This(); + const std = @import("std"); const mem = std.mem; const Allocator = std.mem.Allocator; -const ArrayListUnmanaged = std.ArrayListUnmanaged; const Value = @import("value.zig").Value; -const Type = @import("type.zig").Type; -const TypedValue = @import("TypedValue.zig"); const assert = std.debug.assert; -const log = std.log.scoped(.module); -const BigIntConst = std.math.big.int.Const; -const BigIntMutable = std.math.big.int.Mutable; +const log = std.log.scoped(.compilation); const Target = std.Target; const target_util = @import("target.zig"); const Package = @import("Package.zig"); const link = @import("link.zig"); -const ir = @import("ir.zig"); -const zir = @import("zir.zig"); -const Module = @This(); -const Inst = ir.Inst; -const Body = ir.Body; -const ast = std.zig.ast; const trace = @import("tracy.zig").trace; const liveness = @import("liveness.zig"); -const astgen = @import("astgen.zig"); -const zir_sema = @import("zir_sema.zig"); const build_options = @import("build_options"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const glibc = @import("glibc.zig"); const fatal = @import("main.zig").fatal; +const ZigModule = @import("ZigModule.zig"); /// General-purpose allocator. Used for both temporary and long-term storage. gpa: *Allocator, /// Arena-allocated memory used during initialization. Should be untouched until deinit. arena_state: std.heap.ArenaAllocator.State, -/// Pointer to externally managed resource. `null` if there is no zig file being compiled. -root_pkg: ?*Package, -/// Module owns this resource. -/// The `Scope` is either a `Scope.ZIRModule` or `Scope.File`. -root_scope: *Scope, bin_file: *link.File, -/// It's rare for a decl to be exported, so we save memory by having a sparse map of -/// Decl pointers to details about them being exported. -/// The Export memory is owned by the `export_owners` table; the slice itself is owned by this table. -decl_exports: std.AutoArrayHashMapUnmanaged(*Decl, []*Export) = .{}, -/// We track which export is associated with the given symbol name for quick -/// detection of symbol collisions. -symbol_exports: std.StringArrayHashMapUnmanaged(*Export) = .{}, -/// This models the Decls that perform exports, so that `decl_exports` can be updated when a Decl -/// is modified. Note that the key of this table is not the Decl being exported, but the Decl that -/// is performing the export of another Decl. -/// This table owns the Export memory. -export_owners: std.AutoArrayHashMapUnmanaged(*Decl, []*Export) = .{}, -/// Maps fully qualified namespaced names to the Decl struct for them. -decl_table: std.ArrayHashMapUnmanaged(Scope.NameHash, *Decl, Scope.name_hash_hash, Scope.name_hash_eql, false) = .{}, - c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{}, link_error_flags: link.File.ErrorFlags = .{}, work_queue: std.fifo.LinearFifo(WorkItem, .Dynamic), -/// We optimize memory usage for a compilation with no compile errors by storing the -/// error messages and mapping outside of `Decl`. -/// The ErrorMsg memory is owned by the decl, using Module's general purpose allocator. -/// Note that a Decl can succeed but the Fn it represents can fail. In this case, -/// a Decl can have a failed_decls entry but have analysis status of success. -failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *ErrorMsg) = .{}, -/// Using a map here for consistency with the other fields here. -/// The ErrorMsg memory is owned by the `Scope`, using Module's general purpose allocator. -failed_files: std.AutoArrayHashMapUnmanaged(*Scope, *ErrorMsg) = .{}, -/// Using a map here for consistency with the other fields here. -/// The ErrorMsg memory is owned by the `Export`, using Module's general purpose allocator. -failed_exports: std.AutoArrayHashMapUnmanaged(*Export, *ErrorMsg) = .{}, /// The ErrorMsg memory is owned by the `CObject`, using Module's general purpose allocator. failed_c_objects: std.AutoArrayHashMapUnmanaged(*CObject, *ErrorMsg) = .{}, -/// Incrementing integer used to compare against the corresponding Decl -/// field to determine whether a Decl's status applies to an ongoing update, or a -/// previous analysis. -generation: u32 = 0, - -next_anon_name_index: usize = 0, - -/// Candidates for deletion. After a semantic analysis update completes, this list -/// contains Decls that need to be deleted if they end up having no references to them. -deletion_set: std.ArrayListUnmanaged(*Decl) = .{}, - keep_source_files_loaded: bool, use_clang: bool, sanitize_c: bool, @@ -95,18 +44,15 @@ sanitize_c: bool, clang_passthrough_mode: bool, /// Whether to print clang argvs to stdout. debug_cc: bool, - -/// Error tags and their values, tag names are duped with mod.gpa. -global_error_set: std.StringHashMapUnmanaged(u16) = .{}, +disable_c_depfile: bool, c_source_files: []const CSourceFile, clang_argv: []const []const u8, -cache: std.cache_hash.CacheHash, +cache_parent: *std.cache_hash.Cache, /// Path to own executable for invoking `zig clang`. self_exe_path: ?[]const u8, zig_lib_directory: Directory, zig_cache_directory: Directory, -zig_cache_artifact_directory: Directory, libc_include_dir_list: []const []const u8, rand: *std.rand.Random, @@ -128,7 +74,10 @@ libc_static_lib: ?[]const u8 = null, /// The key is the basename, and the value is the absolute path to the completed build artifact. crt_files: std.StringHashMapUnmanaged([]const u8) = .{}, -pub const InnerError = error{ OutOfMemory, AnalysisFail }; +/// Keeping track of this possibly open resource so we can close it later. +owned_link_dir: ?std.fs.Dir, + +pub const InnerError = ZigModule.InnerError; /// For passing to a C compiler. pub const CSourceFile = struct { @@ -138,14 +87,14 @@ pub const CSourceFile = struct { const WorkItem = union(enum) { /// Write the machine code for a Decl to the output file. - codegen_decl: *Decl, + codegen_decl: *ZigModule.Decl, /// The Decl needs to be analyzed and possibly export itself. /// It may have already be analyzed, or it may have been determined /// to be outdated; in this case perform semantic analysis again. - analyze_decl: *Decl, + analyze_decl: *ZigModule.Decl, /// The source file containing the Decl has been updated, and so the /// Decl may need its line number information updated in the debug info. - update_line_number: *Decl, + update_line_number: *ZigModule.Decl, /// Invoke the Clang compiler to create an object file, which gets linked /// with the Module. c_object: *CObject, @@ -156,192 +105,6 @@ const WorkItem = union(enum) { glibc_so: *const glibc.Lib, }; -pub const Export = struct { - options: std.builtin.ExportOptions, - /// Byte offset into the file that contains the export directive. - src: usize, - /// Represents the position of the export, if any, in the output file. - link: link.File.Elf.Export, - /// The Decl that performs the export. Note that this is *not* the Decl being exported. - owner_decl: *Decl, - /// The Decl being exported. Note this is *not* the Decl performing the export. - exported_decl: *Decl, - status: enum { - in_progress, - failed, - /// Indicates that the failure was due to a temporary issue, such as an I/O error - /// when writing to the output file. Retrying the export may succeed. - failed_retryable, - complete, - }, -}; - -pub const Decl = struct { - /// This name is relative to the containing namespace of the decl. It uses a null-termination - /// to save bytes, since there can be a lot of decls in a compilation. The null byte is not allowed - /// in symbol names, because executable file formats use null-terminated strings for symbol names. - /// All Decls have names, even values that are not bound to a zig namespace. This is necessary for - /// mapping them to an address in the output file. - /// Memory owned by this decl, using Module's allocator. - name: [*:0]const u8, - /// The direct parent container of the Decl. This is either a `Scope.Container` or `Scope.ZIRModule`. - /// Reference to externally owned memory. - scope: *Scope, - /// The AST Node decl index or ZIR Inst index that contains this declaration. - /// Must be recomputed when the corresponding source file is modified. - src_index: usize, - /// The most recent value of the Decl after a successful semantic analysis. - typed_value: union(enum) { - never_succeeded: void, - most_recent: TypedValue.Managed, - }, - /// Represents the "shallow" analysis status. For example, for decls that are functions, - /// the function type is analyzed with this set to `in_progress`, however, the semantic - /// analysis of the function body is performed with this value set to `success`. Functions - /// have their own analysis status field. - analysis: enum { - /// This Decl corresponds to an AST Node that has not been referenced yet, and therefore - /// because of Zig's lazy declaration analysis, it will remain unanalyzed until referenced. - unreferenced, - /// Semantic analysis for this Decl is running right now. This state detects dependency loops. - in_progress, - /// This Decl might be OK but it depends on another one which did not successfully complete - /// semantic analysis. - dependency_failure, - /// Semantic analysis failure. - /// There will be a corresponding ErrorMsg in Module.failed_decls. - sema_failure, - /// There will be a corresponding ErrorMsg in Module.failed_decls. - /// This indicates the failure was something like running out of disk space, - /// and attempting semantic analysis again may succeed. - sema_failure_retryable, - /// There will be a corresponding ErrorMsg in Module.failed_decls. - codegen_failure, - /// There will be a corresponding ErrorMsg in Module.failed_decls. - /// This indicates the failure was something like running out of disk space, - /// and attempting codegen again may succeed. - codegen_failure_retryable, - /// Everything is done. During an update, this Decl may be out of date, depending - /// on its dependencies. The `generation` field can be used to determine if this - /// completion status occurred before or after a given update. - complete, - /// A Module update is in progress, and this Decl has been flagged as being known - /// to require re-analysis. - outdated, - }, - /// This flag is set when this Decl is added to a check_for_deletion set, and cleared - /// when removed. - deletion_flag: bool, - /// Whether the corresponding AST decl has a `pub` keyword. - is_pub: bool, - - /// An integer that can be checked against the corresponding incrementing - /// generation field of Module. This is used to determine whether `complete` status - /// represents pre- or post- re-analysis. - generation: u32, - - /// Represents the position of the code in the output file. - /// This is populated regardless of semantic analysis and code generation. - link: link.File.LinkBlock, - - /// Represents the function in the linked output file, if the `Decl` is a function. - /// This is stored here and not in `Fn` because `Decl` survives across updates but - /// `Fn` does not. - /// TODO Look into making `Fn` a longer lived structure and moving this field there - /// to save on memory usage. - fn_link: link.File.LinkFn, - - contents_hash: std.zig.SrcHash, - - /// The shallow set of other decls whose typed_value could possibly change if this Decl's - /// typed_value is modified. - dependants: DepsTable = .{}, - /// The shallow set of other decls whose typed_value changing indicates that this Decl's - /// typed_value may need to be regenerated. - dependencies: DepsTable = .{}, - - /// The reason this is not `std.AutoArrayHashMapUnmanaged` is a workaround for - /// stage1 compiler giving me: `error: struct 'Module.Decl' depends on itself` - pub const DepsTable = std.ArrayHashMapUnmanaged(*Decl, void, std.array_hash_map.getAutoHashFn(*Decl), std.array_hash_map.getAutoEqlFn(*Decl), false); - - pub fn destroy(self: *Decl, gpa: *Allocator) void { - gpa.free(mem.spanZ(self.name)); - if (self.typedValueManaged()) |tvm| { - tvm.deinit(gpa); - } - self.dependants.deinit(gpa); - self.dependencies.deinit(gpa); - gpa.destroy(self); - } - - pub fn src(self: Decl) usize { - switch (self.scope.tag) { - .container => { - const container = @fieldParentPtr(Scope.Container, "base", self.scope); - const tree = container.file_scope.contents.tree; - // TODO Container should have it's own decls() - const decl_node = tree.root_node.decls()[self.src_index]; - return tree.token_locs[decl_node.firstToken()].start; - }, - .zir_module => { - const zir_module = @fieldParentPtr(Scope.ZIRModule, "base", self.scope); - const module = zir_module.contents.module; - const src_decl = module.decls[self.src_index]; - return src_decl.inst.src; - }, - .none => unreachable, - .file, .block => unreachable, - .gen_zir => unreachable, - .local_val => unreachable, - .local_ptr => unreachable, - .decl => unreachable, - } - } - - pub fn fullyQualifiedNameHash(self: Decl) Scope.NameHash { - return self.scope.fullyQualifiedNameHash(mem.spanZ(self.name)); - } - - pub fn typedValue(self: *Decl) error{AnalysisFail}!TypedValue { - const tvm = self.typedValueManaged() orelse return error.AnalysisFail; - return tvm.typed_value; - } - - pub fn value(self: *Decl) error{AnalysisFail}!Value { - return (try self.typedValue()).val; - } - - pub fn dump(self: *Decl) void { - const loc = std.zig.findLineColumn(self.scope.source.bytes, self.src); - std.debug.print("{}:{}:{} name={} status={}", .{ - self.scope.sub_file_path, - loc.line + 1, - loc.column + 1, - mem.spanZ(self.name), - @tagName(self.analysis), - }); - if (self.typedValueManaged()) |tvm| { - std.debug.print(" ty={} val={}", .{ tvm.typed_value.ty, tvm.typed_value.val }); - } - std.debug.print("\n", .{}); - } - - pub fn typedValueManaged(self: *Decl) ?*TypedValue.Managed { - switch (self.typed_value) { - .most_recent => |*x| return x, - .never_succeeded => return null, - } - } - - fn removeDependant(self: *Decl, other: *Decl) void { - self.dependants.removeAssertDiscard(other); - } - - fn removeDependency(self: *Decl, other: *Decl) void { - self.dependencies.removeAssertDiscard(other); - } -}; - pub const CObject = struct { /// Relative to cwd. Owned by arena. src_path: []const u8, @@ -350,580 +113,41 @@ pub const CObject = struct { arena: std.heap.ArenaAllocator.State, status: union(enum) { new, - /// This is the output object path. Owned by gpa. - success: []u8, - /// There will be a corresponding ErrorMsg in Module.failed_c_objects. - /// This is the C source file contents (used for printing error messages). Owned by gpa. - failure: []u8, + success: struct { + /// The outputted result. Owned by gpa. + object_path: []u8, + /// This is a file system lock on the cache hash manifest representing this + /// object. It prevents other invocations of the Zig compiler from interfering + /// with this object until released. + lock: std.cache_hash.Lock, + }, + /// There will be a corresponding ErrorMsg in Compilation.failed_c_objects. + failure, }, + /// Returns if there was failure. + pub fn clearStatus(self: *CObject, gpa: *Allocator) bool { + switch (self.status) { + .new => return false, + .failure => { + self.status = .new; + return true; + }, + .success => |*success| { + gpa.free(success.object_path); + success.lock.release(); + self.status = .new; + return false; + }, + } + } + pub fn destroy(self: *CObject, gpa: *Allocator) void { - switch (self.status) { - .new => {}, - .failure, .success => |data| gpa.free(data), - } + _ = self.clearStatus(gpa); self.arena.promote(gpa).deinit(); } }; -/// Fn struct memory is owned by the Decl's TypedValue.Managed arena allocator. -pub const Fn = struct { - /// This memory owned by the Decl's TypedValue.Managed arena allocator. - analysis: union(enum) { - queued: *ZIR, - in_progress, - /// There will be a corresponding ErrorMsg in Module.failed_decls - sema_failure, - /// This Fn might be OK but it depends on another Decl which did not successfully complete - /// semantic analysis. - dependency_failure, - success: Body, - }, - owner_decl: *Decl, - - /// This memory is temporary and points to stack memory for the duration - /// of Fn analysis. - pub const Analysis = struct { - inner_block: Scope.Block, - }; - - /// Contains un-analyzed ZIR instructions generated from Zig source AST. - pub const ZIR = struct { - body: zir.Module.Body, - arena: std.heap.ArenaAllocator.State, - }; - - /// For debugging purposes. - pub fn dump(self: *Fn, mod: Module) void { - std.debug.print("Module.Function(name={}) ", .{self.owner_decl.name}); - switch (self.analysis) { - .queued => { - std.debug.print("queued\n", .{}); - }, - .in_progress => { - std.debug.print("in_progress\n", .{}); - }, - else => { - std.debug.print("\n", .{}); - zir.dumpFn(mod, self); - }, - } - } -}; - -pub const Var = struct { - /// if is_extern == true this is undefined - init: Value, - owner_decl: *Decl, - - is_extern: bool, - is_mutable: bool, - is_threadlocal: bool, -}; - -pub const Scope = struct { - tag: Tag, - - pub const NameHash = [16]u8; - - pub fn cast(base: *Scope, comptime T: type) ?*T { - if (base.tag != T.base_tag) - return null; - - return @fieldParentPtr(T, "base", base); - } - - /// Asserts the scope has a parent which is a DeclAnalysis and - /// returns the arena Allocator. - pub fn arena(self: *Scope) *Allocator { - switch (self.tag) { - .block => return self.cast(Block).?.arena, - .decl => return &self.cast(DeclAnalysis).?.arena.allocator, - .gen_zir => return self.cast(GenZIR).?.arena, - .local_val => return self.cast(LocalVal).?.gen_zir.arena, - .local_ptr => return self.cast(LocalPtr).?.gen_zir.arena, - .zir_module => return &self.cast(ZIRModule).?.contents.module.arena.allocator, - .file => unreachable, - .container => unreachable, - .none => unreachable, - } - } - - /// If the scope has a parent which is a `DeclAnalysis`, - /// returns the `Decl`, otherwise returns `null`. - pub fn decl(self: *Scope) ?*Decl { - return switch (self.tag) { - .block => self.cast(Block).?.decl, - .gen_zir => self.cast(GenZIR).?.decl, - .local_val => self.cast(LocalVal).?.gen_zir.decl, - .local_ptr => self.cast(LocalPtr).?.gen_zir.decl, - .decl => self.cast(DeclAnalysis).?.decl, - .zir_module => null, - .file => null, - .container => null, - .none => unreachable, - }; - } - - /// Asserts the scope has a parent which is a ZIRModule or Container and - /// returns it. - pub fn namespace(self: *Scope) *Scope { - switch (self.tag) { - .block => return self.cast(Block).?.decl.scope, - .gen_zir => return self.cast(GenZIR).?.decl.scope, - .local_val => return self.cast(LocalVal).?.gen_zir.decl.scope, - .local_ptr => return self.cast(LocalPtr).?.gen_zir.decl.scope, - .decl => return self.cast(DeclAnalysis).?.decl.scope, - .file => return &self.cast(File).?.root_container.base, - .zir_module, .container => return self, - .none => unreachable, - } - } - - /// Must generate unique bytes with no collisions with other decls. - /// The point of hashing here is only to limit the number of bytes of - /// the unique identifier to a fixed size (16 bytes). - pub fn fullyQualifiedNameHash(self: *Scope, name: []const u8) NameHash { - switch (self.tag) { - .block => unreachable, - .gen_zir => unreachable, - .local_val => unreachable, - .local_ptr => unreachable, - .decl => unreachable, - .file => unreachable, - .zir_module => return self.cast(ZIRModule).?.fullyQualifiedNameHash(name), - .container => return self.cast(Container).?.fullyQualifiedNameHash(name), - .none => unreachable, - } - } - - /// Asserts the scope is a child of a File and has an AST tree and returns the tree. - pub fn tree(self: *Scope) *ast.Tree { - switch (self.tag) { - .file => return self.cast(File).?.contents.tree, - .zir_module => unreachable, - .none => unreachable, - .decl => return self.cast(DeclAnalysis).?.decl.scope.cast(Container).?.file_scope.contents.tree, - .block => return self.cast(Block).?.decl.scope.cast(Container).?.file_scope.contents.tree, - .gen_zir => return self.cast(GenZIR).?.decl.scope.cast(Container).?.file_scope.contents.tree, - .local_val => return self.cast(LocalVal).?.gen_zir.decl.scope.cast(Container).?.file_scope.contents.tree, - .local_ptr => return self.cast(LocalPtr).?.gen_zir.decl.scope.cast(Container).?.file_scope.contents.tree, - .container => return self.cast(Container).?.file_scope.contents.tree, - } - } - - /// Asserts the scope is a child of a `GenZIR` and returns it. - pub fn getGenZIR(self: *Scope) *GenZIR { - return switch (self.tag) { - .block => unreachable, - .gen_zir => self.cast(GenZIR).?, - .local_val => return self.cast(LocalVal).?.gen_zir, - .local_ptr => return self.cast(LocalPtr).?.gen_zir, - .decl => unreachable, - .zir_module => unreachable, - .file => unreachable, - .container => unreachable, - .none => unreachable, - }; - } - - /// Asserts the scope has a parent which is a ZIRModule, Contaienr or File and - /// returns the sub_file_path field. - pub fn subFilePath(base: *Scope) []const u8 { - switch (base.tag) { - .container => return @fieldParentPtr(Container, "base", base).file_scope.sub_file_path, - .file => return @fieldParentPtr(File, "base", base).sub_file_path, - .zir_module => return @fieldParentPtr(ZIRModule, "base", base).sub_file_path, - .none => unreachable, - .block => unreachable, - .gen_zir => unreachable, - .local_val => unreachable, - .local_ptr => unreachable, - .decl => unreachable, - } - } - - pub fn unload(base: *Scope, gpa: *Allocator) void { - switch (base.tag) { - .file => return @fieldParentPtr(File, "base", base).unload(gpa), - .zir_module => return @fieldParentPtr(ZIRModule, "base", base).unload(gpa), - .none => {}, - .block => unreachable, - .gen_zir => unreachable, - .local_val => unreachable, - .local_ptr => unreachable, - .decl => unreachable, - .container => unreachable, - } - } - - pub fn getSource(base: *Scope, module: *Module) ![:0]const u8 { - switch (base.tag) { - .container => return @fieldParentPtr(Container, "base", base).file_scope.getSource(module), - .file => return @fieldParentPtr(File, "base", base).getSource(module), - .zir_module => return @fieldParentPtr(ZIRModule, "base", base).getSource(module), - .none => unreachable, - .gen_zir => unreachable, - .local_val => unreachable, - .local_ptr => unreachable, - .block => unreachable, - .decl => unreachable, - } - } - - /// Asserts the scope is a namespace Scope and removes the Decl from the namespace. - pub fn removeDecl(base: *Scope, child: *Decl) void { - switch (base.tag) { - .container => return @fieldParentPtr(Container, "base", base).removeDecl(child), - .zir_module => return @fieldParentPtr(ZIRModule, "base", base).removeDecl(child), - .none => unreachable, - .file => unreachable, - .block => unreachable, - .gen_zir => unreachable, - .local_val => unreachable, - .local_ptr => unreachable, - .decl => unreachable, - } - } - - /// Asserts the scope is a File or ZIRModule and deinitializes it, then deallocates it. - pub fn destroy(base: *Scope, gpa: *Allocator) void { - switch (base.tag) { - .file => { - const scope_file = @fieldParentPtr(File, "base", base); - scope_file.deinit(gpa); - gpa.destroy(scope_file); - }, - .zir_module => { - const scope_zir_module = @fieldParentPtr(ZIRModule, "base", base); - scope_zir_module.deinit(gpa); - gpa.destroy(scope_zir_module); - }, - .none => { - const scope_none = @fieldParentPtr(None, "base", base); - gpa.destroy(scope_none); - }, - .block => unreachable, - .gen_zir => unreachable, - .local_val => unreachable, - .local_ptr => unreachable, - .decl => unreachable, - .container => unreachable, - } - } - - fn name_hash_hash(x: NameHash) u32 { - return @truncate(u32, @bitCast(u128, x)); - } - - fn name_hash_eql(a: NameHash, b: NameHash) bool { - return @bitCast(u128, a) == @bitCast(u128, b); - } - - pub const Tag = enum { - /// .zir source code. - zir_module, - /// .zig source code. - file, - /// There is no .zig or .zir source code being compiled in this Module. - none, - /// struct, enum or union, every .file contains one of these. - container, - block, - decl, - gen_zir, - local_val, - local_ptr, - }; - - pub const Container = struct { - pub const base_tag: Tag = .container; - base: Scope = Scope{ .tag = base_tag }, - - file_scope: *Scope.File, - - /// Direct children of the file. - decls: std.AutoArrayHashMapUnmanaged(*Decl, void), - - // TODO implement container types and put this in a status union - // ty: Type - - pub fn deinit(self: *Container, gpa: *Allocator) void { - self.decls.deinit(gpa); - self.* = undefined; - } - - pub fn removeDecl(self: *Container, child: *Decl) void { - _ = self.decls.remove(child); - } - - pub fn fullyQualifiedNameHash(self: *Container, name: []const u8) NameHash { - // TODO container scope qualified names. - return std.zig.hashSrc(name); - } - }; - - pub const File = struct { - pub const base_tag: Tag = .file; - base: Scope = Scope{ .tag = base_tag }, - - /// Relative to the owning package's root_src_dir. - /// Reference to external memory, not owned by File. - sub_file_path: []const u8, - source: union(enum) { - unloaded: void, - bytes: [:0]const u8, - }, - contents: union { - not_available: void, - tree: *ast.Tree, - }, - status: enum { - never_loaded, - unloaded_success, - unloaded_parse_failure, - loaded_success, - }, - - root_container: Container, - - pub fn unload(self: *File, gpa: *Allocator) void { - switch (self.status) { - .never_loaded, - .unloaded_parse_failure, - .unloaded_success, - => {}, - - .loaded_success => { - self.contents.tree.deinit(); - self.status = .unloaded_success; - }, - } - switch (self.source) { - .bytes => |bytes| { - gpa.free(bytes); - self.source = .{ .unloaded = {} }; - }, - .unloaded => {}, - } - } - - pub fn deinit(self: *File, gpa: *Allocator) void { - self.root_container.deinit(gpa); - self.unload(gpa); - self.* = undefined; - } - - pub fn dumpSrc(self: *File, src: usize) void { - const loc = std.zig.findLineColumn(self.source.bytes, src); - std.debug.print("{}:{}:{}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 }); - } - - pub fn getSource(self: *File, module: *Module) ![:0]const u8 { - switch (self.source) { - .unloaded => { - const source = try module.root_pkg.?.root_src_directory.handle.readFileAllocOptions( - module.gpa, - self.sub_file_path, - std.math.maxInt(u32), - null, - 1, - 0, - ); - self.source = .{ .bytes = source }; - return source; - }, - .bytes => |bytes| return bytes, - } - } - }; - - /// For when there is no top level scope because there are no .zig files being compiled. - pub const None = struct { - pub const base_tag: Tag = .none; - base: Scope = Scope{ .tag = base_tag }, - }; - - pub const ZIRModule = struct { - pub const base_tag: Tag = .zir_module; - base: Scope = Scope{ .tag = base_tag }, - /// Relative to the owning package's root_src_dir. - /// Reference to external memory, not owned by ZIRModule. - sub_file_path: []const u8, - source: union(enum) { - unloaded: void, - bytes: [:0]const u8, - }, - contents: union { - not_available: void, - module: *zir.Module, - }, - status: enum { - never_loaded, - unloaded_success, - unloaded_parse_failure, - unloaded_sema_failure, - - loaded_sema_failure, - loaded_success, - }, - - /// Even though .zir files only have 1 module, this set is still needed - /// because of anonymous Decls, which can exist in the global set, but - /// not this one. - decls: ArrayListUnmanaged(*Decl), - - pub fn unload(self: *ZIRModule, gpa: *Allocator) void { - switch (self.status) { - .never_loaded, - .unloaded_parse_failure, - .unloaded_sema_failure, - .unloaded_success, - => {}, - - .loaded_success => { - self.contents.module.deinit(gpa); - gpa.destroy(self.contents.module); - self.contents = .{ .not_available = {} }; - self.status = .unloaded_success; - }, - .loaded_sema_failure => { - self.contents.module.deinit(gpa); - gpa.destroy(self.contents.module); - self.contents = .{ .not_available = {} }; - self.status = .unloaded_sema_failure; - }, - } - switch (self.source) { - .bytes => |bytes| { - gpa.free(bytes); - self.source = .{ .unloaded = {} }; - }, - .unloaded => {}, - } - } - - pub fn deinit(self: *ZIRModule, gpa: *Allocator) void { - self.decls.deinit(gpa); - self.unload(gpa); - self.* = undefined; - } - - pub fn removeDecl(self: *ZIRModule, child: *Decl) void { - for (self.decls.items) |item, i| { - if (item == child) { - _ = self.decls.swapRemove(i); - return; - } - } - } - - pub fn dumpSrc(self: *ZIRModule, src: usize) void { - const loc = std.zig.findLineColumn(self.source.bytes, src); - std.debug.print("{}:{}:{}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 }); - } - - pub fn getSource(self: *ZIRModule, module: *Module) ![:0]const u8 { - switch (self.source) { - .unloaded => { - const source = try module.root_pkg.?.root_src_directory.handle.readFileAllocOptions( - module.gpa, - self.sub_file_path, - std.math.maxInt(u32), - null, - 1, - 0, - ); - self.source = .{ .bytes = source }; - return source; - }, - .bytes => |bytes| return bytes, - } - } - - pub fn fullyQualifiedNameHash(self: *ZIRModule, name: []const u8) NameHash { - // ZIR modules only have 1 file with all decls global in the same namespace. - return std.zig.hashSrc(name); - } - }; - - /// This is a temporary structure, references to it are valid only - /// during semantic analysis of the block. - pub const Block = struct { - pub const base_tag: Tag = .block; - base: Scope = Scope{ .tag = base_tag }, - parent: ?*Block, - func: ?*Fn, - decl: *Decl, - instructions: ArrayListUnmanaged(*Inst), - /// Points to the arena allocator of DeclAnalysis - arena: *Allocator, - label: ?Label = null, - is_comptime: bool, - - pub const Label = struct { - zir_block: *zir.Inst.Block, - results: ArrayListUnmanaged(*Inst), - block_inst: *Inst.Block, - }; - }; - - /// This is a temporary structure, references to it are valid only - /// during semantic analysis of the decl. - pub const DeclAnalysis = struct { - pub const base_tag: Tag = .decl; - base: Scope = Scope{ .tag = base_tag }, - decl: *Decl, - arena: std.heap.ArenaAllocator, - }; - - /// This is a temporary structure, references to it are valid only - /// during semantic analysis of the decl. - pub const GenZIR = struct { - pub const base_tag: Tag = .gen_zir; - base: Scope = Scope{ .tag = base_tag }, - /// Parents can be: `GenZIR`, `ZIRModule`, `File` - parent: *Scope, - decl: *Decl, - arena: *Allocator, - /// The first N instructions in a function body ZIR are arg instructions. - instructions: std.ArrayListUnmanaged(*zir.Inst) = .{}, - label: ?Label = null, - - pub const Label = struct { - token: ast.TokenIndex, - block_inst: *zir.Inst.Block, - result_loc: astgen.ResultLoc, - }; - }; - - /// This is always a `const` local and importantly the `inst` is a value type, not a pointer. - /// This structure lives as long as the AST generation of the Block - /// node that contains the variable. - pub const LocalVal = struct { - pub const base_tag: Tag = .local_val; - base: Scope = Scope{ .tag = base_tag }, - /// Parents can be: `LocalVal`, `LocalPtr`, `GenZIR`. - parent: *Scope, - gen_zir: *GenZIR, - name: []const u8, - inst: *zir.Inst, - }; - - /// This could be a `const` or `var` local. It has a pointer instead of a value. - /// This structure lives as long as the AST generation of the Block - /// node that contains the variable. - pub const LocalPtr = struct { - pub const base_tag: Tag = .local_ptr; - base: Scope = Scope{ .tag = base_tag }, - /// Parents can be: `LocalVal`, `LocalPtr`, `GenZIR`. - parent: *Scope, - gen_zir: *GenZIR, - name: []const u8, - ptr: *zir.Inst, - }; -}; - pub const AllErrors = struct { arena: std.heap.ArenaAllocator.State, list: []const Message, @@ -1029,12 +253,12 @@ pub const InitOptions = struct { debug_link: bool = false, stack_size_override: ?u64 = null, self_exe_path: ?[]const u8 = null, - version: std.builtin.Version = .{ .major = 0, .minor = 0, .patch = 0 }, + version: ?std.builtin.Version = null, libc_installation: ?*const LibCInstallation = null, }; pub fn create(gpa: *Allocator, options: InitOptions) !*Module { - const mod: *Module = mod: { + const comp: *Module = comp: { // For allocations that have the same lifetime as Module. This arena is used only during this // initialization and then is freed in deinit(). var arena_allocator = std.heap.ArenaAllocator.init(gpa); @@ -1043,7 +267,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { // We put the `Module` itself in the arena. Freeing the arena will free the module. // It's initialized later after we prepare the initialization options. - const mod = try arena.create(Module); + const comp = try arena.create(Module); const root_name = try arena.dupe(u8, options.root_name); const ofmt = options.object_format orelse options.target.getObjectFormat(); @@ -1153,75 +377,142 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { // For example, one cannot change the target between updates, but one can change source files, // so the target goes into the cache hash, but source files do not. This is so that we can // find the same binary and incrementally update it even if there are modified source files. - // We do this even if outputting to the current directory because (1) this cache_hash instance - // will be the "parent" of other cache_hash instances such as for C objects, (2) we need - // a place for intermediate build artifacts, such as a .o file to be linked with LLD, and (3) - // we need somewhere to store serialization of incremental compilation metadata. - var cache = try std.cache_hash.CacheHash.init(gpa, options.zig_cache_directory.handle, "h"); - errdefer cache.release(); + // We do this even if outputting to the current directory because we need somewhere to store + // incremental compilation metadata. + const cache = try arena.create(std.cache_hash.Cache); + cache.* = .{ + .gpa = gpa, + .manifest_dir = try options.zig_cache_directory.handle.makeOpenPath("h", .{}), + }; + errdefer cache.manifest_dir.close(); - // Now we will prepare hash state initializations to avoid redundantly computing hashes. - // First we add common things between things that apply to zig source and all c source files. - cache.addBytes(build_options.version); - cache.add(options.optimize_mode); - cache.add(options.target.cpu.arch); - cache.addBytes(options.target.cpu.model.name); - cache.add(options.target.cpu.features.ints); - cache.add(options.target.os.tag); - switch (options.target.os.tag) { - .linux => { - cache.add(options.target.os.version_range.linux.range.min); - cache.add(options.target.os.version_range.linux.range.max); - cache.add(options.target.os.version_range.linux.glibc); - }, - .windows => { - cache.add(options.target.os.version_range.windows.min); - cache.add(options.target.os.version_range.windows.max); - }, - .freebsd, - .macosx, - .ios, - .tvos, - .watchos, - .netbsd, - .openbsd, - .dragonfly, - => { - cache.add(options.target.os.version_range.semver.min); - cache.add(options.target.os.version_range.semver.max); - }, - else => {}, - } - cache.add(options.target.abi); - cache.add(ofmt); - cache.add(pic); - cache.add(stack_check); - cache.add(sanitize_c); - cache.add(valgrind); - cache.add(link_mode); - cache.add(options.strip); - cache.add(single_threaded); + // This is shared hasher state common to zig source and all C source files. + cache.hash.addBytes(build_options.version); + cache.hash.addBytes(options.zig_lib_directory.path orelse "."); + cache.hash.add(options.optimize_mode); + cache.hash.add(options.target.cpu.arch); + cache.hash.addBytes(options.target.cpu.model.name); + cache.hash.add(options.target.cpu.features.ints); + cache.hash.add(options.target.os.tag); + cache.hash.add(options.target.abi); + cache.hash.add(ofmt); + cache.hash.add(pic); + cache.hash.add(stack_check); + cache.hash.add(link_mode); + cache.hash.add(options.strip); + cache.hash.add(options.link_libc); + cache.hash.add(options.output_mode); // TODO audit this and make sure everything is in it - // We don't care whether we find something there, just show us the digest. - const digest = (try cache.hit()) orelse cache.final(); + const zig_module: ?*ZigModule = if (options.root_pkg) |root_pkg| blk: { + // Options that are specific to zig source files, that cannot be + // modified between incremental updates. + var hash = cache.hash; - const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); - var artifact_dir = try options.zig_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); - errdefer artifact_dir.close(); - const zig_cache_artifact_directory: Directory = .{ - .handle = artifact_dir, - .path = if (options.zig_cache_directory.path) |p| - try std.fs.path.join(arena, &[_][]const u8{ p, artifact_sub_dir }) - else - artifact_sub_dir, + hash.add(valgrind); + hash.add(single_threaded); + switch (options.target.os.getVersionRange()) { + .linux => |linux| { + hash.add(linux.range.min); + hash.add(linux.range.max); + hash.add(linux.glibc); + }, + .windows => |windows| { + hash.add(windows.min); + hash.add(windows.max); + }, + .semver => |semver| { + hash.add(semver.min); + hash.add(semver.max); + }, + .none => {}, + } + + const digest = hash.final(); + const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); + var artifact_dir = try options.zig_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); + errdefer artifact_dir.close(); + const zig_cache_artifact_directory: Directory = .{ + .handle = artifact_dir, + .path = if (options.zig_cache_directory.path) |p| + try std.fs.path.join(arena, &[_][]const u8{ p, artifact_sub_dir }) + else + artifact_sub_dir, + }; + + // TODO when we implement serialization and deserialization of incremental compilation metadata, + // this is where we would load it. We have open a handle to the directory where + // the output either already is, or will be. + // However we currently do not have serialization of such metadata, so for now + // we set up an empty ZigModule that does the entire compilation fresh. + + const root_scope = rs: { + if (mem.endsWith(u8, root_pkg.root_src_path, ".zig")) { + const root_scope = try gpa.create(ZigModule.Scope.File); + root_scope.* = .{ + .sub_file_path = root_pkg.root_src_path, + .source = .{ .unloaded = {} }, + .contents = .{ .not_available = {} }, + .status = .never_loaded, + .root_container = .{ + .file_scope = root_scope, + .decls = .{}, + }, + }; + break :rs &root_scope.base; + } else if (mem.endsWith(u8, root_pkg.root_src_path, ".zir")) { + const root_scope = try gpa.create(ZigModule.Scope.ZIRModule); + root_scope.* = .{ + .sub_file_path = root_pkg.root_src_path, + .source = .{ .unloaded = {} }, + .contents = .{ .not_available = {} }, + .status = .never_loaded, + .decls = .{}, + }; + break :rs &root_scope.base; + } else { + unreachable; + } + }; + + const zig_module = try arena.create(ZigModule); + zig_module.* = .{ + .gpa = gpa, + .comp = comp, + .root_pkg = root_pkg, + .root_scope = root_scope, + .zig_cache_artifact_directory = zig_cache_artifact_directory, + }; + break :blk zig_module; + } else null; + errdefer if (zig_module) |zm| zm.deinit(); + + // For resource management purposes. + var owned_link_dir: ?std.fs.Dir = null; + errdefer if (owned_link_dir) |*dir| dir.close(); + + const bin_directory = emit_bin.directory orelse blk: { + if (zig_module) |zm| break :blk zm.zig_cache_artifact_directory; + + const digest = cache.hash.peek(); + const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); + var artifact_dir = try options.zig_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); + owned_link_dir = artifact_dir; + const link_artifact_directory: Directory = .{ + .handle = artifact_dir, + .path = if (options.zig_cache_directory.path) |p| + try std.fs.path.join(arena, &[_][]const u8{ p, artifact_sub_dir }) + else + artifact_sub_dir, + }; + break :blk link_artifact_directory; }; const bin_file = try link.File.openPath(gpa, .{ - .directory = emit_bin.directory orelse zig_cache_artifact_directory, + .directory = bin_directory, .sub_path = emit_bin.basename, .root_name = root_name, - .root_pkg = options.root_pkg, + .zig_module = zig_module, .target = options.target, .dynamic_linker = options.dynamic_linker, .output_mode = options.output_mode, @@ -1263,70 +554,33 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { }); errdefer bin_file.destroy(); - // We arena-allocate the root scope so there is no free needed. - const root_scope = blk: { - if (options.root_pkg) |root_pkg| { - if (mem.endsWith(u8, root_pkg.root_src_path, ".zig")) { - const root_scope = try gpa.create(Scope.File); - root_scope.* = .{ - .sub_file_path = root_pkg.root_src_path, - .source = .{ .unloaded = {} }, - .contents = .{ .not_available = {} }, - .status = .never_loaded, - .root_container = .{ - .file_scope = root_scope, - .decls = .{}, - }, - }; - break :blk &root_scope.base; - } else if (mem.endsWith(u8, root_pkg.root_src_path, ".zir")) { - const root_scope = try gpa.create(Scope.ZIRModule); - root_scope.* = .{ - .sub_file_path = root_pkg.root_src_path, - .source = .{ .unloaded = {} }, - .contents = .{ .not_available = {} }, - .status = .never_loaded, - .decls = .{}, - }; - break :blk &root_scope.base; - } else { - unreachable; - } - } else { - const root_scope = try gpa.create(Scope.None); - root_scope.* = .{}; - break :blk &root_scope.base; - } - }; - - mod.* = .{ + comp.* = .{ .gpa = gpa, .arena_state = arena_allocator.state, .zig_lib_directory = options.zig_lib_directory, .zig_cache_directory = options.zig_cache_directory, - .zig_cache_artifact_directory = zig_cache_artifact_directory, - .root_pkg = options.root_pkg, - .root_scope = root_scope, .bin_file = bin_file, .work_queue = std.fifo.LinearFifo(WorkItem, .Dynamic).init(gpa), .keep_source_files_loaded = options.keep_source_files_loaded, .use_clang = use_clang, .clang_argv = options.clang_argv, .c_source_files = options.c_source_files, - .cache = cache, + .cache_parent = cache, .self_exe_path = options.self_exe_path, .libc_include_dir_list = libc_dirs.libc_include_dir_list, .sanitize_c = sanitize_c, .rand = options.rand, .clang_passthrough_mode = options.clang_passthrough_mode, .debug_cc = options.debug_cc, + .disable_c_depfile = options.disable_c_depfile, + .owned_link_dir = owned_link_dir, }; - break :mod mod; + break :comp comp; }; - errdefer mod.destroy(); + errdefer comp.destroy(); // Add a `CObject` for each `c_source_files`. - try mod.c_object_table.ensureCapacity(gpa, options.c_source_files.len); + try comp.c_object_table.ensureCapacity(gpa, options.c_source_files.len); for (options.c_source_files) |c_source_file| { var local_arena = std.heap.ArenaAllocator.init(gpa); errdefer local_arena.deinit(); @@ -1335,28 +589,29 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { c_object.* = .{ .status = .{ .new = {} }, - // TODO why are we duplicating this memory? do we need to? - // look into refactoring to turn these 2 fields simply into a CSourceFile + // TODO look into refactoring to turn these 2 fields simply into a CSourceFile .src_path = try local_arena.allocator.dupe(u8, c_source_file.src_path), .extra_flags = try local_arena.allocator.dupe([]const u8, c_source_file.extra_flags), .arena = local_arena.state, }; - mod.c_object_table.putAssumeCapacityNoClobber(c_object, {}); + comp.c_object_table.putAssumeCapacityNoClobber(c_object, {}); } // If we need to build glibc for the target, add work items for it. // We go through the work queue so that building can be done in parallel. - if (mod.wantBuildGLibCFromSource()) { - try mod.addBuildingGLibCWorkItems(); + if (comp.wantBuildGLibCFromSource()) { + try comp.addBuildingGLibCWorkItems(); } - return mod; + return comp; } pub fn destroy(self: *Module) void { + const optional_zig_module = self.bin_file.options.zig_module; self.bin_file.destroy(); + if (optional_zig_module) |zig_module| zig_module.deinit(); + const gpa = self.gpa; - self.deletion_set.deinit(gpa); self.work_queue.deinit(); { @@ -1368,86 +623,32 @@ pub fn destroy(self: *Module) void { self.crt_files.deinit(gpa); } - for (self.decl_table.items()) |entry| { - entry.value.destroy(gpa); - } - self.decl_table.deinit(gpa); - for (self.c_object_table.items()) |entry| { entry.key.destroy(gpa); } self.c_object_table.deinit(gpa); - for (self.failed_decls.items()) |entry| { - entry.value.destroy(gpa); - } - self.failed_decls.deinit(gpa); - for (self.failed_c_objects.items()) |entry| { entry.value.destroy(gpa); } self.failed_c_objects.deinit(gpa); - for (self.failed_files.items()) |entry| { - entry.value.destroy(gpa); - } - self.failed_files.deinit(gpa); - - for (self.failed_exports.items()) |entry| { - entry.value.destroy(gpa); - } - self.failed_exports.deinit(gpa); - - for (self.decl_exports.items()) |entry| { - const export_list = entry.value; - gpa.free(export_list); - } - self.decl_exports.deinit(gpa); - - for (self.export_owners.items()) |entry| { - freeExportList(gpa, entry.value); - } - self.export_owners.deinit(gpa); - - self.symbol_exports.deinit(gpa); - self.root_scope.destroy(gpa); - - var it = self.global_error_set.iterator(); - while (it.next()) |entry| { - gpa.free(entry.key); - } - self.global_error_set.deinit(gpa); - - self.zig_cache_artifact_directory.handle.close(); - self.cache.release(); + self.cache_parent.manifest_dir.close(); + if (self.owned_link_dir) |*dir| dir.close(); // This destroys `self`. self.arena_state.promote(gpa).deinit(); } -fn freeExportList(gpa: *Allocator, export_list: []*Export) void { - for (export_list) |exp| { - gpa.free(exp.options.name); - gpa.destroy(exp); - } - gpa.free(export_list); -} - pub fn getTarget(self: Module) Target { return self.bin_file.options.target; } -pub fn optimizeMode(self: Module) std.builtin.Mode { - return self.bin_file.options.optimize_mode; -} - /// Detect changes to source files, perform semantic analysis, and update the output files. pub fn update(self: *Module) !void { const tracy = trace(@src()); defer tracy.end(); - self.generation += 1; - // For compiling C objects, we rely on the cache hash system to avoid duplicating work. // TODO Look into caching this data in memory to improve performance. // Add a WorkItem for each C object. @@ -1456,36 +657,42 @@ pub fn update(self: *Module) !void { self.work_queue.writeItemAssumeCapacity(.{ .c_object = entry.key }); } - // TODO Detect which source files changed. - // Until then we simulate a full cache miss. Source files could have been loaded for any reason; - // to force a refresh we unload now. - if (self.root_scope.cast(Scope.File)) |zig_file| { - zig_file.unload(self.gpa); - self.analyzeContainer(&zig_file.root_container) catch |err| switch (err) { - error.AnalysisFail => { - assert(self.totalErrorCount() != 0); - }, - else => |e| return e, - }; - } else if (self.root_scope.cast(Scope.ZIRModule)) |zir_module| { - zir_module.unload(self.gpa); - self.analyzeRootZIRModule(zir_module) catch |err| switch (err) { - error.AnalysisFail => { - assert(self.totalErrorCount() != 0); - }, - else => |e| return e, - }; + if (self.bin_file.options.zig_module) |zig_module| { + zig_module.generation += 1; + + // TODO Detect which source files changed. + // Until then we simulate a full cache miss. Source files could have been loaded for any reason; + // to force a refresh we unload now. + if (zig_module.root_scope.cast(ZigModule.Scope.File)) |zig_file| { + zig_file.unload(zig_module.gpa); + zig_module.analyzeContainer(&zig_file.root_container) catch |err| switch (err) { + error.AnalysisFail => { + assert(self.totalErrorCount() != 0); + }, + else => |e| return e, + }; + } else if (zig_module.root_scope.cast(ZigModule.Scope.ZIRModule)) |zir_module| { + zir_module.unload(zig_module.gpa); + zig_module.analyzeRootZIRModule(zir_module) catch |err| switch (err) { + error.AnalysisFail => { + assert(self.totalErrorCount() != 0); + }, + else => |e| return e, + }; + } } try self.performAllTheWork(); - // Process the deletion set. - while (self.deletion_set.popOrNull()) |decl| { - if (decl.dependants.items().len != 0) { - decl.deletion_flag = false; - continue; + if (self.bin_file.options.zig_module) |zig_module| { + // Process the deletion set. + while (zig_module.deletion_set.popOrNull()) |decl| { + if (decl.dependants.items().len != 0) { + decl.deletion_flag = false; + continue; + } + try zig_module.deleteDecl(decl); } - try self.deleteDecl(decl); } // This is needed before reading the error flags. @@ -1496,7 +703,9 @@ pub fn update(self: *Module) !void { // If there are any errors, we anticipate the source files being loaded // to report error messages. Otherwise we unload all source files to save memory. if (self.totalErrorCount() == 0 and !self.keep_source_files_loaded) { - self.root_scope.unload(self.gpa); + if (self.bin_file.options.zig_module) |zig_module| { + zig_module.root_scope.unload(self.gpa); + } } } @@ -1513,11 +722,20 @@ pub fn makeBinFileWritable(self: *Module) !void { } pub fn totalErrorCount(self: *Module) usize { - const total = self.failed_decls.items().len + - self.failed_c_objects.items().len + - self.failed_files.items().len + - self.failed_exports.items().len; - return if (total == 0) @boolToInt(self.link_error_flags.no_entry_point_found) else total; + var total: usize = self.failed_c_objects.items().len; + + if (self.bin_file.options.zig_module) |zig_module| { + total += zig_module.failed_decls.items().len + + zig_module.failed_exports.items().len + + zig_module.failed_files.items().len; + } + + // The "no entry point found" error only counts if there are no other errors. + if (total == 0) { + return @boolToInt(self.link_error_flags.no_entry_point_found); + } + + return total; } pub fn getAllErrorsAlloc(self: *Module) !AllErrors { @@ -1530,31 +748,32 @@ pub fn getAllErrorsAlloc(self: *Module) !AllErrors { for (self.failed_c_objects.items()) |entry| { const c_object = entry.key; const err_msg = entry.value; - const source = c_object.status.failure; - try AllErrors.add(&arena, &errors, c_object.src_path, source, err_msg.*); + try AllErrors.add(&arena, &errors, c_object.src_path, "", err_msg.*); } - for (self.failed_files.items()) |entry| { - const scope = entry.key; - const err_msg = entry.value; - const source = try scope.getSource(self); - try AllErrors.add(&arena, &errors, scope.subFilePath(), source, err_msg.*); - } - for (self.failed_decls.items()) |entry| { - const decl = entry.key; - const err_msg = entry.value; - const source = try decl.scope.getSource(self); - try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*); - } - for (self.failed_exports.items()) |entry| { - const decl = entry.key.owner_decl; - const err_msg = entry.value; - const source = try decl.scope.getSource(self); - try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*); + if (self.bin_file.options.zig_module) |zig_module| { + for (zig_module.failed_files.items()) |entry| { + const scope = entry.key; + const err_msg = entry.value; + const source = try scope.getSource(zig_module); + try AllErrors.add(&arena, &errors, scope.subFilePath(), source, err_msg.*); + } + for (zig_module.failed_decls.items()) |entry| { + const decl = entry.key; + const err_msg = entry.value; + const source = try decl.scope.getSource(zig_module); + try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*); + } + for (zig_module.failed_exports.items()) |entry| { + const decl = entry.key.owner_decl; + const err_msg = entry.value; + const source = try decl.scope.getSource(zig_module); + try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*); + } } if (errors.items.len == 0 and self.link_error_flags.no_entry_point_found) { const global_err_src_path = blk: { - if (self.root_pkg) |root_pkg| break :blk root_pkg.root_src_path; + if (self.bin_file.options.zig_module) |zig_module| break :blk zig_module.root_pkg.root_src_path; if (self.c_source_files.len != 0) break :blk self.c_source_files[0].src_path; if (self.bin_file.options.objects.len != 0) break :blk self.bin_file.options.objects[0]; break :blk "(no file)"; @@ -1590,9 +809,10 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void { => continue, .complete, .codegen_failure_retryable => { + const zig_module = self.bin_file.options.zig_module.?; if (decl.typed_value.most_recent.typed_value.val.cast(Value.Payload.Function)) |payload| { switch (payload.func.analysis) { - .queued => self.analyzeFnBody(decl, payload.func) catch |err| switch (err) { + .queued => zig_module.analyzeFnBody(decl, payload.func) catch |err| switch (err) { error.AnalysisFail => { assert(payload.func.analysis != .in_progress); continue; @@ -1605,23 +825,23 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void { } // Here we tack on additional allocations to the Decl's arena. The allocations are // lifetime annotations in the ZIR. - var decl_arena = decl.typed_value.most_recent.arena.?.promote(self.gpa); + var decl_arena = decl.typed_value.most_recent.arena.?.promote(zig_module.gpa); defer decl.typed_value.most_recent.arena.?.* = decl_arena.state; log.debug("analyze liveness of {}\n", .{decl.name}); - try liveness.analyze(self.gpa, &decl_arena.allocator, payload.func.analysis.success); + try liveness.analyze(zig_module.gpa, &decl_arena.allocator, payload.func.analysis.success); } assert(decl.typed_value.most_recent.typed_value.ty.hasCodeGenBits()); - self.bin_file.updateDecl(self, decl) catch |err| switch (err) { + self.bin_file.updateDecl(zig_module, decl) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.AnalysisFail => { decl.analysis = .dependency_failure; }, else => { - try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1); - self.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create( - self.gpa, + try zig_module.failed_decls.ensureCapacity(zig_module.gpa, zig_module.failed_decls.items().len + 1); + zig_module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create( + zig_module.gpa, decl.src(), "unable to codegen: {}", .{@errorName(err)}, @@ -1632,16 +852,18 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void { }, }, .analyze_decl => |decl| { - self.ensureDeclAnalyzed(decl) catch |err| switch (err) { + const zig_module = self.bin_file.options.zig_module.?; + zig_module.ensureDeclAnalyzed(decl) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.AnalysisFail => continue, }; }, .update_line_number => |decl| { - self.bin_file.updateDeclLineNumber(self, decl) catch |err| { - try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1); - self.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create( - self.gpa, + const zig_module = self.bin_file.options.zig_module.?; + self.bin_file.updateDeclLineNumber(zig_module, decl) catch |err| { + try zig_module.failed_decls.ensureCapacity(zig_module.gpa, zig_module.failed_decls.items().len + 1); + zig_module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create( + zig_module.gpa, decl.src(), "unable to update line number: {}", .{@errorName(err)}, @@ -1650,21 +872,7 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void { }; }, .c_object => |c_object| { - // Free the previous attempt. - switch (c_object.status) { - .new => {}, - .success => |o_file_path| { - self.gpa.free(o_file_path); - c_object.status = .{ .new = {} }; - }, - .failure => |source| { - self.failed_c_objects.removeAssertDiscard(c_object); - self.gpa.free(source); - - c_object.status = .{ .new = {} }; - }, - } - self.buildCObject(c_object) catch |err| switch (err) { + self.updateCObject(c_object) catch |err| switch (err) { error.AnalysisFail => continue, else => { try self.failed_c_objects.ensureCapacity(self.gpa, self.failed_c_objects.items().len + 1); @@ -1674,7 +882,7 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void { "unable to build C object: {}", .{@errorName(err)}, )); - c_object.status = .{ .failure = "" }; + c_object.status = .{ .failure = {} }; }, }; }, @@ -1692,127 +900,175 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void { }; } -fn buildCObject(mod: *Module, c_object: *CObject) !void { +fn updateCObject(comp: *Compilation, c_object: *CObject) !void { const tracy = trace(@src()); defer tracy.end(); - // TODO this C source file needs its own cache hash instance - if (!build_options.have_llvm) { - return mod.failCObj(c_object, "clang not available: compiler not built with LLVM extensions enabled", .{}); + return comp.failCObj(c_object, "clang not available: compiler not built with LLVM extensions enabled", .{}); } - const self_exe_path = mod.self_exe_path orelse - return mod.failCObj(c_object, "clang compilation disabled", .{}); + const self_exe_path = comp.self_exe_path orelse + return comp.failCObj(c_object, "clang compilation disabled", .{}); - var arena_allocator = std.heap.ArenaAllocator.init(mod.gpa); + if (c_object.clearStatus(comp.gpa)) { + // There was previous failure. + comp.failed_c_objects.removeAssertDiscard(c_object); + } + + var ch = comp.cache_parent.obtain(); + defer ch.deinit(); + + ch.hash.add(comp.sanitize_c); + ch.hash.addListOfBytes(comp.clang_argv); + ch.hash.add(comp.bin_file.options.link_libcpp); + ch.hash.addListOfBytes(comp.libc_include_dir_list); + // TODO + //cache_int(cache_hash, g->code_model); + //cache_bool(cache_hash, codegen_have_frame_pointer(g)); + _ = try ch.addFile(c_object.src_path, null); + { + // Hash the extra flags, with special care to call addFile for file parameters. + // TODO this logic can likely be improved by utilizing clang_options_data.zig. + const file_args = [_][]const u8{"-include"}; + var arg_i: usize = 0; + while (arg_i < c_object.extra_flags.len) : (arg_i += 1) { + const arg = c_object.extra_flags[arg_i]; + ch.hash.addBytes(arg); + for (file_args) |file_arg| { + if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_object.extra_flags.len) { + arg_i += 1; + _ = try ch.addFile(c_object.extra_flags[arg_i], null); + } + } + } + } + + var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); defer arena_allocator.deinit(); const arena = &arena_allocator.allocator; - var argv = std.ArrayList([]const u8).init(mod.gpa); - defer argv.deinit(); - const c_source_basename = std.fs.path.basename(c_object.src_path); // Special case when doing build-obj for just one C file. When there are more than one object // file and building an object we need to link them together, but with just one it should go // directly to the output file. - const direct_o = mod.c_source_files.len == 1 and mod.root_pkg == null and - mod.bin_file.options.output_mode == .Obj and mod.bin_file.options.objects.len == 0; + const direct_o = comp.c_source_files.len == 1 and comp.bin_file.options.zig_module == null and + comp.bin_file.options.output_mode == .Obj and comp.bin_file.options.objects.len == 0; const o_basename_noext = if (direct_o) - mod.bin_file.options.root_name + comp.bin_file.options.root_name else mem.split(c_source_basename, ".").next().?; - const o_basename = try std.fmt.allocPrint(arena, "{}{}", .{ o_basename_noext, mod.getTarget().oFileExt() }); + const o_basename = try std.fmt.allocPrint(arena, "{}{}", .{ o_basename_noext, comp.getTarget().oFileExt() }); - // We can't know the digest until we do the C compiler invocation, so we need a temporary filename. - const out_obj_path = try mod.tmpFilePath(arena, o_basename); + const full_object_path = if (!(try ch.hit()) or comp.disable_c_depfile) blk: { + var argv = std.ArrayList([]const u8).init(comp.gpa); + defer argv.deinit(); - var zig_cache_tmp_dir = try mod.zig_cache_directory.handle.makeOpenPath("tmp", .{}); - defer zig_cache_tmp_dir.close(); + // We can't know the digest until we do the C compiler invocation, so we need a temporary filename. + const out_obj_path = try comp.tmpFilePath(arena, o_basename); - try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang", "-c" }); + try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang", "-c" }); - const ext = classifyFileExt(c_object.src_path); - // TODO capture the .d file and deal with caching stuff - try mod.addCCArgs(arena, &argv, ext, false, null); + const ext = classifyFileExt(c_object.src_path); + // TODO capture the .d file and deal with caching stuff + try comp.addCCArgs(arena, &argv, ext, false, null); - try argv.append("-o"); - try argv.append(out_obj_path); + try argv.append("-o"); + try argv.append(out_obj_path); - try argv.append(c_object.src_path); - try argv.appendSlice(c_object.extra_flags); + try argv.append(c_object.src_path); + try argv.appendSlice(c_object.extra_flags); - if (mod.debug_cc) { - for (argv.items[0 .. argv.items.len - 1]) |arg| { - std.debug.print("{} ", .{arg}); + if (comp.debug_cc) { + for (argv.items[0 .. argv.items.len - 1]) |arg| { + std.debug.print("{} ", .{arg}); + } + std.debug.print("{}\n", .{argv.items[argv.items.len - 1]}); } - std.debug.print("{}\n", .{argv.items[argv.items.len - 1]}); - } - const child = try std.ChildProcess.init(argv.items, arena); - defer child.deinit(); + const child = try std.ChildProcess.init(argv.items, arena); + defer child.deinit(); - if (mod.clang_passthrough_mode) { - child.stdin_behavior = .Inherit; - child.stdout_behavior = .Inherit; - child.stderr_behavior = .Inherit; + if (comp.clang_passthrough_mode) { + child.stdin_behavior = .Inherit; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; - const term = child.spawnAndWait() catch |err| { - return mod.failCObj(c_object, "unable to spawn {}: {}", .{ argv.items[0], @errorName(err) }); + const term = child.spawnAndWait() catch |err| { + return comp.failCObj(c_object, "unable to spawn {}: {}", .{ argv.items[0], @errorName(err) }); + }; + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO make std.process.exit and std.ChildProcess exit code have the same type + // and forward it here. Currently it is u32 vs u8. + std.process.exit(1); + } + }, + else => std.process.exit(1), + } + } else { + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; + + try child.spawn(); + + const stdout_reader = child.stdout.?.reader(); + const stderr_reader = child.stderr.?.reader(); + + // TODO Need to poll to read these streams to prevent a deadlock (or rely on evented I/O). + const stdout = try stdout_reader.readAllAlloc(arena, std.math.maxInt(u32)); + const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); + + const term = child.wait() catch |err| { + return comp.failCObj(c_object, "unable to spawn {}: {}", .{ argv.items[0], @errorName(err) }); + }; + + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO parse clang stderr and turn it into an error message + // and then call failCObjWithOwnedErrorMsg + std.log.err("clang failed with stderr: {}", .{stderr}); + return comp.failCObj(c_object, "clang exited with code {}", .{code}); + } + }, + else => { + std.log.err("clang terminated with stderr: {}", .{stderr}); + return comp.failCObj(c_object, "clang terminated unexpectedly", .{}); + }, + } + } + + // TODO handle .d files + + // Rename into place. + const digest = ch.final(); + const full_object_path = if (comp.zig_cache_directory.path) |p| + try std.fs.path.join(arena, &[_][]const u8{ p, "o", &digest, o_basename }) + else + try std.fs.path.join(arena, &[_][]const u8{ "o", &digest, o_basename }); + try std.fs.rename(out_obj_path, full_object_path); + + ch.writeManifest() catch |err| { + std.log.warn("failed to write cache manifest when compiling '{}': {}", .{ c_object.src_path, @errorName(err) }); }; - switch (term) { - .Exited => |code| { - if (code != 0) { - // TODO make std.process.exit and std.ChildProcess exit code have the same type - // and forward it here. Currently it is u32 vs u8. - std.process.exit(1); - } - }, - else => std.process.exit(1), - } - } else { - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Pipe; - child.stderr_behavior = .Pipe; + break :blk full_object_path; + } else blk: { + const digest = ch.final(); + const full_object_path = if (comp.zig_cache_directory.path) |p| + try std.fs.path.join(arena, &[_][]const u8{ p, "o", &digest, o_basename }) + else + try std.fs.path.join(arena, &[_][]const u8{ "o", &digest, o_basename }); + break :blk full_object_path; + }; - try child.spawn(); - - const stdout_reader = child.stdout.?.reader(); - const stderr_reader = child.stderr.?.reader(); - - // TODO Need to poll to read these streams to prevent a deadlock (or rely on evented I/O). - const stdout = try stdout_reader.readAllAlloc(arena, std.math.maxInt(u32)); - const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); - - const term = child.wait() catch |err| { - return mod.failCObj(c_object, "unable to spawn {}: {}", .{ argv.items[0], @errorName(err) }); - }; - - switch (term) { - .Exited => |code| { - if (code != 0) { - // TODO parse clang stderr and turn it into an error message - // and then call failCObjWithOwnedErrorMsg - std.log.err("clang failed with stderr: {}", .{stderr}); - return mod.failCObj(c_object, "clang exited with code {}", .{code}); - } - }, - else => { - std.log.err("clang terminated with stderr: {}", .{stderr}); - return mod.failCObj(c_object, "clang terminated unexpectedly", .{}); - }, - } - } - - // TODO handle .d files - - // TODO Add renameat capabilities to the std lib in a higher layer than the posix layer. - const tmp_basename = std.fs.path.basename(out_obj_path); - try std.os.renameat(zig_cache_tmp_dir.fd, tmp_basename, mod.zig_cache_artifact_directory.handle.fd, o_basename); - - const success_file_path = try std.fs.path.join(mod.gpa, &[_][]const u8{ - mod.zig_cache_artifact_directory.path.?, o_basename, - }); - c_object.status = .{ .success = success_file_path }; + c_object.status = .{ + .success = .{ + .object_path = full_object_path, + .lock = ch.toOwnedLock(), + }, + }; } fn tmpFilePath(mod: *Module, arena: *Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 { @@ -2016,1913 +1272,6 @@ fn addCCArgs( try argv.appendSlice(mod.clang_argv); } -pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void { - const tracy = trace(@src()); - defer tracy.end(); - - const subsequent_analysis = switch (decl.analysis) { - .in_progress => unreachable, - - .sema_failure, - .sema_failure_retryable, - .codegen_failure, - .dependency_failure, - .codegen_failure_retryable, - => return error.AnalysisFail, - - .complete => return, - - .outdated => blk: { - log.debug("re-analyzing {}\n", .{decl.name}); - - // The exports this Decl performs will be re-discovered, so we remove them here - // prior to re-analysis. - self.deleteDeclExports(decl); - // Dependencies will be re-discovered, so we remove them here prior to re-analysis. - for (decl.dependencies.items()) |entry| { - const dep = entry.key; - dep.removeDependant(decl); - if (dep.dependants.items().len == 0 and !dep.deletion_flag) { - // We don't perform a deletion here, because this Decl or another one - // may end up referencing it before the update is complete. - dep.deletion_flag = true; - try self.deletion_set.append(self.gpa, dep); - } - } - decl.dependencies.clearRetainingCapacity(); - - break :blk true; - }, - - .unreferenced => false, - }; - - const type_changed = if (self.root_scope.cast(Scope.ZIRModule)) |zir_module| - try zir_sema.analyzeZirDecl(self, decl, zir_module.contents.module.decls[decl.src_index]) - else - self.astGenAndAnalyzeDecl(decl) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.AnalysisFail => return error.AnalysisFail, - else => { - try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1); - self.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create( - self.gpa, - decl.src(), - "unable to analyze: {}", - .{@errorName(err)}, - )); - decl.analysis = .sema_failure_retryable; - return error.AnalysisFail; - }, - }; - - if (subsequent_analysis) { - // We may need to chase the dependants and re-analyze them. - // However, if the decl is a function, and the type is the same, we do not need to. - if (type_changed or decl.typed_value.most_recent.typed_value.val.tag() != .function) { - for (decl.dependants.items()) |entry| { - const dep = entry.key; - switch (dep.analysis) { - .unreferenced => unreachable, - .in_progress => unreachable, - .outdated => continue, // already queued for update - - .dependency_failure, - .sema_failure, - .sema_failure_retryable, - .codegen_failure, - .codegen_failure_retryable, - .complete, - => if (dep.generation != self.generation) { - try self.markOutdatedDecl(dep); - }, - } - } - } - } -} - -fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool { - const tracy = trace(@src()); - defer tracy.end(); - - const container_scope = decl.scope.cast(Scope.Container).?; - const tree = try self.getAstTree(container_scope); - const ast_node = tree.root_node.decls()[decl.src_index]; - switch (ast_node.tag) { - .FnProto => { - const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", ast_node); - - decl.analysis = .in_progress; - - // This arena allocator's memory is discarded at the end of this function. It is used - // to determine the type of the function, and hence the type of the decl, which is needed - // to complete the Decl analysis. - var fn_type_scope_arena = std.heap.ArenaAllocator.init(self.gpa); - defer fn_type_scope_arena.deinit(); - var fn_type_scope: Scope.GenZIR = .{ - .decl = decl, - .arena = &fn_type_scope_arena.allocator, - .parent = decl.scope, - }; - defer fn_type_scope.instructions.deinit(self.gpa); - - decl.is_pub = fn_proto.getVisibToken() != null; - const body_node = fn_proto.getBodyNode() orelse - return self.failTok(&fn_type_scope.base, fn_proto.fn_token, "TODO implement extern functions", .{}); - - const param_decls = fn_proto.params(); - const param_types = try fn_type_scope.arena.alloc(*zir.Inst, param_decls.len); - - const fn_src = tree.token_locs[fn_proto.fn_token].start; - const type_type = try astgen.addZIRInstConst(self, &fn_type_scope.base, fn_src, .{ - .ty = Type.initTag(.type), - .val = Value.initTag(.type_type), - }); - const type_type_rl: astgen.ResultLoc = .{ .ty = type_type }; - for (param_decls) |param_decl, i| { - const param_type_node = switch (param_decl.param_type) { - .any_type => |node| return self.failNode(&fn_type_scope.base, node, "TODO implement anytype parameter", .{}), - .type_expr => |node| node, - }; - param_types[i] = try astgen.expr(self, &fn_type_scope.base, type_type_rl, param_type_node); - } - if (fn_proto.getVarArgsToken()) |var_args_token| { - return self.failTok(&fn_type_scope.base, var_args_token, "TODO implement var args", .{}); - } - if (fn_proto.getLibName()) |lib_name| { - return self.failNode(&fn_type_scope.base, lib_name, "TODO implement function library name", .{}); - } - if (fn_proto.getAlignExpr()) |align_expr| { - return self.failNode(&fn_type_scope.base, align_expr, "TODO implement function align expression", .{}); - } - if (fn_proto.getSectionExpr()) |sect_expr| { - return self.failNode(&fn_type_scope.base, sect_expr, "TODO implement function section expression", .{}); - } - if (fn_proto.getCallconvExpr()) |callconv_expr| { - return self.failNode( - &fn_type_scope.base, - callconv_expr, - "TODO implement function calling convention expression", - .{}, - ); - } - const return_type_expr = switch (fn_proto.return_type) { - .Explicit => |node| node, - .InferErrorSet => |node| return self.failNode(&fn_type_scope.base, node, "TODO implement inferred error sets", .{}), - .Invalid => |tok| return self.failTok(&fn_type_scope.base, tok, "unable to parse return type", .{}), - }; - - const return_type_inst = try astgen.expr(self, &fn_type_scope.base, type_type_rl, return_type_expr); - const fn_type_inst = try astgen.addZIRInst(self, &fn_type_scope.base, fn_src, zir.Inst.FnType, .{ - .return_type = return_type_inst, - .param_types = param_types, - }, .{}); - - // We need the memory for the Type to go into the arena for the Decl - var decl_arena = std.heap.ArenaAllocator.init(self.gpa); - errdefer decl_arena.deinit(); - const decl_arena_state = try decl_arena.allocator.create(std.heap.ArenaAllocator.State); - - var block_scope: Scope.Block = .{ - .parent = null, - .func = null, - .decl = decl, - .instructions = .{}, - .arena = &decl_arena.allocator, - .is_comptime = false, - }; - defer block_scope.instructions.deinit(self.gpa); - - const fn_type = try zir_sema.analyzeBodyValueAsType(self, &block_scope, fn_type_inst, .{ - .instructions = fn_type_scope.instructions.items, - }); - const new_func = try decl_arena.allocator.create(Fn); - const fn_payload = try decl_arena.allocator.create(Value.Payload.Function); - - const fn_zir = blk: { - // This scope's arena memory is discarded after the ZIR generation - // pass completes, and semantic analysis of it completes. - var gen_scope_arena = std.heap.ArenaAllocator.init(self.gpa); - errdefer gen_scope_arena.deinit(); - var gen_scope: Scope.GenZIR = .{ - .decl = decl, - .arena = &gen_scope_arena.allocator, - .parent = decl.scope, - }; - defer gen_scope.instructions.deinit(self.gpa); - - // We need an instruction for each parameter, and they must be first in the body. - try gen_scope.instructions.resize(self.gpa, fn_proto.params_len); - var params_scope = &gen_scope.base; - for (fn_proto.params()) |param, i| { - const name_token = param.name_token.?; - const src = tree.token_locs[name_token].start; - const param_name = tree.tokenSlice(name_token); // TODO: call identifierTokenString - const arg = try gen_scope_arena.allocator.create(zir.Inst.Arg); - arg.* = .{ - .base = .{ - .tag = .arg, - .src = src, - }, - .positionals = .{ - .name = param_name, - }, - .kw_args = .{}, - }; - gen_scope.instructions.items[i] = &arg.base; - const sub_scope = try gen_scope_arena.allocator.create(Scope.LocalVal); - sub_scope.* = .{ - .parent = params_scope, - .gen_zir = &gen_scope, - .name = param_name, - .inst = &arg.base, - }; - params_scope = &sub_scope.base; - } - - const body_block = body_node.cast(ast.Node.Block).?; - - try astgen.blockExpr(self, params_scope, body_block); - - if (gen_scope.instructions.items.len == 0 or - !gen_scope.instructions.items[gen_scope.instructions.items.len - 1].tag.isNoReturn()) - { - const src = tree.token_locs[body_block.rbrace].start; - _ = try astgen.addZIRNoOp(self, &gen_scope.base, src, .returnvoid); - } - - const fn_zir = try gen_scope_arena.allocator.create(Fn.ZIR); - fn_zir.* = .{ - .body = .{ - .instructions = try gen_scope.arena.dupe(*zir.Inst, gen_scope.instructions.items), - }, - .arena = gen_scope_arena.state, - }; - break :blk fn_zir; - }; - - new_func.* = .{ - .analysis = .{ .queued = fn_zir }, - .owner_decl = decl, - }; - fn_payload.* = .{ .func = new_func }; - - var prev_type_has_bits = false; - var type_changed = true; - - if (decl.typedValueManaged()) |tvm| { - prev_type_has_bits = tvm.typed_value.ty.hasCodeGenBits(); - type_changed = !tvm.typed_value.ty.eql(fn_type); - - tvm.deinit(self.gpa); - } - - decl_arena_state.* = decl_arena.state; - decl.typed_value = .{ - .most_recent = .{ - .typed_value = .{ - .ty = fn_type, - .val = Value.initPayload(&fn_payload.base), - }, - .arena = decl_arena_state, - }, - }; - decl.analysis = .complete; - decl.generation = self.generation; - - if (fn_type.hasCodeGenBits()) { - // We don't fully codegen the decl until later, but we do need to reserve a global - // offset table index for it. This allows us to codegen decls out of dependency order, - // increasing how many computations can be done in parallel. - try self.bin_file.allocateDeclIndexes(decl); - try self.work_queue.writeItem(.{ .codegen_decl = decl }); - } else if (prev_type_has_bits) { - self.bin_file.freeDecl(decl); - } - - if (fn_proto.getExternExportInlineToken()) |maybe_export_token| { - if (tree.token_ids[maybe_export_token] == .Keyword_export) { - const export_src = tree.token_locs[maybe_export_token].start; - const name_loc = tree.token_locs[fn_proto.getNameToken().?]; - const name = tree.tokenSliceLoc(name_loc); - // The scope needs to have the decl in it. - try self.analyzeExport(&block_scope.base, export_src, name, decl); - } - } - return type_changed; - }, - .VarDecl => { - const var_decl = @fieldParentPtr(ast.Node.VarDecl, "base", ast_node); - - decl.analysis = .in_progress; - - // We need the memory for the Type to go into the arena for the Decl - var decl_arena = std.heap.ArenaAllocator.init(self.gpa); - errdefer decl_arena.deinit(); - const decl_arena_state = try decl_arena.allocator.create(std.heap.ArenaAllocator.State); - - var block_scope: Scope.Block = .{ - .parent = null, - .func = null, - .decl = decl, - .instructions = .{}, - .arena = &decl_arena.allocator, - .is_comptime = true, - }; - defer block_scope.instructions.deinit(self.gpa); - - decl.is_pub = var_decl.getVisibToken() != null; - const is_extern = blk: { - const maybe_extern_token = var_decl.getExternExportToken() orelse - break :blk false; - if (tree.token_ids[maybe_extern_token] != .Keyword_extern) break :blk false; - if (var_decl.getInitNode()) |some| { - return self.failNode(&block_scope.base, some, "extern variables have no initializers", .{}); - } - break :blk true; - }; - if (var_decl.getLibName()) |lib_name| { - assert(is_extern); - return self.failNode(&block_scope.base, lib_name, "TODO implement function library name", .{}); - } - const is_mutable = tree.token_ids[var_decl.mut_token] == .Keyword_var; - const is_threadlocal = if (var_decl.getThreadLocalToken()) |some| blk: { - if (!is_mutable) { - return self.failTok(&block_scope.base, some, "threadlocal variable cannot be constant", .{}); - } - break :blk true; - } else false; - assert(var_decl.getComptimeToken() == null); - if (var_decl.getAlignNode()) |align_expr| { - return self.failNode(&block_scope.base, align_expr, "TODO implement function align expression", .{}); - } - if (var_decl.getSectionNode()) |sect_expr| { - return self.failNode(&block_scope.base, sect_expr, "TODO implement function section expression", .{}); - } - - const var_info: struct { ty: Type, val: ?Value } = if (var_decl.getInitNode()) |init_node| vi: { - var gen_scope_arena = std.heap.ArenaAllocator.init(self.gpa); - defer gen_scope_arena.deinit(); - var gen_scope: Scope.GenZIR = .{ - .decl = decl, - .arena = &gen_scope_arena.allocator, - .parent = decl.scope, - }; - defer gen_scope.instructions.deinit(self.gpa); - - const init_result_loc: astgen.ResultLoc = if (var_decl.getTypeNode()) |type_node| rl: { - const src = tree.token_locs[type_node.firstToken()].start; - const type_type = try astgen.addZIRInstConst(self, &gen_scope.base, src, .{ - .ty = Type.initTag(.type), - .val = Value.initTag(.type_type), - }); - const var_type = try astgen.expr(self, &gen_scope.base, .{ .ty = type_type }, type_node); - break :rl .{ .ty = var_type }; - } else .none; - - const src = tree.token_locs[init_node.firstToken()].start; - const init_inst = try astgen.expr(self, &gen_scope.base, init_result_loc, init_node); - - var inner_block: Scope.Block = .{ - .parent = null, - .func = null, - .decl = decl, - .instructions = .{}, - .arena = &gen_scope_arena.allocator, - .is_comptime = true, - }; - defer inner_block.instructions.deinit(self.gpa); - try zir_sema.analyzeBody(self, &inner_block.base, .{ .instructions = gen_scope.instructions.items }); - - // The result location guarantees the type coercion. - const analyzed_init_inst = init_inst.analyzed_inst.?; - // The is_comptime in the Scope.Block guarantees the result is comptime-known. - const val = analyzed_init_inst.value().?; - - const ty = try analyzed_init_inst.ty.copy(block_scope.arena); - break :vi .{ - .ty = ty, - .val = try val.copy(block_scope.arena), - }; - } else if (!is_extern) { - return self.failTok(&block_scope.base, var_decl.firstToken(), "variables must be initialized", .{}); - } else if (var_decl.getTypeNode()) |type_node| vi: { - // Temporary arena for the zir instructions. - var type_scope_arena = std.heap.ArenaAllocator.init(self.gpa); - defer type_scope_arena.deinit(); - var type_scope: Scope.GenZIR = .{ - .decl = decl, - .arena = &type_scope_arena.allocator, - .parent = decl.scope, - }; - defer type_scope.instructions.deinit(self.gpa); - - const src = tree.token_locs[type_node.firstToken()].start; - const type_type = try astgen.addZIRInstConst(self, &type_scope.base, src, .{ - .ty = Type.initTag(.type), - .val = Value.initTag(.type_type), - }); - const var_type = try astgen.expr(self, &type_scope.base, .{ .ty = type_type }, type_node); - const ty = try zir_sema.analyzeBodyValueAsType(self, &block_scope, var_type, .{ - .instructions = type_scope.instructions.items, - }); - break :vi .{ - .ty = ty, - .val = null, - }; - } else { - return self.failTok(&block_scope.base, var_decl.firstToken(), "unable to infer variable type", .{}); - }; - - if (is_mutable and !var_info.ty.isValidVarType(is_extern)) { - return self.failTok(&block_scope.base, var_decl.firstToken(), "variable of type '{}' must be const", .{var_info.ty}); - } - - var type_changed = true; - if (decl.typedValueManaged()) |tvm| { - type_changed = !tvm.typed_value.ty.eql(var_info.ty); - - tvm.deinit(self.gpa); - } - - const new_variable = try decl_arena.allocator.create(Var); - const var_payload = try decl_arena.allocator.create(Value.Payload.Variable); - new_variable.* = .{ - .owner_decl = decl, - .init = var_info.val orelse undefined, - .is_extern = is_extern, - .is_mutable = is_mutable, - .is_threadlocal = is_threadlocal, - }; - var_payload.* = .{ .variable = new_variable }; - - decl_arena_state.* = decl_arena.state; - decl.typed_value = .{ - .most_recent = .{ - .typed_value = .{ - .ty = var_info.ty, - .val = Value.initPayload(&var_payload.base), - }, - .arena = decl_arena_state, - }, - }; - decl.analysis = .complete; - decl.generation = self.generation; - - if (var_decl.getExternExportToken()) |maybe_export_token| { - if (tree.token_ids[maybe_export_token] == .Keyword_export) { - const export_src = tree.token_locs[maybe_export_token].start; - const name_loc = tree.token_locs[var_decl.name_token]; - const name = tree.tokenSliceLoc(name_loc); - // The scope needs to have the decl in it. - try self.analyzeExport(&block_scope.base, export_src, name, decl); - } - } - return type_changed; - }, - .Comptime => { - const comptime_decl = @fieldParentPtr(ast.Node.Comptime, "base", ast_node); - - decl.analysis = .in_progress; - - // A comptime decl does not store any value so we can just deinit this arena after analysis is done. - var analysis_arena = std.heap.ArenaAllocator.init(self.gpa); - defer analysis_arena.deinit(); - var gen_scope: Scope.GenZIR = .{ - .decl = decl, - .arena = &analysis_arena.allocator, - .parent = decl.scope, - }; - defer gen_scope.instructions.deinit(self.gpa); - - _ = try astgen.comptimeExpr(self, &gen_scope.base, .none, comptime_decl.expr); - - var block_scope: Scope.Block = .{ - .parent = null, - .func = null, - .decl = decl, - .instructions = .{}, - .arena = &analysis_arena.allocator, - .is_comptime = true, - }; - defer block_scope.instructions.deinit(self.gpa); - - _ = try zir_sema.analyzeBody(self, &block_scope.base, .{ - .instructions = gen_scope.instructions.items, - }); - - decl.analysis = .complete; - decl.generation = self.generation; - return true; - }, - .Use => @panic("TODO usingnamespace decl"), - else => unreachable, - } -} - -fn declareDeclDependency(self: *Module, depender: *Decl, dependee: *Decl) !void { - try depender.dependencies.ensureCapacity(self.gpa, depender.dependencies.items().len + 1); - try dependee.dependants.ensureCapacity(self.gpa, dependee.dependants.items().len + 1); - - depender.dependencies.putAssumeCapacity(dependee, {}); - dependee.dependants.putAssumeCapacity(depender, {}); -} - -fn getSrcModule(self: *Module, root_scope: *Scope.ZIRModule) !*zir.Module { - switch (root_scope.status) { - .never_loaded, .unloaded_success => { - try self.failed_files.ensureCapacity(self.gpa, self.failed_files.items().len + 1); - - const source = try root_scope.getSource(self); - - var keep_zir_module = false; - const zir_module = try self.gpa.create(zir.Module); - defer if (!keep_zir_module) self.gpa.destroy(zir_module); - - zir_module.* = try zir.parse(self.gpa, source); - defer if (!keep_zir_module) zir_module.deinit(self.gpa); - - if (zir_module.error_msg) |src_err_msg| { - self.failed_files.putAssumeCapacityNoClobber( - &root_scope.base, - try ErrorMsg.create(self.gpa, src_err_msg.byte_offset, "{}", .{src_err_msg.msg}), - ); - root_scope.status = .unloaded_parse_failure; - return error.AnalysisFail; - } - - root_scope.status = .loaded_success; - root_scope.contents = .{ .module = zir_module }; - keep_zir_module = true; - - return zir_module; - }, - - .unloaded_parse_failure, - .unloaded_sema_failure, - => return error.AnalysisFail, - - .loaded_success, .loaded_sema_failure => return root_scope.contents.module, - } -} - -fn getAstTree(self: *Module, container_scope: *Scope.Container) !*ast.Tree { - const tracy = trace(@src()); - defer tracy.end(); - - const root_scope = container_scope.file_scope; - - switch (root_scope.status) { - .never_loaded, .unloaded_success => { - try self.failed_files.ensureCapacity(self.gpa, self.failed_files.items().len + 1); - - const source = try root_scope.getSource(self); - - var keep_tree = false; - const tree = try std.zig.parse(self.gpa, source); - defer if (!keep_tree) tree.deinit(); - - if (tree.errors.len != 0) { - const parse_err = tree.errors[0]; - - var msg = std.ArrayList(u8).init(self.gpa); - defer msg.deinit(); - - try parse_err.render(tree.token_ids, msg.outStream()); - const err_msg = try self.gpa.create(ErrorMsg); - err_msg.* = .{ - .msg = msg.toOwnedSlice(), - .byte_offset = tree.token_locs[parse_err.loc()].start, - }; - - self.failed_files.putAssumeCapacityNoClobber(&root_scope.base, err_msg); - root_scope.status = .unloaded_parse_failure; - return error.AnalysisFail; - } - - root_scope.status = .loaded_success; - root_scope.contents = .{ .tree = tree }; - keep_tree = true; - - return tree; - }, - - .unloaded_parse_failure => return error.AnalysisFail, - - .loaded_success => return root_scope.contents.tree, - } -} - -fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void { - const tracy = trace(@src()); - defer tracy.end(); - - // We may be analyzing it for the first time, or this may be - // an incremental update. This code handles both cases. - const tree = try self.getAstTree(container_scope); - const decls = tree.root_node.decls(); - - try self.work_queue.ensureUnusedCapacity(decls.len); - try container_scope.decls.ensureCapacity(self.gpa, decls.len); - - // Keep track of the decls that we expect to see in this file so that - // we know which ones have been deleted. - var deleted_decls = std.AutoArrayHashMap(*Decl, void).init(self.gpa); - defer deleted_decls.deinit(); - try deleted_decls.ensureCapacity(container_scope.decls.items().len); - for (container_scope.decls.items()) |entry| { - deleted_decls.putAssumeCapacityNoClobber(entry.key, {}); - } - - for (decls) |src_decl, decl_i| { - if (src_decl.cast(ast.Node.FnProto)) |fn_proto| { - // We will create a Decl for it regardless of analysis status. - const name_tok = fn_proto.getNameToken() orelse { - @panic("TODO missing function name"); - }; - - const name_loc = tree.token_locs[name_tok]; - const name = tree.tokenSliceLoc(name_loc); - const name_hash = container_scope.fullyQualifiedNameHash(name); - const contents_hash = std.zig.hashSrc(tree.getNodeSource(src_decl)); - if (self.decl_table.get(name_hash)) |decl| { - // Update the AST Node index of the decl, even if its contents are unchanged, it may - // have been re-ordered. - decl.src_index = decl_i; - if (deleted_decls.remove(decl) == null) { - decl.analysis = .sema_failure; - const err_msg = try ErrorMsg.create(self.gpa, tree.token_locs[name_tok].start, "redefinition of '{}'", .{decl.name}); - errdefer err_msg.destroy(self.gpa); - try self.failed_decls.putNoClobber(self.gpa, decl, err_msg); - } else { - if (!srcHashEql(decl.contents_hash, contents_hash)) { - try self.markOutdatedDecl(decl); - decl.contents_hash = contents_hash; - } else switch (self.bin_file.tag) { - .coff => { - // TODO Implement for COFF - }, - .elf => if (decl.fn_link.elf.len != 0) { - // TODO Look into detecting when this would be unnecessary by storing enough state - // in `Decl` to notice that the line number did not change. - self.work_queue.writeItemAssumeCapacity(.{ .update_line_number = decl }); - }, - .macho => { - // TODO Implement for MachO - }, - .c, .wasm => {}, - } - } - } else { - const new_decl = try self.createNewDecl(&container_scope.base, name, decl_i, name_hash, contents_hash); - container_scope.decls.putAssumeCapacity(new_decl, {}); - if (fn_proto.getExternExportInlineToken()) |maybe_export_token| { - if (tree.token_ids[maybe_export_token] == .Keyword_export) { - self.work_queue.writeItemAssumeCapacity(.{ .analyze_decl = new_decl }); - } - } - } - } else if (src_decl.castTag(.VarDecl)) |var_decl| { - const name_loc = tree.token_locs[var_decl.name_token]; - const name = tree.tokenSliceLoc(name_loc); - const name_hash = container_scope.fullyQualifiedNameHash(name); - const contents_hash = std.zig.hashSrc(tree.getNodeSource(src_decl)); - if (self.decl_table.get(name_hash)) |decl| { - // Update the AST Node index of the decl, even if its contents are unchanged, it may - // have been re-ordered. - decl.src_index = decl_i; - if (deleted_decls.remove(decl) == null) { - decl.analysis = .sema_failure; - const err_msg = try ErrorMsg.create(self.gpa, name_loc.start, "redefinition of '{}'", .{decl.name}); - errdefer err_msg.destroy(self.gpa); - try self.failed_decls.putNoClobber(self.gpa, decl, err_msg); - } else if (!srcHashEql(decl.contents_hash, contents_hash)) { - try self.markOutdatedDecl(decl); - decl.contents_hash = contents_hash; - } - } else { - const new_decl = try self.createNewDecl(&container_scope.base, name, decl_i, name_hash, contents_hash); - container_scope.decls.putAssumeCapacity(new_decl, {}); - if (var_decl.getExternExportToken()) |maybe_export_token| { - if (tree.token_ids[maybe_export_token] == .Keyword_export) { - self.work_queue.writeItemAssumeCapacity(.{ .analyze_decl = new_decl }); - } - } - } - } else if (src_decl.castTag(.Comptime)) |comptime_node| { - const name_index = self.getNextAnonNameIndex(); - const name = try std.fmt.allocPrint(self.gpa, "__comptime_{}", .{name_index}); - defer self.gpa.free(name); - - const name_hash = container_scope.fullyQualifiedNameHash(name); - const contents_hash = std.zig.hashSrc(tree.getNodeSource(src_decl)); - - const new_decl = try self.createNewDecl(&container_scope.base, name, decl_i, name_hash, contents_hash); - container_scope.decls.putAssumeCapacity(new_decl, {}); - self.work_queue.writeItemAssumeCapacity(.{ .analyze_decl = new_decl }); - } else if (src_decl.castTag(.ContainerField)) |container_field| { - log.err("TODO: analyze container field", .{}); - } else if (src_decl.castTag(.TestDecl)) |test_decl| { - log.err("TODO: analyze test decl", .{}); - } else if (src_decl.castTag(.Use)) |use_decl| { - log.err("TODO: analyze usingnamespace decl", .{}); - } else { - unreachable; - } - } - // Handle explicitly deleted decls from the source code. Not to be confused - // with when we delete decls because they are no longer referenced. - for (deleted_decls.items()) |entry| { - log.debug("noticed '{}' deleted from source\n", .{entry.key.name}); - try self.deleteDecl(entry.key); - } -} - -fn analyzeRootZIRModule(self: *Module, root_scope: *Scope.ZIRModule) !void { - // We may be analyzing it for the first time, or this may be - // an incremental update. This code handles both cases. - const src_module = try self.getSrcModule(root_scope); - - try self.work_queue.ensureUnusedCapacity(src_module.decls.len); - try root_scope.decls.ensureCapacity(self.gpa, src_module.decls.len); - - var exports_to_resolve = std.ArrayList(*zir.Decl).init(self.gpa); - defer exports_to_resolve.deinit(); - - // Keep track of the decls that we expect to see in this file so that - // we know which ones have been deleted. - var deleted_decls = std.AutoArrayHashMap(*Decl, void).init(self.gpa); - defer deleted_decls.deinit(); - try deleted_decls.ensureCapacity(self.decl_table.items().len); - for (self.decl_table.items()) |entry| { - deleted_decls.putAssumeCapacityNoClobber(entry.value, {}); - } - - for (src_module.decls) |src_decl, decl_i| { - const name_hash = root_scope.fullyQualifiedNameHash(src_decl.name); - if (self.decl_table.get(name_hash)) |decl| { - deleted_decls.removeAssertDiscard(decl); - if (!srcHashEql(src_decl.contents_hash, decl.contents_hash)) { - try self.markOutdatedDecl(decl); - decl.contents_hash = src_decl.contents_hash; - } - } else { - const new_decl = try self.createNewDecl( - &root_scope.base, - src_decl.name, - decl_i, - name_hash, - src_decl.contents_hash, - ); - root_scope.decls.appendAssumeCapacity(new_decl); - if (src_decl.inst.cast(zir.Inst.Export)) |export_inst| { - try exports_to_resolve.append(src_decl); - } - } - } - for (exports_to_resolve.items) |export_decl| { - _ = try zir_sema.resolveZirDecl(self, &root_scope.base, export_decl); - } - // Handle explicitly deleted decls from the source code. Not to be confused - // with when we delete decls because they are no longer referenced. - for (deleted_decls.items()) |entry| { - log.debug("noticed '{}' deleted from source\n", .{entry.key.name}); - try self.deleteDecl(entry.key); - } -} - -fn deleteDecl(self: *Module, decl: *Decl) !void { - try self.deletion_set.ensureCapacity(self.gpa, self.deletion_set.items.len + decl.dependencies.items().len); - - // Remove from the namespace it resides in. In the case of an anonymous Decl it will - // not be present in the set, and this does nothing. - decl.scope.removeDecl(decl); - - log.debug("deleting decl '{}'\n", .{decl.name}); - const name_hash = decl.fullyQualifiedNameHash(); - self.decl_table.removeAssertDiscard(name_hash); - // Remove itself from its dependencies, because we are about to destroy the decl pointer. - for (decl.dependencies.items()) |entry| { - const dep = entry.key; - dep.removeDependant(decl); - if (dep.dependants.items().len == 0 and !dep.deletion_flag) { - // We don't recursively perform a deletion here, because during the update, - // another reference to it may turn up. - dep.deletion_flag = true; - self.deletion_set.appendAssumeCapacity(dep); - } - } - // Anything that depends on this deleted decl certainly needs to be re-analyzed. - for (decl.dependants.items()) |entry| { - const dep = entry.key; - dep.removeDependency(decl); - if (dep.analysis != .outdated) { - // TODO Move this failure possibility to the top of the function. - try self.markOutdatedDecl(dep); - } - } - if (self.failed_decls.remove(decl)) |entry| { - entry.value.destroy(self.gpa); - } - self.deleteDeclExports(decl); - self.bin_file.freeDecl(decl); - decl.destroy(self.gpa); -} - -/// Delete all the Export objects that are caused by this Decl. Re-analysis of -/// this Decl will cause them to be re-created (or not). -fn deleteDeclExports(self: *Module, decl: *Decl) void { - const kv = self.export_owners.remove(decl) orelse return; - - for (kv.value) |exp| { - if (self.decl_exports.getEntry(exp.exported_decl)) |decl_exports_kv| { - // Remove exports with owner_decl matching the regenerating decl. - const list = decl_exports_kv.value; - var i: usize = 0; - var new_len = list.len; - while (i < new_len) { - if (list[i].owner_decl == decl) { - mem.copyBackwards(*Export, list[i..], list[i + 1 .. new_len]); - new_len -= 1; - } else { - i += 1; - } - } - decl_exports_kv.value = self.gpa.shrink(list, new_len); - if (new_len == 0) { - self.decl_exports.removeAssertDiscard(exp.exported_decl); - } - } - if (self.bin_file.cast(link.File.Elf)) |elf| { - elf.deleteExport(exp.link); - } - if (self.failed_exports.remove(exp)) |entry| { - entry.value.destroy(self.gpa); - } - _ = self.symbol_exports.remove(exp.options.name); - self.gpa.free(exp.options.name); - self.gpa.destroy(exp); - } - self.gpa.free(kv.value); -} - -fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void { - const tracy = trace(@src()); - defer tracy.end(); - - // Use the Decl's arena for function memory. - var arena = decl.typed_value.most_recent.arena.?.promote(self.gpa); - defer decl.typed_value.most_recent.arena.?.* = arena.state; - var inner_block: Scope.Block = .{ - .parent = null, - .func = func, - .decl = decl, - .instructions = .{}, - .arena = &arena.allocator, - .is_comptime = false, - }; - defer inner_block.instructions.deinit(self.gpa); - - const fn_zir = func.analysis.queued; - defer fn_zir.arena.promote(self.gpa).deinit(); - func.analysis = .{ .in_progress = {} }; - log.debug("set {} to in_progress\n", .{decl.name}); - - try zir_sema.analyzeBody(self, &inner_block.base, fn_zir.body); - - const instructions = try arena.allocator.dupe(*Inst, inner_block.instructions.items); - func.analysis = .{ .success = .{ .instructions = instructions } }; - log.debug("set {} to success\n", .{decl.name}); -} - -fn markOutdatedDecl(self: *Module, decl: *Decl) !void { - log.debug("mark {} outdated\n", .{decl.name}); - try self.work_queue.writeItem(.{ .analyze_decl = decl }); - if (self.failed_decls.remove(decl)) |entry| { - entry.value.destroy(self.gpa); - } - decl.analysis = .outdated; -} - -fn allocateNewDecl( - self: *Module, - scope: *Scope, - src_index: usize, - contents_hash: std.zig.SrcHash, -) !*Decl { - const new_decl = try self.gpa.create(Decl); - new_decl.* = .{ - .name = "", - .scope = scope.namespace(), - .src_index = src_index, - .typed_value = .{ .never_succeeded = {} }, - .analysis = .unreferenced, - .deletion_flag = false, - .contents_hash = contents_hash, - .link = switch (self.bin_file.tag) { - .coff => .{ .coff = link.File.Coff.TextBlock.empty }, - .elf => .{ .elf = link.File.Elf.TextBlock.empty }, - .macho => .{ .macho = link.File.MachO.TextBlock.empty }, - .c => .{ .c = {} }, - .wasm => .{ .wasm = {} }, - }, - .fn_link = switch (self.bin_file.tag) { - .coff => .{ .coff = {} }, - .elf => .{ .elf = link.File.Elf.SrcFn.empty }, - .macho => .{ .macho = link.File.MachO.SrcFn.empty }, - .c => .{ .c = {} }, - .wasm => .{ .wasm = null }, - }, - .generation = 0, - .is_pub = false, - }; - return new_decl; -} - -fn createNewDecl( - self: *Module, - scope: *Scope, - decl_name: []const u8, - src_index: usize, - name_hash: Scope.NameHash, - contents_hash: std.zig.SrcHash, -) !*Decl { - try self.decl_table.ensureCapacity(self.gpa, self.decl_table.items().len + 1); - const new_decl = try self.allocateNewDecl(scope, src_index, contents_hash); - errdefer self.gpa.destroy(new_decl); - new_decl.name = try mem.dupeZ(self.gpa, u8, decl_name); - self.decl_table.putAssumeCapacityNoClobber(name_hash, new_decl); - return new_decl; -} - -/// Get error value for error tag `name`. -pub fn getErrorValue(self: *Module, name: []const u8) !std.StringHashMapUnmanaged(u16).Entry { - const gop = try self.global_error_set.getOrPut(self.gpa, name); - if (gop.found_existing) - return gop.entry.*; - errdefer self.global_error_set.removeAssertDiscard(name); - - gop.entry.key = try self.gpa.dupe(u8, name); - gop.entry.value = @intCast(u16, self.global_error_set.count() - 1); - return gop.entry.*; -} - -pub fn requireFunctionBlock(self: *Module, scope: *Scope, src: usize) !*Scope.Block { - return scope.cast(Scope.Block) orelse - return self.fail(scope, src, "instruction illegal outside function body", .{}); -} - -pub fn requireRuntimeBlock(self: *Module, scope: *Scope, src: usize) !*Scope.Block { - const block = try self.requireFunctionBlock(scope, src); - if (block.is_comptime) { - return self.fail(scope, src, "unable to resolve comptime value", .{}); - } - return block; -} - -pub fn resolveConstValue(self: *Module, scope: *Scope, base: *Inst) !Value { - return (try self.resolveDefinedValue(scope, base)) orelse - return self.fail(scope, base.src, "unable to resolve comptime value", .{}); -} - -pub fn resolveDefinedValue(self: *Module, scope: *Scope, base: *Inst) !?Value { - if (base.value()) |val| { - if (val.isUndef()) { - return self.fail(scope, base.src, "use of undefined value here causes undefined behavior", .{}); - } - return val; - } - return null; -} - -pub fn analyzeExport(self: *Module, scope: *Scope, src: usize, borrowed_symbol_name: []const u8, exported_decl: *Decl) !void { - try self.ensureDeclAnalyzed(exported_decl); - const typed_value = exported_decl.typed_value.most_recent.typed_value; - switch (typed_value.ty.zigTypeTag()) { - .Fn => {}, - else => return self.fail(scope, src, "unable to export type '{}'", .{typed_value.ty}), - } - - try self.decl_exports.ensureCapacity(self.gpa, self.decl_exports.items().len + 1); - try self.export_owners.ensureCapacity(self.gpa, self.export_owners.items().len + 1); - - const new_export = try self.gpa.create(Export); - errdefer self.gpa.destroy(new_export); - - const symbol_name = try self.gpa.dupe(u8, borrowed_symbol_name); - errdefer self.gpa.free(symbol_name); - - const owner_decl = scope.decl().?; - - new_export.* = .{ - .options = .{ .name = symbol_name }, - .src = src, - .link = .{}, - .owner_decl = owner_decl, - .exported_decl = exported_decl, - .status = .in_progress, - }; - - // Add to export_owners table. - const eo_gop = self.export_owners.getOrPutAssumeCapacity(owner_decl); - if (!eo_gop.found_existing) { - eo_gop.entry.value = &[0]*Export{}; - } - eo_gop.entry.value = try self.gpa.realloc(eo_gop.entry.value, eo_gop.entry.value.len + 1); - eo_gop.entry.value[eo_gop.entry.value.len - 1] = new_export; - errdefer eo_gop.entry.value = self.gpa.shrink(eo_gop.entry.value, eo_gop.entry.value.len - 1); - - // Add to exported_decl table. - const de_gop = self.decl_exports.getOrPutAssumeCapacity(exported_decl); - if (!de_gop.found_existing) { - de_gop.entry.value = &[0]*Export{}; - } - de_gop.entry.value = try self.gpa.realloc(de_gop.entry.value, de_gop.entry.value.len + 1); - de_gop.entry.value[de_gop.entry.value.len - 1] = new_export; - errdefer de_gop.entry.value = self.gpa.shrink(de_gop.entry.value, de_gop.entry.value.len - 1); - - if (self.symbol_exports.get(symbol_name)) |_| { - try self.failed_exports.ensureCapacity(self.gpa, self.failed_exports.items().len + 1); - self.failed_exports.putAssumeCapacityNoClobber(new_export, try ErrorMsg.create( - self.gpa, - src, - "exported symbol collision: {}", - .{symbol_name}, - )); - // TODO: add a note - new_export.status = .failed; - return; - } - - try self.symbol_exports.putNoClobber(self.gpa, symbol_name, new_export); - self.bin_file.updateDeclExports(self, exported_decl, de_gop.entry.value) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - else => { - try self.failed_exports.ensureCapacity(self.gpa, self.failed_exports.items().len + 1); - self.failed_exports.putAssumeCapacityNoClobber(new_export, try ErrorMsg.create( - self.gpa, - src, - "unable to export: {}", - .{@errorName(err)}, - )); - new_export.status = .failed_retryable; - }, - }; -} - -pub fn addNoOp( - self: *Module, - block: *Scope.Block, - src: usize, - ty: Type, - comptime tag: Inst.Tag, -) !*Inst { - const inst = try block.arena.create(tag.Type()); - inst.* = .{ - .base = .{ - .tag = tag, - .ty = ty, - .src = src, - }, - }; - try block.instructions.append(self.gpa, &inst.base); - return &inst.base; -} - -pub fn addUnOp( - self: *Module, - block: *Scope.Block, - src: usize, - ty: Type, - tag: Inst.Tag, - operand: *Inst, -) !*Inst { - const inst = try block.arena.create(Inst.UnOp); - inst.* = .{ - .base = .{ - .tag = tag, - .ty = ty, - .src = src, - }, - .operand = operand, - }; - try block.instructions.append(self.gpa, &inst.base); - return &inst.base; -} - -pub fn addBinOp( - self: *Module, - block: *Scope.Block, - src: usize, - ty: Type, - tag: Inst.Tag, - lhs: *Inst, - rhs: *Inst, -) !*Inst { - const inst = try block.arena.create(Inst.BinOp); - inst.* = .{ - .base = .{ - .tag = tag, - .ty = ty, - .src = src, - }, - .lhs = lhs, - .rhs = rhs, - }; - try block.instructions.append(self.gpa, &inst.base); - return &inst.base; -} - -pub fn addArg(self: *Module, block: *Scope.Block, src: usize, ty: Type, name: [*:0]const u8) !*Inst { - const inst = try block.arena.create(Inst.Arg); - inst.* = .{ - .base = .{ - .tag = .arg, - .ty = ty, - .src = src, - }, - .name = name, - }; - try block.instructions.append(self.gpa, &inst.base); - return &inst.base; -} - -pub fn addBr( - self: *Module, - scope_block: *Scope.Block, - src: usize, - target_block: *Inst.Block, - operand: *Inst, -) !*Inst { - const inst = try scope_block.arena.create(Inst.Br); - inst.* = .{ - .base = .{ - .tag = .br, - .ty = Type.initTag(.noreturn), - .src = src, - }, - .operand = operand, - .block = target_block, - }; - try scope_block.instructions.append(self.gpa, &inst.base); - return &inst.base; -} - -pub fn addCondBr( - self: *Module, - block: *Scope.Block, - src: usize, - condition: *Inst, - then_body: ir.Body, - else_body: ir.Body, -) !*Inst { - const inst = try block.arena.create(Inst.CondBr); - inst.* = .{ - .base = .{ - .tag = .condbr, - .ty = Type.initTag(.noreturn), - .src = src, - }, - .condition = condition, - .then_body = then_body, - .else_body = else_body, - }; - try block.instructions.append(self.gpa, &inst.base); - return &inst.base; -} - -pub fn addCall( - self: *Module, - block: *Scope.Block, - src: usize, - ty: Type, - func: *Inst, - args: []const *Inst, -) !*Inst { - const inst = try block.arena.create(Inst.Call); - inst.* = .{ - .base = .{ - .tag = .call, - .ty = ty, - .src = src, - }, - .func = func, - .args = args, - }; - try block.instructions.append(self.gpa, &inst.base); - return &inst.base; -} - -pub fn constInst(self: *Module, scope: *Scope, src: usize, typed_value: TypedValue) !*Inst { - const const_inst = try scope.arena().create(Inst.Constant); - const_inst.* = .{ - .base = .{ - .tag = Inst.Constant.base_tag, - .ty = typed_value.ty, - .src = src, - }, - .val = typed_value.val, - }; - return &const_inst.base; -} - -pub fn constType(self: *Module, scope: *Scope, src: usize, ty: Type) !*Inst { - return self.constInst(scope, src, .{ - .ty = Type.initTag(.type), - .val = try ty.toValue(scope.arena()), - }); -} - -pub fn constVoid(self: *Module, scope: *Scope, src: usize) !*Inst { - return self.constInst(scope, src, .{ - .ty = Type.initTag(.void), - .val = Value.initTag(.void_value), - }); -} - -pub fn constNoReturn(self: *Module, scope: *Scope, src: usize) !*Inst { - return self.constInst(scope, src, .{ - .ty = Type.initTag(.noreturn), - .val = Value.initTag(.unreachable_value), - }); -} - -pub fn constUndef(self: *Module, scope: *Scope, src: usize, ty: Type) !*Inst { - return self.constInst(scope, src, .{ - .ty = ty, - .val = Value.initTag(.undef), - }); -} - -pub fn constBool(self: *Module, scope: *Scope, src: usize, v: bool) !*Inst { - return self.constInst(scope, src, .{ - .ty = Type.initTag(.bool), - .val = ([2]Value{ Value.initTag(.bool_false), Value.initTag(.bool_true) })[@boolToInt(v)], - }); -} - -pub fn constIntUnsigned(self: *Module, scope: *Scope, src: usize, ty: Type, int: u64) !*Inst { - const int_payload = try scope.arena().create(Value.Payload.Int_u64); - int_payload.* = .{ .int = int }; - - return self.constInst(scope, src, .{ - .ty = ty, - .val = Value.initPayload(&int_payload.base), - }); -} - -pub fn constIntSigned(self: *Module, scope: *Scope, src: usize, ty: Type, int: i64) !*Inst { - const int_payload = try scope.arena().create(Value.Payload.Int_i64); - int_payload.* = .{ .int = int }; - - return self.constInst(scope, src, .{ - .ty = ty, - .val = Value.initPayload(&int_payload.base), - }); -} - -pub fn constIntBig(self: *Module, scope: *Scope, src: usize, ty: Type, big_int: BigIntConst) !*Inst { - const val_payload = if (big_int.positive) blk: { - if (big_int.to(u64)) |x| { - return self.constIntUnsigned(scope, src, ty, x); - } else |err| switch (err) { - error.NegativeIntoUnsigned => unreachable, - error.TargetTooSmall => {}, // handled below - } - const big_int_payload = try scope.arena().create(Value.Payload.IntBigPositive); - big_int_payload.* = .{ .limbs = big_int.limbs }; - break :blk &big_int_payload.base; - } else blk: { - if (big_int.to(i64)) |x| { - return self.constIntSigned(scope, src, ty, x); - } else |err| switch (err) { - error.NegativeIntoUnsigned => unreachable, - error.TargetTooSmall => {}, // handled below - } - const big_int_payload = try scope.arena().create(Value.Payload.IntBigNegative); - big_int_payload.* = .{ .limbs = big_int.limbs }; - break :blk &big_int_payload.base; - }; - - return self.constInst(scope, src, .{ - .ty = ty, - .val = Value.initPayload(val_payload), - }); -} - -pub fn createAnonymousDecl( - self: *Module, - scope: *Scope, - decl_arena: *std.heap.ArenaAllocator, - typed_value: TypedValue, -) !*Decl { - const name_index = self.getNextAnonNameIndex(); - const scope_decl = scope.decl().?; - const name = try std.fmt.allocPrint(self.gpa, "{}__anon_{}", .{ scope_decl.name, name_index }); - defer self.gpa.free(name); - const name_hash = scope.namespace().fullyQualifiedNameHash(name); - const src_hash: std.zig.SrcHash = undefined; - const new_decl = try self.createNewDecl(scope, name, scope_decl.src_index, name_hash, src_hash); - const decl_arena_state = try decl_arena.allocator.create(std.heap.ArenaAllocator.State); - - decl_arena_state.* = decl_arena.state; - new_decl.typed_value = .{ - .most_recent = .{ - .typed_value = typed_value, - .arena = decl_arena_state, - }, - }; - new_decl.analysis = .complete; - new_decl.generation = self.generation; - - // TODO: This generates the Decl into the machine code file if it is of a type that is non-zero size. - // We should be able to further improve the compiler to not omit Decls which are only referenced at - // compile-time and not runtime. - if (typed_value.ty.hasCodeGenBits()) { - try self.bin_file.allocateDeclIndexes(new_decl); - try self.work_queue.writeItem(.{ .codegen_decl = new_decl }); - } - - return new_decl; -} - -fn getNextAnonNameIndex(self: *Module) usize { - return @atomicRmw(usize, &self.next_anon_name_index, .Add, 1, .Monotonic); -} - -pub fn lookupDeclName(self: *Module, scope: *Scope, ident_name: []const u8) ?*Decl { - const namespace = scope.namespace(); - const name_hash = namespace.fullyQualifiedNameHash(ident_name); - return self.decl_table.get(name_hash); -} - -pub fn analyzeDeclRef(self: *Module, scope: *Scope, src: usize, decl: *Decl) InnerError!*Inst { - const scope_decl = scope.decl().?; - try self.declareDeclDependency(scope_decl, decl); - self.ensureDeclAnalyzed(decl) catch |err| { - if (scope.cast(Scope.Block)) |block| { - if (block.func) |func| { - func.analysis = .dependency_failure; - } else { - block.decl.analysis = .dependency_failure; - } - } else { - scope_decl.analysis = .dependency_failure; - } - return err; - }; - - const decl_tv = try decl.typedValue(); - if (decl_tv.val.tag() == .variable) { - return self.analyzeVarRef(scope, src, decl_tv); - } - const ty = try self.simplePtrType(scope, src, decl_tv.ty, false, .One); - const val_payload = try scope.arena().create(Value.Payload.DeclRef); - val_payload.* = .{ .decl = decl }; - - return self.constInst(scope, src, .{ - .ty = ty, - .val = Value.initPayload(&val_payload.base), - }); -} - -fn analyzeVarRef(self: *Module, scope: *Scope, src: usize, tv: TypedValue) InnerError!*Inst { - const variable = tv.val.cast(Value.Payload.Variable).?.variable; - - const ty = try self.simplePtrType(scope, src, tv.ty, variable.is_mutable, .One); - if (!variable.is_mutable and !variable.is_extern) { - const val_payload = try scope.arena().create(Value.Payload.RefVal); - val_payload.* = .{ .val = variable.init }; - return self.constInst(scope, src, .{ - .ty = ty, - .val = Value.initPayload(&val_payload.base), - }); - } - - const b = try self.requireRuntimeBlock(scope, src); - const inst = try b.arena.create(Inst.VarPtr); - inst.* = .{ - .base = .{ - .tag = .varptr, - .ty = ty, - .src = src, - }, - .variable = variable, - }; - try b.instructions.append(self.gpa, &inst.base); - return &inst.base; -} - -pub fn analyzeDeref(self: *Module, scope: *Scope, src: usize, ptr: *Inst, ptr_src: usize) InnerError!*Inst { - const elem_ty = switch (ptr.ty.zigTypeTag()) { - .Pointer => ptr.ty.elemType(), - else => return self.fail(scope, ptr_src, "expected pointer, found '{}'", .{ptr.ty}), - }; - if (ptr.value()) |val| { - return self.constInst(scope, src, .{ - .ty = elem_ty, - .val = try val.pointerDeref(scope.arena()), - }); - } - - const b = try self.requireRuntimeBlock(scope, src); - return self.addUnOp(b, src, elem_ty, .load, ptr); -} - -pub fn analyzeDeclRefByName(self: *Module, scope: *Scope, src: usize, decl_name: []const u8) InnerError!*Inst { - const decl = self.lookupDeclName(scope, decl_name) orelse - return self.fail(scope, src, "decl '{}' not found", .{decl_name}); - return self.analyzeDeclRef(scope, src, decl); -} - -pub fn wantSafety(self: *Module, scope: *Scope) bool { - // TODO take into account scope's safety overrides - return switch (self.optimizeMode()) { - .Debug => true, - .ReleaseSafe => true, - .ReleaseFast => false, - .ReleaseSmall => false, - }; -} - -pub fn analyzeIsNull( - self: *Module, - scope: *Scope, - src: usize, - operand: *Inst, - invert_logic: bool, -) InnerError!*Inst { - if (operand.value()) |opt_val| { - const is_null = opt_val.isNull(); - const bool_value = if (invert_logic) !is_null else is_null; - return self.constBool(scope, src, bool_value); - } - const b = try self.requireRuntimeBlock(scope, src); - const inst_tag: Inst.Tag = if (invert_logic) .isnonnull else .isnull; - return self.addUnOp(b, src, Type.initTag(.bool), inst_tag, operand); -} - -pub fn analyzeIsErr(self: *Module, scope: *Scope, src: usize, operand: *Inst) InnerError!*Inst { - return self.fail(scope, src, "TODO implement analysis of iserr", .{}); -} - -pub fn analyzeSlice(self: *Module, scope: *Scope, src: usize, array_ptr: *Inst, start: *Inst, end_opt: ?*Inst, sentinel_opt: ?*Inst) InnerError!*Inst { - const ptr_child = switch (array_ptr.ty.zigTypeTag()) { - .Pointer => array_ptr.ty.elemType(), - else => return self.fail(scope, src, "expected pointer, found '{}'", .{array_ptr.ty}), - }; - - var array_type = ptr_child; - const elem_type = switch (ptr_child.zigTypeTag()) { - .Array => ptr_child.elemType(), - .Pointer => blk: { - if (ptr_child.isSinglePointer()) { - if (ptr_child.elemType().zigTypeTag() == .Array) { - array_type = ptr_child.elemType(); - break :blk ptr_child.elemType().elemType(); - } - - return self.fail(scope, src, "slice of single-item pointer", .{}); - } - break :blk ptr_child.elemType(); - }, - else => return self.fail(scope, src, "slice of non-array type '{}'", .{ptr_child}), - }; - - const slice_sentinel = if (sentinel_opt) |sentinel| blk: { - const casted = try self.coerce(scope, elem_type, sentinel); - break :blk try self.resolveConstValue(scope, casted); - } else null; - - var return_ptr_size: std.builtin.TypeInfo.Pointer.Size = .Slice; - var return_elem_type = elem_type; - if (end_opt) |end| { - if (end.value()) |end_val| { - if (start.value()) |start_val| { - const start_u64 = start_val.toUnsignedInt(); - const end_u64 = end_val.toUnsignedInt(); - if (start_u64 > end_u64) { - return self.fail(scope, src, "out of bounds slice", .{}); - } - - const len = end_u64 - start_u64; - const array_sentinel = if (array_type.zigTypeTag() == .Array and end_u64 == array_type.arrayLen()) - array_type.sentinel() - else - slice_sentinel; - return_elem_type = try self.arrayType(scope, len, array_sentinel, elem_type); - return_ptr_size = .One; - } - } - } - const return_type = try self.ptrType( - scope, - src, - return_elem_type, - if (end_opt == null) slice_sentinel else null, - 0, // TODO alignment - 0, - 0, - !ptr_child.isConstPtr(), - ptr_child.isAllowzeroPtr(), - ptr_child.isVolatilePtr(), - return_ptr_size, - ); - - return self.fail(scope, src, "TODO implement analysis of slice", .{}); -} - -/// Asserts that lhs and rhs types are both numeric. -pub fn cmpNumeric( - self: *Module, - scope: *Scope, - src: usize, - lhs: *Inst, - rhs: *Inst, - op: std.math.CompareOperator, -) !*Inst { - assert(lhs.ty.isNumeric()); - assert(rhs.ty.isNumeric()); - - const lhs_ty_tag = lhs.ty.zigTypeTag(); - const rhs_ty_tag = rhs.ty.zigTypeTag(); - - if (lhs_ty_tag == .Vector and rhs_ty_tag == .Vector) { - if (lhs.ty.arrayLen() != rhs.ty.arrayLen()) { - return self.fail(scope, src, "vector length mismatch: {} and {}", .{ - lhs.ty.arrayLen(), - rhs.ty.arrayLen(), - }); - } - return self.fail(scope, src, "TODO implement support for vectors in cmpNumeric", .{}); - } else if (lhs_ty_tag == .Vector or rhs_ty_tag == .Vector) { - return self.fail(scope, src, "mixed scalar and vector operands to comparison operator: '{}' and '{}'", .{ - lhs.ty, - rhs.ty, - }); - } - - if (lhs.value()) |lhs_val| { - if (rhs.value()) |rhs_val| { - return self.constBool(scope, src, Value.compare(lhs_val, op, rhs_val)); - } - } - - // TODO handle comparisons against lazy zero values - // Some values can be compared against zero without being runtime known or without forcing - // a full resolution of their value, for example `@sizeOf(@Frame(function))` is known to - // always be nonzero, and we benefit from not forcing the full evaluation and stack frame layout - // of this function if we don't need to. - - // It must be a runtime comparison. - const b = try self.requireRuntimeBlock(scope, src); - // For floats, emit a float comparison instruction. - const lhs_is_float = switch (lhs_ty_tag) { - .Float, .ComptimeFloat => true, - else => false, - }; - const rhs_is_float = switch (rhs_ty_tag) { - .Float, .ComptimeFloat => true, - else => false, - }; - if (lhs_is_float and rhs_is_float) { - // Implicit cast the smaller one to the larger one. - const dest_type = x: { - if (lhs_ty_tag == .ComptimeFloat) { - break :x rhs.ty; - } else if (rhs_ty_tag == .ComptimeFloat) { - break :x lhs.ty; - } - if (lhs.ty.floatBits(self.getTarget()) >= rhs.ty.floatBits(self.getTarget())) { - break :x lhs.ty; - } else { - break :x rhs.ty; - } - }; - const casted_lhs = try self.coerce(scope, dest_type, lhs); - const casted_rhs = try self.coerce(scope, dest_type, rhs); - return self.addBinOp(b, src, dest_type, Inst.Tag.fromCmpOp(op), casted_lhs, casted_rhs); - } - // For mixed unsigned integer sizes, implicit cast both operands to the larger integer. - // For mixed signed and unsigned integers, implicit cast both operands to a signed - // integer with + 1 bit. - // For mixed floats and integers, extract the integer part from the float, cast that to - // a signed integer with mantissa bits + 1, and if there was any non-integral part of the float, - // add/subtract 1. - const lhs_is_signed = if (lhs.value()) |lhs_val| - lhs_val.compareWithZero(.lt) - else - (lhs.ty.isFloat() or lhs.ty.isSignedInt()); - const rhs_is_signed = if (rhs.value()) |rhs_val| - rhs_val.compareWithZero(.lt) - else - (rhs.ty.isFloat() or rhs.ty.isSignedInt()); - const dest_int_is_signed = lhs_is_signed or rhs_is_signed; - - var dest_float_type: ?Type = null; - - var lhs_bits: usize = undefined; - if (lhs.value()) |lhs_val| { - if (lhs_val.isUndef()) - return self.constUndef(scope, src, Type.initTag(.bool)); - const is_unsigned = if (lhs_is_float) x: { - var bigint_space: Value.BigIntSpace = undefined; - var bigint = try lhs_val.toBigInt(&bigint_space).toManaged(self.gpa); - defer bigint.deinit(); - const zcmp = lhs_val.orderAgainstZero(); - if (lhs_val.floatHasFraction()) { - switch (op) { - .eq => return self.constBool(scope, src, false), - .neq => return self.constBool(scope, src, true), - else => {}, - } - if (zcmp == .lt) { - try bigint.addScalar(bigint.toConst(), -1); - } else { - try bigint.addScalar(bigint.toConst(), 1); - } - } - lhs_bits = bigint.toConst().bitCountTwosComp(); - break :x (zcmp != .lt); - } else x: { - lhs_bits = lhs_val.intBitCountTwosComp(); - break :x (lhs_val.orderAgainstZero() != .lt); - }; - lhs_bits += @boolToInt(is_unsigned and dest_int_is_signed); - } else if (lhs_is_float) { - dest_float_type = lhs.ty; - } else { - const int_info = lhs.ty.intInfo(self.getTarget()); - lhs_bits = int_info.bits + @boolToInt(!int_info.signed and dest_int_is_signed); - } - - var rhs_bits: usize = undefined; - if (rhs.value()) |rhs_val| { - if (rhs_val.isUndef()) - return self.constUndef(scope, src, Type.initTag(.bool)); - const is_unsigned = if (rhs_is_float) x: { - var bigint_space: Value.BigIntSpace = undefined; - var bigint = try rhs_val.toBigInt(&bigint_space).toManaged(self.gpa); - defer bigint.deinit(); - const zcmp = rhs_val.orderAgainstZero(); - if (rhs_val.floatHasFraction()) { - switch (op) { - .eq => return self.constBool(scope, src, false), - .neq => return self.constBool(scope, src, true), - else => {}, - } - if (zcmp == .lt) { - try bigint.addScalar(bigint.toConst(), -1); - } else { - try bigint.addScalar(bigint.toConst(), 1); - } - } - rhs_bits = bigint.toConst().bitCountTwosComp(); - break :x (zcmp != .lt); - } else x: { - rhs_bits = rhs_val.intBitCountTwosComp(); - break :x (rhs_val.orderAgainstZero() != .lt); - }; - rhs_bits += @boolToInt(is_unsigned and dest_int_is_signed); - } else if (rhs_is_float) { - dest_float_type = rhs.ty; - } else { - const int_info = rhs.ty.intInfo(self.getTarget()); - rhs_bits = int_info.bits + @boolToInt(!int_info.signed and dest_int_is_signed); - } - - const dest_type = if (dest_float_type) |ft| ft else blk: { - const max_bits = std.math.max(lhs_bits, rhs_bits); - const casted_bits = std.math.cast(u16, max_bits) catch |err| switch (err) { - error.Overflow => return self.fail(scope, src, "{} exceeds maximum integer bit count", .{max_bits}), - }; - break :blk try self.makeIntType(scope, dest_int_is_signed, casted_bits); - }; - const casted_lhs = try self.coerce(scope, dest_type, lhs); - const casted_rhs = try self.coerce(scope, dest_type, rhs); - - return self.addBinOp(b, src, Type.initTag(.bool), Inst.Tag.fromCmpOp(op), casted_lhs, casted_rhs); -} - -fn wrapOptional(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst { - if (inst.value()) |val| { - return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = val }); - } - - const b = try self.requireRuntimeBlock(scope, inst.src); - return self.addUnOp(b, inst.src, dest_type, .wrap_optional, inst); -} - -fn makeIntType(self: *Module, scope: *Scope, signed: bool, bits: u16) !Type { - if (signed) { - const int_payload = try scope.arena().create(Type.Payload.IntSigned); - int_payload.* = .{ .bits = bits }; - return Type.initPayload(&int_payload.base); - } else { - const int_payload = try scope.arena().create(Type.Payload.IntUnsigned); - int_payload.* = .{ .bits = bits }; - return Type.initPayload(&int_payload.base); - } -} - -pub fn resolvePeerTypes(self: *Module, scope: *Scope, instructions: []*Inst) !Type { - if (instructions.len == 0) - return Type.initTag(.noreturn); - - if (instructions.len == 1) - return instructions[0].ty; - - var prev_inst = instructions[0]; - for (instructions[1..]) |next_inst| { - if (next_inst.ty.eql(prev_inst.ty)) - continue; - if (next_inst.ty.zigTypeTag() == .NoReturn) - continue; - if (prev_inst.ty.zigTypeTag() == .NoReturn) { - prev_inst = next_inst; - continue; - } - if (next_inst.ty.zigTypeTag() == .Undefined) - continue; - if (prev_inst.ty.zigTypeTag() == .Undefined) { - prev_inst = next_inst; - continue; - } - if (prev_inst.ty.isInt() and - next_inst.ty.isInt() and - prev_inst.ty.isSignedInt() == next_inst.ty.isSignedInt()) - { - if (prev_inst.ty.intInfo(self.getTarget()).bits < next_inst.ty.intInfo(self.getTarget()).bits) { - prev_inst = next_inst; - } - continue; - } - if (prev_inst.ty.isFloat() and next_inst.ty.isFloat()) { - if (prev_inst.ty.floatBits(self.getTarget()) < next_inst.ty.floatBits(self.getTarget())) { - prev_inst = next_inst; - } - continue; - } - - // TODO error notes pointing out each type - return self.fail(scope, next_inst.src, "incompatible types: '{}' and '{}'", .{ prev_inst.ty, next_inst.ty }); - } - - return prev_inst.ty; -} - -pub fn coerce(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst { - // If the types are the same, we can return the operand. - if (dest_type.eql(inst.ty)) - return inst; - - const in_memory_result = coerceInMemoryAllowed(dest_type, inst.ty); - if (in_memory_result == .ok) { - return self.bitcast(scope, dest_type, inst); - } - - // undefined to anything - if (inst.value()) |val| { - if (val.isUndef() or inst.ty.zigTypeTag() == .Undefined) { - return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = val }); - } - } - assert(inst.ty.zigTypeTag() != .Undefined); - - // null to ?T - if (dest_type.zigTypeTag() == .Optional and inst.ty.zigTypeTag() == .Null) { - return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = Value.initTag(.null_value) }); - } - - // T to ?T - if (dest_type.zigTypeTag() == .Optional) { - var buf: Type.Payload.PointerSimple = undefined; - const child_type = dest_type.optionalChild(&buf); - if (child_type.eql(inst.ty)) { - return self.wrapOptional(scope, dest_type, inst); - } else if (try self.coerceNum(scope, child_type, inst)) |some| { - return self.wrapOptional(scope, dest_type, some); - } - } - - // *[N]T to []T - if (inst.ty.isSinglePointer() and dest_type.isSlice() and - (!inst.ty.isConstPtr() or dest_type.isConstPtr())) - { - const array_type = inst.ty.elemType(); - const dst_elem_type = dest_type.elemType(); - if (array_type.zigTypeTag() == .Array and - coerceInMemoryAllowed(dst_elem_type, array_type.elemType()) == .ok) - { - return self.coerceArrayPtrToSlice(scope, dest_type, inst); - } - } - - // comptime known number to other number - if (try self.coerceNum(scope, dest_type, inst)) |some| - return some; - - // integer widening - if (inst.ty.zigTypeTag() == .Int and dest_type.zigTypeTag() == .Int) { - assert(inst.value() == null); // handled above - - const src_info = inst.ty.intInfo(self.getTarget()); - const dst_info = dest_type.intInfo(self.getTarget()); - if ((src_info.signed == dst_info.signed and dst_info.bits >= src_info.bits) or - // small enough unsigned ints can get casted to large enough signed ints - (src_info.signed and !dst_info.signed and dst_info.bits > src_info.bits)) - { - const b = try self.requireRuntimeBlock(scope, inst.src); - return self.addUnOp(b, inst.src, dest_type, .intcast, inst); - } - } - - // float widening - if (inst.ty.zigTypeTag() == .Float and dest_type.zigTypeTag() == .Float) { - assert(inst.value() == null); // handled above - - const src_bits = inst.ty.floatBits(self.getTarget()); - const dst_bits = dest_type.floatBits(self.getTarget()); - if (dst_bits >= src_bits) { - const b = try self.requireRuntimeBlock(scope, inst.src); - return self.addUnOp(b, inst.src, dest_type, .floatcast, inst); - } - } - - return self.fail(scope, inst.src, "expected {}, found {}", .{ dest_type, inst.ty }); -} - -pub fn coerceNum(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !?*Inst { - const val = inst.value() orelse return null; - const src_zig_tag = inst.ty.zigTypeTag(); - const dst_zig_tag = dest_type.zigTypeTag(); - - if (dst_zig_tag == .ComptimeInt or dst_zig_tag == .Int) { - if (src_zig_tag == .Float or src_zig_tag == .ComptimeFloat) { - if (val.floatHasFraction()) { - return self.fail(scope, inst.src, "fractional component prevents float value {} from being casted to type '{}'", .{ val, inst.ty }); - } - return self.fail(scope, inst.src, "TODO float to int", .{}); - } else if (src_zig_tag == .Int or src_zig_tag == .ComptimeInt) { - if (!val.intFitsInType(dest_type, self.getTarget())) { - return self.fail(scope, inst.src, "type {} cannot represent integer value {}", .{ inst.ty, val }); - } - return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = val }); - } - } else if (dst_zig_tag == .ComptimeFloat or dst_zig_tag == .Float) { - if (src_zig_tag == .Float or src_zig_tag == .ComptimeFloat) { - const res = val.floatCast(scope.arena(), dest_type, self.getTarget()) catch |err| switch (err) { - error.Overflow => return self.fail( - scope, - inst.src, - "cast of value {} to type '{}' loses information", - .{ val, dest_type }, - ), - error.OutOfMemory => return error.OutOfMemory, - }; - return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = res }); - } else if (src_zig_tag == .Int or src_zig_tag == .ComptimeInt) { - return self.fail(scope, inst.src, "TODO int to float", .{}); - } - } - return null; -} - -pub fn storePtr(self: *Module, scope: *Scope, src: usize, ptr: *Inst, uncasted_value: *Inst) !*Inst { - if (ptr.ty.isConstPtr()) - return self.fail(scope, src, "cannot assign to constant", .{}); - - const elem_ty = ptr.ty.elemType(); - const value = try self.coerce(scope, elem_ty, uncasted_value); - if (elem_ty.onePossibleValue() != null) - return self.constVoid(scope, src); - - // TODO handle comptime pointer writes - // TODO handle if the element type requires comptime - - const b = try self.requireRuntimeBlock(scope, src); - return self.addBinOp(b, src, Type.initTag(.void), .store, ptr, value); -} - -pub fn bitcast(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst { - if (inst.value()) |val| { - // Keep the comptime Value representation; take the new type. - return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = val }); - } - // TODO validate the type size and other compile errors - const b = try self.requireRuntimeBlock(scope, inst.src); - return self.addUnOp(b, inst.src, dest_type, .bitcast, inst); -} - -fn coerceArrayPtrToSlice(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst { - if (inst.value()) |val| { - // The comptime Value representation is compatible with both types. - return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = val }); - } - return self.fail(scope, inst.src, "TODO implement coerceArrayPtrToSlice runtime instruction", .{}); -} - fn failCObj(mod: *Module, c_object: *CObject, comptime format: []const u8, args: anytype) InnerError { @setCold(true); const err_msg = try ErrorMsg.create(mod.gpa, 0, "unable to build C object: " ++ format, args); @@ -3935,107 +1284,10 @@ fn failCObjWithOwnedErrorMsg(mod: *Module, c_object: *CObject, err_msg: *ErrorMs try mod.failed_c_objects.ensureCapacity(mod.gpa, mod.failed_c_objects.items().len + 1); } mod.failed_c_objects.putAssumeCapacityNoClobber(c_object, err_msg); - c_object.status = .{ .failure = "" }; + c_object.status = .failure; return error.AnalysisFail; } -pub fn fail(self: *Module, scope: *Scope, src: usize, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); - const err_msg = try ErrorMsg.create(self.gpa, src, format, args); - return self.failWithOwnedErrorMsg(scope, src, err_msg); -} - -pub fn failTok( - self: *Module, - scope: *Scope, - token_index: ast.TokenIndex, - comptime format: []const u8, - args: anytype, -) InnerError { - @setCold(true); - const src = scope.tree().token_locs[token_index].start; - return self.fail(scope, src, format, args); -} - -pub fn failNode( - self: *Module, - scope: *Scope, - ast_node: *ast.Node, - comptime format: []const u8, - args: anytype, -) InnerError { - @setCold(true); - const src = scope.tree().token_locs[ast_node.firstToken()].start; - return self.fail(scope, src, format, args); -} - -fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *ErrorMsg) InnerError { - { - errdefer err_msg.destroy(self.gpa); - try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1); - try self.failed_files.ensureCapacity(self.gpa, self.failed_files.items().len + 1); - } - switch (scope.tag) { - .decl => { - const decl = scope.cast(Scope.DeclAnalysis).?.decl; - decl.analysis = .sema_failure; - decl.generation = self.generation; - self.failed_decls.putAssumeCapacityNoClobber(decl, err_msg); - }, - .block => { - const block = scope.cast(Scope.Block).?; - if (block.func) |func| { - func.analysis = .sema_failure; - } else { - block.decl.analysis = .sema_failure; - block.decl.generation = self.generation; - } - self.failed_decls.putAssumeCapacityNoClobber(block.decl, err_msg); - }, - .gen_zir => { - const gen_zir = scope.cast(Scope.GenZIR).?; - gen_zir.decl.analysis = .sema_failure; - gen_zir.decl.generation = self.generation; - self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg); - }, - .local_val => { - const gen_zir = scope.cast(Scope.LocalVal).?.gen_zir; - gen_zir.decl.analysis = .sema_failure; - gen_zir.decl.generation = self.generation; - self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg); - }, - .local_ptr => { - const gen_zir = scope.cast(Scope.LocalPtr).?.gen_zir; - gen_zir.decl.analysis = .sema_failure; - gen_zir.decl.generation = self.generation; - self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg); - }, - .zir_module => { - const zir_module = scope.cast(Scope.ZIRModule).?; - zir_module.status = .loaded_sema_failure; - self.failed_files.putAssumeCapacityNoClobber(scope, err_msg); - }, - .none => unreachable, - .file => unreachable, - .container => unreachable, - } - return error.AnalysisFail; -} - -const InMemoryCoercionResult = enum { - ok, - no_match, -}; - -fn coerceInMemoryAllowed(dest_type: Type, src_type: Type) InMemoryCoercionResult { - if (dest_type.eql(src_type)) - return .ok; - - // TODO: implement more of this function - - return .no_match; -} - pub const ErrorMsg = struct { byte_offset: usize, msg: []const u8, @@ -4066,375 +1318,6 @@ pub const ErrorMsg = struct { } }; -fn srcHashEql(a: std.zig.SrcHash, b: std.zig.SrcHash) bool { - return @bitCast(u128, a) == @bitCast(u128, b); -} - -pub fn intAdd(allocator: *Allocator, lhs: Value, rhs: Value) !Value { - // TODO is this a performance issue? maybe we should try the operation without - // resorting to BigInt first. - var lhs_space: Value.BigIntSpace = undefined; - var rhs_space: Value.BigIntSpace = undefined; - const lhs_bigint = lhs.toBigInt(&lhs_space); - const rhs_bigint = rhs.toBigInt(&rhs_space); - const limbs = try allocator.alloc( - std.math.big.Limb, - std.math.max(lhs_bigint.limbs.len, rhs_bigint.limbs.len) + 1, - ); - var result_bigint = BigIntMutable{ .limbs = limbs, .positive = undefined, .len = undefined }; - result_bigint.add(lhs_bigint, rhs_bigint); - const result_limbs = result_bigint.limbs[0..result_bigint.len]; - - const val_payload = if (result_bigint.positive) blk: { - const val_payload = try allocator.create(Value.Payload.IntBigPositive); - val_payload.* = .{ .limbs = result_limbs }; - break :blk &val_payload.base; - } else blk: { - const val_payload = try allocator.create(Value.Payload.IntBigNegative); - val_payload.* = .{ .limbs = result_limbs }; - break :blk &val_payload.base; - }; - - return Value.initPayload(val_payload); -} - -pub fn intSub(allocator: *Allocator, lhs: Value, rhs: Value) !Value { - // TODO is this a performance issue? maybe we should try the operation without - // resorting to BigInt first. - var lhs_space: Value.BigIntSpace = undefined; - var rhs_space: Value.BigIntSpace = undefined; - const lhs_bigint = lhs.toBigInt(&lhs_space); - const rhs_bigint = rhs.toBigInt(&rhs_space); - const limbs = try allocator.alloc( - std.math.big.Limb, - std.math.max(lhs_bigint.limbs.len, rhs_bigint.limbs.len) + 1, - ); - var result_bigint = BigIntMutable{ .limbs = limbs, .positive = undefined, .len = undefined }; - result_bigint.sub(lhs_bigint, rhs_bigint); - const result_limbs = result_bigint.limbs[0..result_bigint.len]; - - const val_payload = if (result_bigint.positive) blk: { - const val_payload = try allocator.create(Value.Payload.IntBigPositive); - val_payload.* = .{ .limbs = result_limbs }; - break :blk &val_payload.base; - } else blk: { - const val_payload = try allocator.create(Value.Payload.IntBigNegative); - val_payload.* = .{ .limbs = result_limbs }; - break :blk &val_payload.base; - }; - - return Value.initPayload(val_payload); -} - -pub fn floatAdd(self: *Module, scope: *Scope, float_type: Type, src: usize, lhs: Value, rhs: Value) !Value { - var bit_count = switch (float_type.tag()) { - .comptime_float => 128, - else => float_type.floatBits(self.getTarget()), - }; - - const allocator = scope.arena(); - const val_payload = switch (bit_count) { - 16 => { - return self.fail(scope, src, "TODO Implement addition for soft floats", .{}); - }, - 32 => blk: { - const lhs_val = lhs.toFloat(f32); - const rhs_val = rhs.toFloat(f32); - const val_payload = try allocator.create(Value.Payload.Float_32); - val_payload.* = .{ .val = lhs_val + rhs_val }; - break :blk &val_payload.base; - }, - 64 => blk: { - const lhs_val = lhs.toFloat(f64); - const rhs_val = rhs.toFloat(f64); - const val_payload = try allocator.create(Value.Payload.Float_64); - val_payload.* = .{ .val = lhs_val + rhs_val }; - break :blk &val_payload.base; - }, - 128 => { - return self.fail(scope, src, "TODO Implement addition for big floats", .{}); - }, - else => unreachable, - }; - - return Value.initPayload(val_payload); -} - -pub fn floatSub(self: *Module, scope: *Scope, float_type: Type, src: usize, lhs: Value, rhs: Value) !Value { - var bit_count = switch (float_type.tag()) { - .comptime_float => 128, - else => float_type.floatBits(self.getTarget()), - }; - - const allocator = scope.arena(); - const val_payload = switch (bit_count) { - 16 => { - return self.fail(scope, src, "TODO Implement substraction for soft floats", .{}); - }, - 32 => blk: { - const lhs_val = lhs.toFloat(f32); - const rhs_val = rhs.toFloat(f32); - const val_payload = try allocator.create(Value.Payload.Float_32); - val_payload.* = .{ .val = lhs_val - rhs_val }; - break :blk &val_payload.base; - }, - 64 => blk: { - const lhs_val = lhs.toFloat(f64); - const rhs_val = rhs.toFloat(f64); - const val_payload = try allocator.create(Value.Payload.Float_64); - val_payload.* = .{ .val = lhs_val - rhs_val }; - break :blk &val_payload.base; - }, - 128 => { - return self.fail(scope, src, "TODO Implement substraction for big floats", .{}); - }, - else => unreachable, - }; - - return Value.initPayload(val_payload); -} - -pub fn simplePtrType(self: *Module, scope: *Scope, src: usize, elem_ty: Type, mutable: bool, size: std.builtin.TypeInfo.Pointer.Size) Allocator.Error!Type { - if (!mutable and size == .Slice and elem_ty.eql(Type.initTag(.u8))) { - return Type.initTag(.const_slice_u8); - } - // TODO stage1 type inference bug - const T = Type.Tag; - - const type_payload = try scope.arena().create(Type.Payload.PointerSimple); - type_payload.* = .{ - .base = .{ - .tag = switch (size) { - .One => if (mutable) T.single_mut_pointer else T.single_const_pointer, - .Many => if (mutable) T.many_mut_pointer else T.many_const_pointer, - .C => if (mutable) T.c_mut_pointer else T.c_const_pointer, - .Slice => if (mutable) T.mut_slice else T.const_slice, - }, - }, - .pointee_type = elem_ty, - }; - return Type.initPayload(&type_payload.base); -} - -pub fn ptrType( - self: *Module, - scope: *Scope, - src: usize, - elem_ty: Type, - sentinel: ?Value, - @"align": u32, - bit_offset: u16, - host_size: u16, - mutable: bool, - @"allowzero": bool, - @"volatile": bool, - size: std.builtin.TypeInfo.Pointer.Size, -) Allocator.Error!Type { - assert(host_size == 0 or bit_offset < host_size * 8); - - // TODO check if type can be represented by simplePtrType - const type_payload = try scope.arena().create(Type.Payload.Pointer); - type_payload.* = .{ - .pointee_type = elem_ty, - .sentinel = sentinel, - .@"align" = @"align", - .bit_offset = bit_offset, - .host_size = host_size, - .@"allowzero" = @"allowzero", - .mutable = mutable, - .@"volatile" = @"volatile", - .size = size, - }; - return Type.initPayload(&type_payload.base); -} - -pub fn optionalType(self: *Module, scope: *Scope, child_type: Type) Allocator.Error!Type { - return Type.initPayload(switch (child_type.tag()) { - .single_const_pointer => blk: { - const payload = try scope.arena().create(Type.Payload.PointerSimple); - payload.* = .{ - .base = .{ .tag = .optional_single_const_pointer }, - .pointee_type = child_type.elemType(), - }; - break :blk &payload.base; - }, - .single_mut_pointer => blk: { - const payload = try scope.arena().create(Type.Payload.PointerSimple); - payload.* = .{ - .base = .{ .tag = .optional_single_mut_pointer }, - .pointee_type = child_type.elemType(), - }; - break :blk &payload.base; - }, - else => blk: { - const payload = try scope.arena().create(Type.Payload.Optional); - payload.* = .{ - .child_type = child_type, - }; - break :blk &payload.base; - }, - }); -} - -pub fn arrayType(self: *Module, scope: *Scope, len: u64, sentinel: ?Value, elem_type: Type) Allocator.Error!Type { - if (elem_type.eql(Type.initTag(.u8))) { - if (sentinel) |some| { - if (some.eql(Value.initTag(.zero))) { - const payload = try scope.arena().create(Type.Payload.Array_u8_Sentinel0); - payload.* = .{ - .len = len, - }; - return Type.initPayload(&payload.base); - } - } else { - const payload = try scope.arena().create(Type.Payload.Array_u8); - payload.* = .{ - .len = len, - }; - return Type.initPayload(&payload.base); - } - } - - if (sentinel) |some| { - const payload = try scope.arena().create(Type.Payload.ArraySentinel); - payload.* = .{ - .len = len, - .sentinel = some, - .elem_type = elem_type, - }; - return Type.initPayload(&payload.base); - } - - const payload = try scope.arena().create(Type.Payload.Array); - payload.* = .{ - .len = len, - .elem_type = elem_type, - }; - return Type.initPayload(&payload.base); -} - -pub fn errorUnionType(self: *Module, scope: *Scope, error_set: Type, payload: Type) Allocator.Error!Type { - assert(error_set.zigTypeTag() == .ErrorSet); - if (error_set.eql(Type.initTag(.anyerror)) and payload.eql(Type.initTag(.void))) { - return Type.initTag(.anyerror_void_error_union); - } - - const result = try scope.arena().create(Type.Payload.ErrorUnion); - result.* = .{ - .error_set = error_set, - .payload = payload, - }; - return Type.initPayload(&result.base); -} - -pub fn anyframeType(self: *Module, scope: *Scope, return_type: Type) Allocator.Error!Type { - const result = try scope.arena().create(Type.Payload.AnyFrame); - result.* = .{ - .return_type = return_type, - }; - return Type.initPayload(&result.base); -} - -pub fn dumpInst(self: *Module, scope: *Scope, inst: *Inst) void { - const zir_module = scope.namespace(); - const source = zir_module.getSource(self) catch @panic("dumpInst failed to get source"); - const loc = std.zig.findLineColumn(source, inst.src); - if (inst.tag == .constant) { - std.debug.print("constant ty={} val={} src={}:{}:{}\n", .{ - inst.ty, - inst.castTag(.constant).?.val, - zir_module.subFilePath(), - loc.line + 1, - loc.column + 1, - }); - } else if (inst.deaths == 0) { - std.debug.print("{} ty={} src={}:{}:{}\n", .{ - @tagName(inst.tag), - inst.ty, - zir_module.subFilePath(), - loc.line + 1, - loc.column + 1, - }); - } else { - std.debug.print("{} ty={} deaths={b} src={}:{}:{}\n", .{ - @tagName(inst.tag), - inst.ty, - inst.deaths, - zir_module.subFilePath(), - loc.line + 1, - loc.column + 1, - }); - } -} - -pub const PanicId = enum { - unreach, - unwrap_null, -}; - -pub fn addSafetyCheck(mod: *Module, parent_block: *Scope.Block, ok: *Inst, panic_id: PanicId) !void { - const block_inst = try parent_block.arena.create(Inst.Block); - block_inst.* = .{ - .base = .{ - .tag = Inst.Block.base_tag, - .ty = Type.initTag(.void), - .src = ok.src, - }, - .body = .{ - .instructions = try parent_block.arena.alloc(*Inst, 1), // Only need space for the condbr. - }, - }; - - const ok_body: ir.Body = .{ - .instructions = try parent_block.arena.alloc(*Inst, 1), // Only need space for the brvoid. - }; - const brvoid = try parent_block.arena.create(Inst.BrVoid); - brvoid.* = .{ - .base = .{ - .tag = .brvoid, - .ty = Type.initTag(.noreturn), - .src = ok.src, - }, - .block = block_inst, - }; - ok_body.instructions[0] = &brvoid.base; - - var fail_block: Scope.Block = .{ - .parent = parent_block, - .func = parent_block.func, - .decl = parent_block.decl, - .instructions = .{}, - .arena = parent_block.arena, - .is_comptime = parent_block.is_comptime, - }; - defer fail_block.instructions.deinit(mod.gpa); - - _ = try mod.safetyPanic(&fail_block, ok.src, panic_id); - - const fail_body: ir.Body = .{ .instructions = try parent_block.arena.dupe(*Inst, fail_block.instructions.items) }; - - const condbr = try parent_block.arena.create(Inst.CondBr); - condbr.* = .{ - .base = .{ - .tag = .condbr, - .ty = Type.initTag(.noreturn), - .src = ok.src, - }, - .condition = ok, - .then_body = ok_body, - .else_body = fail_body, - }; - block_inst.body.instructions[0] = &condbr.base; - - try parent_block.instructions.append(mod.gpa, &block_inst.base); -} - -pub fn safetyPanic(mod: *Module, block: *Scope.Block, src: usize, panic_id: PanicId) !*Inst { - // TODO Once we have a panic function to call, call it here instead of breakpoint. - _ = try mod.addNoOp(block, src, Type.initTag(.void), .breakpoint); - return mod.addNoOp(block, src, Type.initTag(.noreturn), .unreach); -} - pub const FileExt = enum { c, cpp, diff --git a/src-self-hosted/ZigModule.zig b/src-self-hosted/ZigModule.zig new file mode 100644 index 0000000000..1f21ce7653 --- /dev/null +++ b/src-self-hosted/ZigModule.zig @@ -0,0 +1,3238 @@ +//! TODO This is going to get renamed from ZigModule to Module (but first we have to rename +//! Module to Compilation). +const Module = @This(); +const Compilation = @import("Module.zig"); + +const std = @import("std"); +const mem = std.mem; +const Allocator = std.mem.Allocator; +const ArrayListUnmanaged = std.ArrayListUnmanaged; +const Value = @import("value.zig").Value; +const Type = @import("type.zig").Type; +const TypedValue = @import("TypedValue.zig"); +const assert = std.debug.assert; +const log = std.log.scoped(.module); +const BigIntConst = std.math.big.int.Const; +const BigIntMutable = std.math.big.int.Mutable; +const Target = std.Target; +const Package = @import("Package.zig"); +const link = @import("link.zig"); +const ir = @import("ir.zig"); +const zir = @import("zir.zig"); +const Inst = ir.Inst; +const Body = ir.Body; +const ast = std.zig.ast; +const trace = @import("tracy.zig").trace; +const astgen = @import("astgen.zig"); +const zir_sema = @import("zir_sema.zig"); + +/// General-purpose allocator. Used for both temporary and long-term storage. +gpa: *Allocator, +comp: *Compilation, + +/// Where our incremental compilation metadata serialization will go. +zig_cache_artifact_directory: Compilation.Directory, +/// Pointer to externally managed resource. `null` if there is no zig file being compiled. +root_pkg: *Package, +/// Module owns this resource. +/// The `Scope` is either a `Scope.ZIRModule` or `Scope.File`. +root_scope: *Scope, +/// It's rare for a decl to be exported, so we save memory by having a sparse map of +/// Decl pointers to details about them being exported. +/// The Export memory is owned by the `export_owners` table; the slice itself is owned by this table. +decl_exports: std.AutoArrayHashMapUnmanaged(*Decl, []*Export) = .{}, +/// We track which export is associated with the given symbol name for quick +/// detection of symbol collisions. +symbol_exports: std.StringArrayHashMapUnmanaged(*Export) = .{}, +/// This models the Decls that perform exports, so that `decl_exports` can be updated when a Decl +/// is modified. Note that the key of this table is not the Decl being exported, but the Decl that +/// is performing the export of another Decl. +/// This table owns the Export memory. +export_owners: std.AutoArrayHashMapUnmanaged(*Decl, []*Export) = .{}, +/// Maps fully qualified namespaced names to the Decl struct for them. +decl_table: std.ArrayHashMapUnmanaged(Scope.NameHash, *Decl, Scope.name_hash_hash, Scope.name_hash_eql, false) = .{}, +/// We optimize memory usage for a compilation with no compile errors by storing the +/// error messages and mapping outside of `Decl`. +/// The ErrorMsg memory is owned by the decl, using Module's general purpose allocator. +/// Note that a Decl can succeed but the Fn it represents can fail. In this case, +/// a Decl can have a failed_decls entry but have analysis status of success. +failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *Compilation.ErrorMsg) = .{}, +/// Using a map here for consistency with the other fields here. +/// The ErrorMsg memory is owned by the `Scope`, using Module's general purpose allocator. +failed_files: std.AutoArrayHashMapUnmanaged(*Scope, *Compilation.ErrorMsg) = .{}, +/// Using a map here for consistency with the other fields here. +/// The ErrorMsg memory is owned by the `Export`, using Module's general purpose allocator. +failed_exports: std.AutoArrayHashMapUnmanaged(*Export, *Compilation.ErrorMsg) = .{}, + +next_anon_name_index: usize = 0, + +/// Candidates for deletion. After a semantic analysis update completes, this list +/// contains Decls that need to be deleted if they end up having no references to them. +deletion_set: ArrayListUnmanaged(*Decl) = .{}, + +/// Error tags and their values, tag names are duped with mod.gpa. +global_error_set: std.StringHashMapUnmanaged(u16) = .{}, + +/// Incrementing integer used to compare against the corresponding Decl +/// field to determine whether a Decl's status applies to an ongoing update, or a +/// previous analysis. +generation: u32 = 0, + +pub const Export = struct { + options: std.builtin.ExportOptions, + /// Byte offset into the file that contains the export directive. + src: usize, + /// Represents the position of the export, if any, in the output file. + link: link.File.Elf.Export, + /// The Decl that performs the export. Note that this is *not* the Decl being exported. + owner_decl: *Decl, + /// The Decl being exported. Note this is *not* the Decl performing the export. + exported_decl: *Decl, + status: enum { + in_progress, + failed, + /// Indicates that the failure was due to a temporary issue, such as an I/O error + /// when writing to the output file. Retrying the export may succeed. + failed_retryable, + complete, + }, +}; + +pub const Decl = struct { + /// This name is relative to the containing namespace of the decl. It uses a null-termination + /// to save bytes, since there can be a lot of decls in a compilation. The null byte is not allowed + /// in symbol names, because executable file formats use null-terminated strings for symbol names. + /// All Decls have names, even values that are not bound to a zig namespace. This is necessary for + /// mapping them to an address in the output file. + /// Memory owned by this decl, using Module's allocator. + name: [*:0]const u8, + /// The direct parent container of the Decl. This is either a `Scope.Container` or `Scope.ZIRModule`. + /// Reference to externally owned memory. + scope: *Scope, + /// The AST Node decl index or ZIR Inst index that contains this declaration. + /// Must be recomputed when the corresponding source file is modified. + src_index: usize, + /// The most recent value of the Decl after a successful semantic analysis. + typed_value: union(enum) { + never_succeeded: void, + most_recent: TypedValue.Managed, + }, + /// Represents the "shallow" analysis status. For example, for decls that are functions, + /// the function type is analyzed with this set to `in_progress`, however, the semantic + /// analysis of the function body is performed with this value set to `success`. Functions + /// have their own analysis status field. + analysis: enum { + /// This Decl corresponds to an AST Node that has not been referenced yet, and therefore + /// because of Zig's lazy declaration analysis, it will remain unanalyzed until referenced. + unreferenced, + /// Semantic analysis for this Decl is running right now. This state detects dependency loops. + in_progress, + /// This Decl might be OK but it depends on another one which did not successfully complete + /// semantic analysis. + dependency_failure, + /// Semantic analysis failure. + /// There will be a corresponding ErrorMsg in Module.failed_decls. + sema_failure, + /// There will be a corresponding ErrorMsg in Module.failed_decls. + /// This indicates the failure was something like running out of disk space, + /// and attempting semantic analysis again may succeed. + sema_failure_retryable, + /// There will be a corresponding ErrorMsg in Module.failed_decls. + codegen_failure, + /// There will be a corresponding ErrorMsg in Module.failed_decls. + /// This indicates the failure was something like running out of disk space, + /// and attempting codegen again may succeed. + codegen_failure_retryable, + /// Everything is done. During an update, this Decl may be out of date, depending + /// on its dependencies. The `generation` field can be used to determine if this + /// completion status occurred before or after a given update. + complete, + /// A Module update is in progress, and this Decl has been flagged as being known + /// to require re-analysis. + outdated, + }, + /// This flag is set when this Decl is added to a check_for_deletion set, and cleared + /// when removed. + deletion_flag: bool, + /// Whether the corresponding AST decl has a `pub` keyword. + is_pub: bool, + + /// An integer that can be checked against the corresponding incrementing + /// generation field of Module. This is used to determine whether `complete` status + /// represents pre- or post- re-analysis. + generation: u32, + + /// Represents the position of the code in the output file. + /// This is populated regardless of semantic analysis and code generation. + link: link.File.LinkBlock, + + /// Represents the function in the linked output file, if the `Decl` is a function. + /// This is stored here and not in `Fn` because `Decl` survives across updates but + /// `Fn` does not. + /// TODO Look into making `Fn` a longer lived structure and moving this field there + /// to save on memory usage. + fn_link: link.File.LinkFn, + + contents_hash: std.zig.SrcHash, + + /// The shallow set of other decls whose typed_value could possibly change if this Decl's + /// typed_value is modified. + dependants: DepsTable = .{}, + /// The shallow set of other decls whose typed_value changing indicates that this Decl's + /// typed_value may need to be regenerated. + dependencies: DepsTable = .{}, + + /// The reason this is not `std.AutoArrayHashMapUnmanaged` is a workaround for + /// stage1 compiler giving me: `error: struct 'Module.Decl' depends on itself` + pub const DepsTable = std.ArrayHashMapUnmanaged(*Decl, void, std.array_hash_map.getAutoHashFn(*Decl), std.array_hash_map.getAutoEqlFn(*Decl), false); + + pub fn destroy(self: *Decl, gpa: *Allocator) void { + gpa.free(mem.spanZ(self.name)); + if (self.typedValueManaged()) |tvm| { + tvm.deinit(gpa); + } + self.dependants.deinit(gpa); + self.dependencies.deinit(gpa); + gpa.destroy(self); + } + + pub fn src(self: Decl) usize { + switch (self.scope.tag) { + .container => { + const container = @fieldParentPtr(Scope.Container, "base", self.scope); + const tree = container.file_scope.contents.tree; + // TODO Container should have it's own decls() + const decl_node = tree.root_node.decls()[self.src_index]; + return tree.token_locs[decl_node.firstToken()].start; + }, + .zir_module => { + const zir_module = @fieldParentPtr(Scope.ZIRModule, "base", self.scope); + const module = zir_module.contents.module; + const src_decl = module.decls[self.src_index]; + return src_decl.inst.src; + }, + .file, .block => unreachable, + .gen_zir => unreachable, + .local_val => unreachable, + .local_ptr => unreachable, + .decl => unreachable, + } + } + + pub fn fullyQualifiedNameHash(self: Decl) Scope.NameHash { + return self.scope.fullyQualifiedNameHash(mem.spanZ(self.name)); + } + + pub fn typedValue(self: *Decl) error{AnalysisFail}!TypedValue { + const tvm = self.typedValueManaged() orelse return error.AnalysisFail; + return tvm.typed_value; + } + + pub fn value(self: *Decl) error{AnalysisFail}!Value { + return (try self.typedValue()).val; + } + + pub fn dump(self: *Decl) void { + const loc = std.zig.findLineColumn(self.scope.source.bytes, self.src); + std.debug.print("{}:{}:{} name={} status={}", .{ + self.scope.sub_file_path, + loc.line + 1, + loc.column + 1, + mem.spanZ(self.name), + @tagName(self.analysis), + }); + if (self.typedValueManaged()) |tvm| { + std.debug.print(" ty={} val={}", .{ tvm.typed_value.ty, tvm.typed_value.val }); + } + std.debug.print("\n", .{}); + } + + pub fn typedValueManaged(self: *Decl) ?*TypedValue.Managed { + switch (self.typed_value) { + .most_recent => |*x| return x, + .never_succeeded => return null, + } + } + + fn removeDependant(self: *Decl, other: *Decl) void { + self.dependants.removeAssertDiscard(other); + } + + fn removeDependency(self: *Decl, other: *Decl) void { + self.dependencies.removeAssertDiscard(other); + } +}; + +/// Fn struct memory is owned by the Decl's TypedValue.Managed arena allocator. +pub const Fn = struct { + /// This memory owned by the Decl's TypedValue.Managed arena allocator. + analysis: union(enum) { + queued: *ZIR, + in_progress, + /// There will be a corresponding ErrorMsg in Module.failed_decls + sema_failure, + /// This Fn might be OK but it depends on another Decl which did not successfully complete + /// semantic analysis. + dependency_failure, + success: Body, + }, + owner_decl: *Decl, + + /// This memory is temporary and points to stack memory for the duration + /// of Fn analysis. + pub const Analysis = struct { + inner_block: Scope.Block, + }; + + /// Contains un-analyzed ZIR instructions generated from Zig source AST. + pub const ZIR = struct { + body: zir.Module.Body, + arena: std.heap.ArenaAllocator.State, + }; + + /// For debugging purposes. + pub fn dump(self: *Fn, mod: Module) void { + std.debug.print("Module.Function(name={}) ", .{self.owner_decl.name}); + switch (self.analysis) { + .queued => { + std.debug.print("queued\n", .{}); + }, + .in_progress => { + std.debug.print("in_progress\n", .{}); + }, + else => { + std.debug.print("\n", .{}); + zir.dumpFn(mod, self); + }, + } + } +}; + +pub const Var = struct { + /// if is_extern == true this is undefined + init: Value, + owner_decl: *Decl, + + is_extern: bool, + is_mutable: bool, + is_threadlocal: bool, +}; + +pub const Scope = struct { + tag: Tag, + + pub const NameHash = [16]u8; + + pub fn cast(base: *Scope, comptime T: type) ?*T { + if (base.tag != T.base_tag) + return null; + + return @fieldParentPtr(T, "base", base); + } + + /// Asserts the scope has a parent which is a DeclAnalysis and + /// returns the arena Allocator. + pub fn arena(self: *Scope) *Allocator { + switch (self.tag) { + .block => return self.cast(Block).?.arena, + .decl => return &self.cast(DeclAnalysis).?.arena.allocator, + .gen_zir => return self.cast(GenZIR).?.arena, + .local_val => return self.cast(LocalVal).?.gen_zir.arena, + .local_ptr => return self.cast(LocalPtr).?.gen_zir.arena, + .zir_module => return &self.cast(ZIRModule).?.contents.module.arena.allocator, + .file => unreachable, + .container => unreachable, + } + } + + /// If the scope has a parent which is a `DeclAnalysis`, + /// returns the `Decl`, otherwise returns `null`. + pub fn decl(self: *Scope) ?*Decl { + return switch (self.tag) { + .block => self.cast(Block).?.decl, + .gen_zir => self.cast(GenZIR).?.decl, + .local_val => self.cast(LocalVal).?.gen_zir.decl, + .local_ptr => self.cast(LocalPtr).?.gen_zir.decl, + .decl => self.cast(DeclAnalysis).?.decl, + .zir_module => null, + .file => null, + .container => null, + }; + } + + /// Asserts the scope has a parent which is a ZIRModule or Container and + /// returns it. + pub fn namespace(self: *Scope) *Scope { + switch (self.tag) { + .block => return self.cast(Block).?.decl.scope, + .gen_zir => return self.cast(GenZIR).?.decl.scope, + .local_val => return self.cast(LocalVal).?.gen_zir.decl.scope, + .local_ptr => return self.cast(LocalPtr).?.gen_zir.decl.scope, + .decl => return self.cast(DeclAnalysis).?.decl.scope, + .file => return &self.cast(File).?.root_container.base, + .zir_module, .container => return self, + } + } + + /// Must generate unique bytes with no collisions with other decls. + /// The point of hashing here is only to limit the number of bytes of + /// the unique identifier to a fixed size (16 bytes). + pub fn fullyQualifiedNameHash(self: *Scope, name: []const u8) NameHash { + switch (self.tag) { + .block => unreachable, + .gen_zir => unreachable, + .local_val => unreachable, + .local_ptr => unreachable, + .decl => unreachable, + .file => unreachable, + .zir_module => return self.cast(ZIRModule).?.fullyQualifiedNameHash(name), + .container => return self.cast(Container).?.fullyQualifiedNameHash(name), + } + } + + /// Asserts the scope is a child of a File and has an AST tree and returns the tree. + pub fn tree(self: *Scope) *ast.Tree { + switch (self.tag) { + .file => return self.cast(File).?.contents.tree, + .zir_module => unreachable, + .decl => return self.cast(DeclAnalysis).?.decl.scope.cast(Container).?.file_scope.contents.tree, + .block => return self.cast(Block).?.decl.scope.cast(Container).?.file_scope.contents.tree, + .gen_zir => return self.cast(GenZIR).?.decl.scope.cast(Container).?.file_scope.contents.tree, + .local_val => return self.cast(LocalVal).?.gen_zir.decl.scope.cast(Container).?.file_scope.contents.tree, + .local_ptr => return self.cast(LocalPtr).?.gen_zir.decl.scope.cast(Container).?.file_scope.contents.tree, + .container => return self.cast(Container).?.file_scope.contents.tree, + } + } + + /// Asserts the scope is a child of a `GenZIR` and returns it. + pub fn getGenZIR(self: *Scope) *GenZIR { + return switch (self.tag) { + .block => unreachable, + .gen_zir => self.cast(GenZIR).?, + .local_val => return self.cast(LocalVal).?.gen_zir, + .local_ptr => return self.cast(LocalPtr).?.gen_zir, + .decl => unreachable, + .zir_module => unreachable, + .file => unreachable, + .container => unreachable, + }; + } + + /// Asserts the scope has a parent which is a ZIRModule, Contaienr or File and + /// returns the sub_file_path field. + pub fn subFilePath(base: *Scope) []const u8 { + switch (base.tag) { + .container => return @fieldParentPtr(Container, "base", base).file_scope.sub_file_path, + .file => return @fieldParentPtr(File, "base", base).sub_file_path, + .zir_module => return @fieldParentPtr(ZIRModule, "base", base).sub_file_path, + .block => unreachable, + .gen_zir => unreachable, + .local_val => unreachable, + .local_ptr => unreachable, + .decl => unreachable, + } + } + + pub fn unload(base: *Scope, gpa: *Allocator) void { + switch (base.tag) { + .file => return @fieldParentPtr(File, "base", base).unload(gpa), + .zir_module => return @fieldParentPtr(ZIRModule, "base", base).unload(gpa), + .block => unreachable, + .gen_zir => unreachable, + .local_val => unreachable, + .local_ptr => unreachable, + .decl => unreachable, + .container => unreachable, + } + } + + pub fn getSource(base: *Scope, module: *Module) ![:0]const u8 { + switch (base.tag) { + .container => return @fieldParentPtr(Container, "base", base).file_scope.getSource(module), + .file => return @fieldParentPtr(File, "base", base).getSource(module), + .zir_module => return @fieldParentPtr(ZIRModule, "base", base).getSource(module), + .gen_zir => unreachable, + .local_val => unreachable, + .local_ptr => unreachable, + .block => unreachable, + .decl => unreachable, + } + } + + /// Asserts the scope is a namespace Scope and removes the Decl from the namespace. + pub fn removeDecl(base: *Scope, child: *Decl) void { + switch (base.tag) { + .container => return @fieldParentPtr(Container, "base", base).removeDecl(child), + .zir_module => return @fieldParentPtr(ZIRModule, "base", base).removeDecl(child), + .file => unreachable, + .block => unreachable, + .gen_zir => unreachable, + .local_val => unreachable, + .local_ptr => unreachable, + .decl => unreachable, + } + } + + /// Asserts the scope is a File or ZIRModule and deinitializes it, then deallocates it. + pub fn destroy(base: *Scope, gpa: *Allocator) void { + switch (base.tag) { + .file => { + const scope_file = @fieldParentPtr(File, "base", base); + scope_file.deinit(gpa); + gpa.destroy(scope_file); + }, + .zir_module => { + const scope_zir_module = @fieldParentPtr(ZIRModule, "base", base); + scope_zir_module.deinit(gpa); + gpa.destroy(scope_zir_module); + }, + .block => unreachable, + .gen_zir => unreachable, + .local_val => unreachable, + .local_ptr => unreachable, + .decl => unreachable, + .container => unreachable, + } + } + + fn name_hash_hash(x: NameHash) u32 { + return @truncate(u32, @bitCast(u128, x)); + } + + fn name_hash_eql(a: NameHash, b: NameHash) bool { + return @bitCast(u128, a) == @bitCast(u128, b); + } + + pub const Tag = enum { + /// .zir source code. + zir_module, + /// .zig source code. + file, + /// struct, enum or union, every .file contains one of these. + container, + block, + decl, + gen_zir, + local_val, + local_ptr, + }; + + pub const Container = struct { + pub const base_tag: Tag = .container; + base: Scope = Scope{ .tag = base_tag }, + + file_scope: *Scope.File, + + /// Direct children of the file. + decls: std.AutoArrayHashMapUnmanaged(*Decl, void), + + // TODO implement container types and put this in a status union + // ty: Type + + pub fn deinit(self: *Container, gpa: *Allocator) void { + self.decls.deinit(gpa); + self.* = undefined; + } + + pub fn removeDecl(self: *Container, child: *Decl) void { + _ = self.decls.remove(child); + } + + pub fn fullyQualifiedNameHash(self: *Container, name: []const u8) NameHash { + // TODO container scope qualified names. + return std.zig.hashSrc(name); + } + }; + + pub const File = struct { + pub const base_tag: Tag = .file; + base: Scope = Scope{ .tag = base_tag }, + + /// Relative to the owning package's root_src_dir. + /// Reference to external memory, not owned by File. + sub_file_path: []const u8, + source: union(enum) { + unloaded: void, + bytes: [:0]const u8, + }, + contents: union { + not_available: void, + tree: *ast.Tree, + }, + status: enum { + never_loaded, + unloaded_success, + unloaded_parse_failure, + loaded_success, + }, + + root_container: Container, + + pub fn unload(self: *File, gpa: *Allocator) void { + switch (self.status) { + .never_loaded, + .unloaded_parse_failure, + .unloaded_success, + => {}, + + .loaded_success => { + self.contents.tree.deinit(); + self.status = .unloaded_success; + }, + } + switch (self.source) { + .bytes => |bytes| { + gpa.free(bytes); + self.source = .{ .unloaded = {} }; + }, + .unloaded => {}, + } + } + + pub fn deinit(self: *File, gpa: *Allocator) void { + self.root_container.deinit(gpa); + self.unload(gpa); + self.* = undefined; + } + + pub fn dumpSrc(self: *File, src: usize) void { + const loc = std.zig.findLineColumn(self.source.bytes, src); + std.debug.print("{}:{}:{}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 }); + } + + pub fn getSource(self: *File, module: *Module) ![:0]const u8 { + switch (self.source) { + .unloaded => { + const source = try module.root_pkg.root_src_directory.handle.readFileAllocOptions( + module.gpa, + self.sub_file_path, + std.math.maxInt(u32), + null, + 1, + 0, + ); + self.source = .{ .bytes = source }; + return source; + }, + .bytes => |bytes| return bytes, + } + } + }; + + pub const ZIRModule = struct { + pub const base_tag: Tag = .zir_module; + base: Scope = Scope{ .tag = base_tag }, + /// Relative to the owning package's root_src_dir. + /// Reference to external memory, not owned by ZIRModule. + sub_file_path: []const u8, + source: union(enum) { + unloaded: void, + bytes: [:0]const u8, + }, + contents: union { + not_available: void, + module: *zir.Module, + }, + status: enum { + never_loaded, + unloaded_success, + unloaded_parse_failure, + unloaded_sema_failure, + + loaded_sema_failure, + loaded_success, + }, + + /// Even though .zir files only have 1 module, this set is still needed + /// because of anonymous Decls, which can exist in the global set, but + /// not this one. + decls: ArrayListUnmanaged(*Decl), + + pub fn unload(self: *ZIRModule, gpa: *Allocator) void { + switch (self.status) { + .never_loaded, + .unloaded_parse_failure, + .unloaded_sema_failure, + .unloaded_success, + => {}, + + .loaded_success => { + self.contents.module.deinit(gpa); + gpa.destroy(self.contents.module); + self.contents = .{ .not_available = {} }; + self.status = .unloaded_success; + }, + .loaded_sema_failure => { + self.contents.module.deinit(gpa); + gpa.destroy(self.contents.module); + self.contents = .{ .not_available = {} }; + self.status = .unloaded_sema_failure; + }, + } + switch (self.source) { + .bytes => |bytes| { + gpa.free(bytes); + self.source = .{ .unloaded = {} }; + }, + .unloaded => {}, + } + } + + pub fn deinit(self: *ZIRModule, gpa: *Allocator) void { + self.decls.deinit(gpa); + self.unload(gpa); + self.* = undefined; + } + + pub fn removeDecl(self: *ZIRModule, child: *Decl) void { + for (self.decls.items) |item, i| { + if (item == child) { + _ = self.decls.swapRemove(i); + return; + } + } + } + + pub fn dumpSrc(self: *ZIRModule, src: usize) void { + const loc = std.zig.findLineColumn(self.source.bytes, src); + std.debug.print("{}:{}:{}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 }); + } + + pub fn getSource(self: *ZIRModule, module: *Module) ![:0]const u8 { + switch (self.source) { + .unloaded => { + const source = try module.root_pkg.root_src_directory.handle.readFileAllocOptions( + module.gpa, + self.sub_file_path, + std.math.maxInt(u32), + null, + 1, + 0, + ); + self.source = .{ .bytes = source }; + return source; + }, + .bytes => |bytes| return bytes, + } + } + + pub fn fullyQualifiedNameHash(self: *ZIRModule, name: []const u8) NameHash { + // ZIR modules only have 1 file with all decls global in the same namespace. + return std.zig.hashSrc(name); + } + }; + + /// This is a temporary structure, references to it are valid only + /// during semantic analysis of the block. + pub const Block = struct { + pub const base_tag: Tag = .block; + base: Scope = Scope{ .tag = base_tag }, + parent: ?*Block, + func: ?*Fn, + decl: *Decl, + instructions: ArrayListUnmanaged(*Inst), + /// Points to the arena allocator of DeclAnalysis + arena: *Allocator, + label: ?Label = null, + is_comptime: bool, + + pub const Label = struct { + zir_block: *zir.Inst.Block, + results: ArrayListUnmanaged(*Inst), + block_inst: *Inst.Block, + }; + }; + + /// This is a temporary structure, references to it are valid only + /// during semantic analysis of the decl. + pub const DeclAnalysis = struct { + pub const base_tag: Tag = .decl; + base: Scope = Scope{ .tag = base_tag }, + decl: *Decl, + arena: std.heap.ArenaAllocator, + }; + + /// This is a temporary structure, references to it are valid only + /// during semantic analysis of the decl. + pub const GenZIR = struct { + pub const base_tag: Tag = .gen_zir; + base: Scope = Scope{ .tag = base_tag }, + /// Parents can be: `GenZIR`, `ZIRModule`, `File` + parent: *Scope, + decl: *Decl, + arena: *Allocator, + /// The first N instructions in a function body ZIR are arg instructions. + instructions: std.ArrayListUnmanaged(*zir.Inst) = .{}, + label: ?Label = null, + + pub const Label = struct { + token: ast.TokenIndex, + block_inst: *zir.Inst.Block, + result_loc: astgen.ResultLoc, + }; + }; + + /// This is always a `const` local and importantly the `inst` is a value type, not a pointer. + /// This structure lives as long as the AST generation of the Block + /// node that contains the variable. + pub const LocalVal = struct { + pub const base_tag: Tag = .local_val; + base: Scope = Scope{ .tag = base_tag }, + /// Parents can be: `LocalVal`, `LocalPtr`, `GenZIR`. + parent: *Scope, + gen_zir: *GenZIR, + name: []const u8, + inst: *zir.Inst, + }; + + /// This could be a `const` or `var` local. It has a pointer instead of a value. + /// This structure lives as long as the AST generation of the Block + /// node that contains the variable. + pub const LocalPtr = struct { + pub const base_tag: Tag = .local_ptr; + base: Scope = Scope{ .tag = base_tag }, + /// Parents can be: `LocalVal`, `LocalPtr`, `GenZIR`. + parent: *Scope, + gen_zir: *GenZIR, + name: []const u8, + ptr: *zir.Inst, + }; +}; + +pub const InnerError = error{ OutOfMemory, AnalysisFail }; + +pub fn deinit(self: *Module) void { + const gpa = self.gpa; + + self.zig_cache_artifact_directory.handle.close(); + + self.deletion_set.deinit(gpa); + + for (self.decl_table.items()) |entry| { + entry.value.destroy(gpa); + } + self.decl_table.deinit(gpa); + + for (self.failed_decls.items()) |entry| { + entry.value.destroy(gpa); + } + self.failed_decls.deinit(gpa); + + for (self.failed_files.items()) |entry| { + entry.value.destroy(gpa); + } + self.failed_files.deinit(gpa); + + for (self.failed_exports.items()) |entry| { + entry.value.destroy(gpa); + } + self.failed_exports.deinit(gpa); + + for (self.decl_exports.items()) |entry| { + const export_list = entry.value; + gpa.free(export_list); + } + self.decl_exports.deinit(gpa); + + for (self.export_owners.items()) |entry| { + freeExportList(gpa, entry.value); + } + self.export_owners.deinit(gpa); + + self.symbol_exports.deinit(gpa); + self.root_scope.destroy(gpa); + + var it = self.global_error_set.iterator(); + while (it.next()) |entry| { + gpa.free(entry.key); + } + self.global_error_set.deinit(gpa); +} + +fn freeExportList(gpa: *Allocator, export_list: []*Export) void { + for (export_list) |exp| { + gpa.free(exp.options.name); + gpa.destroy(exp); + } + gpa.free(export_list); +} + +pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void { + const tracy = trace(@src()); + defer tracy.end(); + + const subsequent_analysis = switch (decl.analysis) { + .in_progress => unreachable, + + .sema_failure, + .sema_failure_retryable, + .codegen_failure, + .dependency_failure, + .codegen_failure_retryable, + => return error.AnalysisFail, + + .complete => return, + + .outdated => blk: { + log.debug("re-analyzing {}\n", .{decl.name}); + + // The exports this Decl performs will be re-discovered, so we remove them here + // prior to re-analysis. + self.deleteDeclExports(decl); + // Dependencies will be re-discovered, so we remove them here prior to re-analysis. + for (decl.dependencies.items()) |entry| { + const dep = entry.key; + dep.removeDependant(decl); + if (dep.dependants.items().len == 0 and !dep.deletion_flag) { + // We don't perform a deletion here, because this Decl or another one + // may end up referencing it before the update is complete. + dep.deletion_flag = true; + try self.deletion_set.append(self.gpa, dep); + } + } + decl.dependencies.clearRetainingCapacity(); + + break :blk true; + }, + + .unreferenced => false, + }; + + const type_changed = if (self.root_scope.cast(Scope.ZIRModule)) |zir_module| + try zir_sema.analyzeZirDecl(self, decl, zir_module.contents.module.decls[decl.src_index]) + else + self.astGenAndAnalyzeDecl(decl) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.AnalysisFail => return error.AnalysisFail, + else => { + try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1); + self.failed_decls.putAssumeCapacityNoClobber(decl, try Compilation.ErrorMsg.create( + self.gpa, + decl.src(), + "unable to analyze: {}", + .{@errorName(err)}, + )); + decl.analysis = .sema_failure_retryable; + return error.AnalysisFail; + }, + }; + + if (subsequent_analysis) { + // We may need to chase the dependants and re-analyze them. + // However, if the decl is a function, and the type is the same, we do not need to. + if (type_changed or decl.typed_value.most_recent.typed_value.val.tag() != .function) { + for (decl.dependants.items()) |entry| { + const dep = entry.key; + switch (dep.analysis) { + .unreferenced => unreachable, + .in_progress => unreachable, + .outdated => continue, // already queued for update + + .dependency_failure, + .sema_failure, + .sema_failure_retryable, + .codegen_failure, + .codegen_failure_retryable, + .complete, + => if (dep.generation != self.generation) { + try self.markOutdatedDecl(dep); + }, + } + } + } + } +} + +fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool { + const tracy = trace(@src()); + defer tracy.end(); + + const container_scope = decl.scope.cast(Scope.Container).?; + const tree = try self.getAstTree(container_scope); + const ast_node = tree.root_node.decls()[decl.src_index]; + switch (ast_node.tag) { + .FnProto => { + const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", ast_node); + + decl.analysis = .in_progress; + + // This arena allocator's memory is discarded at the end of this function. It is used + // to determine the type of the function, and hence the type of the decl, which is needed + // to complete the Decl analysis. + var fn_type_scope_arena = std.heap.ArenaAllocator.init(self.gpa); + defer fn_type_scope_arena.deinit(); + var fn_type_scope: Scope.GenZIR = .{ + .decl = decl, + .arena = &fn_type_scope_arena.allocator, + .parent = decl.scope, + }; + defer fn_type_scope.instructions.deinit(self.gpa); + + decl.is_pub = fn_proto.getVisibToken() != null; + const body_node = fn_proto.getBodyNode() orelse + return self.failTok(&fn_type_scope.base, fn_proto.fn_token, "TODO implement extern functions", .{}); + + const param_decls = fn_proto.params(); + const param_types = try fn_type_scope.arena.alloc(*zir.Inst, param_decls.len); + + const fn_src = tree.token_locs[fn_proto.fn_token].start; + const type_type = try astgen.addZIRInstConst(self, &fn_type_scope.base, fn_src, .{ + .ty = Type.initTag(.type), + .val = Value.initTag(.type_type), + }); + const type_type_rl: astgen.ResultLoc = .{ .ty = type_type }; + for (param_decls) |param_decl, i| { + const param_type_node = switch (param_decl.param_type) { + .any_type => |node| return self.failNode(&fn_type_scope.base, node, "TODO implement anytype parameter", .{}), + .type_expr => |node| node, + }; + param_types[i] = try astgen.expr(self, &fn_type_scope.base, type_type_rl, param_type_node); + } + if (fn_proto.getVarArgsToken()) |var_args_token| { + return self.failTok(&fn_type_scope.base, var_args_token, "TODO implement var args", .{}); + } + if (fn_proto.getLibName()) |lib_name| { + return self.failNode(&fn_type_scope.base, lib_name, "TODO implement function library name", .{}); + } + if (fn_proto.getAlignExpr()) |align_expr| { + return self.failNode(&fn_type_scope.base, align_expr, "TODO implement function align expression", .{}); + } + if (fn_proto.getSectionExpr()) |sect_expr| { + return self.failNode(&fn_type_scope.base, sect_expr, "TODO implement function section expression", .{}); + } + if (fn_proto.getCallconvExpr()) |callconv_expr| { + return self.failNode( + &fn_type_scope.base, + callconv_expr, + "TODO implement function calling convention expression", + .{}, + ); + } + const return_type_expr = switch (fn_proto.return_type) { + .Explicit => |node| node, + .InferErrorSet => |node| return self.failNode(&fn_type_scope.base, node, "TODO implement inferred error sets", .{}), + .Invalid => |tok| return self.failTok(&fn_type_scope.base, tok, "unable to parse return type", .{}), + }; + + const return_type_inst = try astgen.expr(self, &fn_type_scope.base, type_type_rl, return_type_expr); + const fn_type_inst = try astgen.addZIRInst(self, &fn_type_scope.base, fn_src, zir.Inst.FnType, .{ + .return_type = return_type_inst, + .param_types = param_types, + }, .{}); + + // We need the memory for the Type to go into the arena for the Decl + var decl_arena = std.heap.ArenaAllocator.init(self.gpa); + errdefer decl_arena.deinit(); + const decl_arena_state = try decl_arena.allocator.create(std.heap.ArenaAllocator.State); + + var block_scope: Scope.Block = .{ + .parent = null, + .func = null, + .decl = decl, + .instructions = .{}, + .arena = &decl_arena.allocator, + .is_comptime = false, + }; + defer block_scope.instructions.deinit(self.gpa); + + const fn_type = try zir_sema.analyzeBodyValueAsType(self, &block_scope, fn_type_inst, .{ + .instructions = fn_type_scope.instructions.items, + }); + const new_func = try decl_arena.allocator.create(Fn); + const fn_payload = try decl_arena.allocator.create(Value.Payload.Function); + + const fn_zir = blk: { + // This scope's arena memory is discarded after the ZIR generation + // pass completes, and semantic analysis of it completes. + var gen_scope_arena = std.heap.ArenaAllocator.init(self.gpa); + errdefer gen_scope_arena.deinit(); + var gen_scope: Scope.GenZIR = .{ + .decl = decl, + .arena = &gen_scope_arena.allocator, + .parent = decl.scope, + }; + defer gen_scope.instructions.deinit(self.gpa); + + // We need an instruction for each parameter, and they must be first in the body. + try gen_scope.instructions.resize(self.gpa, fn_proto.params_len); + var params_scope = &gen_scope.base; + for (fn_proto.params()) |param, i| { + const name_token = param.name_token.?; + const src = tree.token_locs[name_token].start; + const param_name = tree.tokenSlice(name_token); // TODO: call identifierTokenString + const arg = try gen_scope_arena.allocator.create(zir.Inst.Arg); + arg.* = .{ + .base = .{ + .tag = .arg, + .src = src, + }, + .positionals = .{ + .name = param_name, + }, + .kw_args = .{}, + }; + gen_scope.instructions.items[i] = &arg.base; + const sub_scope = try gen_scope_arena.allocator.create(Scope.LocalVal); + sub_scope.* = .{ + .parent = params_scope, + .gen_zir = &gen_scope, + .name = param_name, + .inst = &arg.base, + }; + params_scope = &sub_scope.base; + } + + const body_block = body_node.cast(ast.Node.Block).?; + + try astgen.blockExpr(self, params_scope, body_block); + + if (gen_scope.instructions.items.len == 0 or + !gen_scope.instructions.items[gen_scope.instructions.items.len - 1].tag.isNoReturn()) + { + const src = tree.token_locs[body_block.rbrace].start; + _ = try astgen.addZIRNoOp(self, &gen_scope.base, src, .returnvoid); + } + + const fn_zir = try gen_scope_arena.allocator.create(Fn.ZIR); + fn_zir.* = .{ + .body = .{ + .instructions = try gen_scope.arena.dupe(*zir.Inst, gen_scope.instructions.items), + }, + .arena = gen_scope_arena.state, + }; + break :blk fn_zir; + }; + + new_func.* = .{ + .analysis = .{ .queued = fn_zir }, + .owner_decl = decl, + }; + fn_payload.* = .{ .func = new_func }; + + var prev_type_has_bits = false; + var type_changed = true; + + if (decl.typedValueManaged()) |tvm| { + prev_type_has_bits = tvm.typed_value.ty.hasCodeGenBits(); + type_changed = !tvm.typed_value.ty.eql(fn_type); + + tvm.deinit(self.gpa); + } + + decl_arena_state.* = decl_arena.state; + decl.typed_value = .{ + .most_recent = .{ + .typed_value = .{ + .ty = fn_type, + .val = Value.initPayload(&fn_payload.base), + }, + .arena = decl_arena_state, + }, + }; + decl.analysis = .complete; + decl.generation = self.generation; + + if (fn_type.hasCodeGenBits()) { + // We don't fully codegen the decl until later, but we do need to reserve a global + // offset table index for it. This allows us to codegen decls out of dependency order, + // increasing how many computations can be done in parallel. + try self.comp.bin_file.allocateDeclIndexes(decl); + try self.comp.work_queue.writeItem(.{ .codegen_decl = decl }); + } else if (prev_type_has_bits) { + self.comp.bin_file.freeDecl(decl); + } + + if (fn_proto.getExternExportInlineToken()) |maybe_export_token| { + if (tree.token_ids[maybe_export_token] == .Keyword_export) { + const export_src = tree.token_locs[maybe_export_token].start; + const name_loc = tree.token_locs[fn_proto.getNameToken().?]; + const name = tree.tokenSliceLoc(name_loc); + // The scope needs to have the decl in it. + try self.analyzeExport(&block_scope.base, export_src, name, decl); + } + } + return type_changed; + }, + .VarDecl => { + const var_decl = @fieldParentPtr(ast.Node.VarDecl, "base", ast_node); + + decl.analysis = .in_progress; + + // We need the memory for the Type to go into the arena for the Decl + var decl_arena = std.heap.ArenaAllocator.init(self.gpa); + errdefer decl_arena.deinit(); + const decl_arena_state = try decl_arena.allocator.create(std.heap.ArenaAllocator.State); + + var block_scope: Scope.Block = .{ + .parent = null, + .func = null, + .decl = decl, + .instructions = .{}, + .arena = &decl_arena.allocator, + .is_comptime = true, + }; + defer block_scope.instructions.deinit(self.gpa); + + decl.is_pub = var_decl.getVisibToken() != null; + const is_extern = blk: { + const maybe_extern_token = var_decl.getExternExportToken() orelse + break :blk false; + if (tree.token_ids[maybe_extern_token] != .Keyword_extern) break :blk false; + if (var_decl.getInitNode()) |some| { + return self.failNode(&block_scope.base, some, "extern variables have no initializers", .{}); + } + break :blk true; + }; + if (var_decl.getLibName()) |lib_name| { + assert(is_extern); + return self.failNode(&block_scope.base, lib_name, "TODO implement function library name", .{}); + } + const is_mutable = tree.token_ids[var_decl.mut_token] == .Keyword_var; + const is_threadlocal = if (var_decl.getThreadLocalToken()) |some| blk: { + if (!is_mutable) { + return self.failTok(&block_scope.base, some, "threadlocal variable cannot be constant", .{}); + } + break :blk true; + } else false; + assert(var_decl.getComptimeToken() == null); + if (var_decl.getAlignNode()) |align_expr| { + return self.failNode(&block_scope.base, align_expr, "TODO implement function align expression", .{}); + } + if (var_decl.getSectionNode()) |sect_expr| { + return self.failNode(&block_scope.base, sect_expr, "TODO implement function section expression", .{}); + } + + const var_info: struct { ty: Type, val: ?Value } = if (var_decl.getInitNode()) |init_node| vi: { + var gen_scope_arena = std.heap.ArenaAllocator.init(self.gpa); + defer gen_scope_arena.deinit(); + var gen_scope: Scope.GenZIR = .{ + .decl = decl, + .arena = &gen_scope_arena.allocator, + .parent = decl.scope, + }; + defer gen_scope.instructions.deinit(self.gpa); + + const init_result_loc: astgen.ResultLoc = if (var_decl.getTypeNode()) |type_node| rl: { + const src = tree.token_locs[type_node.firstToken()].start; + const type_type = try astgen.addZIRInstConst(self, &gen_scope.base, src, .{ + .ty = Type.initTag(.type), + .val = Value.initTag(.type_type), + }); + const var_type = try astgen.expr(self, &gen_scope.base, .{ .ty = type_type }, type_node); + break :rl .{ .ty = var_type }; + } else .none; + + const src = tree.token_locs[init_node.firstToken()].start; + const init_inst = try astgen.expr(self, &gen_scope.base, init_result_loc, init_node); + + var inner_block: Scope.Block = .{ + .parent = null, + .func = null, + .decl = decl, + .instructions = .{}, + .arena = &gen_scope_arena.allocator, + .is_comptime = true, + }; + defer inner_block.instructions.deinit(self.gpa); + try zir_sema.analyzeBody(self, &inner_block.base, .{ .instructions = gen_scope.instructions.items }); + + // The result location guarantees the type coercion. + const analyzed_init_inst = init_inst.analyzed_inst.?; + // The is_comptime in the Scope.Block guarantees the result is comptime-known. + const val = analyzed_init_inst.value().?; + + const ty = try analyzed_init_inst.ty.copy(block_scope.arena); + break :vi .{ + .ty = ty, + .val = try val.copy(block_scope.arena), + }; + } else if (!is_extern) { + return self.failTok(&block_scope.base, var_decl.firstToken(), "variables must be initialized", .{}); + } else if (var_decl.getTypeNode()) |type_node| vi: { + // Temporary arena for the zir instructions. + var type_scope_arena = std.heap.ArenaAllocator.init(self.gpa); + defer type_scope_arena.deinit(); + var type_scope: Scope.GenZIR = .{ + .decl = decl, + .arena = &type_scope_arena.allocator, + .parent = decl.scope, + }; + defer type_scope.instructions.deinit(self.gpa); + + const src = tree.token_locs[type_node.firstToken()].start; + const type_type = try astgen.addZIRInstConst(self, &type_scope.base, src, .{ + .ty = Type.initTag(.type), + .val = Value.initTag(.type_type), + }); + const var_type = try astgen.expr(self, &type_scope.base, .{ .ty = type_type }, type_node); + const ty = try zir_sema.analyzeBodyValueAsType(self, &block_scope, var_type, .{ + .instructions = type_scope.instructions.items, + }); + break :vi .{ + .ty = ty, + .val = null, + }; + } else { + return self.failTok(&block_scope.base, var_decl.firstToken(), "unable to infer variable type", .{}); + }; + + if (is_mutable and !var_info.ty.isValidVarType(is_extern)) { + return self.failTok(&block_scope.base, var_decl.firstToken(), "variable of type '{}' must be const", .{var_info.ty}); + } + + var type_changed = true; + if (decl.typedValueManaged()) |tvm| { + type_changed = !tvm.typed_value.ty.eql(var_info.ty); + + tvm.deinit(self.gpa); + } + + const new_variable = try decl_arena.allocator.create(Var); + const var_payload = try decl_arena.allocator.create(Value.Payload.Variable); + new_variable.* = .{ + .owner_decl = decl, + .init = var_info.val orelse undefined, + .is_extern = is_extern, + .is_mutable = is_mutable, + .is_threadlocal = is_threadlocal, + }; + var_payload.* = .{ .variable = new_variable }; + + decl_arena_state.* = decl_arena.state; + decl.typed_value = .{ + .most_recent = .{ + .typed_value = .{ + .ty = var_info.ty, + .val = Value.initPayload(&var_payload.base), + }, + .arena = decl_arena_state, + }, + }; + decl.analysis = .complete; + decl.generation = self.generation; + + if (var_decl.getExternExportToken()) |maybe_export_token| { + if (tree.token_ids[maybe_export_token] == .Keyword_export) { + const export_src = tree.token_locs[maybe_export_token].start; + const name_loc = tree.token_locs[var_decl.name_token]; + const name = tree.tokenSliceLoc(name_loc); + // The scope needs to have the decl in it. + try self.analyzeExport(&block_scope.base, export_src, name, decl); + } + } + return type_changed; + }, + .Comptime => { + const comptime_decl = @fieldParentPtr(ast.Node.Comptime, "base", ast_node); + + decl.analysis = .in_progress; + + // A comptime decl does not store any value so we can just deinit this arena after analysis is done. + var analysis_arena = std.heap.ArenaAllocator.init(self.gpa); + defer analysis_arena.deinit(); + var gen_scope: Scope.GenZIR = .{ + .decl = decl, + .arena = &analysis_arena.allocator, + .parent = decl.scope, + }; + defer gen_scope.instructions.deinit(self.gpa); + + _ = try astgen.comptimeExpr(self, &gen_scope.base, .none, comptime_decl.expr); + + var block_scope: Scope.Block = .{ + .parent = null, + .func = null, + .decl = decl, + .instructions = .{}, + .arena = &analysis_arena.allocator, + .is_comptime = true, + }; + defer block_scope.instructions.deinit(self.gpa); + + _ = try zir_sema.analyzeBody(self, &block_scope.base, .{ + .instructions = gen_scope.instructions.items, + }); + + decl.analysis = .complete; + decl.generation = self.generation; + return true; + }, + .Use => @panic("TODO usingnamespace decl"), + else => unreachable, + } +} + +fn declareDeclDependency(self: *Module, depender: *Decl, dependee: *Decl) !void { + try depender.dependencies.ensureCapacity(self.gpa, depender.dependencies.items().len + 1); + try dependee.dependants.ensureCapacity(self.gpa, dependee.dependants.items().len + 1); + + depender.dependencies.putAssumeCapacity(dependee, {}); + dependee.dependants.putAssumeCapacity(depender, {}); +} + +fn getSrcModule(self: *Module, root_scope: *Scope.ZIRModule) !*zir.Module { + switch (root_scope.status) { + .never_loaded, .unloaded_success => { + try self.failed_files.ensureCapacity(self.gpa, self.failed_files.items().len + 1); + + const source = try root_scope.getSource(self); + + var keep_zir_module = false; + const zir_module = try self.gpa.create(zir.Module); + defer if (!keep_zir_module) self.gpa.destroy(zir_module); + + zir_module.* = try zir.parse(self.gpa, source); + defer if (!keep_zir_module) zir_module.deinit(self.gpa); + + if (zir_module.error_msg) |src_err_msg| { + self.failed_files.putAssumeCapacityNoClobber( + &root_scope.base, + try Compilation.ErrorMsg.create(self.gpa, src_err_msg.byte_offset, "{}", .{src_err_msg.msg}), + ); + root_scope.status = .unloaded_parse_failure; + return error.AnalysisFail; + } + + root_scope.status = .loaded_success; + root_scope.contents = .{ .module = zir_module }; + keep_zir_module = true; + + return zir_module; + }, + + .unloaded_parse_failure, + .unloaded_sema_failure, + => return error.AnalysisFail, + + .loaded_success, .loaded_sema_failure => return root_scope.contents.module, + } +} + +fn getAstTree(self: *Module, container_scope: *Scope.Container) !*ast.Tree { + const tracy = trace(@src()); + defer tracy.end(); + + const root_scope = container_scope.file_scope; + + switch (root_scope.status) { + .never_loaded, .unloaded_success => { + try self.failed_files.ensureCapacity(self.gpa, self.failed_files.items().len + 1); + + const source = try root_scope.getSource(self); + + var keep_tree = false; + const tree = try std.zig.parse(self.gpa, source); + defer if (!keep_tree) tree.deinit(); + + if (tree.errors.len != 0) { + const parse_err = tree.errors[0]; + + var msg = std.ArrayList(u8).init(self.gpa); + defer msg.deinit(); + + try parse_err.render(tree.token_ids, msg.outStream()); + const err_msg = try self.gpa.create(Compilation.ErrorMsg); + err_msg.* = .{ + .msg = msg.toOwnedSlice(), + .byte_offset = tree.token_locs[parse_err.loc()].start, + }; + + self.failed_files.putAssumeCapacityNoClobber(&root_scope.base, err_msg); + root_scope.status = .unloaded_parse_failure; + return error.AnalysisFail; + } + + root_scope.status = .loaded_success; + root_scope.contents = .{ .tree = tree }; + keep_tree = true; + + return tree; + }, + + .unloaded_parse_failure => return error.AnalysisFail, + + .loaded_success => return root_scope.contents.tree, + } +} + +pub fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void { + const tracy = trace(@src()); + defer tracy.end(); + + // We may be analyzing it for the first time, or this may be + // an incremental update. This code handles both cases. + const tree = try self.getAstTree(container_scope); + const decls = tree.root_node.decls(); + + try self.comp.work_queue.ensureUnusedCapacity(decls.len); + try container_scope.decls.ensureCapacity(self.gpa, decls.len); + + // Keep track of the decls that we expect to see in this file so that + // we know which ones have been deleted. + var deleted_decls = std.AutoArrayHashMap(*Decl, void).init(self.gpa); + defer deleted_decls.deinit(); + try deleted_decls.ensureCapacity(container_scope.decls.items().len); + for (container_scope.decls.items()) |entry| { + deleted_decls.putAssumeCapacityNoClobber(entry.key, {}); + } + + for (decls) |src_decl, decl_i| { + if (src_decl.cast(ast.Node.FnProto)) |fn_proto| { + // We will create a Decl for it regardless of analysis status. + const name_tok = fn_proto.getNameToken() orelse { + @panic("TODO missing function name"); + }; + + const name_loc = tree.token_locs[name_tok]; + const name = tree.tokenSliceLoc(name_loc); + const name_hash = container_scope.fullyQualifiedNameHash(name); + const contents_hash = std.zig.hashSrc(tree.getNodeSource(src_decl)); + if (self.decl_table.get(name_hash)) |decl| { + // Update the AST Node index of the decl, even if its contents are unchanged, it may + // have been re-ordered. + decl.src_index = decl_i; + if (deleted_decls.remove(decl) == null) { + decl.analysis = .sema_failure; + const err_msg = try Compilation.ErrorMsg.create(self.gpa, tree.token_locs[name_tok].start, "redefinition of '{}'", .{decl.name}); + errdefer err_msg.destroy(self.gpa); + try self.failed_decls.putNoClobber(self.gpa, decl, err_msg); + } else { + if (!srcHashEql(decl.contents_hash, contents_hash)) { + try self.markOutdatedDecl(decl); + decl.contents_hash = contents_hash; + } else switch (self.comp.bin_file.tag) { + .coff => { + // TODO Implement for COFF + }, + .elf => if (decl.fn_link.elf.len != 0) { + // TODO Look into detecting when this would be unnecessary by storing enough state + // in `Decl` to notice that the line number did not change. + self.comp.work_queue.writeItemAssumeCapacity(.{ .update_line_number = decl }); + }, + .macho => { + // TODO Implement for MachO + }, + .c, .wasm => {}, + } + } + } else { + const new_decl = try self.createNewDecl(&container_scope.base, name, decl_i, name_hash, contents_hash); + container_scope.decls.putAssumeCapacity(new_decl, {}); + if (fn_proto.getExternExportInlineToken()) |maybe_export_token| { + if (tree.token_ids[maybe_export_token] == .Keyword_export) { + self.comp.work_queue.writeItemAssumeCapacity(.{ .analyze_decl = new_decl }); + } + } + } + } else if (src_decl.castTag(.VarDecl)) |var_decl| { + const name_loc = tree.token_locs[var_decl.name_token]; + const name = tree.tokenSliceLoc(name_loc); + const name_hash = container_scope.fullyQualifiedNameHash(name); + const contents_hash = std.zig.hashSrc(tree.getNodeSource(src_decl)); + if (self.decl_table.get(name_hash)) |decl| { + // Update the AST Node index of the decl, even if its contents are unchanged, it may + // have been re-ordered. + decl.src_index = decl_i; + if (deleted_decls.remove(decl) == null) { + decl.analysis = .sema_failure; + const err_msg = try Compilation.ErrorMsg.create(self.gpa, name_loc.start, "redefinition of '{}'", .{decl.name}); + errdefer err_msg.destroy(self.gpa); + try self.failed_decls.putNoClobber(self.gpa, decl, err_msg); + } else if (!srcHashEql(decl.contents_hash, contents_hash)) { + try self.markOutdatedDecl(decl); + decl.contents_hash = contents_hash; + } + } else { + const new_decl = try self.createNewDecl(&container_scope.base, name, decl_i, name_hash, contents_hash); + container_scope.decls.putAssumeCapacity(new_decl, {}); + if (var_decl.getExternExportToken()) |maybe_export_token| { + if (tree.token_ids[maybe_export_token] == .Keyword_export) { + self.comp.work_queue.writeItemAssumeCapacity(.{ .analyze_decl = new_decl }); + } + } + } + } else if (src_decl.castTag(.Comptime)) |comptime_node| { + const name_index = self.getNextAnonNameIndex(); + const name = try std.fmt.allocPrint(self.gpa, "__comptime_{}", .{name_index}); + defer self.gpa.free(name); + + const name_hash = container_scope.fullyQualifiedNameHash(name); + const contents_hash = std.zig.hashSrc(tree.getNodeSource(src_decl)); + + const new_decl = try self.createNewDecl(&container_scope.base, name, decl_i, name_hash, contents_hash); + container_scope.decls.putAssumeCapacity(new_decl, {}); + self.comp.work_queue.writeItemAssumeCapacity(.{ .analyze_decl = new_decl }); + } else if (src_decl.castTag(.ContainerField)) |container_field| { + log.err("TODO: analyze container field", .{}); + } else if (src_decl.castTag(.TestDecl)) |test_decl| { + log.err("TODO: analyze test decl", .{}); + } else if (src_decl.castTag(.Use)) |use_decl| { + log.err("TODO: analyze usingnamespace decl", .{}); + } else { + unreachable; + } + } + // Handle explicitly deleted decls from the source code. Not to be confused + // with when we delete decls because they are no longer referenced. + for (deleted_decls.items()) |entry| { + log.debug("noticed '{}' deleted from source\n", .{entry.key.name}); + try self.deleteDecl(entry.key); + } +} + +pub fn analyzeRootZIRModule(self: *Module, root_scope: *Scope.ZIRModule) !void { + // We may be analyzing it for the first time, or this may be + // an incremental update. This code handles both cases. + const src_module = try self.getSrcModule(root_scope); + + try self.comp.work_queue.ensureUnusedCapacity(src_module.decls.len); + try root_scope.decls.ensureCapacity(self.gpa, src_module.decls.len); + + var exports_to_resolve = std.ArrayList(*zir.Decl).init(self.gpa); + defer exports_to_resolve.deinit(); + + // Keep track of the decls that we expect to see in this file so that + // we know which ones have been deleted. + var deleted_decls = std.AutoArrayHashMap(*Decl, void).init(self.gpa); + defer deleted_decls.deinit(); + try deleted_decls.ensureCapacity(self.decl_table.items().len); + for (self.decl_table.items()) |entry| { + deleted_decls.putAssumeCapacityNoClobber(entry.value, {}); + } + + for (src_module.decls) |src_decl, decl_i| { + const name_hash = root_scope.fullyQualifiedNameHash(src_decl.name); + if (self.decl_table.get(name_hash)) |decl| { + deleted_decls.removeAssertDiscard(decl); + if (!srcHashEql(src_decl.contents_hash, decl.contents_hash)) { + try self.markOutdatedDecl(decl); + decl.contents_hash = src_decl.contents_hash; + } + } else { + const new_decl = try self.createNewDecl( + &root_scope.base, + src_decl.name, + decl_i, + name_hash, + src_decl.contents_hash, + ); + root_scope.decls.appendAssumeCapacity(new_decl); + if (src_decl.inst.cast(zir.Inst.Export)) |export_inst| { + try exports_to_resolve.append(src_decl); + } + } + } + for (exports_to_resolve.items) |export_decl| { + _ = try zir_sema.resolveZirDecl(self, &root_scope.base, export_decl); + } + // Handle explicitly deleted decls from the source code. Not to be confused + // with when we delete decls because they are no longer referenced. + for (deleted_decls.items()) |entry| { + log.debug("noticed '{}' deleted from source\n", .{entry.key.name}); + try self.deleteDecl(entry.key); + } +} + +pub fn deleteDecl(self: *Module, decl: *Decl) !void { + try self.deletion_set.ensureCapacity(self.gpa, self.deletion_set.items.len + decl.dependencies.items().len); + + // Remove from the namespace it resides in. In the case of an anonymous Decl it will + // not be present in the set, and this does nothing. + decl.scope.removeDecl(decl); + + log.debug("deleting decl '{}'\n", .{decl.name}); + const name_hash = decl.fullyQualifiedNameHash(); + self.decl_table.removeAssertDiscard(name_hash); + // Remove itself from its dependencies, because we are about to destroy the decl pointer. + for (decl.dependencies.items()) |entry| { + const dep = entry.key; + dep.removeDependant(decl); + if (dep.dependants.items().len == 0 and !dep.deletion_flag) { + // We don't recursively perform a deletion here, because during the update, + // another reference to it may turn up. + dep.deletion_flag = true; + self.deletion_set.appendAssumeCapacity(dep); + } + } + // Anything that depends on this deleted decl certainly needs to be re-analyzed. + for (decl.dependants.items()) |entry| { + const dep = entry.key; + dep.removeDependency(decl); + if (dep.analysis != .outdated) { + // TODO Move this failure possibility to the top of the function. + try self.markOutdatedDecl(dep); + } + } + if (self.failed_decls.remove(decl)) |entry| { + entry.value.destroy(self.gpa); + } + self.deleteDeclExports(decl); + self.comp.bin_file.freeDecl(decl); + decl.destroy(self.gpa); +} + +/// Delete all the Export objects that are caused by this Decl. Re-analysis of +/// this Decl will cause them to be re-created (or not). +fn deleteDeclExports(self: *Module, decl: *Decl) void { + const kv = self.export_owners.remove(decl) orelse return; + + for (kv.value) |exp| { + if (self.decl_exports.getEntry(exp.exported_decl)) |decl_exports_kv| { + // Remove exports with owner_decl matching the regenerating decl. + const list = decl_exports_kv.value; + var i: usize = 0; + var new_len = list.len; + while (i < new_len) { + if (list[i].owner_decl == decl) { + mem.copyBackwards(*Export, list[i..], list[i + 1 .. new_len]); + new_len -= 1; + } else { + i += 1; + } + } + decl_exports_kv.value = self.gpa.shrink(list, new_len); + if (new_len == 0) { + self.decl_exports.removeAssertDiscard(exp.exported_decl); + } + } + if (self.comp.bin_file.cast(link.File.Elf)) |elf| { + elf.deleteExport(exp.link); + } + if (self.failed_exports.remove(exp)) |entry| { + entry.value.destroy(self.gpa); + } + _ = self.symbol_exports.remove(exp.options.name); + self.gpa.free(exp.options.name); + self.gpa.destroy(exp); + } + self.gpa.free(kv.value); +} + +pub fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void { + const tracy = trace(@src()); + defer tracy.end(); + + // Use the Decl's arena for function memory. + var arena = decl.typed_value.most_recent.arena.?.promote(self.gpa); + defer decl.typed_value.most_recent.arena.?.* = arena.state; + var inner_block: Scope.Block = .{ + .parent = null, + .func = func, + .decl = decl, + .instructions = .{}, + .arena = &arena.allocator, + .is_comptime = false, + }; + defer inner_block.instructions.deinit(self.gpa); + + const fn_zir = func.analysis.queued; + defer fn_zir.arena.promote(self.gpa).deinit(); + func.analysis = .{ .in_progress = {} }; + log.debug("set {} to in_progress\n", .{decl.name}); + + try zir_sema.analyzeBody(self, &inner_block.base, fn_zir.body); + + const instructions = try arena.allocator.dupe(*Inst, inner_block.instructions.items); + func.analysis = .{ .success = .{ .instructions = instructions } }; + log.debug("set {} to success\n", .{decl.name}); +} + +fn markOutdatedDecl(self: *Module, decl: *Decl) !void { + log.debug("mark {} outdated\n", .{decl.name}); + try self.comp.work_queue.writeItem(.{ .analyze_decl = decl }); + if (self.failed_decls.remove(decl)) |entry| { + entry.value.destroy(self.gpa); + } + decl.analysis = .outdated; +} + +fn allocateNewDecl( + self: *Module, + scope: *Scope, + src_index: usize, + contents_hash: std.zig.SrcHash, +) !*Decl { + const new_decl = try self.gpa.create(Decl); + new_decl.* = .{ + .name = "", + .scope = scope.namespace(), + .src_index = src_index, + .typed_value = .{ .never_succeeded = {} }, + .analysis = .unreferenced, + .deletion_flag = false, + .contents_hash = contents_hash, + .link = switch (self.comp.bin_file.tag) { + .coff => .{ .coff = link.File.Coff.TextBlock.empty }, + .elf => .{ .elf = link.File.Elf.TextBlock.empty }, + .macho => .{ .macho = link.File.MachO.TextBlock.empty }, + .c => .{ .c = {} }, + .wasm => .{ .wasm = {} }, + }, + .fn_link = switch (self.comp.bin_file.tag) { + .coff => .{ .coff = {} }, + .elf => .{ .elf = link.File.Elf.SrcFn.empty }, + .macho => .{ .macho = link.File.MachO.SrcFn.empty }, + .c => .{ .c = {} }, + .wasm => .{ .wasm = null }, + }, + .generation = 0, + .is_pub = false, + }; + return new_decl; +} + +fn createNewDecl( + self: *Module, + scope: *Scope, + decl_name: []const u8, + src_index: usize, + name_hash: Scope.NameHash, + contents_hash: std.zig.SrcHash, +) !*Decl { + try self.decl_table.ensureCapacity(self.gpa, self.decl_table.items().len + 1); + const new_decl = try self.allocateNewDecl(scope, src_index, contents_hash); + errdefer self.gpa.destroy(new_decl); + new_decl.name = try mem.dupeZ(self.gpa, u8, decl_name); + self.decl_table.putAssumeCapacityNoClobber(name_hash, new_decl); + return new_decl; +} + +/// Get error value for error tag `name`. +pub fn getErrorValue(self: *Module, name: []const u8) !std.StringHashMapUnmanaged(u16).Entry { + const gop = try self.global_error_set.getOrPut(self.gpa, name); + if (gop.found_existing) + return gop.entry.*; + errdefer self.global_error_set.removeAssertDiscard(name); + + gop.entry.key = try self.gpa.dupe(u8, name); + gop.entry.value = @intCast(u16, self.global_error_set.count() - 1); + return gop.entry.*; +} + +pub fn requireFunctionBlock(self: *Module, scope: *Scope, src: usize) !*Scope.Block { + return scope.cast(Scope.Block) orelse + return self.fail(scope, src, "instruction illegal outside function body", .{}); +} + +pub fn requireRuntimeBlock(self: *Module, scope: *Scope, src: usize) !*Scope.Block { + const block = try self.requireFunctionBlock(scope, src); + if (block.is_comptime) { + return self.fail(scope, src, "unable to resolve comptime value", .{}); + } + return block; +} + +pub fn resolveConstValue(self: *Module, scope: *Scope, base: *Inst) !Value { + return (try self.resolveDefinedValue(scope, base)) orelse + return self.fail(scope, base.src, "unable to resolve comptime value", .{}); +} + +pub fn resolveDefinedValue(self: *Module, scope: *Scope, base: *Inst) !?Value { + if (base.value()) |val| { + if (val.isUndef()) { + return self.fail(scope, base.src, "use of undefined value here causes undefined behavior", .{}); + } + return val; + } + return null; +} + +pub fn analyzeExport(self: *Module, scope: *Scope, src: usize, borrowed_symbol_name: []const u8, exported_decl: *Decl) !void { + try self.ensureDeclAnalyzed(exported_decl); + const typed_value = exported_decl.typed_value.most_recent.typed_value; + switch (typed_value.ty.zigTypeTag()) { + .Fn => {}, + else => return self.fail(scope, src, "unable to export type '{}'", .{typed_value.ty}), + } + + try self.decl_exports.ensureCapacity(self.gpa, self.decl_exports.items().len + 1); + try self.export_owners.ensureCapacity(self.gpa, self.export_owners.items().len + 1); + + const new_export = try self.gpa.create(Export); + errdefer self.gpa.destroy(new_export); + + const symbol_name = try self.gpa.dupe(u8, borrowed_symbol_name); + errdefer self.gpa.free(symbol_name); + + const owner_decl = scope.decl().?; + + new_export.* = .{ + .options = .{ .name = symbol_name }, + .src = src, + .link = .{}, + .owner_decl = owner_decl, + .exported_decl = exported_decl, + .status = .in_progress, + }; + + // Add to export_owners table. + const eo_gop = self.export_owners.getOrPutAssumeCapacity(owner_decl); + if (!eo_gop.found_existing) { + eo_gop.entry.value = &[0]*Export{}; + } + eo_gop.entry.value = try self.gpa.realloc(eo_gop.entry.value, eo_gop.entry.value.len + 1); + eo_gop.entry.value[eo_gop.entry.value.len - 1] = new_export; + errdefer eo_gop.entry.value = self.gpa.shrink(eo_gop.entry.value, eo_gop.entry.value.len - 1); + + // Add to exported_decl table. + const de_gop = self.decl_exports.getOrPutAssumeCapacity(exported_decl); + if (!de_gop.found_existing) { + de_gop.entry.value = &[0]*Export{}; + } + de_gop.entry.value = try self.gpa.realloc(de_gop.entry.value, de_gop.entry.value.len + 1); + de_gop.entry.value[de_gop.entry.value.len - 1] = new_export; + errdefer de_gop.entry.value = self.gpa.shrink(de_gop.entry.value, de_gop.entry.value.len - 1); + + if (self.symbol_exports.get(symbol_name)) |_| { + try self.failed_exports.ensureCapacity(self.gpa, self.failed_exports.items().len + 1); + self.failed_exports.putAssumeCapacityNoClobber(new_export, try Compilation.ErrorMsg.create( + self.gpa, + src, + "exported symbol collision: {}", + .{symbol_name}, + )); + // TODO: add a note + new_export.status = .failed; + return; + } + + try self.symbol_exports.putNoClobber(self.gpa, symbol_name, new_export); + self.comp.bin_file.updateDeclExports(self, exported_decl, de_gop.entry.value) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => { + try self.failed_exports.ensureCapacity(self.gpa, self.failed_exports.items().len + 1); + self.failed_exports.putAssumeCapacityNoClobber(new_export, try Compilation.ErrorMsg.create( + self.gpa, + src, + "unable to export: {}", + .{@errorName(err)}, + )); + new_export.status = .failed_retryable; + }, + }; +} + +pub fn addNoOp( + self: *Module, + block: *Scope.Block, + src: usize, + ty: Type, + comptime tag: Inst.Tag, +) !*Inst { + const inst = try block.arena.create(tag.Type()); + inst.* = .{ + .base = .{ + .tag = tag, + .ty = ty, + .src = src, + }, + }; + try block.instructions.append(self.gpa, &inst.base); + return &inst.base; +} + +pub fn addUnOp( + self: *Module, + block: *Scope.Block, + src: usize, + ty: Type, + tag: Inst.Tag, + operand: *Inst, +) !*Inst { + const inst = try block.arena.create(Inst.UnOp); + inst.* = .{ + .base = .{ + .tag = tag, + .ty = ty, + .src = src, + }, + .operand = operand, + }; + try block.instructions.append(self.gpa, &inst.base); + return &inst.base; +} + +pub fn addBinOp( + self: *Module, + block: *Scope.Block, + src: usize, + ty: Type, + tag: Inst.Tag, + lhs: *Inst, + rhs: *Inst, +) !*Inst { + const inst = try block.arena.create(Inst.BinOp); + inst.* = .{ + .base = .{ + .tag = tag, + .ty = ty, + .src = src, + }, + .lhs = lhs, + .rhs = rhs, + }; + try block.instructions.append(self.gpa, &inst.base); + return &inst.base; +} + +pub fn addArg(self: *Module, block: *Scope.Block, src: usize, ty: Type, name: [*:0]const u8) !*Inst { + const inst = try block.arena.create(Inst.Arg); + inst.* = .{ + .base = .{ + .tag = .arg, + .ty = ty, + .src = src, + }, + .name = name, + }; + try block.instructions.append(self.gpa, &inst.base); + return &inst.base; +} + +pub fn addBr( + self: *Module, + scope_block: *Scope.Block, + src: usize, + target_block: *Inst.Block, + operand: *Inst, +) !*Inst { + const inst = try scope_block.arena.create(Inst.Br); + inst.* = .{ + .base = .{ + .tag = .br, + .ty = Type.initTag(.noreturn), + .src = src, + }, + .operand = operand, + .block = target_block, + }; + try scope_block.instructions.append(self.gpa, &inst.base); + return &inst.base; +} + +pub fn addCondBr( + self: *Module, + block: *Scope.Block, + src: usize, + condition: *Inst, + then_body: ir.Body, + else_body: ir.Body, +) !*Inst { + const inst = try block.arena.create(Inst.CondBr); + inst.* = .{ + .base = .{ + .tag = .condbr, + .ty = Type.initTag(.noreturn), + .src = src, + }, + .condition = condition, + .then_body = then_body, + .else_body = else_body, + }; + try block.instructions.append(self.gpa, &inst.base); + return &inst.base; +} + +pub fn addCall( + self: *Module, + block: *Scope.Block, + src: usize, + ty: Type, + func: *Inst, + args: []const *Inst, +) !*Inst { + const inst = try block.arena.create(Inst.Call); + inst.* = .{ + .base = .{ + .tag = .call, + .ty = ty, + .src = src, + }, + .func = func, + .args = args, + }; + try block.instructions.append(self.gpa, &inst.base); + return &inst.base; +} + +pub fn constInst(self: *Module, scope: *Scope, src: usize, typed_value: TypedValue) !*Inst { + const const_inst = try scope.arena().create(Inst.Constant); + const_inst.* = .{ + .base = .{ + .tag = Inst.Constant.base_tag, + .ty = typed_value.ty, + .src = src, + }, + .val = typed_value.val, + }; + return &const_inst.base; +} + +pub fn constType(self: *Module, scope: *Scope, src: usize, ty: Type) !*Inst { + return self.constInst(scope, src, .{ + .ty = Type.initTag(.type), + .val = try ty.toValue(scope.arena()), + }); +} + +pub fn constVoid(self: *Module, scope: *Scope, src: usize) !*Inst { + return self.constInst(scope, src, .{ + .ty = Type.initTag(.void), + .val = Value.initTag(.void_value), + }); +} + +pub fn constNoReturn(self: *Module, scope: *Scope, src: usize) !*Inst { + return self.constInst(scope, src, .{ + .ty = Type.initTag(.noreturn), + .val = Value.initTag(.unreachable_value), + }); +} + +pub fn constUndef(self: *Module, scope: *Scope, src: usize, ty: Type) !*Inst { + return self.constInst(scope, src, .{ + .ty = ty, + .val = Value.initTag(.undef), + }); +} + +pub fn constBool(self: *Module, scope: *Scope, src: usize, v: bool) !*Inst { + return self.constInst(scope, src, .{ + .ty = Type.initTag(.bool), + .val = ([2]Value{ Value.initTag(.bool_false), Value.initTag(.bool_true) })[@boolToInt(v)], + }); +} + +pub fn constIntUnsigned(self: *Module, scope: *Scope, src: usize, ty: Type, int: u64) !*Inst { + const int_payload = try scope.arena().create(Value.Payload.Int_u64); + int_payload.* = .{ .int = int }; + + return self.constInst(scope, src, .{ + .ty = ty, + .val = Value.initPayload(&int_payload.base), + }); +} + +pub fn constIntSigned(self: *Module, scope: *Scope, src: usize, ty: Type, int: i64) !*Inst { + const int_payload = try scope.arena().create(Value.Payload.Int_i64); + int_payload.* = .{ .int = int }; + + return self.constInst(scope, src, .{ + .ty = ty, + .val = Value.initPayload(&int_payload.base), + }); +} + +pub fn constIntBig(self: *Module, scope: *Scope, src: usize, ty: Type, big_int: BigIntConst) !*Inst { + const val_payload = if (big_int.positive) blk: { + if (big_int.to(u64)) |x| { + return self.constIntUnsigned(scope, src, ty, x); + } else |err| switch (err) { + error.NegativeIntoUnsigned => unreachable, + error.TargetTooSmall => {}, // handled below + } + const big_int_payload = try scope.arena().create(Value.Payload.IntBigPositive); + big_int_payload.* = .{ .limbs = big_int.limbs }; + break :blk &big_int_payload.base; + } else blk: { + if (big_int.to(i64)) |x| { + return self.constIntSigned(scope, src, ty, x); + } else |err| switch (err) { + error.NegativeIntoUnsigned => unreachable, + error.TargetTooSmall => {}, // handled below + } + const big_int_payload = try scope.arena().create(Value.Payload.IntBigNegative); + big_int_payload.* = .{ .limbs = big_int.limbs }; + break :blk &big_int_payload.base; + }; + + return self.constInst(scope, src, .{ + .ty = ty, + .val = Value.initPayload(val_payload), + }); +} + +pub fn createAnonymousDecl( + self: *Module, + scope: *Scope, + decl_arena: *std.heap.ArenaAllocator, + typed_value: TypedValue, +) !*Decl { + const name_index = self.getNextAnonNameIndex(); + const scope_decl = scope.decl().?; + const name = try std.fmt.allocPrint(self.gpa, "{}__anon_{}", .{ scope_decl.name, name_index }); + defer self.gpa.free(name); + const name_hash = scope.namespace().fullyQualifiedNameHash(name); + const src_hash: std.zig.SrcHash = undefined; + const new_decl = try self.createNewDecl(scope, name, scope_decl.src_index, name_hash, src_hash); + const decl_arena_state = try decl_arena.allocator.create(std.heap.ArenaAllocator.State); + + decl_arena_state.* = decl_arena.state; + new_decl.typed_value = .{ + .most_recent = .{ + .typed_value = typed_value, + .arena = decl_arena_state, + }, + }; + new_decl.analysis = .complete; + new_decl.generation = self.generation; + + // TODO: This generates the Decl into the machine code file if it is of a type that is non-zero size. + // We should be able to further improve the compiler to not omit Decls which are only referenced at + // compile-time and not runtime. + if (typed_value.ty.hasCodeGenBits()) { + try self.comp.bin_file.allocateDeclIndexes(new_decl); + try self.comp.work_queue.writeItem(.{ .codegen_decl = new_decl }); + } + + return new_decl; +} + +fn getNextAnonNameIndex(self: *Module) usize { + return @atomicRmw(usize, &self.next_anon_name_index, .Add, 1, .Monotonic); +} + +pub fn lookupDeclName(self: *Module, scope: *Scope, ident_name: []const u8) ?*Decl { + const namespace = scope.namespace(); + const name_hash = namespace.fullyQualifiedNameHash(ident_name); + return self.decl_table.get(name_hash); +} + +pub fn analyzeDeclRef(self: *Module, scope: *Scope, src: usize, decl: *Decl) InnerError!*Inst { + const scope_decl = scope.decl().?; + try self.declareDeclDependency(scope_decl, decl); + self.ensureDeclAnalyzed(decl) catch |err| { + if (scope.cast(Scope.Block)) |block| { + if (block.func) |func| { + func.analysis = .dependency_failure; + } else { + block.decl.analysis = .dependency_failure; + } + } else { + scope_decl.analysis = .dependency_failure; + } + return err; + }; + + const decl_tv = try decl.typedValue(); + if (decl_tv.val.tag() == .variable) { + return self.analyzeVarRef(scope, src, decl_tv); + } + const ty = try self.simplePtrType(scope, src, decl_tv.ty, false, .One); + const val_payload = try scope.arena().create(Value.Payload.DeclRef); + val_payload.* = .{ .decl = decl }; + + return self.constInst(scope, src, .{ + .ty = ty, + .val = Value.initPayload(&val_payload.base), + }); +} + +fn analyzeVarRef(self: *Module, scope: *Scope, src: usize, tv: TypedValue) InnerError!*Inst { + const variable = tv.val.cast(Value.Payload.Variable).?.variable; + + const ty = try self.simplePtrType(scope, src, tv.ty, variable.is_mutable, .One); + if (!variable.is_mutable and !variable.is_extern) { + const val_payload = try scope.arena().create(Value.Payload.RefVal); + val_payload.* = .{ .val = variable.init }; + return self.constInst(scope, src, .{ + .ty = ty, + .val = Value.initPayload(&val_payload.base), + }); + } + + const b = try self.requireRuntimeBlock(scope, src); + const inst = try b.arena.create(Inst.VarPtr); + inst.* = .{ + .base = .{ + .tag = .varptr, + .ty = ty, + .src = src, + }, + .variable = variable, + }; + try b.instructions.append(self.gpa, &inst.base); + return &inst.base; +} + +pub fn analyzeDeref(self: *Module, scope: *Scope, src: usize, ptr: *Inst, ptr_src: usize) InnerError!*Inst { + const elem_ty = switch (ptr.ty.zigTypeTag()) { + .Pointer => ptr.ty.elemType(), + else => return self.fail(scope, ptr_src, "expected pointer, found '{}'", .{ptr.ty}), + }; + if (ptr.value()) |val| { + return self.constInst(scope, src, .{ + .ty = elem_ty, + .val = try val.pointerDeref(scope.arena()), + }); + } + + const b = try self.requireRuntimeBlock(scope, src); + return self.addUnOp(b, src, elem_ty, .load, ptr); +} + +pub fn analyzeDeclRefByName(self: *Module, scope: *Scope, src: usize, decl_name: []const u8) InnerError!*Inst { + const decl = self.lookupDeclName(scope, decl_name) orelse + return self.fail(scope, src, "decl '{}' not found", .{decl_name}); + return self.analyzeDeclRef(scope, src, decl); +} + +pub fn wantSafety(self: *Module, scope: *Scope) bool { + // TODO take into account scope's safety overrides + return switch (self.optimizeMode()) { + .Debug => true, + .ReleaseSafe => true, + .ReleaseFast => false, + .ReleaseSmall => false, + }; +} + +pub fn analyzeIsNull( + self: *Module, + scope: *Scope, + src: usize, + operand: *Inst, + invert_logic: bool, +) InnerError!*Inst { + if (operand.value()) |opt_val| { + const is_null = opt_val.isNull(); + const bool_value = if (invert_logic) !is_null else is_null; + return self.constBool(scope, src, bool_value); + } + const b = try self.requireRuntimeBlock(scope, src); + const inst_tag: Inst.Tag = if (invert_logic) .isnonnull else .isnull; + return self.addUnOp(b, src, Type.initTag(.bool), inst_tag, operand); +} + +pub fn analyzeIsErr(self: *Module, scope: *Scope, src: usize, operand: *Inst) InnerError!*Inst { + return self.fail(scope, src, "TODO implement analysis of iserr", .{}); +} + +pub fn analyzeSlice(self: *Module, scope: *Scope, src: usize, array_ptr: *Inst, start: *Inst, end_opt: ?*Inst, sentinel_opt: ?*Inst) InnerError!*Inst { + const ptr_child = switch (array_ptr.ty.zigTypeTag()) { + .Pointer => array_ptr.ty.elemType(), + else => return self.fail(scope, src, "expected pointer, found '{}'", .{array_ptr.ty}), + }; + + var array_type = ptr_child; + const elem_type = switch (ptr_child.zigTypeTag()) { + .Array => ptr_child.elemType(), + .Pointer => blk: { + if (ptr_child.isSinglePointer()) { + if (ptr_child.elemType().zigTypeTag() == .Array) { + array_type = ptr_child.elemType(); + break :blk ptr_child.elemType().elemType(); + } + + return self.fail(scope, src, "slice of single-item pointer", .{}); + } + break :blk ptr_child.elemType(); + }, + else => return self.fail(scope, src, "slice of non-array type '{}'", .{ptr_child}), + }; + + const slice_sentinel = if (sentinel_opt) |sentinel| blk: { + const casted = try self.coerce(scope, elem_type, sentinel); + break :blk try self.resolveConstValue(scope, casted); + } else null; + + var return_ptr_size: std.builtin.TypeInfo.Pointer.Size = .Slice; + var return_elem_type = elem_type; + if (end_opt) |end| { + if (end.value()) |end_val| { + if (start.value()) |start_val| { + const start_u64 = start_val.toUnsignedInt(); + const end_u64 = end_val.toUnsignedInt(); + if (start_u64 > end_u64) { + return self.fail(scope, src, "out of bounds slice", .{}); + } + + const len = end_u64 - start_u64; + const array_sentinel = if (array_type.zigTypeTag() == .Array and end_u64 == array_type.arrayLen()) + array_type.sentinel() + else + slice_sentinel; + return_elem_type = try self.arrayType(scope, len, array_sentinel, elem_type); + return_ptr_size = .One; + } + } + } + const return_type = try self.ptrType( + scope, + src, + return_elem_type, + if (end_opt == null) slice_sentinel else null, + 0, // TODO alignment + 0, + 0, + !ptr_child.isConstPtr(), + ptr_child.isAllowzeroPtr(), + ptr_child.isVolatilePtr(), + return_ptr_size, + ); + + return self.fail(scope, src, "TODO implement analysis of slice", .{}); +} + +/// Asserts that lhs and rhs types are both numeric. +pub fn cmpNumeric( + self: *Module, + scope: *Scope, + src: usize, + lhs: *Inst, + rhs: *Inst, + op: std.math.CompareOperator, +) !*Inst { + assert(lhs.ty.isNumeric()); + assert(rhs.ty.isNumeric()); + + const lhs_ty_tag = lhs.ty.zigTypeTag(); + const rhs_ty_tag = rhs.ty.zigTypeTag(); + + if (lhs_ty_tag == .Vector and rhs_ty_tag == .Vector) { + if (lhs.ty.arrayLen() != rhs.ty.arrayLen()) { + return self.fail(scope, src, "vector length mismatch: {} and {}", .{ + lhs.ty.arrayLen(), + rhs.ty.arrayLen(), + }); + } + return self.fail(scope, src, "TODO implement support for vectors in cmpNumeric", .{}); + } else if (lhs_ty_tag == .Vector or rhs_ty_tag == .Vector) { + return self.fail(scope, src, "mixed scalar and vector operands to comparison operator: '{}' and '{}'", .{ + lhs.ty, + rhs.ty, + }); + } + + if (lhs.value()) |lhs_val| { + if (rhs.value()) |rhs_val| { + return self.constBool(scope, src, Value.compare(lhs_val, op, rhs_val)); + } + } + + // TODO handle comparisons against lazy zero values + // Some values can be compared against zero without being runtime known or without forcing + // a full resolution of their value, for example `@sizeOf(@Frame(function))` is known to + // always be nonzero, and we benefit from not forcing the full evaluation and stack frame layout + // of this function if we don't need to. + + // It must be a runtime comparison. + const b = try self.requireRuntimeBlock(scope, src); + // For floats, emit a float comparison instruction. + const lhs_is_float = switch (lhs_ty_tag) { + .Float, .ComptimeFloat => true, + else => false, + }; + const rhs_is_float = switch (rhs_ty_tag) { + .Float, .ComptimeFloat => true, + else => false, + }; + if (lhs_is_float and rhs_is_float) { + // Implicit cast the smaller one to the larger one. + const dest_type = x: { + if (lhs_ty_tag == .ComptimeFloat) { + break :x rhs.ty; + } else if (rhs_ty_tag == .ComptimeFloat) { + break :x lhs.ty; + } + if (lhs.ty.floatBits(self.getTarget()) >= rhs.ty.floatBits(self.getTarget())) { + break :x lhs.ty; + } else { + break :x rhs.ty; + } + }; + const casted_lhs = try self.coerce(scope, dest_type, lhs); + const casted_rhs = try self.coerce(scope, dest_type, rhs); + return self.addBinOp(b, src, dest_type, Inst.Tag.fromCmpOp(op), casted_lhs, casted_rhs); + } + // For mixed unsigned integer sizes, implicit cast both operands to the larger integer. + // For mixed signed and unsigned integers, implicit cast both operands to a signed + // integer with + 1 bit. + // For mixed floats and integers, extract the integer part from the float, cast that to + // a signed integer with mantissa bits + 1, and if there was any non-integral part of the float, + // add/subtract 1. + const lhs_is_signed = if (lhs.value()) |lhs_val| + lhs_val.compareWithZero(.lt) + else + (lhs.ty.isFloat() or lhs.ty.isSignedInt()); + const rhs_is_signed = if (rhs.value()) |rhs_val| + rhs_val.compareWithZero(.lt) + else + (rhs.ty.isFloat() or rhs.ty.isSignedInt()); + const dest_int_is_signed = lhs_is_signed or rhs_is_signed; + + var dest_float_type: ?Type = null; + + var lhs_bits: usize = undefined; + if (lhs.value()) |lhs_val| { + if (lhs_val.isUndef()) + return self.constUndef(scope, src, Type.initTag(.bool)); + const is_unsigned = if (lhs_is_float) x: { + var bigint_space: Value.BigIntSpace = undefined; + var bigint = try lhs_val.toBigInt(&bigint_space).toManaged(self.gpa); + defer bigint.deinit(); + const zcmp = lhs_val.orderAgainstZero(); + if (lhs_val.floatHasFraction()) { + switch (op) { + .eq => return self.constBool(scope, src, false), + .neq => return self.constBool(scope, src, true), + else => {}, + } + if (zcmp == .lt) { + try bigint.addScalar(bigint.toConst(), -1); + } else { + try bigint.addScalar(bigint.toConst(), 1); + } + } + lhs_bits = bigint.toConst().bitCountTwosComp(); + break :x (zcmp != .lt); + } else x: { + lhs_bits = lhs_val.intBitCountTwosComp(); + break :x (lhs_val.orderAgainstZero() != .lt); + }; + lhs_bits += @boolToInt(is_unsigned and dest_int_is_signed); + } else if (lhs_is_float) { + dest_float_type = lhs.ty; + } else { + const int_info = lhs.ty.intInfo(self.getTarget()); + lhs_bits = int_info.bits + @boolToInt(!int_info.signed and dest_int_is_signed); + } + + var rhs_bits: usize = undefined; + if (rhs.value()) |rhs_val| { + if (rhs_val.isUndef()) + return self.constUndef(scope, src, Type.initTag(.bool)); + const is_unsigned = if (rhs_is_float) x: { + var bigint_space: Value.BigIntSpace = undefined; + var bigint = try rhs_val.toBigInt(&bigint_space).toManaged(self.gpa); + defer bigint.deinit(); + const zcmp = rhs_val.orderAgainstZero(); + if (rhs_val.floatHasFraction()) { + switch (op) { + .eq => return self.constBool(scope, src, false), + .neq => return self.constBool(scope, src, true), + else => {}, + } + if (zcmp == .lt) { + try bigint.addScalar(bigint.toConst(), -1); + } else { + try bigint.addScalar(bigint.toConst(), 1); + } + } + rhs_bits = bigint.toConst().bitCountTwosComp(); + break :x (zcmp != .lt); + } else x: { + rhs_bits = rhs_val.intBitCountTwosComp(); + break :x (rhs_val.orderAgainstZero() != .lt); + }; + rhs_bits += @boolToInt(is_unsigned and dest_int_is_signed); + } else if (rhs_is_float) { + dest_float_type = rhs.ty; + } else { + const int_info = rhs.ty.intInfo(self.getTarget()); + rhs_bits = int_info.bits + @boolToInt(!int_info.signed and dest_int_is_signed); + } + + const dest_type = if (dest_float_type) |ft| ft else blk: { + const max_bits = std.math.max(lhs_bits, rhs_bits); + const casted_bits = std.math.cast(u16, max_bits) catch |err| switch (err) { + error.Overflow => return self.fail(scope, src, "{} exceeds maximum integer bit count", .{max_bits}), + }; + break :blk try self.makeIntType(scope, dest_int_is_signed, casted_bits); + }; + const casted_lhs = try self.coerce(scope, dest_type, lhs); + const casted_rhs = try self.coerce(scope, dest_type, rhs); + + return self.addBinOp(b, src, Type.initTag(.bool), Inst.Tag.fromCmpOp(op), casted_lhs, casted_rhs); +} + +fn wrapOptional(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst { + if (inst.value()) |val| { + return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = val }); + } + + const b = try self.requireRuntimeBlock(scope, inst.src); + return self.addUnOp(b, inst.src, dest_type, .wrap_optional, inst); +} + +fn makeIntType(self: *Module, scope: *Scope, signed: bool, bits: u16) !Type { + if (signed) { + const int_payload = try scope.arena().create(Type.Payload.IntSigned); + int_payload.* = .{ .bits = bits }; + return Type.initPayload(&int_payload.base); + } else { + const int_payload = try scope.arena().create(Type.Payload.IntUnsigned); + int_payload.* = .{ .bits = bits }; + return Type.initPayload(&int_payload.base); + } +} + +pub fn resolvePeerTypes(self: *Module, scope: *Scope, instructions: []*Inst) !Type { + if (instructions.len == 0) + return Type.initTag(.noreturn); + + if (instructions.len == 1) + return instructions[0].ty; + + var prev_inst = instructions[0]; + for (instructions[1..]) |next_inst| { + if (next_inst.ty.eql(prev_inst.ty)) + continue; + if (next_inst.ty.zigTypeTag() == .NoReturn) + continue; + if (prev_inst.ty.zigTypeTag() == .NoReturn) { + prev_inst = next_inst; + continue; + } + if (next_inst.ty.zigTypeTag() == .Undefined) + continue; + if (prev_inst.ty.zigTypeTag() == .Undefined) { + prev_inst = next_inst; + continue; + } + if (prev_inst.ty.isInt() and + next_inst.ty.isInt() and + prev_inst.ty.isSignedInt() == next_inst.ty.isSignedInt()) + { + if (prev_inst.ty.intInfo(self.getTarget()).bits < next_inst.ty.intInfo(self.getTarget()).bits) { + prev_inst = next_inst; + } + continue; + } + if (prev_inst.ty.isFloat() and next_inst.ty.isFloat()) { + if (prev_inst.ty.floatBits(self.getTarget()) < next_inst.ty.floatBits(self.getTarget())) { + prev_inst = next_inst; + } + continue; + } + + // TODO error notes pointing out each type + return self.fail(scope, next_inst.src, "incompatible types: '{}' and '{}'", .{ prev_inst.ty, next_inst.ty }); + } + + return prev_inst.ty; +} + +pub fn coerce(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst { + // If the types are the same, we can return the operand. + if (dest_type.eql(inst.ty)) + return inst; + + const in_memory_result = coerceInMemoryAllowed(dest_type, inst.ty); + if (in_memory_result == .ok) { + return self.bitcast(scope, dest_type, inst); + } + + // undefined to anything + if (inst.value()) |val| { + if (val.isUndef() or inst.ty.zigTypeTag() == .Undefined) { + return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = val }); + } + } + assert(inst.ty.zigTypeTag() != .Undefined); + + // null to ?T + if (dest_type.zigTypeTag() == .Optional and inst.ty.zigTypeTag() == .Null) { + return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = Value.initTag(.null_value) }); + } + + // T to ?T + if (dest_type.zigTypeTag() == .Optional) { + var buf: Type.Payload.PointerSimple = undefined; + const child_type = dest_type.optionalChild(&buf); + if (child_type.eql(inst.ty)) { + return self.wrapOptional(scope, dest_type, inst); + } else if (try self.coerceNum(scope, child_type, inst)) |some| { + return self.wrapOptional(scope, dest_type, some); + } + } + + // *[N]T to []T + if (inst.ty.isSinglePointer() and dest_type.isSlice() and + (!inst.ty.isConstPtr() or dest_type.isConstPtr())) + { + const array_type = inst.ty.elemType(); + const dst_elem_type = dest_type.elemType(); + if (array_type.zigTypeTag() == .Array and + coerceInMemoryAllowed(dst_elem_type, array_type.elemType()) == .ok) + { + return self.coerceArrayPtrToSlice(scope, dest_type, inst); + } + } + + // comptime known number to other number + if (try self.coerceNum(scope, dest_type, inst)) |some| + return some; + + // integer widening + if (inst.ty.zigTypeTag() == .Int and dest_type.zigTypeTag() == .Int) { + assert(inst.value() == null); // handled above + + const src_info = inst.ty.intInfo(self.getTarget()); + const dst_info = dest_type.intInfo(self.getTarget()); + if ((src_info.signed == dst_info.signed and dst_info.bits >= src_info.bits) or + // small enough unsigned ints can get casted to large enough signed ints + (src_info.signed and !dst_info.signed and dst_info.bits > src_info.bits)) + { + const b = try self.requireRuntimeBlock(scope, inst.src); + return self.addUnOp(b, inst.src, dest_type, .intcast, inst); + } + } + + // float widening + if (inst.ty.zigTypeTag() == .Float and dest_type.zigTypeTag() == .Float) { + assert(inst.value() == null); // handled above + + const src_bits = inst.ty.floatBits(self.getTarget()); + const dst_bits = dest_type.floatBits(self.getTarget()); + if (dst_bits >= src_bits) { + const b = try self.requireRuntimeBlock(scope, inst.src); + return self.addUnOp(b, inst.src, dest_type, .floatcast, inst); + } + } + + return self.fail(scope, inst.src, "expected {}, found {}", .{ dest_type, inst.ty }); +} + +pub fn coerceNum(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !?*Inst { + const val = inst.value() orelse return null; + const src_zig_tag = inst.ty.zigTypeTag(); + const dst_zig_tag = dest_type.zigTypeTag(); + + if (dst_zig_tag == .ComptimeInt or dst_zig_tag == .Int) { + if (src_zig_tag == .Float or src_zig_tag == .ComptimeFloat) { + if (val.floatHasFraction()) { + return self.fail(scope, inst.src, "fractional component prevents float value {} from being casted to type '{}'", .{ val, inst.ty }); + } + return self.fail(scope, inst.src, "TODO float to int", .{}); + } else if (src_zig_tag == .Int or src_zig_tag == .ComptimeInt) { + if (!val.intFitsInType(dest_type, self.getTarget())) { + return self.fail(scope, inst.src, "type {} cannot represent integer value {}", .{ inst.ty, val }); + } + return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = val }); + } + } else if (dst_zig_tag == .ComptimeFloat or dst_zig_tag == .Float) { + if (src_zig_tag == .Float or src_zig_tag == .ComptimeFloat) { + const res = val.floatCast(scope.arena(), dest_type, self.getTarget()) catch |err| switch (err) { + error.Overflow => return self.fail( + scope, + inst.src, + "cast of value {} to type '{}' loses information", + .{ val, dest_type }, + ), + error.OutOfMemory => return error.OutOfMemory, + }; + return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = res }); + } else if (src_zig_tag == .Int or src_zig_tag == .ComptimeInt) { + return self.fail(scope, inst.src, "TODO int to float", .{}); + } + } + return null; +} + +pub fn storePtr(self: *Module, scope: *Scope, src: usize, ptr: *Inst, uncasted_value: *Inst) !*Inst { + if (ptr.ty.isConstPtr()) + return self.fail(scope, src, "cannot assign to constant", .{}); + + const elem_ty = ptr.ty.elemType(); + const value = try self.coerce(scope, elem_ty, uncasted_value); + if (elem_ty.onePossibleValue() != null) + return self.constVoid(scope, src); + + // TODO handle comptime pointer writes + // TODO handle if the element type requires comptime + + const b = try self.requireRuntimeBlock(scope, src); + return self.addBinOp(b, src, Type.initTag(.void), .store, ptr, value); +} + +pub fn bitcast(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst { + if (inst.value()) |val| { + // Keep the comptime Value representation; take the new type. + return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = val }); + } + // TODO validate the type size and other compile errors + const b = try self.requireRuntimeBlock(scope, inst.src); + return self.addUnOp(b, inst.src, dest_type, .bitcast, inst); +} + +fn coerceArrayPtrToSlice(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst { + if (inst.value()) |val| { + // The comptime Value representation is compatible with both types. + return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = val }); + } + return self.fail(scope, inst.src, "TODO implement coerceArrayPtrToSlice runtime instruction", .{}); +} + +pub fn fail(self: *Module, scope: *Scope, src: usize, comptime format: []const u8, args: anytype) InnerError { + @setCold(true); + const err_msg = try Compilation.ErrorMsg.create(self.gpa, src, format, args); + return self.failWithOwnedErrorMsg(scope, src, err_msg); +} + +pub fn failTok( + self: *Module, + scope: *Scope, + token_index: ast.TokenIndex, + comptime format: []const u8, + args: anytype, +) InnerError { + @setCold(true); + const src = scope.tree().token_locs[token_index].start; + return self.fail(scope, src, format, args); +} + +pub fn failNode( + self: *Module, + scope: *Scope, + ast_node: *ast.Node, + comptime format: []const u8, + args: anytype, +) InnerError { + @setCold(true); + const src = scope.tree().token_locs[ast_node.firstToken()].start; + return self.fail(scope, src, format, args); +} + +fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Compilation.ErrorMsg) InnerError { + { + errdefer err_msg.destroy(self.gpa); + try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1); + try self.failed_files.ensureCapacity(self.gpa, self.failed_files.items().len + 1); + } + switch (scope.tag) { + .decl => { + const decl = scope.cast(Scope.DeclAnalysis).?.decl; + decl.analysis = .sema_failure; + decl.generation = self.generation; + self.failed_decls.putAssumeCapacityNoClobber(decl, err_msg); + }, + .block => { + const block = scope.cast(Scope.Block).?; + if (block.func) |func| { + func.analysis = .sema_failure; + } else { + block.decl.analysis = .sema_failure; + block.decl.generation = self.generation; + } + self.failed_decls.putAssumeCapacityNoClobber(block.decl, err_msg); + }, + .gen_zir => { + const gen_zir = scope.cast(Scope.GenZIR).?; + gen_zir.decl.analysis = .sema_failure; + gen_zir.decl.generation = self.generation; + self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg); + }, + .local_val => { + const gen_zir = scope.cast(Scope.LocalVal).?.gen_zir; + gen_zir.decl.analysis = .sema_failure; + gen_zir.decl.generation = self.generation; + self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg); + }, + .local_ptr => { + const gen_zir = scope.cast(Scope.LocalPtr).?.gen_zir; + gen_zir.decl.analysis = .sema_failure; + gen_zir.decl.generation = self.generation; + self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg); + }, + .zir_module => { + const zir_module = scope.cast(Scope.ZIRModule).?; + zir_module.status = .loaded_sema_failure; + self.failed_files.putAssumeCapacityNoClobber(scope, err_msg); + }, + .file => unreachable, + .container => unreachable, + } + return error.AnalysisFail; +} + +const InMemoryCoercionResult = enum { + ok, + no_match, +}; + +fn coerceInMemoryAllowed(dest_type: Type, src_type: Type) InMemoryCoercionResult { + if (dest_type.eql(src_type)) + return .ok; + + // TODO: implement more of this function + + return .no_match; +} + +fn srcHashEql(a: std.zig.SrcHash, b: std.zig.SrcHash) bool { + return @bitCast(u128, a) == @bitCast(u128, b); +} + +pub fn intAdd(allocator: *Allocator, lhs: Value, rhs: Value) !Value { + // TODO is this a performance issue? maybe we should try the operation without + // resorting to BigInt first. + var lhs_space: Value.BigIntSpace = undefined; + var rhs_space: Value.BigIntSpace = undefined; + const lhs_bigint = lhs.toBigInt(&lhs_space); + const rhs_bigint = rhs.toBigInt(&rhs_space); + const limbs = try allocator.alloc( + std.math.big.Limb, + std.math.max(lhs_bigint.limbs.len, rhs_bigint.limbs.len) + 1, + ); + var result_bigint = BigIntMutable{ .limbs = limbs, .positive = undefined, .len = undefined }; + result_bigint.add(lhs_bigint, rhs_bigint); + const result_limbs = result_bigint.limbs[0..result_bigint.len]; + + const val_payload = if (result_bigint.positive) blk: { + const val_payload = try allocator.create(Value.Payload.IntBigPositive); + val_payload.* = .{ .limbs = result_limbs }; + break :blk &val_payload.base; + } else blk: { + const val_payload = try allocator.create(Value.Payload.IntBigNegative); + val_payload.* = .{ .limbs = result_limbs }; + break :blk &val_payload.base; + }; + + return Value.initPayload(val_payload); +} + +pub fn intSub(allocator: *Allocator, lhs: Value, rhs: Value) !Value { + // TODO is this a performance issue? maybe we should try the operation without + // resorting to BigInt first. + var lhs_space: Value.BigIntSpace = undefined; + var rhs_space: Value.BigIntSpace = undefined; + const lhs_bigint = lhs.toBigInt(&lhs_space); + const rhs_bigint = rhs.toBigInt(&rhs_space); + const limbs = try allocator.alloc( + std.math.big.Limb, + std.math.max(lhs_bigint.limbs.len, rhs_bigint.limbs.len) + 1, + ); + var result_bigint = BigIntMutable{ .limbs = limbs, .positive = undefined, .len = undefined }; + result_bigint.sub(lhs_bigint, rhs_bigint); + const result_limbs = result_bigint.limbs[0..result_bigint.len]; + + const val_payload = if (result_bigint.positive) blk: { + const val_payload = try allocator.create(Value.Payload.IntBigPositive); + val_payload.* = .{ .limbs = result_limbs }; + break :blk &val_payload.base; + } else blk: { + const val_payload = try allocator.create(Value.Payload.IntBigNegative); + val_payload.* = .{ .limbs = result_limbs }; + break :blk &val_payload.base; + }; + + return Value.initPayload(val_payload); +} + +pub fn floatAdd(self: *Module, scope: *Scope, float_type: Type, src: usize, lhs: Value, rhs: Value) !Value { + var bit_count = switch (float_type.tag()) { + .comptime_float => 128, + else => float_type.floatBits(self.getTarget()), + }; + + const allocator = scope.arena(); + const val_payload = switch (bit_count) { + 16 => { + return self.fail(scope, src, "TODO Implement addition for soft floats", .{}); + }, + 32 => blk: { + const lhs_val = lhs.toFloat(f32); + const rhs_val = rhs.toFloat(f32); + const val_payload = try allocator.create(Value.Payload.Float_32); + val_payload.* = .{ .val = lhs_val + rhs_val }; + break :blk &val_payload.base; + }, + 64 => blk: { + const lhs_val = lhs.toFloat(f64); + const rhs_val = rhs.toFloat(f64); + const val_payload = try allocator.create(Value.Payload.Float_64); + val_payload.* = .{ .val = lhs_val + rhs_val }; + break :blk &val_payload.base; + }, + 128 => { + return self.fail(scope, src, "TODO Implement addition for big floats", .{}); + }, + else => unreachable, + }; + + return Value.initPayload(val_payload); +} + +pub fn floatSub(self: *Module, scope: *Scope, float_type: Type, src: usize, lhs: Value, rhs: Value) !Value { + var bit_count = switch (float_type.tag()) { + .comptime_float => 128, + else => float_type.floatBits(self.getTarget()), + }; + + const allocator = scope.arena(); + const val_payload = switch (bit_count) { + 16 => { + return self.fail(scope, src, "TODO Implement substraction for soft floats", .{}); + }, + 32 => blk: { + const lhs_val = lhs.toFloat(f32); + const rhs_val = rhs.toFloat(f32); + const val_payload = try allocator.create(Value.Payload.Float_32); + val_payload.* = .{ .val = lhs_val - rhs_val }; + break :blk &val_payload.base; + }, + 64 => blk: { + const lhs_val = lhs.toFloat(f64); + const rhs_val = rhs.toFloat(f64); + const val_payload = try allocator.create(Value.Payload.Float_64); + val_payload.* = .{ .val = lhs_val - rhs_val }; + break :blk &val_payload.base; + }, + 128 => { + return self.fail(scope, src, "TODO Implement substraction for big floats", .{}); + }, + else => unreachable, + }; + + return Value.initPayload(val_payload); +} + +pub fn simplePtrType(self: *Module, scope: *Scope, src: usize, elem_ty: Type, mutable: bool, size: std.builtin.TypeInfo.Pointer.Size) Allocator.Error!Type { + if (!mutable and size == .Slice and elem_ty.eql(Type.initTag(.u8))) { + return Type.initTag(.const_slice_u8); + } + // TODO stage1 type inference bug + const T = Type.Tag; + + const type_payload = try scope.arena().create(Type.Payload.PointerSimple); + type_payload.* = .{ + .base = .{ + .tag = switch (size) { + .One => if (mutable) T.single_mut_pointer else T.single_const_pointer, + .Many => if (mutable) T.many_mut_pointer else T.many_const_pointer, + .C => if (mutable) T.c_mut_pointer else T.c_const_pointer, + .Slice => if (mutable) T.mut_slice else T.const_slice, + }, + }, + .pointee_type = elem_ty, + }; + return Type.initPayload(&type_payload.base); +} + +pub fn ptrType( + self: *Module, + scope: *Scope, + src: usize, + elem_ty: Type, + sentinel: ?Value, + @"align": u32, + bit_offset: u16, + host_size: u16, + mutable: bool, + @"allowzero": bool, + @"volatile": bool, + size: std.builtin.TypeInfo.Pointer.Size, +) Allocator.Error!Type { + assert(host_size == 0 or bit_offset < host_size * 8); + + // TODO check if type can be represented by simplePtrType + const type_payload = try scope.arena().create(Type.Payload.Pointer); + type_payload.* = .{ + .pointee_type = elem_ty, + .sentinel = sentinel, + .@"align" = @"align", + .bit_offset = bit_offset, + .host_size = host_size, + .@"allowzero" = @"allowzero", + .mutable = mutable, + .@"volatile" = @"volatile", + .size = size, + }; + return Type.initPayload(&type_payload.base); +} + +pub fn optionalType(self: *Module, scope: *Scope, child_type: Type) Allocator.Error!Type { + return Type.initPayload(switch (child_type.tag()) { + .single_const_pointer => blk: { + const payload = try scope.arena().create(Type.Payload.PointerSimple); + payload.* = .{ + .base = .{ .tag = .optional_single_const_pointer }, + .pointee_type = child_type.elemType(), + }; + break :blk &payload.base; + }, + .single_mut_pointer => blk: { + const payload = try scope.arena().create(Type.Payload.PointerSimple); + payload.* = .{ + .base = .{ .tag = .optional_single_mut_pointer }, + .pointee_type = child_type.elemType(), + }; + break :blk &payload.base; + }, + else => blk: { + const payload = try scope.arena().create(Type.Payload.Optional); + payload.* = .{ + .child_type = child_type, + }; + break :blk &payload.base; + }, + }); +} + +pub fn arrayType(self: *Module, scope: *Scope, len: u64, sentinel: ?Value, elem_type: Type) Allocator.Error!Type { + if (elem_type.eql(Type.initTag(.u8))) { + if (sentinel) |some| { + if (some.eql(Value.initTag(.zero))) { + const payload = try scope.arena().create(Type.Payload.Array_u8_Sentinel0); + payload.* = .{ + .len = len, + }; + return Type.initPayload(&payload.base); + } + } else { + const payload = try scope.arena().create(Type.Payload.Array_u8); + payload.* = .{ + .len = len, + }; + return Type.initPayload(&payload.base); + } + } + + if (sentinel) |some| { + const payload = try scope.arena().create(Type.Payload.ArraySentinel); + payload.* = .{ + .len = len, + .sentinel = some, + .elem_type = elem_type, + }; + return Type.initPayload(&payload.base); + } + + const payload = try scope.arena().create(Type.Payload.Array); + payload.* = .{ + .len = len, + .elem_type = elem_type, + }; + return Type.initPayload(&payload.base); +} + +pub fn errorUnionType(self: *Module, scope: *Scope, error_set: Type, payload: Type) Allocator.Error!Type { + assert(error_set.zigTypeTag() == .ErrorSet); + if (error_set.eql(Type.initTag(.anyerror)) and payload.eql(Type.initTag(.void))) { + return Type.initTag(.anyerror_void_error_union); + } + + const result = try scope.arena().create(Type.Payload.ErrorUnion); + result.* = .{ + .error_set = error_set, + .payload = payload, + }; + return Type.initPayload(&result.base); +} + +pub fn anyframeType(self: *Module, scope: *Scope, return_type: Type) Allocator.Error!Type { + const result = try scope.arena().create(Type.Payload.AnyFrame); + result.* = .{ + .return_type = return_type, + }; + return Type.initPayload(&result.base); +} + +pub fn dumpInst(self: *Module, scope: *Scope, inst: *Inst) void { + const zir_module = scope.namespace(); + const source = zir_module.getSource(self) catch @panic("dumpInst failed to get source"); + const loc = std.zig.findLineColumn(source, inst.src); + if (inst.tag == .constant) { + std.debug.print("constant ty={} val={} src={}:{}:{}\n", .{ + inst.ty, + inst.castTag(.constant).?.val, + zir_module.subFilePath(), + loc.line + 1, + loc.column + 1, + }); + } else if (inst.deaths == 0) { + std.debug.print("{} ty={} src={}:{}:{}\n", .{ + @tagName(inst.tag), + inst.ty, + zir_module.subFilePath(), + loc.line + 1, + loc.column + 1, + }); + } else { + std.debug.print("{} ty={} deaths={b} src={}:{}:{}\n", .{ + @tagName(inst.tag), + inst.ty, + inst.deaths, + zir_module.subFilePath(), + loc.line + 1, + loc.column + 1, + }); + } +} + +pub const PanicId = enum { + unreach, + unwrap_null, +}; + +pub fn addSafetyCheck(mod: *Module, parent_block: *Scope.Block, ok: *Inst, panic_id: PanicId) !void { + const block_inst = try parent_block.arena.create(Inst.Block); + block_inst.* = .{ + .base = .{ + .tag = Inst.Block.base_tag, + .ty = Type.initTag(.void), + .src = ok.src, + }, + .body = .{ + .instructions = try parent_block.arena.alloc(*Inst, 1), // Only need space for the condbr. + }, + }; + + const ok_body: ir.Body = .{ + .instructions = try parent_block.arena.alloc(*Inst, 1), // Only need space for the brvoid. + }; + const brvoid = try parent_block.arena.create(Inst.BrVoid); + brvoid.* = .{ + .base = .{ + .tag = .brvoid, + .ty = Type.initTag(.noreturn), + .src = ok.src, + }, + .block = block_inst, + }; + ok_body.instructions[0] = &brvoid.base; + + var fail_block: Scope.Block = .{ + .parent = parent_block, + .func = parent_block.func, + .decl = parent_block.decl, + .instructions = .{}, + .arena = parent_block.arena, + .is_comptime = parent_block.is_comptime, + }; + defer fail_block.instructions.deinit(mod.gpa); + + _ = try mod.safetyPanic(&fail_block, ok.src, panic_id); + + const fail_body: ir.Body = .{ .instructions = try parent_block.arena.dupe(*Inst, fail_block.instructions.items) }; + + const condbr = try parent_block.arena.create(Inst.CondBr); + condbr.* = .{ + .base = .{ + .tag = .condbr, + .ty = Type.initTag(.noreturn), + .src = ok.src, + }, + .condition = ok, + .then_body = ok_body, + .else_body = fail_body, + }; + block_inst.body.instructions[0] = &condbr.base; + + try parent_block.instructions.append(mod.gpa, &block_inst.base); +} + +pub fn safetyPanic(mod: *Module, block: *Scope.Block, src: usize, panic_id: PanicId) !*Inst { + // TODO Once we have a panic function to call, call it here instead of breakpoint. + _ = try mod.addNoOp(block, src, Type.initTag(.void), .breakpoint); + return mod.addNoOp(block, src, Type.initTag(.noreturn), .unreach); +} + +pub fn getTarget(self: Module) Target { + return self.comp.bin_file.options.target; +} + +pub fn optimizeMode(self: Module) std.builtin.Mode { + return self.comp.bin_file.options.optimize_mode; +} diff --git a/src-self-hosted/astgen.zig b/src-self-hosted/astgen.zig index 2c091a86ec..f39612b7ef 100644 --- a/src-self-hosted/astgen.zig +++ b/src-self-hosted/astgen.zig @@ -6,7 +6,7 @@ const Type = @import("type.zig").Type; const TypedValue = @import("TypedValue.zig"); const assert = std.debug.assert; const zir = @import("zir.zig"); -const Module = @import("Module.zig"); +const Module = @import("ZigModule.zig"); const ast = std.zig.ast; const trace = @import("tracy.zig").trace; const Scope = Module.Scope; diff --git a/src-self-hosted/codegen.zig b/src-self-hosted/codegen.zig index 9405a5f72c..f35092639b 100644 --- a/src-self-hosted/codegen.zig +++ b/src-self-hosted/codegen.zig @@ -7,8 +7,9 @@ const Type = @import("type.zig").Type; const Value = @import("value.zig").Value; const TypedValue = @import("TypedValue.zig"); const link = @import("link.zig"); -const Module = @import("Module.zig"); -const ErrorMsg = Module.ErrorMsg; +const Module = @import("ZigModule.zig"); +const Compilation = @import("Module.zig"); +const ErrorMsg = Compilation.ErrorMsg; const Target = std.Target; const Allocator = mem.Allocator; const trace = @import("tracy.zig").trace; @@ -50,7 +51,7 @@ pub const Result = union(enum) { appended: void, /// The value is available externally, `code` is unused. externally_managed: []const u8, - fail: *Module.ErrorMsg, + fail: *ErrorMsg, }; pub const GenerateSymbolError = error{ diff --git a/src-self-hosted/codegen/c.zig b/src-self-hosted/codegen/c.zig index 34ddcfbb3b..4ad2a6dafc 100644 --- a/src-self-hosted/codegen/c.zig +++ b/src-self-hosted/codegen/c.zig @@ -1,7 +1,7 @@ const std = @import("std"); const link = @import("../link.zig"); -const Module = @import("../Module.zig"); +const Module = @import("../ZigModule.zig"); const Inst = @import("../ir.zig").Inst; const Value = @import("../value.zig").Value; diff --git a/src-self-hosted/codegen/wasm.zig b/src-self-hosted/codegen/wasm.zig index e55e904934..5f44aa8c65 100644 --- a/src-self-hosted/codegen/wasm.zig +++ b/src-self-hosted/codegen/wasm.zig @@ -5,7 +5,8 @@ const assert = std.debug.assert; const leb = std.debug.leb; const mem = std.mem; -const Decl = @import("../Module.zig").Decl; +const Module = @import("../ZigModule.zig"); +const Decl = Module.Decl; const Inst = @import("../ir.zig").Inst; const Type = @import("../type.zig").Type; const Value = @import("../value.zig").Value; diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index 9d0a0d45fd..d366f74286 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -5,6 +5,7 @@ const mem = std.mem; const Module = @import("Module.zig"); const path = std.fs.path; const build_options = @import("build_options"); +const trace = @import("tracy.zig").trace; pub const Lib = struct { name: []const u8, @@ -54,6 +55,9 @@ pub const LoadMetaDataError = error{ /// This function will emit a log error when there is a problem with the zig installation and then return /// `error.ZigInstallationCorrupt`. pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!*ABI { + const tracy = trace(@src()); + defer tracy.end(); + var arena_allocator = std.heap.ArenaAllocator.init(gpa); errdefer arena_allocator.deinit(); const arena = &arena_allocator.allocator; @@ -584,6 +588,9 @@ fn lib_path(mod: *Module, arena: *Allocator, sub_path: []const u8) ![]const u8 { } fn build_libc_object(mod: *Module, basename: []const u8, c_source_file: Module.CSourceFile) !void { + const tracy = trace(@src()); + defer tracy.end(); + // TODO: This is extracted into a local variable to work around a stage1 miscompilation. const emit_bin = Module.EmitLoc{ .directory = null, // Put it in the cache directory. @@ -618,8 +625,11 @@ fn build_libc_object(mod: *Module, basename: []const u8, c_source_file: Module.C try sub_module.update(); try mod.crt_files.ensureCapacity(mod.gpa, mod.crt_files.count() + 1); - const artifact_path = try std.fs.path.join(mod.gpa, &[_][]const u8{ - sub_module.zig_cache_artifact_directory.path.?, basename, - }); + const artifact_path = if (sub_module.bin_file.options.directory.path) |p| + try std.fs.path.join(mod.gpa, &[_][]const u8{ p, basename }) + else + try mod.gpa.dupe(u8, basename); + + // TODO obtain a lock on the artifact and put that in crt_files as well. mod.crt_files.putAssumeCapacityNoClobber(basename, artifact_path); } diff --git a/src-self-hosted/ir.zig b/src-self-hosted/ir.zig index 26afa52929..2b07110a90 100644 --- a/src-self-hosted/ir.zig +++ b/src-self-hosted/ir.zig @@ -1,7 +1,7 @@ const std = @import("std"); const Value = @import("value.zig").Value; const Type = @import("type.zig").Type; -const Module = @import("Module.zig"); +const Module = @import("ZigModule.zig"); const assert = std.debug.assert; const codegen = @import("codegen.zig"); const ast = std.zig.ast; diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 626ec6cca8..50f133f792 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1,6 +1,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; -const Module = @import("Module.zig"); +const Compilation = @import("Module.zig"); +const ZigModule = @import("ZigModule.zig"); const fs = std.fs; const trace = @import("tracy.zig").trace; const Package = @import("Package.zig"); @@ -12,7 +13,7 @@ pub const producer_string = if (std.builtin.is_test) "zig test" else "zig " ++ b pub const Options = struct { /// Where the output will go. - directory: Module.Directory, + directory: Compilation.Directory, /// Path to the output file, relative to `directory`. sub_path: []const u8, target: std.Target, @@ -21,7 +22,9 @@ pub const Options = struct { object_format: std.builtin.ObjectFormat, optimize_mode: std.builtin.Mode, root_name: []const u8, - root_pkg: ?*const Package, + /// Not every Compilation compiles .zig code! For example you could do `zig build-exe foo.o`. + /// TODO rename Module to Compilation and then (as a separate commit) ZigModule to Module. + zig_module: ?*ZigModule, dynamic_linker: ?[]const u8 = null, /// Used for calculating how much space to reserve for symbols in case the binary file /// does not already have a symbol table. @@ -71,7 +74,7 @@ pub const Options = struct { lib_dirs: []const []const u8 = &[0][]const u8{}, rpath_list: []const []const u8 = &[0][]const u8{}, - version: std.builtin.Version, + version: ?std.builtin.Version, libc_installation: ?*const LibCInstallation, pub fn effectiveOutputMode(options: Options) std.builtin.OutputMode { @@ -184,7 +187,7 @@ pub const File = struct { /// May be called before or after updateDeclExports but must be called /// after allocateDeclIndexes for any given Decl. - pub fn updateDecl(base: *File, module: *Module, decl: *Module.Decl) !void { + pub fn updateDecl(base: *File, module: *ZigModule, decl: *ZigModule.Decl) !void { switch (base.tag) { .coff => return @fieldParentPtr(Coff, "base", base).updateDecl(module, decl), .elf => return @fieldParentPtr(Elf, "base", base).updateDecl(module, decl), @@ -194,7 +197,7 @@ pub const File = struct { } } - pub fn updateDeclLineNumber(base: *File, module: *Module, decl: *Module.Decl) !void { + pub fn updateDeclLineNumber(base: *File, module: *ZigModule, decl: *ZigModule.Decl) !void { switch (base.tag) { .coff => return @fieldParentPtr(Coff, "base", base).updateDeclLineNumber(module, decl), .elf => return @fieldParentPtr(Elf, "base", base).updateDeclLineNumber(module, decl), @@ -205,7 +208,7 @@ pub const File = struct { /// Must be called before any call to updateDecl or updateDeclExports for /// any given Decl. - pub fn allocateDeclIndexes(base: *File, decl: *Module.Decl) !void { + pub fn allocateDeclIndexes(base: *File, decl: *ZigModule.Decl) !void { switch (base.tag) { .coff => return @fieldParentPtr(Coff, "base", base).allocateDeclIndexes(decl), .elf => return @fieldParentPtr(Elf, "base", base).allocateDeclIndexes(decl), @@ -256,20 +259,20 @@ pub const File = struct { } } - pub fn flush(base: *File, module: *Module) !void { + pub fn flush(base: *File, comp: *Compilation) !void { const tracy = trace(@src()); defer tracy.end(); try switch (base.tag) { - .coff => @fieldParentPtr(Coff, "base", base).flush(module), - .elf => @fieldParentPtr(Elf, "base", base).flush(module), - .macho => @fieldParentPtr(MachO, "base", base).flush(module), - .c => @fieldParentPtr(C, "base", base).flush(module), - .wasm => @fieldParentPtr(Wasm, "base", base).flush(module), + .coff => @fieldParentPtr(Coff, "base", base).flush(comp), + .elf => @fieldParentPtr(Elf, "base", base).flush(comp), + .macho => @fieldParentPtr(MachO, "base", base).flush(comp), + .c => @fieldParentPtr(C, "base", base).flush(comp), + .wasm => @fieldParentPtr(Wasm, "base", base).flush(comp), }; } - pub fn freeDecl(base: *File, decl: *Module.Decl) void { + pub fn freeDecl(base: *File, decl: *ZigModule.Decl) void { switch (base.tag) { .coff => @fieldParentPtr(Coff, "base", base).freeDecl(decl), .elf => @fieldParentPtr(Elf, "base", base).freeDecl(decl), @@ -293,9 +296,9 @@ pub const File = struct { /// allocateDeclIndexes for any given Decl. pub fn updateDeclExports( base: *File, - module: *Module, - decl: *const Module.Decl, - exports: []const *Module.Export, + module: *ZigModule, + decl: *const ZigModule.Decl, + exports: []const *ZigModule.Export, ) !void { switch (base.tag) { .coff => return @fieldParentPtr(Coff, "base", base).updateDeclExports(module, decl, exports), @@ -306,7 +309,7 @@ pub const File = struct { } } - pub fn getDeclVAddr(base: *File, decl: *const Module.Decl) u64 { + pub fn getDeclVAddr(base: *File, decl: *const ZigModule.Decl) u64 { switch (base.tag) { .coff => return @fieldParentPtr(Coff, "base", base).getDeclVAddr(decl), .elf => return @fieldParentPtr(Elf, "base", base).getDeclVAddr(decl), diff --git a/src-self-hosted/link/C.zig b/src-self-hosted/link/C.zig index 277fea7080..edfa055578 100644 --- a/src-self-hosted/link/C.zig +++ b/src-self-hosted/link/C.zig @@ -2,7 +2,8 @@ const std = @import("std"); const mem = std.mem; const assert = std.debug.assert; const Allocator = std.mem.Allocator; -const Module = @import("../Module.zig"); +const Module = @import("../ZigModule.zig"); +const Compilation = @import("../Module.zig"); const fs = std.fs; const codegen = @import("../codegen/c.zig"); const link = @import("../link.zig"); @@ -20,7 +21,7 @@ main: std.ArrayList(u8), called: std.StringHashMap(void), need_stddef: bool = false, need_stdint: bool = false, -error_msg: *Module.ErrorMsg = undefined, +error_msg: *Compilation.ErrorMsg = undefined, pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*File { assert(options.object_format == .c); @@ -51,7 +52,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio } pub fn fail(self: *C, src: usize, comptime format: []const u8, args: anytype) error{ AnalysisFail, OutOfMemory } { - self.error_msg = try Module.ErrorMsg.create(self.base.allocator, src, format, args); + self.error_msg = try Compilation.ErrorMsg.create(self.base.allocator, src, format, args); return error.AnalysisFail; } @@ -71,7 +72,7 @@ pub fn updateDecl(self: *C, module: *Module, decl: *Module.Decl) !void { }; } -pub fn flush(self: *C, module: *Module) !void { +pub fn flush(self: *C, comp: *Compilation) !void { const writer = self.base.file.?.writer(); try writer.writeAll(@embedFile("cbe.h")); var includes = false; diff --git a/src-self-hosted/link/Coff.zig b/src-self-hosted/link/Coff.zig index 6d7e87e898..7e3fc2fa72 100644 --- a/src-self-hosted/link/Coff.zig +++ b/src-self-hosted/link/Coff.zig @@ -7,7 +7,8 @@ const assert = std.debug.assert; const fs = std.fs; const trace = @import("../tracy.zig").trace; -const Module = @import("../Module.zig"); +const Module = @import("../ZigModule.zig"); +const Compilation = @import("../Module.zig"); const codegen = @import("../codegen.zig"); const link = @import("../link.zig"); @@ -732,7 +733,7 @@ pub fn updateDeclExports(self: *Coff, module: *Module, decl: *const Module.Decl, try module.failed_exports.ensureCapacity(module.gpa, module.failed_exports.items().len + 1); module.failed_exports.putAssumeCapacityNoClobber( exp, - try Module.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: ExportOptions.section", .{}), + try Compilation.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: ExportOptions.section", .{}), ); continue; } @@ -743,14 +744,14 @@ pub fn updateDeclExports(self: *Coff, module: *Module, decl: *const Module.Decl, try module.failed_exports.ensureCapacity(module.gpa, module.failed_exports.items().len + 1); module.failed_exports.putAssumeCapacityNoClobber( exp, - try Module.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: Exports other than '_start'", .{}), + try Compilation.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: Exports other than '_start'", .{}), ); continue; } } } -pub fn flush(self: *Coff, module: *Module) !void { +pub fn flush(self: *Coff, comp: *Compilation) !void { if (self.text_section_size_dirty) { // Write the new raw size in the .text header var buf: [4]u8 = undefined; diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index d46053493a..cd8391c925 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -3,7 +3,8 @@ const mem = std.mem; const assert = std.debug.assert; const Allocator = std.mem.Allocator; const ir = @import("../ir.zig"); -const Module = @import("../Module.zig"); +const Module = @import("../ZigModule.zig"); +const Compilation = @import("../Module.zig"); const fs = std.fs; const elf = std.elf; const codegen = @import("../codegen.zig"); @@ -122,6 +123,9 @@ dbg_info_decl_free_list: std.AutoHashMapUnmanaged(*TextBlock, void) = .{}, dbg_info_decl_first: ?*TextBlock = null, dbg_info_decl_last: ?*TextBlock = null, +/// Prevents other processes from clobbering the output file this is linking. +lock: ?std.cache_hash.Lock = null, + /// `alloc_num / alloc_den` is the factor of padding when allocating. const alloc_num = 4; const alloc_den = 3; @@ -285,7 +289,21 @@ fn createFile(allocator: *Allocator, file: fs.File, options: link.Options) !Elf return self; } +pub fn releaseLock(self: *Elf) void { + if (self.lock) |*lock| { + lock.release(); + self.lock = null; + } +} + +pub fn toOwnedLock(self: *Elf) std.cache_hash.Lock { + const lock = self.lock.?; + self.lock = null; + return lock; +} + pub fn deinit(self: *Elf) void { + self.releaseLock(); self.sections.deinit(self.base.allocator); self.program_headers.deinit(self.base.allocator); self.shstrtab.deinit(self.base.allocator); @@ -709,20 +727,24 @@ pub const abbrev_base_type = 4; pub const abbrev_pad1 = 5; pub const abbrev_parameter = 6; -pub fn flush(self: *Elf, module: *Module) !void { +pub fn flush(self: *Elf, comp: *Compilation) !void { if (build_options.have_llvm and self.base.options.use_lld) { - return self.linkWithLLD(module); + return self.linkWithLLD(comp); } else { switch (self.base.options.effectiveOutputMode()) { .Exe, .Obj => {}, .Lib => return error.TODOImplementWritingLibFiles, } - return self.flushInner(module); + return self.flushInner(comp); } } /// Commit pending changes and write headers. -fn flushInner(self: *Elf, module: *Module) !void { +fn flushInner(self: *Elf, comp: *Compilation) !void { + // TODO This linker code currently assumes there is only 1 compilation unit and it corresponds to the + // Zig source code. + const zig_module = self.base.options.zig_module orelse return error.LinkingWithoutZigSourceUnimplemented; + const target_endian = self.base.options.target.cpu.arch.endian(); const foreign_endian = target_endian != std.Target.current.cpu.arch.endian(); const ptr_width_bytes: u8 = self.ptrWidthBytes(); @@ -844,8 +866,8 @@ fn flushInner(self: *Elf, module: *Module) !void { }, } // Write the form for the compile unit, which must match the abbrev table above. - const name_strp = try self.makeDebugString(self.base.options.root_pkg.?.root_src_path); - const comp_dir_strp = try self.makeDebugString(self.base.options.root_pkg.?.root_src_directory.path.?); + const name_strp = try self.makeDebugString(zig_module.root_pkg.root_src_path); + const comp_dir_strp = try self.makeDebugString(zig_module.root_pkg.root_src_directory.path.?); const producer_strp = try self.makeDebugString(link.producer_string); // Currently only one compilation unit is supported, so the address range is simply // identical to the main program header virtual address and memory size. @@ -1014,7 +1036,7 @@ fn flushInner(self: *Elf, module: *Module) !void { 0, // include_directories (none except the compilation unit cwd) }); // file_names[0] - di_buf.appendSliceAssumeCapacity(self.base.options.root_pkg.?.root_src_path); // relative path name + di_buf.appendSliceAssumeCapacity(zig_module.root_pkg.root_src_path); // relative path name di_buf.appendSliceAssumeCapacity(&[_]u8{ 0, // null byte for the relative path name 0, // directory_index @@ -1199,11 +1221,105 @@ fn flushInner(self: *Elf, module: *Module) !void { assert(!self.debug_strtab_dirty); } -fn linkWithLLD(self: *Elf, module: *Module) !void { +fn linkWithLLD(self: *Elf, comp: *Compilation) !void { var arena_allocator = std.heap.ArenaAllocator.init(self.base.allocator); defer arena_allocator.deinit(); const arena = &arena_allocator.allocator; + const directory = self.base.options.directory; // Just an alias to make it shorter to type. + + // If there is no Zig code to compile, then we should skip flushing the output file because it + // will not be part of the linker line anyway. + const zig_module_obj_path: ?[]const u8 = if (self.base.options.zig_module) |module| blk: { + try self.flushInner(comp); + + const obj_basename = self.base.intermediary_basename.?; + const full_obj_path = if (directory.path) |dir_path| + try std.fs.path.join(arena, &[_][]const u8{dir_path, obj_basename}) + else + obj_basename; + break :blk full_obj_path; + } else null; + + // Here we want to determine whether we can save time by not invoking LLD when the + // output is unchanged. None of the linker options or the object files that are being + // linked are in the hash that namespaces the directory we are outputting to. Therefore, + // we must hash those now, and the resulting digest will form the "id" of the linking + // job we are about to perform. + // After a successful link, we store the id in the metadata of a symlink named "id.txt" in + // the artifact directory. So, now, we check if this symlink exists, and if it matches + // our digest. If so, we can skip linking. Otherwise, we proceed with invoking LLD. + const id_symlink_basename = "id.txt"; + + // We are about to obtain this lock, so here we give other processes a chance first. + self.releaseLock(); + + var ch = comp.cache_parent.obtain(); + defer ch.deinit(); + + const is_lib = self.base.options.output_mode == .Lib; + const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib; + const have_dynamic_linker = self.base.options.link_libc and + self.base.options.link_mode == .Dynamic and (is_dyn_lib or self.base.options.output_mode == .Exe); + + try ch.addOptionalFile(self.base.options.linker_script); + try ch.addOptionalFile(self.base.options.version_script); + try ch.addListOfFiles(self.base.options.objects); + for (comp.c_object_table.items()) |entry| switch (entry.key.status) { + .new => unreachable, + .failure => return error.NotAllCSourceFilesAvailableToLink, + .success => |success| _ = try ch.addFile(success.object_path, null), + }; + try ch.addOptionalFile(zig_module_obj_path); + // We can skip hashing libc and libc++ components that we are in charge of building from Zig + // installation sources because they are always a product of the compiler version + target information. + ch.hash.addOptional(self.base.options.stack_size_override); + ch.hash.addOptional(self.base.options.gc_sections); + ch.hash.add(self.base.options.eh_frame_hdr); + ch.hash.add(self.base.options.rdynamic); + ch.hash.addListOfBytes(self.base.options.extra_lld_args); + ch.hash.addListOfBytes(self.base.options.lib_dirs); + ch.hash.add(self.base.options.z_nodelete); + ch.hash.add(self.base.options.z_defs); + if (self.base.options.link_libc) { + ch.hash.add(self.base.options.libc_installation != null); + if (self.base.options.libc_installation) |libc_installation| { + ch.hash.addBytes(libc_installation.crt_dir.?); + } + if (have_dynamic_linker) { + ch.hash.addOptionalBytes(self.base.options.dynamic_linker); + } + } + if (is_dyn_lib) { + ch.hash.addOptionalBytes(self.base.options.override_soname); + ch.hash.addOptional(self.base.options.version); + } + ch.hash.addListOfBytes(self.base.options.system_libs); + ch.hash.addOptional(self.base.options.allow_shlib_undefined); + ch.hash.add(self.base.options.bind_global_refs_locally); + + // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock. + _ = try ch.hit(); + const digest = ch.final(); + + var prev_digest_buf: [digest.len]u8 = undefined; + const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch blk: { + // Handle this as a cache miss. + mem.set(u8, &prev_digest_buf, 0); + break :blk &prev_digest_buf; + }; + if (mem.eql(u8, prev_digest, &digest)) { + // Hot diggity dog! The output binary is already there. + self.lock = ch.toOwnedLock(); + return; + } + + // We are about to change the output file to be different, so we invalidate the build hash now. + directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) { + error.FileNotFound => {}, + else => |e| return e, + }; + const target = self.base.options.target; const is_obj = self.base.options.output_mode == .Obj; @@ -1272,8 +1388,6 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { try argv.append(arg); } - const is_lib = self.base.options.output_mode == .Lib; - const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib; if (self.base.options.link_mode == .Static) { if (target.cpu.arch.isARM() or target.cpu.arch.isThumb()) { try argv.append("-Bstatic"); @@ -1288,7 +1402,7 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { try argv.append("-pie"); } - const full_out_path = if (self.base.options.directory.path) |dir_path| + const full_out_path = if (directory.path) |dir_path| try std.fs.path.join(arena, &[_][]const u8{dir_path, self.base.options.sub_path}) else self.base.options.sub_path; @@ -1311,13 +1425,14 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { break :o "Scrt1.o"; } }; - try argv.append(try module.get_libc_crt_file(arena, crt1o)); + try argv.append(try comp.get_libc_crt_file(arena, crt1o)); if (target_util.libc_needs_crti_crtn(target)) { - try argv.append(try module.get_libc_crt_file(arena, "crti.o")); + try argv.append(try comp.get_libc_crt_file(arena, "crti.o")); } } // TODO rpaths + // TODO add to cache hash above too //for (size_t i = 0; i < g->rpath_list.length; i += 1) { // Buf *rpath = g->rpath_list.at(i); // add_rpath(lj, rpath); @@ -1354,7 +1469,7 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { try argv.append(libc_installation.crt_dir.?); } - if (self.base.options.link_mode == .Dynamic and (is_dyn_lib or self.base.options.output_mode == .Exe)) { + if (have_dynamic_linker) { if (self.base.options.dynamic_linker) |dynamic_linker| { try argv.append("-dynamic-linker"); try argv.append(dynamic_linker); @@ -1363,9 +1478,10 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { } if (is_dyn_lib) { - const soname = self.base.options.override_soname orelse - try std.fmt.allocPrint(arena, "lib{}.so.{}", .{self.base.options.root_name, - self.base.options.version.major,}); + const soname = self.base.options.override_soname orelse if (self.base.options.version) |ver| + try std.fmt.allocPrint(arena, "lib{}.so.{}", .{self.base.options.root_name, ver.major}) + else + try std.fmt.allocPrint(arena, "lib{}.so", .{self.base.options.root_name}); try argv.append("-soname"); try argv.append(soname); @@ -1378,28 +1494,14 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { // Positional arguments to the linker such as object files. try argv.appendSlice(self.base.options.objects); - for (module.c_object_table.items()) |entry| { - const c_object = entry.key; - switch (c_object.status) { - .new => unreachable, - .failure => return error.NotAllCSourceFilesAvailableToLink, - .success => |full_obj_path| { - try argv.append(full_obj_path); - }, - } - } + for (comp.c_object_table.items()) |entry| switch (entry.key.status) { + .new => unreachable, + .failure => unreachable, // Checked during cache hashing. + .success => |success| try argv.append(success.object_path), + }; - // If there is no Zig code to compile, then we should skip flushing the output file because it - // will not be part of the linker line anyway. - if (module.root_pkg != null) { - try self.flushInner(module); - - const obj_basename = self.base.intermediary_basename.?; - const full_obj_path = if (self.base.options.directory.path) |dir_path| - try std.fs.path.join(arena, &[_][]const u8{dir_path, obj_basename}) - else - obj_basename; - try argv.append(full_obj_path); + if (zig_module_obj_path) |p| { + try argv.append(p); } // TODO compiler-rt and libc @@ -1419,7 +1521,7 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { // By this time, we depend on these libs being dynamically linked libraries and not static libraries // (the check for that needs to be earlier), but they could be full paths to .so files, in which // case we want to avoid prepending "-l". - const ext = Module.classifyFileExt(link_lib); + const ext = Compilation.classifyFileExt(link_lib); const arg = if (ext == .so) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib}); argv.appendAssumeCapacity(arg); } @@ -1427,8 +1529,8 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { if (!is_obj) { // libc++ dep if (self.base.options.link_libcpp) { - try argv.append(module.libcxxabi_static_lib.?); - try argv.append(module.libcxx_static_lib.?); + try argv.append(comp.libcxxabi_static_lib.?); + try argv.append(comp.libcxx_static_lib.?); } // libc dep @@ -1448,15 +1550,15 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { try argv.append("-lpthread"); } } else if (target.isGnuLibC()) { - try argv.append(module.libunwind_static_lib.?); + try argv.append(comp.libunwind_static_lib.?); // TODO here we need to iterate over the glibc libs and add the .so files to the linker line. std.log.warn("TODO port add_glibc_libs to stage2", .{}); - try argv.append(try module.get_libc_crt_file(arena, "libc_nonshared.a")); + try argv.append(try comp.get_libc_crt_file(arena, "libc_nonshared.a")); } else if (target.isMusl()) { - try argv.append(module.libunwind_static_lib.?); - try argv.append(module.libc_static_lib.?); + try argv.append(comp.libunwind_static_lib.?); + try argv.append(comp.libc_static_lib.?); } else if (self.base.options.link_libcpp) { - try argv.append(module.libunwind_static_lib.?); + try argv.append(comp.libunwind_static_lib.?); } else { unreachable; // Compiler was supposed to emit an error for not being able to provide libc. } @@ -1466,9 +1568,9 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { // crt end if (link_in_crt) { if (target.isAndroid()) { - try argv.append(try module.get_libc_crt_file(arena, "crtend_android.o")); + try argv.append(try comp.get_libc_crt_file(arena, "crtend_android.o")); } else if (target_util.libc_needs_crti_crtn(target)) { - try argv.append(try module.get_libc_crt_file(arena, "crtn.o")); + try argv.append(try comp.get_libc_crt_file(arena, "crtn.o")); } } @@ -1500,6 +1602,19 @@ fn linkWithLLD(self: *Elf, module: *Module) !void { const ZigLLDLink = @import("../llvm.zig").ZigLLDLink; const ok = ZigLLDLink(.ELF, new_argv.ptr, new_argv.len, append_diagnostic, 0, 0); if (!ok) return error.LLDReportedFailure; + + // Update the dangling symlink "id.txt" with the digest. If it fails we can continue; it only + // means that the next invocation will have an unnecessary cache miss. + directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { + std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); + }; + // Again failure here only means an unnecessary cache miss. + ch.writeManifest() catch |err| { + std.log.warn("failed to write cache manifest when linking: {}", .{ @errorName(err) }); + }; + // We hang on to this lock so that the output file path can be used without + // other processes clobbering it. + self.lock = ch.toOwnedLock(); } fn append_diagnostic(context: usize, ptr: [*]const u8, len: usize) callconv(.C) void { @@ -2396,7 +2511,7 @@ pub fn updateDeclExports( try module.failed_exports.ensureCapacity(module.gpa, module.failed_exports.items().len + 1); module.failed_exports.putAssumeCapacityNoClobber( exp, - try Module.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: ExportOptions.section", .{}), + try Compilation.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: ExportOptions.section", .{}), ); continue; } @@ -2414,7 +2529,7 @@ pub fn updateDeclExports( try module.failed_exports.ensureCapacity(module.gpa, module.failed_exports.items().len + 1); module.failed_exports.putAssumeCapacityNoClobber( exp, - try Module.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: GlobalLinkage.LinkOnce", .{}), + try Compilation.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: GlobalLinkage.LinkOnce", .{}), ); continue; }, @@ -2722,8 +2837,8 @@ fn dbgLineNeededHeaderBytes(self: Elf) u32 { directory_count * 8 + file_name_count * 8 + // These are encoded as DW.FORM_string rather than DW.FORM_strp as we would like // because of a workaround for readelf and gdb failing to understand DWARFv5 correctly. - self.base.options.root_pkg.?.root_src_directory.path.?.len + - self.base.options.root_pkg.?.root_src_path.len); + self.base.options.zig_module.?.root_pkg.root_src_directory.path.?.len + + self.base.options.zig_module.?.root_pkg.root_src_path.len); } fn dbgInfoNeededHeaderBytes(self: Elf) u32 { diff --git a/src-self-hosted/link/MachO.zig b/src-self-hosted/link/MachO.zig index b57f3d303a..c412c84e9e 100644 --- a/src-self-hosted/link/MachO.zig +++ b/src-self-hosted/link/MachO.zig @@ -12,7 +12,8 @@ const mem = std.mem; const trace = @import("../tracy.zig").trace; const Type = @import("../type.zig").Type; -const Module = @import("../Module.zig"); +const Module = @import("../ZigModule.zig"); +const Compilation = @import("../Module.zig"); const link = @import("../link.zig"); const File = link.File; @@ -205,7 +206,7 @@ fn createFile(allocator: *Allocator, file: fs.File, options: link.Options) !Mach return self; } -pub fn flush(self: *MachO, module: *Module) !void { +pub fn flush(self: *MachO, comp: *Compilation) !void { switch (self.base.options.output_mode) { .Exe => { var last_cmd_offset: usize = @sizeOf(macho.mach_header_64); diff --git a/src-self-hosted/link/Wasm.zig b/src-self-hosted/link/Wasm.zig index bea36723ff..f3586489d6 100644 --- a/src-self-hosted/link/Wasm.zig +++ b/src-self-hosted/link/Wasm.zig @@ -6,7 +6,8 @@ const assert = std.debug.assert; const fs = std.fs; const leb = std.debug.leb; -const Module = @import("../Module.zig"); +const Module = @import("../ZigModule.zig"); +const Compilation = @import("../Module.zig"); const codegen = @import("../codegen/wasm.zig"); const link = @import("../link.zig"); @@ -126,7 +127,7 @@ pub fn freeDecl(self: *Wasm, decl: *Module.Decl) void { decl.fn_link.wasm = null; } -pub fn flush(self: *Wasm, module: *Module) !void { +pub fn flush(self: *Wasm, comp: *Compilation) !void { const file = self.base.file.?; const header_size = 5 + 1; @@ -164,7 +165,7 @@ pub fn flush(self: *Wasm, module: *Module) !void { } // Export section - { + if (self.base.options.zig_module) |module| { const header_offset = try reserveVecSectionHeader(file); const writer = file.writer(); var count: u32 = 0; diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 154e09bc68..6453cdaeb1 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -268,6 +268,7 @@ pub fn buildOutputType( var link_mode: ?std.builtin.LinkMode = null; var root_src_file: ?[]const u8 = null; var version: std.builtin.Version = .{ .major = 0, .minor = 0, .patch = 0 }; + var have_version = false; var strip = false; var single_threaded = false; var watch = false; @@ -445,6 +446,7 @@ pub fn buildOutputType( version = std.builtin.Version.parse(args[i]) catch |err| { fatal("unable to parse --version '{}': {}", .{ args[i], @errorName(err) }); }; + have_version = true; } else if (mem.eql(u8, arg, "-target")) { if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; @@ -799,6 +801,7 @@ pub fn buildOutputType( version.major = std.fmt.parseInt(u32, linker_args.items[i], 10) catch |err| { fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); }; + have_version = true; } else if (mem.eql(u8, arg, "--minor-image-version")) { i += 1; if (i >= linker_args.items.len) { @@ -807,6 +810,7 @@ pub fn buildOutputType( version.minor = std.fmt.parseInt(u32, linker_args.items[i], 10) catch |err| { fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); }; + have_version = true; } else if (mem.eql(u8, arg, "--stack")) { i += 1; if (i >= linker_args.items.len) { @@ -1161,7 +1165,7 @@ pub fn buildOutputType( .self_exe_path = self_exe_path, .rand = &default_prng.random, .clang_passthrough_mode = arg_mode != .build, - .version = version, + .version = if (have_version) version else null, .libc_installation = if (libc_installation) |*lci| lci else null, .debug_cc = debug_cc, .debug_link = debug_link, @@ -1228,7 +1232,9 @@ fn updateModule(gpa: *Allocator, module: *Module, zir_out_path: ?[]const u8) !vo } if (zir_out_path) |zop| { - var new_zir_module = try zir.emit(gpa, module); + const zig_module = module.bin_file.options.zig_module orelse + fatal("-femit-zir with no zig source code", .{}); + var new_zir_module = try zir.emit(gpa, zig_module); defer new_zir_module.deinit(gpa); const baf = try io.BufferedAtomicFile.create(gpa, fs.cwd(), zop, .{}); diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 3fd6aa3bab..40eb833273 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -549,7 +549,7 @@ pub const TestContext = struct { update_node.estimated_total_items = 5; var emit_node = update_node.start("emit", null); emit_node.activate(); - var new_zir_module = try zir.emit(allocator, module); + var new_zir_module = try zir.emit(allocator, module.bin_file.options.zig_module.?); defer new_zir_module.deinit(allocator); emit_node.end(); diff --git a/src-self-hosted/type.zig b/src-self-hosted/type.zig index 4966395512..0c3d077477 100644 --- a/src-self-hosted/type.zig +++ b/src-self-hosted/type.zig @@ -3,7 +3,7 @@ const Value = @import("value.zig").Value; const assert = std.debug.assert; const Allocator = std.mem.Allocator; const Target = std.Target; -const Module = @import("Module.zig"); +const Module = @import("ZigModule.zig"); /// This is the raw data, with no bookkeeping, no memory awareness, no de-duplication. /// It's important for this type to be small. diff --git a/src-self-hosted/value.zig b/src-self-hosted/value.zig index b65aa06bea..2344b813b3 100644 --- a/src-self-hosted/value.zig +++ b/src-self-hosted/value.zig @@ -6,7 +6,7 @@ const BigIntConst = std.math.big.int.Const; const BigIntMutable = std.math.big.int.Mutable; const Target = std.Target; const Allocator = std.mem.Allocator; -const Module = @import("Module.zig"); +const Module = @import("ZigModule.zig"); /// This is the raw data, with no bookkeeping, no memory awareness, /// no de-duplication, and no type system awareness. diff --git a/src-self-hosted/zir.zig b/src-self-hosted/zir.zig index 7e723fc674..f543f8b2ce 100644 --- a/src-self-hosted/zir.zig +++ b/src-self-hosted/zir.zig @@ -10,7 +10,7 @@ const Type = @import("type.zig").Type; const Value = @import("value.zig").Value; const TypedValue = @import("TypedValue.zig"); const ir = @import("ir.zig"); -const IrModule = @import("Module.zig"); +const IrModule = @import("ZigModule.zig"); /// This struct is relevent only for the ZIR Module text format. It is not used for /// semantic analysis of Zig source code. diff --git a/src-self-hosted/zir_sema.zig b/src-self-hosted/zir_sema.zig index c99da39c04..6a93bc2a5e 100644 --- a/src-self-hosted/zir_sema.zig +++ b/src-self-hosted/zir_sema.zig @@ -16,7 +16,7 @@ const TypedValue = @import("TypedValue.zig"); const assert = std.debug.assert; const ir = @import("ir.zig"); const zir = @import("zir.zig"); -const Module = @import("Module.zig"); +const Module = @import("ZigModule.zig"); const Inst = ir.Inst; const Body = ir.Body; const trace = @import("tracy.zig").trace; @@ -199,10 +199,10 @@ pub fn analyzeZirDecl(mod: *Module, decl: *Decl, src_decl: *zir.Decl) InnerError // We don't fully codegen the decl until later, but we do need to reserve a global // offset table index for it. This allows us to codegen decls out of dependency order, // increasing how many computations can be done in parallel. - try mod.bin_file.allocateDeclIndexes(decl); - try mod.work_queue.writeItem(.{ .codegen_decl = decl }); + try mod.comp.bin_file.allocateDeclIndexes(decl); + try mod.comp.work_queue.writeItem(.{ .codegen_decl = decl }); } else if (prev_type_has_bits) { - mod.bin_file.freeDecl(decl); + mod.comp.bin_file.freeDecl(decl); } return type_changed; diff --git a/test/stage2/test.zig b/test/stage2/test.zig index ad81e463b9..b37eb63a2e 100644 --- a/test/stage2/test.zig +++ b/test/stage2/test.zig @@ -1,8 +1,11 @@ const std = @import("std"); const TestContext = @import("../../src-self-hosted/test.zig").TestContext; -// self-hosted does not yet support PE executable files / COFF object files -// or mach-o files. So we do these test cases cross compiling for x86_64-linux. +// Self-hosted has differing levels of support for various architectures. For now we pass explicit +// target parameters to each test case. At some point we will take this to the next level and have +// a set of targets that all test cases run on unless specifically overridden. For now, each test +// case applies to only the specified target. + const linux_x64 = std.zig.CrossTarget{ .cpu_arch = .x86_64, .os_tag = .linux, From 4d59f775289b50a9529e801a6998dcd181945efb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2020 19:49:52 -0700 Subject: [PATCH 021/210] stage2: rename Module to Compilation --- .../{Module.zig => Compilation.zig} | 124 +++++++-------- src-self-hosted/Package.zig | 4 +- src-self-hosted/ZigModule.zig | 6 +- src-self-hosted/codegen.zig | 2 +- src-self-hosted/glibc.zig | 150 +++++++++--------- src-self-hosted/introspect.zig | 14 +- src-self-hosted/link.zig | 3 +- src-self-hosted/link/C.zig | 2 +- src-self-hosted/link/Coff.zig | 2 +- src-self-hosted/link/Elf.zig | 2 +- src-self-hosted/link/MachO.zig | 2 +- src-self-hosted/link/Wasm.zig | 2 +- src-self-hosted/main.zig | 50 +++--- src-self-hosted/test.zig | 32 ++-- 14 files changed, 195 insertions(+), 200 deletions(-) rename src-self-hosted/{Module.zig => Compilation.zig} (93%) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Compilation.zig similarity index 93% rename from src-self-hosted/Module.zig rename to src-self-hosted/Compilation.zig index 557b5dcfcd..ab105deb07 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Compilation.zig @@ -1,5 +1,3 @@ -//! TODO This is going to get renamed from Module to Compilation. -const Module = @This(); const Compilation = @This(); const std = @import("std"); @@ -31,7 +29,7 @@ link_error_flags: link.File.ErrorFlags = .{}, work_queue: std.fifo.LinearFifo(WorkItem, .Dynamic), -/// The ErrorMsg memory is owned by the `CObject`, using Module's general purpose allocator. +/// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator. failed_c_objects: std.AutoArrayHashMapUnmanaged(*CObject, *ErrorMsg) = .{}, keep_source_files_loaded: bool, @@ -39,7 +37,7 @@ use_clang: bool, sanitize_c: bool, /// When this is `true` it means invoking clang as a sub-process is expected to inherit /// stdin, stdout, stderr, and if it returns non success, to forward the exit code. -/// Otherwise we attempt to parse the error messages and expose them via the Module API. +/// Otherwise we attempt to parse the error messages and expose them via the Compilation API. /// This is `true` for `zig cc`, `zig c++`, and `zig translate-c`. clang_passthrough_mode: bool, /// Whether to print clang argvs to stdout. @@ -96,7 +94,7 @@ const WorkItem = union(enum) { /// Decl may need its line number information updated in the debug info. update_line_number: *ZigModule.Decl, /// Invoke the Clang compiler to create an object file, which gets linked - /// with the Module. + /// with the Compilation. c_object: *CObject, /// one of the glibc static objects @@ -192,8 +190,8 @@ pub const Directory = struct { pub const EmitLoc = struct { /// If this is `null` it means the file will be output to the cache directory. - /// When provided, both the open file handle and the path name must outlive the `Module`. - directory: ?Module.Directory, + /// When provided, both the open file handle and the path name must outlive the `Compilation`. + directory: ?Compilation.Directory, /// This may not have sub-directories in it. basename: []const u8, }; @@ -257,17 +255,17 @@ pub const InitOptions = struct { libc_installation: ?*const LibCInstallation = null, }; -pub fn create(gpa: *Allocator, options: InitOptions) !*Module { - const comp: *Module = comp: { - // For allocations that have the same lifetime as Module. This arena is used only during this +pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { + const comp: *Compilation = comp: { + // For allocations that have the same lifetime as Compilation. This arena is used only during this // initialization and then is freed in deinit(). var arena_allocator = std.heap.ArenaAllocator.init(gpa); errdefer arena_allocator.deinit(); const arena = &arena_allocator.allocator; - // We put the `Module` itself in the arena. Freeing the arena will free the module. + // We put the `Compilation` itself in the arena. Freeing the arena will free the module. // It's initialized later after we prepare the initialization options. - const comp = try arena.create(Module); + const comp = try arena.create(Compilation); const root_name = try arena.dupe(u8, options.root_name); const ofmt = options.object_format orelse options.target.getObjectFormat(); @@ -606,7 +604,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Module { return comp; } -pub fn destroy(self: *Module) void { +pub fn destroy(self: *Compilation) void { const optional_zig_module = self.bin_file.options.zig_module; self.bin_file.destroy(); if (optional_zig_module) |zig_module| zig_module.deinit(); @@ -640,12 +638,12 @@ pub fn destroy(self: *Module) void { self.arena_state.promote(gpa).deinit(); } -pub fn getTarget(self: Module) Target { +pub fn getTarget(self: Compilation) Target { return self.bin_file.options.target; } /// Detect changes to source files, perform semantic analysis, and update the output files. -pub fn update(self: *Module) !void { +pub fn update(self: *Compilation) !void { const tracy = trace(@src()); defer tracy.end(); @@ -713,15 +711,15 @@ pub fn update(self: *Module) !void { /// binary is concerned. This will remove the write flag, or close the file, /// or whatever is needed so that it can be executed. /// After this, one must call` makeFileWritable` before calling `update`. -pub fn makeBinFileExecutable(self: *Module) !void { +pub fn makeBinFileExecutable(self: *Compilation) !void { return self.bin_file.makeExecutable(); } -pub fn makeBinFileWritable(self: *Module) !void { +pub fn makeBinFileWritable(self: *Compilation) !void { return self.bin_file.makeWritable(); } -pub fn totalErrorCount(self: *Module) usize { +pub fn totalErrorCount(self: *Compilation) usize { var total: usize = self.failed_c_objects.items().len; if (self.bin_file.options.zig_module) |zig_module| { @@ -738,7 +736,7 @@ pub fn totalErrorCount(self: *Module) usize { return total; } -pub fn getAllErrorsAlloc(self: *Module) !AllErrors { +pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors { var arena = std.heap.ArenaAllocator.init(self.gpa); errdefer arena.deinit(); @@ -795,7 +793,7 @@ pub fn getAllErrorsAlloc(self: *Module) !AllErrors { }; } -pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void { +pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { while (self.work_queue.readItem()) |work_item| switch (work_item) { .codegen_decl => |decl| switch (decl.analysis) { .unreferenced => unreachable, @@ -1071,25 +1069,25 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { }; } -fn tmpFilePath(mod: *Module, arena: *Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 { +fn tmpFilePath(comp: *Compilation, arena: *Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 { const s = std.fs.path.sep_str; return std.fmt.allocPrint( arena, "{}" ++ s ++ "tmp" ++ s ++ "{x}-{}", - .{ mod.zig_cache_directory.path.?, mod.rand.int(u64), suffix }, + .{ comp.zig_cache_directory.path.?, comp.rand.int(u64), suffix }, ); } /// Add common C compiler args between translate-c and C object compilation. fn addCCArgs( - mod: *Module, + comp: *Compilation, arena: *Allocator, argv: *std.ArrayList([]const u8), ext: FileExt, translate_c: bool, out_dep_path: ?[]const u8, ) !void { - const target = mod.getTarget(); + const target = comp.getTarget(); if (translate_c) { try argv.appendSlice(&[_][]const u8{ "-x", "c" }); @@ -1106,27 +1104,27 @@ fn addCCArgs( // We don't ever put `-fcolor-diagnostics` or `-fno-color-diagnostics` because in passthrough mode // we want Clang to infer it, and in normal mode we always want it off, which will be true since // clang will detect stderr as a pipe rather than a terminal. - if (!mod.clang_passthrough_mode) { + if (!comp.clang_passthrough_mode) { // Make stderr more easily parseable. try argv.append("-fno-caret-diagnostics"); } - if (mod.bin_file.options.function_sections) { + if (comp.bin_file.options.function_sections) { try argv.append("-ffunction-sections"); } - try argv.ensureCapacity(argv.items.len + mod.bin_file.options.framework_dirs.len * 2); - for (mod.bin_file.options.framework_dirs) |framework_dir| { + try argv.ensureCapacity(argv.items.len + comp.bin_file.options.framework_dirs.len * 2); + for (comp.bin_file.options.framework_dirs) |framework_dir| { argv.appendAssumeCapacity("-iframework"); argv.appendAssumeCapacity(framework_dir); } - if (mod.bin_file.options.link_libcpp) { + if (comp.bin_file.options.link_libcpp) { const libcxx_include_path = try std.fs.path.join(arena, &[_][]const u8{ - mod.zig_lib_directory.path.?, "libcxx", "include", + comp.zig_lib_directory.path.?, "libcxx", "include", }); const libcxxabi_include_path = try std.fs.path.join(arena, &[_][]const u8{ - mod.zig_lib_directory.path.?, "libcxxabi", "include", + comp.zig_lib_directory.path.?, "libcxxabi", "include", }); try argv.append("-isystem"); @@ -1150,11 +1148,11 @@ fn addCCArgs( // According to Rich Felker libc headers are supposed to go before C language headers. // However as noted by @dimenus, appending libc headers before c_headers breaks intrinsics // and other compiler specific items. - const c_headers_dir = try std.fs.path.join(arena, &[_][]const u8{ mod.zig_lib_directory.path.?, "include" }); + const c_headers_dir = try std.fs.path.join(arena, &[_][]const u8{ comp.zig_lib_directory.path.?, "include" }); try argv.append("-isystem"); try argv.append(c_headers_dir); - for (mod.libc_include_dir_list) |include_dir| { + for (comp.libc_include_dir_list) |include_dir| { try argv.append("-isystem"); try argv.append(include_dir); } @@ -1206,28 +1204,28 @@ fn addCCArgs( try argv.append("-Wno-pragma-pack"); } - if (!mod.bin_file.options.strip) { + if (!comp.bin_file.options.strip) { try argv.append("-g"); } - if (mod.haveFramePointer()) { + if (comp.haveFramePointer()) { try argv.append("-fno-omit-frame-pointer"); } else { try argv.append("-fomit-frame-pointer"); } - if (mod.sanitize_c) { + if (comp.sanitize_c) { try argv.append("-fsanitize=undefined"); try argv.append("-fsanitize-trap=undefined"); } - switch (mod.bin_file.options.optimize_mode) { + switch (comp.bin_file.options.optimize_mode) { .Debug => { // windows c runtime requires -D_DEBUG if using debug libraries try argv.append("-D_DEBUG"); try argv.append("-Og"); - if (mod.bin_file.options.link_libc) { + if (comp.bin_file.options.link_libc) { try argv.append("-fstack-protector-strong"); try argv.append("--param"); try argv.append("ssp-buffer-size=4"); @@ -1239,7 +1237,7 @@ fn addCCArgs( // See the comment in the BuildModeFastRelease case for why we pass -O2 rather // than -O3 here. try argv.append("-O2"); - if (mod.bin_file.options.link_libc) { + if (comp.bin_file.options.link_libc) { try argv.append("-D_FORTIFY_SOURCE=2"); try argv.append("-fstack-protector-strong"); try argv.append("--param"); @@ -1265,25 +1263,25 @@ fn addCCArgs( }, } - if (target_util.supports_fpic(target) and mod.bin_file.options.pic) { + if (target_util.supports_fpic(target) and comp.bin_file.options.pic) { try argv.append("-fPIC"); } - try argv.appendSlice(mod.clang_argv); + try argv.appendSlice(comp.clang_argv); } -fn failCObj(mod: *Module, c_object: *CObject, comptime format: []const u8, args: anytype) InnerError { +fn failCObj(comp: *Compilation, c_object: *CObject, comptime format: []const u8, args: anytype) InnerError { @setCold(true); - const err_msg = try ErrorMsg.create(mod.gpa, 0, "unable to build C object: " ++ format, args); - return mod.failCObjWithOwnedErrorMsg(c_object, err_msg); + const err_msg = try ErrorMsg.create(comp.gpa, 0, "unable to build C object: " ++ format, args); + return comp.failCObjWithOwnedErrorMsg(c_object, err_msg); } -fn failCObjWithOwnedErrorMsg(mod: *Module, c_object: *CObject, err_msg: *ErrorMsg) InnerError { +fn failCObjWithOwnedErrorMsg(comp: *Compilation, c_object: *CObject, err_msg: *ErrorMsg) InnerError { { - errdefer err_msg.destroy(mod.gpa); - try mod.failed_c_objects.ensureCapacity(mod.gpa, mod.failed_c_objects.items().len + 1); + errdefer err_msg.destroy(comp.gpa); + try comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1); } - mod.failed_c_objects.putAssumeCapacityNoClobber(c_object, err_msg); + comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, err_msg); c_object.status = .failure; return error.AnalysisFail; } @@ -1389,9 +1387,9 @@ test "classifyFileExt" { std.testing.expectEqual(FileExt.unknown, classifyFileExt("foo.so.1.2.3~")); } -fn haveFramePointer(mod: *Module) bool { - return switch (mod.bin_file.options.optimize_mode) { - .Debug, .ReleaseSafe => !mod.bin_file.options.strip, +fn haveFramePointer(comp: *Compilation) bool { + return switch (comp.bin_file.options.optimize_mode) { + .Debug, .ReleaseSafe => !comp.bin_file.options.strip, .ReleaseSmall, .ReleaseFast => false, }; } @@ -1496,17 +1494,17 @@ fn detectLibCFromLibCInstallation(arena: *Allocator, target: Target, lci: *const }; } -pub fn get_libc_crt_file(mod: *Module, arena: *Allocator, basename: []const u8) ![]const u8 { - if (mod.wantBuildGLibCFromSource()) { - return mod.crt_files.get(basename).?; +pub fn get_libc_crt_file(comp: *Compilation, arena: *Allocator, basename: []const u8) ![]const u8 { + if (comp.wantBuildGLibCFromSource()) { + return comp.crt_files.get(basename).?; } - const lci = mod.bin_file.options.libc_installation orelse return error.LibCInstallationNotAvailable; + const lci = comp.bin_file.options.libc_installation orelse return error.LibCInstallationNotAvailable; const crt_dir_path = lci.crt_dir orelse return error.LibCInstallationMissingCRTDir; const full_path = try std.fs.path.join(arena, &[_][]const u8{ crt_dir_path, basename }); return full_path; } -fn addBuildingGLibCWorkItems(mod: *Module) !void { +fn addBuildingGLibCWorkItems(comp: *Compilation) !void { const static_file_work_items = [_]WorkItem{ .{ .glibc_crt_file = .crti_o }, .{ .glibc_crt_file = .crtn_o }, @@ -1515,15 +1513,15 @@ fn addBuildingGLibCWorkItems(mod: *Module) !void { .{ .glibc_crt_file = .scrt1_o }, .{ .glibc_crt_file = .libc_nonshared_a }, }; - try mod.work_queue.ensureUnusedCapacity(static_file_work_items.len + glibc.libs.len); - mod.work_queue.writeAssumeCapacity(&static_file_work_items); + try comp.work_queue.ensureUnusedCapacity(static_file_work_items.len + glibc.libs.len); + comp.work_queue.writeAssumeCapacity(&static_file_work_items); for (glibc.libs) |*glibc_so| { - mod.work_queue.writeItemAssumeCapacity(.{ .glibc_so = glibc_so }); + comp.work_queue.writeItemAssumeCapacity(.{ .glibc_so = glibc_so }); } } -fn wantBuildGLibCFromSource(mod: *Module) bool { - return mod.bin_file.options.link_libc and - mod.bin_file.options.libc_installation == null and - mod.bin_file.options.target.isGnuLibC(); +fn wantBuildGLibCFromSource(comp: *Compilation) bool { + return comp.bin_file.options.link_libc and + comp.bin_file.options.libc_installation == null and + comp.bin_file.options.target.isGnuLibC(); } diff --git a/src-self-hosted/Package.zig b/src-self-hosted/Package.zig index 11b1436293..027aaafe7d 100644 --- a/src-self-hosted/Package.zig +++ b/src-self-hosted/Package.zig @@ -1,6 +1,6 @@ pub const Table = std.StringHashMapUnmanaged(*Package); -root_src_directory: Module.Directory, +root_src_directory: Compilation.Directory, /// Relative to `root_src_directory`. root_src_path: []u8, table: Table, @@ -56,4 +56,4 @@ const mem = std.mem; const Allocator = std.mem.Allocator; const assert = std.debug.assert; const Package = @This(); -const Module = @import("Module.zig"); +const Compilation = @import("Compilation.zig"); diff --git a/src-self-hosted/ZigModule.zig b/src-self-hosted/ZigModule.zig index 1f21ce7653..53d35fb41b 100644 --- a/src-self-hosted/ZigModule.zig +++ b/src-self-hosted/ZigModule.zig @@ -1,9 +1,7 @@ -//! TODO This is going to get renamed from ZigModule to Module (but first we have to rename -//! Module to Compilation). +//! TODO This is going to get renamed from ZigModule to Module const Module = @This(); -const Compilation = @import("Module.zig"); - const std = @import("std"); +const Compilation = @import("Compilation.zig"); const mem = std.mem; const Allocator = std.mem.Allocator; const ArrayListUnmanaged = std.ArrayListUnmanaged; diff --git a/src-self-hosted/codegen.zig b/src-self-hosted/codegen.zig index f35092639b..212843a38c 100644 --- a/src-self-hosted/codegen.zig +++ b/src-self-hosted/codegen.zig @@ -8,7 +8,7 @@ const Value = @import("value.zig").Value; const TypedValue = @import("TypedValue.zig"); const link = @import("link.zig"); const Module = @import("ZigModule.zig"); -const Compilation = @import("Module.zig"); +const Compilation = @import("Compilation.zig"); const ErrorMsg = Compilation.ErrorMsg; const Target = std.Target; const Allocator = mem.Allocator; diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index d366f74286..3577442121 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -2,7 +2,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const target_util = @import("target.zig"); const mem = std.mem; -const Module = @import("Module.zig"); +const Compilation = @import("Compilation.zig"); const path = std.fs.path; const build_options = @import("build_options"); const trace = @import("tracy.zig").trace; @@ -246,11 +246,11 @@ pub const CRTFile = enum { libc_nonshared_a, }; -pub fn buildCRTFile(mod: *Module, crt_file: CRTFile) !void { +pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { if (!build_options.have_llvm) { return error.ZigCompilerNotBuiltWithLLVMExtensions; } - const gpa = mod.gpa; + const gpa = comp.gpa; var arena_allocator = std.heap.ArenaAllocator.init(gpa); errdefer arena_allocator.deinit(); const arena = &arena_allocator.allocator; @@ -258,29 +258,29 @@ pub fn buildCRTFile(mod: *Module, crt_file: CRTFile) !void { switch (crt_file) { .crti_o => { var args = std.ArrayList([]const u8).init(arena); - try add_include_dirs(mod, arena, &args); + try add_include_dirs(comp, arena, &args); try args.appendSlice(&[_][]const u8{ "-D_LIBC_REENTRANT", "-include", - try lib_path(mod, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-modules.h"), + try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-modules.h"), "-DMODULE_NAME=libc", "-Wno-nonportable-include-path", "-include", - try lib_path(mod, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-symbols.h"), + try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-symbols.h"), "-DTOP_NAMESPACE=glibc", "-DASSEMBLER", "-g", "-Wa,--noexecstack", }); - const c_source_file: Module.CSourceFile = .{ - .src_path = try start_asm_path(mod, arena, "crti.S"), + const c_source_file: Compilation.CSourceFile = .{ + .src_path = try start_asm_path(comp, arena, "crti.S"), .extra_flags = args.items, }; - return build_libc_object(mod, "crti.o", c_source_file); + return build_libc_object(comp, "crti.o", c_source_file); }, .crtn_o => { var args = std.ArrayList([]const u8).init(arena); - try add_include_dirs(mod, arena, &args); + try add_include_dirs(comp, arena, &args); try args.appendSlice(&[_][]const u8{ "-D_LIBC_REENTRANT", "-DMODULE_NAME=libc", @@ -289,23 +289,23 @@ pub fn buildCRTFile(mod: *Module, crt_file: CRTFile) !void { "-g", "-Wa,--noexecstack", }); - const c_source_file: Module.CSourceFile = .{ - .src_path = try start_asm_path(mod, arena, "crtn.S"), + const c_source_file: Compilation.CSourceFile = .{ + .src_path = try start_asm_path(comp, arena, "crtn.S"), .extra_flags = args.items, }; - return build_libc_object(mod, "crtn.o", c_source_file); + return build_libc_object(comp, "crtn.o", c_source_file); }, .start_os => { var args = std.ArrayList([]const u8).init(arena); - try add_include_dirs(mod, arena, &args); + try add_include_dirs(comp, arena, &args); try args.appendSlice(&[_][]const u8{ "-D_LIBC_REENTRANT", "-include", - try lib_path(mod, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-modules.h"), + try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-modules.h"), "-DMODULE_NAME=libc", "-Wno-nonportable-include-path", "-include", - try lib_path(mod, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-symbols.h"), + try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-symbols.h"), "-DPIC", "-DSHARED", "-DTOP_NAMESPACE=glibc", @@ -313,19 +313,19 @@ pub fn buildCRTFile(mod: *Module, crt_file: CRTFile) !void { "-g", "-Wa,--noexecstack", }); - const c_source_file: Module.CSourceFile = .{ - .src_path = try start_asm_path(mod, arena, "start.S"), + const c_source_file: Compilation.CSourceFile = .{ + .src_path = try start_asm_path(comp, arena, "start.S"), .extra_flags = args.items, }; - return build_libc_object(mod, "start.os", c_source_file); + return build_libc_object(comp, "start.os", c_source_file); }, .abi_note_o => { var args = std.ArrayList([]const u8).init(arena); try args.appendSlice(&[_][]const u8{ "-I", - try lib_path(mod, arena, lib_libc_glibc ++ "glibc" ++ path.sep_str ++ "csu"), + try lib_path(comp, arena, lib_libc_glibc ++ "glibc" ++ path.sep_str ++ "csu"), }); - try add_include_dirs(mod, arena, &args); + try add_include_dirs(comp, arena, &args); try args.appendSlice(&[_][]const u8{ "-D_LIBC_REENTRANT", "-DMODULE_NAME=libc", @@ -334,11 +334,11 @@ pub fn buildCRTFile(mod: *Module, crt_file: CRTFile) !void { "-g", "-Wa,--noexecstack", }); - const c_source_file: Module.CSourceFile = .{ - .src_path = try lib_path(mod, arena, lib_libc_glibc ++ "csu" ++ path.sep_str ++ "abi-note.S"), + const c_source_file: Compilation.CSourceFile = .{ + .src_path = try lib_path(comp, arena, lib_libc_glibc ++ "csu" ++ path.sep_str ++ "abi-note.S"), .extra_flags = args.items, }; - return build_libc_object(mod, "abi-note.o", c_source_file); + return build_libc_object(comp, "abi-note.o", c_source_file); }, .scrt1_o => { return error.Unimplemented; // TODO @@ -349,8 +349,8 @@ pub fn buildCRTFile(mod: *Module, crt_file: CRTFile) !void { } } -fn start_asm_path(mod: *Module, arena: *Allocator, basename: []const u8) ![]const u8 { - const arch = mod.getTarget().cpu.arch; +fn start_asm_path(comp: *Compilation, arena: *Allocator, basename: []const u8) ![]const u8 { + const arch = comp.getTarget().cpu.arch; const is_ppc = arch == .powerpc or arch == .powerpc64 or arch == .powerpc64le; const is_aarch64 = arch == .aarch64 or arch == .aarch64_be; const is_sparc = arch == .sparc or arch == .sparcel or arch == .sparcv9; @@ -359,7 +359,7 @@ fn start_asm_path(mod: *Module, arena: *Allocator, basename: []const u8) ![]cons const s = path.sep_str; var result = std.ArrayList(u8).init(arena); - try result.appendSlice(mod.zig_lib_directory.path.?); + try result.appendSlice(comp.zig_lib_directory.path.?); try result.appendSlice(s ++ "libc" ++ s ++ "glibc" ++ s ++ "sysdeps" ++ s); if (is_sparc) { if (is_64) { @@ -392,76 +392,76 @@ fn start_asm_path(mod: *Module, arena: *Allocator, basename: []const u8) ![]cons return result.items; } -fn add_include_dirs(mod: *Module, arena: *Allocator, args: *std.ArrayList([]const u8)) error{OutOfMemory}!void { - const target = mod.getTarget(); +fn add_include_dirs(comp: *Compilation, arena: *Allocator, args: *std.ArrayList([]const u8)) error{OutOfMemory}!void { + const target = comp.getTarget(); const arch = target.cpu.arch; const opt_nptl: ?[]const u8 = if (target.os.tag == .linux) "nptl" else "htl"; - const glibc = try lib_path(mod, arena, lib_libc ++ "glibc"); + const glibc = try lib_path(comp, arena, lib_libc ++ "glibc"); const s = path.sep_str; try args.append("-I"); - try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "include")); + try args.append(try lib_path(comp, arena, lib_libc_glibc ++ "include")); if (target.os.tag == .linux) { - try add_include_dirs_arch(arena, args, arch, null, try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix" ++ s ++ "sysv" ++ s ++ "linux")); + try add_include_dirs_arch(arena, args, arch, null, try lib_path(comp, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix" ++ s ++ "sysv" ++ s ++ "linux")); } if (opt_nptl) |nptl| { - try add_include_dirs_arch(arena, args, arch, nptl, try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps")); + try add_include_dirs_arch(arena, args, arch, nptl, try lib_path(comp, arena, lib_libc_glibc ++ "sysdeps")); } if (target.os.tag == .linux) { try args.append("-I"); - try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ + try args.append(try lib_path(comp, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix" ++ s ++ "sysv" ++ s ++ "linux" ++ s ++ "generic")); try args.append("-I"); - try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ + try args.append(try lib_path(comp, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix" ++ s ++ "sysv" ++ s ++ "linux" ++ s ++ "include")); try args.append("-I"); - try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ + try args.append(try lib_path(comp, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix" ++ s ++ "sysv" ++ s ++ "linux")); } if (opt_nptl) |nptl| { try args.append("-I"); - try args.append(try path.join(arena, &[_][]const u8{ mod.zig_lib_directory.path.?, lib_libc_glibc ++ "sysdeps", nptl })); + try args.append(try path.join(arena, &[_][]const u8{ comp.zig_lib_directory.path.?, lib_libc_glibc ++ "sysdeps", nptl })); } try args.append("-I"); - try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "pthread")); + try args.append(try lib_path(comp, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "pthread")); try args.append("-I"); - try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix" ++ s ++ "sysv")); + try args.append(try lib_path(comp, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix" ++ s ++ "sysv")); - try add_include_dirs_arch(arena, args, arch, null, try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix")); + try add_include_dirs_arch(arena, args, arch, null, try lib_path(comp, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix")); try args.append("-I"); - try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix")); + try args.append(try lib_path(comp, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "unix")); - try add_include_dirs_arch(arena, args, arch, null, try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps")); + try add_include_dirs_arch(arena, args, arch, null, try lib_path(comp, arena, lib_libc_glibc ++ "sysdeps")); try args.append("-I"); - try args.append(try lib_path(mod, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "generic")); + try args.append(try lib_path(comp, arena, lib_libc_glibc ++ "sysdeps" ++ s ++ "generic")); try args.append("-I"); - try args.append(try path.join(arena, &[_][]const u8{ mod.zig_lib_directory.path.?, lib_libc ++ "glibc" })); + try args.append(try path.join(arena, &[_][]const u8{ comp.zig_lib_directory.path.?, lib_libc ++ "glibc" })); try args.append("-I"); try args.append(try std.fmt.allocPrint(arena, "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{}-{}-{}", .{ - mod.zig_lib_directory.path.?, @tagName(arch), @tagName(target.os.tag), @tagName(target.abi), + comp.zig_lib_directory.path.?, @tagName(arch), @tagName(target.os.tag), @tagName(target.abi), })); try args.append("-I"); - try args.append(try lib_path(mod, arena, lib_libc ++ "include" ++ s ++ "generic-glibc")); + try args.append(try lib_path(comp, arena, lib_libc ++ "include" ++ s ++ "generic-glibc")); try args.append("-I"); try args.append(try std.fmt.allocPrint(arena, "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{}-linux-any", .{ - mod.zig_lib_directory.path.?, @tagName(arch), + comp.zig_lib_directory.path.?, @tagName(arch), })); try args.append("-I"); - try args.append(try lib_path(mod, arena, lib_libc ++ "include" ++ s ++ "any-linux-any")); + try args.append(try lib_path(comp, arena, lib_libc ++ "include" ++ s ++ "any-linux-any")); } fn add_include_dirs_arch( @@ -576,60 +576,60 @@ fn add_include_dirs_arch( } } -fn path_from_lib(mod: *Module, arena: *Allocator, sub_path: []const u8) ![]const u8 { - return path.join(arena, &[_][]const u8{ mod.zig_lib_directory.path.?, sub_path }); +fn path_from_lib(comp: *Compilation, arena: *Allocator, sub_path: []const u8) ![]const u8 { + return path.join(arena, &[_][]const u8{ comp.zig_lib_directory.path.?, sub_path }); } const lib_libc = "libc" ++ path.sep_str; const lib_libc_glibc = lib_libc ++ "glibc" ++ path.sep_str; -fn lib_path(mod: *Module, arena: *Allocator, sub_path: []const u8) ![]const u8 { - return path.join(arena, &[_][]const u8{ mod.zig_lib_directory.path.?, sub_path }); +fn lib_path(comp: *Compilation, arena: *Allocator, sub_path: []const u8) ![]const u8 { + return path.join(arena, &[_][]const u8{ comp.zig_lib_directory.path.?, sub_path }); } -fn build_libc_object(mod: *Module, basename: []const u8, c_source_file: Module.CSourceFile) !void { +fn build_libc_object(comp: *Compilation, basename: []const u8, c_source_file: Compilation.CSourceFile) !void { const tracy = trace(@src()); defer tracy.end(); // TODO: This is extracted into a local variable to work around a stage1 miscompilation. - const emit_bin = Module.EmitLoc{ + const emit_bin = Compilation.EmitLoc{ .directory = null, // Put it in the cache directory. .basename = basename, }; - const sub_module = try Module.create(mod.gpa, .{ + const sub_compilation = try Compilation.create(comp.gpa, .{ // TODO use the global cache directory here - .zig_cache_directory = mod.zig_cache_directory, - .zig_lib_directory = mod.zig_lib_directory, - .target = mod.getTarget(), + .zig_cache_directory = comp.zig_cache_directory, + .zig_lib_directory = comp.zig_lib_directory, + .target = comp.getTarget(), .root_name = mem.split(basename, ".").next().?, .root_pkg = null, .output_mode = .Obj, - .rand = mod.rand, - .libc_installation = mod.bin_file.options.libc_installation, + .rand = comp.rand, + .libc_installation = comp.bin_file.options.libc_installation, .emit_bin = emit_bin, - .optimize_mode = mod.bin_file.options.optimize_mode, + .optimize_mode = comp.bin_file.options.optimize_mode, .want_sanitize_c = false, .want_stack_check = false, .want_valgrind = false, - .want_pic = mod.bin_file.options.pic, + .want_pic = comp.bin_file.options.pic, .emit_h = null, - .strip = mod.bin_file.options.strip, - .is_native_os = mod.bin_file.options.is_native_os, - .self_exe_path = mod.self_exe_path, - .c_source_files = &[1]Module.CSourceFile{c_source_file}, - .debug_cc = mod.debug_cc, - .debug_link = mod.bin_file.options.debug_link, + .strip = comp.bin_file.options.strip, + .is_native_os = comp.bin_file.options.is_native_os, + .self_exe_path = comp.self_exe_path, + .c_source_files = &[1]Compilation.CSourceFile{c_source_file}, + .debug_cc = comp.debug_cc, + .debug_link = comp.bin_file.options.debug_link, }); - defer sub_module.destroy(); + defer sub_compilation.destroy(); - try sub_module.update(); + try sub_compilation.update(); - try mod.crt_files.ensureCapacity(mod.gpa, mod.crt_files.count() + 1); - const artifact_path = if (sub_module.bin_file.options.directory.path) |p| - try std.fs.path.join(mod.gpa, &[_][]const u8{ p, basename }) + try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1); + const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p| + try std.fs.path.join(comp.gpa, &[_][]const u8{ p, basename }) else - try mod.gpa.dupe(u8, basename); + try comp.gpa.dupe(u8, basename); // TODO obtain a lock on the artifact and put that in crt_files as well. - mod.crt_files.putAssumeCapacityNoClobber(basename, artifact_path); + comp.crt_files.putAssumeCapacityNoClobber(basename, artifact_path); } diff --git a/src-self-hosted/introspect.zig b/src-self-hosted/introspect.zig index 3dd040881c..8041142a67 100644 --- a/src-self-hosted/introspect.zig +++ b/src-self-hosted/introspect.zig @@ -2,12 +2,12 @@ const std = @import("std"); const mem = std.mem; const fs = std.fs; const CacheHash = std.cache_hash.CacheHash; -const Module = @import("Module.zig"); +const Compilation = @import("Compilation.zig"); /// Returns the sub_path that worked, or `null` if none did. /// The path of the returned Directory is relative to `base`. /// The handle of the returned Directory is open. -fn testZigInstallPrefix(base_dir: fs.Dir) ?Module.Directory { +fn testZigInstallPrefix(base_dir: fs.Dir) ?Compilation.Directory { const test_index_file = "std" ++ fs.path.sep_str ++ "std.zig"; zig_dir: { @@ -19,7 +19,7 @@ fn testZigInstallPrefix(base_dir: fs.Dir) ?Module.Directory { break :zig_dir; }; file.close(); - return Module.Directory{ .handle = test_zig_dir, .path = lib_zig }; + return Compilation.Directory{ .handle = test_zig_dir, .path = lib_zig }; } // Try lib/std/std.zig @@ -29,11 +29,11 @@ fn testZigInstallPrefix(base_dir: fs.Dir) ?Module.Directory { return null; }; file.close(); - return Module.Directory{ .handle = test_zig_dir, .path = "lib" }; + return Compilation.Directory{ .handle = test_zig_dir, .path = "lib" }; } /// Both the directory handle and the path are newly allocated resources which the caller now owns. -pub fn findZigLibDir(gpa: *mem.Allocator) !Module.Directory { +pub fn findZigLibDir(gpa: *mem.Allocator) !Compilation.Directory { const self_exe_path = try fs.selfExePathAlloc(gpa); defer gpa.free(self_exe_path); @@ -44,7 +44,7 @@ pub fn findZigLibDir(gpa: *mem.Allocator) !Module.Directory { pub fn findZigLibDirFromSelfExe( allocator: *mem.Allocator, self_exe_path: []const u8, -) error{ OutOfMemory, FileNotFound }!Module.Directory { +) error{ OutOfMemory, FileNotFound }!Compilation.Directory { const cwd = fs.cwd(); var cur_path: []const u8 = self_exe_path; while (fs.path.dirname(cur_path)) |dirname| : (cur_path = dirname) { @@ -52,7 +52,7 @@ pub fn findZigLibDirFromSelfExe( defer base_dir.close(); const sub_directory = testZigInstallPrefix(base_dir) orelse continue; - return Module.Directory{ + return Compilation.Directory{ .handle = sub_directory.handle, .path = try fs.path.join(allocator, &[_][]const u8{ dirname, sub_directory.path.? }), }; diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 50f133f792..b2d9a0416f 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1,6 +1,6 @@ const std = @import("std"); const Allocator = std.mem.Allocator; -const Compilation = @import("Module.zig"); +const Compilation = @import("Compilation.zig"); const ZigModule = @import("ZigModule.zig"); const fs = std.fs; const trace = @import("tracy.zig").trace; @@ -23,7 +23,6 @@ pub const Options = struct { optimize_mode: std.builtin.Mode, root_name: []const u8, /// Not every Compilation compiles .zig code! For example you could do `zig build-exe foo.o`. - /// TODO rename Module to Compilation and then (as a separate commit) ZigModule to Module. zig_module: ?*ZigModule, dynamic_linker: ?[]const u8 = null, /// Used for calculating how much space to reserve for symbols in case the binary file diff --git a/src-self-hosted/link/C.zig b/src-self-hosted/link/C.zig index edfa055578..6cf406f353 100644 --- a/src-self-hosted/link/C.zig +++ b/src-self-hosted/link/C.zig @@ -3,7 +3,7 @@ const mem = std.mem; const assert = std.debug.assert; const Allocator = std.mem.Allocator; const Module = @import("../ZigModule.zig"); -const Compilation = @import("../Module.zig"); +const Compilation = @import("../Compilation.zig"); const fs = std.fs; const codegen = @import("../codegen/c.zig"); const link = @import("../link.zig"); diff --git a/src-self-hosted/link/Coff.zig b/src-self-hosted/link/Coff.zig index 7e3fc2fa72..7d45f61f24 100644 --- a/src-self-hosted/link/Coff.zig +++ b/src-self-hosted/link/Coff.zig @@ -8,7 +8,7 @@ const fs = std.fs; const trace = @import("../tracy.zig").trace; const Module = @import("../ZigModule.zig"); -const Compilation = @import("../Module.zig"); +const Compilation = @import("../Compilation.zig"); const codegen = @import("../codegen.zig"); const link = @import("../link.zig"); diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index cd8391c925..b88790a8cc 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -4,7 +4,7 @@ const assert = std.debug.assert; const Allocator = std.mem.Allocator; const ir = @import("../ir.zig"); const Module = @import("../ZigModule.zig"); -const Compilation = @import("../Module.zig"); +const Compilation = @import("../Compilation.zig"); const fs = std.fs; const elf = std.elf; const codegen = @import("../codegen.zig"); diff --git a/src-self-hosted/link/MachO.zig b/src-self-hosted/link/MachO.zig index c412c84e9e..dd70cc7c2b 100644 --- a/src-self-hosted/link/MachO.zig +++ b/src-self-hosted/link/MachO.zig @@ -13,7 +13,7 @@ const trace = @import("../tracy.zig").trace; const Type = @import("../type.zig").Type; const Module = @import("../ZigModule.zig"); -const Compilation = @import("../Module.zig"); +const Compilation = @import("../Compilation.zig"); const link = @import("../link.zig"); const File = link.File; diff --git a/src-self-hosted/link/Wasm.zig b/src-self-hosted/link/Wasm.zig index f3586489d6..85634daca5 100644 --- a/src-self-hosted/link/Wasm.zig +++ b/src-self-hosted/link/Wasm.zig @@ -7,7 +7,7 @@ const fs = std.fs; const leb = std.debug.leb; const Module = @import("../ZigModule.zig"); -const Compilation = @import("../Module.zig"); +const Compilation = @import("../Compilation.zig"); const codegen = @import("../codegen/wasm.zig"); const link = @import("../link.zig"); diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 6453cdaeb1..d0bce3e19f 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -7,7 +7,7 @@ const process = std.process; const Allocator = mem.Allocator; const ArrayList = std.ArrayList; const ast = std.zig.ast; -const Module = @import("Module.zig"); +const Compilation = @import("Compilation.zig"); const link = @import("link.zig"); const Package = @import("Package.zig"); const zir = @import("zir.zig"); @@ -331,7 +331,7 @@ pub fn buildOutputType( var rpath_list = std.ArrayList([]const u8).init(gpa); defer rpath_list.deinit(); - var c_source_files = std.ArrayList(Module.CSourceFile).init(gpa); + var c_source_files = std.ArrayList(Compilation.CSourceFile).init(gpa); defer c_source_files.deinit(); var link_objects = std.ArrayList([]const u8).init(gpa); @@ -566,7 +566,7 @@ pub fn buildOutputType( mem.endsWith(u8, arg, ".lib")) { try link_objects.append(arg); - } else if (Module.hasAsmExt(arg) or Module.hasCExt(arg) or Module.hasCppExt(arg)) { + } else if (Compilation.hasAsmExt(arg) or Compilation.hasCExt(arg) or Compilation.hasCppExt(arg)) { // TODO a way to pass extra flags on the CLI try c_source_files.append(.{ .src_path = arg }); } else if (mem.endsWith(u8, arg, ".so") or @@ -611,7 +611,7 @@ pub fn buildOutputType( try clang_argv.appendSlice(it.other_args); }, .positional => { - const file_ext = Module.classifyFileExt(mem.spanZ(it.only_arg)); + const file_ext = Compilation.classifyFileExt(mem.spanZ(it.only_arg)); switch (file_ext) { .assembly, .c, .cpp, .ll, .bc, .h => try c_source_files.append(.{ .src_path = it.only_arg }), .unknown, .so => try link_objects.append(it.only_arg), @@ -998,9 +998,9 @@ pub fn buildOutputType( var cleanup_emit_bin_dir: ?fs.Dir = null; defer if (cleanup_emit_bin_dir) |*dir| dir.close(); - const emit_bin_loc: ?Module.EmitLoc = switch (emit_bin) { + const emit_bin_loc: ?Compilation.EmitLoc = switch (emit_bin) { .no => null, - .yes_default_path => Module.EmitLoc{ + .yes_default_path => Compilation.EmitLoc{ .directory = .{ .path = null, .handle = fs.cwd() }, .basename = try std.zig.binNameAlloc( arena, @@ -1016,7 +1016,7 @@ pub fn buildOutputType( if (fs.path.dirname(full_path)) |dirname| { const handle = try fs.cwd().openDir(dirname, .{}); cleanup_emit_bin_dir = handle; - break :b Module.EmitLoc{ + break :b Compilation.EmitLoc{ .basename = basename, .directory = .{ .path = dirname, @@ -1024,7 +1024,7 @@ pub fn buildOutputType( }, }; } else { - break :b Module.EmitLoc{ + break :b Compilation.EmitLoc{ .basename = basename, .directory = .{ .path = null, .handle = fs.cwd() }, }; @@ -1035,9 +1035,9 @@ pub fn buildOutputType( var cleanup_emit_h_dir: ?fs.Dir = null; defer if (cleanup_emit_h_dir) |*dir| dir.close(); - const emit_h_loc: ?Module.EmitLoc = switch (emit_h) { + const emit_h_loc: ?Compilation.EmitLoc = switch (emit_h) { .no => null, - .yes_default_path => Module.EmitLoc{ + .yes_default_path => Compilation.EmitLoc{ .directory = .{ .path = null, .handle = fs.cwd() }, .basename = try std.fmt.allocPrint(arena, "{}.h", .{root_name}), }, @@ -1046,7 +1046,7 @@ pub fn buildOutputType( if (fs.path.dirname(full_path)) |dirname| { const handle = try fs.cwd().openDir(dirname, .{}); cleanup_emit_h_dir = handle; - break :b Module.EmitLoc{ + break :b Compilation.EmitLoc{ .basename = basename, .directory = .{ .path = dirname, @@ -1054,7 +1054,7 @@ pub fn buildOutputType( }, }; } else { - break :b Module.EmitLoc{ + break :b Compilation.EmitLoc{ .basename = basename, .directory = .{ .path = null, .handle = fs.cwd() }, }; @@ -1103,7 +1103,7 @@ pub fn buildOutputType( const cache_parent_dir = if (root_pkg) |pkg| pkg.root_src_directory.handle else fs.cwd(); var cache_dir = try cache_parent_dir.makeOpenPath("zig-cache", .{}); defer cache_dir.close(); - const zig_cache_directory: Module.Directory = .{ + const zig_cache_directory: Compilation.Directory = .{ .handle = cache_dir, .path = blk: { if (root_pkg) |pkg| { @@ -1115,7 +1115,7 @@ pub fn buildOutputType( }, }; - const module = Module.create(gpa, .{ + const comp = Compilation.create(gpa, .{ .zig_lib_directory = zig_lib_directory, .zig_cache_directory = zig_cache_directory, .root_name = root_name, @@ -1170,15 +1170,15 @@ pub fn buildOutputType( .debug_cc = debug_cc, .debug_link = debug_link, }) catch |err| { - fatal("unable to create module: {}", .{@errorName(err)}); + fatal("unable to create compilation: {}", .{@errorName(err)}); }; - defer module.destroy(); + defer comp.destroy(); const stdin = std.io.getStdIn().inStream(); const stderr = std.io.getStdErr().outStream(); var repl_buf: [1024]u8 = undefined; - try updateModule(gpa, module, zir_out_path); + try updateModule(gpa, comp, zir_out_path); if (build_options.have_llvm and only_pp_or_asm) { // this may include dumping the output to stdout @@ -1188,7 +1188,7 @@ pub fn buildOutputType( while (watch) { try stderr.print("🦎 ", .{}); if (output_mode == .Exe) { - try module.makeBinFileExecutable(); + try comp.makeBinFileExecutable(); } if (stdin.readUntilDelimiterOrEof(&repl_buf, '\n') catch |err| { try stderr.print("\nUnable to parse command: {}\n", .{@errorName(err)}); @@ -1198,9 +1198,9 @@ pub fn buildOutputType( if (mem.eql(u8, actual_line, "update")) { if (output_mode == .Exe) { - try module.makeBinFileWritable(); + try comp.makeBinFileWritable(); } - try updateModule(gpa, module, zir_out_path); + try updateModule(gpa, comp, zir_out_path); } else if (mem.eql(u8, actual_line, "exit")) { break; } else if (mem.eql(u8, actual_line, "help")) { @@ -1214,11 +1214,11 @@ pub fn buildOutputType( } } -fn updateModule(gpa: *Allocator, module: *Module, zir_out_path: ?[]const u8) !void { - try module.update(); +fn updateModule(gpa: *Allocator, comp: *Compilation, zir_out_path: ?[]const u8) !void { + try comp.update(); - var errors = try module.getAllErrorsAlloc(); - defer errors.deinit(module.gpa); + var errors = try comp.getAllErrorsAlloc(); + defer errors.deinit(comp.gpa); if (errors.list.len != 0) { for (errors.list) |full_err_msg| { @@ -1232,7 +1232,7 @@ fn updateModule(gpa: *Allocator, module: *Module, zir_out_path: ?[]const u8) !vo } if (zir_out_path) |zop| { - const zig_module = module.bin_file.options.zig_module orelse + const zig_module = comp.bin_file.options.zig_module orelse fatal("-femit-zir with no zig source code", .{}); var new_zir_module = try zir.emit(gpa, zig_module); defer new_zir_module.deinit(gpa); diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 40eb833273..edff8aa262 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -1,6 +1,6 @@ const std = @import("std"); const link = @import("link.zig"); -const Module = @import("Module.zig"); +const Compilation = @import("Compilation.zig"); const Allocator = std.mem.Allocator; const zir = @import("zir.zig"); const Package = @import("Package.zig"); @@ -61,7 +61,7 @@ pub const TestContext = struct { ZIR, }; - /// A Case consists of a set of *updates*. The same Module is used for each + /// A Case consists of a set of *updates*. The same Compilation is used for each /// update, so each update's source is treated as a single file being /// updated by the test harness and incrementally compiled. pub const Case = struct { @@ -437,7 +437,7 @@ pub const TestContext = struct { allocator: *Allocator, root_node: *std.Progress.Node, case: Case, - zig_lib_directory: Module.Directory, + zig_lib_directory: Compilation.Directory, rand: *std.rand.Random, ) !void { const target_info = try std.zig.system.NativeTargetInfo.detect(allocator, case.target); @@ -453,7 +453,7 @@ pub const TestContext = struct { var cache_dir = try tmp.dir.makeOpenPath("zig-cache", .{}); defer cache_dir.close(); const bogus_path = "bogus"; // TODO this will need to be fixed before we can test LLVM extensions - const zig_cache_directory: Module.Directory = .{ + const zig_cache_directory: Compilation.Directory = .{ .handle = cache_dir, .path = try std.fs.path.join(arena, &[_][]const u8{ bogus_path, "zig-cache" }), }; @@ -465,15 +465,15 @@ pub const TestContext = struct { const ofmt: ?std.builtin.ObjectFormat = if (case.cbe) .c else null; const bin_name = try std.zig.binNameAlloc(arena, "test_case", target, case.output_mode, null, ofmt); - const emit_directory: Module.Directory = .{ + const emit_directory: Compilation.Directory = .{ .path = bogus_path, .handle = tmp.dir, }; - const emit_bin: Module.EmitLoc = .{ + const emit_bin: Compilation.EmitLoc = .{ .directory = emit_directory, .basename = bin_name, }; - const module = try Module.create(allocator, .{ + const comp = try Compilation.create(allocator, .{ .zig_cache_directory = zig_cache_directory, .zig_lib_directory = zig_lib_directory, .rand = rand, @@ -491,7 +491,7 @@ pub const TestContext = struct { .object_format = ofmt, .is_native_os = case.target.isNativeOs(), }); - defer module.destroy(); + defer comp.destroy(); for (case.updates.items) |update, update_index| { var update_node = root_node.start("update", 3); @@ -505,20 +505,20 @@ pub const TestContext = struct { var module_node = update_node.start("parse/analysis/codegen", null); module_node.activate(); - try module.makeBinFileWritable(); - try module.update(); + try comp.makeBinFileWritable(); + try comp.update(); module_node.end(); if (update.case != .Error) { - var all_errors = try module.getAllErrorsAlloc(); + var all_errors = try comp.getAllErrorsAlloc(); defer all_errors.deinit(allocator); if (all_errors.list.len != 0) { - std.debug.print("\nErrors occurred updating the module:\n================\n", .{}); + std.debug.print("\nErrors occurred updating the compilation:\n================\n", .{}); for (all_errors.list) |err| { std.debug.print(":{}:{}: error: {}\n================\n", .{ err.line + 1, err.column + 1, err.msg }); } if (case.cbe) { - const C = module.bin_file.cast(link.File.C).?; + const C = comp.bin_file.cast(link.File.C).?; std.debug.print("Generated C: \n===============\n{}\n\n===========\n\n", .{C.main.items}); } std.debug.print("Test failed.\n", .{}); @@ -549,7 +549,7 @@ pub const TestContext = struct { update_node.estimated_total_items = 5; var emit_node = update_node.start("emit", null); emit_node.activate(); - var new_zir_module = try zir.emit(allocator, module.bin_file.options.zig_module.?); + var new_zir_module = try zir.emit(allocator, comp.bin_file.options.zig_module.?); defer new_zir_module.deinit(allocator); emit_node.end(); @@ -584,7 +584,7 @@ pub const TestContext = struct { for (handled_errors) |*h| { h.* = false; } - var all_errors = try module.getAllErrorsAlloc(); + var all_errors = try comp.getAllErrorsAlloc(); defer all_errors.deinit(allocator); for (all_errors.list) |a| { for (e) |ex, i| { @@ -666,7 +666,7 @@ pub const TestContext = struct { }, } - try module.makeBinFileExecutable(); + try comp.makeBinFileExecutable(); break :x try std.ChildProcess.exec(.{ .allocator = allocator, From 2456df5f4ebf9fef147d86736974e95403e2d40d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2020 19:56:35 -0700 Subject: [PATCH 022/210] stage2: rename ZigModule to Module --- src-self-hosted/Compilation.zig | 116 +++++++++--------- src-self-hosted/{ZigModule.zig => Module.zig} | 1 - src-self-hosted/astgen.zig | 2 +- src-self-hosted/codegen.zig | 2 +- src-self-hosted/codegen/c.zig | 2 +- src-self-hosted/codegen/wasm.zig | 2 +- src-self-hosted/ir.zig | 2 +- src-self-hosted/link.zig | 20 +-- src-self-hosted/link/C.zig | 2 +- src-self-hosted/link/Coff.zig | 2 +- src-self-hosted/link/Elf.zig | 20 +-- src-self-hosted/link/MachO.zig | 2 +- src-self-hosted/link/Wasm.zig | 4 +- src-self-hosted/main.zig | 4 +- src-self-hosted/test.zig | 2 +- src-self-hosted/type.zig | 2 +- src-self-hosted/value.zig | 2 +- src-self-hosted/zir.zig | 2 +- src-self-hosted/zir_sema.zig | 2 +- 19 files changed, 95 insertions(+), 96 deletions(-) rename src-self-hosted/{ZigModule.zig => Module.zig} (99%) diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index ab105deb07..9a46b88619 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -16,7 +16,7 @@ const build_options = @import("build_options"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const glibc = @import("glibc.zig"); const fatal = @import("main.zig").fatal; -const ZigModule = @import("ZigModule.zig"); +const Module = @import("Module.zig"); /// General-purpose allocator. Used for both temporary and long-term storage. gpa: *Allocator, @@ -75,7 +75,7 @@ crt_files: std.StringHashMapUnmanaged([]const u8) = .{}, /// Keeping track of this possibly open resource so we can close it later. owned_link_dir: ?std.fs.Dir, -pub const InnerError = ZigModule.InnerError; +pub const InnerError = Module.InnerError; /// For passing to a C compiler. pub const CSourceFile = struct { @@ -85,14 +85,14 @@ pub const CSourceFile = struct { const WorkItem = union(enum) { /// Write the machine code for a Decl to the output file. - codegen_decl: *ZigModule.Decl, + codegen_decl: *Module.Decl, /// The Decl needs to be analyzed and possibly export itself. /// It may have already be analyzed, or it may have been determined /// to be outdated; in this case perform semantic analysis again. - analyze_decl: *ZigModule.Decl, + analyze_decl: *Module.Decl, /// The source file containing the Decl has been updated, and so the /// Decl may need its line number information updated in the debug info. - update_line_number: *ZigModule.Decl, + update_line_number: *Module.Decl, /// Invoke the Clang compiler to create an object file, which gets linked /// with the Compilation. c_object: *CObject, @@ -402,7 +402,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { cache.hash.add(options.output_mode); // TODO audit this and make sure everything is in it - const zig_module: ?*ZigModule = if (options.root_pkg) |root_pkg| blk: { + const module: ?*Module = if (options.root_pkg) |root_pkg| blk: { // Options that are specific to zig source files, that cannot be // modified between incremental updates. var hash = cache.hash; @@ -442,11 +442,11 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { // this is where we would load it. We have open a handle to the directory where // the output either already is, or will be. // However we currently do not have serialization of such metadata, so for now - // we set up an empty ZigModule that does the entire compilation fresh. + // we set up an empty Module that does the entire compilation fresh. const root_scope = rs: { if (mem.endsWith(u8, root_pkg.root_src_path, ".zig")) { - const root_scope = try gpa.create(ZigModule.Scope.File); + const root_scope = try gpa.create(Module.Scope.File); root_scope.* = .{ .sub_file_path = root_pkg.root_src_path, .source = .{ .unloaded = {} }, @@ -459,7 +459,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { }; break :rs &root_scope.base; } else if (mem.endsWith(u8, root_pkg.root_src_path, ".zir")) { - const root_scope = try gpa.create(ZigModule.Scope.ZIRModule); + const root_scope = try gpa.create(Module.Scope.ZIRModule); root_scope.* = .{ .sub_file_path = root_pkg.root_src_path, .source = .{ .unloaded = {} }, @@ -473,24 +473,24 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { } }; - const zig_module = try arena.create(ZigModule); - zig_module.* = .{ + const module = try arena.create(Module); + module.* = .{ .gpa = gpa, .comp = comp, .root_pkg = root_pkg, .root_scope = root_scope, .zig_cache_artifact_directory = zig_cache_artifact_directory, }; - break :blk zig_module; + break :blk module; } else null; - errdefer if (zig_module) |zm| zm.deinit(); + errdefer if (module) |zm| zm.deinit(); // For resource management purposes. var owned_link_dir: ?std.fs.Dir = null; errdefer if (owned_link_dir) |*dir| dir.close(); const bin_directory = emit_bin.directory orelse blk: { - if (zig_module) |zm| break :blk zm.zig_cache_artifact_directory; + if (module) |zm| break :blk zm.zig_cache_artifact_directory; const digest = cache.hash.peek(); const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); @@ -510,7 +510,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .directory = bin_directory, .sub_path = emit_bin.basename, .root_name = root_name, - .zig_module = zig_module, + .module = module, .target = options.target, .dynamic_linker = options.dynamic_linker, .output_mode = options.output_mode, @@ -605,9 +605,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { } pub fn destroy(self: *Compilation) void { - const optional_zig_module = self.bin_file.options.zig_module; + const optional_module = self.bin_file.options.module; self.bin_file.destroy(); - if (optional_zig_module) |zig_module| zig_module.deinit(); + if (optional_module) |module| module.deinit(); const gpa = self.gpa; self.work_queue.deinit(); @@ -655,23 +655,23 @@ pub fn update(self: *Compilation) !void { self.work_queue.writeItemAssumeCapacity(.{ .c_object = entry.key }); } - if (self.bin_file.options.zig_module) |zig_module| { - zig_module.generation += 1; + if (self.bin_file.options.module) |module| { + module.generation += 1; // TODO Detect which source files changed. // Until then we simulate a full cache miss. Source files could have been loaded for any reason; // to force a refresh we unload now. - if (zig_module.root_scope.cast(ZigModule.Scope.File)) |zig_file| { - zig_file.unload(zig_module.gpa); - zig_module.analyzeContainer(&zig_file.root_container) catch |err| switch (err) { + if (module.root_scope.cast(Module.Scope.File)) |zig_file| { + zig_file.unload(module.gpa); + module.analyzeContainer(&zig_file.root_container) catch |err| switch (err) { error.AnalysisFail => { assert(self.totalErrorCount() != 0); }, else => |e| return e, }; - } else if (zig_module.root_scope.cast(ZigModule.Scope.ZIRModule)) |zir_module| { - zir_module.unload(zig_module.gpa); - zig_module.analyzeRootZIRModule(zir_module) catch |err| switch (err) { + } else if (module.root_scope.cast(Module.Scope.ZIRModule)) |zir_module| { + zir_module.unload(module.gpa); + module.analyzeRootZIRModule(zir_module) catch |err| switch (err) { error.AnalysisFail => { assert(self.totalErrorCount() != 0); }, @@ -682,14 +682,14 @@ pub fn update(self: *Compilation) !void { try self.performAllTheWork(); - if (self.bin_file.options.zig_module) |zig_module| { + if (self.bin_file.options.module) |module| { // Process the deletion set. - while (zig_module.deletion_set.popOrNull()) |decl| { + while (module.deletion_set.popOrNull()) |decl| { if (decl.dependants.items().len != 0) { decl.deletion_flag = false; continue; } - try zig_module.deleteDecl(decl); + try module.deleteDecl(decl); } } @@ -701,8 +701,8 @@ pub fn update(self: *Compilation) !void { // If there are any errors, we anticipate the source files being loaded // to report error messages. Otherwise we unload all source files to save memory. if (self.totalErrorCount() == 0 and !self.keep_source_files_loaded) { - if (self.bin_file.options.zig_module) |zig_module| { - zig_module.root_scope.unload(self.gpa); + if (self.bin_file.options.module) |module| { + module.root_scope.unload(self.gpa); } } } @@ -722,10 +722,10 @@ pub fn makeBinFileWritable(self: *Compilation) !void { pub fn totalErrorCount(self: *Compilation) usize { var total: usize = self.failed_c_objects.items().len; - if (self.bin_file.options.zig_module) |zig_module| { - total += zig_module.failed_decls.items().len + - zig_module.failed_exports.items().len + - zig_module.failed_files.items().len; + if (self.bin_file.options.module) |module| { + total += module.failed_decls.items().len + + module.failed_exports.items().len + + module.failed_files.items().len; } // The "no entry point found" error only counts if there are no other errors. @@ -748,30 +748,30 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors { const err_msg = entry.value; try AllErrors.add(&arena, &errors, c_object.src_path, "", err_msg.*); } - if (self.bin_file.options.zig_module) |zig_module| { - for (zig_module.failed_files.items()) |entry| { + if (self.bin_file.options.module) |module| { + for (module.failed_files.items()) |entry| { const scope = entry.key; const err_msg = entry.value; - const source = try scope.getSource(zig_module); + const source = try scope.getSource(module); try AllErrors.add(&arena, &errors, scope.subFilePath(), source, err_msg.*); } - for (zig_module.failed_decls.items()) |entry| { + for (module.failed_decls.items()) |entry| { const decl = entry.key; const err_msg = entry.value; - const source = try decl.scope.getSource(zig_module); + const source = try decl.scope.getSource(module); try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*); } - for (zig_module.failed_exports.items()) |entry| { + for (module.failed_exports.items()) |entry| { const decl = entry.key.owner_decl; const err_msg = entry.value; - const source = try decl.scope.getSource(zig_module); + const source = try decl.scope.getSource(module); try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*); } } if (errors.items.len == 0 and self.link_error_flags.no_entry_point_found) { const global_err_src_path = blk: { - if (self.bin_file.options.zig_module) |zig_module| break :blk zig_module.root_pkg.root_src_path; + if (self.bin_file.options.module) |module| break :blk module.root_pkg.root_src_path; if (self.c_source_files.len != 0) break :blk self.c_source_files[0].src_path; if (self.bin_file.options.objects.len != 0) break :blk self.bin_file.options.objects[0]; break :blk "(no file)"; @@ -807,10 +807,10 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { => continue, .complete, .codegen_failure_retryable => { - const zig_module = self.bin_file.options.zig_module.?; + const module = self.bin_file.options.module.?; if (decl.typed_value.most_recent.typed_value.val.cast(Value.Payload.Function)) |payload| { switch (payload.func.analysis) { - .queued => zig_module.analyzeFnBody(decl, payload.func) catch |err| switch (err) { + .queued => module.analyzeFnBody(decl, payload.func) catch |err| switch (err) { error.AnalysisFail => { assert(payload.func.analysis != .in_progress); continue; @@ -823,23 +823,23 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { } // Here we tack on additional allocations to the Decl's arena. The allocations are // lifetime annotations in the ZIR. - var decl_arena = decl.typed_value.most_recent.arena.?.promote(zig_module.gpa); + var decl_arena = decl.typed_value.most_recent.arena.?.promote(module.gpa); defer decl.typed_value.most_recent.arena.?.* = decl_arena.state; log.debug("analyze liveness of {}\n", .{decl.name}); - try liveness.analyze(zig_module.gpa, &decl_arena.allocator, payload.func.analysis.success); + try liveness.analyze(module.gpa, &decl_arena.allocator, payload.func.analysis.success); } assert(decl.typed_value.most_recent.typed_value.ty.hasCodeGenBits()); - self.bin_file.updateDecl(zig_module, decl) catch |err| switch (err) { + self.bin_file.updateDecl(module, decl) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.AnalysisFail => { decl.analysis = .dependency_failure; }, else => { - try zig_module.failed_decls.ensureCapacity(zig_module.gpa, zig_module.failed_decls.items().len + 1); - zig_module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create( - zig_module.gpa, + try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1); + module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create( + module.gpa, decl.src(), "unable to codegen: {}", .{@errorName(err)}, @@ -850,18 +850,18 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { }, }, .analyze_decl => |decl| { - const zig_module = self.bin_file.options.zig_module.?; - zig_module.ensureDeclAnalyzed(decl) catch |err| switch (err) { + const module = self.bin_file.options.module.?; + module.ensureDeclAnalyzed(decl) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.AnalysisFail => continue, }; }, .update_line_number => |decl| { - const zig_module = self.bin_file.options.zig_module.?; - self.bin_file.updateDeclLineNumber(zig_module, decl) catch |err| { - try zig_module.failed_decls.ensureCapacity(zig_module.gpa, zig_module.failed_decls.items().len + 1); - zig_module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create( - zig_module.gpa, + const module = self.bin_file.options.module.?; + self.bin_file.updateDeclLineNumber(module, decl) catch |err| { + try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1); + module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create( + module.gpa, decl.src(), "unable to update line number: {}", .{@errorName(err)}, @@ -949,7 +949,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { // Special case when doing build-obj for just one C file. When there are more than one object // file and building an object we need to link them together, but with just one it should go // directly to the output file. - const direct_o = comp.c_source_files.len == 1 and comp.bin_file.options.zig_module == null and + const direct_o = comp.c_source_files.len == 1 and comp.bin_file.options.module == null and comp.bin_file.options.output_mode == .Obj and comp.bin_file.options.objects.len == 0; const o_basename_noext = if (direct_o) comp.bin_file.options.root_name diff --git a/src-self-hosted/ZigModule.zig b/src-self-hosted/Module.zig similarity index 99% rename from src-self-hosted/ZigModule.zig rename to src-self-hosted/Module.zig index 53d35fb41b..ebe7cdfb1e 100644 --- a/src-self-hosted/ZigModule.zig +++ b/src-self-hosted/Module.zig @@ -1,4 +1,3 @@ -//! TODO This is going to get renamed from ZigModule to Module const Module = @This(); const std = @import("std"); const Compilation = @import("Compilation.zig"); diff --git a/src-self-hosted/astgen.zig b/src-self-hosted/astgen.zig index f39612b7ef..2c091a86ec 100644 --- a/src-self-hosted/astgen.zig +++ b/src-self-hosted/astgen.zig @@ -6,7 +6,7 @@ const Type = @import("type.zig").Type; const TypedValue = @import("TypedValue.zig"); const assert = std.debug.assert; const zir = @import("zir.zig"); -const Module = @import("ZigModule.zig"); +const Module = @import("Module.zig"); const ast = std.zig.ast; const trace = @import("tracy.zig").trace; const Scope = Module.Scope; diff --git a/src-self-hosted/codegen.zig b/src-self-hosted/codegen.zig index 212843a38c..a1d3cc2fc4 100644 --- a/src-self-hosted/codegen.zig +++ b/src-self-hosted/codegen.zig @@ -7,7 +7,7 @@ const Type = @import("type.zig").Type; const Value = @import("value.zig").Value; const TypedValue = @import("TypedValue.zig"); const link = @import("link.zig"); -const Module = @import("ZigModule.zig"); +const Module = @import("Module.zig"); const Compilation = @import("Compilation.zig"); const ErrorMsg = Compilation.ErrorMsg; const Target = std.Target; diff --git a/src-self-hosted/codegen/c.zig b/src-self-hosted/codegen/c.zig index 4ad2a6dafc..34ddcfbb3b 100644 --- a/src-self-hosted/codegen/c.zig +++ b/src-self-hosted/codegen/c.zig @@ -1,7 +1,7 @@ const std = @import("std"); const link = @import("../link.zig"); -const Module = @import("../ZigModule.zig"); +const Module = @import("../Module.zig"); const Inst = @import("../ir.zig").Inst; const Value = @import("../value.zig").Value; diff --git a/src-self-hosted/codegen/wasm.zig b/src-self-hosted/codegen/wasm.zig index 5f44aa8c65..4ea8838409 100644 --- a/src-self-hosted/codegen/wasm.zig +++ b/src-self-hosted/codegen/wasm.zig @@ -5,7 +5,7 @@ const assert = std.debug.assert; const leb = std.debug.leb; const mem = std.mem; -const Module = @import("../ZigModule.zig"); +const Module = @import("../Module.zig"); const Decl = Module.Decl; const Inst = @import("../ir.zig").Inst; const Type = @import("../type.zig").Type; diff --git a/src-self-hosted/ir.zig b/src-self-hosted/ir.zig index 2b07110a90..26afa52929 100644 --- a/src-self-hosted/ir.zig +++ b/src-self-hosted/ir.zig @@ -1,7 +1,7 @@ const std = @import("std"); const Value = @import("value.zig").Value; const Type = @import("type.zig").Type; -const Module = @import("ZigModule.zig"); +const Module = @import("Module.zig"); const assert = std.debug.assert; const codegen = @import("codegen.zig"); const ast = std.zig.ast; diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index b2d9a0416f..4078b263e9 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1,7 +1,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const Compilation = @import("Compilation.zig"); -const ZigModule = @import("ZigModule.zig"); +const Module = @import("Module.zig"); const fs = std.fs; const trace = @import("tracy.zig").trace; const Package = @import("Package.zig"); @@ -23,7 +23,7 @@ pub const Options = struct { optimize_mode: std.builtin.Mode, root_name: []const u8, /// Not every Compilation compiles .zig code! For example you could do `zig build-exe foo.o`. - zig_module: ?*ZigModule, + module: ?*Module, dynamic_linker: ?[]const u8 = null, /// Used for calculating how much space to reserve for symbols in case the binary file /// does not already have a symbol table. @@ -186,7 +186,7 @@ pub const File = struct { /// May be called before or after updateDeclExports but must be called /// after allocateDeclIndexes for any given Decl. - pub fn updateDecl(base: *File, module: *ZigModule, decl: *ZigModule.Decl) !void { + pub fn updateDecl(base: *File, module: *Module, decl: *Module.Decl) !void { switch (base.tag) { .coff => return @fieldParentPtr(Coff, "base", base).updateDecl(module, decl), .elf => return @fieldParentPtr(Elf, "base", base).updateDecl(module, decl), @@ -196,7 +196,7 @@ pub const File = struct { } } - pub fn updateDeclLineNumber(base: *File, module: *ZigModule, decl: *ZigModule.Decl) !void { + pub fn updateDeclLineNumber(base: *File, module: *Module, decl: *Module.Decl) !void { switch (base.tag) { .coff => return @fieldParentPtr(Coff, "base", base).updateDeclLineNumber(module, decl), .elf => return @fieldParentPtr(Elf, "base", base).updateDeclLineNumber(module, decl), @@ -207,7 +207,7 @@ pub const File = struct { /// Must be called before any call to updateDecl or updateDeclExports for /// any given Decl. - pub fn allocateDeclIndexes(base: *File, decl: *ZigModule.Decl) !void { + pub fn allocateDeclIndexes(base: *File, decl: *Module.Decl) !void { switch (base.tag) { .coff => return @fieldParentPtr(Coff, "base", base).allocateDeclIndexes(decl), .elf => return @fieldParentPtr(Elf, "base", base).allocateDeclIndexes(decl), @@ -271,7 +271,7 @@ pub const File = struct { }; } - pub fn freeDecl(base: *File, decl: *ZigModule.Decl) void { + pub fn freeDecl(base: *File, decl: *Module.Decl) void { switch (base.tag) { .coff => @fieldParentPtr(Coff, "base", base).freeDecl(decl), .elf => @fieldParentPtr(Elf, "base", base).freeDecl(decl), @@ -295,9 +295,9 @@ pub const File = struct { /// allocateDeclIndexes for any given Decl. pub fn updateDeclExports( base: *File, - module: *ZigModule, - decl: *const ZigModule.Decl, - exports: []const *ZigModule.Export, + module: *Module, + decl: *const Module.Decl, + exports: []const *Module.Export, ) !void { switch (base.tag) { .coff => return @fieldParentPtr(Coff, "base", base).updateDeclExports(module, decl, exports), @@ -308,7 +308,7 @@ pub const File = struct { } } - pub fn getDeclVAddr(base: *File, decl: *const ZigModule.Decl) u64 { + pub fn getDeclVAddr(base: *File, decl: *const Module.Decl) u64 { switch (base.tag) { .coff => return @fieldParentPtr(Coff, "base", base).getDeclVAddr(decl), .elf => return @fieldParentPtr(Elf, "base", base).getDeclVAddr(decl), diff --git a/src-self-hosted/link/C.zig b/src-self-hosted/link/C.zig index 6cf406f353..54c244d3e5 100644 --- a/src-self-hosted/link/C.zig +++ b/src-self-hosted/link/C.zig @@ -2,7 +2,7 @@ const std = @import("std"); const mem = std.mem; const assert = std.debug.assert; const Allocator = std.mem.Allocator; -const Module = @import("../ZigModule.zig"); +const Module = @import("../Module.zig"); const Compilation = @import("../Compilation.zig"); const fs = std.fs; const codegen = @import("../codegen/c.zig"); diff --git a/src-self-hosted/link/Coff.zig b/src-self-hosted/link/Coff.zig index 7d45f61f24..308c9c2332 100644 --- a/src-self-hosted/link/Coff.zig +++ b/src-self-hosted/link/Coff.zig @@ -7,7 +7,7 @@ const assert = std.debug.assert; const fs = std.fs; const trace = @import("../tracy.zig").trace; -const Module = @import("../ZigModule.zig"); +const Module = @import("../Module.zig"); const Compilation = @import("../Compilation.zig"); const codegen = @import("../codegen.zig"); const link = @import("../link.zig"); diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index b88790a8cc..2131f44f3d 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -3,7 +3,7 @@ const mem = std.mem; const assert = std.debug.assert; const Allocator = std.mem.Allocator; const ir = @import("../ir.zig"); -const Module = @import("../ZigModule.zig"); +const Module = @import("../Module.zig"); const Compilation = @import("../Compilation.zig"); const fs = std.fs; const elf = std.elf; @@ -743,7 +743,7 @@ pub fn flush(self: *Elf, comp: *Compilation) !void { fn flushInner(self: *Elf, comp: *Compilation) !void { // TODO This linker code currently assumes there is only 1 compilation unit and it corresponds to the // Zig source code. - const zig_module = self.base.options.zig_module orelse return error.LinkingWithoutZigSourceUnimplemented; + const module = self.base.options.module orelse return error.LinkingWithoutZigSourceUnimplemented; const target_endian = self.base.options.target.cpu.arch.endian(); const foreign_endian = target_endian != std.Target.current.cpu.arch.endian(); @@ -866,8 +866,8 @@ fn flushInner(self: *Elf, comp: *Compilation) !void { }, } // Write the form for the compile unit, which must match the abbrev table above. - const name_strp = try self.makeDebugString(zig_module.root_pkg.root_src_path); - const comp_dir_strp = try self.makeDebugString(zig_module.root_pkg.root_src_directory.path.?); + const name_strp = try self.makeDebugString(module.root_pkg.root_src_path); + const comp_dir_strp = try self.makeDebugString(module.root_pkg.root_src_directory.path.?); const producer_strp = try self.makeDebugString(link.producer_string); // Currently only one compilation unit is supported, so the address range is simply // identical to the main program header virtual address and memory size. @@ -1036,7 +1036,7 @@ fn flushInner(self: *Elf, comp: *Compilation) !void { 0, // include_directories (none except the compilation unit cwd) }); // file_names[0] - di_buf.appendSliceAssumeCapacity(zig_module.root_pkg.root_src_path); // relative path name + di_buf.appendSliceAssumeCapacity(module.root_pkg.root_src_path); // relative path name di_buf.appendSliceAssumeCapacity(&[_]u8{ 0, // null byte for the relative path name 0, // directory_index @@ -1230,7 +1230,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { // If there is no Zig code to compile, then we should skip flushing the output file because it // will not be part of the linker line anyway. - const zig_module_obj_path: ?[]const u8 = if (self.base.options.zig_module) |module| blk: { + const module_obj_path: ?[]const u8 = if (self.base.options.module) |module| blk: { try self.flushInner(comp); const obj_basename = self.base.intermediary_basename.?; @@ -1270,7 +1270,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { .failure => return error.NotAllCSourceFilesAvailableToLink, .success => |success| _ = try ch.addFile(success.object_path, null), }; - try ch.addOptionalFile(zig_module_obj_path); + try ch.addOptionalFile(module_obj_path); // We can skip hashing libc and libc++ components that we are in charge of building from Zig // installation sources because they are always a product of the compiler version + target information. ch.hash.addOptional(self.base.options.stack_size_override); @@ -1500,7 +1500,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { .success => |success| try argv.append(success.object_path), }; - if (zig_module_obj_path) |p| { + if (module_obj_path) |p| { try argv.append(p); } @@ -2837,8 +2837,8 @@ fn dbgLineNeededHeaderBytes(self: Elf) u32 { directory_count * 8 + file_name_count * 8 + // These are encoded as DW.FORM_string rather than DW.FORM_strp as we would like // because of a workaround for readelf and gdb failing to understand DWARFv5 correctly. - self.base.options.zig_module.?.root_pkg.root_src_directory.path.?.len + - self.base.options.zig_module.?.root_pkg.root_src_path.len); + self.base.options.module.?.root_pkg.root_src_directory.path.?.len + + self.base.options.module.?.root_pkg.root_src_path.len); } fn dbgInfoNeededHeaderBytes(self: Elf) u32 { diff --git a/src-self-hosted/link/MachO.zig b/src-self-hosted/link/MachO.zig index dd70cc7c2b..0c7e3ce2ed 100644 --- a/src-self-hosted/link/MachO.zig +++ b/src-self-hosted/link/MachO.zig @@ -12,7 +12,7 @@ const mem = std.mem; const trace = @import("../tracy.zig").trace; const Type = @import("../type.zig").Type; -const Module = @import("../ZigModule.zig"); +const Module = @import("../Module.zig"); const Compilation = @import("../Compilation.zig"); const link = @import("../link.zig"); const File = link.File; diff --git a/src-self-hosted/link/Wasm.zig b/src-self-hosted/link/Wasm.zig index 85634daca5..25e623bebb 100644 --- a/src-self-hosted/link/Wasm.zig +++ b/src-self-hosted/link/Wasm.zig @@ -6,7 +6,7 @@ const assert = std.debug.assert; const fs = std.fs; const leb = std.debug.leb; -const Module = @import("../ZigModule.zig"); +const Module = @import("../Module.zig"); const Compilation = @import("../Compilation.zig"); const codegen = @import("../codegen/wasm.zig"); const link = @import("../link.zig"); @@ -165,7 +165,7 @@ pub fn flush(self: *Wasm, comp: *Compilation) !void { } // Export section - if (self.base.options.zig_module) |module| { + if (self.base.options.module) |module| { const header_offset = try reserveVecSectionHeader(file); const writer = file.writer(); var count: u32 = 0; diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index d0bce3e19f..b7ec9ee712 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -1232,9 +1232,9 @@ fn updateModule(gpa: *Allocator, comp: *Compilation, zir_out_path: ?[]const u8) } if (zir_out_path) |zop| { - const zig_module = comp.bin_file.options.zig_module orelse + const module = comp.bin_file.options.module orelse fatal("-femit-zir with no zig source code", .{}); - var new_zir_module = try zir.emit(gpa, zig_module); + var new_zir_module = try zir.emit(gpa, module); defer new_zir_module.deinit(gpa); const baf = try io.BufferedAtomicFile.create(gpa, fs.cwd(), zop, .{}); diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index edff8aa262..154a839a3c 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -549,7 +549,7 @@ pub const TestContext = struct { update_node.estimated_total_items = 5; var emit_node = update_node.start("emit", null); emit_node.activate(); - var new_zir_module = try zir.emit(allocator, comp.bin_file.options.zig_module.?); + var new_zir_module = try zir.emit(allocator, comp.bin_file.options.module.?); defer new_zir_module.deinit(allocator); emit_node.end(); diff --git a/src-self-hosted/type.zig b/src-self-hosted/type.zig index 0c3d077477..4966395512 100644 --- a/src-self-hosted/type.zig +++ b/src-self-hosted/type.zig @@ -3,7 +3,7 @@ const Value = @import("value.zig").Value; const assert = std.debug.assert; const Allocator = std.mem.Allocator; const Target = std.Target; -const Module = @import("ZigModule.zig"); +const Module = @import("Module.zig"); /// This is the raw data, with no bookkeeping, no memory awareness, no de-duplication. /// It's important for this type to be small. diff --git a/src-self-hosted/value.zig b/src-self-hosted/value.zig index 2344b813b3..b65aa06bea 100644 --- a/src-self-hosted/value.zig +++ b/src-self-hosted/value.zig @@ -6,7 +6,7 @@ const BigIntConst = std.math.big.int.Const; const BigIntMutable = std.math.big.int.Mutable; const Target = std.Target; const Allocator = std.mem.Allocator; -const Module = @import("ZigModule.zig"); +const Module = @import("Module.zig"); /// This is the raw data, with no bookkeeping, no memory awareness, /// no de-duplication, and no type system awareness. diff --git a/src-self-hosted/zir.zig b/src-self-hosted/zir.zig index f543f8b2ce..7e723fc674 100644 --- a/src-self-hosted/zir.zig +++ b/src-self-hosted/zir.zig @@ -10,7 +10,7 @@ const Type = @import("type.zig").Type; const Value = @import("value.zig").Value; const TypedValue = @import("TypedValue.zig"); const ir = @import("ir.zig"); -const IrModule = @import("ZigModule.zig"); +const IrModule = @import("Module.zig"); /// This struct is relevent only for the ZIR Module text format. It is not used for /// semantic analysis of Zig source code. diff --git a/src-self-hosted/zir_sema.zig b/src-self-hosted/zir_sema.zig index 6a93bc2a5e..10543d2ee6 100644 --- a/src-self-hosted/zir_sema.zig +++ b/src-self-hosted/zir_sema.zig @@ -16,7 +16,7 @@ const TypedValue = @import("TypedValue.zig"); const assert = std.debug.assert; const ir = @import("ir.zig"); const zir = @import("zir.zig"); -const Module = @import("ZigModule.zig"); +const Module = @import("Module.zig"); const Inst = ir.Inst; const Body = ir.Body; const trace = @import("tracy.zig").trace; From 046dce9cefa221e46f59d3f2f3132556dec6432c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2020 21:13:24 -0700 Subject: [PATCH 023/210] stage2: fix not creating cache o dir before writing to it also remove non-working debug CLI options --- src-self-hosted/Compilation.zig | 43 ++++++++++++++++++++++----------- src-self-hosted/glibc.zig | 17 +++++++++++++ src-self-hosted/link/Elf.zig | 16 +++++------- src-self-hosted/main.zig | 30 +++-------------------- 4 files changed, 55 insertions(+), 51 deletions(-) diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 9a46b88619..d8c0bec199 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -156,6 +156,15 @@ pub const AllErrors = struct { column: usize, byte_offset: usize, msg: []const u8, + + pub fn renderToStdErr(self: Message) void { + std.debug.print("{}:{}:{}: error: {}\n", .{ + self.src_path, + self.line + 1, + self.column + 1, + self.msg, + }); + } }; pub fn deinit(self: *AllErrors, gpa: *Allocator) void { @@ -693,6 +702,12 @@ pub fn update(self: *Compilation) !void { } } + if (self.totalErrorCount() != 0) { + // Skip flushing. + self.link_error_flags = .{}; + return; + } + // This is needed before reading the error flags. try self.bin_file.flush(self); @@ -957,12 +972,14 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { mem.split(c_source_basename, ".").next().?; const o_basename = try std.fmt.allocPrint(arena, "{}{}", .{ o_basename_noext, comp.getTarget().oFileExt() }); - const full_object_path = if (!(try ch.hit()) or comp.disable_c_depfile) blk: { + const digest = if ((try ch.hit()) and !comp.disable_c_depfile) ch.final() else blk: { var argv = std.ArrayList([]const u8).init(comp.gpa); defer argv.deinit(); // We can't know the digest until we do the C compiler invocation, so we need a temporary filename. const out_obj_path = try comp.tmpFilePath(arena, o_basename); + var zig_cache_tmp_dir = try comp.zig_cache_directory.handle.makeOpenPath("tmp", .{}); + defer zig_cache_tmp_dir.close(); try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang", "-c" }); @@ -1042,25 +1059,23 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { // Rename into place. const digest = ch.final(); - const full_object_path = if (comp.zig_cache_directory.path) |p| - try std.fs.path.join(arena, &[_][]const u8{ p, "o", &digest, o_basename }) - else - try std.fs.path.join(arena, &[_][]const u8{ "o", &digest, o_basename }); - try std.fs.rename(out_obj_path, full_object_path); + const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); + var o_dir = try comp.zig_cache_directory.handle.makeOpenPath(o_sub_path, .{}); + defer o_dir.close(); + // TODO Add renameat capabilities to the std lib in a higher layer than the posix layer. + const tmp_basename = std.fs.path.basename(out_obj_path); + try std.os.renameat(zig_cache_tmp_dir.fd, tmp_basename, o_dir.fd, o_basename); ch.writeManifest() catch |err| { std.log.warn("failed to write cache manifest when compiling '{}': {}", .{ c_object.src_path, @errorName(err) }); }; - break :blk full_object_path; - } else blk: { - const digest = ch.final(); - const full_object_path = if (comp.zig_cache_directory.path) |p| - try std.fs.path.join(arena, &[_][]const u8{ p, "o", &digest, o_basename }) - else - try std.fs.path.join(arena, &[_][]const u8{ "o", &digest, o_basename }); - break :blk full_object_path; + break :blk digest; }; + const full_object_path = if (comp.zig_cache_directory.path) |p| + try std.fs.path.join(comp.gpa, &[_][]const u8{ p, "o", &digest, o_basename }) + else + try std.fs.path.join(comp.gpa, &[_][]const u8{ "o", &digest, o_basename }); c_object.status = .{ .success = .{ .object_path = full_object_path, diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index 3577442121..23759ea6ad 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -619,11 +619,28 @@ fn build_libc_object(comp: *Compilation, basename: []const u8, c_source_file: Co .c_source_files = &[1]Compilation.CSourceFile{c_source_file}, .debug_cc = comp.debug_cc, .debug_link = comp.bin_file.options.debug_link, + .clang_passthrough_mode = comp.clang_passthrough_mode, }); defer sub_compilation.destroy(); try sub_compilation.update(); + // Look for compilation errors in this sub_compilation + var errors = try sub_compilation.getAllErrorsAlloc(); + defer errors.deinit(sub_compilation.gpa); + + if (errors.list.len != 0) { + for (errors.list) |full_err_msg| { + std.log.err("{}:{}:{}: error: {}\n", .{ + full_err_msg.src_path, + full_err_msg.line + 1, + full_err_msg.column + 1, + full_err_msg.msg, + }); + } + return error.BuildingLibCObjectFailed; + } + try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1); const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p| try std.fs.path.join(comp.gpa, &[_][]const u8{ p, basename }) diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 2131f44f3d..37ae1b6d69 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -1265,11 +1265,9 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { try ch.addOptionalFile(self.base.options.linker_script); try ch.addOptionalFile(self.base.options.version_script); try ch.addListOfFiles(self.base.options.objects); - for (comp.c_object_table.items()) |entry| switch (entry.key.status) { - .new => unreachable, - .failure => return error.NotAllCSourceFilesAvailableToLink, - .success => |success| _ = try ch.addFile(success.object_path, null), - }; + for (comp.c_object_table.items()) |entry| { + _ = try ch.addFile(entry.key.status.success.object_path, null); + } try ch.addOptionalFile(module_obj_path); // We can skip hashing libc and libc++ components that we are in charge of building from Zig // installation sources because they are always a product of the compiler version + target information. @@ -1494,11 +1492,9 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { // Positional arguments to the linker such as object files. try argv.appendSlice(self.base.options.objects); - for (comp.c_object_table.items()) |entry| switch (entry.key.status) { - .new => unreachable, - .failure => unreachable, // Checked during cache hashing. - .success => |success| try argv.append(success.object_path), - }; + for (comp.c_object_table.items()) |entry| { + try argv.append(entry.key.status.success.object_path); + } if (module_obj_path) |p| { try argv.append(p); diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index b7ec9ee712..918e92422a 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -228,12 +228,8 @@ const usage_build_generic = \\ \\Debug Options (Zig Compiler Development): \\ -ftime-report Print timing diagnostics - \\ --debug-tokenize verbose tokenization - \\ --debug-ast-tree verbose parsing into an AST (tree view) - \\ --debug-ast-fmt verbose parsing into an AST (render source) - \\ --debug-ir verbose Zig IR - \\ --debug-link verbose linking - \\ --debug-codegen verbose machine code generation + \\ --debug-link Verbose linker invocation + \\ --debug-cc Verbose C compiler invocation \\ ; @@ -272,12 +268,7 @@ pub fn buildOutputType( var strip = false; var single_threaded = false; var watch = false; - var debug_tokenize = false; - var debug_ast_tree = false; - var debug_ast_fmt = false; var debug_link = false; - var debug_ir = false; - var debug_codegen = false; var debug_cc = false; var time_report = false; var emit_bin: Emit = .yes_default_path; @@ -531,18 +522,8 @@ pub fn buildOutputType( link_eh_frame_hdr = true; } else if (mem.eql(u8, arg, "-Bsymbolic")) { linker_bind_global_refs_locally = true; - } else if (mem.eql(u8, arg, "--debug-tokenize")) { - debug_tokenize = true; - } else if (mem.eql(u8, arg, "--debug-ast-tree")) { - debug_ast_tree = true; - } else if (mem.eql(u8, arg, "--debug-ast-fmt")) { - debug_ast_fmt = true; } else if (mem.eql(u8, arg, "--debug-link")) { debug_link = true; - } else if (mem.eql(u8, arg, "--debug-ir")) { - debug_ir = true; - } else if (mem.eql(u8, arg, "--debug-codegen")) { - debug_codegen = true; } else if (mem.eql(u8, arg, "--debug-cc")) { debug_cc = true; } else if (mem.startsWith(u8, arg, "-T")) { @@ -1222,12 +1203,7 @@ fn updateModule(gpa: *Allocator, comp: *Compilation, zir_out_path: ?[]const u8) if (errors.list.len != 0) { for (errors.list) |full_err_msg| { - std.debug.print("{}:{}:{}: error: {}\n", .{ - full_err_msg.src_path, - full_err_msg.line + 1, - full_err_msg.column + 1, - full_err_msg.msg, - }); + full_err_msg.renderToStdErr(); } } From 2627778b251deff3426b5dc21a5a591e0c823b3b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2020 22:15:03 -0700 Subject: [PATCH 024/210] stage2: don't create empty object files when no zig source --- src-self-hosted/Compilation.zig | 9 ++-- src-self-hosted/link.zig | 34 +++++++------- src-self-hosted/link/C.zig | 4 +- src-self-hosted/link/Coff.zig | 80 +++++++++++---------------------- src-self-hosted/link/Elf.zig | 53 +++++++++++----------- src-self-hosted/link/MachO.zig | 63 +++++++------------------- src-self-hosted/link/Wasm.zig | 20 ++++++--- 7 files changed, 108 insertions(+), 155 deletions(-) diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index d8c0bec199..c4eeaf7354 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -1072,13 +1072,14 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { break :blk digest; }; - const full_object_path = if (comp.zig_cache_directory.path) |p| - try std.fs.path.join(comp.gpa, &[_][]const u8{ p, "o", &digest, o_basename }) + const components = if (comp.zig_cache_directory.path) |p| + &[_][]const u8{ p, "o", &digest, o_basename } else - try std.fs.path.join(comp.gpa, &[_][]const u8{ "o", &digest, o_basename }); + &[_][]const u8{ "o", &digest, o_basename }; + c_object.status = .{ .success = .{ - .object_path = full_object_path, + .object_path = try std.fs.path.join(comp.gpa, components), .lock = ch.toOwnedLock(), }, }; diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 4078b263e9..8158e7ee55 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -126,17 +126,29 @@ pub const File = struct { pub fn openPath(allocator: *Allocator, options: Options) !*File { const use_lld = build_options.have_llvm and options.use_lld; // comptime known false when !have_llvm const sub_path = if (use_lld) blk: { + if (options.module == null) { + // No point in opening a file, we would not write anything to it. Initialize with empty. + return switch (options.object_format) { + .coff, .pe => &(try Coff.createEmpty(allocator, options)).base, + .elf => &(try Elf.createEmpty(allocator, options)).base, + .macho => &(try MachO.createEmpty(allocator, options)).base, + .wasm => &(try Wasm.createEmpty(allocator, options)).base, + .c => unreachable, // Reported error earlier. + .hex => return error.HexObjectFormatUnimplemented, + .raw => return error.RawObjectFormatUnimplemented, + }; + } // Open a temporary object file, not the final output file because we want to link with LLD. break :blk try std.fmt.allocPrint(allocator, "{}{}", .{ options.sub_path, options.target.oFileExt() }); } else options.sub_path; errdefer if (use_lld) allocator.free(sub_path); const file: *File = switch (options.object_format) { - .coff, .pe => try Coff.openPath(allocator, sub_path, options), - .elf => try Elf.openPath(allocator, sub_path, options), - .macho => try MachO.openPath(allocator, sub_path, options), - .wasm => try Wasm.openPath(allocator, sub_path, options), - .c => try C.openPath(allocator, sub_path, options), + .coff, .pe => &(try Coff.openPath(allocator, sub_path, options)).base, + .elf => &(try Elf.openPath(allocator, sub_path, options)).base, + .macho => &(try MachO.openPath(allocator, sub_path, options)).base, + .wasm => &(try Wasm.openPath(allocator, sub_path, options)).base, + .c => &(try C.openPath(allocator, sub_path, options)).base, .hex => return error.HexObjectFormatUnimplemented, .raw => return error.RawObjectFormatUnimplemented, }; @@ -216,19 +228,9 @@ pub const File = struct { } } - pub fn deinit(base: *File) void { - switch (base.tag) { - .coff => @fieldParentPtr(Coff, "base", base).deinit(), - .elf => @fieldParentPtr(Elf, "base", base).deinit(), - .macho => @fieldParentPtr(MachO, "base", base).deinit(), - .c => @fieldParentPtr(C, "base", base).deinit(), - .wasm => @fieldParentPtr(Wasm, "base", base).deinit(), - } + pub fn destroy(base: *File) void { if (base.file) |f| f.close(); if (base.intermediary_basename) |sub_path| base.allocator.free(sub_path); - } - - pub fn destroy(base: *File) void { switch (base.tag) { .coff => { const parent = @fieldParentPtr(Coff, "base", base); diff --git a/src-self-hosted/link/C.zig b/src-self-hosted/link/C.zig index 54c244d3e5..f2c3d30a61 100644 --- a/src-self-hosted/link/C.zig +++ b/src-self-hosted/link/C.zig @@ -23,7 +23,7 @@ need_stddef: bool = false, need_stdint: bool = false, error_msg: *Compilation.ErrorMsg = undefined, -pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*File { +pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*C { assert(options.object_format == .c); if (options.use_llvm) return error.LLVMHasNoCBackend; @@ -48,7 +48,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio .called = std.StringHashMap(void).init(allocator), }; - return &c_file.base; + return c_file; } pub fn fail(self: *C, src: usize, comptime format: []const u8, args: anytype) error{ AnalysisFail, OutOfMemory } { diff --git a/src-self-hosted/link/Coff.zig b/src-self-hosted/link/Coff.zig index 308c9c2332..ffec465155 100644 --- a/src-self-hosted/link/Coff.zig +++ b/src-self-hosted/link/Coff.zig @@ -28,7 +28,7 @@ pub const base_tag: link.File.Tag = .coff; const msdos_stub = @embedFile("msdos-stub.bin"); base: link.File, -ptr_width: enum { p32, p64 }, +ptr_width: PtrWidth, error_flags: link.File.ErrorFlags = .{}, text_block_free_list: std.ArrayListUnmanaged(*TextBlock) = .{}, @@ -64,6 +64,8 @@ text_section_size_dirty: bool = false, /// and needs to be updated in the optional header. size_of_image_dirty: bool = false, +pub const PtrWidth = enum { p32, p64 }; + pub const TextBlock = struct { /// Offset of the code relative to the start of the text section text_offset: u32, @@ -111,7 +113,7 @@ pub const TextBlock = struct { pub const SrcFn = void; -pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*link.File { +pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*Coff { assert(options.object_format == .coff); if (options.use_llvm) return error.LLVM_BackendIsTODO_ForCoff; // TODO @@ -124,66 +126,17 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio }); errdefer file.close(); - var coff_file = try allocator.create(Coff); - errdefer allocator.destroy(coff_file); + const self = try createEmpty(allocator, options); + errdefer self.base.destroy(); - coff_file.* = openFile(allocator, file, options) catch |err| switch (err) { - error.IncrFailed => try createFile(allocator, file, options), - else => |e| return e, - }; + self.base.file = file; - return &coff_file.base; -} - -/// Returns error.IncrFailed if incremental update could not be performed. -fn openFile(allocator: *Allocator, file: fs.File, options: link.Options) !Coff { - switch (options.output_mode) { - .Exe => {}, - .Obj => return error.IncrFailed, - .Lib => return error.IncrFailed, - } - var self: Coff = .{ - .base = .{ - .file = file, - .tag = .coff, - .options = options, - .allocator = allocator, - }, - .ptr_width = switch (options.target.cpu.arch.ptrBitWidth()) { - 32 => .p32, - 64 => .p64, - else => return error.UnsupportedELFArchitecture, - }, - }; - errdefer self.deinit(); - - // TODO implement reading the PE/COFF file - return error.IncrFailed; -} - -/// Truncates the existing file contents and overwrites the contents. -/// Returns an error if `file` is not already open with +read +write +seek abilities. -fn createFile(allocator: *Allocator, file: fs.File, options: link.Options) !Coff { // TODO Write object specific relocations, COFF symbol table, then enable object file output. switch (options.output_mode) { .Exe => {}, .Obj => return error.TODOImplementWritingObjFiles, .Lib => return error.TODOImplementWritingLibFiles, } - var self: Coff = .{ - .base = .{ - .tag = .coff, - .options = options, - .allocator = allocator, - .file = file, - }, - .ptr_width = switch (options.target.cpu.arch.ptrBitWidth()) { - 32 => .p32, - 64 => .p64, - else => return error.UnsupportedCOFFArchitecture, - }, - }; - errdefer self.deinit(); var coff_file_header_offset: u32 = 0; if (options.output_mode == .Exe) { @@ -426,6 +379,25 @@ fn createFile(allocator: *Allocator, file: fs.File, options: link.Options) !Coff return self; } +pub fn createEmpty(gpa: *Allocator, options: link.Options) !*Coff { + const ptr_width: PtrWidth = switch (options.target.cpu.arch.ptrBitWidth()) { + 0...32 => .p32, + 33...64 => .p64, + else => return error.UnsupportedCOFFArchitecture, + }; + const self = try gpa.create(Coff); + self.* = .{ + .base = .{ + .tag = .coff, + .options = options, + .allocator = gpa, + .file = null, + }, + .ptr_width = ptr_width, + }; + return self; +} + pub fn allocateDeclIndexes(self: *Coff, decl: *Module.Decl) !void { try self.offset_table.ensureCapacity(self.base.allocator, self.offset_table.items.len + 1); diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 37ae1b6d69..d419cbc297 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -31,7 +31,7 @@ pub const base_tag: File.Tag = .elf; base: File, -ptr_width: enum { p32, p64 }, +ptr_width: PtrWidth, /// Stored in native-endian format, depending on target endianness needs to be bswapped on read/write. /// Same order as in the file. @@ -136,6 +136,8 @@ const alloc_den = 3; const minimum_text_block_size = 64; const min_text_capacity = minimum_text_block_size * alloc_num / alloc_den; +pub const PtrWidth = enum { p32, p64 }; + pub const TextBlock = struct { /// Each decl always gets a local symbol with the fully qualified name. /// The vaddr and size are found here directly. @@ -222,7 +224,7 @@ pub const SrcFn = struct { }; }; -pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*File { +pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*Elf { assert(options.object_format == .elf); if (options.use_llvm) return error.LLVMBackendUnimplementedForELF; // TODO @@ -234,31 +236,11 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio }); errdefer file.close(); - var elf_file = try allocator.create(Elf); - errdefer allocator.destroy(elf_file); + const self = try createEmpty(allocator, options); + errdefer self.base.destroy(); - elf_file.* = try createFile(allocator, file, options); - return &elf_file.base; -} - -/// Truncates the existing file contents and overwrites the contents. -/// Returns an error if `file` is not already open with +read +write +seek abilities. -fn createFile(allocator: *Allocator, file: fs.File, options: link.Options) !Elf { - var self: Elf = .{ - .base = .{ - .tag = .elf, - .options = options, - .allocator = allocator, - .file = file, - }, - .ptr_width = switch (options.target.cpu.arch.ptrBitWidth()) { - 0 ... 32 => .p32, - 33 ... 64 => .p64, - else => return error.UnsupportedELFArchitecture, - }, - .shdr_table_dirty = true, - }; - errdefer self.deinit(); + self.base.file = file; + self.shdr_table_dirty = true; // Index 0 is always a null symbol. try self.local_symbols.append(allocator, .{ @@ -289,6 +271,25 @@ fn createFile(allocator: *Allocator, file: fs.File, options: link.Options) !Elf return self; } +pub fn createEmpty(gpa: *Allocator, options: link.Options) !*Elf { + const ptr_width: PtrWidth = switch (options.target.cpu.arch.ptrBitWidth()) { + 0 ... 32 => .p32, + 33 ... 64 => .p64, + else => return error.UnsupportedELFArchitecture, + }; + const self = try gpa.create(Elf); + self.* = .{ + .base = .{ + .tag = .elf, + .options = options, + .allocator = gpa, + .file = null, + }, + .ptr_width = ptr_width, + }; + return self; +} + pub fn releaseLock(self: *Elf) void { if (self.lock) |*lock| { lock.release(); diff --git a/src-self-hosted/link/MachO.zig b/src-self-hosted/link/MachO.zig index 0c7e3ce2ed..08a948ed0e 100644 --- a/src-self-hosted/link/MachO.zig +++ b/src-self-hosted/link/MachO.zig @@ -135,7 +135,7 @@ pub const SrcFn = struct { pub const empty = SrcFn{}; }; -pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*File { +pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*MachO { assert(options.object_format == .macho); if (options.use_llvm) return error.LLVM_BackendIsTODO_ForMachO; // TODO @@ -148,64 +148,35 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio }); errdefer file.close(); - var macho_file = try allocator.create(MachO); - errdefer allocator.destroy(macho_file); + const self = try createEmpty(allocator, options); + errdefer self.base.destroy(); - macho_file.* = openFile(allocator, file, options) catch |err| switch (err) { - error.IncrFailed => try createFile(allocator, file, options), - else => |e| return e, - }; + self.base.file = file; - return &macho_file.base; -} - -/// Returns error.IncrFailed if incremental update could not be performed. -fn openFile(allocator: *Allocator, file: fs.File, options: link.Options) !MachO { - switch (options.output_mode) { - .Exe => {}, - .Obj => {}, - .Lib => return error.IncrFailed, - } - var self: MachO = .{ - .base = .{ - .file = file, - .tag = .macho, - .options = options, - .allocator = allocator, - }, - }; - errdefer self.deinit(); - - // TODO implement reading the macho file - return error.IncrFailed; - //try self.populateMissingMetadata(); - //return self; -} - -/// Truncates the existing file contents and overwrites the contents. -/// Returns an error if `file` is not already open with +read +write +seek abilities. -fn createFile(allocator: *Allocator, file: fs.File, options: link.Options) !MachO { switch (options.output_mode) { .Exe => {}, .Obj => {}, .Lib => return error.TODOImplementWritingLibFiles, } - var self: MachO = .{ - .base = .{ - .file = file, - .tag = .macho, - .options = options, - .allocator = allocator, - }, - }; - errdefer self.deinit(); - try self.populateMissingMetadata(); return self; } +pub fn createEmpty(gpa: *Allocator, options: link.Options) !*MachO { + const self = try gpa.create(MachO); + self.* = .{ + .base = .{ + .tag = .macho, + .options = options, + .allocator = gpa, + .file = null, + }, + }; + return self; +} + pub fn flush(self: *MachO, comp: *Compilation) !void { switch (self.base.options.output_mode) { .Exe => { diff --git a/src-self-hosted/link/Wasm.zig b/src-self-hosted/link/Wasm.zig index 25e623bebb..2351c6a3ea 100644 --- a/src-self-hosted/link/Wasm.zig +++ b/src-self-hosted/link/Wasm.zig @@ -50,7 +50,7 @@ base: link.File, /// TODO: can/should we access some data structure in Module directly? funcs: std.ArrayListUnmanaged(*Module.Decl) = .{}, -pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*link.File { +pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*Wasm { assert(options.object_format == .wasm); if (options.use_llvm) return error.LLVM_BackendIsTODO_ForWasm; // TODO @@ -60,21 +60,27 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio const file = try options.directory.handle.createFile(sub_path, .{ .truncate = true, .read = true }); errdefer file.close(); - const wasm = try allocator.create(Wasm); - errdefer allocator.destroy(wasm); + const wasm = try createEmpty(allocator, options); + errdefer wasm.base.destroy(); + + wasm.base.file = file; try file.writeAll(&(spec.magic ++ spec.version)); + return wasm; +} + +pub fn createEmpty(gpa: *Allocator, options: link.Options) !*Wasm { + const wasm = try gpa.create(Wasm); wasm.* = .{ .base = .{ .tag = .wasm, .options = options, - .file = file, - .allocator = allocator, + .file = null, + .allocator = gpa, }, }; - - return &wasm.base; + return wasm; } pub fn deinit(self: *Wasm) void { From 97ea5da18d0a26d55447082d37f6a0945da42d1f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2020 22:38:50 -0700 Subject: [PATCH 025/210] stage2: fix bad include path for glibc abi-note.S --- src-self-hosted/glibc.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index 23759ea6ad..0f7fabdcac 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -323,7 +323,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { var args = std.ArrayList([]const u8).init(arena); try args.appendSlice(&[_][]const u8{ "-I", - try lib_path(comp, arena, lib_libc_glibc ++ "glibc" ++ path.sep_str ++ "csu"), + try lib_path(comp, arena, lib_libc_glibc ++ "csu"), }); try add_include_dirs(comp, arena, &args); try args.appendSlice(&[_][]const u8{ From e5aef96293336fa25e6f094bf82d189176e8d1a7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2020 22:54:16 -0700 Subject: [PATCH 026/210] stage2: CRT files retain locks on the build artifacts --- src-self-hosted/Compilation.zig | 17 ++++++++++++++--- src-self-hosted/glibc.zig | 5 ++++- src-self-hosted/link.zig | 18 ++++++++++++++++++ src-self-hosted/link/Elf.zig | 27 ++++++--------------------- 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index c4eeaf7354..a3a99aee6f 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -70,13 +70,24 @@ libc_static_lib: ?[]const u8 = null, /// For example `Scrt1.o` and `libc.so.6`. These are populated after building libc from source, /// The set of needed CRT (C runtime) files differs depending on the target and compilation settings. /// The key is the basename, and the value is the absolute path to the completed build artifact. -crt_files: std.StringHashMapUnmanaged([]const u8) = .{}, +crt_files: std.StringHashMapUnmanaged(CRTFile) = .{}, /// Keeping track of this possibly open resource so we can close it later. owned_link_dir: ?std.fs.Dir, pub const InnerError = Module.InnerError; +pub const CRTFile = struct { + lock: std.cache_hash.Lock, + full_object_path: []const u8, + + fn deinit(self: *CRTFile, gpa: *Allocator) void { + self.lock.release(); + gpa.free(self.full_object_path); + self.* = undefined; + } +}; + /// For passing to a C compiler. pub const CSourceFile = struct { src_path: []const u8, @@ -625,7 +636,7 @@ pub fn destroy(self: *Compilation) void { var it = self.crt_files.iterator(); while (it.next()) |entry| { gpa.free(entry.key); - gpa.free(entry.value); + entry.value.deinit(gpa); } self.crt_files.deinit(gpa); } @@ -1512,7 +1523,7 @@ fn detectLibCFromLibCInstallation(arena: *Allocator, target: Target, lci: *const pub fn get_libc_crt_file(comp: *Compilation, arena: *Allocator, basename: []const u8) ![]const u8 { if (comp.wantBuildGLibCFromSource()) { - return comp.crt_files.get(basename).?; + return comp.crt_files.get(basename).?.full_object_path; } const lci = comp.bin_file.options.libc_installation orelse return error.LibCInstallationNotAvailable; const crt_dir_path = lci.crt_dir orelse return error.LibCInstallationMissingCRTDir; diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index 0f7fabdcac..cd25baef8c 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -648,5 +648,8 @@ fn build_libc_object(comp: *Compilation, basename: []const u8, c_source_file: Co try comp.gpa.dupe(u8, basename); // TODO obtain a lock on the artifact and put that in crt_files as well. - comp.crt_files.putAssumeCapacityNoClobber(basename, artifact_path); + comp.crt_files.putAssumeCapacityNoClobber(basename, .{ + .full_object_path = artifact_path, + .lock = sub_compilation.bin_file.toOwnedLock(), + }); } diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 8158e7ee55..f75274eb58 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -90,6 +90,10 @@ pub const File = struct { /// this location, and then this path can be placed on the LLD linker line. intermediary_basename: ?[]const u8 = null, + /// Prevents other processes from clobbering files in the output directory + /// of this linking operation. + lock: ?std.cache_hash.Lock = null, + pub const LinkBlock = union { elf: Elf.TextBlock, coff: Coff.TextBlock, @@ -228,7 +232,21 @@ pub const File = struct { } } + pub fn releaseLock(self: *File) void { + if (self.lock) |*lock| { + lock.release(); + self.lock = null; + } + } + + pub fn toOwnedLock(self: *File) std.cache_hash.Lock { + const lock = self.lock.?; + self.lock = null; + return lock; + } + pub fn destroy(base: *File) void { + base.releaseLock(); if (base.file) |f| f.close(); if (base.intermediary_basename) |sub_path| base.allocator.free(sub_path); switch (base.tag) { diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index d419cbc297..e3a28a5fd4 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -123,9 +123,6 @@ dbg_info_decl_free_list: std.AutoHashMapUnmanaged(*TextBlock, void) = .{}, dbg_info_decl_first: ?*TextBlock = null, dbg_info_decl_last: ?*TextBlock = null, -/// Prevents other processes from clobbering the output file this is linking. -lock: ?std.cache_hash.Lock = null, - /// `alloc_num / alloc_den` is the factor of padding when allocating. const alloc_num = 4; const alloc_den = 3; @@ -290,21 +287,7 @@ pub fn createEmpty(gpa: *Allocator, options: link.Options) !*Elf { return self; } -pub fn releaseLock(self: *Elf) void { - if (self.lock) |*lock| { - lock.release(); - self.lock = null; - } -} - -pub fn toOwnedLock(self: *Elf) std.cache_hash.Lock { - const lock = self.lock.?; - self.lock = null; - return lock; -} - pub fn deinit(self: *Elf) void { - self.releaseLock(); self.sections.deinit(self.base.allocator); self.program_headers.deinit(self.base.allocator); self.shstrtab.deinit(self.base.allocator); @@ -1253,7 +1236,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { const id_symlink_basename = "id.txt"; // We are about to obtain this lock, so here we give other processes a chance first. - self.releaseLock(); + self.base.releaseLock(); var ch = comp.cache_parent.obtain(); defer ch.deinit(); @@ -1302,14 +1285,16 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { const digest = ch.final(); var prev_digest_buf: [digest.len]u8 = undefined; - const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch blk: { + const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { + log.debug("ELF LLD new_digest={} readlink error: {}", .{digest, @errorName(err)}); // Handle this as a cache miss. mem.set(u8, &prev_digest_buf, 0); break :blk &prev_digest_buf; }; + log.debug("ELF LLD prev_digest={} new_digest={}", .{prev_digest, digest}); if (mem.eql(u8, prev_digest, &digest)) { // Hot diggity dog! The output binary is already there. - self.lock = ch.toOwnedLock(); + self.base.lock = ch.toOwnedLock(); return; } @@ -1611,7 +1596,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { }; // We hang on to this lock so that the output file path can be used without // other processes clobbering it. - self.lock = ch.toOwnedLock(); + self.base.lock = ch.toOwnedLock(); } fn append_diagnostic(context: usize, ptr: [*]const u8, len: usize) callconv(.C) void { From 0379d7b4317e7e12d27b66f8a44a443d328ce539 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2020 23:04:15 -0700 Subject: [PATCH 027/210] stage2: don't bother building glibc when only doing build-obj --- src-self-hosted/Compilation.zig | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index a3a99aee6f..82d8b0d5a0 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -1548,7 +1548,12 @@ fn addBuildingGLibCWorkItems(comp: *Compilation) !void { } fn wantBuildGLibCFromSource(comp: *Compilation) bool { - return comp.bin_file.options.link_libc and + const is_exe_or_dyn_lib = switch (comp.bin_file.options.output_mode) { + .Obj => false, + .Lib => comp.bin_file.options.link_mode == .Dynamic, + .Exe => true, + }; + return comp.bin_file.options.link_libc and is_exe_or_dyn_lib and comp.bin_file.options.libc_installation == null and comp.bin_file.options.target.isGnuLibC(); } From 060c91b97f25e22781f4b9e7c90bd0642594b722 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 13 Sep 2020 23:28:28 -0700 Subject: [PATCH 028/210] stage2: namespace cache dir with C source path This is not strictly necessary but it increases the likelihood of cache hits because foo.c and bar.c now will have different cache directories and can be updated independently without clobbering each other's cache data. --- src-self-hosted/Compilation.zig | 19 ++++++++++++++++++- src-self-hosted/link/Elf.zig | 6 +++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 82d8b0d5a0..4066fc8cf1 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -427,6 +427,13 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { // modified between incremental updates. var hash = cache.hash; + // Here we put the root source file path name, but *not* with addFile. We want the + // hash to be the same regardless of the contents of the source file, because + // incremental compilation will handle it, but we do want to namespace different + // source file names because they are likely different compilations and therefore this + // would be likely to cause cache hits. + hash.addBytes(root_pkg.root_src_path); + hash.addOptionalBytes(root_pkg.root_src_directory.path); hash.add(valgrind); hash.add(single_threaded); switch (options.target.os.getVersionRange()) { @@ -512,7 +519,17 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { const bin_directory = emit_bin.directory orelse blk: { if (module) |zm| break :blk zm.zig_cache_artifact_directory; - const digest = cache.hash.peek(); + // We could use the cache hash as is no problem, however, we increase + // the likelihood of cache hits by adding the first C source file + // path name (not contents) to the hash. This way if the user is compiling + // foo.c and bar.c as separate compilations, they get different cache + // directories. + var hash = cache.hash; + if (options.c_source_files.len >= 1) { + hash.addBytes(options.c_source_files[0].src_path); + } + + const digest = hash.final(); const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); var artifact_dir = try options.zig_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); owned_link_dir = artifact_dir; diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index e3a28a5fd4..3b016e3c8e 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -1288,15 +1288,15 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { log.debug("ELF LLD new_digest={} readlink error: {}", .{digest, @errorName(err)}); // Handle this as a cache miss. - mem.set(u8, &prev_digest_buf, 0); - break :blk &prev_digest_buf; + break :blk prev_digest_buf[0..0]; }; - log.debug("ELF LLD prev_digest={} new_digest={}", .{prev_digest, digest}); if (mem.eql(u8, prev_digest, &digest)) { + log.debug("ELF LLD digest={} match - skipping invocation", .{digest}); // Hot diggity dog! The output binary is already there. self.base.lock = ch.toOwnedLock(); return; } + log.debug("ELF LLD prev_digest={} new_digest={}", .{prev_digest, digest}); // We are about to change the output file to be different, so we invalidate the build hash now. directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) { From f2e380380e034a3f1a3e35857220edd78439a949 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 00:24:03 -0700 Subject: [PATCH 029/210] stage2: building glibc Scrt1.o --- src-self-hosted/Compilation.zig | 2 - src-self-hosted/glibc.zig | 120 ++++++++++++++++---------------- 2 files changed, 61 insertions(+), 61 deletions(-) diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 4066fc8cf1..abd12bf370 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -1552,8 +1552,6 @@ fn addBuildingGLibCWorkItems(comp: *Compilation) !void { const static_file_work_items = [_]WorkItem{ .{ .glibc_crt_file = .crti_o }, .{ .glibc_crt_file = .crtn_o }, - .{ .glibc_crt_file = .start_os }, - .{ .glibc_crt_file = .abi_note_o }, .{ .glibc_crt_file = .scrt1_o }, .{ .glibc_crt_file = .libc_nonshared_a }, }; diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index cd25baef8c..29d96bb375 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -240,8 +240,6 @@ fn findLib(name: []const u8) ?*const Lib { pub const CRTFile = enum { crti_o, crtn_o, - start_os, - abi_note_o, scrt1_o, libc_nonshared_a, }; @@ -272,11 +270,12 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-g", "-Wa,--noexecstack", }); - const c_source_file: Compilation.CSourceFile = .{ - .src_path = try start_asm_path(comp, arena, "crti.S"), - .extra_flags = args.items, - }; - return build_libc_object(comp, "crti.o", c_source_file); + return build_libc_object(comp, "crti.o", &[1]Compilation.CSourceFile{ + .{ + .src_path = try start_asm_path(comp, arena, "crti.S"), + .extra_flags = args.items, + }, + }); }, .crtn_o => { var args = std.ArrayList([]const u8).init(arena); @@ -289,59 +288,58 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-g", "-Wa,--noexecstack", }); - const c_source_file: Compilation.CSourceFile = .{ - .src_path = try start_asm_path(comp, arena, "crtn.S"), - .extra_flags = args.items, - }; - return build_libc_object(comp, "crtn.o", c_source_file); - }, - .start_os => { - var args = std.ArrayList([]const u8).init(arena); - try add_include_dirs(comp, arena, &args); - try args.appendSlice(&[_][]const u8{ - "-D_LIBC_REENTRANT", - "-include", - try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-modules.h"), - "-DMODULE_NAME=libc", - "-Wno-nonportable-include-path", - "-include", - try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-symbols.h"), - "-DPIC", - "-DSHARED", - "-DTOP_NAMESPACE=glibc", - "-DASSEMBLER", - "-g", - "-Wa,--noexecstack", + return build_libc_object(comp, "crtn.o", &[1]Compilation.CSourceFile{ + .{ + .src_path = try start_asm_path(comp, arena, "crtn.S"), + .extra_flags = args.items, + }, }); - const c_source_file: Compilation.CSourceFile = .{ - .src_path = try start_asm_path(comp, arena, "start.S"), - .extra_flags = args.items, - }; - return build_libc_object(comp, "start.os", c_source_file); - }, - .abi_note_o => { - var args = std.ArrayList([]const u8).init(arena); - try args.appendSlice(&[_][]const u8{ - "-I", - try lib_path(comp, arena, lib_libc_glibc ++ "csu"), - }); - try add_include_dirs(comp, arena, &args); - try args.appendSlice(&[_][]const u8{ - "-D_LIBC_REENTRANT", - "-DMODULE_NAME=libc", - "-DTOP_NAMESPACE=glibc", - "-DASSEMBLER", - "-g", - "-Wa,--noexecstack", - }); - const c_source_file: Compilation.CSourceFile = .{ - .src_path = try lib_path(comp, arena, lib_libc_glibc ++ "csu" ++ path.sep_str ++ "abi-note.S"), - .extra_flags = args.items, - }; - return build_libc_object(comp, "abi-note.o", c_source_file); }, .scrt1_o => { - return error.Unimplemented; // TODO + const start_os: Compilation.CSourceFile = blk: { + var args = std.ArrayList([]const u8).init(arena); + try add_include_dirs(comp, arena, &args); + try args.appendSlice(&[_][]const u8{ + "-D_LIBC_REENTRANT", + "-include", + try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-modules.h"), + "-DMODULE_NAME=libc", + "-Wno-nonportable-include-path", + "-include", + try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-symbols.h"), + "-DPIC", + "-DSHARED", + "-DTOP_NAMESPACE=glibc", + "-DASSEMBLER", + "-g", + "-Wa,--noexecstack", + }); + break :blk .{ + .src_path = try start_asm_path(comp, arena, "start.S"), + .extra_flags = args.items, + }; + }; + const abi_note_o: Compilation.CSourceFile = blk: { + var args = std.ArrayList([]const u8).init(arena); + try args.appendSlice(&[_][]const u8{ + "-I", + try lib_path(comp, arena, lib_libc_glibc ++ "csu"), + }); + try add_include_dirs(comp, arena, &args); + try args.appendSlice(&[_][]const u8{ + "-D_LIBC_REENTRANT", + "-DMODULE_NAME=libc", + "-DTOP_NAMESPACE=glibc", + "-DASSEMBLER", + "-g", + "-Wa,--noexecstack", + }); + break :blk .{ + .src_path = try lib_path(comp, arena, lib_libc_glibc ++ "csu" ++ path.sep_str ++ "abi-note.S"), + .extra_flags = args.items, + }; + }; + return build_libc_object(comp, "Scrt1.o", &[_]Compilation.CSourceFile{ start_os, abi_note_o }); }, .libc_nonshared_a => { return error.Unimplemented; // TODO @@ -587,7 +585,11 @@ fn lib_path(comp: *Compilation, arena: *Allocator, sub_path: []const u8) ![]cons return path.join(arena, &[_][]const u8{ comp.zig_lib_directory.path.?, sub_path }); } -fn build_libc_object(comp: *Compilation, basename: []const u8, c_source_file: Compilation.CSourceFile) !void { +fn build_libc_object( + comp: *Compilation, + basename: []const u8, + c_source_files: []const Compilation.CSourceFile, +) !void { const tracy = trace(@src()); defer tracy.end(); @@ -616,7 +618,7 @@ fn build_libc_object(comp: *Compilation, basename: []const u8, c_source_file: Co .strip = comp.bin_file.options.strip, .is_native_os = comp.bin_file.options.is_native_os, .self_exe_path = comp.self_exe_path, - .c_source_files = &[1]Compilation.CSourceFile{c_source_file}, + .c_source_files = c_source_files, .debug_cc = comp.debug_cc, .debug_link = comp.bin_file.options.debug_link, .clang_passthrough_mode = comp.clang_passthrough_mode, From c58e9951effddacfd5cef6c05019bc3f312747f8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 10:31:34 -0700 Subject: [PATCH 030/210] revert bogus `enum` keywords in zig_llvm.h --- src/zig_llvm.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zig_llvm.h b/src/zig_llvm.h index 13c4de0535..4de048a3d8 100644 --- a/src/zig_llvm.h +++ b/src/zig_llvm.h @@ -59,8 +59,8 @@ enum ZigLLVMABIType { }; ZIG_EXTERN_C LLVMTargetMachineRef ZigLLVMCreateTargetMachine(LLVMTargetRef T, const char *Triple, - const char *CPU, const char *Features, enum LLVMCodeGenOptLevel Level, enum LLVMRelocMode Reloc, - enum LLVMCodeModel CodeModel, bool function_sections, enum ZigLLVMABIType float_abi, const char *abi_name); + const char *CPU, const char *Features, LLVMCodeGenOptLevel Level, LLVMRelocMode Reloc, + LLVMCodeModel CodeModel, bool function_sections, enum ZigLLVMABIType float_abi, const char *abi_name); ZIG_EXTERN_C LLVMTypeRef ZigLLVMTokenTypeInContext(LLVMContextRef context_ref); From 04f6a26955bbb947983cbf3e0681d9092aaaa13f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 10:42:29 -0700 Subject: [PATCH 031/210] fix stage1 regressions in this branch also prepare for supporting linking into archives --- src-self-hosted/Compilation.zig | 9 +++++++- src-self-hosted/link.zig | 39 ++++++++++++++++++++------------- src-self-hosted/link/C.zig | 4 ++++ src-self-hosted/link/Coff.zig | 3 +++ src-self-hosted/link/Elf.zig | 17 +++++++------- src-self-hosted/link/MachO.zig | 3 +++ src-self-hosted/link/Wasm.zig | 4 ++++ src-self-hosted/llvm.zig | 11 ++++++++-- src/codegen.cpp | 2 +- 9 files changed, 64 insertions(+), 28 deletions(-) diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index abd12bf370..1f42c87849 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -305,6 +305,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { options.system_libs.len != 0 or options.link_libc or options.link_libcpp or options.link_eh_frame_hdr or + options.output_mode == .Lib or + options.lld_argv.len != 0 or options.linker_script != null or options.version_script != null) { break :blk true; @@ -320,12 +322,17 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { break :blk false; }; + const is_exe_or_dyn_lib = switch (options.output_mode) { + .Obj => false, + .Lib => (options.link_mode orelse .Static) == .Dynamic, + .Exe => true, + }; const must_dynamic_link = dl: { if (target_util.cannotDynamicLink(options.target)) break :dl false; if (target_util.osRequiresLibC(options.target)) break :dl true; - if (options.link_libc and options.target.isGnuLibC()) + if (is_exe_or_dyn_lib and options.link_libc and options.target.isGnuLibC()) break :dl true; if (options.system_libs.len != 0) break :dl true; diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index f75274eb58..832015faae 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -279,16 +279,19 @@ pub const File = struct { } pub fn flush(base: *File, comp: *Compilation) !void { - const tracy = trace(@src()); - defer tracy.end(); - - try switch (base.tag) { - .coff => @fieldParentPtr(Coff, "base", base).flush(comp), - .elf => @fieldParentPtr(Elf, "base", base).flush(comp), - .macho => @fieldParentPtr(MachO, "base", base).flush(comp), - .c => @fieldParentPtr(C, "base", base).flush(comp), - .wasm => @fieldParentPtr(Wasm, "base", base).flush(comp), - }; + const use_lld = build_options.have_llvm and base.options.use_lld; + if (base.options.output_mode == .Lib and base.options.link_mode == .Static and + !base.options.target.isWasm()) + { + return base.linkAsArchive(comp); + } + switch (base.tag) { + .coff => return @fieldParentPtr(Coff, "base", base).flush(comp), + .elf => return @fieldParentPtr(Elf, "base", base).flush(comp), + .macho => return @fieldParentPtr(MachO, "base", base).flush(comp), + .c => return @fieldParentPtr(C, "base", base).flush(comp), + .wasm => return @fieldParentPtr(Wasm, "base", base).flush(comp), + } } pub fn freeDecl(base: *File, decl: *Module.Decl) void { @@ -302,13 +305,13 @@ pub const File = struct { } pub fn errorFlags(base: *File) ErrorFlags { - return switch (base.tag) { - .coff => @fieldParentPtr(Coff, "base", base).error_flags, - .elf => @fieldParentPtr(Elf, "base", base).error_flags, - .macho => @fieldParentPtr(MachO, "base", base).error_flags, + switch (base.tag) { + .coff => return @fieldParentPtr(Coff, "base", base).error_flags, + .elf => return @fieldParentPtr(Elf, "base", base).error_flags, + .macho => return @fieldParentPtr(MachO, "base", base).error_flags, .c => return .{ .no_entry_point_found = false }, .wasm => return ErrorFlags{}, - }; + } } /// May be called before or after updateDecl, but must be called after @@ -338,6 +341,12 @@ pub const File = struct { } } + fn linkAsArchive(base: *File, comp: *Compilation) !void { + // TODO follow pattern from ELF linkWithLLD + // ZigLLVMWriteArchive + return error.TODOMakeArchive; + } + pub const Tag = enum { coff, elf, diff --git a/src-self-hosted/link/C.zig b/src-self-hosted/link/C.zig index f2c3d30a61..97990f34fb 100644 --- a/src-self-hosted/link/C.zig +++ b/src-self-hosted/link/C.zig @@ -7,6 +7,7 @@ const Compilation = @import("../Compilation.zig"); const fs = std.fs; const codegen = @import("../codegen/c.zig"); const link = @import("../link.zig"); +const trace = @import("../tracy.zig").trace; const File = link.File; const C = @This(); @@ -73,6 +74,9 @@ pub fn updateDecl(self: *C, module: *Module, decl: *Module.Decl) !void { } pub fn flush(self: *C, comp: *Compilation) !void { + const tracy = trace(@src()); + defer tracy.end(); + const writer = self.base.file.?.writer(); try writer.writeAll(@embedFile("cbe.h")); var includes = false; diff --git a/src-self-hosted/link/Coff.zig b/src-self-hosted/link/Coff.zig index ffec465155..b927ca2586 100644 --- a/src-self-hosted/link/Coff.zig +++ b/src-self-hosted/link/Coff.zig @@ -724,6 +724,9 @@ pub fn updateDeclExports(self: *Coff, module: *Module, decl: *const Module.Decl, } pub fn flush(self: *Coff, comp: *Compilation) !void { + const tracy = trace(@src()); + defer tracy.end(); + if (self.text_section_size_dirty) { // Write the new raw size in the .text header var buf: [4]u8 = undefined; diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 3b016e3c8e..49cf02d7c4 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -725,6 +725,9 @@ pub fn flush(self: *Elf, comp: *Compilation) !void { /// Commit pending changes and write headers. fn flushInner(self: *Elf, comp: *Compilation) !void { + const tracy = trace(@src()); + defer tracy.end(); + // TODO This linker code currently assumes there is only 1 compilation unit and it corresponds to the // Zig source code. const module = self.base.options.module orelse return error.LinkingWithoutZigSourceUnimplemented; @@ -1206,6 +1209,9 @@ fn flushInner(self: *Elf, comp: *Compilation) !void { } fn linkWithLLD(self: *Elf, comp: *Compilation) !void { + const tracy = trace(@src()); + defer tracy.end(); + var arena_allocator = std.heap.ArenaAllocator.init(self.base.allocator); defer arena_allocator.deinit(); const arena = &arena_allocator.allocator; @@ -1315,13 +1321,6 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { if (is_obj) { try argv.append("-r"); } - if (self.base.options.output_mode == .Lib and - self.base.options.link_mode == .Static and - !target.isWasm()) - { - // TODO port the code from link.cpp - return error.TODOMakeArchive; - } const link_in_crt = self.base.options.link_libc and self.base.options.output_mode == .Exe; try argv.append("-error-limit=0"); @@ -1581,8 +1580,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { new_argv[i] = try arena.dupeZ(u8, arg); } - const ZigLLDLink = @import("../llvm.zig").ZigLLDLink; - const ok = ZigLLDLink(.ELF, new_argv.ptr, new_argv.len, append_diagnostic, 0, 0); + const llvm = @import("../llvm.zig"); + const ok = llvm.Link(.ELF, new_argv.ptr, new_argv.len, append_diagnostic, 0, 0); if (!ok) return error.LLDReportedFailure; // Update the dangling symlink "id.txt" with the digest. If it fails we can continue; it only diff --git a/src-self-hosted/link/MachO.zig b/src-self-hosted/link/MachO.zig index 08a948ed0e..ff3fa2810d 100644 --- a/src-self-hosted/link/MachO.zig +++ b/src-self-hosted/link/MachO.zig @@ -178,6 +178,9 @@ pub fn createEmpty(gpa: *Allocator, options: link.Options) !*MachO { } pub fn flush(self: *MachO, comp: *Compilation) !void { + const tracy = trace(@src()); + defer tracy.end(); + switch (self.base.options.output_mode) { .Exe => { var last_cmd_offset: usize = @sizeOf(macho.mach_header_64); diff --git a/src-self-hosted/link/Wasm.zig b/src-self-hosted/link/Wasm.zig index 2351c6a3ea..b4bb2c3195 100644 --- a/src-self-hosted/link/Wasm.zig +++ b/src-self-hosted/link/Wasm.zig @@ -10,6 +10,7 @@ const Module = @import("../Module.zig"); const Compilation = @import("../Compilation.zig"); const codegen = @import("../codegen/wasm.zig"); const link = @import("../link.zig"); +const trace = @import("../tracy.zig").trace; /// Various magic numbers defined by the wasm spec const spec = struct { @@ -134,6 +135,9 @@ pub fn freeDecl(self: *Wasm, decl: *Module.Decl) void { } pub fn flush(self: *Wasm, comp: *Compilation) !void { + const tracy = trace(@src()); + defer tracy.end(); + const file = self.base.file.?; const header_size = 5 + 1; diff --git a/src-self-hosted/llvm.zig b/src-self-hosted/llvm.zig index d65e046169..d247e87c28 100644 --- a/src-self-hosted/llvm.zig +++ b/src-self-hosted/llvm.zig @@ -1,8 +1,9 @@ //! We do this instead of @cImport because the self-hosted compiler is easier //! to bootstrap if it does not depend on translate-c. +pub const Link = ZigLLDLink; pub extern fn ZigLLDLink( - oformat: ZigLLVM_ObjectFormatType, + oformat: ObjectFormatType, args: [*:null]const ?[*:0]const u8, arg_count: usize, append_diagnostic: fn (context: usize, ptr: [*]const u8, len: usize) callconv(.C) void, @@ -10,7 +11,7 @@ pub extern fn ZigLLDLink( context_stderr: usize, ) bool; -pub const ZigLLVM_ObjectFormatType = extern enum(c_int) { +pub const ObjectFormatType = extern enum(c_int) { Unknown, COFF, ELF, @@ -18,3 +19,9 @@ pub const ZigLLVM_ObjectFormatType = extern enum(c_int) { Wasm, XCOFF, }; + +pub const GetHostCPUName = LLVMGetHostCPUName; +extern fn LLVMGetHostCPUName() ?[*:0]u8; + +pub const GetNativeFeatures = ZigLLVMGetNativeFeatures; +extern fn ZigLLVMGetNativeFeatures() ?[*:0]u8; diff --git a/src/codegen.cpp b/src/codegen.cpp index b5c1ca3a41..ce6eeb1def 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -9013,7 +9013,7 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { if (g->zig_target->os_builtin_str != nullptr) { buf_append_str(contents, g->zig_target->os_builtin_str); } else { - buf_appendf(contents, "Target.Os.defaultVersionRange(.%s);\n", cur_os); + buf_appendf(contents, "Target.Os.Tag.defaultVersionRange(.%s);\n", cur_os); } } buf_appendf(contents, "pub const object_format = ObjectFormat.%s;\n", cur_obj_fmt); From 778bb4bc9c9ceb62426c0ed48c079142b713b910 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 11:05:51 -0700 Subject: [PATCH 032/210] move std.cache_hash from std to stage2 The API is pretty specific to the implementationt details of the self-hosted compiler. I don't want to have to independently support and maintain this as part of the standard library, and be obligated to not make breaking changes to it with changes to the implementation of stage2. --- lib/std/std.zig | 1 - .../Cache.zig | 40 ++++++++----------- src-self-hosted/Compilation.zig | 9 +++-- src-self-hosted/introspect.zig | 1 - src-self-hosted/link.zig | 5 ++- 5 files changed, 25 insertions(+), 31 deletions(-) rename lib/std/cache_hash.zig => src-self-hosted/Cache.zig (97%) diff --git a/lib/std/std.zig b/lib/std/std.zig index 4236b29298..56b75e5656 100644 --- a/lib/std/std.zig +++ b/lib/std/std.zig @@ -48,7 +48,6 @@ pub const base64 = @import("base64.zig"); pub const build = @import("build.zig"); pub const builtin = @import("builtin.zig"); pub const c = @import("c.zig"); -pub const cache_hash = @import("cache_hash.zig"); pub const coff = @import("coff.zig"); pub const compress = @import("compress.zig"); pub const crypto = @import("crypto.zig"); diff --git a/lib/std/cache_hash.zig b/src-self-hosted/Cache.zig similarity index 97% rename from lib/std/cache_hash.zig rename to src-self-hosted/Cache.zig index 2075bedc22..b41c0a312f 100644 --- a/lib/std/cache_hash.zig +++ b/src-self-hosted/Cache.zig @@ -1,9 +1,9 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2015-2020 Zig Contributors -// This file is part of [zig](https://ziglang.org/), which is MIT licensed. -// The MIT license requires this copyright notice to be included in all copies -// and substantial portions of the software. -const std = @import("std.zig"); +gpa: *Allocator, +manifest_dir: fs.Dir, +hash: HashHelper = .{}, + +const Cache = @This(); +const std = @import("std"); const crypto = std.crypto; const fs = std.fs; const base64 = std.base64; @@ -13,6 +13,17 @@ const mem = std.mem; const fmt = std.fmt; const Allocator = std.mem.Allocator; +/// Be sure to call `CacheHash.deinit` after successful initialization. +pub fn obtain(cache: *const Cache) CacheHash { + return CacheHash{ + .cache = cache, + .hash = cache.hash, + .manifest_file = null, + .manifest_dirty = false, + .b64_digest = undefined, + }; +} + pub const base64_encoder = fs.base64_encoder; pub const base64_decoder = fs.base64_decoder; /// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 @@ -50,23 +61,6 @@ pub const File = struct { } }; -pub const Cache = struct { - gpa: *Allocator, - manifest_dir: fs.Dir, - hash: HashHelper = .{}, - - /// Be sure to call `CacheHash.deinit` after successful initialization. - pub fn obtain(cache: *const Cache) CacheHash { - return CacheHash{ - .cache = cache, - .hash = cache.hash, - .manifest_file = null, - .manifest_dirty = false, - .b64_digest = undefined, - }; - } -}; - pub const HashHelper = struct { hasher: Hasher = hasher_init, diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 1f42c87849..606eefa4ee 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -17,6 +17,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const glibc = @import("glibc.zig"); const fatal = @import("main.zig").fatal; const Module = @import("Module.zig"); +const Cache = @import("Cache.zig"); /// General-purpose allocator. Used for both temporary and long-term storage. gpa: *Allocator, @@ -46,7 +47,7 @@ disable_c_depfile: bool, c_source_files: []const CSourceFile, clang_argv: []const []const u8, -cache_parent: *std.cache_hash.Cache, +cache_parent: *Cache, /// Path to own executable for invoking `zig clang`. self_exe_path: ?[]const u8, zig_lib_directory: Directory, @@ -78,7 +79,7 @@ owned_link_dir: ?std.fs.Dir, pub const InnerError = Module.InnerError; pub const CRTFile = struct { - lock: std.cache_hash.Lock, + lock: Cache.Lock, full_object_path: []const u8, fn deinit(self: *CRTFile, gpa: *Allocator) void { @@ -128,7 +129,7 @@ pub const CObject = struct { /// This is a file system lock on the cache hash manifest representing this /// object. It prevents other invocations of the Zig compiler from interfering /// with this object until released. - lock: std.cache_hash.Lock, + lock: Cache.Lock, }, /// There will be a corresponding ErrorMsg in Compilation.failed_c_objects. failure, @@ -404,7 +405,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { // find the same binary and incrementally update it even if there are modified source files. // We do this even if outputting to the current directory because we need somewhere to store // incremental compilation metadata. - const cache = try arena.create(std.cache_hash.Cache); + const cache = try arena.create(Cache); cache.* = .{ .gpa = gpa, .manifest_dir = try options.zig_cache_directory.handle.makeOpenPath("h", .{}), diff --git a/src-self-hosted/introspect.zig b/src-self-hosted/introspect.zig index 8041142a67..067326ebb6 100644 --- a/src-self-hosted/introspect.zig +++ b/src-self-hosted/introspect.zig @@ -1,7 +1,6 @@ const std = @import("std"); const mem = std.mem; const fs = std.fs; -const CacheHash = std.cache_hash.CacheHash; const Compilation = @import("Compilation.zig"); /// Returns the sub_path that worked, or `null` if none did. diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 832015faae..a7393cc913 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -6,6 +6,7 @@ const fs = std.fs; const trace = @import("tracy.zig").trace; const Package = @import("Package.zig"); const Type = @import("type.zig").Type; +const Cache = @import("Cache.zig"); const build_options = @import("build_options"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; @@ -92,7 +93,7 @@ pub const File = struct { /// Prevents other processes from clobbering files in the output directory /// of this linking operation. - lock: ?std.cache_hash.Lock = null, + lock: ?Cache.Lock = null, pub const LinkBlock = union { elf: Elf.TextBlock, @@ -239,7 +240,7 @@ pub const File = struct { } } - pub fn toOwnedLock(self: *File) std.cache_hash.Lock { + pub fn toOwnedLock(self: *File) Cache.Lock { const lock = self.lock.?; self.lock = null; return lock; From 40cb712d13aff4bfe83256858ad6b18d82e70211 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 11:10:38 -0700 Subject: [PATCH 033/210] disable sourcehut freebsd CI checks Drew won't give us enough RAM for stage1 to build stage2. We'll still have freebsd builds available on releases but we're going to lose freebsd CI testing for master branch builds until we fully switch over to stage2 (and have lower memory usage). Let me know if anyone wants to run a SourceHut instance and give zig access to run on slightly more powerful machines. We need about 8 GiB RAM to run the CI test suite for now. After we're fully self hosted I expect to re-enable this. --- .builds/freebsd.yml | 8 -------- ci/srht/update_download_page | 12 ++++++------ 2 files changed, 6 insertions(+), 14 deletions(-) delete mode 100644 .builds/freebsd.yml diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml deleted file mode 100644 index 9b936657fa..0000000000 --- a/.builds/freebsd.yml +++ /dev/null @@ -1,8 +0,0 @@ -image: freebsd/latest -secrets: - - 6c60aaee-92e7-4e7d-812c-114817689b4d - - dd0bd962-7664-4d3e-b0f3-41c9ee96b8b8 -sources: - - https://github.com/ziglang/zig -tasks: - - build: cd zig && ./ci/srht/freebsd_script diff --git a/ci/srht/update_download_page b/ci/srht/update_download_page index 77436a3772..12cf639d2e 100755 --- a/ci/srht/update_download_page +++ b/ci/srht/update_download_page @@ -13,7 +13,7 @@ AARCH64_LINUX_JSON_URL="https://ziglang.org/builds/aarch64-linux-$VERSION.json" X86_64_LINUX_JSON_URL="https://ziglang.org/builds/x86_64-linux-$VERSION.json" X86_64_WINDOWS_JSON_URL="https://ziglang.org/builds/x86_64-windows-$VERSION.json" X86_64_MACOS_JSON_URL="https://ziglang.org/builds/x86_64-macos-$VERSION.json" -X86_64_FREEBSD_JSON_URL="https://ziglang.org/builds/x86_64-freebsd-$VERSION.json" +#X86_64_FREEBSD_JSON_URL="https://ziglang.org/builds/x86_64-freebsd-$VERSION.json" # If any of these fail, it's not really this job failing; rather we have detected # that this job will be called again later when other jobs have completed. @@ -21,7 +21,7 @@ curl --fail -I "$AARCH64_LINUX_JSON_URL" >/dev/null || exit 0 curl --fail -I "$X86_64_LINUX_JSON_URL" >/dev/null || exit 0 curl --fail -I "$X86_64_WINDOWS_JSON_URL" >/dev/null || exit 0 curl --fail -I "$X86_64_MACOS_JSON_URL" >/dev/null || exit 0 -curl --fail -I "$X86_64_FREEBSD_JSON_URL" >/dev/null || exit 0 +#curl --fail -I "$X86_64_FREEBSD_JSON_URL" >/dev/null || exit 0 # Without --user, this gave me: # ERROR: Could not install packages due to an EnvironmentError: [Errno 13] Permission denied @@ -63,10 +63,10 @@ export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)" export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)" export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)" -X86_64_FREEBSD_JSON=$(curl --fail "$X86_64_FREEBSD_JSON_URL" || exit 1) -export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" | jq .tarball -r)" -export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" | jq .size -r)" -export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" | jq .shasum -r)" +#X86_64_FREEBSD_JSON=$(curl --fail "$X86_64_FREEBSD_JSON_URL" || exit 1) +#export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" | jq .tarball -r)" +#export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" | jq .size -r)" +#export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" | jq .shasum -r)" git clone https://github.com/ziglang/www.ziglang.org --depth 1 cd www.ziglang.org From 26798018b780846eb0613b2061835985bf0c58ea Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 15:25:30 -0700 Subject: [PATCH 034/210] stage2: implement writing archive files --- src-self-hosted/Compilation.zig | 2 + src-self-hosted/link.zig | 123 ++++++++++++++++++++++++++++++-- src-self-hosted/link/C.zig | 4 ++ src-self-hosted/link/Coff.zig | 9 +++ src-self-hosted/link/Elf.zig | 9 ++- src-self-hosted/link/MachO.zig | 11 ++- src-self-hosted/link/Wasm.zig | 9 +++ src-self-hosted/llvm.zig | 49 ++++++++++++- src-self-hosted/target.zig | 41 +++++++++++ 9 files changed, 246 insertions(+), 11 deletions(-) diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 606eefa4ee..49747cecb2 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -535,6 +535,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { var hash = cache.hash; if (options.c_source_files.len >= 1) { hash.addBytes(options.c_source_files[0].src_path); + } else if (options.link_objects.len >= 1) { + hash.addBytes(options.link_objects[0]); } const digest = hash.final(); diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index a7393cc913..4d3fd5085d 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const mem = std.mem; const Allocator = std.mem.Allocator; const Compilation = @import("Compilation.zig"); const Module = @import("Module.zig"); @@ -9,6 +10,7 @@ const Type = @import("type.zig").Type; const Cache = @import("Cache.zig"); const build_options = @import("build_options"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; +const log = std.log.scoped(.link); pub const producer_string = if (std.builtin.is_test) "zig test" else "zig " ++ build_options.version; @@ -279,9 +281,11 @@ pub const File = struct { } } + /// Commit pending changes and write headers. Takes into account final output mode + /// and `use_lld`, not only `effectiveOutputMode`. pub fn flush(base: *File, comp: *Compilation) !void { const use_lld = build_options.have_llvm and base.options.use_lld; - if (base.options.output_mode == .Lib and base.options.link_mode == .Static and + if (use_lld and base.options.output_mode == .Lib and base.options.link_mode == .Static and !base.options.target.isWasm()) { return base.linkAsArchive(comp); @@ -295,6 +299,18 @@ pub const File = struct { } } + /// Commit pending changes and write headers. Works based on `effectiveOutputMode` + /// rather than final output mode. + pub fn flushModule(base: *File, comp: *Compilation) !void { + switch (base.tag) { + .coff => return @fieldParentPtr(Coff, "base", base).flushModule(comp), + .elf => return @fieldParentPtr(Elf, "base", base).flushModule(comp), + .macho => return @fieldParentPtr(MachO, "base", base).flushModule(comp), + .c => return @fieldParentPtr(C, "base", base).flushModule(comp), + .wasm => return @fieldParentPtr(Wasm, "base", base).flushModule(comp), + } + } + pub fn freeDecl(base: *File, decl: *Module.Decl) void { switch (base.tag) { .coff => @fieldParentPtr(Coff, "base", base).freeDecl(decl), @@ -343,9 +359,108 @@ pub const File = struct { } fn linkAsArchive(base: *File, comp: *Compilation) !void { - // TODO follow pattern from ELF linkWithLLD - // ZigLLVMWriteArchive - return error.TODOMakeArchive; + const tracy = trace(@src()); + defer tracy.end(); + + var arena_allocator = std.heap.ArenaAllocator.init(base.allocator); + defer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + const directory = base.options.directory; // Just an alias to make it shorter to type. + + // If there is no Zig code to compile, then we should skip flushing the output file because it + // will not be part of the linker line anyway. + const module_obj_path: ?[]const u8 = if (base.options.module) |module| blk: { + try base.flushModule(comp); + + const obj_basename = base.intermediary_basename.?; + const full_obj_path = if (directory.path) |dir_path| + try std.fs.path.join(arena, &[_][]const u8{ dir_path, obj_basename }) + else + obj_basename; + break :blk full_obj_path; + } else null; + + // This function follows the same pattern as link.Elf.linkWithLLD so if you want some + // insight as to what's going on here you can read that function body which is more + // well-commented. + + const id_symlink_basename = "llvm-ar.id"; + + base.releaseLock(); + + var ch = comp.cache_parent.obtain(); + defer ch.deinit(); + + try ch.addListOfFiles(base.options.objects); + for (comp.c_object_table.items()) |entry| { + _ = try ch.addFile(entry.key.status.success.object_path, null); + } + try ch.addOptionalFile(module_obj_path); + + // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock. + _ = try ch.hit(); + const digest = ch.final(); + + var prev_digest_buf: [digest.len]u8 = undefined; + const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| b: { + log.debug("archive new_digest={} readlink error: {}", .{ digest, @errorName(err) }); + break :b prev_digest_buf[0..0]; + }; + if (mem.eql(u8, prev_digest, &digest)) { + log.debug("archive digest={} match - skipping invocation", .{digest}); + base.lock = ch.toOwnedLock(); + return; + } + + // We are about to change the output file to be different, so we invalidate the build hash now. + directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) { + error.FileNotFound => {}, + else => |e| return e, + }; + + var object_files = std.ArrayList([*:0]const u8).init(base.allocator); + defer object_files.deinit(); + + try object_files.ensureCapacity(base.options.objects.len + comp.c_object_table.items().len + 1); + for (base.options.objects) |obj_path| { + object_files.appendAssumeCapacity(try arena.dupeZ(u8, obj_path)); + } + for (comp.c_object_table.items()) |entry| { + object_files.appendAssumeCapacity(try arena.dupeZ(u8, entry.key.status.success.object_path)); + } + if (module_obj_path) |p| { + object_files.appendAssumeCapacity(try arena.dupeZ(u8, p)); + } + + const full_out_path = if (directory.path) |dir_path| + try std.fs.path.join(arena, &[_][]const u8{ dir_path, base.options.sub_path }) + else + base.options.sub_path; + const full_out_path_z = try arena.dupeZ(u8, full_out_path); + + if (base.options.debug_link) { + std.debug.print("ar rcs {}", .{full_out_path_z}); + for (object_files.items) |arg| { + std.debug.print(" {}", .{arg}); + } + std.debug.print("\n", .{}); + } + + const llvm = @import("llvm.zig"); + const os_type = @import("target.zig").osToLLVM(base.options.target.os.tag); + const bad = llvm.WriteArchive(full_out_path_z, object_files.items.ptr, object_files.items.len, os_type); + if (bad) return error.UnableToWriteArchive; + + directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { + std.log.warn("failed to save archive hash digest symlink: {}", .{@errorName(err)}); + }; + + ch.writeManifest() catch |err| { + std.log.warn("failed to write cache manifest when archiving: {}", .{@errorName(err)}); + }; + + base.lock = ch.toOwnedLock(); } pub const Tag = enum { diff --git a/src-self-hosted/link/C.zig b/src-self-hosted/link/C.zig index 97990f34fb..d5d1249244 100644 --- a/src-self-hosted/link/C.zig +++ b/src-self-hosted/link/C.zig @@ -74,6 +74,10 @@ pub fn updateDecl(self: *C, module: *Module, decl: *Module.Decl) !void { } pub fn flush(self: *C, comp: *Compilation) !void { + return self.flushModule(comp); +} + +pub fn flushModule(self: *C, comp: *Compilation) !void { const tracy = trace(@src()); defer tracy.end(); diff --git a/src-self-hosted/link/Coff.zig b/src-self-hosted/link/Coff.zig index b927ca2586..31726a5712 100644 --- a/src-self-hosted/link/Coff.zig +++ b/src-self-hosted/link/Coff.zig @@ -11,6 +11,7 @@ const Module = @import("../Module.zig"); const Compilation = @import("../Compilation.zig"); const codegen = @import("../codegen.zig"); const link = @import("../link.zig"); +const build_options = @import("build_options"); const allocation_padding = 4 / 3; const minimum_text_block_size = 64 * allocation_padding; @@ -724,6 +725,14 @@ pub fn updateDeclExports(self: *Coff, module: *Module, decl: *const Module.Decl, } pub fn flush(self: *Coff, comp: *Compilation) !void { + if (build_options.have_llvm and self.base.options.use_lld) { + return error.CoffLinkingWithLLDUnimplemented; + } else { + return self.flushModule(comp); + } +} + +pub fn flushModule(self: *Coff, comp: *Compilation) !void { const tracy = trace(@src()); defer tracy.end(); diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 49cf02d7c4..8a75796bc3 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -719,12 +719,11 @@ pub fn flush(self: *Elf, comp: *Compilation) !void { .Exe, .Obj => {}, .Lib => return error.TODOImplementWritingLibFiles, } - return self.flushInner(comp); + return self.flushModule(comp); } } -/// Commit pending changes and write headers. -fn flushInner(self: *Elf, comp: *Compilation) !void { +pub fn flushModule(self: *Elf, comp: *Compilation) !void { const tracy = trace(@src()); defer tracy.end(); @@ -1221,7 +1220,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { // If there is no Zig code to compile, then we should skip flushing the output file because it // will not be part of the linker line anyway. const module_obj_path: ?[]const u8 = if (self.base.options.module) |module| blk: { - try self.flushInner(comp); + try self.flushModule(comp); const obj_basename = self.base.intermediary_basename.?; const full_obj_path = if (directory.path) |dir_path| @@ -1239,7 +1238,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { // After a successful link, we store the id in the metadata of a symlink named "id.txt" in // the artifact directory. So, now, we check if this symlink exists, and if it matches // our digest. If so, we can skip linking. Otherwise, we proceed with invoking LLD. - const id_symlink_basename = "id.txt"; + const id_symlink_basename = "lld.id"; // We are about to obtain this lock, so here we give other processes a chance first. self.base.releaseLock(); diff --git a/src-self-hosted/link/MachO.zig b/src-self-hosted/link/MachO.zig index ff3fa2810d..3b70a0d710 100644 --- a/src-self-hosted/link/MachO.zig +++ b/src-self-hosted/link/MachO.zig @@ -9,9 +9,10 @@ const macho = std.macho; const codegen = @import("../codegen.zig"); const math = std.math; const mem = std.mem; + const trace = @import("../tracy.zig").trace; const Type = @import("../type.zig").Type; - +const build_options = @import("build_options"); const Module = @import("../Module.zig"); const Compilation = @import("../Compilation.zig"); const link = @import("../link.zig"); @@ -178,6 +179,14 @@ pub fn createEmpty(gpa: *Allocator, options: link.Options) !*MachO { } pub fn flush(self: *MachO, comp: *Compilation) !void { + if (build_options.have_llvm and self.base.options.use_lld) { + return error.MachOLLDLinkingUnimplemented; + } else { + return self.flushModule(comp); + } +} + +pub fn flushModule(self: *MachO, comp: *Compilation) !void { const tracy = trace(@src()); defer tracy.end(); diff --git a/src-self-hosted/link/Wasm.zig b/src-self-hosted/link/Wasm.zig index b4bb2c3195..1160e471fe 100644 --- a/src-self-hosted/link/Wasm.zig +++ b/src-self-hosted/link/Wasm.zig @@ -11,6 +11,7 @@ const Compilation = @import("../Compilation.zig"); const codegen = @import("../codegen/wasm.zig"); const link = @import("../link.zig"); const trace = @import("../tracy.zig").trace; +const build_options = @import("build_options"); /// Various magic numbers defined by the wasm spec const spec = struct { @@ -135,6 +136,14 @@ pub fn freeDecl(self: *Wasm, decl: *Module.Decl) void { } pub fn flush(self: *Wasm, comp: *Compilation) !void { + if (build_options.have_llvm and self.base.options.use_lld) { + return error.WasmLinkingWithLLDUnimplemented; + } else { + return self.flushModule(comp); + } +} + +pub fn flushModule(self: *Wasm, comp: *Compilation) !void { const tracy = trace(@src()); defer tracy.end(); diff --git a/src-self-hosted/llvm.zig b/src-self-hosted/llvm.zig index d247e87c28..64a6d4e8b5 100644 --- a/src-self-hosted/llvm.zig +++ b/src-self-hosted/llvm.zig @@ -2,7 +2,7 @@ //! to bootstrap if it does not depend on translate-c. pub const Link = ZigLLDLink; -pub extern fn ZigLLDLink( +extern fn ZigLLDLink( oformat: ObjectFormatType, args: [*:null]const ?[*:0]const u8, arg_count: usize, @@ -25,3 +25,50 @@ extern fn LLVMGetHostCPUName() ?[*:0]u8; pub const GetNativeFeatures = ZigLLVMGetNativeFeatures; extern fn ZigLLVMGetNativeFeatures() ?[*:0]u8; + +pub const WriteArchive = ZigLLVMWriteArchive; +extern fn ZigLLVMWriteArchive( + archive_name: [*:0]const u8, + file_names_ptr: [*]const [*:0]const u8, + file_names_len: usize, + os_type: OSType, +) bool; + +pub const OSType = extern enum(c_int) { + UnknownOS = 0, + Ananas = 1, + CloudABI = 2, + Darwin = 3, + DragonFly = 4, + FreeBSD = 5, + Fuchsia = 6, + IOS = 7, + KFreeBSD = 8, + Linux = 9, + Lv2 = 10, + MacOSX = 11, + NetBSD = 12, + OpenBSD = 13, + Solaris = 14, + Win32 = 15, + Haiku = 16, + Minix = 17, + RTEMS = 18, + NaCl = 19, + CNK = 20, + AIX = 21, + CUDA = 22, + NVCL = 23, + AMDHSA = 24, + PS4 = 25, + ELFIAMCU = 26, + TvOS = 27, + WatchOS = 28, + Mesa3D = 29, + Contiki = 30, + AMDPAL = 31, + HermitCore = 32, + Hurd = 33, + WASI = 34, + Emscripten = 35, +}; diff --git a/src-self-hosted/target.zig b/src-self-hosted/target.zig index b09c363d8a..8ea880e898 100644 --- a/src-self-hosted/target.zig +++ b/src-self-hosted/target.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const llvm = @import("llvm.zig"); pub const ArchOsAbi = struct { arch: std.Target.Cpu.Arch, @@ -168,3 +169,43 @@ pub fn supportsStackProbing(target: std.Target) bool { return target.os.tag != .windows and target.os.tag != .uefi and (target.cpu.arch == .i386 or target.cpu.arch == .x86_64); } + +pub fn osToLLVM(os_tag: std.Target.Os.Tag) llvm.OSType { + return switch (os_tag) { + .freestanding, .other => .UnknownOS, + .windows, .uefi => .Win32, + .ananas => .Ananas, + .cloudabi => .CloudABI, + .dragonfly => .DragonFly, + .freebsd => .FreeBSD, + .fuchsia => .Fuchsia, + .ios => .IOS, + .kfreebsd => .KFreeBSD, + .linux => .Linux, + .lv2 => .Lv2, + .macosx => .MacOSX, + .netbsd => .NetBSD, + .openbsd => .OpenBSD, + .solaris => .Solaris, + .haiku => .Haiku, + .minix => .Minix, + .rtems => .RTEMS, + .nacl => .NaCl, + .cnk => .CNK, + .aix => .AIX, + .cuda => .CUDA, + .nvcl => .NVCL, + .amdhsa => .AMDHSA, + .ps4 => .PS4, + .elfiamcu => .ELFIAMCU, + .tvos => .TvOS, + .watchos => .WatchOS, + .mesa3d => .Mesa3D, + .contiki => .Contiki, + .amdpal => .AMDPAL, + .hermit => .HermitCore, + .hurd => .Hurd, + .wasi => .WASI, + .emscripten => .Emscripten, + }; +} From dffdb2844e0163f58507b1f4a51693f4086b7949 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 18:06:19 -0700 Subject: [PATCH 035/210] track all TODO comments in BRANCH_TODO file Before merging, do this for every item in the file: * solve the issue, or * convert the task to a github issue and update the comment to link to the issue (and remove "TODO" text from the comment). Then delete the file. Related: #363 --- BRANCH_TODO | 51 +++++++++++++++++++++++++++ lib/std/child_process.zig | 4 +-- src-self-hosted/Compilation.zig | 8 ++--- src-self-hosted/glibc.zig | 9 +++-- src-self-hosted/libc_installation.zig | 2 +- src-self-hosted/main.zig | 2 +- 6 files changed, 62 insertions(+), 14 deletions(-) create mode 100644 BRANCH_TODO diff --git a/BRANCH_TODO b/BRANCH_TODO new file mode 100644 index 0000000000..2e9da59df3 --- /dev/null +++ b/BRANCH_TODO @@ -0,0 +1,51 @@ + * refactor 2 CObject fields to use CSourceFile + * integrate code model and have_frame_pointer to main() and c objects + * integrate target features into building C source files + * integrate target features into building assembly code + * handle .d files from c objects + * glibc .so files + * support rpaths in ELF linker code + * build & link against compiler-rt + * build & link againstn freestanding libc + * add CLI support for a way to pass extra flags to c source files + * implement the workaround for using LLVM to detect native CPU features + * self-host main.cpp + * capture lld stdout/stderr better + * musl + * mingw-w64 + * port the stage1 os.cpp code that raises the open fd limit + * use global zig-cache dir for crt files + * `zig translate-c` + * make sure zig cc works + - using it as a preprocessor (-E) + - @breakpoint(); // TODO the first arg is empty string right? skip past that. + - try building some software + * MachO LLD linking + * COFF LLD linking + * WASM LLD linking + * implement proper parsing of LLD stderr/stdout and exposing compile errors + * implement proper parsing of clang stderr/stdout and exposing compile errors + * implement proper compile errors for failing to build glibc crt files and shared libs + * skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd) + * self-host link.cpp and building libcs (#4313 and #4314). using the `zig cc` command will set a flag indicating a preference for the llvm backend, which will include linking with LLD. At least for now. If zig's self-hosted linker ever gets on par with the likes of ld and lld, we can make it always be used even for zig cc. + * improve the stage2 tests to support testing with LLVM extensions enabled + * multi-thread building C objects + * support cross compiling stage2 with `zig build` + * implement emit-h in stage2 + * implement -fno-emit-bin + * audit the base cache hash + * implement serialization/deserialization of incremental compilation metadata + * incremental compilation - implement detection of which source files changed + * improve the cache hash logic for c objects with respect to extra flags and file parameters + * LLVM codegen backend: put a sub-arch in the triple in some cases + * rework libc_installation.zig abstraction to use std.log instead of taking a stderr stream + * implement an LLVM backend for stage2 + * implement outputting dynamic libraries in self-hosted linker + * implement outputting static libraries (archive files) in self-hosted linker + * support linking against object files in self-hosted linker + * avoid invoking lld when it's just 1 object file (the `zig cc -c` case) + * `zig fmt --check` should output to stdout not stderr. + * main.zig: If there was an argsAllocZ we could avoid this allocation + * improve robustness of response file parsing + * there are a couple panic("TODO") in clang options parsing + * std.testing needs improvement to support exposing directory path for its tmp dir (look for "bogus") diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 9219b05088..82ed938f33 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -213,7 +213,7 @@ pub const ChildProcess = struct { const stdout_in = child.stdout.?.inStream(); const stderr_in = child.stderr.?.inStream(); - // TODO need to poll to read these streams to prevent a deadlock (or rely on evented I/O). + // TODO https://github.com/ziglang/zig/issues/6343 const stdout = try stdout_in.readAllAlloc(args.allocator, args.max_output_bytes); errdefer args.allocator.free(stdout); const stderr = try stderr_in.readAllAlloc(args.allocator, args.max_output_bytes); @@ -485,7 +485,7 @@ pub const ChildProcess = struct { const any_ignore = (self.stdin_behavior == StdIo.Ignore or self.stdout_behavior == StdIo.Ignore or self.stderr_behavior == StdIo.Ignore); const nul_handle = if (any_ignore) - // "\Device\Null" or "\??\NUL" + // "\Device\Null" or "\??\NUL" windows.OpenFile(&[_]u16{ '\\', 'D', 'e', 'v', 'i', 'c', 'e', '\\', 'N', 'u', 'l', 'l' }, .{ .access_mask = windows.GENERIC_READ | windows.SYNCHRONIZE, .share_access = windows.FILE_SHARE_READ, diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 49747cecb2..85db6495b9 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -695,7 +695,6 @@ pub fn update(self: *Compilation) !void { defer tracy.end(); // For compiling C objects, we rely on the cache hash system to avoid duplicating work. - // TODO Look into caching this data in memory to improve performance. // Add a WorkItem for each C object. try self.work_queue.ensureUnusedCapacity(self.c_object_table.items().len); for (self.c_object_table.items()) |entry| { @@ -1052,8 +1051,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { switch (term) { .Exited => |code| { if (code != 0) { - // TODO make std.process.exit and std.ChildProcess exit code have the same type - // and forward it here. Currently it is u32 vs u8. + // TODO https://github.com/ziglang/zig/issues/6342 std.process.exit(1); } }, @@ -1069,7 +1067,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { const stdout_reader = child.stdout.?.reader(); const stderr_reader = child.stderr.?.reader(); - // TODO Need to poll to read these streams to prevent a deadlock (or rely on evented I/O). + // TODO https://github.com/ziglang/zig/issues/6343 const stdout = try stdout_reader.readAllAlloc(arena, std.math.maxInt(u32)); const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); @@ -1100,7 +1098,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); var o_dir = try comp.zig_cache_directory.handle.makeOpenPath(o_sub_path, .{}); defer o_dir.close(); - // TODO Add renameat capabilities to the std lib in a higher layer than the posix layer. + // TODO https://github.com/ziglang/zig/issues/6344 const tmp_basename = std.fs.path.basename(out_obj_path); try std.os.renameat(zig_cache_tmp_dir.fd, tmp_basename, o_dir.fd, o_basename); diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index 29d96bb375..80e66fa28e 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -270,7 +270,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-g", "-Wa,--noexecstack", }); - return build_libc_object(comp, "crti.o", &[1]Compilation.CSourceFile{ + return build_crt_file(comp, "crti.o", &[1]Compilation.CSourceFile{ .{ .src_path = try start_asm_path(comp, arena, "crti.S"), .extra_flags = args.items, @@ -288,7 +288,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-g", "-Wa,--noexecstack", }); - return build_libc_object(comp, "crtn.o", &[1]Compilation.CSourceFile{ + return build_crt_file(comp, "crtn.o", &[1]Compilation.CSourceFile{ .{ .src_path = try start_asm_path(comp, arena, "crtn.S"), .extra_flags = args.items, @@ -339,7 +339,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }; }; - return build_libc_object(comp, "Scrt1.o", &[_]Compilation.CSourceFile{ start_os, abi_note_o }); + return build_crt_file(comp, "Scrt1.o", &[_]Compilation.CSourceFile{ start_os, abi_note_o }); }, .libc_nonshared_a => { return error.Unimplemented; // TODO @@ -585,7 +585,7 @@ fn lib_path(comp: *Compilation, arena: *Allocator, sub_path: []const u8) ![]cons return path.join(arena, &[_][]const u8{ comp.zig_lib_directory.path.?, sub_path }); } -fn build_libc_object( +fn build_crt_file( comp: *Compilation, basename: []const u8, c_source_files: []const Compilation.CSourceFile, @@ -649,7 +649,6 @@ fn build_libc_object( else try comp.gpa.dupe(u8, basename); - // TODO obtain a lock on the artifact and put that in crt_files as well. comp.crt_files.putAssumeCapacityNoClobber(basename, .{ .full_object_path = artifact_path, .lock = sub_compilation.bin_file.toOwnedLock(), diff --git a/src-self-hosted/libc_installation.zig b/src-self-hosted/libc_installation.zig index 125f514060..eb59ef7e6a 100644 --- a/src-self-hosted/libc_installation.zig +++ b/src-self-hosted/libc_installation.zig @@ -11,7 +11,7 @@ const is_gnu = Target.current.isGnu(); usingnamespace @import("windows_sdk.zig"); -// TODO Rework this abstraction to use std.log instead of taking a stderr stream. +// TODO https://github.com/ziglang/zig/issues/6345 /// See the render function implementation for documentation of the fields. pub const LibCInstallation = struct { diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 918e92422a..6ec2c45531 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -1615,7 +1615,7 @@ pub const info_zen = extern "c" fn ZigClang_main(argc: c_int, argv: [*:null]?[*:0]u8) c_int; -/// TODO make it so the return value can be !noreturn +/// TODO https://github.com/ziglang/zig/issues/3257 fn punt_to_clang(arena: *Allocator, args: []const []const u8) error{OutOfMemory} { if (!build_options.have_llvm) fatal("`zig cc` and `zig c++` unavailable: compiler not built with LLVM extensions enabled", .{}); From 29d9743f9d3a36cbeb1b2618f011551b11474e4d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 19:43:26 -0700 Subject: [PATCH 036/210] Revert "disable sourcehut freebsd CI checks" This reverts commit 40cb712d13aff4bfe83256858ad6b18d82e70211. Thanks to Ava & Luna of Lavatech, we don't need to resort to this, they have graciously given zig a SourceHut instance to use that gives us 8GB RAM. --- .builds/freebsd.yml | 8 ++++++++ ci/srht/update_download_page | 12 ++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 .builds/freebsd.yml diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml new file mode 100644 index 0000000000..9b936657fa --- /dev/null +++ b/.builds/freebsd.yml @@ -0,0 +1,8 @@ +image: freebsd/latest +secrets: + - 6c60aaee-92e7-4e7d-812c-114817689b4d + - dd0bd962-7664-4d3e-b0f3-41c9ee96b8b8 +sources: + - https://github.com/ziglang/zig +tasks: + - build: cd zig && ./ci/srht/freebsd_script diff --git a/ci/srht/update_download_page b/ci/srht/update_download_page index 12cf639d2e..77436a3772 100755 --- a/ci/srht/update_download_page +++ b/ci/srht/update_download_page @@ -13,7 +13,7 @@ AARCH64_LINUX_JSON_URL="https://ziglang.org/builds/aarch64-linux-$VERSION.json" X86_64_LINUX_JSON_URL="https://ziglang.org/builds/x86_64-linux-$VERSION.json" X86_64_WINDOWS_JSON_URL="https://ziglang.org/builds/x86_64-windows-$VERSION.json" X86_64_MACOS_JSON_URL="https://ziglang.org/builds/x86_64-macos-$VERSION.json" -#X86_64_FREEBSD_JSON_URL="https://ziglang.org/builds/x86_64-freebsd-$VERSION.json" +X86_64_FREEBSD_JSON_URL="https://ziglang.org/builds/x86_64-freebsd-$VERSION.json" # If any of these fail, it's not really this job failing; rather we have detected # that this job will be called again later when other jobs have completed. @@ -21,7 +21,7 @@ curl --fail -I "$AARCH64_LINUX_JSON_URL" >/dev/null || exit 0 curl --fail -I "$X86_64_LINUX_JSON_URL" >/dev/null || exit 0 curl --fail -I "$X86_64_WINDOWS_JSON_URL" >/dev/null || exit 0 curl --fail -I "$X86_64_MACOS_JSON_URL" >/dev/null || exit 0 -#curl --fail -I "$X86_64_FREEBSD_JSON_URL" >/dev/null || exit 0 +curl --fail -I "$X86_64_FREEBSD_JSON_URL" >/dev/null || exit 0 # Without --user, this gave me: # ERROR: Could not install packages due to an EnvironmentError: [Errno 13] Permission denied @@ -63,10 +63,10 @@ export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)" export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)" export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)" -#X86_64_FREEBSD_JSON=$(curl --fail "$X86_64_FREEBSD_JSON_URL" || exit 1) -#export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" | jq .tarball -r)" -#export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" | jq .size -r)" -#export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" | jq .shasum -r)" +X86_64_FREEBSD_JSON=$(curl --fail "$X86_64_FREEBSD_JSON_URL" || exit 1) +export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" | jq .tarball -r)" +export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" | jq .size -r)" +export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" | jq .shasum -r)" git clone https://github.com/ziglang/www.ziglang.org --depth 1 cd www.ziglang.org From 0e94530c515dab388828becacc0fe295c5f2b917 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 19:52:36 -0700 Subject: [PATCH 037/210] stage2: refactor 2 CObject fields to use CSourceFile --- BRANCH_TODO | 1 - src-self-hosted/Compilation.zig | 30 +++++++++++------------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 2e9da59df3..221d2ec28b 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ - * refactor 2 CObject fields to use CSourceFile * integrate code model and have_frame_pointer to main() and c objects * integrate target features into building C source files * integrate target features into building assembly code diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 85db6495b9..90ff6c9a80 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -117,10 +117,7 @@ const WorkItem = union(enum) { pub const CObject = struct { /// Relative to cwd. Owned by arena. - src_path: []const u8, - /// Owned by arena. - extra_flags: []const []const u8, - arena: std.heap.ArenaAllocator.State, + src: CSourceFile, status: union(enum) { new, success: struct { @@ -154,7 +151,7 @@ pub const CObject = struct { pub fn destroy(self: *CObject, gpa: *Allocator) void { _ = self.clearStatus(gpa); - self.arena.promote(gpa).deinit(); + gpa.destroy(self); } }; @@ -627,17 +624,12 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { // Add a `CObject` for each `c_source_files`. try comp.c_object_table.ensureCapacity(gpa, options.c_source_files.len); for (options.c_source_files) |c_source_file| { - var local_arena = std.heap.ArenaAllocator.init(gpa); - errdefer local_arena.deinit(); - - const c_object = try local_arena.allocator.create(CObject); + const c_object = try gpa.create(CObject); + errdefer gpa.destroy(c_object); c_object.* = .{ .status = .{ .new = {} }, - // TODO look into refactoring to turn these 2 fields simply into a CSourceFile - .src_path = try local_arena.allocator.dupe(u8, c_source_file.src_path), - .extra_flags = try local_arena.allocator.dupe([]const u8, c_source_file.extra_flags), - .arena = local_arena.state, + .src = c_source_file, }; comp.c_object_table.putAssumeCapacityNoClobber(c_object, {}); } @@ -798,7 +790,7 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors { for (self.failed_c_objects.items()) |entry| { const c_object = entry.key; const err_msg = entry.value; - try AllErrors.add(&arena, &errors, c_object.src_path, "", err_msg.*); + try AllErrors.add(&arena, &errors, c_object.src.src_path, "", err_msg.*); } if (self.bin_file.options.module) |module| { for (module.failed_files.items()) |entry| { @@ -981,13 +973,13 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { // TODO this logic can likely be improved by utilizing clang_options_data.zig. const file_args = [_][]const u8{"-include"}; var arg_i: usize = 0; - while (arg_i < c_object.extra_flags.len) : (arg_i += 1) { - const arg = c_object.extra_flags[arg_i]; + while (arg_i < c_object.src.extra_flags.len) : (arg_i += 1) { + const arg = c_object.src.extra_flags[arg_i]; ch.hash.addBytes(arg); for (file_args) |file_arg| { - if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_object.extra_flags.len) { + if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_object.src.extra_flags.len) { arg_i += 1; - _ = try ch.addFile(c_object.extra_flags[arg_i], null); + _ = try ch.addFile(c_object.src.extra_flags[arg_i], null); } } } @@ -1028,7 +1020,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { try argv.append(out_obj_path); try argv.append(c_object.src_path); - try argv.appendSlice(c_object.extra_flags); + try argv.appendSlice(c_object.src.extra_flags); if (comp.debug_cc) { for (argv.items[0 .. argv.items.len - 1]) |arg| { From 6acd903a9510d0e145b7bba3a9d42d4b9c07cc34 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 21:21:39 -0700 Subject: [PATCH 038/210] stage2: support for machine code model CLI --- BRANCH_TODO | 1 - src-self-hosted/Compilation.zig | 26 ++++++++++++++++++-------- src-self-hosted/link.zig | 1 + src-self-hosted/main.zig | 16 ++++++++++++++++ 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 221d2ec28b..c5af55bbf2 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ - * integrate code model and have_frame_pointer to main() and c objects * integrate target features into building C source files * integrate target features into building assembly code * handle .d files from c objects diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 90ff6c9a80..484f43e4ab 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -271,6 +271,7 @@ pub const InitOptions = struct { self_exe_path: ?[]const u8 = null, version: ?std.builtin.Version = null, libc_installation: ?*const LibCInstallation = null, + machine_code_model: std.builtin.CodeModel = .default, }; pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { @@ -319,6 +320,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { // We would also want to prefer LLVM for architectures that we don't have self-hosted support for too. break :blk false; }; + if (!use_llvm and options.machine_code_model != .default) { + return error.MachineCodeModelNotSupported; + } const is_exe_or_dyn_lib = switch (options.output_mode) { .Obj => false, @@ -425,6 +429,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { cache.hash.add(options.strip); cache.hash.add(options.link_libc); cache.hash.add(options.output_mode); + cache.hash.add(options.machine_code_model); // TODO audit this and make sure everything is in it const module: ?*Module = if (options.root_pkg) |root_pkg| blk: { @@ -593,6 +598,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .stack_check = stack_check, .single_threaded = single_threaded, .debug_link = options.debug_link, + .machine_code_model = options.machine_code_model, }); errdefer bin_file.destroy(); @@ -964,10 +970,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { ch.hash.addListOfBytes(comp.clang_argv); ch.hash.add(comp.bin_file.options.link_libcpp); ch.hash.addListOfBytes(comp.libc_include_dir_list); - // TODO - //cache_int(cache_hash, g->code_model); - //cache_bool(cache_hash, codegen_have_frame_pointer(g)); - _ = try ch.addFile(c_object.src_path, null); + _ = try ch.addFile(c_object.src.src_path, null); { // Hash the extra flags, with special care to call addFile for file parameters. // TODO this logic can likely be improved by utilizing clang_options_data.zig. @@ -989,7 +992,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { defer arena_allocator.deinit(); const arena = &arena_allocator.allocator; - const c_source_basename = std.fs.path.basename(c_object.src_path); + const c_source_basename = std.fs.path.basename(c_object.src.src_path); // Special case when doing build-obj for just one C file. When there are more than one object // file and building an object we need to link them together, but with just one it should go // directly to the output file. @@ -1012,14 +1015,14 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang", "-c" }); - const ext = classifyFileExt(c_object.src_path); + const ext = classifyFileExt(c_object.src.src_path); // TODO capture the .d file and deal with caching stuff try comp.addCCArgs(arena, &argv, ext, false, null); try argv.append("-o"); try argv.append(out_obj_path); - try argv.append(c_object.src_path); + try argv.append(c_object.src.src_path); try argv.appendSlice(c_object.src.extra_flags); if (comp.debug_cc) { @@ -1095,7 +1098,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { try std.os.renameat(zig_cache_tmp_dir.fd, tmp_basename, o_dir.fd, o_basename); ch.writeManifest() catch |err| { - std.log.warn("failed to write cache manifest when compiling '{}': {}", .{ c_object.src_path, @errorName(err) }); + std.log.warn("failed to write cache manifest when compiling '{}': {}", .{ c_object.src.src_path, @errorName(err) }); }; break :blk digest; }; @@ -1219,6 +1222,10 @@ fn addCCArgs( // flag = SplitIterator_next(&it); // } //} + const mcmodel = comp.bin_file.options.machine_code_model; + if (mcmodel != .default) { + try argv.append(try std.fmt.allocPrint(arena, "-mcmodel={}", .{@tagName(mcmodel)})); + } if (translate_c) { // This gives us access to preprocessing entities, presumably at the cost of performance. try argv.append("-Xclang"); @@ -1432,6 +1439,9 @@ test "classifyFileExt" { } fn haveFramePointer(comp: *Compilation) bool { + // If you complicate this logic make sure you update the parent cache hash. + // Right now it's not in the cache hash because the value depends on optimize_mode + // and strip which are both already part of the hash. return switch (comp.bin_file.options.optimize_mode) { .Debug, .ReleaseSafe => !comp.bin_file.options.strip, .ReleaseSmall, .ReleaseFast => false, diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 4d3fd5085d..cdeb2b5e5c 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -24,6 +24,7 @@ pub const Options = struct { link_mode: std.builtin.LinkMode, object_format: std.builtin.ObjectFormat, optimize_mode: std.builtin.Mode, + machine_code_model: std.builtin.CodeModel, root_name: []const u8, /// Not every Compilation compiles .zig code! For example you could do `zig build-exe foo.o`. module: ?*Module, diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 6ec2c45531..fcf548a2d6 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -183,6 +183,9 @@ const usage_build_generic = \\Compile Options: \\ -target [name] -- see the targets command \\ -mcpu [cpu] Specify target CPU and feature set + \\ -mcmodel=[default|tiny| Limit range of code and data virtual addresses + \\ small|kernel| + \\ medium|large] \\ --name [name] Override output name \\ --mode [mode] Set the build mode \\ Debug (default) optimizations off, safety on @@ -306,6 +309,7 @@ pub fn buildOutputType( var use_clang: ?bool = null; var link_eh_frame_hdr = false; var libc_paths_file: ?[]const u8 = null; + var machine_code_model: std.builtin.CodeModel = .default; var system_libs = std.ArrayList([]const u8).init(gpa); defer system_libs.deinit(); @@ -446,10 +450,16 @@ pub fn buildOutputType( if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; target_mcpu = args[i]; + } else if (mem.eql(u8, arg, "-mcmodel")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + machine_code_model = parseCodeModel(args[i]); } else if (mem.startsWith(u8, arg, "-ofmt=")) { target_ofmt = arg["-ofmt=".len..]; } else if (mem.startsWith(u8, arg, "-mcpu=")) { target_mcpu = arg["-mcpu=".len..]; + } else if (mem.startsWith(u8, arg, "-mcmodel=")) { + machine_code_model = parseCodeModel(arg["-mcmodel=".len..]); } else if (mem.eql(u8, arg, "--dynamic-linker")) { if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; @@ -1150,6 +1160,7 @@ pub fn buildOutputType( .libc_installation = if (libc_installation) |*lci| lci else null, .debug_cc = debug_cc, .debug_link = debug_link, + .machine_code_model = machine_code_model, }) catch |err| { fatal("unable to create compilation: {}", .{@errorName(err)}); }; @@ -1917,3 +1928,8 @@ fn is_libcpp_lib_name(target: std.Target, name: []const u8) bool { eqlIgnoreCase(ignore_case, name, "stdc++") or eqlIgnoreCase(ignore_case, name, "c++abi"); } + +fn parseCodeModel(arg: []const u8) std.builtin.CodeModel { + return std.meta.stringToEnum(std.builtin.CodeModel, arg) orelse + fatal("unsupported machine code model: '{}'", .{arg}); +} From 1ad60c43868212e3c4162196686f3017f80efc02 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 22:57:08 -0700 Subject: [PATCH 039/210] integrate target features into building C source files --- BRANCH_TODO | 1 - src-self-hosted/Compilation.zig | 27 ++++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index c5af55bbf2..5ec11da3f0 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ - * integrate target features into building C source files * integrate target features into building assembly code * handle .d files from c objects * glibc .so files diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 484f43e4ab..48bc20630b 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -1209,19 +1209,20 @@ fn addCCArgs( "-Xclang", "-target-cpu", "-Xclang", llvm_name, }); } - // TODO CLI args for target features - //if (g->zig_target->llvm_cpu_features != nullptr) { - // // https://github.com/ziglang/zig/issues/5017 - // SplitIterator it = memSplit(str(g->zig_target->llvm_cpu_features), str(",")); - // Optional> flag = SplitIterator_next(&it); - // while (flag.is_some) { - // try argv.append("-Xclang"); - // try argv.append("-target-feature"); - // try argv.append("-Xclang"); - // try argv.append(buf_ptr(buf_create_from_slice(flag.value))); - // flag = SplitIterator_next(&it); - // } - //} + + // It would be really nice if there was a more compact way to communicate this info to Clang. + const all_features_list = target.cpu.arch.allFeaturesList(); + try argv.ensureCapacity(argv.items.len + all_features_list.len * 4); + for (all_features_list) |feature, index_usize| { + const index = @intCast(std.Target.Cpu.Feature.Set.Index, index_usize); + const is_enabled = target.cpu.features.isEnabled(index); + + if (feature.llvm_name) |llvm_name| { + argv.appendSliceAssumeCapacity(&[_][]const u8{ "-Xclang", "-target-feature", "-Xclang" }); + const plus_or_minus = "-+"[@boolToInt(is_enabled)]; + try argv.append(try std.fmt.allocPrint(arena, "{c}{s}", .{ plus_or_minus, llvm_name })); + } + } const mcmodel = comp.bin_file.options.machine_code_model; if (mcmodel != .default) { try argv.append(try std.fmt.allocPrint(arena, "-mcmodel={}", .{@tagName(mcmodel)})); From 6e9396e32b4f31eb57ac18da510fff1ad24637a0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 23:03:13 -0700 Subject: [PATCH 040/210] integrate target features into building assembly code This brings us up to par from what stage1 does. There will still be an open issue for completing this. --- BRANCH_TODO | 2 +- src-self-hosted/Compilation.zig | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 5ec11da3f0..c3f80a204d 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ - * integrate target features into building assembly code * handle .d files from c objects * glibc .so files * support rpaths in ELF linker code @@ -46,3 +45,4 @@ * improve robustness of response file parsing * there are a couple panic("TODO") in clang options parsing * std.testing needs improvement to support exposing directory path for its tmp dir (look for "bogus") + * integrate target features into building assembly code diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 48bc20630b..94c6f7909f 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -1241,10 +1241,20 @@ fn addCCArgs( }, .so, .assembly, .ll, .bc, .unknown => {}, } - // TODO CLI args for cpu features when compiling assembly - //for (size_t i = 0; i < g->zig_target->llvm_cpu_features_asm_len; i += 1) { - // try argv.append(g->zig_target->llvm_cpu_features_asm_ptr[i]); - //} + // Argh, why doesn't the assembler accept the list of CPU features?! + // I don't see a way to do this other than hard coding everything. + switch (target.cpu.arch) { + .riscv32, .riscv64 => { + if (std.Target.riscv.featureSetHas(target.cpu.features, .relax)) { + try argv.append("-mrelax"); + } else { + try argv.append("-mno-relax"); + } + }, + else => { + // TODO + }, + } if (target.os.tag == .freestanding) { try argv.append("-ffreestanding"); From 4fa8cc736966544da381c90a6111158179fc550b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2020 00:40:33 -0700 Subject: [PATCH 041/210] stage2: don't depend on windows SDK C++ code when unavailable --- BRANCH_TODO | 3 +++ src-self-hosted/libc_installation.zig | 2 ++ 2 files changed, 5 insertions(+) diff --git a/BRANCH_TODO b/BRANCH_TODO index c3f80a204d..9f6bf9ffef 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -46,3 +46,6 @@ * there are a couple panic("TODO") in clang options parsing * std.testing needs improvement to support exposing directory path for its tmp dir (look for "bogus") * integrate target features into building assembly code + * libc_installation.zig: make it look for msvc only if msvc abi is chosen + * switch the default C ABI for windows to be mingw-w64 + * port windows_sdk.cpp to zig diff --git a/src-self-hosted/libc_installation.zig b/src-self-hosted/libc_installation.zig index eb59ef7e6a..8e380af936 100644 --- a/src-self-hosted/libc_installation.zig +++ b/src-self-hosted/libc_installation.zig @@ -169,6 +169,8 @@ pub const LibCInstallation = struct { var self: LibCInstallation = .{}; if (is_windows) { + if (!build_options.have_llvm) + return error.WindowsSdkNotFound; var sdk: *ZigWindowsSDK = undefined; switch (zig_find_windows_sdk(&sdk)) { .None => { From 1da9e0fcaf05fb78e4634278f51b6fe43509bd6f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2020 00:41:02 -0700 Subject: [PATCH 042/210] stage2: building glibc libc_nonshared.a CRT file --- src-self-hosted/glibc.zig | 98 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 5 deletions(-) diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index 80e66fa28e..6ecaf9721e 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -270,7 +270,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-g", "-Wa,--noexecstack", }); - return build_crt_file(comp, "crti.o", &[1]Compilation.CSourceFile{ + return build_crt_file(comp, "crti.o", .Obj, &[1]Compilation.CSourceFile{ .{ .src_path = try start_asm_path(comp, arena, "crti.S"), .extra_flags = args.items, @@ -288,7 +288,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-g", "-Wa,--noexecstack", }); - return build_crt_file(comp, "crtn.o", &[1]Compilation.CSourceFile{ + return build_crt_file(comp, "crtn.o", .Obj, &[1]Compilation.CSourceFile{ .{ .src_path = try start_asm_path(comp, arena, "crtn.S"), .extra_flags = args.items, @@ -339,10 +339,97 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }; }; - return build_crt_file(comp, "Scrt1.o", &[_]Compilation.CSourceFile{ start_os, abi_note_o }); + return build_crt_file(comp, "Scrt1.o", .Obj, &[_]Compilation.CSourceFile{ start_os, abi_note_o }); }, .libc_nonshared_a => { - return error.Unimplemented; // TODO + const deps = [_][]const u8{ + lib_libc_glibc ++ "stdlib" ++ path.sep_str ++ "atexit.c", + lib_libc_glibc ++ "stdlib" ++ path.sep_str ++ "at_quick_exit.c", + lib_libc_glibc ++ "io" ++ path.sep_str ++ "stat.c", + lib_libc_glibc ++ "io" ++ path.sep_str ++ "fstat.c", + lib_libc_glibc ++ "io" ++ path.sep_str ++ "lstat.c", + lib_libc_glibc ++ "io" ++ path.sep_str ++ "stat64.c", + lib_libc_glibc ++ "io" ++ path.sep_str ++ "fstat64.c", + lib_libc_glibc ++ "io" ++ path.sep_str ++ "lstat64.c", + lib_libc_glibc ++ "io" ++ path.sep_str ++ "fstatat.c", + lib_libc_glibc ++ "io" ++ path.sep_str ++ "fstatat64.c", + lib_libc_glibc ++ "io" ++ path.sep_str ++ "mknod.c", + lib_libc_glibc ++ "io" ++ path.sep_str ++ "mknodat.c", + lib_libc_glibc ++ "nptl" ++ path.sep_str ++ "pthread_atfork.c", + lib_libc_glibc ++ "debug" ++ path.sep_str ++ "stack_chk_fail_local.c", + }; + + var c_source_files: [deps.len + 1]Compilation.CSourceFile = undefined; + + c_source_files[0] = blk: { + var args = std.ArrayList([]const u8).init(arena); + try args.appendSlice(&[_][]const u8{ + "-std=gnu11", + "-fgnu89-inline", + "-g", + "-O2", + "-fmerge-all-constants", + "-fno-stack-protector", + "-fmath-errno", + "-fno-stack-protector", + "-I", + try lib_path(comp, arena, lib_libc_glibc ++ "csu"), + }); + try add_include_dirs(comp, arena, &args); + try args.appendSlice(&[_][]const u8{ + "-DSTACK_PROTECTOR_LEVEL=0", + "-fPIC", + "-fno-stack-protector", + "-ftls-model=initial-exec", + "-D_LIBC_REENTRANT", + "-include", + try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-modules.h"), + "-DMODULE_NAME=libc", + "-Wno-nonportable-include-path", + "-include", + try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-symbols.h"), + "-DPIC", + "-DLIBC_NONSHARED=1", + "-DTOP_NAMESPACE=glibc", + }); + break :blk .{ + .src_path = try lib_path(comp, arena, lib_libc_glibc ++ "csu" ++ path.sep_str ++ "elf-init.c"), + .extra_flags = args.items, + }; + }; + + for (deps) |dep, i| { + var args = std.ArrayList([]const u8).init(arena); + try args.appendSlice(&[_][]const u8{ + "-std=gnu11", + "-fgnu89-inline", + "-g", + "-O2", + "-fmerge-all-constants", + "-fno-stack-protector", + "-fmath-errno", + "-ftls-model=initial-exec", + "-Wno-ignored-attributes", + }); + try add_include_dirs(comp, arena, &args); + try args.appendSlice(&[_][]const u8{ + "-D_LIBC_REENTRANT", + "-include", + try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-modules.h"), + "-DMODULE_NAME=libc", + "-Wno-nonportable-include-path", + "-include", + try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-symbols.h"), + "-DPIC", + "-DLIBC_NONSHARED=1", + "-DTOP_NAMESPACE=glibc", + }); + c_source_files[i + 1] = .{ + .src_path = try lib_path(comp, arena, dep), + .extra_flags = args.items, + }; + } + return build_crt_file(comp, "libc_nonshared.a", .Lib, &c_source_files); }, } } @@ -588,6 +675,7 @@ fn lib_path(comp: *Compilation, arena: *Allocator, sub_path: []const u8) ![]cons fn build_crt_file( comp: *Compilation, basename: []const u8, + output_mode: std.builtin.OutputMode, c_source_files: []const Compilation.CSourceFile, ) !void { const tracy = trace(@src()); @@ -605,7 +693,7 @@ fn build_crt_file( .target = comp.getTarget(), .root_name = mem.split(basename, ".").next().?, .root_pkg = null, - .output_mode = .Obj, + .output_mode = output_mode, .rand = comp.rand, .libc_installation = comp.bin_file.options.libc_installation, .emit_bin = emit_bin, From b2860aa3e4d86e8712192c8a2068867e3ed04ef2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2020 12:52:58 -0700 Subject: [PATCH 043/210] stage2: add missing import to libc_installation.zig --- src-self-hosted/libc_installation.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/src-self-hosted/libc_installation.zig b/src-self-hosted/libc_installation.zig index 8e380af936..05ac8a8101 100644 --- a/src-self-hosted/libc_installation.zig +++ b/src-self-hosted/libc_installation.zig @@ -4,6 +4,7 @@ const Target = std.Target; const fs = std.fs; const Allocator = std.mem.Allocator; const Batch = std.event.Batch; +const build_options = @import("build_options"); const is_darwin = Target.current.isDarwin(); const is_windows = Target.current.os.tag == .windows; From c803d334d0b2f376006aa7709c7e282b52e9c987 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2020 13:40:01 -0700 Subject: [PATCH 044/210] update the zen of zig --- doc/langref.html.in | 5 +++-- src-self-hosted/main.zig | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 10bc81e6df..038aadd45f 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -11392,8 +11392,9 @@ keyword <- KEYWORD_align / KEYWORD_and / KEYWORD_anyframe / KEYWORD_anytype
  • Incremental improvements.
  • Avoid local maximums.
  • Reduce the amount one must remember.
    • -
  • Minimize energy spent on coding style.
    • -
  • Resource deallocation must succeed.
    • +
  • Focus on code rather than style.
    • +
  • Resource allocation may fail; resource deallocation must succeed.
    • +
  • Memory is a resource.
  • Together we serve the users.
    • {#header_close#} diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index fcf548a2d6..ab7456263c 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -1617,8 +1617,9 @@ pub const info_zen = \\ * Incremental improvements. \\ * Avoid local maximums. \\ * Reduce the amount one must remember. - \\ * Minimize energy spent on coding style. - \\ * Resource deallocation must succeed. + \\ * Focus on code rather than style. + \\ * Resource allocation may fail; resource deallocation must succeed. + \\ * Memory is a resource. \\ * Together we serve the users. \\ \\ From f82b1831f7d39d20451869d7aa94bd84de45fe12 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2020 15:24:13 -0700 Subject: [PATCH 045/210] std: handle sharing violation when deleting a file on windows --- lib/std/os/windows.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index bd9dc8b32e..405f7438f5 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -760,6 +760,7 @@ pub const DeleteFileError = error{ FileNotFound, AccessDenied, NameTooLong, + /// Also known as sharing violation. FileBusy, Unexpected, NotDir, @@ -824,6 +825,7 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil .INVALID_PARAMETER => unreachable, .FILE_IS_A_DIRECTORY => return error.IsDir, .NOT_A_DIRECTORY => return error.NotDir, + .SHARING_VIOLATION => return error.FileBusy, else => return unexpectedStatus(rc), } } From a0b43ff3b3c22cb5c57e6728d6cb35d722f22a3b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2020 15:24:34 -0700 Subject: [PATCH 046/210] stage2: eliminate one failure path in building c object --- src-self-hosted/Compilation.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 94c6f7909f..5fd99c2b90 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -1220,7 +1220,8 @@ fn addCCArgs( if (feature.llvm_name) |llvm_name| { argv.appendSliceAssumeCapacity(&[_][]const u8{ "-Xclang", "-target-feature", "-Xclang" }); const plus_or_minus = "-+"[@boolToInt(is_enabled)]; - try argv.append(try std.fmt.allocPrint(arena, "{c}{s}", .{ plus_or_minus, llvm_name })); + const arg = try std.fmt.allocPrint(arena, "{c}{s}", .{ plus_or_minus, llvm_name }); + argv.appendAssumeCapacity(arg); } } const mcmodel = comp.bin_file.options.machine_code_model; From 99a2fc2cde4c193d11322b2b22086fb4bc99f9fc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2020 18:02:42 -0700 Subject: [PATCH 047/210] stage2: implement .d file parsing for C objects --- BRANCH_TODO | 1 - src-self-hosted/Cache.zig | 39 + src-self-hosted/Compilation.zig | 44 +- .../{dep_tokenizer.zig => DepTokenizer.zig} | 754 +++++++++--------- 4 files changed, 436 insertions(+), 402 deletions(-) rename src-self-hosted/{dep_tokenizer.zig => DepTokenizer.zig} (54%) diff --git a/BRANCH_TODO b/BRANCH_TODO index 9f6bf9ffef..07d0eb2781 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ - * handle .d files from c objects * glibc .so files * support rpaths in ELF linker code * build & link against compiler-rt diff --git a/src-self-hosted/Cache.zig b/src-self-hosted/Cache.zig index b41c0a312f..8ad26a8f05 100644 --- a/src-self-hosted/Cache.zig +++ b/src-self-hosted/Cache.zig @@ -444,6 +444,45 @@ pub const CacheHash = struct { try self.populateFileHash(new_ch_file); } + pub fn addDepFilePost(self: *CacheHash, dir: fs.Dir, dep_file_basename: []const u8) !void { + assert(self.manifest_file != null); + + const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, MANIFEST_FILE_SIZE_MAX); + defer self.cache.gpa.free(dep_file_contents); + + const DepTokenizer = @import("DepTokenizer.zig"); + var it = DepTokenizer.init(self.cache.gpa, dep_file_contents); + defer it.deinit(); + + // Skip first token: target. + { + const opt_result = it.next() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.InvalidInput => { + std.log.err("failed parsing {}: {}: {}", .{ dep_file_basename, @errorName(err), it.error_text }); + return error.InvalidDepFile; + }, + }; + _ = opt_result orelse return; // Empty dep file OK. + } + // Process 0+ preqreqs. + // Clang is invoked in single-source mode so we never get more targets. + while (true) { + const opt_result = it.next() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.InvalidInput => { + std.log.err("failed parsing {}: {}: {}", .{ dep_file_basename, @errorName(err), it.error_text }); + return error.InvalidDepFile; + }, + }; + const result = opt_result orelse return; + switch (result.id) { + .target => return, + .prereq => try self.addFilePost(result.bytes), + } + } + } + /// Returns a base64 encoded hash of the inputs. pub fn final(self: *CacheHash) [BASE64_DIGEST_LEN]u8 { assert(self.manifest_file != null); diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 5fd99c2b90..b07137fdfa 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -1016,8 +1016,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang", "-c" }); const ext = classifyFileExt(c_object.src.src_path); - // TODO capture the .d file and deal with caching stuff - try comp.addCCArgs(arena, &argv, ext, false, null); + const out_dep_path: ?[]const u8 = if (comp.disable_c_depfile or !ext.clangSupportsDepFile()) + null + else + try std.fmt.allocPrint(arena, "{}.d", .{out_obj_path}); + try comp.addCCArgs(arena, &argv, ext, false, out_dep_path); try argv.append("-o"); try argv.append(out_obj_path); @@ -1086,7 +1089,15 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { } } - // TODO handle .d files + if (out_dep_path) |dep_file_path| { + const dep_basename = std.fs.path.basename(dep_file_path); + // Add the files depended on to the cache system. + try ch.addDepFilePost(zig_cache_tmp_dir, dep_basename); + // Just to save disk space, we delete the file because it is never needed again. + zig_cache_tmp_dir.deleteFile(dep_basename) catch |err| { + std.log.warn("failed to delete '{}': {}", .{ dep_file_path, @errorName(err) }); + }; + } // Rename into place. const digest = ch.final(); @@ -1118,11 +1129,12 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { fn tmpFilePath(comp: *Compilation, arena: *Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 { const s = std.fs.path.sep_str; - return std.fmt.allocPrint( - arena, - "{}" ++ s ++ "tmp" ++ s ++ "{x}-{}", - .{ comp.zig_cache_directory.path.?, comp.rand.int(u64), suffix }, - ); + const rand_int = comp.rand.int(u64); + if (comp.zig_cache_directory.path) |p| { + return std.fmt.allocPrint(arena, "{}" ++ s ++ "tmp" ++ s ++ "{x}-{s}", .{ p, rand_int, suffix }); + } else { + return std.fmt.allocPrint(arena, "tmp" ++ s ++ "{x}-{s}", .{ rand_int, suffix }); + } } /// Add common C compiler args between translate-c and C object compilation. @@ -1233,15 +1245,12 @@ fn addCCArgs( try argv.append("-Xclang"); try argv.append("-detailed-preprocessing-record"); } - if (out_dep_path) |p| { - try argv.append("-MD"); - try argv.append("-MV"); - try argv.append("-MF"); - try argv.append(p); - } }, .so, .assembly, .ll, .bc, .unknown => {}, } + if (out_dep_path) |p| { + try argv.appendSlice(&[_][]const u8{ "-MD", "-MV", "-MF", p }); + } // Argh, why doesn't the assembler accept the list of CPU features?! // I don't see a way to do this other than hard coding everything. switch (target.cpu.arch) { @@ -1388,6 +1397,13 @@ pub const FileExt = enum { assembly, so, unknown, + + pub fn clangSupportsDepFile(ext: FileExt) bool { + return switch (ext) { + .c, .cpp, .h => true, + .ll, .bc, .assembly, .so, .unknown => false, + }; + } }; pub fn hasCExt(filename: []const u8) bool { diff --git a/src-self-hosted/dep_tokenizer.zig b/src-self-hosted/DepTokenizer.zig similarity index 54% rename from src-self-hosted/dep_tokenizer.zig rename to src-self-hosted/DepTokenizer.zig index 20324cbf0c..262e7d558b 100644 --- a/src-self-hosted/dep_tokenizer.zig +++ b/src-self-hosted/DepTokenizer.zig @@ -1,360 +1,362 @@ +const Tokenizer = @This(); + +arena: std.heap.ArenaAllocator, +index: usize, +bytes: []const u8, +error_text: []const u8, +state: State, + const std = @import("std"); const testing = std.testing; +const assert = std.debug.assert; -pub const Tokenizer = struct { - arena: std.heap.ArenaAllocator, - index: usize, - bytes: []const u8, - error_text: []const u8, - state: State, +pub fn init(allocator: *std.mem.Allocator, bytes: []const u8) Tokenizer { + return Tokenizer{ + .arena = std.heap.ArenaAllocator.init(allocator), + .index = 0, + .bytes = bytes, + .error_text = "", + .state = State{ .lhs = {} }, + }; +} - pub fn init(allocator: *std.mem.Allocator, bytes: []const u8) Tokenizer { - return Tokenizer{ - .arena = std.heap.ArenaAllocator.init(allocator), - .index = 0, - .bytes = bytes, - .error_text = "", - .state = State{ .lhs = {} }, - }; - } +pub fn deinit(self: *Tokenizer) void { + self.arena.deinit(); +} - pub fn deinit(self: *Tokenizer) void { - self.arena.deinit(); - } - - pub fn next(self: *Tokenizer) Error!?Token { - while (self.index < self.bytes.len) { - const char = self.bytes[self.index]; - while (true) { - switch (self.state) { - .lhs => switch (char) { - '\t', '\n', '\r', ' ' => { - // silently ignore whitespace - break; // advance - }, - else => { - self.state = State{ .target = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) }; - }, +pub fn next(self: *Tokenizer) Error!?Token { + while (self.index < self.bytes.len) { + const char = self.bytes[self.index]; + while (true) { + switch (self.state) { + .lhs => switch (char) { + '\t', '\n', '\r', ' ' => { + // silently ignore whitespace + break; // advance }, - .target => |*target| switch (char) { - '\t', '\n', '\r', ' ' => { - return self.errorIllegalChar(self.index, char, "invalid target", .{}); - }, - '$' => { - self.state = State{ .target_dollar_sign = target.* }; - break; // advance - }, - '\\' => { - self.state = State{ .target_reverse_solidus = target.* }; - break; // advance - }, - ':' => { - self.state = State{ .target_colon = target.* }; - break; // advance - }, - else => { - try target.append(char); - break; // advance - }, + else => { + self.state = State{ .target = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) }; }, - .target_reverse_solidus => |*target| switch (char) { - '\t', '\n', '\r' => { - return self.errorIllegalChar(self.index, char, "bad target escape", .{}); - }, - ' ', '#', '\\' => { - try target.append(char); - self.state = State{ .target = target.* }; - break; // advance - }, - '$' => { - try target.appendSlice(self.bytes[self.index - 1 .. self.index]); - self.state = State{ .target_dollar_sign = target.* }; - break; // advance - }, - else => { - try target.appendSlice(self.bytes[self.index - 1 .. self.index + 1]); - self.state = State{ .target = target.* }; - break; // advance - }, + }, + .target => |*target| switch (char) { + '\t', '\n', '\r', ' ' => { + return self.errorIllegalChar(self.index, char, "invalid target", .{}); }, - .target_dollar_sign => |*target| switch (char) { - '$' => { - try target.append(char); - self.state = State{ .target = target.* }; - break; // advance - }, - else => { - return self.errorIllegalChar(self.index, char, "expecting '$'", .{}); - }, + '$' => { + self.state = State{ .target_dollar_sign = target.* }; + break; // advance }, - .target_colon => |*target| switch (char) { - '\n', '\r' => { - const bytes = target.span(); - if (bytes.len != 0) { - self.state = State{ .lhs = {} }; - return Token{ .id = .target, .bytes = bytes }; - } - // silently ignore null target + '\\' => { + self.state = State{ .target_reverse_solidus = target.* }; + break; // advance + }, + ':' => { + self.state = State{ .target_colon = target.* }; + break; // advance + }, + else => { + try target.append(char); + break; // advance + }, + }, + .target_reverse_solidus => |*target| switch (char) { + '\t', '\n', '\r' => { + return self.errorIllegalChar(self.index, char, "bad target escape", .{}); + }, + ' ', '#', '\\' => { + try target.append(char); + self.state = State{ .target = target.* }; + break; // advance + }, + '$' => { + try target.appendSlice(self.bytes[self.index - 1 .. self.index]); + self.state = State{ .target_dollar_sign = target.* }; + break; // advance + }, + else => { + try target.appendSlice(self.bytes[self.index - 1 .. self.index + 1]); + self.state = State{ .target = target.* }; + break; // advance + }, + }, + .target_dollar_sign => |*target| switch (char) { + '$' => { + try target.append(char); + self.state = State{ .target = target.* }; + break; // advance + }, + else => { + return self.errorIllegalChar(self.index, char, "expecting '$'", .{}); + }, + }, + .target_colon => |*target| switch (char) { + '\n', '\r' => { + const bytes = target.span(); + if (bytes.len != 0) { self.state = State{ .lhs = {} }; - continue; - }, - '\\' => { - self.state = State{ .target_colon_reverse_solidus = target.* }; - break; // advance - }, - else => { - const bytes = target.span(); - if (bytes.len != 0) { - self.state = State{ .rhs = {} }; - return Token{ .id = .target, .bytes = bytes }; - } - // silently ignore null target + return Token{ .id = .target, .bytes = bytes }; + } + // silently ignore null target + self.state = State{ .lhs = {} }; + continue; + }, + '\\' => { + self.state = State{ .target_colon_reverse_solidus = target.* }; + break; // advance + }, + else => { + const bytes = target.span(); + if (bytes.len != 0) { + self.state = State{ .rhs = {} }; + return Token{ .id = .target, .bytes = bytes }; + } + // silently ignore null target + self.state = State{ .lhs = {} }; + continue; + }, + }, + .target_colon_reverse_solidus => |*target| switch (char) { + '\n', '\r' => { + const bytes = target.span(); + if (bytes.len != 0) { self.state = State{ .lhs = {} }; - continue; - }, + return Token{ .id = .target, .bytes = bytes }; + } + // silently ignore null target + self.state = State{ .lhs = {} }; + continue; }, - .target_colon_reverse_solidus => |*target| switch (char) { - '\n', '\r' => { - const bytes = target.span(); - if (bytes.len != 0) { - self.state = State{ .lhs = {} }; - return Token{ .id = .target, .bytes = bytes }; - } - // silently ignore null target - self.state = State{ .lhs = {} }; - continue; - }, - else => { - try target.appendSlice(self.bytes[self.index - 2 .. self.index + 1]); - self.state = State{ .target = target.* }; - break; - }, + else => { + try target.appendSlice(self.bytes[self.index - 2 .. self.index + 1]); + self.state = State{ .target = target.* }; + break; }, - .rhs => switch (char) { - '\t', ' ' => { - // silently ignore horizontal whitespace - break; // advance - }, - '\n', '\r' => { - self.state = State{ .lhs = {} }; - continue; - }, - '\\' => { - self.state = State{ .rhs_continuation = {} }; - break; // advance - }, - '"' => { - self.state = State{ .prereq_quote = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) }; - break; // advance - }, - else => { - self.state = State{ .prereq = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) }; - }, + }, + .rhs => switch (char) { + '\t', ' ' => { + // silently ignore horizontal whitespace + break; // advance }, - .rhs_continuation => switch (char) { - '\n' => { - self.state = State{ .rhs = {} }; - break; // advance - }, - '\r' => { - self.state = State{ .rhs_continuation_linefeed = {} }; - break; // advance - }, - else => { - return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{}); - }, + '\n', '\r' => { + self.state = State{ .lhs = {} }; + continue; }, - .rhs_continuation_linefeed => switch (char) { - '\n' => { - self.state = State{ .rhs = {} }; - break; // advance - }, - else => { - return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{}); - }, + '\\' => { + self.state = State{ .rhs_continuation = {} }; + break; // advance }, - .prereq_quote => |*prereq| switch (char) { - '"' => { - const bytes = prereq.span(); - self.index += 1; - self.state = State{ .rhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; - }, - else => { - try prereq.append(char); - break; // advance - }, + '"' => { + self.state = State{ .prereq_quote = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) }; + break; // advance }, - .prereq => |*prereq| switch (char) { - '\t', ' ' => { - const bytes = prereq.span(); - self.state = State{ .rhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; - }, - '\n', '\r' => { - const bytes = prereq.span(); - self.state = State{ .lhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; - }, - '\\' => { - self.state = State{ .prereq_continuation = prereq.* }; - break; // advance - }, - else => { - try prereq.append(char); - break; // advance - }, + else => { + self.state = State{ .prereq = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) }; }, - .prereq_continuation => |*prereq| switch (char) { - '\n' => { - const bytes = prereq.span(); - self.index += 1; - self.state = State{ .rhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; - }, - '\r' => { - self.state = State{ .prereq_continuation_linefeed = prereq.* }; - break; // advance - }, - else => { - // not continuation - try prereq.appendSlice(self.bytes[self.index - 1 .. self.index + 1]); - self.state = State{ .prereq = prereq.* }; - break; // advance - }, + }, + .rhs_continuation => switch (char) { + '\n' => { + self.state = State{ .rhs = {} }; + break; // advance }, - .prereq_continuation_linefeed => |prereq| switch (char) { - '\n' => { - const bytes = prereq.span(); - self.index += 1; - self.state = State{ .rhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; - }, - else => { - return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{}); - }, + '\r' => { + self.state = State{ .rhs_continuation_linefeed = {} }; + break; // advance }, - } + else => { + return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{}); + }, + }, + .rhs_continuation_linefeed => switch (char) { + '\n' => { + self.state = State{ .rhs = {} }; + break; // advance + }, + else => { + return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{}); + }, + }, + .prereq_quote => |*prereq| switch (char) { + '"' => { + const bytes = prereq.span(); + self.index += 1; + self.state = State{ .rhs = {} }; + return Token{ .id = .prereq, .bytes = bytes }; + }, + else => { + try prereq.append(char); + break; // advance + }, + }, + .prereq => |*prereq| switch (char) { + '\t', ' ' => { + const bytes = prereq.span(); + self.state = State{ .rhs = {} }; + return Token{ .id = .prereq, .bytes = bytes }; + }, + '\n', '\r' => { + const bytes = prereq.span(); + self.state = State{ .lhs = {} }; + return Token{ .id = .prereq, .bytes = bytes }; + }, + '\\' => { + self.state = State{ .prereq_continuation = prereq.* }; + break; // advance + }, + else => { + try prereq.append(char); + break; // advance + }, + }, + .prereq_continuation => |*prereq| switch (char) { + '\n' => { + const bytes = prereq.span(); + self.index += 1; + self.state = State{ .rhs = {} }; + return Token{ .id = .prereq, .bytes = bytes }; + }, + '\r' => { + self.state = State{ .prereq_continuation_linefeed = prereq.* }; + break; // advance + }, + else => { + // not continuation + try prereq.appendSlice(self.bytes[self.index - 1 .. self.index + 1]); + self.state = State{ .prereq = prereq.* }; + break; // advance + }, + }, + .prereq_continuation_linefeed => |prereq| switch (char) { + '\n' => { + const bytes = prereq.span(); + self.index += 1; + self.state = State{ .rhs = {} }; + return Token{ .id = .prereq, .bytes = bytes }; + }, + else => { + return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{}); + }, + }, } - self.index += 1; } - - // eof, handle maybe incomplete token - if (self.index == 0) return null; - const idx = self.index - 1; - switch (self.state) { - .lhs, - .rhs, - .rhs_continuation, - .rhs_continuation_linefeed, - => {}, - .target => |target| { - return self.errorPosition(idx, target.span(), "incomplete target", .{}); - }, - .target_reverse_solidus, - .target_dollar_sign, - => { - const index = self.index - 1; - return self.errorIllegalChar(idx, self.bytes[idx], "incomplete escape", .{}); - }, - .target_colon => |target| { - const bytes = target.span(); - if (bytes.len != 0) { - self.index += 1; - self.state = State{ .rhs = {} }; - return Token{ .id = .target, .bytes = bytes }; - } - // silently ignore null target - self.state = State{ .lhs = {} }; - }, - .target_colon_reverse_solidus => |target| { - const bytes = target.span(); - if (bytes.len != 0) { - self.index += 1; - self.state = State{ .rhs = {} }; - return Token{ .id = .target, .bytes = bytes }; - } - // silently ignore null target - self.state = State{ .lhs = {} }; - }, - .prereq_quote => |prereq| { - return self.errorPosition(idx, prereq.span(), "incomplete quoted prerequisite", .{}); - }, - .prereq => |prereq| { - const bytes = prereq.span(); - self.state = State{ .lhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; - }, - .prereq_continuation => |prereq| { - const bytes = prereq.span(); - self.state = State{ .lhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; - }, - .prereq_continuation_linefeed => |prereq| { - const bytes = prereq.span(); - self.state = State{ .lhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; - }, - } - return null; + self.index += 1; } - fn errorf(self: *Tokenizer, comptime fmt: []const u8, args: anytype) Error { - self.error_text = try std.fmt.allocPrintZ(&self.arena.allocator, fmt, args); - return Error.InvalidInput; + // eof, handle maybe incomplete token + if (self.index == 0) return null; + const idx = self.index - 1; + switch (self.state) { + .lhs, + .rhs, + .rhs_continuation, + .rhs_continuation_linefeed, + => {}, + .target => |target| { + return self.errorPosition(idx, target.span(), "incomplete target", .{}); + }, + .target_reverse_solidus, + .target_dollar_sign, + => { + const index = self.index - 1; + return self.errorIllegalChar(idx, self.bytes[idx], "incomplete escape", .{}); + }, + .target_colon => |target| { + const bytes = target.span(); + if (bytes.len != 0) { + self.index += 1; + self.state = State{ .rhs = {} }; + return Token{ .id = .target, .bytes = bytes }; + } + // silently ignore null target + self.state = State{ .lhs = {} }; + }, + .target_colon_reverse_solidus => |target| { + const bytes = target.span(); + if (bytes.len != 0) { + self.index += 1; + self.state = State{ .rhs = {} }; + return Token{ .id = .target, .bytes = bytes }; + } + // silently ignore null target + self.state = State{ .lhs = {} }; + }, + .prereq_quote => |prereq| { + return self.errorPosition(idx, prereq.span(), "incomplete quoted prerequisite", .{}); + }, + .prereq => |prereq| { + const bytes = prereq.span(); + self.state = State{ .lhs = {} }; + return Token{ .id = .prereq, .bytes = bytes }; + }, + .prereq_continuation => |prereq| { + const bytes = prereq.span(); + self.state = State{ .lhs = {} }; + return Token{ .id = .prereq, .bytes = bytes }; + }, + .prereq_continuation_linefeed => |prereq| { + const bytes = prereq.span(); + self.state = State{ .lhs = {} }; + return Token{ .id = .prereq, .bytes = bytes }; + }, } + return null; +} - fn errorPosition(self: *Tokenizer, position: usize, bytes: []const u8, comptime fmt: []const u8, args: anytype) Error { - var buffer = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0); - try buffer.outStream().print(fmt, args); - try buffer.appendSlice(" '"); - var out = makeOutput(std.ArrayListSentineled(u8, 0).appendSlice, &buffer); - try printCharValues(&out, bytes); - try buffer.appendSlice("'"); - try buffer.outStream().print(" at position {}", .{position - (bytes.len - 1)}); - self.error_text = buffer.span(); - return Error.InvalidInput; - } +fn errorf(self: *Tokenizer, comptime fmt: []const u8, args: anytype) Error { + self.error_text = try std.fmt.allocPrintZ(&self.arena.allocator, fmt, args); + return Error.InvalidInput; +} - fn errorIllegalChar(self: *Tokenizer, position: usize, char: u8, comptime fmt: []const u8, args: anytype) Error { - var buffer = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0); - try buffer.appendSlice("illegal char "); - try printUnderstandableChar(&buffer, char); - try buffer.outStream().print(" at position {}", .{position}); - if (fmt.len != 0) try buffer.outStream().print(": " ++ fmt, args); - self.error_text = buffer.span(); - return Error.InvalidInput; - } +fn errorPosition(self: *Tokenizer, position: usize, bytes: []const u8, comptime fmt: []const u8, args: anytype) Error { + var buffer = std.ArrayList(u8).init(&self.arena.allocator); + try buffer.outStream().print(fmt, args); + try buffer.appendSlice(" '"); + const out = buffer.writer(); + try printCharValues(out, bytes); + try buffer.appendSlice("'"); + try buffer.outStream().print(" at position {}", .{position - (bytes.len - 1)}); + try buffer.append(0); + self.error_text = buffer.items[0 .. buffer.items.len - 1 :0]; + return Error.InvalidInput; +} - const Error = error{ - OutOfMemory, - InvalidInput, - }; +fn errorIllegalChar(self: *Tokenizer, position: usize, char: u8, comptime fmt: []const u8, args: anytype) Error { + var buffer = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0); + try buffer.appendSlice("illegal char "); + try printUnderstandableChar(&buffer, char); + try buffer.outStream().print(" at position {}", .{position}); + if (fmt.len != 0) try buffer.outStream().print(": " ++ fmt, args); + self.error_text = buffer.span(); + return Error.InvalidInput; +} - const State = union(enum) { - lhs: void, - target: std.ArrayListSentineled(u8, 0), - target_reverse_solidus: std.ArrayListSentineled(u8, 0), - target_dollar_sign: std.ArrayListSentineled(u8, 0), - target_colon: std.ArrayListSentineled(u8, 0), - target_colon_reverse_solidus: std.ArrayListSentineled(u8, 0), - rhs: void, - rhs_continuation: void, - rhs_continuation_linefeed: void, - prereq_quote: std.ArrayListSentineled(u8, 0), - prereq: std.ArrayListSentineled(u8, 0), - prereq_continuation: std.ArrayListSentineled(u8, 0), - prereq_continuation_linefeed: std.ArrayListSentineled(u8, 0), - }; +const Error = error{ + OutOfMemory, + InvalidInput, +}; - const Token = struct { - id: ID, - bytes: []const u8, +const State = union(enum) { + lhs: void, + target: std.ArrayListSentineled(u8, 0), + target_reverse_solidus: std.ArrayListSentineled(u8, 0), + target_dollar_sign: std.ArrayListSentineled(u8, 0), + target_colon: std.ArrayListSentineled(u8, 0), + target_colon_reverse_solidus: std.ArrayListSentineled(u8, 0), + rhs: void, + rhs_continuation: void, + rhs_continuation_linefeed: void, + prereq_quote: std.ArrayListSentineled(u8, 0), + prereq: std.ArrayListSentineled(u8, 0), + prereq_continuation: std.ArrayListSentineled(u8, 0), + prereq_continuation_linefeed: std.ArrayListSentineled(u8, 0), +}; - const ID = enum { - target, - prereq, - }; +pub const Token = struct { + id: ID, + bytes: []const u8, + + pub const ID = enum { + target, + prereq, }; }; @@ -836,7 +838,7 @@ test "error prereq - continuation expecting end-of-line" { // - tokenize input, emit textual representation, and compare to expect fn depTokenizer(input: []const u8, expect: []const u8) !void { - var arena_allocator = std.heap.ArenaAllocator.init(std.heap.page_allocator); + var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator); const arena = &arena_allocator.allocator; defer arena_allocator.deinit(); @@ -872,13 +874,13 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void { return; } - var out = makeOutput(std.fs.File.write, try std.io.getStdErr()); + const out = std.io.getStdErr().writer(); - try out.write("\n"); - try printSection(&out, "<<<< input", input); - try printSection(&out, "==== expect", expect); - try printSection(&out, ">>>> got", got); - try printRuler(&out); + try out.writeAll("\n"); + try printSection(out, "<<<< input", input); + try printSection(out, "==== expect", expect); + try printSection(out, ">>>> got", got); + try printRuler(out); testing.expect(false); } @@ -887,29 +889,29 @@ fn printSection(out: anytype, label: []const u8, bytes: []const u8) !void { try printLabel(out, label, bytes); try hexDump(out, bytes); try printRuler(out); - try out.write(bytes); - try out.write("\n"); + try out.writeAll(bytes); + try out.writeAll("\n"); } fn printLabel(out: anytype, label: []const u8, bytes: []const u8) !void { var buf: [80]u8 = undefined; var text = try std.fmt.bufPrint(buf[0..], "{} {} bytes ", .{ label, bytes.len }); - try out.write(text); + try out.writeAll(text); var i: usize = text.len; const end = 79; while (i < 79) : (i += 1) { - try out.write([_]u8{label[0]}); + try out.writeAll(&[_]u8{label[0]}); } - try out.write("\n"); + try out.writeAll("\n"); } fn printRuler(out: anytype) !void { var i: usize = 0; const end = 79; while (i < 79) : (i += 1) { - try out.write("-"); + try out.writeAll("-"); } - try out.write("\n"); + try out.writeAll("\n"); } fn hexDump(out: anytype, bytes: []const u8) !void { @@ -924,80 +926,80 @@ fn hexDump(out: anytype, bytes: []const u8) !void { const n = bytes.len & 0x0f; if (n > 0) { try printDecValue(out, offset, 8); - try out.write(":"); - try out.write(" "); + try out.writeAll(":"); + try out.writeAll(" "); var end1 = std.math.min(offset + n, offset + 8); for (bytes[offset..end1]) |b| { - try out.write(" "); + try out.writeAll(" "); try printHexValue(out, b, 2); } var end2 = offset + n; if (end2 > end1) { - try out.write(" "); + try out.writeAll(" "); for (bytes[end1..end2]) |b| { - try out.write(" "); + try out.writeAll(" "); try printHexValue(out, b, 2); } } const short = 16 - n; var i: usize = 0; while (i < short) : (i += 1) { - try out.write(" "); + try out.writeAll(" "); } if (end2 > end1) { - try out.write(" |"); + try out.writeAll(" |"); } else { - try out.write(" |"); + try out.writeAll(" |"); } try printCharValues(out, bytes[offset..end2]); - try out.write("|\n"); + try out.writeAll("|\n"); offset += n; } try printDecValue(out, offset, 8); - try out.write(":"); - try out.write("\n"); + try out.writeAll(":"); + try out.writeAll("\n"); } fn hexDump16(out: anytype, offset: usize, bytes: []const u8) !void { try printDecValue(out, offset, 8); - try out.write(":"); - try out.write(" "); + try out.writeAll(":"); + try out.writeAll(" "); for (bytes[0..8]) |b| { - try out.write(" "); + try out.writeAll(" "); try printHexValue(out, b, 2); } - try out.write(" "); + try out.writeAll(" "); for (bytes[8..16]) |b| { - try out.write(" "); + try out.writeAll(" "); try printHexValue(out, b, 2); } - try out.write(" |"); + try out.writeAll(" |"); try printCharValues(out, bytes); - try out.write("|\n"); + try out.writeAll("|\n"); } fn printDecValue(out: anytype, value: u64, width: u8) !void { var buffer: [20]u8 = undefined; - const len = std.fmt.formatIntBuf(buffer[0..], value, 10, false, width); - try out.write(buffer[0..len]); + const len = std.fmt.formatIntBuf(buffer[0..], value, 10, false, .{ .width = width, .fill = '0' }); + try out.writeAll(buffer[0..len]); } fn printHexValue(out: anytype, value: u64, width: u8) !void { var buffer: [16]u8 = undefined; - const len = std.fmt.formatIntBuf(buffer[0..], value, 16, false, width); - try out.write(buffer[0..len]); + const len = std.fmt.formatIntBuf(buffer[0..], value, 16, false, .{ .width = width, .fill = '0' }); + try out.writeAll(buffer[0..len]); } fn printCharValues(out: anytype, bytes: []const u8) !void { for (bytes) |b| { - try out.write(&[_]u8{printable_char_tab[b]}); + try out.writeAll(&[_]u8{printable_char_tab[b]}); } } fn printUnderstandableChar(buffer: *std.ArrayListSentineled(u8, 0), char: u8) !void { if (!std.ascii.isPrint(char) or char == ' ') { - try buffer.outStream().print("\\x{X:2}", .{char}); + try buffer.outStream().print("\\x{X:0>2}", .{char}); } else { try buffer.appendSlice("'"); try buffer.append(printable_char_tab[char]); @@ -1013,27 +1015,5 @@ const printable_char_tab: []const u8 = "................................................................"; // zig fmt: on comptime { - std.debug.assert(printable_char_tab.len == 256); -} - -// Make an output var that wraps a context and output function. -// output: must be a function that takes a `self` idiom parameter -// and a bytes parameter -// context: must be that self -fn makeOutput(comptime output: anytype, context: anytype) Output(output, @TypeOf(context)) { - return Output(output, @TypeOf(context)){ - .context = context, - }; -} - -fn Output(comptime output_func: anytype, comptime Context: type) type { - return struct { - context: Context, - - pub const output = output_func; - - fn write(self: @This(), bytes: []const u8) !void { - try output_func(self.context, bytes); - } - }; + assert(printable_char_tab.len == 256); } From da0fde59b6ffde5d505f664aa27b728aa2b1cc2a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2020 18:08:29 -0700 Subject: [PATCH 048/210] stage2: update to new LibCInstallation API --- BRANCH_TODO | 1 - src-self-hosted/main.zig | 5 ++--- src-self-hosted/stage2.zig | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 07d0eb2781..49fdbe2f51 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -33,7 +33,6 @@ * incremental compilation - implement detection of which source files changed * improve the cache hash logic for c objects with respect to extra flags and file parameters * LLVM codegen backend: put a sub-arch in the triple in some cases - * rework libc_installation.zig abstraction to use std.log instead of taking a stderr stream * implement an LLVM backend for stage2 * implement outputting dynamic libraries in self-hosted linker * implement outputting static libraries (archive files) in self-hosted linker diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index ab7456263c..127fa1ae08 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -1086,7 +1086,7 @@ pub fn buildOutputType( defer if (libc_installation) |*l| l.deinit(gpa); if (libc_paths_file) |paths_file| { - libc_installation = LibCInstallation.parse(gpa, paths_file, io.getStdErr().writer()) catch |err| { + libc_installation = LibCInstallation.parse(gpa, paths_file) catch |err| { fatal("unable to parse libc paths file: {}", .{@errorName(err)}); }; } @@ -1269,8 +1269,7 @@ pub fn cmdLibC(gpa: *Allocator, args: []const []const u8) !void { } } if (input_file) |libc_file| { - const stderr = std.io.getStdErr().writer(); - var libc = LibCInstallation.parse(gpa, libc_file, stderr) catch |err| { + var libc = LibCInstallation.parse(gpa, libc_file) catch |err| { fatal("unable to parse libc file: {}", .{@errorName(err)}); }; defer libc.deinit(gpa); diff --git a/src-self-hosted/stage2.zig b/src-self-hosted/stage2.zig index 623da30e2c..bfa37b603e 100644 --- a/src-self-hosted/stage2.zig +++ b/src-self-hosted/stage2.zig @@ -12,7 +12,7 @@ const ArrayListSentineled = std.ArrayListSentineled; const Target = std.Target; const CrossTarget = std.zig.CrossTarget; const self_hosted_main = @import("main.zig"); -const DepTokenizer = @import("dep_tokenizer.zig").Tokenizer; +const DepTokenizer = @import("DepTokenizer.zig"); const assert = std.debug.assert; const LibCInstallation = @import("libc_installation.zig").LibCInstallation; @@ -21,7 +21,7 @@ var stderr: fs.File.OutStream = undefined; var stdout: fs.File.OutStream = undefined; comptime { - _ = @import("dep_tokenizer.zig"); + _ = @import("DepTokenizer.zig"); } // ABI warning From 95941c4e7021588f224f0559c6f66de3a205b972 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2020 03:02:46 -0700 Subject: [PATCH 049/210] stage2: building glibc shared objects * caching system: use 16 bytes siphash final(), there was a bug in the std lib that wasn't catching undefined values for 18 bytes. fixed in master branch. * fix caching system unit test logic to not cause error.TextBusy on windows * port the logic from stage1 for building glibc shared objects * add is_native_os to the base cache hash * fix incorrectly freeing crt_files key (which is always a reference to global static constant data) * fix 2 use-after-free in loading glibc metadata * fix memory leak in buildCRTFile (errdefer instead of defer on arena) --- BRANCH_TODO | 11 +- ci/azure/windows_mingw_script | 4 + src-self-hosted/Cache.zig | 293 +++++++++++++++++--------------- src-self-hosted/Compilation.zig | 51 +++--- src-self-hosted/glibc.zig | 273 +++++++++++++++++++++++++++-- 5 files changed, 446 insertions(+), 186 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 49fdbe2f51..3275c48a53 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,7 @@ * glibc .so files + - stage1 C++ code integration + - ok file + * use hex for cache hash file paths * support rpaths in ELF linker code * build & link against compiler-rt * build & link againstn freestanding libc @@ -21,7 +24,6 @@ * implement proper parsing of LLD stderr/stdout and exposing compile errors * implement proper parsing of clang stderr/stdout and exposing compile errors * implement proper compile errors for failing to build glibc crt files and shared libs - * skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd) * self-host link.cpp and building libcs (#4313 and #4314). using the `zig cc` command will set a flag indicating a preference for the llvm backend, which will include linking with LLD. At least for now. If zig's self-hosted linker ever gets on par with the likes of ld and lld, we can make it always be used even for zig cc. * improve the stage2 tests to support testing with LLVM extensions enabled * multi-thread building C objects @@ -47,3 +49,10 @@ * libc_installation.zig: make it look for msvc only if msvc abi is chosen * switch the default C ABI for windows to be mingw-w64 * port windows_sdk.cpp to zig + * change glibc log errors to normal exposed compile errors + * update Package to use Compilation.Directory in create() + - skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd) + (maybe make it an explicit option and have main.zig disable it) + - make it possible for Package to not openDir and reference already existing resources. + * rename src/ to src/stage1/ + * rename src-self-hosted/ to src/ diff --git a/ci/azure/windows_mingw_script b/ci/azure/windows_mingw_script index 7883766427..f7cecf96c9 100644 --- a/ci/azure/windows_mingw_script +++ b/ci/azure/windows_mingw_script @@ -18,4 +18,8 @@ cmake .. -G 'MSYS Makefiles' -DCMAKE_BUILD_TYPE=RelWithDebInfo $CMAKEFLAGS -DCMA make -j$(nproc) install +# I saw a failure due to `git diff` being > 400 KB instead of empty as expected so this is to debug it. +git status +git diff | head -n100 + ./zig build test-behavior -Dskip-non-native -Dskip-release diff --git a/src-self-hosted/Cache.zig b/src-self-hosted/Cache.zig index 8ad26a8f05..95925851ff 100644 --- a/src-self-hosted/Cache.zig +++ b/src-self-hosted/Cache.zig @@ -26,9 +26,9 @@ pub fn obtain(cache: *const Cache) CacheHash { pub const base64_encoder = fs.base64_encoder; pub const base64_decoder = fs.base64_decoder; -/// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 -/// We round up to 18 to avoid the `==` padding after base64 encoding. -pub const BIN_DIGEST_LEN = 18; +/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 +/// Currently we use SipHash and so this value must be 16 not any higher. +pub const BIN_DIGEST_LEN = 16; pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN); const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024; @@ -87,14 +87,29 @@ pub const HashHelper = struct { hh.add(x.major); hh.add(x.minor); hh.add(x.patch); - return; }, - else => {}, - } - - switch (@typeInfo(@TypeOf(x))) { - .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)), - else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))), + std.Target.Os.TaggedVersionRange => { + switch (x) { + .linux => |linux| { + hh.add(linux.range.min); + hh.add(linux.range.max); + hh.add(linux.glibc); + }, + .windows => |windows| { + hh.add(windows.min); + hh.add(windows.max); + }, + .semver => |semver| { + hh.add(semver.min); + hh.add(semver.max); + }, + .none => {}, + } + }, + else => switch (@typeInfo(@TypeOf(x))) { + .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)), + else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))), + }, } } @@ -613,44 +628,46 @@ test "cache file and then recall it" { var digest1: [BASE64_DIGEST_LEN]u8 = undefined; var digest2: [BASE64_DIGEST_LEN]u8 = undefined; - var cache = Cache{ - .gpa = testing.allocator, - .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), - }; - defer cache.manifest_dir.close(); - { - var ch = cache.obtain(); - defer ch.deinit(); + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + defer cache.manifest_dir.close(); - ch.hash.add(true); - ch.hash.add(@as(u16, 1234)); - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file, null); + { + var ch = cache.obtain(); + defer ch.deinit(); - // There should be nothing in the cache - testing.expectEqual(false, try ch.hit()); + ch.hash.add(true); + ch.hash.add(@as(u16, 1234)); + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file, null); - digest1 = ch.final(); - try ch.writeManifest(); + // There should be nothing in the cache + testing.expectEqual(false, try ch.hit()); + + digest1 = ch.final(); + try ch.writeManifest(); + } + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.add(true); + ch.hash.add(@as(u16, 1234)); + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file, null); + + // Cache hit! We just "built" the same file + testing.expect(try ch.hit()); + digest2 = ch.final(); + + try ch.writeManifest(); + } + + testing.expectEqual(digest1, digest2); } - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.add(true); - ch.hash.add(@as(u16, 1234)); - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file, null); - - // Cache hit! We just "built" the same file - testing.expect(try ch.hit()); - digest2 = ch.final(); - - try ch.writeManifest(); - } - - testing.expectEqual(digest1, digest2); try cwd.deleteTree(temp_manifest_dir); try cwd.deleteFile(temp_file); @@ -693,51 +710,53 @@ test "check that changing a file makes cache fail" { var digest1: [BASE64_DIGEST_LEN]u8 = undefined; var digest2: [BASE64_DIGEST_LEN]u8 = undefined; - var cache = Cache{ - .gpa = testing.allocator, - .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), - }; - defer cache.manifest_dir.close(); - { - var ch = cache.obtain(); - defer ch.deinit(); + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + defer cache.manifest_dir.close(); - ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addFile(temp_file, 100); + { + var ch = cache.obtain(); + defer ch.deinit(); - // There should be nothing in the cache - testing.expectEqual(false, try ch.hit()); + ch.hash.addBytes("1234"); + const temp_file_idx = try ch.addFile(temp_file, 100); - testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?)); + // There should be nothing in the cache + testing.expectEqual(false, try ch.hit()); - digest1 = ch.final(); + testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?)); - try ch.writeManifest(); + digest1 = ch.final(); + + try ch.writeManifest(); + } + + try cwd.writeFile(temp_file, updated_temp_file_contents); + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + const temp_file_idx = try ch.addFile(temp_file, 100); + + // A file that we depend on has been updated, so the cache should not contain an entry for it + testing.expectEqual(false, try ch.hit()); + + // The cache system does not keep the contents of re-hashed input files. + testing.expect(ch.files.items[temp_file_idx].contents == null); + + digest2 = ch.final(); + + try ch.writeManifest(); + } + + testing.expect(!mem.eql(u8, digest1[0..], digest2[0..])); } - try cwd.writeFile(temp_file, updated_temp_file_contents); - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addFile(temp_file, 100); - - // A file that we depend on has been updated, so the cache should not contain an entry for it - testing.expectEqual(false, try ch.hit()); - - // The cache system does not keep the contents of re-hashed input files. - testing.expect(ch.files.items[temp_file_idx].contents == null); - - digest2 = ch.final(); - - try ch.writeManifest(); - } - - testing.expect(!mem.eql(u8, digest1[0..], digest2[0..])); - try cwd.deleteTree(temp_manifest_dir); try cwd.deleteTree(temp_file); } @@ -749,7 +768,7 @@ test "no file inputs" { } const cwd = fs.cwd(); const temp_manifest_dir = "no_file_inputs_manifest_dir"; - defer cwd.deleteTree(temp_manifest_dir) catch unreachable; + defer cwd.deleteTree(temp_manifest_dir) catch {}; var digest1: [BASE64_DIGEST_LEN]u8 = undefined; var digest2: [BASE64_DIGEST_LEN]u8 = undefined; @@ -810,67 +829,69 @@ test "CacheHashes with files added after initial hash work" { var digest2: [BASE64_DIGEST_LEN]u8 = undefined; var digest3: [BASE64_DIGEST_LEN]u8 = undefined; - var cache = Cache{ - .gpa = testing.allocator, - .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), - }; - defer cache.manifest_dir.close(); - { - var ch = cache.obtain(); - defer ch.deinit(); + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + defer cache.manifest_dir.close(); - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file1, null); + { + var ch = cache.obtain(); + defer ch.deinit(); - // There should be nothing in the cache - testing.expectEqual(false, try ch.hit()); + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file1, null); - _ = try ch.addFilePost(temp_file2); + // There should be nothing in the cache + testing.expectEqual(false, try ch.hit()); - digest1 = ch.final(); - try ch.writeManifest(); + _ = try ch.addFilePost(temp_file2); + + digest1 = ch.final(); + try ch.writeManifest(); + } + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file1, null); + + testing.expect(try ch.hit()); + digest2 = ch.final(); + + try ch.writeManifest(); + } + testing.expect(mem.eql(u8, &digest1, &digest2)); + + // Modify the file added after initial hash + const ts2 = std.time.nanoTimestamp(); + try cwd.writeFile(temp_file2, "Hello world the second, updated\n"); + + while (isProblematicTimestamp(ts2)) { + std.time.sleep(1); + } + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file1, null); + + // A file that we depend on has been updated, so the cache should not contain an entry for it + testing.expectEqual(false, try ch.hit()); + + _ = try ch.addFilePost(temp_file2); + + digest3 = ch.final(); + + try ch.writeManifest(); + } + + testing.expect(!mem.eql(u8, &digest1, &digest3)); } - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file1, null); - - testing.expect(try ch.hit()); - digest2 = ch.final(); - - try ch.writeManifest(); - } - testing.expect(mem.eql(u8, &digest1, &digest2)); - - // Modify the file added after initial hash - const ts2 = std.time.nanoTimestamp(); - try cwd.writeFile(temp_file2, "Hello world the second, updated\n"); - - while (isProblematicTimestamp(ts2)) { - std.time.sleep(1); - } - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file1, null); - - // A file that we depend on has been updated, so the cache should not contain an entry for it - testing.expectEqual(false, try ch.hit()); - - _ = try ch.addFilePost(temp_file2); - - digest3 = ch.final(); - - try ch.writeManifest(); - } - - testing.expect(!mem.eql(u8, &digest1, &digest3)); try cwd.deleteTree(temp_manifest_dir); try cwd.deleteFile(temp_file1); diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index b07137fdfa..d99a85e1d3 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -68,7 +68,9 @@ libunwind_static_lib: ?[]const u8 = null, /// and resolved before calling linker.flush(). libc_static_lib: ?[]const u8 = null, -/// For example `Scrt1.o` and `libc.so.6`. These are populated after building libc from source, +glibc_so_files: ?glibc.BuiltSharedObjects = null, + +/// For example `Scrt1.o` and `libc_nonshared.a`. These are populated after building libc from source, /// The set of needed CRT (C runtime) files differs depending on the target and compilation settings. /// The key is the basename, and the value is the absolute path to the completed build artifact. crt_files: std.StringHashMapUnmanaged(CRTFile) = .{}, @@ -111,8 +113,8 @@ const WorkItem = union(enum) { /// one of the glibc static objects glibc_crt_file: glibc.CRTFile, - /// one of the glibc shared objects - glibc_so: *const glibc.Lib, + /// all of the glibc shared objects + glibc_shared_objects, }; pub const CObject = struct { @@ -272,6 +274,9 @@ pub const InitOptions = struct { version: ?std.builtin.Version = null, libc_installation: ?*const LibCInstallation = null, machine_code_model: std.builtin.CodeModel = .default, + /// TODO Once self-hosted Zig is capable enough, we can remove this special-case + /// hack in favor of more general compilation options. + stage1_is_dummy_so: bool = false, }; pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { @@ -421,6 +426,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { cache.hash.addBytes(options.target.cpu.model.name); cache.hash.add(options.target.cpu.features.ints); cache.hash.add(options.target.os.tag); + cache.hash.add(options.is_native_os); cache.hash.add(options.target.abi); cache.hash.add(ofmt); cache.hash.add(pic); @@ -446,22 +452,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { hash.addOptionalBytes(root_pkg.root_src_directory.path); hash.add(valgrind); hash.add(single_threaded); - switch (options.target.os.getVersionRange()) { - .linux => |linux| { - hash.add(linux.range.min); - hash.add(linux.range.max); - hash.add(linux.glibc); - }, - .windows => |windows| { - hash.add(windows.min); - hash.add(windows.max); - }, - .semver => |semver| { - hash.add(semver.min); - hash.add(semver.max); - }, - .none => {}, - } + hash.add(options.target.os.getVersionRange()); const digest = hash.final(); const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); @@ -660,7 +651,6 @@ pub fn destroy(self: *Compilation) void { { var it = self.crt_files.iterator(); while (it.next()) |entry| { - gpa.free(entry.key); entry.value.deinit(gpa); } self.crt_files.deinit(gpa); @@ -936,14 +926,15 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { }, .glibc_crt_file => |crt_file| { glibc.buildCRTFile(self, crt_file) catch |err| { - // This is a problem with the Zig installation. It's mostly OK to crash here, - // but TODO because it would be even better if we could recover gracefully - // from temporary problems such as out-of-disk-space. + // TODO Expose this as a normal compile error rather than crashing here. fatal("unable to build glibc CRT file: {}", .{@errorName(err)}); }; }, - .glibc_so => |glibc_lib| { - fatal("TODO build glibc shared object '{}.so.{}'", .{ glibc_lib.name, glibc_lib.sover }); + .glibc_shared_objects => { + glibc.buildSharedObjects(self) catch |err| { + // TODO Expose this as a normal compile error rather than crashing here. + fatal("unable to build glibc shared objects: {}", .{@errorName(err)}); + }; }, }; } @@ -1587,17 +1578,13 @@ pub fn get_libc_crt_file(comp: *Compilation, arena: *Allocator, basename: []cons } fn addBuildingGLibCWorkItems(comp: *Compilation) !void { - const static_file_work_items = [_]WorkItem{ + try comp.work_queue.write(&[_]WorkItem{ .{ .glibc_crt_file = .crti_o }, .{ .glibc_crt_file = .crtn_o }, .{ .glibc_crt_file = .scrt1_o }, .{ .glibc_crt_file = .libc_nonshared_a }, - }; - try comp.work_queue.ensureUnusedCapacity(static_file_work_items.len + glibc.libs.len); - comp.work_queue.writeAssumeCapacity(&static_file_work_items); - for (glibc.libs) |*glibc_so| { - comp.work_queue.writeItemAssumeCapacity(.{ .glibc_so = glibc_so }); - } + .{ .glibc_shared_objects = {} }, + }); } fn wantBuildGLibCFromSource(comp: *Compilation) bool { diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index 6ecaf9721e..31876bf4b2 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -1,11 +1,15 @@ const std = @import("std"); const Allocator = std.mem.Allocator; -const target_util = @import("target.zig"); const mem = std.mem; -const Compilation = @import("Compilation.zig"); const path = std.fs.path; +const assert = std.debug.assert; + +const target_util = @import("target.zig"); +const Compilation = @import("Compilation.zig"); const build_options = @import("build_options"); const trace = @import("tracy.zig").trace; +const Cache = @import("Cache.zig"); +const Package = @import("Package.zig"); pub const Lib = struct { name: []const u8, @@ -83,14 +87,14 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError! }; defer gpa.free(vers_txt_contents); - const fns_txt_contents = glibc_dir.readFileAlloc(gpa, "fns.txt", max_txt_size) catch |err| switch (err) { + // Arena allocated because the result contains references to function names. + const fns_txt_contents = glibc_dir.readFileAlloc(arena, "fns.txt", max_txt_size) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => { std.log.err("unable to read fns.txt: {}", .{@errorName(err)}); return error.ZigInstallationCorrupt; }, }; - defer gpa.free(fns_txt_contents); const abi_txt_contents = glibc_dir.readFileAlloc(gpa, "abi.txt", max_txt_size) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, @@ -183,7 +187,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError! .os = .linux, .abi = abi_tag, }; - try version_table.put(arena, triple, ver_list_base.ptr); + try version_table.put(gpa, triple, ver_list_base.ptr); } break :blk ver_list_base; }; @@ -250,7 +254,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { } const gpa = comp.gpa; var arena_allocator = std.heap.ArenaAllocator.init(gpa); - errdefer arena_allocator.deinit(); + defer arena_allocator.deinit(); const arena = &arena_allocator.allocator; switch (crt_file) { @@ -713,6 +717,252 @@ fn build_crt_file( }); defer sub_compilation.destroy(); + try updateSubCompilation(sub_compilation); + + try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1); + const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p| + try path.join(comp.gpa, &[_][]const u8{ p, basename }) + else + try comp.gpa.dupe(u8, basename); + + comp.crt_files.putAssumeCapacityNoClobber(basename, .{ + .full_object_path = artifact_path, + .lock = sub_compilation.bin_file.toOwnedLock(), + }); +} + +pub const BuiltSharedObjects = struct { + lock: Cache.Lock, + dir_path: []u8, + + pub fn deinit(self: *BuiltSharedObjects, gpa: *Allocator) void { + self.lock.release(); + gpa.free(self.dir_path); + self.* = undefined; + } +}; + +const all_map_basename = "all.map"; + +pub fn buildSharedObjects(comp: *Compilation) !void { + const tracy = trace(@src()); + defer tracy.end(); + + var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); + defer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + const target = comp.getTarget(); + const target_version = target.os.version_range.linux.glibc; + + // TODO use the global cache directory here + var cache_parent: Cache = .{ + .gpa = comp.gpa, + .manifest_dir = comp.cache_parent.manifest_dir, + }; + var cache = cache_parent.obtain(); + defer cache.deinit(); + cache.hash.addBytes(build_options.version); + cache.hash.addBytes(comp.zig_lib_directory.path orelse "."); + cache.hash.add(target.cpu.arch); + cache.hash.addBytes(target.cpu.model.name); + cache.hash.add(target.cpu.features.ints); + cache.hash.add(target.abi); + cache.hash.add(target_version); + + const hit = try cache.hit(); + const digest = cache.final(); + const o_sub_path = try path.join(arena, &[_][]const u8{ "o", &digest }); + if (!hit) { + var o_directory: Compilation.Directory = .{ + .handle = try comp.zig_cache_directory.handle.makeOpenPath(o_sub_path, .{}), + .path = try path.join(arena, &[_][]const u8{ comp.zig_cache_directory.path.?, o_sub_path }), + }; + defer o_directory.handle.close(); + + const metadata = try loadMetaData(comp.gpa, comp.zig_lib_directory.handle); + defer metadata.destroy(comp.gpa); + + const ver_list_base = metadata.version_table.get(.{ + .arch = target.cpu.arch, + .os = target.os.tag, + .abi = target.abi, + }) orelse return error.GLibCUnavailableForThisTarget; + const target_ver_index = for (metadata.all_versions) |ver, i| { + switch (ver.order(target_version)) { + .eq => break i, + .lt => continue, + .gt => { + // TODO Expose via compile error mechanism instead of log. + std.log.warn("invalid target glibc version: {}", .{target_version}); + return error.InvalidTargetGLibCVersion; + }, + } + } else blk: { + const latest_index = metadata.all_versions.len - 1; + std.log.warn("zig cannot build new glibc version {}; providing instead {}", .{ + target_version, metadata.all_versions[latest_index], + }); + break :blk latest_index; + }; + { + var map_contents = std.ArrayList(u8).init(arena); + for (metadata.all_versions) |ver| { + if (ver.patch == 0) { + try map_contents.writer().print("GLIBC_{d}.{d} {{ }};\n", .{ ver.major, ver.minor }); + } else { + try map_contents.writer().print("GLIBC_{d}.{d}.{d} {{ }};\n", .{ ver.major, ver.minor, ver.patch }); + } + } + try o_directory.handle.writeFile(all_map_basename, map_contents.items); + map_contents.deinit(); // The most recent allocation of an arena can be freed :) + } + var zig_body = std.ArrayList(u8).init(comp.gpa); + defer zig_body.deinit(); + var zig_footer = std.ArrayList(u8).init(comp.gpa); + defer zig_footer.deinit(); + for (libs) |*lib| { + zig_body.shrinkRetainingCapacity(0); + zig_footer.shrinkRetainingCapacity(0); + + try zig_body.appendSlice( + \\comptime { + \\ asm ( + \\ + ); + for (metadata.all_functions) |*libc_fn, fn_i| { + if (libc_fn.lib != lib) continue; + + const ver_list = ver_list_base[fn_i]; + // Pick the default symbol version: + // - If there are no versions, don't emit it + // - Take the greatest one <= than the target one + // - If none of them is <= than the + // specified one don't pick any default version + if (ver_list.len == 0) continue; + var chosen_def_ver_index: u8 = 255; + { + var ver_i: u8 = 0; + while (ver_i < ver_list.len) : (ver_i += 1) { + const ver_index = ver_list.versions[ver_i]; + if ((chosen_def_ver_index == 255 or ver_index > chosen_def_ver_index) and + target_ver_index >= ver_index) + { + chosen_def_ver_index = ver_index; + } + } + } + { + var ver_i: u8 = 0; + while (ver_i < ver_list.len) : (ver_i += 1) { + const ver_index = ver_list.versions[ver_i]; + const ver = metadata.all_versions[ver_index]; + const sym_name = libc_fn.name; + const stub_name = if (ver.patch == 0) + try std.fmt.allocPrint(arena, "{s}_{d}_{d}", .{ sym_name, ver.major, ver.minor }) + else + try std.fmt.allocPrint(arena, "{s}_{d}_{d}_{d}", .{ sym_name, ver.major, ver.minor, ver.patch }); + + try zig_footer.writer().print("export fn {s}() void {{}}\n", .{stub_name}); + + // Default symbol version definition vs normal symbol version definition + const want_two_ats = chosen_def_ver_index != 255 and ver_index == chosen_def_ver_index; + const at_sign_str = "@@"[0 .. @boolToInt(want_two_ats) + @as(usize, 1)]; + if (ver.patch == 0) { + try zig_body.writer().print(" \\\\ .symver {s}, {s}{s}GLIBC_{d}.{d}\n", .{ + stub_name, sym_name, at_sign_str, ver.major, ver.minor, + }); + } else { + try zig_body.writer().print(" \\\\ .symver {s}, {s}{s}GLIBC_{d}.{d}.{d}\n", .{ + stub_name, sym_name, at_sign_str, ver.major, ver.minor, ver.patch, + }); + } + // Hide the stub to keep the symbol table clean + try zig_body.writer().print(" \\\\ .hidden {s}\n", .{stub_name}); + } + } + } + + try zig_body.appendSlice( + \\ ); + \\} + \\ + ); + try zig_body.appendSlice(zig_footer.items); + + var lib_name_buf: [32]u8 = undefined; // Larger than each of the names "c", "pthread", etc. + const zig_file_basename = std.fmt.bufPrint(&lib_name_buf, "{s}.zig", .{lib.name}) catch unreachable; + try o_directory.handle.writeFile(zig_file_basename, zig_body.items); + + try buildSharedLib(comp, arena, comp.zig_cache_directory, o_directory, zig_file_basename, lib); + } + cache.writeManifest() catch |err| { + std.log.warn("glibc shared objects: failed to write cache manifest: {}", .{@errorName(err)}); + }; + } + + assert(comp.glibc_so_files == null); + comp.glibc_so_files = BuiltSharedObjects{ + .lock = cache.toOwnedLock(), + .dir_path = try path.join(comp.gpa, &[_][]const u8{ comp.zig_cache_directory.path.?, o_sub_path }), + }; +} + +fn buildSharedLib( + comp: *Compilation, + arena: *Allocator, + zig_cache_directory: Compilation.Directory, + bin_directory: Compilation.Directory, + zig_file_basename: []const u8, + lib: *const Lib, +) !void { + const tracy = trace(@src()); + defer tracy.end(); + + const emit_bin = Compilation.EmitLoc{ + .directory = bin_directory, + .basename = try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ lib.name, lib.sover }), + }; + const version: std.builtin.Version = .{ .major = lib.sover, .minor = 0, .patch = 0 }; + const ld_basename = path.basename(comp.getTarget().standardDynamicLinkerPath().get().?); + const override_soname = if (mem.eql(u8, lib.name, "ld")) ld_basename else null; + const map_file_path = try path.join(arena, &[_][]const u8{ bin_directory.path.?, all_map_basename }); + // TODO we should be able to just give the open directory to Package + const root_pkg = try Package.create(comp.gpa, std.fs.cwd(), bin_directory.path.?, zig_file_basename); + defer root_pkg.destroy(comp.gpa); + const sub_compilation = try Compilation.create(comp.gpa, .{ + .zig_cache_directory = zig_cache_directory, + .zig_lib_directory = comp.zig_lib_directory, + .target = comp.getTarget(), + .root_name = lib.name, + .root_pkg = null, + .output_mode = .Lib, + .link_mode = .Dynamic, + .rand = comp.rand, + .libc_installation = comp.bin_file.options.libc_installation, + .emit_bin = emit_bin, + .optimize_mode = comp.bin_file.options.optimize_mode, + .want_sanitize_c = false, + .want_stack_check = false, + .want_valgrind = false, + .emit_h = null, + .strip = comp.bin_file.options.strip, + .is_native_os = false, + .self_exe_path = comp.self_exe_path, + .debug_cc = comp.debug_cc, + .debug_link = comp.bin_file.options.debug_link, + .clang_passthrough_mode = comp.clang_passthrough_mode, + .version = version, + .stage1_is_dummy_so = true, + .version_script = map_file_path, + .override_soname = override_soname, + }); + defer sub_compilation.destroy(); + + try updateSubCompilation(sub_compilation); +} + +fn updateSubCompilation(sub_compilation: *Compilation) !void { try sub_compilation.update(); // Look for compilation errors in this sub_compilation @@ -730,15 +980,4 @@ fn build_crt_file( } return error.BuildingLibCObjectFailed; } - - try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1); - const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p| - try std.fs.path.join(comp.gpa, &[_][]const u8{ p, basename }) - else - try comp.gpa.dupe(u8, basename); - - comp.crt_files.putAssumeCapacityNoClobber(basename, .{ - .full_object_path = artifact_path, - .lock = sub_compilation.bin_file.toOwnedLock(), - }); } From 80d8515e3a9aa1edf2549a190e8e4d495d8028ca Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2020 10:49:43 -0700 Subject: [PATCH 050/210] ci: disable git autocrlf See also commit 2c8495b4bb261d440bc6d1a6b0415a64c8d99222 --- BRANCH_TODO | 2 ++ ci/azure/windows_mingw_script | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 3275c48a53..037def2191 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,6 @@ * glibc .so files + - without stage1 c++ code integration it should fail with a better error message + instead of lld not getting the object file on the linker line for some reason. - stage1 C++ code integration - ok file * use hex for cache hash file paths diff --git a/ci/azure/windows_mingw_script b/ci/azure/windows_mingw_script index f7cecf96c9..a6bfd81477 100644 --- a/ci/azure/windows_mingw_script +++ b/ci/azure/windows_mingw_script @@ -7,6 +7,11 @@ pacman --noconfirm --needed -S git base-devel mingw-w64-x86_64-toolchain mingw-w git config core.abbrev 9 +# Git is wrong for autocrlf being enabled by default on Windows. +# git is mangling files on Windows by default. +# This is the second bug I've tracked down to being caused by autocrlf. +git config core.autocrlf false + ZIGBUILDDIR="$(pwd)/build" PREFIX="$ZIGBUILDDIR/dist" CMAKEFLAGS="-DCMAKE_COLOR_MAKEFILE=OFF -DCMAKE_INSTALL_PREFIX=$PREFIX -DZIG_STATIC=ON" @@ -18,8 +23,4 @@ cmake .. -G 'MSYS Makefiles' -DCMAKE_BUILD_TYPE=RelWithDebInfo $CMAKEFLAGS -DCMA make -j$(nproc) install -# I saw a failure due to `git diff` being > 400 KB instead of empty as expected so this is to debug it. -git status -git diff | head -n100 - ./zig build test-behavior -Dskip-non-native -Dskip-release From a863d899e121f11fd21c3bd14afe47a9a0ff23b5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2020 11:58:26 -0700 Subject: [PATCH 051/210] ci: undo mangled autocrlf files before building --- ci/azure/windows_mingw_script | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/azure/windows_mingw_script b/ci/azure/windows_mingw_script index a6bfd81477..900baebee3 100644 --- a/ci/azure/windows_mingw_script +++ b/ci/azure/windows_mingw_script @@ -11,6 +11,8 @@ git config core.abbrev 9 # git is mangling files on Windows by default. # This is the second bug I've tracked down to being caused by autocrlf. git config core.autocrlf false +# Too late; the files are already mangled. +git checkout . ZIGBUILDDIR="$(pwd)/build" PREFIX="$ZIGBUILDDIR/dist" From 883dcb8b18ca2f1cfa7208273f7b5dad18532a7e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2020 12:31:42 -0700 Subject: [PATCH 052/210] stage2 Cache: use hex instead of base64 for file paths --- BRANCH_TODO | 1 - src-self-hosted/Cache.zig | 87 ++++++++++++++++++--------------------- 2 files changed, 41 insertions(+), 47 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 037def2191..0f9354e32e 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -3,7 +3,6 @@ instead of lld not getting the object file on the linker line for some reason. - stage1 C++ code integration - ok file - * use hex for cache hash file paths * support rpaths in ELF linker code * build & link against compiler-rt * build & link againstn freestanding libc diff --git a/src-self-hosted/Cache.zig b/src-self-hosted/Cache.zig index 95925851ff..125353ebe2 100644 --- a/src-self-hosted/Cache.zig +++ b/src-self-hosted/Cache.zig @@ -6,7 +6,6 @@ const Cache = @This(); const std = @import("std"); const crypto = std.crypto; const fs = std.fs; -const base64 = std.base64; const assert = std.debug.assert; const testing = std.testing; const mem = std.mem; @@ -20,18 +19,15 @@ pub fn obtain(cache: *const Cache) CacheHash { .hash = cache.hash, .manifest_file = null, .manifest_dirty = false, - .b64_digest = undefined, + .hex_digest = undefined, }; } -pub const base64_encoder = fs.base64_encoder; -pub const base64_decoder = fs.base64_decoder; /// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 -/// Currently we use SipHash and so this value must be 16 not any higher. -pub const BIN_DIGEST_LEN = 16; -pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN); +pub const bin_digest_len = 16; +pub const hex_digest_len = bin_digest_len * 2; -const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024; +const manifest_file_size_max = 50 * 1024 * 1024; /// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it /// provides enough collision resistance for the CacheHash use cases, while being one of our @@ -45,7 +41,7 @@ pub const File = struct { path: ?[]const u8, max_file_size: ?usize, stat: fs.File.Stat, - bin_digest: [BIN_DIGEST_LEN]u8, + bin_digest: [bin_digest_len]u8, contents: ?[]const u8, pub fn deinit(self: *File, allocator: *Allocator) void { @@ -118,20 +114,19 @@ pub const HashHelper = struct { hh.add(optional orelse return); } - /// Returns a base64 encoded hash of the inputs, without modifying state. - pub fn peek(hh: HashHelper) [BASE64_DIGEST_LEN]u8 { + /// Returns a hex encoded hash of the inputs, without modifying state. + pub fn peek(hh: HashHelper) [hex_digest_len]u8 { var copy = hh; return copy.final(); } - /// Returns a base64 encoded hash of the inputs, mutating the state of the hasher. - pub fn final(hh: *HashHelper) [BASE64_DIGEST_LEN]u8 { - var bin_digest: [BIN_DIGEST_LEN]u8 = undefined; + /// Returns a hex encoded hash of the inputs, mutating the state of the hasher. + pub fn final(hh: *HashHelper) [hex_digest_len]u8 { + var bin_digest: [bin_digest_len]u8 = undefined; hh.hasher.final(&bin_digest); - var out_digest: [BASE64_DIGEST_LEN]u8 = undefined; - base64_encoder.encode(&out_digest, &bin_digest); - + var out_digest: [hex_digest_len]u8 = undefined; + _ = std.fmt.bufPrint(&out_digest, "{x}", .{bin_digest}) catch unreachable; return out_digest; } }; @@ -155,7 +150,7 @@ pub const CacheHash = struct { manifest_file: ?fs.File, manifest_dirty: bool, files: std.ArrayListUnmanaged(File) = .{}, - b64_digest: [BASE64_DIGEST_LEN]u8, + hex_digest: [hex_digest_len]u8, /// Add a file as a dependency of process being cached. When `hit` is /// called, the file's contents will be checked to ensure that it matches @@ -205,7 +200,7 @@ pub const CacheHash = struct { } /// Check the cache to see if the input exists in it. If it exists, returns `true`. - /// A base64 encoding of its hash is available by calling `final`. + /// A hex encoding of its hash is available by calling `final`. /// /// This function will also acquire an exclusive lock to the manifest file. This means /// that a process holding a CacheHash will block any other process attempting to @@ -218,18 +213,18 @@ pub const CacheHash = struct { assert(self.manifest_file == null); const ext = ".txt"; - var manifest_file_path: [self.b64_digest.len + ext.len]u8 = undefined; + var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined; - var bin_digest: [BIN_DIGEST_LEN]u8 = undefined; + var bin_digest: [bin_digest_len]u8 = undefined; self.hash.hasher.final(&bin_digest); - base64_encoder.encode(self.b64_digest[0..], &bin_digest); + _ = std.fmt.bufPrint(&self.hex_digest, "{x}", .{bin_digest}) catch unreachable; self.hash.hasher = hasher_init; self.hash.hasher.update(&bin_digest); - mem.copy(u8, &manifest_file_path, &self.b64_digest); - manifest_file_path[self.b64_digest.len..][0..ext.len].* = ext.*; + mem.copy(u8, &manifest_file_path, &self.hex_digest); + manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*; if (self.files.items.len != 0) { self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ @@ -258,7 +253,7 @@ pub const CacheHash = struct { }; } - const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, MANIFEST_FILE_SIZE_MAX); + const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, manifest_file_size_max); defer self.cache.gpa.free(file_contents); const input_file_count = self.files.items.len; @@ -290,7 +285,7 @@ pub const CacheHash = struct { cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat; cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat; cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat; - base64_decoder.decode(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat; + std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat; if (file_path.len == 0) { return error.InvalidFormat; @@ -325,7 +320,7 @@ pub const CacheHash = struct { cache_hash_file.stat.inode = 0; } - var actual_digest: [BIN_DIGEST_LEN]u8 = undefined; + var actual_digest: [bin_digest_len]u8 = undefined; try hashFile(this_file, &actual_digest); if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) { @@ -462,7 +457,7 @@ pub const CacheHash = struct { pub fn addDepFilePost(self: *CacheHash, dir: fs.Dir, dep_file_basename: []const u8) !void { assert(self.manifest_file != null); - const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, MANIFEST_FILE_SIZE_MAX); + const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max); defer self.cache.gpa.free(dep_file_contents); const DepTokenizer = @import("DepTokenizer.zig"); @@ -498,8 +493,8 @@ pub const CacheHash = struct { } } - /// Returns a base64 encoded hash of the inputs. - pub fn final(self: *CacheHash) [BASE64_DIGEST_LEN]u8 { + /// Returns a hex encoded hash of the inputs. + pub fn final(self: *CacheHash) [hex_digest_len]u8 { assert(self.manifest_file != null); // We don't close the manifest file yet, because we want to @@ -508,11 +503,11 @@ pub const CacheHash = struct { // cache_release is called we still might be working on creating // the artifacts to cache. - var bin_digest: [BIN_DIGEST_LEN]u8 = undefined; + var bin_digest: [bin_digest_len]u8 = undefined; self.hash.hasher.final(&bin_digest); - var out_digest: [BASE64_DIGEST_LEN]u8 = undefined; - base64_encoder.encode(&out_digest, &bin_digest); + var out_digest: [hex_digest_len]u8 = undefined; + _ = std.fmt.bufPrint(&out_digest, "{x}", .{bin_digest}) catch unreachable; return out_digest; } @@ -521,18 +516,18 @@ pub const CacheHash = struct { assert(self.manifest_file != null); if (!self.manifest_dirty) return; - var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined; + var encoded_digest: [hex_digest_len]u8 = undefined; var contents = std.ArrayList(u8).init(self.cache.gpa); var writer = contents.writer(); defer contents.deinit(); for (self.files.items) |file| { - base64_encoder.encode(encoded_digest[0..], &file.bin_digest); - try writer.print("{} {} {} {} {}\n", .{ + _ = std.fmt.bufPrint(&encoded_digest, "{x}", .{file.bin_digest}) catch unreachable; + try writer.print("{d} {d} {d} {s} {s}\n", .{ file.stat.size, file.stat.inode, file.stat.mtime, - encoded_digest[0..], + &encoded_digest, file.path, }); } @@ -625,8 +620,8 @@ test "cache file and then recall it" { std.time.sleep(1); } - var digest1: [BASE64_DIGEST_LEN]u8 = undefined; - var digest2: [BASE64_DIGEST_LEN]u8 = undefined; + var digest1: [hex_digest_len]u8 = undefined; + var digest2: [hex_digest_len]u8 = undefined; { var cache = Cache{ @@ -707,8 +702,8 @@ test "check that changing a file makes cache fail" { std.time.sleep(1); } - var digest1: [BASE64_DIGEST_LEN]u8 = undefined; - var digest2: [BASE64_DIGEST_LEN]u8 = undefined; + var digest1: [hex_digest_len]u8 = undefined; + var digest2: [hex_digest_len]u8 = undefined; { var cache = Cache{ @@ -770,8 +765,8 @@ test "no file inputs" { const temp_manifest_dir = "no_file_inputs_manifest_dir"; defer cwd.deleteTree(temp_manifest_dir) catch {}; - var digest1: [BASE64_DIGEST_LEN]u8 = undefined; - var digest2: [BASE64_DIGEST_LEN]u8 = undefined; + var digest1: [hex_digest_len]u8 = undefined; + var digest2: [hex_digest_len]u8 = undefined; var cache = Cache{ .gpa = testing.allocator, @@ -825,9 +820,9 @@ test "CacheHashes with files added after initial hash work" { std.time.sleep(1); } - var digest1: [BASE64_DIGEST_LEN]u8 = undefined; - var digest2: [BASE64_DIGEST_LEN]u8 = undefined; - var digest3: [BASE64_DIGEST_LEN]u8 = undefined; + var digest1: [hex_digest_len]u8 = undefined; + var digest2: [hex_digest_len]u8 = undefined; + var digest3: [hex_digest_len]u8 = undefined; { var cache = Cache{ From 670260aab6e3ee720b4ef6d6d0234f48ed83b76b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2020 13:41:28 -0700 Subject: [PATCH 053/210] stage2: std.log.err is an error not a warning --- src-self-hosted/main.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 127fa1ae08..132a7d3947 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -81,8 +81,8 @@ pub fn log( // We only recognize 4 log levels in this application. const level_txt = switch (level) { - .emerg, .alert, .crit => "error", - .err, .warn => "warning", + .emerg, .alert, .crit, .err => "error", + .warn => "warning", .notice, .info => "info", .debug => "debug", }; From 3256b3c48536407e64b641b55bde064a9ac34606 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2020 13:41:53 -0700 Subject: [PATCH 054/210] stage2: glibc shared objects use ok file to detect cache hits --- src-self-hosted/glibc.zig | 45 +++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index 31876bf4b2..bf7f0f6450 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -773,13 +773,26 @@ pub fn buildSharedObjects(comp: *Compilation) !void { const hit = try cache.hit(); const digest = cache.final(); const o_sub_path = try path.join(arena, &[_][]const u8{ "o", &digest }); - if (!hit) { - var o_directory: Compilation.Directory = .{ - .handle = try comp.zig_cache_directory.handle.makeOpenPath(o_sub_path, .{}), - .path = try path.join(arena, &[_][]const u8{ comp.zig_cache_directory.path.?, o_sub_path }), - }; - defer o_directory.handle.close(); + // Even if we get a hit, it doesn't guarantee that we finished the job last time. + // We use the presence of an "ok" file to determine if it is a true hit. + + var o_directory: Compilation.Directory = .{ + .handle = try comp.zig_cache_directory.handle.makeOpenPath(o_sub_path, .{}), + .path = try path.join(arena, &[_][]const u8{ comp.zig_cache_directory.path.?, o_sub_path }), + }; + defer o_directory.handle.close(); + + const ok_basename = "ok"; + const actual_hit = if (hit) blk: { + o_directory.handle.access(ok_basename, .{}) catch |err| switch (err) { + error.FileNotFound => break :blk false, + else => |e| return e, + }; + break :blk true; + } else false; + + if (!actual_hit) { const metadata = try loadMetaData(comp.gpa, comp.zig_lib_directory.handle); defer metadata.destroy(comp.gpa); @@ -869,16 +882,16 @@ pub fn buildSharedObjects(comp: *Compilation) !void { const want_two_ats = chosen_def_ver_index != 255 and ver_index == chosen_def_ver_index; const at_sign_str = "@@"[0 .. @boolToInt(want_two_ats) + @as(usize, 1)]; if (ver.patch == 0) { - try zig_body.writer().print(" \\\\ .symver {s}, {s}{s}GLIBC_{d}.{d}\n", .{ + try zig_body.writer().print(" \\\\.symver {s}, {s}{s}GLIBC_{d}.{d}\n", .{ stub_name, sym_name, at_sign_str, ver.major, ver.minor, }); } else { - try zig_body.writer().print(" \\\\ .symver {s}, {s}{s}GLIBC_{d}.{d}.{d}\n", .{ + try zig_body.writer().print(" \\\\.symver {s}, {s}{s}GLIBC_{d}.{d}.{d}\n", .{ stub_name, sym_name, at_sign_str, ver.major, ver.minor, ver.patch, }); } // Hide the stub to keep the symbol table clean - try zig_body.writer().print(" \\\\ .hidden {s}\n", .{stub_name}); + try zig_body.writer().print(" \\\\.hidden {s}\n", .{stub_name}); } } } @@ -896,9 +909,13 @@ pub fn buildSharedObjects(comp: *Compilation) !void { try buildSharedLib(comp, arena, comp.zig_cache_directory, o_directory, zig_file_basename, lib); } - cache.writeManifest() catch |err| { - std.log.warn("glibc shared objects: failed to write cache manifest: {}", .{@errorName(err)}); - }; + // No need to write the manifest because there are no file inputs associated with this cache hash. + // However we do need to write the ok file now. + if (o_directory.handle.createFile(ok_basename, .{})) |file| { + file.close(); + } else |err| { + std.log.warn("glibc shared objects: failed to mark completion: {}", .{@errorName(err)}); + } } assert(comp.glibc_so_files == null); @@ -935,7 +952,7 @@ fn buildSharedLib( .zig_lib_directory = comp.zig_lib_directory, .target = comp.getTarget(), .root_name = lib.name, - .root_pkg = null, + .root_pkg = root_pkg, .output_mode = .Lib, .link_mode = .Dynamic, .rand = comp.rand, @@ -971,7 +988,7 @@ fn updateSubCompilation(sub_compilation: *Compilation) !void { if (errors.list.len != 0) { for (errors.list) |full_err_msg| { - std.log.err("{}:{}:{}: error: {}\n", .{ + std.log.err("{}:{}:{}: {}\n", .{ full_err_msg.src_path, full_err_msg.line + 1, full_err_msg.column + 1, From 17f094ec5bce9737f733de08ce0ca049f6d2a186 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2020 14:33:13 -0700 Subject: [PATCH 055/210] stage2: build glibc shared objects using assembly files closes #6358 --- BRANCH_TODO | 7 +-- src-self-hosted/glibc.zig | 94 +++++++++++++++++++++++---------------- 2 files changed, 58 insertions(+), 43 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 0f9354e32e..7292231d1f 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,8 +1,5 @@ - * glibc .so files - - without stage1 c++ code integration it should fail with a better error message - instead of lld not getting the object file on the linker line for some reason. - - stage1 C++ code integration - - ok file + * libunwind + * stage1 C++ code integration * support rpaths in ELF linker code * build & link against compiler-rt * build & link againstn freestanding libc diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index bf7f0f6450..f629400a1a 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -744,6 +744,9 @@ pub const BuiltSharedObjects = struct { const all_map_basename = "all.map"; +// TODO Turn back on zig fmt when https://github.com/ziglang/zig/issues/5948 is implemented. +// zig fmt: off + pub fn buildSharedObjects(comp: *Compilation) !void { const tracy = trace(@src()); defer tracy.end(); @@ -765,8 +768,6 @@ pub fn buildSharedObjects(comp: *Compilation) !void { cache.hash.addBytes(build_options.version); cache.hash.addBytes(comp.zig_lib_directory.path orelse "."); cache.hash.add(target.cpu.arch); - cache.hash.addBytes(target.cpu.model.name); - cache.hash.add(target.cpu.features.ints); cache.hash.add(target.abi); cache.hash.add(target_version); @@ -832,17 +833,9 @@ pub fn buildSharedObjects(comp: *Compilation) !void { } var zig_body = std.ArrayList(u8).init(comp.gpa); defer zig_body.deinit(); - var zig_footer = std.ArrayList(u8).init(comp.gpa); - defer zig_footer.deinit(); for (libs) |*lib| { zig_body.shrinkRetainingCapacity(0); - zig_footer.shrinkRetainingCapacity(0); - try zig_body.appendSlice( - \\comptime { - \\ asm ( - \\ - ); for (metadata.all_functions) |*libc_fn, fn_i| { if (libc_fn.lib != lib) continue; @@ -868,46 +861,66 @@ pub fn buildSharedObjects(comp: *Compilation) !void { { var ver_i: u8 = 0; while (ver_i < ver_list.len) : (ver_i += 1) { + // Example: + // .globl _Exit_2_2_5 + // .type _Exit_2_2_5, @function; + // .symver _Exit_2_2_5, _Exit@@GLIBC_2.2.5 + // .hidden _Exit_2_2_5 + // _Exit_2_2_5: const ver_index = ver_list.versions[ver_i]; const ver = metadata.all_versions[ver_index]; const sym_name = libc_fn.name; - const stub_name = if (ver.patch == 0) - try std.fmt.allocPrint(arena, "{s}_{d}_{d}", .{ sym_name, ver.major, ver.minor }) - else - try std.fmt.allocPrint(arena, "{s}_{d}_{d}_{d}", .{ sym_name, ver.major, ver.minor, ver.patch }); - - try zig_footer.writer().print("export fn {s}() void {{}}\n", .{stub_name}); - // Default symbol version definition vs normal symbol version definition const want_two_ats = chosen_def_ver_index != 255 and ver_index == chosen_def_ver_index; const at_sign_str = "@@"[0 .. @boolToInt(want_two_ats) + @as(usize, 1)]; + if (ver.patch == 0) { - try zig_body.writer().print(" \\\\.symver {s}, {s}{s}GLIBC_{d}.{d}\n", .{ - stub_name, sym_name, at_sign_str, ver.major, ver.minor, + const sym_plus_ver = try std.fmt.allocPrint( + arena, "{s}_{d}_{d}", + .{sym_name, ver.major, ver.minor}, + ); + try zig_body.writer().print( + \\.globl {s} + \\.type {s}, @function; + \\.symver {s}, {s}{s}GLIBC_{d}.{d} + \\.hidden {s} + \\{s}: + \\ + , .{ + sym_plus_ver, + sym_plus_ver, + sym_plus_ver, sym_name, at_sign_str, ver.major, ver.minor, + sym_plus_ver, + sym_plus_ver, }); } else { - try zig_body.writer().print(" \\\\.symver {s}, {s}{s}GLIBC_{d}.{d}.{d}\n", .{ - stub_name, sym_name, at_sign_str, ver.major, ver.minor, ver.patch, + const sym_plus_ver = try std.fmt.allocPrint(arena, "{s}_{d}_{d}_{d}", + .{sym_name, ver.major, ver.minor, ver.patch}, + ); + try zig_body.writer().print( + \\.globl {s} + \\.type {s}, @function; + \\.symver {s}, {s}{s}GLIBC_{d}.{d}.{d} + \\.hidden {s} + \\{s}: + \\ + , .{ + sym_plus_ver, + sym_plus_ver, + sym_plus_ver, sym_name, at_sign_str, ver.major, ver.minor, ver.patch, + sym_plus_ver, + sym_plus_ver, }); } - // Hide the stub to keep the symbol table clean - try zig_body.writer().print(" \\\\.hidden {s}\n", .{stub_name}); } } } - try zig_body.appendSlice( - \\ ); - \\} - \\ - ); - try zig_body.appendSlice(zig_footer.items); - var lib_name_buf: [32]u8 = undefined; // Larger than each of the names "c", "pthread", etc. - const zig_file_basename = std.fmt.bufPrint(&lib_name_buf, "{s}.zig", .{lib.name}) catch unreachable; - try o_directory.handle.writeFile(zig_file_basename, zig_body.items); + const asm_file_basename = std.fmt.bufPrint(&lib_name_buf, "{s}.s", .{lib.name}) catch unreachable; + try o_directory.handle.writeFile(asm_file_basename, zig_body.items); - try buildSharedLib(comp, arena, comp.zig_cache_directory, o_directory, zig_file_basename, lib); + try buildSharedLib(comp, arena, comp.zig_cache_directory, o_directory, asm_file_basename, lib); } // No need to write the manifest because there are no file inputs associated with this cache hash. // However we do need to write the ok file now. @@ -925,12 +938,14 @@ pub fn buildSharedObjects(comp: *Compilation) !void { }; } +// zig fmt: on + fn buildSharedLib( comp: *Compilation, arena: *Allocator, zig_cache_directory: Compilation.Directory, bin_directory: Compilation.Directory, - zig_file_basename: []const u8, + asm_file_basename: []const u8, lib: *const Lib, ) !void { const tracy = trace(@src()); @@ -944,15 +959,17 @@ fn buildSharedLib( const ld_basename = path.basename(comp.getTarget().standardDynamicLinkerPath().get().?); const override_soname = if (mem.eql(u8, lib.name, "ld")) ld_basename else null; const map_file_path = try path.join(arena, &[_][]const u8{ bin_directory.path.?, all_map_basename }); - // TODO we should be able to just give the open directory to Package - const root_pkg = try Package.create(comp.gpa, std.fs.cwd(), bin_directory.path.?, zig_file_basename); - defer root_pkg.destroy(comp.gpa); + const c_source_files = [1]Compilation.CSourceFile{ + .{ + .src_path = try path.join(arena, &[_][]const u8{ bin_directory.path.?, asm_file_basename }), + }, + }; const sub_compilation = try Compilation.create(comp.gpa, .{ .zig_cache_directory = zig_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = comp.getTarget(), .root_name = lib.name, - .root_pkg = root_pkg, + .root_pkg = null, .output_mode = .Lib, .link_mode = .Dynamic, .rand = comp.rand, @@ -973,6 +990,7 @@ fn buildSharedLib( .stage1_is_dummy_so = true, .version_script = map_file_path, .override_soname = override_soname, + .c_source_files = &c_source_files, }); defer sub_compilation.destroy(); From 17d40ecb4964dbba0f3b1482f2c68253cfc69edf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2020 17:17:37 -0700 Subject: [PATCH 056/210] stage2: building libunwind.a and put glibc shared objects on the elf linker line --- BRANCH_TODO | 13 +-- build.zig | 32 +++++-- src-self-hosted/Compilation.zig | 42 +++++++++- src-self-hosted/glibc.zig | 5 +- src-self-hosted/libunwind.zig | 144 ++++++++++++++++++++++++++++++++ src-self-hosted/link/Elf.zig | 30 ++++--- 6 files changed, 234 insertions(+), 32 deletions(-) create mode 100644 src-self-hosted/libunwind.zig diff --git a/BRANCH_TODO b/BRANCH_TODO index 7292231d1f..0b4c13f8ae 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,7 +1,10 @@ - * libunwind - * stage1 C++ code integration + * make sure zig cc works + - using it as a preprocessor (-E) + - @breakpoint(); // TODO the first arg is empty string right? skip past that. + - try building some software * support rpaths in ELF linker code * build & link against compiler-rt + - stage1 C++ code integration * build & link againstn freestanding libc * add CLI support for a way to pass extra flags to c source files * implement the workaround for using LLVM to detect native CPU features @@ -12,10 +15,6 @@ * port the stage1 os.cpp code that raises the open fd limit * use global zig-cache dir for crt files * `zig translate-c` - * make sure zig cc works - - using it as a preprocessor (-E) - - @breakpoint(); // TODO the first arg is empty string right? skip past that. - - try building some software * MachO LLD linking * COFF LLD linking * WASM LLD linking @@ -54,3 +53,5 @@ - make it possible for Package to not openDir and reference already existing resources. * rename src/ to src/stage1/ * rename src-self-hosted/ to src/ + * improve Directory.join to only use 1 allocation in a clean way. + * tracy builds with lc++ diff --git a/build.zig b/build.zig index 961e143545..17ee040e8f 100644 --- a/build.zig +++ b/build.zig @@ -73,6 +73,9 @@ pub fn build(b: *Builder) !void { if (only_install_lib_files) return; + const tracy = b.option([]const u8, "tracy", "Enable Tracy integration. Supply path to Tracy source"); + const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse enable_llvm; + var exe = b.addExecutable("zig", "src-self-hosted/main.zig"); exe.install(); exe.setBuildMode(mode); @@ -90,10 +93,8 @@ pub fn build(b: *Builder) !void { var ctx = parseConfigH(b, config_h_text); ctx.llvm = try findLLVM(b, ctx.llvm_config_exe); - try configureStage2(b, exe, ctx); + try configureStage2(b, exe, ctx, tracy != null); } - const tracy = b.option([]const u8, "tracy", "Enable Tracy integration. Supply path to Tracy source"); - const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse enable_llvm; if (link_libc) { exe.linkLibC(); test_stage2.linkLibC(); @@ -151,7 +152,9 @@ pub fn build(b: *Builder) !void { ) catch unreachable; exe.addIncludeDir(tracy_path); exe.addCSourceFile(client_cpp, &[_][]const u8{ "-DTRACY_ENABLE=1", "-fno-sanitize=undefined" }); - exe.linkSystemLibraryName("c++"); + if (!enable_llvm) { + exe.linkSystemLibraryName("c++"); + } exe.linkLibC(); } @@ -344,7 +347,7 @@ fn findLLVM(b: *Builder, llvm_config_exe: []const u8) !LibraryDep { return result; } -fn configureStage2(b: *Builder, exe: anytype, ctx: Context) !void { +fn configureStage2(b: *Builder, exe: anytype, ctx: Context, need_cpp_includes: bool) !void { exe.addIncludeDir("src"); exe.addIncludeDir(ctx.cmake_binary_dir); addCppLib(b, exe, ctx.cmake_binary_dir, "zig_cpp"); @@ -377,7 +380,7 @@ fn configureStage2(b: *Builder, exe: anytype, ctx: Context) !void { if (exe.target.getOsTag() == .linux) { // First we try to static link against gcc libstdc++. If that doesn't work, // we fall back to -lc++ and cross our fingers. - addCxxKnownPath(b, ctx, exe, "libstdc++.a", "") catch |err| switch (err) { + addCxxKnownPath(b, ctx, exe, "libstdc++.a", "", need_cpp_includes) catch |err| switch (err) { error.RequiredLibraryNotFound => { exe.linkSystemLibrary("c++"); }, @@ -386,12 +389,12 @@ fn configureStage2(b: *Builder, exe: anytype, ctx: Context) !void { exe.linkSystemLibrary("pthread"); } else if (exe.target.isFreeBSD()) { - try addCxxKnownPath(b, ctx, exe, "libc++.a", null); + try addCxxKnownPath(b, ctx, exe, "libc++.a", null, need_cpp_includes); exe.linkSystemLibrary("pthread"); } else if (exe.target.isDarwin()) { - if (addCxxKnownPath(b, ctx, exe, "libgcc_eh.a", "")) { + if (addCxxKnownPath(b, ctx, exe, "libgcc_eh.a", "", need_cpp_includes)) { // Compiler is GCC. - try addCxxKnownPath(b, ctx, exe, "libstdc++.a", null); + try addCxxKnownPath(b, ctx, exe, "libstdc++.a", null, need_cpp_includes); exe.linkSystemLibrary("pthread"); // TODO LLD cannot perform this link. // See https://github.com/ziglang/zig/issues/1535 @@ -417,6 +420,7 @@ fn addCxxKnownPath( exe: anytype, objname: []const u8, errtxt: ?[]const u8, + need_cpp_includes: bool, ) !void { const path_padded = try b.exec(&[_][]const u8{ ctx.cxx_compiler, @@ -432,6 +436,16 @@ fn addCxxKnownPath( return error.RequiredLibraryNotFound; } exe.addObjectFile(path_unpadded); + + // TODO a way to integrate with system c++ include files here + // cc -E -Wp,-v -xc++ /dev/null + if (need_cpp_includes) { + // I used these temporarily for testing something but we obviously need a + // more general purpose solution here. + //exe.addIncludeDir("/nix/store/b3zsk4ihlpiimv3vff86bb5bxghgdzb9-gcc-9.2.0/lib/gcc/x86_64-unknown-linux-gnu/9.2.0/../../../../include/c++/9.2.0"); + //exe.addIncludeDir("/nix/store/b3zsk4ihlpiimv3vff86bb5bxghgdzb9-gcc-9.2.0/lib/gcc/x86_64-unknown-linux-gnu/9.2.0/../../../../include/c++/9.2.0/x86_64-unknown-linux-gnu"); + //exe.addIncludeDir("/nix/store/b3zsk4ihlpiimv3vff86bb5bxghgdzb9-gcc-9.2.0/lib/gcc/x86_64-unknown-linux-gnu/9.2.0/../../../../include/c++/9.2.0/backward"); + } } const Context = struct { diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index d99a85e1d3..92e9d89da0 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -15,6 +15,7 @@ const liveness = @import("liveness.zig"); const build_options = @import("build_options"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const glibc = @import("glibc.zig"); +const libunwind = @import("libunwind.zig"); const fatal = @import("main.zig").fatal; const Module = @import("Module.zig"); const Cache = @import("Cache.zig"); @@ -63,7 +64,7 @@ libcxx_static_lib: ?[]const u8 = null, libcxxabi_static_lib: ?[]const u8 = null, /// Populated when we build libunwind.a. A WorkItem to build this is placed in the queue /// and resolved before calling linker.flush(). -libunwind_static_lib: ?[]const u8 = null, +libunwind_static_lib: ?CRTFile = null, /// Populated when we build c.a. A WorkItem to build this is placed in the queue /// and resolved before calling linker.flush(). libc_static_lib: ?[]const u8 = null, @@ -115,6 +116,8 @@ const WorkItem = union(enum) { glibc_crt_file: glibc.CRTFile, /// all of the glibc shared objects glibc_shared_objects, + + libunwind: void, }; pub const CObject = struct { @@ -206,6 +209,17 @@ pub const Directory = struct { /// `null` means cwd. path: ?[]const u8, handle: std.fs.Dir, + + pub fn join(self: Directory, allocator: *Allocator, paths: []const []const u8) ![]u8 { + if (self.path) |p| { + // TODO clean way to do this with only 1 allocation + const part2 = try std.fs.path.join(allocator, paths); + defer allocator.free(part2); + return std.fs.path.join(allocator, &[_][]const u8{ p, part2 }); + } else { + return std.fs.path.join(allocator, paths); + } + } }; pub const EmitLoc = struct { @@ -274,9 +288,6 @@ pub const InitOptions = struct { version: ?std.builtin.Version = null, libc_installation: ?*const LibCInstallation = null, machine_code_model: std.builtin.CodeModel = .default, - /// TODO Once self-hosted Zig is capable enough, we can remove this special-case - /// hack in favor of more general compilation options. - stage1_is_dummy_so: bool = false, }; pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { @@ -636,6 +647,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { if (comp.wantBuildGLibCFromSource()) { try comp.addBuildingGLibCWorkItems(); } + if (comp.wantBuildLibUnwindFromSource()) { + try comp.work_queue.writeItem(.{ .libunwind = {} }); + } return comp; } @@ -656,6 +670,10 @@ pub fn destroy(self: *Compilation) void { self.crt_files.deinit(gpa); } + if (self.libunwind_static_lib) |*unwind_crt_file| { + unwind_crt_file.deinit(gpa); + } + for (self.c_object_table.items()) |entry| { entry.key.destroy(gpa); } @@ -936,6 +954,12 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { fatal("unable to build glibc shared objects: {}", .{@errorName(err)}); }; }, + .libunwind => { + libunwind.buildStaticLib(self) catch |err| { + // TODO Expose this as a normal compile error rather than crashing here. + fatal("unable to build libunwind: {}", .{@errorName(err)}); + }; + }, }; } @@ -1597,3 +1621,13 @@ fn wantBuildGLibCFromSource(comp: *Compilation) bool { comp.bin_file.options.libc_installation == null and comp.bin_file.options.target.isGnuLibC(); } + +fn wantBuildLibUnwindFromSource(comp: *Compilation) bool { + const is_exe_or_dyn_lib = switch (comp.bin_file.options.output_mode) { + .Obj => false, + .Lib => comp.bin_file.options.link_mode == .Dynamic, + .Exe => true, + }; + return comp.bin_file.options.link_libc and is_exe_or_dyn_lib and + comp.bin_file.options.libc_installation == null; +} diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index f629400a1a..5bda249080 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -751,6 +751,10 @@ pub fn buildSharedObjects(comp: *Compilation) !void { const tracy = trace(@src()); defer tracy.end(); + if (!build_options.have_llvm) { + return error.ZigCompilerNotBuiltWithLLVMExtensions; + } + var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); defer arena_allocator.deinit(); const arena = &arena_allocator.allocator; @@ -987,7 +991,6 @@ fn buildSharedLib( .debug_link = comp.bin_file.options.debug_link, .clang_passthrough_mode = comp.clang_passthrough_mode, .version = version, - .stage1_is_dummy_so = true, .version_script = map_file_path, .override_soname = override_soname, .c_source_files = &c_source_files, diff --git a/src-self-hosted/libunwind.zig b/src-self-hosted/libunwind.zig new file mode 100644 index 0000000000..19f77da2dc --- /dev/null +++ b/src-self-hosted/libunwind.zig @@ -0,0 +1,144 @@ +const std = @import("std"); +const path = std.fs.path; +const assert = std.debug.assert; + +const target_util = @import("target.zig"); +const Compilation = @import("Compilation.zig"); +const build_options = @import("build_options"); +const trace = @import("tracy.zig").trace; + +pub fn buildStaticLib(comp: *Compilation) !void { + const tracy = trace(@src()); + defer tracy.end(); + + if (!build_options.have_llvm) { + return error.ZigCompilerNotBuiltWithLLVMExtensions; + } + + var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); + defer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + const root_name = "unwind"; + const output_mode = .Lib; + const link_mode = .Static; + const target = comp.getTarget(); + const basename = try std.zig.binNameAlloc(arena, root_name, target, output_mode, link_mode, null); + + const emit_bin = Compilation.EmitLoc{ + .directory = null, // Put it in the cache directory. + .basename = basename, + }; + + const unwind_src_list = [_][]const u8{ + "libunwind" ++ path.sep_str ++ "src" ++ path.sep_str ++ "libunwind.cpp", + "libunwind" ++ path.sep_str ++ "src" ++ path.sep_str ++ "Unwind-EHABI.cpp", + "libunwind" ++ path.sep_str ++ "src" ++ path.sep_str ++ "Unwind-seh.cpp", + "libunwind" ++ path.sep_str ++ "src" ++ path.sep_str ++ "UnwindLevel1.c", + "libunwind" ++ path.sep_str ++ "src" ++ path.sep_str ++ "UnwindLevel1-gcc-ext.c", + "libunwind" ++ path.sep_str ++ "src" ++ path.sep_str ++ "Unwind-sjlj.c", + "libunwind" ++ path.sep_str ++ "src" ++ path.sep_str ++ "UnwindRegistersRestore.S", + "libunwind" ++ path.sep_str ++ "src" ++ path.sep_str ++ "UnwindRegistersSave.S", + }; + + var c_source_files: [unwind_src_list.len]Compilation.CSourceFile = undefined; + for (unwind_src_list) |unwind_src, i| { + var cflags = std.ArrayList([]const u8).init(arena); + + switch (Compilation.classifyFileExt(unwind_src)) { + .c => { + try cflags.append("-std=c99"); + }, + .cpp => { + try cflags.appendSlice(&[_][]const u8{ + "-fno-rtti", + "-I", + try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libcxx", "include" }), + }); + }, + .assembly => {}, + else => unreachable, // You can see the entire list of files just above. + } + try cflags.append("-I"); + try cflags.append(try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libunwind", "include" })); + if (target_util.supports_fpic(target)) { + try cflags.append("-fPIC"); + } + try cflags.append("-D_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS"); + try cflags.append("-Wa,--noexecstack"); + + // This is intentionally always defined because the macro definition means, should it only + // build for the target specified by compiler defines. Since we pass -target the compiler + // defines will be correct. + try cflags.append("-D_LIBUNWIND_IS_NATIVE_ONLY"); + + if (comp.bin_file.options.optimize_mode == .Debug) { + try cflags.append("-D_DEBUG"); + } + if (comp.bin_file.options.single_threaded) { + try cflags.append("-D_LIBUNWIND_HAS_NO_THREADS"); + } + try cflags.append("-Wno-bitwise-conditional-parentheses"); + + c_source_files[i] = .{ + .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{unwind_src}), + .extra_flags = cflags.items, + }; + } + const sub_compilation = try Compilation.create(comp.gpa, .{ + // TODO use the global cache directory here + .zig_cache_directory = comp.zig_cache_directory, + .zig_lib_directory = comp.zig_lib_directory, + .target = target, + .root_name = root_name, + .root_pkg = null, + .output_mode = output_mode, + .rand = comp.rand, + .libc_installation = comp.bin_file.options.libc_installation, + .emit_bin = emit_bin, + .optimize_mode = comp.bin_file.options.optimize_mode, + .link_mode = link_mode, + .want_sanitize_c = false, + .want_stack_check = false, + .want_valgrind = false, + .want_pic = comp.bin_file.options.pic, + .emit_h = null, + .strip = comp.bin_file.options.strip, + .is_native_os = comp.bin_file.options.is_native_os, + .self_exe_path = comp.self_exe_path, + .c_source_files = &c_source_files, + .debug_cc = comp.debug_cc, + .debug_link = comp.bin_file.options.debug_link, + .clang_passthrough_mode = comp.clang_passthrough_mode, + .link_libc = true, + }); + defer sub_compilation.destroy(); + + try updateSubCompilation(sub_compilation); + + assert(comp.libunwind_static_lib == null); + comp.libunwind_static_lib = Compilation.CRTFile{ + .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{basename}), + .lock = sub_compilation.bin_file.toOwnedLock(), + }; +} + +fn updateSubCompilation(sub_compilation: *Compilation) !void { + try sub_compilation.update(); + + // Look for compilation errors in this sub_compilation + var errors = try sub_compilation.getAllErrorsAlloc(); + defer errors.deinit(sub_compilation.gpa); + + if (errors.list.len != 0) { + for (errors.list) |full_err_msg| { + std.log.err("{}:{}:{}: {}\n", .{ + full_err_msg.src_path, + full_err_msg.line + 1, + full_err_msg.column + 1, + full_err_msg.msg, + }); + } + return error.BuildingLibCObjectFailed; + } +} diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 8a75796bc3..5c2c73d740 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -1,26 +1,28 @@ +const Elf = @This(); + const std = @import("std"); const mem = std.mem; const assert = std.debug.assert; const Allocator = std.mem.Allocator; +const fs = std.fs; +const elf = std.elf; +const log = std.log.scoped(.link); +const DW = std.dwarf; +const leb128 = std.debug.leb; + const ir = @import("../ir.zig"); const Module = @import("../Module.zig"); const Compilation = @import("../Compilation.zig"); -const fs = std.fs; -const elf = std.elf; const codegen = @import("../codegen.zig"); -const log = std.log.scoped(.link); -const DW = std.dwarf; const trace = @import("../tracy.zig").trace; -const leb128 = std.debug.leb; const Package = @import("../Package.zig"); const Value = @import("../value.zig").Value; const Type = @import("../type.zig").Type; const link = @import("../link.zig"); const File = link.File; -const Elf = @This(); const build_options = @import("build_options"); const target_util = @import("../target.zig"); -const fatal = @import("main.zig").fatal; +const glibc = @import("../glibc.zig"); const default_entry_addr = 0x8000000; @@ -1530,15 +1532,19 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { try argv.append("-lpthread"); } } else if (target.isGnuLibC()) { - try argv.append(comp.libunwind_static_lib.?); - // TODO here we need to iterate over the glibc libs and add the .so files to the linker line. - std.log.warn("TODO port add_glibc_libs to stage2", .{}); + try argv.append(comp.libunwind_static_lib.?.full_object_path); + for (glibc.libs) |lib| { + const lib_path = try std.fmt.allocPrint(arena, "{s}{c}lib{s}.so.{d}", .{ + comp.glibc_so_files.?.dir_path, fs.path.sep, lib.name, lib.sover, + }); + try argv.append(lib_path); + } try argv.append(try comp.get_libc_crt_file(arena, "libc_nonshared.a")); } else if (target.isMusl()) { - try argv.append(comp.libunwind_static_lib.?); + try argv.append(comp.libunwind_static_lib.?.full_object_path); try argv.append(comp.libc_static_lib.?); } else if (self.base.options.link_libcpp) { - try argv.append(comp.libunwind_static_lib.?); + try argv.append(comp.libunwind_static_lib.?.full_object_path); } else { unreachable; // Compiler was supposed to emit an error for not being able to provide libc. } From 07eb2c65f6ee3d2ea81a59f776689b1d919d3280 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2020 19:24:09 -0700 Subject: [PATCH 057/210] stage2: don't add unused args to assembly compilations --- src-self-hosted/Compilation.zig | 147 ++++++++++++++++---------------- src-self-hosted/main.zig | 5 +- 2 files changed, 75 insertions(+), 77 deletions(-) diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 92e9d89da0..8e18e7fffc 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -1170,10 +1170,6 @@ fn addCCArgs( if (ext == .cpp) { try argv.append("-nostdinc++"); } - try argv.appendSlice(&[_][]const u8{ - "-nostdinc", - "-fno-spell-checking", - }); // We don't ever put `-fcolor-diagnostics` or `-fno-color-diagnostics` because in passthrough mode // we want Clang to infer it, and in normal mode we always want it off, which will be true since @@ -1219,6 +1215,11 @@ fn addCCArgs( switch (ext) { .c, .cpp, .h => { + try argv.appendSlice(&[_][]const u8{ + "-nostdinc", + "-fno-spell-checking", + }); + // According to Rich Felker libc headers are supposed to go before C language headers. // However as noted by @dimenus, appending libc headers before c_headers breaks intrinsics // and other compiler specific items. @@ -1260,6 +1261,75 @@ fn addCCArgs( try argv.append("-Xclang"); try argv.append("-detailed-preprocessing-record"); } + + // windows.h has files such as pshpack1.h which do #pragma packing, triggering a clang warning. + // So for this target, we disable this warning. + if (target.os.tag == .windows and target.abi.isGnu()) { + try argv.append("-Wno-pragma-pack"); + } + + if (!comp.bin_file.options.strip) { + try argv.append("-g"); + } + + if (comp.haveFramePointer()) { + try argv.append("-fno-omit-frame-pointer"); + } else { + try argv.append("-fomit-frame-pointer"); + } + + if (comp.sanitize_c) { + try argv.append("-fsanitize=undefined"); + try argv.append("-fsanitize-trap=undefined"); + } + + switch (comp.bin_file.options.optimize_mode) { + .Debug => { + // windows c runtime requires -D_DEBUG if using debug libraries + try argv.append("-D_DEBUG"); + try argv.append("-Og"); + + if (comp.bin_file.options.link_libc) { + try argv.append("-fstack-protector-strong"); + try argv.append("--param"); + try argv.append("ssp-buffer-size=4"); + } else { + try argv.append("-fno-stack-protector"); + } + }, + .ReleaseSafe => { + // See the comment in the BuildModeFastRelease case for why we pass -O2 rather + // than -O3 here. + try argv.append("-O2"); + if (comp.bin_file.options.link_libc) { + try argv.append("-D_FORTIFY_SOURCE=2"); + try argv.append("-fstack-protector-strong"); + try argv.append("--param"); + try argv.append("ssp-buffer-size=4"); + } else { + try argv.append("-fno-stack-protector"); + } + }, + .ReleaseFast => { + try argv.append("-DNDEBUG"); + // Here we pass -O2 rather than -O3 because, although we do the equivalent of + // -O3 in Zig code, the justification for the difference here is that Zig + // has better detection and prevention of undefined behavior, so -O3 is safer for + // Zig code than it is for C code. Also, C programmers are used to their code + // running in -O2 and thus the -O3 path has been tested less. + try argv.append("-O2"); + try argv.append("-fno-stack-protector"); + }, + .ReleaseSmall => { + try argv.append("-DNDEBUG"); + try argv.append("-Os"); + try argv.append("-fno-stack-protector"); + }, + } + + if (target_util.supports_fpic(target) and comp.bin_file.options.pic) { + try argv.append("-fPIC"); + } }, .so, .assembly, .ll, .bc, .unknown => {}, } @@ -1285,75 +1355,6 @@ fn addCCArgs( try argv.append("-ffreestanding"); } - // windows.h has files such as pshpack1.h which do #pragma packing, triggering a clang warning. - // So for this target, we disable this warning. - if (target.os.tag == .windows and target.abi.isGnu()) { - try argv.append("-Wno-pragma-pack"); - } - - if (!comp.bin_file.options.strip) { - try argv.append("-g"); - } - - if (comp.haveFramePointer()) { - try argv.append("-fno-omit-frame-pointer"); - } else { - try argv.append("-fomit-frame-pointer"); - } - - if (comp.sanitize_c) { - try argv.append("-fsanitize=undefined"); - try argv.append("-fsanitize-trap=undefined"); - } - - switch (comp.bin_file.options.optimize_mode) { - .Debug => { - // windows c runtime requires -D_DEBUG if using debug libraries - try argv.append("-D_DEBUG"); - try argv.append("-Og"); - - if (comp.bin_file.options.link_libc) { - try argv.append("-fstack-protector-strong"); - try argv.append("--param"); - try argv.append("ssp-buffer-size=4"); - } else { - try argv.append("-fno-stack-protector"); - } - }, - .ReleaseSafe => { - // See the comment in the BuildModeFastRelease case for why we pass -O2 rather - // than -O3 here. - try argv.append("-O2"); - if (comp.bin_file.options.link_libc) { - try argv.append("-D_FORTIFY_SOURCE=2"); - try argv.append("-fstack-protector-strong"); - try argv.append("--param"); - try argv.append("ssp-buffer-size=4"); - } else { - try argv.append("-fno-stack-protector"); - } - }, - .ReleaseFast => { - try argv.append("-DNDEBUG"); - // Here we pass -O2 rather than -O3 because, although we do the equivalent of - // -O3 in Zig code, the justification for the difference here is that Zig - // has better detection and prevention of undefined behavior, so -O3 is safer for - // Zig code than it is for C code. Also, C programmers are used to their code - // running in -O2 and thus the -O3 path has been tested less. - try argv.append("-O2"); - try argv.append("-fno-stack-protector"); - }, - .ReleaseSmall => { - try argv.append("-DNDEBUG"); - try argv.append("-Os"); - try argv.append("-fno-stack-protector"); - }, - } - - if (target_util.supports_fpic(target) and comp.bin_file.options.pic) { - try argv.append("-fPIC"); - } - try argv.appendSlice(comp.clang_argv); } diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 132a7d3947..0efb529b59 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -73,10 +73,7 @@ pub fn log( const ok = comptime for (build_options.log_scopes) |log_scope| { if (mem.eql(u8, log_scope, scope_name)) break true; - } else false; - - if (!ok) - return; + } else return; } // We only recognize 4 log levels in this application. From 01a7affb87f74e33e674891d77d4b720d3014013 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2020 20:18:06 -0700 Subject: [PATCH 058/210] stage2: ask for a higher open fd limit Here's the doc comment from the commit: For one example of why this is handy, consider the case of building musl libc. We keep a lock open for each of the object files in the form of a file descriptor until they are finally put into an archive file. This is to allow a zig-cache garbage collector to run concurrently to zig processes, and to allow multiple zig processes to run concurrently with each other, without clobbering each other. This code is disabled until #6361 is implemented (getrlimit/setrlimit are not yet added to the standard library). --- BRANCH_TODO | 1 - src-self-hosted/main.zig | 45 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 0b4c13f8ae..c2fbdc713d 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -12,7 +12,6 @@ * capture lld stdout/stderr better * musl * mingw-w64 - * port the stage1 os.cpp code that raises the open fd limit * use global zig-cache dir for crt files * `zig translate-c` * MachO LLD linking diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 0efb529b59..764daa42fd 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -1103,6 +1103,8 @@ pub fn buildOutputType( }, }; + gimmeMoreOfThoseSweetSweetFileDescriptors(); + const comp = Compilation.create(gpa, .{ .zig_lib_directory = zig_lib_directory, .zig_cache_directory = zig_cache_directory, @@ -1930,3 +1932,46 @@ fn parseCodeModel(arg: []const u8) std.builtin.CodeModel { return std.meta.stringToEnum(std.builtin.CodeModel, arg) orelse fatal("unsupported machine code model: '{}'", .{arg}); } + +/// Raise the open file descriptor limit. Ask and ye shall receive. +/// For one example of why this is handy, consider the case of building musl libc. +/// We keep a lock open for each of the object files in the form of a file descriptor +/// until they are finally put into an archive file. This is to allow a zig-cache +/// garbage collector to run concurrently to zig processes, and to allow multiple +/// zig processes to run concurrently with each other, without clobbering each other. +fn gimmeMoreOfThoseSweetSweetFileDescriptors() void { + switch (std.Target.current.os.tag) { + .windows, .wasi, .uefi, .other, .freestanding => return, + // std lib is missing getrlimit/setrlimit. + // https://github.com/ziglang/zig/issues/6361 + //else => {}, + else => return, + } + const posix = std.os; + var lim = posix.getrlimit(posix.RLIMIT_NOFILE, &lim) catch return; // Oh well; we tried. + if (lim.cur == lim.max) return; + while (true) { + // Do a binary search for the limit. + var min: posix.rlim_t = lim.cur; + var max: posix.rlim_t = 1 << 20; + // But if there's a defined upper bound, don't search, just set it. + if (lim.max != posix.RLIM_INFINITY) { + min = lim.max; + max = lim.max; + } + while (true) { + lim.cur = min + (max - min) / 2; + if (posix.setrlimit(posix.RLIMIT_NOFILE, lim)) |_| { + min = lim.cur; + } else |_| { + max = lim.cur; + } + if (min + 1 < max) continue; + return; + } + } +} + +test "fds" { + gimmeMoreOfThoseSweetSweetFileDescriptors(); +} From bc01887376c227a5f42715f48471dcca14b3d31b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2020 20:28:51 -0700 Subject: [PATCH 059/210] stage2: verify -Wl zig cc behavior --- BRANCH_TODO | 1 - src-self-hosted/main.zig | 1 - 2 files changed, 2 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index c2fbdc713d..1934cfd02d 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,6 +1,5 @@ * make sure zig cc works - using it as a preprocessor (-E) - - @breakpoint(); // TODO the first arg is empty string right? skip past that. - try building some software * support rpaths in ELF linker code * build & link against compiler-rt diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 764daa42fd..c9b872cc7e 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -627,7 +627,6 @@ pub fn buildOutputType( .rdynamic => rdynamic = true, .wl => { var split_it = mem.split(it.only_arg, ","); - @breakpoint(); // TODO the first arg is empty string right? skip past that. while (split_it.next()) |linker_arg| { try linker_args.append(linker_arg); } From dc478687d95ee0d495cd26182edae78a01db59af Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 17 Sep 2020 18:29:38 -0700 Subject: [PATCH 060/210] delete all stage1 c++ code not directly related to compiling stage2 Deleted 16,000+ lines of c++ code, including: * an implementation of blake hashing * the cache hash system * compiler.cpp * all the linking code, and everything having to do with building glibc, musl, and mingw-w64 * much of the stage1 compiler internals got slimmed down since it now assumes it is always outputting an object file. More stuff: * stage1 is now built with a different strategy: we have a tiny zig0.cpp which is a slimmed down version of what stage1 main.cpp used to be. Its only purpose is to build stage2 zig code into an object file, which is then linked by the host build system (cmake) into stage1. zig0.cpp uses the same C API that stage2 now has access to, so that stage2 zig code can call into stage1 c++ code. - stage1.h is - stage2.h is - stage1.zig is the main entry point for the Zig/C++ hybrid compiler. It has the functions exported from Zig, called in C++, and bindings for the functions exported from C++, called from Zig. * removed the memory profiling instrumentation from stage1. Abandon ship! * Re-added the sections to the README about how to build stage2 and stage3. * stage2 now knows as a comptime boolean whether it is being compiled as part of stage1 or as stage2. - TODO use this flag to call into stage1 for compiling zig code. * introduce -fdll-export-fns and -fno-dll-export-fns and clarify its relationship to link_mode (static/dynamic) * implement depending on LLVM to detect native target cpu features when LLVM extensions are enabled and zig lacks CPU feature detection for that target architecture. * C importing is broken, will need some stage2 support to function again. --- BRANCH_TODO | 24 +- CMakeLists.txt | 71 +- README.md | 37 + build.zig | 2 + src-self-hosted/Compilation.zig | 18 +- src-self-hosted/libcxx.zig | 66 + src-self-hosted/link.zig | 1 + src-self-hosted/main.zig | 112 +- src-self-hosted/musl.zig | 1843 +++++++++++++++++++ src-self-hosted/stage1.zig | 495 +++++ src-self-hosted/stage2.zig | 1054 ----------- src/all_types.hpp | 135 +- src/analyze.cpp | 49 +- src/analyze.hpp | 3 - src/blake2.h | 196 -- src/blake2b.c | 539 ------ src/cache_hash.cpp | 595 ------ src/cache_hash.hpp | 83 - src/codegen.cpp | 2069 ++------------------- src/codegen.hpp | 35 +- src/compiler.cpp | 196 -- src/compiler.hpp | 24 - src/config.h.in | 2 - src/config.zig.in | 1 + src/dump_analysis.cpp | 12 +- src/empty.cpp | 0 src/errmsg.hpp | 7 +- src/glibc.cpp | 392 ---- src/glibc.hpp | 50 - src/heap.cpp | 66 +- src/heap.hpp | 19 - src/install_files.h | 1907 -------------------- src/ir.cpp | 181 +- src/link.cpp | 2985 ------------------------------- src/main.cpp | 1474 --------------- src/mem.cpp | 13 - src/mem.hpp | 9 - src/mem_profile.cpp | 181 -- src/mem_profile.hpp | 71 - src/mem_type_info.hpp | 109 -- src/stage1.cpp | 127 ++ src/stage1.h | 217 +++ src/stage2.cpp | 131 +- src/stage2.h | 182 +- src/target.cpp | 127 -- src/target.hpp | 30 - src/zig0.cpp | 307 ++++ 47 files changed, 3448 insertions(+), 12799 deletions(-) create mode 100644 src-self-hosted/libcxx.zig create mode 100644 src-self-hosted/musl.zig create mode 100644 src-self-hosted/stage1.zig delete mode 100644 src-self-hosted/stage2.zig delete mode 100644 src/blake2.h delete mode 100644 src/blake2b.c delete mode 100644 src/cache_hash.cpp delete mode 100644 src/cache_hash.hpp delete mode 100644 src/compiler.cpp delete mode 100644 src/compiler.hpp create mode 100644 src/empty.cpp delete mode 100644 src/glibc.cpp delete mode 100644 src/glibc.hpp delete mode 100644 src/install_files.h delete mode 100644 src/link.cpp delete mode 100644 src/main.cpp delete mode 100644 src/mem_profile.cpp delete mode 100644 src/mem_profile.hpp create mode 100644 src/stage1.cpp create mode 100644 src/stage1.h create mode 100644 src/zig0.cpp diff --git a/BRANCH_TODO b/BRANCH_TODO index 1934cfd02d..56f4661259 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,3 +1,21 @@ + * `zig builtin` + * `zig translate-c` + * `zig test` + * `zig run` + * `zig init-lib` + * `zig init-exe` + * `zig build` + * `-ftime-report` + * -fstack-report print stack size diagnostics\n" + * -fdump-analysis write analysis.json file with type information\n" + * -femit-docs create a docs/ dir with html documentation\n" + * -fno-emit-docs do not produce docs/ dir with html documentation\n" + * -femit-asm output .s (assembly code)\n" + * -fno-emit-asm (default) do not output .s (assembly code)\n" + * -femit-llvm-ir produce a .ll file with LLVM IR\n" + * -fno-emit-llvm-ir (default) do not produce a .ll file with LLVM IR\n" + * --cache-dir [path] override the local cache directory\n" + * move main.cpp to stage2 * make sure zig cc works - using it as a preprocessor (-E) - try building some software @@ -6,26 +24,24 @@ - stage1 C++ code integration * build & link againstn freestanding libc * add CLI support for a way to pass extra flags to c source files - * implement the workaround for using LLVM to detect native CPU features - * self-host main.cpp * capture lld stdout/stderr better * musl * mingw-w64 * use global zig-cache dir for crt files - * `zig translate-c` * MachO LLD linking * COFF LLD linking * WASM LLD linking * implement proper parsing of LLD stderr/stdout and exposing compile errors * implement proper parsing of clang stderr/stdout and exposing compile errors * implement proper compile errors for failing to build glibc crt files and shared libs - * self-host link.cpp and building libcs (#4313 and #4314). using the `zig cc` command will set a flag indicating a preference for the llvm backend, which will include linking with LLD. At least for now. If zig's self-hosted linker ever gets on par with the likes of ld and lld, we can make it always be used even for zig cc. * improve the stage2 tests to support testing with LLVM extensions enabled * multi-thread building C objects * support cross compiling stage2 with `zig build` * implement emit-h in stage2 * implement -fno-emit-bin * audit the base cache hash + * audit the CLI options for stage2 + * implement serialization/deserialization of incremental compilation metadata * incremental compilation - implement detection of which source files changed * improve the cache hash logic for c objects with respect to extra flags and file parameters diff --git a/CMakeLists.txt b/CMakeLists.txt index ceaecf5552..c8746338d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,7 +51,6 @@ message("Configuring zig version ${ZIG_VERSION}") set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)") set(ZIG_STATIC_LLVM off CACHE BOOL "Prefer linking against static LLVM libraries") -set(ZIG_ENABLE_MEM_PROFILE off CACHE BOOL "Activate memory usage instrumentation") set(ZIG_PREFER_CLANG_CPP_DYLIB off CACHE BOOL "Try to link against -lclang-cpp") set(ZIG_WORKAROUND_4799 off CACHE BOOL "workaround for https://github.com/ziglang/zig/issues/4799") set(ZIG_WORKAROUND_POLLY_SO off CACHE STRING "workaround for https://github.com/ziglang/zig/issues/4799") @@ -71,11 +70,6 @@ string(REGEX REPLACE "\\\\" "\\\\\\\\" ZIG_LIBC_INCLUDE_DIR_ESCAPED "${ZIG_LIBC_ option(ZIG_TEST_COVERAGE "Build Zig with test coverage instrumentation" OFF) -# Zig no longer has embedded LLD. This option is kept for package maintainers -# so that they don't have to update their scripts in case we ever re-introduce -# LLD to the tree. This option does nothing. -option(ZIG_FORCE_EXTERNAL_LLD "does nothing" OFF) - set(ZIG_TARGET_TRIPLE "native" CACHE STRING "arch-os-abi to output binaries for") set(ZIG_TARGET_MCPU "baseline" CACHE STRING "-mcpu parameter to output binaries for") set(ZIG_EXECUTABLE "" CACHE STRING "(when cross compiling) path to already-built zig binary") @@ -261,15 +255,11 @@ include_directories("${CMAKE_SOURCE_DIR}/deps/dbg-macro") find_package(Threads) -# CMake doesn't let us create an empty executable, so we hang on to this one separately. -set(ZIG_MAIN_SRC "${CMAKE_SOURCE_DIR}/src/main.cpp") - -# This is our shim which will be replaced by libstage2 written in Zig. -set(ZIG0_SHIM_SRC "${CMAKE_SOURCE_DIR}/src/stage2.cpp") - -if(ZIG_ENABLE_MEM_PROFILE) - set(ZIG_SOURCES_MEM_PROFILE "${CMAKE_SOURCE_DIR}/src/mem_profile.cpp") -endif() +# This is our shim which will be replaced by stage1.zig. +set(ZIG0_SOURCES + "${CMAKE_SOURCE_DIR}/src/zig0.cpp" + "${CMAKE_SOURCE_DIR}/src/stage2.cpp" +) set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/analyze.cpp" @@ -277,37 +267,34 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/bigfloat.cpp" "${CMAKE_SOURCE_DIR}/src/bigint.cpp" "${CMAKE_SOURCE_DIR}/src/buffer.cpp" - "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" - "${CMAKE_SOURCE_DIR}/src/compiler.cpp" "${CMAKE_SOURCE_DIR}/src/dump_analysis.cpp" "${CMAKE_SOURCE_DIR}/src/errmsg.cpp" "${CMAKE_SOURCE_DIR}/src/error.cpp" - "${CMAKE_SOURCE_DIR}/src/glibc.cpp" "${CMAKE_SOURCE_DIR}/src/heap.cpp" "${CMAKE_SOURCE_DIR}/src/ir.cpp" "${CMAKE_SOURCE_DIR}/src/ir_print.cpp" - "${CMAKE_SOURCE_DIR}/src/link.cpp" "${CMAKE_SOURCE_DIR}/src/mem.cpp" "${CMAKE_SOURCE_DIR}/src/os.cpp" "${CMAKE_SOURCE_DIR}/src/parser.cpp" "${CMAKE_SOURCE_DIR}/src/range_set.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1.cpp" "${CMAKE_SOURCE_DIR}/src/target.cpp" "${CMAKE_SOURCE_DIR}/src/tokenizer.cpp" "${CMAKE_SOURCE_DIR}/src/util.cpp" "${CMAKE_SOURCE_DIR}/src/softfloat_ext.cpp" - "${ZIG_SOURCES_MEM_PROFILE}" ) set(OPTIMIZED_C_SOURCES - "${CMAKE_SOURCE_DIR}/src/blake2b.c" "${CMAKE_SOURCE_DIR}/src/parse_f128.c" ) set(ZIG_CPP_SOURCES + # These are planned to stay even when we are self-hosted. "${CMAKE_SOURCE_DIR}/src/zig_llvm.cpp" "${CMAKE_SOURCE_DIR}/src/zig_clang.cpp" "${CMAKE_SOURCE_DIR}/src/zig_clang_driver.cpp" "${CMAKE_SOURCE_DIR}/src/zig_clang_cc1_main.cpp" "${CMAKE_SOURCE_DIR}/src/zig_clang_cc1as_main.cpp" + # https://github.com/ziglang/zig/issues/6363 "${CMAKE_SOURCE_DIR}/src/windows_sdk.cpp" ) @@ -447,7 +434,7 @@ if(MSVC OR MINGW) target_link_libraries(zigcompiler LINK_PUBLIC version) endif() -add_executable(zig0 "${ZIG_MAIN_SRC}" "${ZIG0_SHIM_SRC}") +add_executable(zig0 ${ZIG0_SOURCES}) set_target_properties(zig0 PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} LINK_FLAGS ${EXE_LDFLAGS} @@ -455,37 +442,36 @@ set_target_properties(zig0 PROPERTIES target_link_libraries(zig0 zigcompiler) if(MSVC) - set(LIBSTAGE2 "${CMAKE_BINARY_DIR}/zigstage2.lib") + set(ZIG1_OBJECT "${CMAKE_BINARY_DIR}/zig1.obj") else() - set(LIBSTAGE2 "${CMAKE_BINARY_DIR}/libzigstage2.a") + set(ZIG1_OBJECT "${CMAKE_BINARY_DIR}/zig1.o") endif() if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") - set(LIBSTAGE2_RELEASE_ARG "") + set(ZIG1_RELEASE_ARG "") else() - set(LIBSTAGE2_RELEASE_ARG --release-fast --strip) + set(ZIG1_RELEASE_ARG --release-fast --strip) endif() -set(BUILD_LIBSTAGE2_ARGS "build-lib" - "src-self-hosted/stage2.zig" +set(BUILD_ZIG1_ARGS + "src-self-hosted/stage1.zig" -target "${ZIG_TARGET_TRIPLE}" "-mcpu=${ZIG_TARGET_MCPU}" - --name zigstage2 + --name zig1 --override-lib-dir "${CMAKE_SOURCE_DIR}/lib" - --cache on --output-dir "${CMAKE_BINARY_DIR}" - ${LIBSTAGE2_RELEASE_ARG} - --bundle-compiler-rt - -fPIC + ${ZIG1_RELEASE_ARG} -lc --pkg-begin build_options "${ZIG_CONFIG_ZIG_OUT}" --pkg-end + --pkg-begin compiler_rt "${CMAKE_SOURCE_DIR}/lib/std/special/compiler_rt.zig" + --pkg-end ) if("${ZIG_TARGET_TRIPLE}" STREQUAL "native") - add_custom_target(zig_build_libstage2 ALL - COMMAND zig0 ${BUILD_LIBSTAGE2_ARGS} + add_custom_target(zig_build_zig1 ALL + COMMAND zig0 ${BUILD_ZIG1_ARGS} DEPENDS zig0 - BYPRODUCTS "${LIBSTAGE2}" + BYPRODUCTS "${ZIG1_OBJECT}" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ) set(ZIG_EXECUTABLE "${zig_BINARY_DIR}/zig") @@ -493,26 +479,27 @@ if("${ZIG_TARGET_TRIPLE}" STREQUAL "native") set(ZIG_EXECUTABLE "${ZIG_EXECUTABLE}.exe") endif() else() - add_custom_target(zig_build_libstage2 ALL - COMMAND "${ZIG_EXECUTABLE}" ${BUILD_LIBSTAGE2_ARGS} - BYPRODUCTS "${LIBSTAGE2}" + add_custom_target(zig_build_zig1 ALL + COMMAND "${ZIG_EXECUTABLE}" ${BUILD_ZIG1_ARGS} + BYPRODUCTS "${ZIG1_OBJECT}" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ) endif() -add_executable(zig "${ZIG_MAIN_SRC}") +# cmake won't let us configure an executable without C sources. +add_executable(zig "${CMAKE_SOURCE_DIR}/src/empty.cpp") set_target_properties(zig PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} LINK_FLAGS ${EXE_LDFLAGS} ) -target_link_libraries(zig zigcompiler "${LIBSTAGE2}") +target_link_libraries(zig zigcompiler "${ZIG1_OBJECT}") if(MSVC) target_link_libraries(zig ntdll.lib) elseif(MINGW) target_link_libraries(zig ntdll) endif() -add_dependencies(zig zig_build_libstage2) +add_dependencies(zig zig_build_zig1) install(TARGETS zig DESTINATION bin) diff --git a/README.md b/README.md index 8031aa790e..242f78d59e 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ Note that you can ### Stage 1: Build Zig from C++ Source Code +This step must be repeated when you make changes to any of the C++ source code. + #### Dependencies ##### POSIX @@ -77,6 +79,41 @@ Hopefully this will be fixed upstream with LLVM 10.0.1. See https://github.com/ziglang/zig/wiki/Building-Zig-on-Windows +### Stage 2: Build Self-Hosted Zig from Zig Source Code + +Now we use the stage1 binary: + +``` +zig build --prefix $(pwd)/stage2 -Denable-llvm +``` + +This produces `stage2/bin/zig` which can be used for testing and development. +Once it is feature complete, it will be used to build stage 3 - the final compiler +binary. + +### Stage 3: Rebuild Self-Hosted Zig Using the Self-Hosted Compiler + +*Note: Stage 2 compiler is not yet able to build Stage 3. Building Stage 3 is +not yet supported.* + +Once the self-hosted compiler can build itself, this will be the actual +compiler binary that we will install to the system. Until then, users should +use stage 1. + +#### Debug / Development Build + +``` +stage2/bin/zig build +``` + +This produces `zig-cache/bin/zig`. + +#### Release / Install Build + +``` +stage2/bin/zig build install -Drelease +``` + ## License The ultimate goal of the Zig project is to serve users. As a first-order diff --git a/build.zig b/build.zig index 17ee040e8f..9e5f2425c2 100644 --- a/build.zig +++ b/build.zig @@ -145,6 +145,7 @@ pub fn build(b: *Builder) !void { exe.addBuildOption([]const []const u8, "log_scopes", log_scopes); exe.addBuildOption([]const []const u8, "zir_dumps", zir_dumps); exe.addBuildOption(bool, "enable_tracy", tracy != null); + exe.addBuildOption(bool, "is_stage1", false); if (tracy) |tracy_path| { const client_cpp = fs.path.join( b.allocator, @@ -165,6 +166,7 @@ pub fn build(b: *Builder) !void { const is_wasmtime_enabled = b.option(bool, "enable-wasmtime", "Use Wasmtime to enable and run WASI libstd tests") orelse false; const glibc_multi_dir = b.option([]const u8, "enable-foreign-glibc", "Provide directory with glibc installations to run cross compiled tests that link glibc"); + test_stage2.addBuildOption(bool, "is_stage1", false); test_stage2.addBuildOption(bool, "have_llvm", enable_llvm); test_stage2.addBuildOption(bool, "enable_qemu", is_qemu_enabled); test_stage2.addBuildOption(bool, "enable_wine", is_wine_enabled); diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 8e18e7fffc..05f76f08f3 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -244,6 +244,7 @@ pub const InitOptions = struct { /// `null` means to not emit a C header file. emit_h: ?EmitLoc = null, link_mode: ?std.builtin.LinkMode = null, + dll_export_fns: ?bool = false, object_format: ?std.builtin.ObjectFormat = null, optimize_mode: std.builtin.Mode = .Debug, keep_source_files_loaded: bool = false, @@ -340,9 +341,13 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { return error.MachineCodeModelNotSupported; } + const is_dyn_lib = switch (options.output_mode) { + .Obj, .Exe => false, + .Lib => (options.link_mode orelse .Static) == .Dynamic, + }; const is_exe_or_dyn_lib = switch (options.output_mode) { .Obj => false, - .Lib => (options.link_mode orelse .Static) == .Dynamic, + .Lib => is_dyn_lib, .Exe => true, }; const must_dynamic_link = dl: { @@ -365,6 +370,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { break :blk lm; } else default_link_mode; + const dll_export_fns = if (options.dll_export_fns) |explicit| explicit else is_dyn_lib; + const libc_dirs = try detectLibCIncludeDirs( arena, options.zig_lib_directory.path.?, @@ -379,7 +386,12 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { break :b true; break :b link_mode == .Dynamic; }; - const pic = options.want_pic orelse must_pic; + const pic = if (options.want_pic) |explicit| pic: { + if (!explicit and must_pic) { + return error.TargetRequiresPIC; + } + break :pic explicit; + } else must_pic; if (options.emit_h != null) fatal("-femit-h not supported yet", .{}); // TODO @@ -464,6 +476,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { hash.add(valgrind); hash.add(single_threaded); hash.add(options.target.os.getVersionRange()); + hash.add(dll_export_fns); const digest = hash.final(); const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); @@ -601,6 +614,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .single_threaded = single_threaded, .debug_link = options.debug_link, .machine_code_model = options.machine_code_model, + .dll_export_fns = dll_export_fns, }); errdefer bin_file.destroy(); diff --git a/src-self-hosted/libcxx.zig b/src-self-hosted/libcxx.zig new file mode 100644 index 0000000000..c7dc24ae9f --- /dev/null +++ b/src-self-hosted/libcxx.zig @@ -0,0 +1,66 @@ +//! TODO build libcxx and libcxxabi from source + +pub const libcxxabi_files = [_][]const u8{ + "src/abort_message.cpp", + "src/cxa_aux_runtime.cpp", + "src/cxa_default_handlers.cpp", + "src/cxa_demangle.cpp", + "src/cxa_exception.cpp", + "src/cxa_exception_storage.cpp", + "src/cxa_guard.cpp", + "src/cxa_handlers.cpp", + "src/cxa_noexception.cpp", + "src/cxa_personality.cpp", + "src/cxa_thread_atexit.cpp", + "src/cxa_unexpected.cpp", + "src/cxa_vector.cpp", + "src/cxa_virtual.cpp", + "src/fallback_malloc.cpp", + "src/private_typeinfo.cpp", + "src/stdlib_exception.cpp", + "src/stdlib_stdexcept.cpp", + "src/stdlib_typeinfo.cpp", +}; + +pub const libcxx_files = [_][]const u8{ + "src/algorithm.cpp", + "src/any.cpp", + "src/bind.cpp", + "src/charconv.cpp", + "src/chrono.cpp", + "src/condition_variable.cpp", + "src/condition_variable_destructor.cpp", + "src/debug.cpp", + "src/exception.cpp", + "src/experimental/memory_resource.cpp", + "src/filesystem/directory_iterator.cpp", + "src/filesystem/operations.cpp", + "src/functional.cpp", + "src/future.cpp", + "src/hash.cpp", + "src/ios.cpp", + "src/iostream.cpp", + "src/locale.cpp", + "src/memory.cpp", + "src/mutex.cpp", + "src/mutex_destructor.cpp", + "src/new.cpp", + "src/optional.cpp", + "src/random.cpp", + "src/regex.cpp", + "src/shared_mutex.cpp", + "src/stdexcept.cpp", + "src/string.cpp", + "src/strstream.cpp", + "src/support/solaris/xlocale.cpp", + "src/support/win32/locale_win32.cpp", + "src/support/win32/support.cpp", + "src/support/win32/thread_win32.cpp", + "src/system_error.cpp", + "src/thread.cpp", + "src/typeinfo.cpp", + "src/utility.cpp", + "src/valarray.cpp", + "src/variant.cpp", + "src/vector.cpp", +}; diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index cdeb2b5e5c..069a90872e 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -62,6 +62,7 @@ pub const Options = struct { stack_check: bool, single_threaded: bool, debug_link: bool = false, + dll_export_fns: bool, gc_sections: ?bool = null, allow_shlib_undefined: ?bool = null, linker_script: ?[]const u8 = null, diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index c9b872cc7e..db68c867b2 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -92,7 +92,7 @@ pub fn log( var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){}; -pub fn main() !void { +pub fn main() anyerror!void { const gpa = if (std.builtin.link_libc) std.heap.c_allocator else &general_purpose_allocator.allocator; defer if (!std.builtin.link_libc) { _ = general_purpose_allocator.deinit(); @@ -102,7 +102,10 @@ pub fn main() !void { const arena = &arena_instance.allocator; const args = try process.argsAlloc(arena); + return mainArgs(gpa, arena, args); +} +pub fn mainArgs(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !void { if (args.len <= 1) { std.log.info("{}", .{usage}); fatal("expected command argument", .{}); @@ -131,7 +134,7 @@ pub fn main() !void { } else if (mem.eql(u8, cmd, "libc")) { return cmdLibC(gpa, cmd_args); } else if (mem.eql(u8, cmd, "targets")) { - const info = try std.zig.system.NativeTargetInfo.detect(arena, .{}); + const info = try detectNativeTargetInfo(arena, .{}); const stdout = io.getStdOut().outStream(); return @import("print_targets.zig").cmdTargets(arena, cmd_args, stdout, info.target); } else if (mem.eql(u8, cmd, "version")) { @@ -183,7 +186,7 @@ const usage_build_generic = \\ -mcmodel=[default|tiny| Limit range of code and data virtual addresses \\ small|kernel| \\ medium|large] - \\ --name [name] Override output name + \\ --name [name] Override root name (not a file path) \\ --mode [mode] Set the build mode \\ Debug (default) optimizations off, safety on \\ ReleaseFast Optimizations on, safety off @@ -197,7 +200,9 @@ const usage_build_generic = \\ -fno-sanitize-c Disable C undefined behavior detection in safe builds \\ -fvalgrind Include valgrind client requests in release builds \\ -fno-valgrind Omit valgrind client requests in debug builds - \\ --strip Exclude debug symbols + \\ -fdll-export-fns Mark exported functions as DLL exports (Windows) + \\ -fno-dll-export-fns Force-disable marking exported functions as DLL exports + \\ --strip Omit debug symbols \\ --single-threaded Code assumes it is only used single-threaded \\ -ofmt=[mode] Override target object format \\ elf Executable and Linking Format @@ -262,6 +267,7 @@ pub fn buildOutputType( var build_mode: std.builtin.Mode = .Debug; var provided_name: ?[]const u8 = null; var link_mode: ?std.builtin.LinkMode = null; + var dll_export_fns: ?bool = null; var root_src_file: ?[]const u8 = null; var version: std.builtin.Version = .{ .major = 0, .minor = 0, .patch = 0 }; var have_version = false; @@ -521,6 +527,10 @@ pub fn buildOutputType( link_mode = .Dynamic; } else if (mem.eql(u8, arg, "-static")) { link_mode = .Static; + } else if (mem.eql(u8, arg, "-fdll-export-fns")) { + dll_export_fns = true; + } else if (mem.eql(u8, arg, "-fno-dll-export-fns")) { + dll_export_fns = false; } else if (mem.eql(u8, arg, "--strip")) { strip = true; } else if (mem.eql(u8, arg, "--single-threaded")) { @@ -902,14 +912,7 @@ pub fn buildOutputType( else => |e| return e, }; - const target_info = try std.zig.system.NativeTargetInfo.detect(gpa, cross_target); - if (target_info.cpu_detection_unimplemented) { - // TODO We want to just use detected_info.target but implementing - // CPU model & feature detection is todo so here we rely on LLVM. - // TODO The workaround to use LLVM to detect features needs to be used for - // `zig targets` as well. - fatal("CPU features detection is not yet available for this system without LLVM extensions", .{}); - } + const target_info = try detectNativeTargetInfo(gpa, cross_target); if (target_info.target.os.tag != .freestanding) { if (ensure_libc_on_non_freestanding) @@ -1116,6 +1119,7 @@ pub fn buildOutputType( .emit_bin = emit_bin_loc, .emit_h = emit_h_loc, .link_mode = link_mode, + .dll_export_fns = dll_export_fns, .object_format = object_format, .optimize_mode = build_mode, .keep_source_files_loaded = zir_out_path != null, @@ -1974,3 +1978,87 @@ fn gimmeMoreOfThoseSweetSweetFileDescriptors() void { test "fds" { gimmeMoreOfThoseSweetSweetFileDescriptors(); } + +fn detectNativeCpuWithLLVM( + arch: std.Target.Cpu.Arch, + llvm_cpu_name_z: ?[*:0]const u8, + llvm_cpu_features_opt: ?[*:0]const u8, +) !std.Target.Cpu { + var result = std.Target.Cpu.baseline(arch); + + if (llvm_cpu_name_z) |cpu_name_z| { + const llvm_cpu_name = mem.spanZ(cpu_name_z); + + for (arch.allCpuModels()) |model| { + const this_llvm_name = model.llvm_name orelse continue; + if (mem.eql(u8, this_llvm_name, llvm_cpu_name)) { + // Here we use the non-dependencies-populated set, + // so that subtracting features later in this function + // affect the prepopulated set. + result = std.Target.Cpu{ + .arch = arch, + .model = model, + .features = model.features, + }; + break; + } + } + } + + const all_features = arch.allFeaturesList(); + + if (llvm_cpu_features_opt) |llvm_cpu_features| { + var it = mem.tokenize(mem.spanZ(llvm_cpu_features), ","); + while (it.next()) |decorated_llvm_feat| { + var op: enum { + add, + sub, + } = undefined; + var llvm_feat: []const u8 = undefined; + if (mem.startsWith(u8, decorated_llvm_feat, "+")) { + op = .add; + llvm_feat = decorated_llvm_feat[1..]; + } else if (mem.startsWith(u8, decorated_llvm_feat, "-")) { + op = .sub; + llvm_feat = decorated_llvm_feat[1..]; + } else { + return error.InvalidLlvmCpuFeaturesFormat; + } + for (all_features) |feature, index_usize| { + const this_llvm_name = feature.llvm_name orelse continue; + if (mem.eql(u8, llvm_feat, this_llvm_name)) { + const index = @intCast(std.Target.Cpu.Feature.Set.Index, index_usize); + switch (op) { + .add => result.features.addFeature(index), + .sub => result.features.removeFeature(index), + } + break; + } + } + } + } + + result.features.populateDependencies(all_features); + return result; +} + +fn detectNativeTargetInfo(gpa: *Allocator, cross_target: std.zig.CrossTarget) !std.zig.system.NativeTargetInfo { + var info = try std.zig.system.NativeTargetInfo.detect(gpa, cross_target); + if (info.cpu_detection_unimplemented) { + const arch = std.Target.current.cpu.arch; + + // We want to just use detected_info.target but implementing + // CPU model & feature detection is todo so here we rely on LLVM. + // https://github.com/ziglang/zig/issues/4591 + if (!build_options.have_llvm) + fatal("CPU features detection is not yet available for {} without LLVM extensions", .{@tagName(arch)}); + + const llvm = @import("llvm.zig"); + const llvm_cpu_name = llvm.GetHostCPUName(); + const llvm_cpu_features = llvm.GetNativeFeatures(); + info.target.cpu = try detectNativeCpuWithLLVM(arch, llvm_cpu_name, llvm_cpu_features); + cross_target.updateCpuFeatures(&info.target.cpu.features); + info.target.cpu.arch = cross_target.getCpuArch(); + } + return info; +} diff --git a/src-self-hosted/musl.zig b/src-self-hosted/musl.zig new file mode 100644 index 0000000000..88536b90fd --- /dev/null +++ b/src-self-hosted/musl.zig @@ -0,0 +1,1843 @@ +//! TODO build musl libc from source + +pub const src_files = [_][]const u8{ + "musl/src/aio/aio.c", + "musl/src/aio/aio_suspend.c", + "musl/src/aio/lio_listio.c", + "musl/src/complex/__cexp.c", + "musl/src/complex/__cexpf.c", + "musl/src/complex/cabs.c", + "musl/src/complex/cabsf.c", + "musl/src/complex/cabsl.c", + "musl/src/complex/cacos.c", + "musl/src/complex/cacosf.c", + "musl/src/complex/cacosh.c", + "musl/src/complex/cacoshf.c", + "musl/src/complex/cacoshl.c", + "musl/src/complex/cacosl.c", + "musl/src/complex/carg.c", + "musl/src/complex/cargf.c", + "musl/src/complex/cargl.c", + "musl/src/complex/casin.c", + "musl/src/complex/casinf.c", + "musl/src/complex/casinh.c", + "musl/src/complex/casinhf.c", + "musl/src/complex/casinhl.c", + "musl/src/complex/casinl.c", + "musl/src/complex/catan.c", + "musl/src/complex/catanf.c", + "musl/src/complex/catanh.c", + "musl/src/complex/catanhf.c", + "musl/src/complex/catanhl.c", + "musl/src/complex/catanl.c", + "musl/src/complex/ccos.c", + "musl/src/complex/ccosf.c", + "musl/src/complex/ccosh.c", + "musl/src/complex/ccoshf.c", + "musl/src/complex/ccoshl.c", + "musl/src/complex/ccosl.c", + "musl/src/complex/cexp.c", + "musl/src/complex/cexpf.c", + "musl/src/complex/cexpl.c", + "musl/src/complex/cimag.c", + "musl/src/complex/cimagf.c", + "musl/src/complex/cimagl.c", + "musl/src/complex/clog.c", + "musl/src/complex/clogf.c", + "musl/src/complex/clogl.c", + "musl/src/complex/conj.c", + "musl/src/complex/conjf.c", + "musl/src/complex/conjl.c", + "musl/src/complex/cpow.c", + "musl/src/complex/cpowf.c", + "musl/src/complex/cpowl.c", + "musl/src/complex/cproj.c", + "musl/src/complex/cprojf.c", + "musl/src/complex/cprojl.c", + "musl/src/complex/creal.c", + "musl/src/complex/crealf.c", + "musl/src/complex/creall.c", + "musl/src/complex/csin.c", + "musl/src/complex/csinf.c", + "musl/src/complex/csinh.c", + "musl/src/complex/csinhf.c", + "musl/src/complex/csinhl.c", + "musl/src/complex/csinl.c", + "musl/src/complex/csqrt.c", + "musl/src/complex/csqrtf.c", + "musl/src/complex/csqrtl.c", + "musl/src/complex/ctan.c", + "musl/src/complex/ctanf.c", + "musl/src/complex/ctanh.c", + "musl/src/complex/ctanhf.c", + "musl/src/complex/ctanhl.c", + "musl/src/complex/ctanl.c", + "musl/src/conf/confstr.c", + "musl/src/conf/fpathconf.c", + "musl/src/conf/legacy.c", + "musl/src/conf/pathconf.c", + "musl/src/conf/sysconf.c", + "musl/src/crypt/crypt.c", + "musl/src/crypt/crypt_blowfish.c", + "musl/src/crypt/crypt_des.c", + "musl/src/crypt/crypt_md5.c", + "musl/src/crypt/crypt_r.c", + "musl/src/crypt/crypt_sha256.c", + "musl/src/crypt/crypt_sha512.c", + "musl/src/crypt/encrypt.c", + "musl/src/ctype/__ctype_b_loc.c", + "musl/src/ctype/__ctype_get_mb_cur_max.c", + "musl/src/ctype/__ctype_tolower_loc.c", + "musl/src/ctype/__ctype_toupper_loc.c", + "musl/src/ctype/isalnum.c", + "musl/src/ctype/isalpha.c", + "musl/src/ctype/isascii.c", + "musl/src/ctype/isblank.c", + "musl/src/ctype/iscntrl.c", + "musl/src/ctype/isdigit.c", + "musl/src/ctype/isgraph.c", + "musl/src/ctype/islower.c", + "musl/src/ctype/isprint.c", + "musl/src/ctype/ispunct.c", + "musl/src/ctype/isspace.c", + "musl/src/ctype/isupper.c", + "musl/src/ctype/iswalnum.c", + "musl/src/ctype/iswalpha.c", + "musl/src/ctype/iswblank.c", + "musl/src/ctype/iswcntrl.c", + "musl/src/ctype/iswctype.c", + "musl/src/ctype/iswdigit.c", + "musl/src/ctype/iswgraph.c", + "musl/src/ctype/iswlower.c", + "musl/src/ctype/iswprint.c", + "musl/src/ctype/iswpunct.c", + "musl/src/ctype/iswspace.c", + "musl/src/ctype/iswupper.c", + "musl/src/ctype/iswxdigit.c", + "musl/src/ctype/isxdigit.c", + "musl/src/ctype/toascii.c", + "musl/src/ctype/tolower.c", + "musl/src/ctype/toupper.c", + "musl/src/ctype/towctrans.c", + "musl/src/ctype/wcswidth.c", + "musl/src/ctype/wctrans.c", + "musl/src/ctype/wcwidth.c", + "musl/src/dirent/alphasort.c", + "musl/src/dirent/closedir.c", + "musl/src/dirent/dirfd.c", + "musl/src/dirent/fdopendir.c", + "musl/src/dirent/opendir.c", + "musl/src/dirent/readdir.c", + "musl/src/dirent/readdir_r.c", + "musl/src/dirent/rewinddir.c", + "musl/src/dirent/scandir.c", + "musl/src/dirent/seekdir.c", + "musl/src/dirent/telldir.c", + "musl/src/dirent/versionsort.c", + "musl/src/env/__environ.c", + "musl/src/env/__init_tls.c", + "musl/src/env/__libc_start_main.c", + "musl/src/env/__reset_tls.c", + "musl/src/env/__stack_chk_fail.c", + "musl/src/env/clearenv.c", + "musl/src/env/getenv.c", + "musl/src/env/putenv.c", + "musl/src/env/secure_getenv.c", + "musl/src/env/setenv.c", + "musl/src/env/unsetenv.c", + "musl/src/errno/__errno_location.c", + "musl/src/errno/strerror.c", + "musl/src/exit/_Exit.c", + "musl/src/exit/abort.c", + "musl/src/exit/arm/__aeabi_atexit.c", + "musl/src/exit/assert.c", + "musl/src/exit/at_quick_exit.c", + "musl/src/exit/atexit.c", + "musl/src/exit/exit.c", + "musl/src/exit/quick_exit.c", + "musl/src/fcntl/creat.c", + "musl/src/fcntl/fcntl.c", + "musl/src/fcntl/open.c", + "musl/src/fcntl/openat.c", + "musl/src/fcntl/posix_fadvise.c", + "musl/src/fcntl/posix_fallocate.c", + "musl/src/fenv/__flt_rounds.c", + "musl/src/fenv/aarch64/fenv.s", + "musl/src/fenv/arm/fenv-hf.S", + "musl/src/fenv/arm/fenv.c", + "musl/src/fenv/fegetexceptflag.c", + "musl/src/fenv/feholdexcept.c", + "musl/src/fenv/fenv.c", + "musl/src/fenv/fesetexceptflag.c", + "musl/src/fenv/fesetround.c", + "musl/src/fenv/feupdateenv.c", + "musl/src/fenv/i386/fenv.s", + "musl/src/fenv/m68k/fenv.c", + "musl/src/fenv/mips/fenv-sf.c", + "musl/src/fenv/mips/fenv.S", + "musl/src/fenv/mips64/fenv-sf.c", + "musl/src/fenv/mips64/fenv.S", + "musl/src/fenv/mipsn32/fenv-sf.c", + "musl/src/fenv/mipsn32/fenv.S", + "musl/src/fenv/powerpc/fenv-sf.c", + "musl/src/fenv/powerpc/fenv.S", + "musl/src/fenv/powerpc64/fenv.c", + "musl/src/fenv/riscv64/fenv-sf.c", + "musl/src/fenv/riscv64/fenv.S", + "musl/src/fenv/s390x/fenv.c", + "musl/src/fenv/sh/fenv-nofpu.c", + "musl/src/fenv/sh/fenv.S", + "musl/src/fenv/x32/fenv.s", + "musl/src/fenv/x86_64/fenv.s", + "musl/src/internal/defsysinfo.c", + "musl/src/internal/floatscan.c", + "musl/src/internal/i386/defsysinfo.s", + "musl/src/internal/intscan.c", + "musl/src/internal/libc.c", + "musl/src/internal/procfdname.c", + "musl/src/internal/sh/__shcall.c", + "musl/src/internal/shgetc.c", + "musl/src/internal/syscall_ret.c", + "musl/src/internal/vdso.c", + "musl/src/internal/version.c", + "musl/src/ipc/ftok.c", + "musl/src/ipc/msgctl.c", + "musl/src/ipc/msgget.c", + "musl/src/ipc/msgrcv.c", + "musl/src/ipc/msgsnd.c", + "musl/src/ipc/semctl.c", + "musl/src/ipc/semget.c", + "musl/src/ipc/semop.c", + "musl/src/ipc/semtimedop.c", + "musl/src/ipc/shmat.c", + "musl/src/ipc/shmctl.c", + "musl/src/ipc/shmdt.c", + "musl/src/ipc/shmget.c", + "musl/src/ldso/__dlsym.c", + "musl/src/ldso/aarch64/dlsym.s", + "musl/src/ldso/aarch64/tlsdesc.s", + "musl/src/ldso/arm/dlsym.s", + "musl/src/ldso/arm/dlsym_time64.S", + "musl/src/ldso/arm/find_exidx.c", + "musl/src/ldso/arm/tlsdesc.S", + "musl/src/ldso/dl_iterate_phdr.c", + "musl/src/ldso/dladdr.c", + "musl/src/ldso/dlclose.c", + "musl/src/ldso/dlerror.c", + "musl/src/ldso/dlinfo.c", + "musl/src/ldso/dlopen.c", + "musl/src/ldso/dlsym.c", + "musl/src/ldso/i386/dlsym.s", + "musl/src/ldso/i386/dlsym_time64.S", + "musl/src/ldso/i386/tlsdesc.s", + "musl/src/ldso/m68k/dlsym.s", + "musl/src/ldso/m68k/dlsym_time64.S", + "musl/src/ldso/microblaze/dlsym.s", + "musl/src/ldso/microblaze/dlsym_time64.S", + "musl/src/ldso/mips/dlsym.s", + "musl/src/ldso/mips/dlsym_time64.S", + "musl/src/ldso/mips64/dlsym.s", + "musl/src/ldso/mipsn32/dlsym.s", + "musl/src/ldso/mipsn32/dlsym_time64.S", + "musl/src/ldso/or1k/dlsym.s", + "musl/src/ldso/or1k/dlsym_time64.S", + "musl/src/ldso/powerpc/dlsym.s", + "musl/src/ldso/powerpc/dlsym_time64.S", + "musl/src/ldso/powerpc64/dlsym.s", + "musl/src/ldso/riscv64/dlsym.s", + "musl/src/ldso/s390x/dlsym.s", + "musl/src/ldso/sh/dlsym.s", + "musl/src/ldso/sh/dlsym_time64.S", + "musl/src/ldso/tlsdesc.c", + "musl/src/ldso/x32/dlsym.s", + "musl/src/ldso/x86_64/dlsym.s", + "musl/src/ldso/x86_64/tlsdesc.s", + "musl/src/legacy/cuserid.c", + "musl/src/legacy/daemon.c", + "musl/src/legacy/err.c", + "musl/src/legacy/euidaccess.c", + "musl/src/legacy/ftw.c", + "musl/src/legacy/futimes.c", + "musl/src/legacy/getdtablesize.c", + "musl/src/legacy/getloadavg.c", + "musl/src/legacy/getpagesize.c", + "musl/src/legacy/getpass.c", + "musl/src/legacy/getusershell.c", + "musl/src/legacy/isastream.c", + "musl/src/legacy/lutimes.c", + "musl/src/legacy/ulimit.c", + "musl/src/legacy/utmpx.c", + "musl/src/legacy/valloc.c", + "musl/src/linux/adjtime.c", + "musl/src/linux/adjtimex.c", + "musl/src/linux/arch_prctl.c", + "musl/src/linux/brk.c", + "musl/src/linux/cache.c", + "musl/src/linux/cap.c", + "musl/src/linux/chroot.c", + "musl/src/linux/clock_adjtime.c", + "musl/src/linux/clone.c", + "musl/src/linux/copy_file_range.c", + "musl/src/linux/epoll.c", + "musl/src/linux/eventfd.c", + "musl/src/linux/fallocate.c", + "musl/src/linux/fanotify.c", + "musl/src/linux/flock.c", + "musl/src/linux/getdents.c", + "musl/src/linux/getrandom.c", + "musl/src/linux/inotify.c", + "musl/src/linux/ioperm.c", + "musl/src/linux/iopl.c", + "musl/src/linux/klogctl.c", + "musl/src/linux/membarrier.c", + "musl/src/linux/memfd_create.c", + "musl/src/linux/mlock2.c", + "musl/src/linux/module.c", + "musl/src/linux/mount.c", + "musl/src/linux/name_to_handle_at.c", + "musl/src/linux/open_by_handle_at.c", + "musl/src/linux/personality.c", + "musl/src/linux/pivot_root.c", + "musl/src/linux/ppoll.c", + "musl/src/linux/prctl.c", + "musl/src/linux/prlimit.c", + "musl/src/linux/process_vm.c", + "musl/src/linux/ptrace.c", + "musl/src/linux/quotactl.c", + "musl/src/linux/readahead.c", + "musl/src/linux/reboot.c", + "musl/src/linux/remap_file_pages.c", + "musl/src/linux/sbrk.c", + "musl/src/linux/sendfile.c", + "musl/src/linux/setfsgid.c", + "musl/src/linux/setfsuid.c", + "musl/src/linux/setgroups.c", + "musl/src/linux/sethostname.c", + "musl/src/linux/setns.c", + "musl/src/linux/settimeofday.c", + "musl/src/linux/signalfd.c", + "musl/src/linux/splice.c", + "musl/src/linux/stime.c", + "musl/src/linux/swap.c", + "musl/src/linux/sync_file_range.c", + "musl/src/linux/syncfs.c", + "musl/src/linux/sysinfo.c", + "musl/src/linux/tee.c", + "musl/src/linux/timerfd.c", + "musl/src/linux/unshare.c", + "musl/src/linux/utimes.c", + "musl/src/linux/vhangup.c", + "musl/src/linux/vmsplice.c", + "musl/src/linux/wait3.c", + "musl/src/linux/wait4.c", + "musl/src/linux/x32/sysinfo.c", + "musl/src/linux/xattr.c", + "musl/src/locale/__lctrans.c", + "musl/src/locale/__mo_lookup.c", + "musl/src/locale/bind_textdomain_codeset.c", + "musl/src/locale/c_locale.c", + "musl/src/locale/catclose.c", + "musl/src/locale/catgets.c", + "musl/src/locale/catopen.c", + "musl/src/locale/dcngettext.c", + "musl/src/locale/duplocale.c", + "musl/src/locale/freelocale.c", + "musl/src/locale/iconv.c", + "musl/src/locale/iconv_close.c", + "musl/src/locale/langinfo.c", + "musl/src/locale/locale_map.c", + "musl/src/locale/localeconv.c", + "musl/src/locale/newlocale.c", + "musl/src/locale/pleval.c", + "musl/src/locale/setlocale.c", + "musl/src/locale/strcoll.c", + "musl/src/locale/strfmon.c", + "musl/src/locale/strxfrm.c", + "musl/src/locale/textdomain.c", + "musl/src/locale/uselocale.c", + "musl/src/locale/wcscoll.c", + "musl/src/locale/wcsxfrm.c", + "musl/src/malloc/aligned_alloc.c", + "musl/src/malloc/expand_heap.c", + "musl/src/malloc/lite_malloc.c", + "musl/src/malloc/malloc.c", + "musl/src/malloc/malloc_usable_size.c", + "musl/src/malloc/memalign.c", + "musl/src/malloc/posix_memalign.c", + "musl/src/math/__cos.c", + "musl/src/math/__cosdf.c", + "musl/src/math/__cosl.c", + "musl/src/math/__expo2.c", + "musl/src/math/__expo2f.c", + "musl/src/math/__fpclassify.c", + "musl/src/math/__fpclassifyf.c", + "musl/src/math/__fpclassifyl.c", + "musl/src/math/__invtrigl.c", + "musl/src/math/__math_divzero.c", + "musl/src/math/__math_divzerof.c", + "musl/src/math/__math_invalid.c", + "musl/src/math/__math_invalidf.c", + "musl/src/math/__math_oflow.c", + "musl/src/math/__math_oflowf.c", + "musl/src/math/__math_uflow.c", + "musl/src/math/__math_uflowf.c", + "musl/src/math/__math_xflow.c", + "musl/src/math/__math_xflowf.c", + "musl/src/math/__polevll.c", + "musl/src/math/__rem_pio2.c", + "musl/src/math/__rem_pio2_large.c", + "musl/src/math/__rem_pio2f.c", + "musl/src/math/__rem_pio2l.c", + "musl/src/math/__signbit.c", + "musl/src/math/__signbitf.c", + "musl/src/math/__signbitl.c", + "musl/src/math/__sin.c", + "musl/src/math/__sindf.c", + "musl/src/math/__sinl.c", + "musl/src/math/__tan.c", + "musl/src/math/__tandf.c", + "musl/src/math/__tanl.c", + "musl/src/math/aarch64/ceil.c", + "musl/src/math/aarch64/ceilf.c", + "musl/src/math/aarch64/fabs.c", + "musl/src/math/aarch64/fabsf.c", + "musl/src/math/aarch64/floor.c", + "musl/src/math/aarch64/floorf.c", + "musl/src/math/aarch64/fma.c", + "musl/src/math/aarch64/fmaf.c", + "musl/src/math/aarch64/fmax.c", + "musl/src/math/aarch64/fmaxf.c", + "musl/src/math/aarch64/fmin.c", + "musl/src/math/aarch64/fminf.c", + "musl/src/math/aarch64/llrint.c", + "musl/src/math/aarch64/llrintf.c", + "musl/src/math/aarch64/llround.c", + "musl/src/math/aarch64/llroundf.c", + "musl/src/math/aarch64/lrint.c", + "musl/src/math/aarch64/lrintf.c", + "musl/src/math/aarch64/lround.c", + "musl/src/math/aarch64/lroundf.c", + "musl/src/math/aarch64/nearbyint.c", + "musl/src/math/aarch64/nearbyintf.c", + "musl/src/math/aarch64/rint.c", + "musl/src/math/aarch64/rintf.c", + "musl/src/math/aarch64/round.c", + "musl/src/math/aarch64/roundf.c", + "musl/src/math/aarch64/sqrt.c", + "musl/src/math/aarch64/sqrtf.c", + "musl/src/math/aarch64/trunc.c", + "musl/src/math/aarch64/truncf.c", + "musl/src/math/acos.c", + "musl/src/math/acosf.c", + "musl/src/math/acosh.c", + "musl/src/math/acoshf.c", + "musl/src/math/acoshl.c", + "musl/src/math/acosl.c", + "musl/src/math/arm/fabs.c", + "musl/src/math/arm/fabsf.c", + "musl/src/math/arm/fma.c", + "musl/src/math/arm/fmaf.c", + "musl/src/math/arm/sqrt.c", + "musl/src/math/arm/sqrtf.c", + "musl/src/math/asin.c", + "musl/src/math/asinf.c", + "musl/src/math/asinh.c", + "musl/src/math/asinhf.c", + "musl/src/math/asinhl.c", + "musl/src/math/asinl.c", + "musl/src/math/atan.c", + "musl/src/math/atan2.c", + "musl/src/math/atan2f.c", + "musl/src/math/atan2l.c", + "musl/src/math/atanf.c", + "musl/src/math/atanh.c", + "musl/src/math/atanhf.c", + "musl/src/math/atanhl.c", + "musl/src/math/atanl.c", + "musl/src/math/cbrt.c", + "musl/src/math/cbrtf.c", + "musl/src/math/cbrtl.c", + "musl/src/math/ceil.c", + "musl/src/math/ceilf.c", + "musl/src/math/ceill.c", + "musl/src/math/copysign.c", + "musl/src/math/copysignf.c", + "musl/src/math/copysignl.c", + "musl/src/math/cos.c", + "musl/src/math/cosf.c", + "musl/src/math/cosh.c", + "musl/src/math/coshf.c", + "musl/src/math/coshl.c", + "musl/src/math/cosl.c", + "musl/src/math/erf.c", + "musl/src/math/erff.c", + "musl/src/math/erfl.c", + "musl/src/math/exp.c", + "musl/src/math/exp10.c", + "musl/src/math/exp10f.c", + "musl/src/math/exp10l.c", + "musl/src/math/exp2.c", + "musl/src/math/exp2f.c", + "musl/src/math/exp2f_data.c", + "musl/src/math/exp2l.c", + "musl/src/math/exp_data.c", + "musl/src/math/expf.c", + "musl/src/math/expl.c", + "musl/src/math/expm1.c", + "musl/src/math/expm1f.c", + "musl/src/math/expm1l.c", + "musl/src/math/fabs.c", + "musl/src/math/fabsf.c", + "musl/src/math/fabsl.c", + "musl/src/math/fdim.c", + "musl/src/math/fdimf.c", + "musl/src/math/fdiml.c", + "musl/src/math/finite.c", + "musl/src/math/finitef.c", + "musl/src/math/floor.c", + "musl/src/math/floorf.c", + "musl/src/math/floorl.c", + "musl/src/math/fma.c", + "musl/src/math/fmaf.c", + "musl/src/math/fmal.c", + "musl/src/math/fmax.c", + "musl/src/math/fmaxf.c", + "musl/src/math/fmaxl.c", + "musl/src/math/fmin.c", + "musl/src/math/fminf.c", + "musl/src/math/fminl.c", + "musl/src/math/fmod.c", + "musl/src/math/fmodf.c", + "musl/src/math/fmodl.c", + "musl/src/math/frexp.c", + "musl/src/math/frexpf.c", + "musl/src/math/frexpl.c", + "musl/src/math/hypot.c", + "musl/src/math/hypotf.c", + "musl/src/math/hypotl.c", + "musl/src/math/i386/__invtrigl.s", + "musl/src/math/i386/acos.s", + "musl/src/math/i386/acosf.s", + "musl/src/math/i386/acosl.s", + "musl/src/math/i386/asin.s", + "musl/src/math/i386/asinf.s", + "musl/src/math/i386/asinl.s", + "musl/src/math/i386/atan.s", + "musl/src/math/i386/atan2.s", + "musl/src/math/i386/atan2f.s", + "musl/src/math/i386/atan2l.s", + "musl/src/math/i386/atanf.s", + "musl/src/math/i386/atanl.s", + "musl/src/math/i386/ceil.s", + "musl/src/math/i386/ceilf.s", + "musl/src/math/i386/ceill.s", + "musl/src/math/i386/exp2l.s", + "musl/src/math/i386/exp_ld.s", + "musl/src/math/i386/expl.s", + "musl/src/math/i386/expm1l.s", + "musl/src/math/i386/fabs.s", + "musl/src/math/i386/fabsf.s", + "musl/src/math/i386/fabsl.s", + "musl/src/math/i386/floor.s", + "musl/src/math/i386/floorf.s", + "musl/src/math/i386/floorl.s", + "musl/src/math/i386/fmod.s", + "musl/src/math/i386/fmodf.s", + "musl/src/math/i386/fmodl.s", + "musl/src/math/i386/hypot.s", + "musl/src/math/i386/hypotf.s", + "musl/src/math/i386/ldexp.s", + "musl/src/math/i386/ldexpf.s", + "musl/src/math/i386/ldexpl.s", + "musl/src/math/i386/llrint.s", + "musl/src/math/i386/llrintf.s", + "musl/src/math/i386/llrintl.s", + "musl/src/math/i386/log.s", + "musl/src/math/i386/log10.s", + "musl/src/math/i386/log10f.s", + "musl/src/math/i386/log10l.s", + "musl/src/math/i386/log1p.s", + "musl/src/math/i386/log1pf.s", + "musl/src/math/i386/log1pl.s", + "musl/src/math/i386/log2.s", + "musl/src/math/i386/log2f.s", + "musl/src/math/i386/log2l.s", + "musl/src/math/i386/logf.s", + "musl/src/math/i386/logl.s", + "musl/src/math/i386/lrint.s", + "musl/src/math/i386/lrintf.s", + "musl/src/math/i386/lrintl.s", + "musl/src/math/i386/remainder.s", + "musl/src/math/i386/remainderf.s", + "musl/src/math/i386/remainderl.s", + "musl/src/math/i386/remquo.s", + "musl/src/math/i386/remquof.s", + "musl/src/math/i386/remquol.s", + "musl/src/math/i386/rint.s", + "musl/src/math/i386/rintf.s", + "musl/src/math/i386/rintl.s", + "musl/src/math/i386/scalbln.s", + "musl/src/math/i386/scalblnf.s", + "musl/src/math/i386/scalblnl.s", + "musl/src/math/i386/scalbn.s", + "musl/src/math/i386/scalbnf.s", + "musl/src/math/i386/scalbnl.s", + "musl/src/math/i386/sqrt.s", + "musl/src/math/i386/sqrtf.s", + "musl/src/math/i386/sqrtl.s", + "musl/src/math/i386/trunc.s", + "musl/src/math/i386/truncf.s", + "musl/src/math/i386/truncl.s", + "musl/src/math/ilogb.c", + "musl/src/math/ilogbf.c", + "musl/src/math/ilogbl.c", + "musl/src/math/j0.c", + "musl/src/math/j0f.c", + "musl/src/math/j1.c", + "musl/src/math/j1f.c", + "musl/src/math/jn.c", + "musl/src/math/jnf.c", + "musl/src/math/ldexp.c", + "musl/src/math/ldexpf.c", + "musl/src/math/ldexpl.c", + "musl/src/math/lgamma.c", + "musl/src/math/lgamma_r.c", + "musl/src/math/lgammaf.c", + "musl/src/math/lgammaf_r.c", + "musl/src/math/lgammal.c", + "musl/src/math/llrint.c", + "musl/src/math/llrintf.c", + "musl/src/math/llrintl.c", + "musl/src/math/llround.c", + "musl/src/math/llroundf.c", + "musl/src/math/llroundl.c", + "musl/src/math/log.c", + "musl/src/math/log10.c", + "musl/src/math/log10f.c", + "musl/src/math/log10l.c", + "musl/src/math/log1p.c", + "musl/src/math/log1pf.c", + "musl/src/math/log1pl.c", + "musl/src/math/log2.c", + "musl/src/math/log2_data.c", + "musl/src/math/log2f.c", + "musl/src/math/log2f_data.c", + "musl/src/math/log2l.c", + "musl/src/math/log_data.c", + "musl/src/math/logb.c", + "musl/src/math/logbf.c", + "musl/src/math/logbl.c", + "musl/src/math/logf.c", + "musl/src/math/logf_data.c", + "musl/src/math/logl.c", + "musl/src/math/lrint.c", + "musl/src/math/lrintf.c", + "musl/src/math/lrintl.c", + "musl/src/math/lround.c", + "musl/src/math/lroundf.c", + "musl/src/math/lroundl.c", + "musl/src/math/mips/fabs.c", + "musl/src/math/mips/fabsf.c", + "musl/src/math/mips/sqrt.c", + "musl/src/math/mips/sqrtf.c", + "musl/src/math/modf.c", + "musl/src/math/modff.c", + "musl/src/math/modfl.c", + "musl/src/math/nan.c", + "musl/src/math/nanf.c", + "musl/src/math/nanl.c", + "musl/src/math/nearbyint.c", + "musl/src/math/nearbyintf.c", + "musl/src/math/nearbyintl.c", + "musl/src/math/nextafter.c", + "musl/src/math/nextafterf.c", + "musl/src/math/nextafterl.c", + "musl/src/math/nexttoward.c", + "musl/src/math/nexttowardf.c", + "musl/src/math/nexttowardl.c", + "musl/src/math/pow.c", + "musl/src/math/pow_data.c", + "musl/src/math/powerpc/fabs.c", + "musl/src/math/powerpc/fabsf.c", + "musl/src/math/powerpc/fma.c", + "musl/src/math/powerpc/fmaf.c", + "musl/src/math/powerpc/sqrt.c", + "musl/src/math/powerpc/sqrtf.c", + "musl/src/math/powerpc64/ceil.c", + "musl/src/math/powerpc64/ceilf.c", + "musl/src/math/powerpc64/fabs.c", + "musl/src/math/powerpc64/fabsf.c", + "musl/src/math/powerpc64/floor.c", + "musl/src/math/powerpc64/floorf.c", + "musl/src/math/powerpc64/fma.c", + "musl/src/math/powerpc64/fmaf.c", + "musl/src/math/powerpc64/fmax.c", + "musl/src/math/powerpc64/fmaxf.c", + "musl/src/math/powerpc64/fmin.c", + "musl/src/math/powerpc64/fminf.c", + "musl/src/math/powerpc64/lrint.c", + "musl/src/math/powerpc64/lrintf.c", + "musl/src/math/powerpc64/lround.c", + "musl/src/math/powerpc64/lroundf.c", + "musl/src/math/powerpc64/round.c", + "musl/src/math/powerpc64/roundf.c", + "musl/src/math/powerpc64/sqrt.c", + "musl/src/math/powerpc64/sqrtf.c", + "musl/src/math/powerpc64/trunc.c", + "musl/src/math/powerpc64/truncf.c", + "musl/src/math/powf.c", + "musl/src/math/powf_data.c", + "musl/src/math/powl.c", + "musl/src/math/remainder.c", + "musl/src/math/remainderf.c", + "musl/src/math/remainderl.c", + "musl/src/math/remquo.c", + "musl/src/math/remquof.c", + "musl/src/math/remquol.c", + "musl/src/math/rint.c", + "musl/src/math/rintf.c", + "musl/src/math/rintl.c", + "musl/src/math/riscv64/copysign.c", + "musl/src/math/riscv64/copysignf.c", + "musl/src/math/riscv64/fabs.c", + "musl/src/math/riscv64/fabsf.c", + "musl/src/math/riscv64/fma.c", + "musl/src/math/riscv64/fmaf.c", + "musl/src/math/riscv64/fmax.c", + "musl/src/math/riscv64/fmaxf.c", + "musl/src/math/riscv64/fmin.c", + "musl/src/math/riscv64/fminf.c", + "musl/src/math/riscv64/sqrt.c", + "musl/src/math/riscv64/sqrtf.c", + "musl/src/math/round.c", + "musl/src/math/roundf.c", + "musl/src/math/roundl.c", + "musl/src/math/s390x/ceil.c", + "musl/src/math/s390x/ceilf.c", + "musl/src/math/s390x/ceill.c", + "musl/src/math/s390x/fabs.c", + "musl/src/math/s390x/fabsf.c", + "musl/src/math/s390x/fabsl.c", + "musl/src/math/s390x/floor.c", + "musl/src/math/s390x/floorf.c", + "musl/src/math/s390x/floorl.c", + "musl/src/math/s390x/fma.c", + "musl/src/math/s390x/fmaf.c", + "musl/src/math/s390x/nearbyint.c", + "musl/src/math/s390x/nearbyintf.c", + "musl/src/math/s390x/nearbyintl.c", + "musl/src/math/s390x/rint.c", + "musl/src/math/s390x/rintf.c", + "musl/src/math/s390x/rintl.c", + "musl/src/math/s390x/round.c", + "musl/src/math/s390x/roundf.c", + "musl/src/math/s390x/roundl.c", + "musl/src/math/s390x/sqrt.c", + "musl/src/math/s390x/sqrtf.c", + "musl/src/math/s390x/sqrtl.c", + "musl/src/math/s390x/trunc.c", + "musl/src/math/s390x/truncf.c", + "musl/src/math/s390x/truncl.c", + "musl/src/math/scalb.c", + "musl/src/math/scalbf.c", + "musl/src/math/scalbln.c", + "musl/src/math/scalblnf.c", + "musl/src/math/scalblnl.c", + "musl/src/math/scalbn.c", + "musl/src/math/scalbnf.c", + "musl/src/math/scalbnl.c", + "musl/src/math/signgam.c", + "musl/src/math/significand.c", + "musl/src/math/significandf.c", + "musl/src/math/sin.c", + "musl/src/math/sincos.c", + "musl/src/math/sincosf.c", + "musl/src/math/sincosl.c", + "musl/src/math/sinf.c", + "musl/src/math/sinh.c", + "musl/src/math/sinhf.c", + "musl/src/math/sinhl.c", + "musl/src/math/sinl.c", + "musl/src/math/sqrt.c", + "musl/src/math/sqrtf.c", + "musl/src/math/sqrtl.c", + "musl/src/math/tan.c", + "musl/src/math/tanf.c", + "musl/src/math/tanh.c", + "musl/src/math/tanhf.c", + "musl/src/math/tanhl.c", + "musl/src/math/tanl.c", + "musl/src/math/tgamma.c", + "musl/src/math/tgammaf.c", + "musl/src/math/tgammal.c", + "musl/src/math/trunc.c", + "musl/src/math/truncf.c", + "musl/src/math/truncl.c", + "musl/src/math/x32/__invtrigl.s", + "musl/src/math/x32/acosl.s", + "musl/src/math/x32/asinl.s", + "musl/src/math/x32/atan2l.s", + "musl/src/math/x32/atanl.s", + "musl/src/math/x32/ceill.s", + "musl/src/math/x32/exp2l.s", + "musl/src/math/x32/expl.s", + "musl/src/math/x32/expm1l.s", + "musl/src/math/x32/fabs.s", + "musl/src/math/x32/fabsf.s", + "musl/src/math/x32/fabsl.s", + "musl/src/math/x32/floorl.s", + "musl/src/math/x32/fma.c", + "musl/src/math/x32/fmaf.c", + "musl/src/math/x32/fmodl.s", + "musl/src/math/x32/llrint.s", + "musl/src/math/x32/llrintf.s", + "musl/src/math/x32/llrintl.s", + "musl/src/math/x32/log10l.s", + "musl/src/math/x32/log1pl.s", + "musl/src/math/x32/log2l.s", + "musl/src/math/x32/logl.s", + "musl/src/math/x32/lrint.s", + "musl/src/math/x32/lrintf.s", + "musl/src/math/x32/lrintl.s", + "musl/src/math/x32/remainderl.s", + "musl/src/math/x32/rintl.s", + "musl/src/math/x32/sqrt.s", + "musl/src/math/x32/sqrtf.s", + "musl/src/math/x32/sqrtl.s", + "musl/src/math/x32/truncl.s", + "musl/src/math/x86_64/__invtrigl.s", + "musl/src/math/x86_64/acosl.s", + "musl/src/math/x86_64/asinl.s", + "musl/src/math/x86_64/atan2l.s", + "musl/src/math/x86_64/atanl.s", + "musl/src/math/x86_64/ceill.s", + "musl/src/math/x86_64/exp2l.s", + "musl/src/math/x86_64/expl.s", + "musl/src/math/x86_64/expm1l.s", + "musl/src/math/x86_64/fabs.s", + "musl/src/math/x86_64/fabsf.s", + "musl/src/math/x86_64/fabsl.s", + "musl/src/math/x86_64/floorl.s", + "musl/src/math/x86_64/fma.c", + "musl/src/math/x86_64/fmaf.c", + "musl/src/math/x86_64/fmodl.s", + "musl/src/math/x86_64/llrint.s", + "musl/src/math/x86_64/llrintf.s", + "musl/src/math/x86_64/llrintl.s", + "musl/src/math/x86_64/log10l.s", + "musl/src/math/x86_64/log1pl.s", + "musl/src/math/x86_64/log2l.s", + "musl/src/math/x86_64/logl.s", + "musl/src/math/x86_64/lrint.s", + "musl/src/math/x86_64/lrintf.s", + "musl/src/math/x86_64/lrintl.s", + "musl/src/math/x86_64/remainderl.s", + "musl/src/math/x86_64/rintl.s", + "musl/src/math/x86_64/sqrt.s", + "musl/src/math/x86_64/sqrtf.s", + "musl/src/math/x86_64/sqrtl.s", + "musl/src/math/x86_64/truncl.s", + "musl/src/misc/a64l.c", + "musl/src/misc/basename.c", + "musl/src/misc/dirname.c", + "musl/src/misc/ffs.c", + "musl/src/misc/ffsl.c", + "musl/src/misc/ffsll.c", + "musl/src/misc/fmtmsg.c", + "musl/src/misc/forkpty.c", + "musl/src/misc/get_current_dir_name.c", + "musl/src/misc/getauxval.c", + "musl/src/misc/getdomainname.c", + "musl/src/misc/getentropy.c", + "musl/src/misc/gethostid.c", + "musl/src/misc/getopt.c", + "musl/src/misc/getopt_long.c", + "musl/src/misc/getpriority.c", + "musl/src/misc/getresgid.c", + "musl/src/misc/getresuid.c", + "musl/src/misc/getrlimit.c", + "musl/src/misc/getrusage.c", + "musl/src/misc/getsubopt.c", + "musl/src/misc/initgroups.c", + "musl/src/misc/ioctl.c", + "musl/src/misc/issetugid.c", + "musl/src/misc/lockf.c", + "musl/src/misc/login_tty.c", + "musl/src/misc/mntent.c", + "musl/src/misc/nftw.c", + "musl/src/misc/openpty.c", + "musl/src/misc/ptsname.c", + "musl/src/misc/pty.c", + "musl/src/misc/realpath.c", + "musl/src/misc/setdomainname.c", + "musl/src/misc/setpriority.c", + "musl/src/misc/setrlimit.c", + "musl/src/misc/syscall.c", + "musl/src/misc/syslog.c", + "musl/src/misc/uname.c", + "musl/src/misc/wordexp.c", + "musl/src/mman/madvise.c", + "musl/src/mman/mincore.c", + "musl/src/mman/mlock.c", + "musl/src/mman/mlockall.c", + "musl/src/mman/mmap.c", + "musl/src/mman/mprotect.c", + "musl/src/mman/mremap.c", + "musl/src/mman/msync.c", + "musl/src/mman/munlock.c", + "musl/src/mman/munlockall.c", + "musl/src/mman/munmap.c", + "musl/src/mman/posix_madvise.c", + "musl/src/mman/shm_open.c", + "musl/src/mq/mq_close.c", + "musl/src/mq/mq_getattr.c", + "musl/src/mq/mq_notify.c", + "musl/src/mq/mq_open.c", + "musl/src/mq/mq_receive.c", + "musl/src/mq/mq_send.c", + "musl/src/mq/mq_setattr.c", + "musl/src/mq/mq_timedreceive.c", + "musl/src/mq/mq_timedsend.c", + "musl/src/mq/mq_unlink.c", + "musl/src/multibyte/btowc.c", + "musl/src/multibyte/c16rtomb.c", + "musl/src/multibyte/c32rtomb.c", + "musl/src/multibyte/internal.c", + "musl/src/multibyte/mblen.c", + "musl/src/multibyte/mbrlen.c", + "musl/src/multibyte/mbrtoc16.c", + "musl/src/multibyte/mbrtoc32.c", + "musl/src/multibyte/mbrtowc.c", + "musl/src/multibyte/mbsinit.c", + "musl/src/multibyte/mbsnrtowcs.c", + "musl/src/multibyte/mbsrtowcs.c", + "musl/src/multibyte/mbstowcs.c", + "musl/src/multibyte/mbtowc.c", + "musl/src/multibyte/wcrtomb.c", + "musl/src/multibyte/wcsnrtombs.c", + "musl/src/multibyte/wcsrtombs.c", + "musl/src/multibyte/wcstombs.c", + "musl/src/multibyte/wctob.c", + "musl/src/multibyte/wctomb.c", + "musl/src/network/accept.c", + "musl/src/network/accept4.c", + "musl/src/network/bind.c", + "musl/src/network/connect.c", + "musl/src/network/dn_comp.c", + "musl/src/network/dn_expand.c", + "musl/src/network/dn_skipname.c", + "musl/src/network/dns_parse.c", + "musl/src/network/ent.c", + "musl/src/network/ether.c", + "musl/src/network/freeaddrinfo.c", + "musl/src/network/gai_strerror.c", + "musl/src/network/getaddrinfo.c", + "musl/src/network/gethostbyaddr.c", + "musl/src/network/gethostbyaddr_r.c", + "musl/src/network/gethostbyname.c", + "musl/src/network/gethostbyname2.c", + "musl/src/network/gethostbyname2_r.c", + "musl/src/network/gethostbyname_r.c", + "musl/src/network/getifaddrs.c", + "musl/src/network/getnameinfo.c", + "musl/src/network/getpeername.c", + "musl/src/network/getservbyname.c", + "musl/src/network/getservbyname_r.c", + "musl/src/network/getservbyport.c", + "musl/src/network/getservbyport_r.c", + "musl/src/network/getsockname.c", + "musl/src/network/getsockopt.c", + "musl/src/network/h_errno.c", + "musl/src/network/herror.c", + "musl/src/network/hstrerror.c", + "musl/src/network/htonl.c", + "musl/src/network/htons.c", + "musl/src/network/if_freenameindex.c", + "musl/src/network/if_indextoname.c", + "musl/src/network/if_nameindex.c", + "musl/src/network/if_nametoindex.c", + "musl/src/network/in6addr_any.c", + "musl/src/network/in6addr_loopback.c", + "musl/src/network/inet_addr.c", + "musl/src/network/inet_aton.c", + "musl/src/network/inet_legacy.c", + "musl/src/network/inet_ntoa.c", + "musl/src/network/inet_ntop.c", + "musl/src/network/inet_pton.c", + "musl/src/network/listen.c", + "musl/src/network/lookup_ipliteral.c", + "musl/src/network/lookup_name.c", + "musl/src/network/lookup_serv.c", + "musl/src/network/netlink.c", + "musl/src/network/netname.c", + "musl/src/network/ns_parse.c", + "musl/src/network/ntohl.c", + "musl/src/network/ntohs.c", + "musl/src/network/proto.c", + "musl/src/network/recv.c", + "musl/src/network/recvfrom.c", + "musl/src/network/recvmmsg.c", + "musl/src/network/recvmsg.c", + "musl/src/network/res_init.c", + "musl/src/network/res_mkquery.c", + "musl/src/network/res_msend.c", + "musl/src/network/res_query.c", + "musl/src/network/res_querydomain.c", + "musl/src/network/res_send.c", + "musl/src/network/res_state.c", + "musl/src/network/resolvconf.c", + "musl/src/network/send.c", + "musl/src/network/sendmmsg.c", + "musl/src/network/sendmsg.c", + "musl/src/network/sendto.c", + "musl/src/network/serv.c", + "musl/src/network/setsockopt.c", + "musl/src/network/shutdown.c", + "musl/src/network/sockatmark.c", + "musl/src/network/socket.c", + "musl/src/network/socketpair.c", + "musl/src/passwd/fgetgrent.c", + "musl/src/passwd/fgetpwent.c", + "musl/src/passwd/fgetspent.c", + "musl/src/passwd/getgr_a.c", + "musl/src/passwd/getgr_r.c", + "musl/src/passwd/getgrent.c", + "musl/src/passwd/getgrent_a.c", + "musl/src/passwd/getgrouplist.c", + "musl/src/passwd/getpw_a.c", + "musl/src/passwd/getpw_r.c", + "musl/src/passwd/getpwent.c", + "musl/src/passwd/getpwent_a.c", + "musl/src/passwd/getspent.c", + "musl/src/passwd/getspnam.c", + "musl/src/passwd/getspnam_r.c", + "musl/src/passwd/lckpwdf.c", + "musl/src/passwd/nscd_query.c", + "musl/src/passwd/putgrent.c", + "musl/src/passwd/putpwent.c", + "musl/src/passwd/putspent.c", + "musl/src/prng/__rand48_step.c", + "musl/src/prng/__seed48.c", + "musl/src/prng/drand48.c", + "musl/src/prng/lcong48.c", + "musl/src/prng/lrand48.c", + "musl/src/prng/mrand48.c", + "musl/src/prng/rand.c", + "musl/src/prng/rand_r.c", + "musl/src/prng/random.c", + "musl/src/prng/seed48.c", + "musl/src/prng/srand48.c", + "musl/src/process/arm/vfork.s", + "musl/src/process/execl.c", + "musl/src/process/execle.c", + "musl/src/process/execlp.c", + "musl/src/process/execv.c", + "musl/src/process/execve.c", + "musl/src/process/execvp.c", + "musl/src/process/fexecve.c", + "musl/src/process/fork.c", + "musl/src/process/i386/vfork.s", + "musl/src/process/posix_spawn.c", + "musl/src/process/posix_spawn_file_actions_addchdir.c", + "musl/src/process/posix_spawn_file_actions_addclose.c", + "musl/src/process/posix_spawn_file_actions_adddup2.c", + "musl/src/process/posix_spawn_file_actions_addfchdir.c", + "musl/src/process/posix_spawn_file_actions_addopen.c", + "musl/src/process/posix_spawn_file_actions_destroy.c", + "musl/src/process/posix_spawn_file_actions_init.c", + "musl/src/process/posix_spawnattr_destroy.c", + "musl/src/process/posix_spawnattr_getflags.c", + "musl/src/process/posix_spawnattr_getpgroup.c", + "musl/src/process/posix_spawnattr_getsigdefault.c", + "musl/src/process/posix_spawnattr_getsigmask.c", + "musl/src/process/posix_spawnattr_init.c", + "musl/src/process/posix_spawnattr_sched.c", + "musl/src/process/posix_spawnattr_setflags.c", + "musl/src/process/posix_spawnattr_setpgroup.c", + "musl/src/process/posix_spawnattr_setsigdefault.c", + "musl/src/process/posix_spawnattr_setsigmask.c", + "musl/src/process/posix_spawnp.c", + "musl/src/process/s390x/vfork.s", + "musl/src/process/sh/vfork.s", + "musl/src/process/system.c", + "musl/src/process/vfork.c", + "musl/src/process/wait.c", + "musl/src/process/waitid.c", + "musl/src/process/waitpid.c", + "musl/src/process/x32/vfork.s", + "musl/src/process/x86_64/vfork.s", + "musl/src/regex/fnmatch.c", + "musl/src/regex/glob.c", + "musl/src/regex/regcomp.c", + "musl/src/regex/regerror.c", + "musl/src/regex/regexec.c", + "musl/src/regex/tre-mem.c", + "musl/src/sched/affinity.c", + "musl/src/sched/sched_cpucount.c", + "musl/src/sched/sched_get_priority_max.c", + "musl/src/sched/sched_getcpu.c", + "musl/src/sched/sched_getparam.c", + "musl/src/sched/sched_getscheduler.c", + "musl/src/sched/sched_rr_get_interval.c", + "musl/src/sched/sched_setparam.c", + "musl/src/sched/sched_setscheduler.c", + "musl/src/sched/sched_yield.c", + "musl/src/search/hsearch.c", + "musl/src/search/insque.c", + "musl/src/search/lsearch.c", + "musl/src/search/tdelete.c", + "musl/src/search/tdestroy.c", + "musl/src/search/tfind.c", + "musl/src/search/tsearch.c", + "musl/src/search/twalk.c", + "musl/src/select/poll.c", + "musl/src/select/pselect.c", + "musl/src/select/select.c", + "musl/src/setjmp/aarch64/longjmp.s", + "musl/src/setjmp/aarch64/setjmp.s", + "musl/src/setjmp/arm/longjmp.S", + "musl/src/setjmp/arm/setjmp.S", + "musl/src/setjmp/i386/longjmp.s", + "musl/src/setjmp/i386/setjmp.s", + "musl/src/setjmp/longjmp.c", + "musl/src/setjmp/m68k/longjmp.s", + "musl/src/setjmp/m68k/setjmp.s", + "musl/src/setjmp/microblaze/longjmp.s", + "musl/src/setjmp/microblaze/setjmp.s", + "musl/src/setjmp/mips/longjmp.S", + "musl/src/setjmp/mips/setjmp.S", + "musl/src/setjmp/mips64/longjmp.S", + "musl/src/setjmp/mips64/setjmp.S", + "musl/src/setjmp/mipsn32/longjmp.S", + "musl/src/setjmp/mipsn32/setjmp.S", + "musl/src/setjmp/or1k/longjmp.s", + "musl/src/setjmp/or1k/setjmp.s", + "musl/src/setjmp/powerpc/longjmp.S", + "musl/src/setjmp/powerpc/setjmp.S", + "musl/src/setjmp/powerpc64/longjmp.s", + "musl/src/setjmp/powerpc64/setjmp.s", + "musl/src/setjmp/riscv64/longjmp.S", + "musl/src/setjmp/riscv64/setjmp.S", + "musl/src/setjmp/s390x/longjmp.s", + "musl/src/setjmp/s390x/setjmp.s", + "musl/src/setjmp/setjmp.c", + "musl/src/setjmp/sh/longjmp.S", + "musl/src/setjmp/sh/setjmp.S", + "musl/src/setjmp/x32/longjmp.s", + "musl/src/setjmp/x32/setjmp.s", + "musl/src/setjmp/x86_64/longjmp.s", + "musl/src/setjmp/x86_64/setjmp.s", + "musl/src/signal/aarch64/restore.s", + "musl/src/signal/aarch64/sigsetjmp.s", + "musl/src/signal/arm/restore.s", + "musl/src/signal/arm/sigsetjmp.s", + "musl/src/signal/block.c", + "musl/src/signal/getitimer.c", + "musl/src/signal/i386/restore.s", + "musl/src/signal/i386/sigsetjmp.s", + "musl/src/signal/kill.c", + "musl/src/signal/killpg.c", + "musl/src/signal/m68k/sigsetjmp.s", + "musl/src/signal/microblaze/restore.s", + "musl/src/signal/microblaze/sigsetjmp.s", + "musl/src/signal/mips/restore.s", + "musl/src/signal/mips/sigsetjmp.s", + "musl/src/signal/mips64/restore.s", + "musl/src/signal/mips64/sigsetjmp.s", + "musl/src/signal/mipsn32/restore.s", + "musl/src/signal/mipsn32/sigsetjmp.s", + "musl/src/signal/or1k/sigsetjmp.s", + "musl/src/signal/powerpc/restore.s", + "musl/src/signal/powerpc/sigsetjmp.s", + "musl/src/signal/powerpc64/restore.s", + "musl/src/signal/powerpc64/sigsetjmp.s", + "musl/src/signal/psiginfo.c", + "musl/src/signal/psignal.c", + "musl/src/signal/raise.c", + "musl/src/signal/restore.c", + "musl/src/signal/riscv64/restore.s", + "musl/src/signal/riscv64/sigsetjmp.s", + "musl/src/signal/s390x/restore.s", + "musl/src/signal/s390x/sigsetjmp.s", + "musl/src/signal/setitimer.c", + "musl/src/signal/sh/restore.s", + "musl/src/signal/sh/sigsetjmp.s", + "musl/src/signal/sigaction.c", + "musl/src/signal/sigaddset.c", + "musl/src/signal/sigaltstack.c", + "musl/src/signal/sigandset.c", + "musl/src/signal/sigdelset.c", + "musl/src/signal/sigemptyset.c", + "musl/src/signal/sigfillset.c", + "musl/src/signal/sighold.c", + "musl/src/signal/sigignore.c", + "musl/src/signal/siginterrupt.c", + "musl/src/signal/sigisemptyset.c", + "musl/src/signal/sigismember.c", + "musl/src/signal/siglongjmp.c", + "musl/src/signal/signal.c", + "musl/src/signal/sigorset.c", + "musl/src/signal/sigpause.c", + "musl/src/signal/sigpending.c", + "musl/src/signal/sigprocmask.c", + "musl/src/signal/sigqueue.c", + "musl/src/signal/sigrelse.c", + "musl/src/signal/sigrtmax.c", + "musl/src/signal/sigrtmin.c", + "musl/src/signal/sigset.c", + "musl/src/signal/sigsetjmp.c", + "musl/src/signal/sigsetjmp_tail.c", + "musl/src/signal/sigsuspend.c", + "musl/src/signal/sigtimedwait.c", + "musl/src/signal/sigwait.c", + "musl/src/signal/sigwaitinfo.c", + "musl/src/signal/x32/getitimer.c", + "musl/src/signal/x32/restore.s", + "musl/src/signal/x32/setitimer.c", + "musl/src/signal/x32/sigsetjmp.s", + "musl/src/signal/x86_64/restore.s", + "musl/src/signal/x86_64/sigsetjmp.s", + "musl/src/stat/__xstat.c", + "musl/src/stat/chmod.c", + "musl/src/stat/fchmod.c", + "musl/src/stat/fchmodat.c", + "musl/src/stat/fstat.c", + "musl/src/stat/fstatat.c", + "musl/src/stat/futimens.c", + "musl/src/stat/futimesat.c", + "musl/src/stat/lchmod.c", + "musl/src/stat/lstat.c", + "musl/src/stat/mkdir.c", + "musl/src/stat/mkdirat.c", + "musl/src/stat/mkfifo.c", + "musl/src/stat/mkfifoat.c", + "musl/src/stat/mknod.c", + "musl/src/stat/mknodat.c", + "musl/src/stat/stat.c", + "musl/src/stat/statvfs.c", + "musl/src/stat/umask.c", + "musl/src/stat/utimensat.c", + "musl/src/stdio/__fclose_ca.c", + "musl/src/stdio/__fdopen.c", + "musl/src/stdio/__fmodeflags.c", + "musl/src/stdio/__fopen_rb_ca.c", + "musl/src/stdio/__lockfile.c", + "musl/src/stdio/__overflow.c", + "musl/src/stdio/__stdio_close.c", + "musl/src/stdio/__stdio_exit.c", + "musl/src/stdio/__stdio_read.c", + "musl/src/stdio/__stdio_seek.c", + "musl/src/stdio/__stdio_write.c", + "musl/src/stdio/__stdout_write.c", + "musl/src/stdio/__string_read.c", + "musl/src/stdio/__toread.c", + "musl/src/stdio/__towrite.c", + "musl/src/stdio/__uflow.c", + "musl/src/stdio/asprintf.c", + "musl/src/stdio/clearerr.c", + "musl/src/stdio/dprintf.c", + "musl/src/stdio/ext.c", + "musl/src/stdio/ext2.c", + "musl/src/stdio/fclose.c", + "musl/src/stdio/feof.c", + "musl/src/stdio/ferror.c", + "musl/src/stdio/fflush.c", + "musl/src/stdio/fgetc.c", + "musl/src/stdio/fgetln.c", + "musl/src/stdio/fgetpos.c", + "musl/src/stdio/fgets.c", + "musl/src/stdio/fgetwc.c", + "musl/src/stdio/fgetws.c", + "musl/src/stdio/fileno.c", + "musl/src/stdio/flockfile.c", + "musl/src/stdio/fmemopen.c", + "musl/src/stdio/fopen.c", + "musl/src/stdio/fopencookie.c", + "musl/src/stdio/fprintf.c", + "musl/src/stdio/fputc.c", + "musl/src/stdio/fputs.c", + "musl/src/stdio/fputwc.c", + "musl/src/stdio/fputws.c", + "musl/src/stdio/fread.c", + "musl/src/stdio/freopen.c", + "musl/src/stdio/fscanf.c", + "musl/src/stdio/fseek.c", + "musl/src/stdio/fsetpos.c", + "musl/src/stdio/ftell.c", + "musl/src/stdio/ftrylockfile.c", + "musl/src/stdio/funlockfile.c", + "musl/src/stdio/fwide.c", + "musl/src/stdio/fwprintf.c", + "musl/src/stdio/fwrite.c", + "musl/src/stdio/fwscanf.c", + "musl/src/stdio/getc.c", + "musl/src/stdio/getc_unlocked.c", + "musl/src/stdio/getchar.c", + "musl/src/stdio/getchar_unlocked.c", + "musl/src/stdio/getdelim.c", + "musl/src/stdio/getline.c", + "musl/src/stdio/gets.c", + "musl/src/stdio/getw.c", + "musl/src/stdio/getwc.c", + "musl/src/stdio/getwchar.c", + "musl/src/stdio/ofl.c", + "musl/src/stdio/ofl_add.c", + "musl/src/stdio/open_memstream.c", + "musl/src/stdio/open_wmemstream.c", + "musl/src/stdio/pclose.c", + "musl/src/stdio/perror.c", + "musl/src/stdio/popen.c", + "musl/src/stdio/printf.c", + "musl/src/stdio/putc.c", + "musl/src/stdio/putc_unlocked.c", + "musl/src/stdio/putchar.c", + "musl/src/stdio/putchar_unlocked.c", + "musl/src/stdio/puts.c", + "musl/src/stdio/putw.c", + "musl/src/stdio/putwc.c", + "musl/src/stdio/putwchar.c", + "musl/src/stdio/remove.c", + "musl/src/stdio/rename.c", + "musl/src/stdio/rewind.c", + "musl/src/stdio/scanf.c", + "musl/src/stdio/setbuf.c", + "musl/src/stdio/setbuffer.c", + "musl/src/stdio/setlinebuf.c", + "musl/src/stdio/setvbuf.c", + "musl/src/stdio/snprintf.c", + "musl/src/stdio/sprintf.c", + "musl/src/stdio/sscanf.c", + "musl/src/stdio/stderr.c", + "musl/src/stdio/stdin.c", + "musl/src/stdio/stdout.c", + "musl/src/stdio/swprintf.c", + "musl/src/stdio/swscanf.c", + "musl/src/stdio/tempnam.c", + "musl/src/stdio/tmpfile.c", + "musl/src/stdio/tmpnam.c", + "musl/src/stdio/ungetc.c", + "musl/src/stdio/ungetwc.c", + "musl/src/stdio/vasprintf.c", + "musl/src/stdio/vdprintf.c", + "musl/src/stdio/vfprintf.c", + "musl/src/stdio/vfscanf.c", + "musl/src/stdio/vfwprintf.c", + "musl/src/stdio/vfwscanf.c", + "musl/src/stdio/vprintf.c", + "musl/src/stdio/vscanf.c", + "musl/src/stdio/vsnprintf.c", + "musl/src/stdio/vsprintf.c", + "musl/src/stdio/vsscanf.c", + "musl/src/stdio/vswprintf.c", + "musl/src/stdio/vswscanf.c", + "musl/src/stdio/vwprintf.c", + "musl/src/stdio/vwscanf.c", + "musl/src/stdio/wprintf.c", + "musl/src/stdio/wscanf.c", + "musl/src/stdlib/abs.c", + "musl/src/stdlib/atof.c", + "musl/src/stdlib/atoi.c", + "musl/src/stdlib/atol.c", + "musl/src/stdlib/atoll.c", + "musl/src/stdlib/bsearch.c", + "musl/src/stdlib/div.c", + "musl/src/stdlib/ecvt.c", + "musl/src/stdlib/fcvt.c", + "musl/src/stdlib/gcvt.c", + "musl/src/stdlib/imaxabs.c", + "musl/src/stdlib/imaxdiv.c", + "musl/src/stdlib/labs.c", + "musl/src/stdlib/ldiv.c", + "musl/src/stdlib/llabs.c", + "musl/src/stdlib/lldiv.c", + "musl/src/stdlib/qsort.c", + "musl/src/stdlib/strtod.c", + "musl/src/stdlib/strtol.c", + "musl/src/stdlib/wcstod.c", + "musl/src/stdlib/wcstol.c", + "musl/src/string/arm/__aeabi_memcpy.s", + "musl/src/string/arm/__aeabi_memset.s", + "musl/src/string/arm/memcpy.c", + "musl/src/string/arm/memcpy_le.S", + "musl/src/string/bcmp.c", + "musl/src/string/bcopy.c", + "musl/src/string/bzero.c", + "musl/src/string/explicit_bzero.c", + "musl/src/string/i386/memcpy.s", + "musl/src/string/i386/memmove.s", + "musl/src/string/i386/memset.s", + "musl/src/string/index.c", + "musl/src/string/memccpy.c", + "musl/src/string/memchr.c", + "musl/src/string/memcmp.c", + "musl/src/string/memcpy.c", + "musl/src/string/memmem.c", + "musl/src/string/memmove.c", + "musl/src/string/mempcpy.c", + "musl/src/string/memrchr.c", + "musl/src/string/memset.c", + "musl/src/string/rindex.c", + "musl/src/string/stpcpy.c", + "musl/src/string/stpncpy.c", + "musl/src/string/strcasecmp.c", + "musl/src/string/strcasestr.c", + "musl/src/string/strcat.c", + "musl/src/string/strchr.c", + "musl/src/string/strchrnul.c", + "musl/src/string/strcmp.c", + "musl/src/string/strcpy.c", + "musl/src/string/strcspn.c", + "musl/src/string/strdup.c", + "musl/src/string/strerror_r.c", + "musl/src/string/strlcat.c", + "musl/src/string/strlcpy.c", + "musl/src/string/strlen.c", + "musl/src/string/strncasecmp.c", + "musl/src/string/strncat.c", + "musl/src/string/strncmp.c", + "musl/src/string/strncpy.c", + "musl/src/string/strndup.c", + "musl/src/string/strnlen.c", + "musl/src/string/strpbrk.c", + "musl/src/string/strrchr.c", + "musl/src/string/strsep.c", + "musl/src/string/strsignal.c", + "musl/src/string/strspn.c", + "musl/src/string/strstr.c", + "musl/src/string/strtok.c", + "musl/src/string/strtok_r.c", + "musl/src/string/strverscmp.c", + "musl/src/string/swab.c", + "musl/src/string/wcpcpy.c", + "musl/src/string/wcpncpy.c", + "musl/src/string/wcscasecmp.c", + "musl/src/string/wcscasecmp_l.c", + "musl/src/string/wcscat.c", + "musl/src/string/wcschr.c", + "musl/src/string/wcscmp.c", + "musl/src/string/wcscpy.c", + "musl/src/string/wcscspn.c", + "musl/src/string/wcsdup.c", + "musl/src/string/wcslen.c", + "musl/src/string/wcsncasecmp.c", + "musl/src/string/wcsncasecmp_l.c", + "musl/src/string/wcsncat.c", + "musl/src/string/wcsncmp.c", + "musl/src/string/wcsncpy.c", + "musl/src/string/wcsnlen.c", + "musl/src/string/wcspbrk.c", + "musl/src/string/wcsrchr.c", + "musl/src/string/wcsspn.c", + "musl/src/string/wcsstr.c", + "musl/src/string/wcstok.c", + "musl/src/string/wcswcs.c", + "musl/src/string/wmemchr.c", + "musl/src/string/wmemcmp.c", + "musl/src/string/wmemcpy.c", + "musl/src/string/wmemmove.c", + "musl/src/string/wmemset.c", + "musl/src/string/x86_64/memcpy.s", + "musl/src/string/x86_64/memmove.s", + "musl/src/string/x86_64/memset.s", + "musl/src/temp/__randname.c", + "musl/src/temp/mkdtemp.c", + "musl/src/temp/mkostemp.c", + "musl/src/temp/mkostemps.c", + "musl/src/temp/mkstemp.c", + "musl/src/temp/mkstemps.c", + "musl/src/temp/mktemp.c", + "musl/src/termios/cfgetospeed.c", + "musl/src/termios/cfmakeraw.c", + "musl/src/termios/cfsetospeed.c", + "musl/src/termios/tcdrain.c", + "musl/src/termios/tcflow.c", + "musl/src/termios/tcflush.c", + "musl/src/termios/tcgetattr.c", + "musl/src/termios/tcgetsid.c", + "musl/src/termios/tcsendbreak.c", + "musl/src/termios/tcsetattr.c", + "musl/src/thread/__lock.c", + "musl/src/thread/__set_thread_area.c", + "musl/src/thread/__syscall_cp.c", + "musl/src/thread/__timedwait.c", + "musl/src/thread/__tls_get_addr.c", + "musl/src/thread/__unmapself.c", + "musl/src/thread/__wait.c", + "musl/src/thread/aarch64/__set_thread_area.s", + "musl/src/thread/aarch64/__unmapself.s", + "musl/src/thread/aarch64/clone.s", + "musl/src/thread/aarch64/syscall_cp.s", + "musl/src/thread/arm/__aeabi_read_tp.s", + "musl/src/thread/arm/__set_thread_area.c", + "musl/src/thread/arm/__unmapself.s", + "musl/src/thread/arm/atomics.s", + "musl/src/thread/arm/clone.s", + "musl/src/thread/arm/syscall_cp.s", + "musl/src/thread/call_once.c", + "musl/src/thread/clone.c", + "musl/src/thread/cnd_broadcast.c", + "musl/src/thread/cnd_destroy.c", + "musl/src/thread/cnd_init.c", + "musl/src/thread/cnd_signal.c", + "musl/src/thread/cnd_timedwait.c", + "musl/src/thread/cnd_wait.c", + "musl/src/thread/default_attr.c", + "musl/src/thread/i386/__set_thread_area.s", + "musl/src/thread/i386/__unmapself.s", + "musl/src/thread/i386/clone.s", + "musl/src/thread/i386/syscall_cp.s", + "musl/src/thread/i386/tls.s", + "musl/src/thread/lock_ptc.c", + "musl/src/thread/m68k/__m68k_read_tp.s", + "musl/src/thread/m68k/clone.s", + "musl/src/thread/m68k/syscall_cp.s", + "musl/src/thread/microblaze/__set_thread_area.s", + "musl/src/thread/microblaze/__unmapself.s", + "musl/src/thread/microblaze/clone.s", + "musl/src/thread/microblaze/syscall_cp.s", + "musl/src/thread/mips/__unmapself.s", + "musl/src/thread/mips/clone.s", + "musl/src/thread/mips/syscall_cp.s", + "musl/src/thread/mips64/__unmapself.s", + "musl/src/thread/mips64/clone.s", + "musl/src/thread/mips64/syscall_cp.s", + "musl/src/thread/mipsn32/__unmapself.s", + "musl/src/thread/mipsn32/clone.s", + "musl/src/thread/mipsn32/syscall_cp.s", + "musl/src/thread/mtx_destroy.c", + "musl/src/thread/mtx_init.c", + "musl/src/thread/mtx_lock.c", + "musl/src/thread/mtx_timedlock.c", + "musl/src/thread/mtx_trylock.c", + "musl/src/thread/mtx_unlock.c", + "musl/src/thread/or1k/__set_thread_area.s", + "musl/src/thread/or1k/__unmapself.s", + "musl/src/thread/or1k/clone.s", + "musl/src/thread/or1k/syscall_cp.s", + "musl/src/thread/powerpc/__set_thread_area.s", + "musl/src/thread/powerpc/__unmapself.s", + "musl/src/thread/powerpc/clone.s", + "musl/src/thread/powerpc/syscall_cp.s", + "musl/src/thread/powerpc64/__set_thread_area.s", + "musl/src/thread/powerpc64/__unmapself.s", + "musl/src/thread/powerpc64/clone.s", + "musl/src/thread/powerpc64/syscall_cp.s", + "musl/src/thread/pthread_atfork.c", + "musl/src/thread/pthread_attr_destroy.c", + "musl/src/thread/pthread_attr_get.c", + "musl/src/thread/pthread_attr_init.c", + "musl/src/thread/pthread_attr_setdetachstate.c", + "musl/src/thread/pthread_attr_setguardsize.c", + "musl/src/thread/pthread_attr_setinheritsched.c", + "musl/src/thread/pthread_attr_setschedparam.c", + "musl/src/thread/pthread_attr_setschedpolicy.c", + "musl/src/thread/pthread_attr_setscope.c", + "musl/src/thread/pthread_attr_setstack.c", + "musl/src/thread/pthread_attr_setstacksize.c", + "musl/src/thread/pthread_barrier_destroy.c", + "musl/src/thread/pthread_barrier_init.c", + "musl/src/thread/pthread_barrier_wait.c", + "musl/src/thread/pthread_barrierattr_destroy.c", + "musl/src/thread/pthread_barrierattr_init.c", + "musl/src/thread/pthread_barrierattr_setpshared.c", + "musl/src/thread/pthread_cancel.c", + "musl/src/thread/pthread_cleanup_push.c", + "musl/src/thread/pthread_cond_broadcast.c", + "musl/src/thread/pthread_cond_destroy.c", + "musl/src/thread/pthread_cond_init.c", + "musl/src/thread/pthread_cond_signal.c", + "musl/src/thread/pthread_cond_timedwait.c", + "musl/src/thread/pthread_cond_wait.c", + "musl/src/thread/pthread_condattr_destroy.c", + "musl/src/thread/pthread_condattr_init.c", + "musl/src/thread/pthread_condattr_setclock.c", + "musl/src/thread/pthread_condattr_setpshared.c", + "musl/src/thread/pthread_create.c", + "musl/src/thread/pthread_detach.c", + "musl/src/thread/pthread_equal.c", + "musl/src/thread/pthread_getattr_np.c", + "musl/src/thread/pthread_getconcurrency.c", + "musl/src/thread/pthread_getcpuclockid.c", + "musl/src/thread/pthread_getschedparam.c", + "musl/src/thread/pthread_getspecific.c", + "musl/src/thread/pthread_join.c", + "musl/src/thread/pthread_key_create.c", + "musl/src/thread/pthread_kill.c", + "musl/src/thread/pthread_mutex_consistent.c", + "musl/src/thread/pthread_mutex_destroy.c", + "musl/src/thread/pthread_mutex_getprioceiling.c", + "musl/src/thread/pthread_mutex_init.c", + "musl/src/thread/pthread_mutex_lock.c", + "musl/src/thread/pthread_mutex_setprioceiling.c", + "musl/src/thread/pthread_mutex_timedlock.c", + "musl/src/thread/pthread_mutex_trylock.c", + "musl/src/thread/pthread_mutex_unlock.c", + "musl/src/thread/pthread_mutexattr_destroy.c", + "musl/src/thread/pthread_mutexattr_init.c", + "musl/src/thread/pthread_mutexattr_setprotocol.c", + "musl/src/thread/pthread_mutexattr_setpshared.c", + "musl/src/thread/pthread_mutexattr_setrobust.c", + "musl/src/thread/pthread_mutexattr_settype.c", + "musl/src/thread/pthread_once.c", + "musl/src/thread/pthread_rwlock_destroy.c", + "musl/src/thread/pthread_rwlock_init.c", + "musl/src/thread/pthread_rwlock_rdlock.c", + "musl/src/thread/pthread_rwlock_timedrdlock.c", + "musl/src/thread/pthread_rwlock_timedwrlock.c", + "musl/src/thread/pthread_rwlock_tryrdlock.c", + "musl/src/thread/pthread_rwlock_trywrlock.c", + "musl/src/thread/pthread_rwlock_unlock.c", + "musl/src/thread/pthread_rwlock_wrlock.c", + "musl/src/thread/pthread_rwlockattr_destroy.c", + "musl/src/thread/pthread_rwlockattr_init.c", + "musl/src/thread/pthread_rwlockattr_setpshared.c", + "musl/src/thread/pthread_self.c", + "musl/src/thread/pthread_setattr_default_np.c", + "musl/src/thread/pthread_setcancelstate.c", + "musl/src/thread/pthread_setcanceltype.c", + "musl/src/thread/pthread_setconcurrency.c", + "musl/src/thread/pthread_setname_np.c", + "musl/src/thread/pthread_setschedparam.c", + "musl/src/thread/pthread_setschedprio.c", + "musl/src/thread/pthread_setspecific.c", + "musl/src/thread/pthread_sigmask.c", + "musl/src/thread/pthread_spin_destroy.c", + "musl/src/thread/pthread_spin_init.c", + "musl/src/thread/pthread_spin_lock.c", + "musl/src/thread/pthread_spin_trylock.c", + "musl/src/thread/pthread_spin_unlock.c", + "musl/src/thread/pthread_testcancel.c", + "musl/src/thread/riscv64/__set_thread_area.s", + "musl/src/thread/riscv64/__unmapself.s", + "musl/src/thread/riscv64/clone.s", + "musl/src/thread/riscv64/syscall_cp.s", + "musl/src/thread/s390x/__set_thread_area.s", + "musl/src/thread/s390x/__tls_get_offset.s", + "musl/src/thread/s390x/__unmapself.s", + "musl/src/thread/s390x/clone.s", + "musl/src/thread/s390x/syscall_cp.s", + "musl/src/thread/sem_destroy.c", + "musl/src/thread/sem_getvalue.c", + "musl/src/thread/sem_init.c", + "musl/src/thread/sem_open.c", + "musl/src/thread/sem_post.c", + "musl/src/thread/sem_timedwait.c", + "musl/src/thread/sem_trywait.c", + "musl/src/thread/sem_unlink.c", + "musl/src/thread/sem_wait.c", + "musl/src/thread/sh/__set_thread_area.c", + "musl/src/thread/sh/__unmapself.c", + "musl/src/thread/sh/__unmapself_mmu.s", + "musl/src/thread/sh/atomics.s", + "musl/src/thread/sh/clone.s", + "musl/src/thread/sh/syscall_cp.s", + "musl/src/thread/synccall.c", + "musl/src/thread/syscall_cp.c", + "musl/src/thread/thrd_create.c", + "musl/src/thread/thrd_exit.c", + "musl/src/thread/thrd_join.c", + "musl/src/thread/thrd_sleep.c", + "musl/src/thread/thrd_yield.c", + "musl/src/thread/tls.c", + "musl/src/thread/tss_create.c", + "musl/src/thread/tss_delete.c", + "musl/src/thread/tss_set.c", + "musl/src/thread/vmlock.c", + "musl/src/thread/x32/__set_thread_area.s", + "musl/src/thread/x32/__unmapself.s", + "musl/src/thread/x32/clone.s", + "musl/src/thread/x32/syscall_cp.s", + "musl/src/thread/x86_64/__set_thread_area.s", + "musl/src/thread/x86_64/__unmapself.s", + "musl/src/thread/x86_64/clone.s", + "musl/src/thread/x86_64/syscall_cp.s", + "musl/src/time/__map_file.c", + "musl/src/time/__month_to_secs.c", + "musl/src/time/__secs_to_tm.c", + "musl/src/time/__tm_to_secs.c", + "musl/src/time/__tz.c", + "musl/src/time/__year_to_secs.c", + "musl/src/time/asctime.c", + "musl/src/time/asctime_r.c", + "musl/src/time/clock.c", + "musl/src/time/clock_getcpuclockid.c", + "musl/src/time/clock_getres.c", + "musl/src/time/clock_gettime.c", + "musl/src/time/clock_nanosleep.c", + "musl/src/time/clock_settime.c", + "musl/src/time/ctime.c", + "musl/src/time/ctime_r.c", + "musl/src/time/difftime.c", + "musl/src/time/ftime.c", + "musl/src/time/getdate.c", + "musl/src/time/gettimeofday.c", + "musl/src/time/gmtime.c", + "musl/src/time/gmtime_r.c", + "musl/src/time/localtime.c", + "musl/src/time/localtime_r.c", + "musl/src/time/mktime.c", + "musl/src/time/nanosleep.c", + "musl/src/time/strftime.c", + "musl/src/time/strptime.c", + "musl/src/time/time.c", + "musl/src/time/timegm.c", + "musl/src/time/timer_create.c", + "musl/src/time/timer_delete.c", + "musl/src/time/timer_getoverrun.c", + "musl/src/time/timer_gettime.c", + "musl/src/time/timer_settime.c", + "musl/src/time/times.c", + "musl/src/time/timespec_get.c", + "musl/src/time/utime.c", + "musl/src/time/wcsftime.c", + "musl/src/unistd/_exit.c", + "musl/src/unistd/access.c", + "musl/src/unistd/acct.c", + "musl/src/unistd/alarm.c", + "musl/src/unistd/chdir.c", + "musl/src/unistd/chown.c", + "musl/src/unistd/close.c", + "musl/src/unistd/ctermid.c", + "musl/src/unistd/dup.c", + "musl/src/unistd/dup2.c", + "musl/src/unistd/dup3.c", + "musl/src/unistd/faccessat.c", + "musl/src/unistd/fchdir.c", + "musl/src/unistd/fchown.c", + "musl/src/unistd/fchownat.c", + "musl/src/unistd/fdatasync.c", + "musl/src/unistd/fsync.c", + "musl/src/unistd/ftruncate.c", + "musl/src/unistd/getcwd.c", + "musl/src/unistd/getegid.c", + "musl/src/unistd/geteuid.c", + "musl/src/unistd/getgid.c", + "musl/src/unistd/getgroups.c", + "musl/src/unistd/gethostname.c", + "musl/src/unistd/getlogin.c", + "musl/src/unistd/getlogin_r.c", + "musl/src/unistd/getpgid.c", + "musl/src/unistd/getpgrp.c", + "musl/src/unistd/getpid.c", + "musl/src/unistd/getppid.c", + "musl/src/unistd/getsid.c", + "musl/src/unistd/getuid.c", + "musl/src/unistd/isatty.c", + "musl/src/unistd/lchown.c", + "musl/src/unistd/link.c", + "musl/src/unistd/linkat.c", + "musl/src/unistd/lseek.c", + "musl/src/unistd/mips/pipe.s", + "musl/src/unistd/mips64/pipe.s", + "musl/src/unistd/mipsn32/lseek.c", + "musl/src/unistd/mipsn32/pipe.s", + "musl/src/unistd/nice.c", + "musl/src/unistd/pause.c", + "musl/src/unistd/pipe.c", + "musl/src/unistd/pipe2.c", + "musl/src/unistd/posix_close.c", + "musl/src/unistd/pread.c", + "musl/src/unistd/preadv.c", + "musl/src/unistd/pwrite.c", + "musl/src/unistd/pwritev.c", + "musl/src/unistd/read.c", + "musl/src/unistd/readlink.c", + "musl/src/unistd/readlinkat.c", + "musl/src/unistd/readv.c", + "musl/src/unistd/renameat.c", + "musl/src/unistd/rmdir.c", + "musl/src/unistd/setegid.c", + "musl/src/unistd/seteuid.c", + "musl/src/unistd/setgid.c", + "musl/src/unistd/setpgid.c", + "musl/src/unistd/setpgrp.c", + "musl/src/unistd/setregid.c", + "musl/src/unistd/setresgid.c", + "musl/src/unistd/setresuid.c", + "musl/src/unistd/setreuid.c", + "musl/src/unistd/setsid.c", + "musl/src/unistd/setuid.c", + "musl/src/unistd/setxid.c", + "musl/src/unistd/sh/pipe.s", + "musl/src/unistd/sleep.c", + "musl/src/unistd/symlink.c", + "musl/src/unistd/symlinkat.c", + "musl/src/unistd/sync.c", + "musl/src/unistd/tcgetpgrp.c", + "musl/src/unistd/tcsetpgrp.c", + "musl/src/unistd/truncate.c", + "musl/src/unistd/ttyname.c", + "musl/src/unistd/ttyname_r.c", + "musl/src/unistd/ualarm.c", + "musl/src/unistd/unlink.c", + "musl/src/unistd/unlinkat.c", + "musl/src/unistd/usleep.c", + "musl/src/unistd/write.c", + "musl/src/unistd/writev.c", + "musl/src/unistd/x32/lseek.c", +}; +pub const compat_time32_files = [_][]const u8{ + "musl/compat/time32/__xstat.c", + "musl/compat/time32/adjtime32.c", + "musl/compat/time32/adjtimex_time32.c", + "musl/compat/time32/aio_suspend_time32.c", + "musl/compat/time32/clock_adjtime32.c", + "musl/compat/time32/clock_getres_time32.c", + "musl/compat/time32/clock_gettime32.c", + "musl/compat/time32/clock_nanosleep_time32.c", + "musl/compat/time32/clock_settime32.c", + "musl/compat/time32/cnd_timedwait_time32.c", + "musl/compat/time32/ctime32.c", + "musl/compat/time32/ctime32_r.c", + "musl/compat/time32/difftime32.c", + "musl/compat/time32/fstat_time32.c", + "musl/compat/time32/fstatat_time32.c", + "musl/compat/time32/ftime32.c", + "musl/compat/time32/futimens_time32.c", + "musl/compat/time32/futimes_time32.c", + "musl/compat/time32/futimesat_time32.c", + "musl/compat/time32/getitimer_time32.c", + "musl/compat/time32/getrusage_time32.c", + "musl/compat/time32/gettimeofday_time32.c", + "musl/compat/time32/gmtime32.c", + "musl/compat/time32/gmtime32_r.c", + "musl/compat/time32/localtime32.c", + "musl/compat/time32/localtime32_r.c", + "musl/compat/time32/lstat_time32.c", + "musl/compat/time32/lutimes_time32.c", + "musl/compat/time32/mktime32.c", + "musl/compat/time32/mq_timedreceive_time32.c", + "musl/compat/time32/mq_timedsend_time32.c", + "musl/compat/time32/mtx_timedlock_time32.c", + "musl/compat/time32/nanosleep_time32.c", + "musl/compat/time32/ppoll_time32.c", + "musl/compat/time32/pselect_time32.c", + "musl/compat/time32/pthread_cond_timedwait_time32.c", + "musl/compat/time32/pthread_mutex_timedlock_time32.c", + "musl/compat/time32/pthread_rwlock_timedrdlock_time32.c", + "musl/compat/time32/pthread_rwlock_timedwrlock_time32.c", + "musl/compat/time32/pthread_timedjoin_np_time32.c", + "musl/compat/time32/recvmmsg_time32.c", + "musl/compat/time32/sched_rr_get_interval_time32.c", + "musl/compat/time32/select_time32.c", + "musl/compat/time32/sem_timedwait_time32.c", + "musl/compat/time32/semtimedop_time32.c", + "musl/compat/time32/setitimer_time32.c", + "musl/compat/time32/settimeofday_time32.c", + "musl/compat/time32/sigtimedwait_time32.c", + "musl/compat/time32/stat_time32.c", + "musl/compat/time32/stime32.c", + "musl/compat/time32/thrd_sleep_time32.c", + "musl/compat/time32/time32.c", + "musl/compat/time32/time32gm.c", + "musl/compat/time32/timer_gettime32.c", + "musl/compat/time32/timer_settime32.c", + "musl/compat/time32/timerfd_gettime32.c", + "musl/compat/time32/timerfd_settime32.c", + "musl/compat/time32/timespec_get_time32.c", + "musl/compat/time32/utime_time32.c", + "musl/compat/time32/utimensat_time32.c", + "musl/compat/time32/utimes_time32.c", + "musl/compat/time32/wait3_time32.c", + "musl/compat/time32/wait4_time32.c", +}; diff --git a/src-self-hosted/stage1.zig b/src-self-hosted/stage1.zig new file mode 100644 index 0000000000..c2ce99db22 --- /dev/null +++ b/src-self-hosted/stage1.zig @@ -0,0 +1,495 @@ +//! This is the main entry point for the Zig/C++ hybrid compiler (stage1). +//! It has the functions exported from Zig, called in C++, and bindings for +//! the functions exported from C++, called from Zig. + +const std = @import("std"); +const assert = std.debug.assert; +const mem = std.mem; +const build_options = @import("build_options"); +const stage2 = @import("main.zig"); +const fatal = stage2.fatal; +const CrossTarget = std.zig.CrossTarget; +const Target = std.Target; + +comptime { + assert(std.builtin.link_libc); + assert(build_options.is_stage1); + _ = @import("compiler_rt"); +} + +pub const log = stage2.log; +pub const log_level = stage2.log_level; + +pub export fn main(argc: c_int, argv: [*]const [*:0]const u8) c_int { + std.debug.maybeEnableSegfaultHandler(); + + const gpa = std.heap.c_allocator; + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = &arena_instance.allocator; + + const args = arena.alloc([]const u8, @intCast(usize, argc)) catch fatal("out of memory", .{}); + for (args) |*arg, i| { + arg.* = mem.spanZ(argv[i]); + } + stage2.mainArgs(gpa, arena, args) catch |err| fatal("{}", .{err}); + return 0; +} + +// ABI warning +export fn stage2_panic(ptr: [*]const u8, len: usize) void { + @panic(ptr[0..len]); +} + +// ABI warning +const Error = extern enum { + None, + OutOfMemory, + InvalidFormat, + SemanticAnalyzeFail, + AccessDenied, + Interrupted, + SystemResources, + FileNotFound, + FileSystem, + FileTooBig, + DivByZero, + Overflow, + PathAlreadyExists, + Unexpected, + ExactDivRemainder, + NegativeDenominator, + ShiftedOutOneBits, + CCompileErrors, + EndOfFile, + IsDir, + NotDir, + UnsupportedOperatingSystem, + SharingViolation, + PipeBusy, + PrimitiveTypeNotFound, + CacheUnavailable, + PathTooLong, + CCompilerCannotFindFile, + NoCCompilerInstalled, + ReadingDepFile, + InvalidDepFile, + MissingArchitecture, + MissingOperatingSystem, + UnknownArchitecture, + UnknownOperatingSystem, + UnknownABI, + InvalidFilename, + DiskQuota, + DiskSpace, + UnexpectedWriteFailure, + UnexpectedSeekFailure, + UnexpectedFileTruncationFailure, + Unimplemented, + OperationAborted, + BrokenPipe, + NoSpaceLeft, + NotLazy, + IsAsync, + ImportOutsidePkgPath, + UnknownCpuModel, + UnknownCpuFeature, + InvalidCpuFeatures, + InvalidLlvmCpuFeaturesFormat, + UnknownApplicationBinaryInterface, + ASTUnitFailure, + BadPathName, + SymLinkLoop, + ProcessFdQuotaExceeded, + SystemFdQuotaExceeded, + NoDevice, + DeviceBusy, + UnableToSpawnCCompiler, + CCompilerExitCode, + CCompilerCrashed, + CCompilerCannotFindHeaders, + LibCRuntimeNotFound, + LibCStdLibHeaderNotFound, + LibCKernel32LibNotFound, + UnsupportedArchitecture, + WindowsSdkNotFound, + UnknownDynamicLinkerPath, + TargetHasNoDynamicLinker, + InvalidAbiVersion, + InvalidOperatingSystemVersion, + UnknownClangOption, + NestedResponseFile, + ZigIsTheCCompiler, + FileBusy, + Locked, +}; + +// ABI warning +export fn stage2_attach_segfault_handler() void { + if (std.debug.runtime_safety and std.debug.have_segfault_handling_support) { + std.debug.attachSegfaultHandler(); + } +} + +// ABI warning +export fn stage2_progress_create() *std.Progress { + const ptr = std.heap.c_allocator.create(std.Progress) catch @panic("out of memory"); + ptr.* = std.Progress{}; + return ptr; +} + +// ABI warning +export fn stage2_progress_destroy(progress: *std.Progress) void { + std.heap.c_allocator.destroy(progress); +} + +// ABI warning +export fn stage2_progress_start_root( + progress: *std.Progress, + name_ptr: [*]const u8, + name_len: usize, + estimated_total_items: usize, +) *std.Progress.Node { + return progress.start( + name_ptr[0..name_len], + if (estimated_total_items == 0) null else estimated_total_items, + ) catch @panic("timer unsupported"); +} + +// ABI warning +export fn stage2_progress_disable_tty(progress: *std.Progress) void { + progress.terminal = null; +} + +// ABI warning +export fn stage2_progress_start( + node: *std.Progress.Node, + name_ptr: [*]const u8, + name_len: usize, + estimated_total_items: usize, +) *std.Progress.Node { + const child_node = std.heap.c_allocator.create(std.Progress.Node) catch @panic("out of memory"); + child_node.* = node.start( + name_ptr[0..name_len], + if (estimated_total_items == 0) null else estimated_total_items, + ); + child_node.activate(); + return child_node; +} + +// ABI warning +export fn stage2_progress_end(node: *std.Progress.Node) void { + node.end(); + if (&node.context.root != node) { + std.heap.c_allocator.destroy(node); + } +} + +// ABI warning +export fn stage2_progress_complete_one(node: *std.Progress.Node) void { + node.completeOne(); +} + +// ABI warning +export fn stage2_progress_update_node(node: *std.Progress.Node, done_count: usize, total_count: usize) void { + node.completed_items = done_count; + node.estimated_total_items = total_count; + node.activate(); + node.context.maybeRefresh(); +} + +// ABI warning +const Stage2Target = extern struct { + arch: c_int, + vendor: c_int, + + abi: c_int, + os: c_int, + + is_native_os: bool, + is_native_cpu: bool, + + llvm_cpu_name: ?[*:0]const u8, + llvm_cpu_features: ?[*:0]const u8, + cpu_builtin_str: ?[*:0]const u8, + os_builtin_str: ?[*:0]const u8, + + dynamic_linker: ?[*:0]const u8, + + llvm_cpu_features_asm_ptr: [*]const [*:0]const u8, + llvm_cpu_features_asm_len: usize, + + fn fromTarget(self: *Stage2Target, cross_target: CrossTarget) !void { + const allocator = std.heap.c_allocator; + + var dynamic_linker: ?[*:0]u8 = null; + const target = try crossTargetToTarget(cross_target, &dynamic_linker); + + const generic_arch_name = target.cpu.arch.genericName(); + var cpu_builtin_str_buffer = try std.ArrayListSentineled(u8, 0).allocPrint(allocator, + \\Cpu{{ + \\ .arch = .{}, + \\ .model = &Target.{}.cpu.{}, + \\ .features = Target.{}.featureSet(&[_]Target.{}.Feature{{ + \\ + , .{ + @tagName(target.cpu.arch), + generic_arch_name, + target.cpu.model.name, + generic_arch_name, + generic_arch_name, + }); + defer cpu_builtin_str_buffer.deinit(); + + var llvm_features_buffer = try std.ArrayListSentineled(u8, 0).initSize(allocator, 0); + defer llvm_features_buffer.deinit(); + + // Unfortunately we have to do the work twice, because Clang does not support + // the same command line parameters for CPU features when assembling code as it does + // when compiling C code. + var asm_features_list = std.ArrayList([*:0]const u8).init(allocator); + defer asm_features_list.deinit(); + + for (target.cpu.arch.allFeaturesList()) |feature, index_usize| { + const index = @intCast(Target.Cpu.Feature.Set.Index, index_usize); + const is_enabled = target.cpu.features.isEnabled(index); + + if (feature.llvm_name) |llvm_name| { + const plus_or_minus = "-+"[@boolToInt(is_enabled)]; + try llvm_features_buffer.append(plus_or_minus); + try llvm_features_buffer.appendSlice(llvm_name); + try llvm_features_buffer.appendSlice(","); + } + + if (is_enabled) { + // TODO some kind of "zig identifier escape" function rather than + // unconditionally using @"" syntax + try cpu_builtin_str_buffer.appendSlice(" .@\""); + try cpu_builtin_str_buffer.appendSlice(feature.name); + try cpu_builtin_str_buffer.appendSlice("\",\n"); + } + } + + switch (target.cpu.arch) { + .riscv32, .riscv64 => { + if (Target.riscv.featureSetHas(target.cpu.features, .relax)) { + try asm_features_list.append("-mrelax"); + } else { + try asm_features_list.append("-mno-relax"); + } + }, + else => { + // TODO + // Argh, why doesn't the assembler accept the list of CPU features?! + // I don't see a way to do this other than hard coding everything. + }, + } + + try cpu_builtin_str_buffer.appendSlice( + \\ }), + \\}; + \\ + ); + + assert(mem.endsWith(u8, llvm_features_buffer.span(), ",")); + llvm_features_buffer.shrink(llvm_features_buffer.len() - 1); + + var os_builtin_str_buffer = try std.ArrayListSentineled(u8, 0).allocPrint(allocator, + \\Os{{ + \\ .tag = .{}, + \\ .version_range = .{{ + , .{@tagName(target.os.tag)}); + defer os_builtin_str_buffer.deinit(); + + // We'll re-use the OS version range builtin string for the cache hash. + const os_builtin_str_ver_start_index = os_builtin_str_buffer.len(); + + @setEvalBranchQuota(2000); + switch (target.os.tag) { + .freestanding, + .ananas, + .cloudabi, + .dragonfly, + .fuchsia, + .ios, + .kfreebsd, + .lv2, + .solaris, + .haiku, + .minix, + .rtems, + .nacl, + .cnk, + .aix, + .cuda, + .nvcl, + .amdhsa, + .ps4, + .elfiamcu, + .tvos, + .watchos, + .mesa3d, + .contiki, + .amdpal, + .hermit, + .hurd, + .wasi, + .emscripten, + .uefi, + .other, + => try os_builtin_str_buffer.appendSlice(" .none = {} }\n"), + + .freebsd, + .macosx, + .netbsd, + .openbsd, + => try os_builtin_str_buffer.outStream().print( + \\ .semver = .{{ + \\ .min = .{{ + \\ .major = {}, + \\ .minor = {}, + \\ .patch = {}, + \\ }}, + \\ .max = .{{ + \\ .major = {}, + \\ .minor = {}, + \\ .patch = {}, + \\ }}, + \\ }}}}, + \\ + , .{ + target.os.version_range.semver.min.major, + target.os.version_range.semver.min.minor, + target.os.version_range.semver.min.patch, + + target.os.version_range.semver.max.major, + target.os.version_range.semver.max.minor, + target.os.version_range.semver.max.patch, + }), + + .linux => try os_builtin_str_buffer.outStream().print( + \\ .linux = .{{ + \\ .range = .{{ + \\ .min = .{{ + \\ .major = {}, + \\ .minor = {}, + \\ .patch = {}, + \\ }}, + \\ .max = .{{ + \\ .major = {}, + \\ .minor = {}, + \\ .patch = {}, + \\ }}, + \\ }}, + \\ .glibc = .{{ + \\ .major = {}, + \\ .minor = {}, + \\ .patch = {}, + \\ }}, + \\ }}}}, + \\ + , .{ + target.os.version_range.linux.range.min.major, + target.os.version_range.linux.range.min.minor, + target.os.version_range.linux.range.min.patch, + + target.os.version_range.linux.range.max.major, + target.os.version_range.linux.range.max.minor, + target.os.version_range.linux.range.max.patch, + + target.os.version_range.linux.glibc.major, + target.os.version_range.linux.glibc.minor, + target.os.version_range.linux.glibc.patch, + }), + + .windows => try os_builtin_str_buffer.outStream().print( + \\ .windows = .{{ + \\ .min = {s}, + \\ .max = {s}, + \\ }}}}, + \\ + , .{ + target.os.version_range.windows.min, + target.os.version_range.windows.max, + }), + } + try os_builtin_str_buffer.appendSlice("};\n"); + + const glibc_or_darwin_version = blk: { + if (target.isGnuLibC()) { + const stage1_glibc = try std.heap.c_allocator.create(Stage2SemVer); + const stage2_glibc = target.os.version_range.linux.glibc; + stage1_glibc.* = .{ + .major = stage2_glibc.major, + .minor = stage2_glibc.minor, + .patch = stage2_glibc.patch, + }; + break :blk stage1_glibc; + } else if (target.isDarwin()) { + const stage1_semver = try std.heap.c_allocator.create(Stage2SemVer); + const stage2_semver = target.os.version_range.semver.min; + stage1_semver.* = .{ + .major = stage2_semver.major, + .minor = stage2_semver.minor, + .patch = stage2_semver.patch, + }; + break :blk stage1_semver; + } else { + break :blk null; + } + }; + + const std_dl = target.standardDynamicLinkerPath(); + const std_dl_z = if (std_dl.get()) |dl| + (try mem.dupeZ(std.heap.c_allocator, u8, dl)).ptr + else + null; + + const asm_features = asm_features_list.toOwnedSlice(); + self.* = .{ + .arch = @enumToInt(target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch + .vendor = 0, + .os = @enumToInt(target.os.tag), + .abi = @enumToInt(target.abi), + .llvm_cpu_name = if (target.cpu.model.llvm_name) |s| s.ptr else null, + .llvm_cpu_features = llvm_features_buffer.toOwnedSlice().ptr, + .llvm_cpu_features_asm_ptr = asm_features.ptr, + .llvm_cpu_features_asm_len = asm_features.len, + .cpu_builtin_str = cpu_builtin_str_buffer.toOwnedSlice().ptr, + .os_builtin_str = os_builtin_str_buffer.toOwnedSlice().ptr, + .is_native_os = cross_target.isNativeOs(), + .is_native_cpu = cross_target.isNativeCpu(), + .glibc_or_darwin_version = glibc_or_darwin_version, + .dynamic_linker = dynamic_linker, + .standard_dynamic_linker_path = std_dl_z, + }; + } +}; + +fn crossTargetToTarget(cross_target: CrossTarget, dynamic_linker_ptr: *?[*:0]u8) !Target { + var info = try std.zig.system.NativeTargetInfo.detect(std.heap.c_allocator, cross_target); + if (info.cpu_detection_unimplemented) { + // TODO We want to just use detected_info.target but implementing + // CPU model & feature detection is todo so here we rely on LLVM. + const llvm = @import("llvm.zig"); + const llvm_cpu_name = llvm.GetHostCPUName(); + const llvm_cpu_features = llvm.GetNativeFeatures(); + const arch = Target.current.cpu.arch; + info.target.cpu = try detectNativeCpuWithLLVM(arch, llvm_cpu_name, llvm_cpu_features); + cross_target.updateCpuFeatures(&info.target.cpu.features); + info.target.cpu.arch = cross_target.getCpuArch(); + } + if (info.dynamic_linker.get()) |dl| { + dynamic_linker_ptr.* = try mem.dupeZ(std.heap.c_allocator, u8, dl); + } else { + dynamic_linker_ptr.* = null; + } + return info.target; +} + +// ABI warning +const Stage2SemVer = extern struct { + major: u32, + minor: u32, + patch: u32, +}; diff --git a/src-self-hosted/stage2.zig b/src-self-hosted/stage2.zig deleted file mode 100644 index bfa37b603e..0000000000 --- a/src-self-hosted/stage2.zig +++ /dev/null @@ -1,1054 +0,0 @@ -// This is Zig code that is used by both stage1 and stage2. -// The prototypes in src/userland.h must match these definitions. - -const std = @import("std"); -const io = std.io; -const mem = std.mem; -const fs = std.fs; -const process = std.process; -const Allocator = mem.Allocator; -const ArrayList = std.ArrayList; -const ArrayListSentineled = std.ArrayListSentineled; -const Target = std.Target; -const CrossTarget = std.zig.CrossTarget; -const self_hosted_main = @import("main.zig"); -const DepTokenizer = @import("DepTokenizer.zig"); -const assert = std.debug.assert; -const LibCInstallation = @import("libc_installation.zig").LibCInstallation; - -var stderr_file: fs.File = undefined; -var stderr: fs.File.OutStream = undefined; -var stdout: fs.File.OutStream = undefined; - -comptime { - _ = @import("DepTokenizer.zig"); -} - -// ABI warning -export fn stage2_zen(ptr: *[*]const u8, len: *usize) void { - const info_zen = @import("main.zig").info_zen; - ptr.* = info_zen; - len.* = info_zen.len; -} - -// ABI warning -export fn stage2_panic(ptr: [*]const u8, len: usize) void { - @panic(ptr[0..len]); -} - -// ABI warning -const Error = extern enum { - None, - OutOfMemory, - InvalidFormat, - SemanticAnalyzeFail, - AccessDenied, - Interrupted, - SystemResources, - FileNotFound, - FileSystem, - FileTooBig, - DivByZero, - Overflow, - PathAlreadyExists, - Unexpected, - ExactDivRemainder, - NegativeDenominator, - ShiftedOutOneBits, - CCompileErrors, - EndOfFile, - IsDir, - NotDir, - UnsupportedOperatingSystem, - SharingViolation, - PipeBusy, - PrimitiveTypeNotFound, - CacheUnavailable, - PathTooLong, - CCompilerCannotFindFile, - NoCCompilerInstalled, - ReadingDepFile, - InvalidDepFile, - MissingArchitecture, - MissingOperatingSystem, - UnknownArchitecture, - UnknownOperatingSystem, - UnknownABI, - InvalidFilename, - DiskQuota, - DiskSpace, - UnexpectedWriteFailure, - UnexpectedSeekFailure, - UnexpectedFileTruncationFailure, - Unimplemented, - OperationAborted, - BrokenPipe, - NoSpaceLeft, - NotLazy, - IsAsync, - ImportOutsidePkgPath, - UnknownCpuModel, - UnknownCpuFeature, - InvalidCpuFeatures, - InvalidLlvmCpuFeaturesFormat, - UnknownApplicationBinaryInterface, - ASTUnitFailure, - BadPathName, - SymLinkLoop, - ProcessFdQuotaExceeded, - SystemFdQuotaExceeded, - NoDevice, - DeviceBusy, - UnableToSpawnCCompiler, - CCompilerExitCode, - CCompilerCrashed, - CCompilerCannotFindHeaders, - LibCRuntimeNotFound, - LibCStdLibHeaderNotFound, - LibCKernel32LibNotFound, - UnsupportedArchitecture, - WindowsSdkNotFound, - UnknownDynamicLinkerPath, - TargetHasNoDynamicLinker, - InvalidAbiVersion, - InvalidOperatingSystemVersion, - UnknownClangOption, - NestedResponseFile, - ZigIsTheCCompiler, - FileBusy, - Locked, -}; - -const FILE = std.c.FILE; -const ast = std.zig.ast; -const translate_c = @import("translate_c.zig"); - -/// Args should have a null terminating last arg. -export fn stage2_translate_c( - out_ast: **ast.Tree, - out_errors_ptr: *[*]translate_c.ClangErrMsg, - out_errors_len: *usize, - args_begin: [*]?[*]const u8, - args_end: [*]?[*]const u8, - resources_path: [*:0]const u8, -) Error { - var errors: []translate_c.ClangErrMsg = &[0]translate_c.ClangErrMsg{}; - out_ast.* = translate_c.translate(std.heap.c_allocator, args_begin, args_end, &errors, resources_path) catch |err| switch (err) { - error.SemanticAnalyzeFail => { - out_errors_ptr.* = errors.ptr; - out_errors_len.* = errors.len; - return .CCompileErrors; - }, - error.ASTUnitFailure => return .ASTUnitFailure, - error.OutOfMemory => return .OutOfMemory, - }; - return .None; -} - -export fn stage2_free_clang_errors(errors_ptr: [*]translate_c.ClangErrMsg, errors_len: usize) void { - translate_c.freeErrors(errors_ptr[0..errors_len]); -} - -export fn stage2_render_ast(tree: *ast.Tree, output_file: *FILE) Error { - const c_out_stream = std.io.cOutStream(output_file); - _ = std.zig.render(std.heap.c_allocator, c_out_stream, tree) catch |e| switch (e) { - error.WouldBlock => unreachable, // stage1 opens stuff in exclusively blocking mode - error.NotOpenForWriting => unreachable, - error.SystemResources => return .SystemResources, - error.OperationAborted => return .OperationAborted, - error.BrokenPipe => return .BrokenPipe, - error.DiskQuota => return .DiskQuota, - error.FileTooBig => return .FileTooBig, - error.NoSpaceLeft => return .NoSpaceLeft, - error.AccessDenied => return .AccessDenied, - error.OutOfMemory => return .OutOfMemory, - error.Unexpected => return .Unexpected, - error.InputOutput => return .FileSystem, - }; - return .None; -} - -export fn stage2_fmt(argc: c_int, argv: [*]const [*:0]const u8) c_int { - if (std.debug.runtime_safety) { - fmtMain(argc, argv) catch unreachable; - } else { - fmtMain(argc, argv) catch |e| { - std.debug.warn("{}\n", .{@errorName(e)}); - return -1; - }; - } - return 0; -} - -fn argvToArrayList(allocator: *Allocator, argc: c_int, argv: [*]const [*:0]const u8) !ArrayList([]const u8) { - var args_list = std.ArrayList([]const u8).init(allocator); - const argc_usize = @intCast(usize, argc); - var arg_i: usize = 0; - while (arg_i < argc_usize) : (arg_i += 1) { - try args_list.append(mem.spanZ(argv[arg_i])); - } - - return args_list; -} - -fn fmtMain(argc: c_int, argv: [*]const [*:0]const u8) !void { - const allocator = std.heap.c_allocator; - - var args_list = try argvToArrayList(allocator, argc, argv); - defer args_list.deinit(); - - const args = args_list.span()[2..]; - return self_hosted_main.cmdFmt(allocator, args); -} - -export fn stage2_DepTokenizer_init(input: [*]const u8, len: usize) stage2_DepTokenizer { - const t = std.heap.c_allocator.create(DepTokenizer) catch @panic("failed to create .d tokenizer"); - t.* = DepTokenizer.init(std.heap.c_allocator, input[0..len]); - return stage2_DepTokenizer{ - .handle = t, - }; -} - -export fn stage2_DepTokenizer_deinit(self: *stage2_DepTokenizer) void { - self.handle.deinit(); -} - -export fn stage2_DepTokenizer_next(self: *stage2_DepTokenizer) stage2_DepNextResult { - const otoken = self.handle.next() catch { - const textz = std.ArrayListSentineled(u8, 0).init(&self.handle.arena.allocator, self.handle.error_text) catch @panic("failed to create .d tokenizer error text"); - return stage2_DepNextResult{ - .type_id = .error_, - .textz = textz.span().ptr, - }; - }; - const token = otoken orelse { - return stage2_DepNextResult{ - .type_id = .null_, - .textz = undefined, - }; - }; - const textz = std.ArrayListSentineled(u8, 0).init(&self.handle.arena.allocator, token.bytes) catch @panic("failed to create .d tokenizer token text"); - return stage2_DepNextResult{ - .type_id = switch (token.id) { - .target => .target, - .prereq => .prereq, - }, - .textz = textz.span().ptr, - }; -} - -const stage2_DepTokenizer = extern struct { - handle: *DepTokenizer, -}; - -const stage2_DepNextResult = extern struct { - type_id: TypeId, - - // when type_id == error --> error text - // when type_id == null --> undefined - // when type_id == target --> target pathname - // when type_id == prereq --> prereq pathname - textz: [*]const u8, - - const TypeId = extern enum { - error_, - null_, - target, - prereq, - }; -}; - -// ABI warning -export fn stage2_attach_segfault_handler() void { - if (std.debug.runtime_safety and std.debug.have_segfault_handling_support) { - std.debug.attachSegfaultHandler(); - } -} - -// ABI warning -export fn stage2_progress_create() *std.Progress { - const ptr = std.heap.c_allocator.create(std.Progress) catch @panic("out of memory"); - ptr.* = std.Progress{}; - return ptr; -} - -// ABI warning -export fn stage2_progress_destroy(progress: *std.Progress) void { - std.heap.c_allocator.destroy(progress); -} - -// ABI warning -export fn stage2_progress_start_root( - progress: *std.Progress, - name_ptr: [*]const u8, - name_len: usize, - estimated_total_items: usize, -) *std.Progress.Node { - return progress.start( - name_ptr[0..name_len], - if (estimated_total_items == 0) null else estimated_total_items, - ) catch @panic("timer unsupported"); -} - -// ABI warning -export fn stage2_progress_disable_tty(progress: *std.Progress) void { - progress.terminal = null; -} - -// ABI warning -export fn stage2_progress_start( - node: *std.Progress.Node, - name_ptr: [*]const u8, - name_len: usize, - estimated_total_items: usize, -) *std.Progress.Node { - const child_node = std.heap.c_allocator.create(std.Progress.Node) catch @panic("out of memory"); - child_node.* = node.start( - name_ptr[0..name_len], - if (estimated_total_items == 0) null else estimated_total_items, - ); - child_node.activate(); - return child_node; -} - -// ABI warning -export fn stage2_progress_end(node: *std.Progress.Node) void { - node.end(); - if (&node.context.root != node) { - std.heap.c_allocator.destroy(node); - } -} - -// ABI warning -export fn stage2_progress_complete_one(node: *std.Progress.Node) void { - node.completeOne(); -} - -// ABI warning -export fn stage2_progress_update_node(node: *std.Progress.Node, done_count: usize, total_count: usize) void { - node.completed_items = done_count; - node.estimated_total_items = total_count; - node.activate(); - node.context.maybeRefresh(); -} - -fn detectNativeCpuWithLLVM( - arch: Target.Cpu.Arch, - llvm_cpu_name_z: ?[*:0]const u8, - llvm_cpu_features_opt: ?[*:0]const u8, -) !Target.Cpu { - var result = Target.Cpu.baseline(arch); - - if (llvm_cpu_name_z) |cpu_name_z| { - const llvm_cpu_name = mem.spanZ(cpu_name_z); - - for (arch.allCpuModels()) |model| { - const this_llvm_name = model.llvm_name orelse continue; - if (mem.eql(u8, this_llvm_name, llvm_cpu_name)) { - // Here we use the non-dependencies-populated set, - // so that subtracting features later in this function - // affect the prepopulated set. - result = Target.Cpu{ - .arch = arch, - .model = model, - .features = model.features, - }; - break; - } - } - } - - const all_features = arch.allFeaturesList(); - - if (llvm_cpu_features_opt) |llvm_cpu_features| { - var it = mem.tokenize(mem.spanZ(llvm_cpu_features), ","); - while (it.next()) |decorated_llvm_feat| { - var op: enum { - add, - sub, - } = undefined; - var llvm_feat: []const u8 = undefined; - if (mem.startsWith(u8, decorated_llvm_feat, "+")) { - op = .add; - llvm_feat = decorated_llvm_feat[1..]; - } else if (mem.startsWith(u8, decorated_llvm_feat, "-")) { - op = .sub; - llvm_feat = decorated_llvm_feat[1..]; - } else { - return error.InvalidLlvmCpuFeaturesFormat; - } - for (all_features) |feature, index_usize| { - const this_llvm_name = feature.llvm_name orelse continue; - if (mem.eql(u8, llvm_feat, this_llvm_name)) { - const index = @intCast(Target.Cpu.Feature.Set.Index, index_usize); - switch (op) { - .add => result.features.addFeature(index), - .sub => result.features.removeFeature(index), - } - break; - } - } - } - } - - result.features.populateDependencies(all_features); - return result; -} - -export fn stage2_env(argc: c_int, argv: [*]const [*:0]const u8) c_int { - const allocator = std.heap.c_allocator; - - var args_list = argvToArrayList(allocator, argc, argv) catch |err| { - std.debug.print("unable to parse arguments: {}\n", .{@errorName(err)}); - return -1; - }; - defer args_list.deinit(); - - const args = args_list.span()[2..]; - - @import("print_env.zig").cmdEnv(allocator, args, std.io.getStdOut().outStream()) catch |err| { - std.debug.print("unable to print info: {}\n", .{@errorName(err)}); - return -1; - }; - - return 0; -} - -export fn stage2_cc(argc: c_int, argv: [*]const [*:0]const u8, is_cpp: bool) c_int { - const allocator = std.heap.c_allocator; - - var args_list = argvToArrayList(allocator, argc, argv) catch |err| { - std.debug.print("unable to parse arguments: {}\n", .{@errorName(err)}); - return -1; - }; - defer args_list.deinit(); - - self_hosted_main.buildOutputType(allocator, allocator, args_list.items, if (is_cpp) .cpp else .cc) catch |err| { - std.debug.print("zig cc failure: {}\n", .{@errorName(err)}); - return -1; - }; - - return 0; -} - -// ABI warning -export fn stage2_cmd_targets( - zig_triple: ?[*:0]const u8, - mcpu: ?[*:0]const u8, - dynamic_linker: ?[*:0]const u8, -) c_int { - cmdTargets(zig_triple, mcpu, dynamic_linker) catch |err| { - std.debug.warn("unable to list targets: {}\n", .{@errorName(err)}); - return -1; - }; - return 0; -} - -fn cmdTargets( - zig_triple_oz: ?[*:0]const u8, - mcpu_oz: ?[*:0]const u8, - dynamic_linker_oz: ?[*:0]const u8, -) !void { - const cross_target = try stage2CrossTarget(zig_triple_oz, mcpu_oz, dynamic_linker_oz); - var dynamic_linker: ?[*:0]u8 = null; - const target = try crossTargetToTarget(cross_target, &dynamic_linker); - return @import("print_targets.zig").cmdTargets( - std.heap.c_allocator, - &[0][]u8{}, - std.io.getStdOut().outStream(), - target, - ); -} - -// ABI warning -export fn stage2_target_parse( - target: *Stage2Target, - zig_triple: ?[*:0]const u8, - mcpu: ?[*:0]const u8, - dynamic_linker: ?[*:0]const u8, -) Error { - stage2TargetParse(target, zig_triple, mcpu, dynamic_linker) catch |err| switch (err) { - error.OutOfMemory => return .OutOfMemory, - error.UnknownArchitecture => return .UnknownArchitecture, - error.UnknownOperatingSystem => return .UnknownOperatingSystem, - error.UnknownApplicationBinaryInterface => return .UnknownApplicationBinaryInterface, - error.MissingOperatingSystem => return .MissingOperatingSystem, - error.InvalidLlvmCpuFeaturesFormat => return .InvalidLlvmCpuFeaturesFormat, - error.UnexpectedExtraField => return .SemanticAnalyzeFail, - error.InvalidAbiVersion => return .InvalidAbiVersion, - error.InvalidOperatingSystemVersion => return .InvalidOperatingSystemVersion, - error.FileSystem => return .FileSystem, - error.SymLinkLoop => return .SymLinkLoop, - error.SystemResources => return .SystemResources, - error.ProcessFdQuotaExceeded => return .ProcessFdQuotaExceeded, - error.SystemFdQuotaExceeded => return .SystemFdQuotaExceeded, - error.DeviceBusy => return .DeviceBusy, - }; - return .None; -} - -fn stage2CrossTarget( - zig_triple_oz: ?[*:0]const u8, - mcpu_oz: ?[*:0]const u8, - dynamic_linker_oz: ?[*:0]const u8, -) !CrossTarget { - const mcpu = mem.spanZ(mcpu_oz); - const dynamic_linker = mem.spanZ(dynamic_linker_oz); - var diags: CrossTarget.ParseOptions.Diagnostics = .{}; - const target: CrossTarget = CrossTarget.parse(.{ - .arch_os_abi = mem.spanZ(zig_triple_oz) orelse "native", - .cpu_features = mcpu, - .dynamic_linker = dynamic_linker, - .diagnostics = &diags, - }) catch |err| switch (err) { - error.UnknownCpuModel => { - std.debug.warn("Unknown CPU: '{}'\nAvailable CPUs for architecture '{}':\n", .{ - diags.cpu_name.?, - @tagName(diags.arch.?), - }); - for (diags.arch.?.allCpuModels()) |cpu| { - std.debug.warn(" {}\n", .{cpu.name}); - } - process.exit(1); - }, - error.UnknownCpuFeature => { - std.debug.warn( - \\Unknown CPU feature: '{}' - \\Available CPU features for architecture '{}': - \\ - , .{ - diags.unknown_feature_name, - @tagName(diags.arch.?), - }); - for (diags.arch.?.allFeaturesList()) |feature| { - std.debug.warn(" {}: {}\n", .{ feature.name, feature.description }); - } - process.exit(1); - }, - else => |e| return e, - }; - - return target; -} - -fn stage2TargetParse( - stage1_target: *Stage2Target, - zig_triple_oz: ?[*:0]const u8, - mcpu_oz: ?[*:0]const u8, - dynamic_linker_oz: ?[*:0]const u8, -) !void { - const target = try stage2CrossTarget(zig_triple_oz, mcpu_oz, dynamic_linker_oz); - try stage1_target.fromTarget(target); -} - -// ABI warning -const Stage2LibCInstallation = extern struct { - include_dir: [*]const u8, - include_dir_len: usize, - sys_include_dir: [*]const u8, - sys_include_dir_len: usize, - crt_dir: [*]const u8, - crt_dir_len: usize, - msvc_lib_dir: [*]const u8, - msvc_lib_dir_len: usize, - kernel32_lib_dir: [*]const u8, - kernel32_lib_dir_len: usize, - - fn initFromStage2(self: *Stage2LibCInstallation, libc: LibCInstallation) void { - if (libc.include_dir) |s| { - self.include_dir = s.ptr; - self.include_dir_len = s.len; - } else { - self.include_dir = ""; - self.include_dir_len = 0; - } - if (libc.sys_include_dir) |s| { - self.sys_include_dir = s.ptr; - self.sys_include_dir_len = s.len; - } else { - self.sys_include_dir = ""; - self.sys_include_dir_len = 0; - } - if (libc.crt_dir) |s| { - self.crt_dir = s.ptr; - self.crt_dir_len = s.len; - } else { - self.crt_dir = ""; - self.crt_dir_len = 0; - } - if (libc.msvc_lib_dir) |s| { - self.msvc_lib_dir = s.ptr; - self.msvc_lib_dir_len = s.len; - } else { - self.msvc_lib_dir = ""; - self.msvc_lib_dir_len = 0; - } - if (libc.kernel32_lib_dir) |s| { - self.kernel32_lib_dir = s.ptr; - self.kernel32_lib_dir_len = s.len; - } else { - self.kernel32_lib_dir = ""; - self.kernel32_lib_dir_len = 0; - } - } - - fn toStage2(self: Stage2LibCInstallation) LibCInstallation { - var libc: LibCInstallation = .{}; - if (self.include_dir_len != 0) { - libc.include_dir = self.include_dir[0..self.include_dir_len]; - } - if (self.sys_include_dir_len != 0) { - libc.sys_include_dir = self.sys_include_dir[0..self.sys_include_dir_len]; - } - if (self.crt_dir_len != 0) { - libc.crt_dir = self.crt_dir[0..self.crt_dir_len]; - } - if (self.msvc_lib_dir_len != 0) { - libc.msvc_lib_dir = self.msvc_lib_dir[0..self.msvc_lib_dir_len]; - } - if (self.kernel32_lib_dir_len != 0) { - libc.kernel32_lib_dir = self.kernel32_lib_dir[0..self.kernel32_lib_dir_len]; - } - return libc; - } -}; - -// ABI warning -export fn stage2_libc_parse(stage1_libc: *Stage2LibCInstallation, libc_file_z: [*:0]const u8) Error { - const libc_file = mem.spanZ(libc_file_z); - var libc = LibCInstallation.parse(std.heap.c_allocator, libc_file) catch |err| switch (err) { - error.ParseError => return .SemanticAnalyzeFail, - error.FileTooBig => return .FileTooBig, - error.InputOutput => return .FileSystem, - error.NoSpaceLeft => return .NoSpaceLeft, - error.AccessDenied => return .AccessDenied, - error.BrokenPipe => return .BrokenPipe, - error.SystemResources => return .SystemResources, - error.OperationAborted => return .OperationAborted, - error.WouldBlock => unreachable, - error.NotOpenForReading => unreachable, - error.Unexpected => return .Unexpected, - error.IsDir => return .IsDir, - error.ConnectionResetByPeer => unreachable, - error.ConnectionTimedOut => unreachable, - error.OutOfMemory => return .OutOfMemory, - error.Unseekable => unreachable, - error.SharingViolation => return .SharingViolation, - error.PathAlreadyExists => unreachable, - error.FileNotFound => return .FileNotFound, - error.PipeBusy => return .PipeBusy, - error.NameTooLong => return .PathTooLong, - error.InvalidUtf8 => return .BadPathName, - error.BadPathName => return .BadPathName, - error.SymLinkLoop => return .SymLinkLoop, - error.ProcessFdQuotaExceeded => return .ProcessFdQuotaExceeded, - error.SystemFdQuotaExceeded => return .SystemFdQuotaExceeded, - error.NoDevice => return .NoDevice, - error.NotDir => return .NotDir, - error.DeviceBusy => return .DeviceBusy, - error.FileLocksNotSupported => unreachable, - }; - stage1_libc.initFromStage2(libc); - return .None; -} - -// ABI warning -export fn stage2_libc_find_native(stage1_libc: *Stage2LibCInstallation) Error { - var libc = LibCInstallation.findNative(.{ - .allocator = std.heap.c_allocator, - .verbose = true, - }) catch |err| switch (err) { - error.OutOfMemory => return .OutOfMemory, - error.FileSystem => return .FileSystem, - error.UnableToSpawnCCompiler => return .UnableToSpawnCCompiler, - error.CCompilerExitCode => return .CCompilerExitCode, - error.CCompilerCrashed => return .CCompilerCrashed, - error.CCompilerCannotFindHeaders => return .CCompilerCannotFindHeaders, - error.LibCRuntimeNotFound => return .LibCRuntimeNotFound, - error.LibCStdLibHeaderNotFound => return .LibCStdLibHeaderNotFound, - error.LibCKernel32LibNotFound => return .LibCKernel32LibNotFound, - error.UnsupportedArchitecture => return .UnsupportedArchitecture, - error.WindowsSdkNotFound => return .WindowsSdkNotFound, - error.ZigIsTheCCompiler => return .ZigIsTheCCompiler, - }; - stage1_libc.initFromStage2(libc); - return .None; -} - -// ABI warning -export fn stage2_libc_render(stage1_libc: *Stage2LibCInstallation, output_file: *FILE) Error { - var libc = stage1_libc.toStage2(); - const c_out_stream = std.io.cOutStream(output_file); - libc.render(c_out_stream) catch |err| switch (err) { - error.WouldBlock => unreachable, // stage1 opens stuff in exclusively blocking mode - error.NotOpenForWriting => unreachable, - error.SystemResources => return .SystemResources, - error.OperationAborted => return .OperationAborted, - error.BrokenPipe => return .BrokenPipe, - error.DiskQuota => return .DiskQuota, - error.FileTooBig => return .FileTooBig, - error.NoSpaceLeft => return .NoSpaceLeft, - error.AccessDenied => return .AccessDenied, - error.Unexpected => return .Unexpected, - error.InputOutput => return .FileSystem, - }; - return .None; -} - -// ABI warning -const Stage2Target = extern struct { - arch: c_int, - vendor: c_int, - - abi: c_int, - os: c_int, - - is_native_os: bool, - is_native_cpu: bool, - - glibc_or_darwin_version: ?*Stage2SemVer, - - llvm_cpu_name: ?[*:0]const u8, - llvm_cpu_features: ?[*:0]const u8, - cpu_builtin_str: ?[*:0]const u8, - cache_hash: ?[*:0]const u8, - cache_hash_len: usize, - os_builtin_str: ?[*:0]const u8, - - dynamic_linker: ?[*:0]const u8, - standard_dynamic_linker_path: ?[*:0]const u8, - - llvm_cpu_features_asm_ptr: [*]const [*:0]const u8, - llvm_cpu_features_asm_len: usize, - - fn fromTarget(self: *Stage2Target, cross_target: CrossTarget) !void { - const allocator = std.heap.c_allocator; - - var dynamic_linker: ?[*:0]u8 = null; - const target = try crossTargetToTarget(cross_target, &dynamic_linker); - - var cache_hash = try std.ArrayListSentineled(u8, 0).allocPrint(allocator, "{}\n{}\n", .{ - target.cpu.model.name, - target.cpu.features.asBytes(), - }); - defer cache_hash.deinit(); - - const generic_arch_name = target.cpu.arch.genericName(); - var cpu_builtin_str_buffer = try std.ArrayListSentineled(u8, 0).allocPrint(allocator, - \\Cpu{{ - \\ .arch = .{}, - \\ .model = &Target.{}.cpu.{}, - \\ .features = Target.{}.featureSet(&[_]Target.{}.Feature{{ - \\ - , .{ - @tagName(target.cpu.arch), - generic_arch_name, - target.cpu.model.name, - generic_arch_name, - generic_arch_name, - }); - defer cpu_builtin_str_buffer.deinit(); - - var llvm_features_buffer = try std.ArrayListSentineled(u8, 0).initSize(allocator, 0); - defer llvm_features_buffer.deinit(); - - // Unfortunately we have to do the work twice, because Clang does not support - // the same command line parameters for CPU features when assembling code as it does - // when compiling C code. - var asm_features_list = std.ArrayList([*:0]const u8).init(allocator); - defer asm_features_list.deinit(); - - for (target.cpu.arch.allFeaturesList()) |feature, index_usize| { - const index = @intCast(Target.Cpu.Feature.Set.Index, index_usize); - const is_enabled = target.cpu.features.isEnabled(index); - - if (feature.llvm_name) |llvm_name| { - const plus_or_minus = "-+"[@boolToInt(is_enabled)]; - try llvm_features_buffer.append(plus_or_minus); - try llvm_features_buffer.appendSlice(llvm_name); - try llvm_features_buffer.appendSlice(","); - } - - if (is_enabled) { - // TODO some kind of "zig identifier escape" function rather than - // unconditionally using @"" syntax - try cpu_builtin_str_buffer.appendSlice(" .@\""); - try cpu_builtin_str_buffer.appendSlice(feature.name); - try cpu_builtin_str_buffer.appendSlice("\",\n"); - } - } - - switch (target.cpu.arch) { - .riscv32, .riscv64 => { - if (std.Target.riscv.featureSetHas(target.cpu.features, .relax)) { - try asm_features_list.append("-mrelax"); - } else { - try asm_features_list.append("-mno-relax"); - } - }, - else => { - // TODO - // Argh, why doesn't the assembler accept the list of CPU features?! - // I don't see a way to do this other than hard coding everything. - }, - } - - try cpu_builtin_str_buffer.appendSlice( - \\ }), - \\}; - \\ - ); - - assert(mem.endsWith(u8, llvm_features_buffer.span(), ",")); - llvm_features_buffer.shrink(llvm_features_buffer.len() - 1); - - var os_builtin_str_buffer = try std.ArrayListSentineled(u8, 0).allocPrint(allocator, - \\Os{{ - \\ .tag = .{}, - \\ .version_range = .{{ - , .{@tagName(target.os.tag)}); - defer os_builtin_str_buffer.deinit(); - - // We'll re-use the OS version range builtin string for the cache hash. - const os_builtin_str_ver_start_index = os_builtin_str_buffer.len(); - - @setEvalBranchQuota(2000); - switch (target.os.tag) { - .freestanding, - .ananas, - .cloudabi, - .dragonfly, - .fuchsia, - .ios, - .kfreebsd, - .lv2, - .solaris, - .haiku, - .minix, - .rtems, - .nacl, - .cnk, - .aix, - .cuda, - .nvcl, - .amdhsa, - .ps4, - .elfiamcu, - .tvos, - .watchos, - .mesa3d, - .contiki, - .amdpal, - .hermit, - .hurd, - .wasi, - .emscripten, - .uefi, - .other, - => try os_builtin_str_buffer.appendSlice(" .none = {} }\n"), - - .freebsd, - .macosx, - .netbsd, - .openbsd, - => try os_builtin_str_buffer.outStream().print( - \\ .semver = .{{ - \\ .min = .{{ - \\ .major = {}, - \\ .minor = {}, - \\ .patch = {}, - \\ }}, - \\ .max = .{{ - \\ .major = {}, - \\ .minor = {}, - \\ .patch = {}, - \\ }}, - \\ }}}}, - \\ - , .{ - target.os.version_range.semver.min.major, - target.os.version_range.semver.min.minor, - target.os.version_range.semver.min.patch, - - target.os.version_range.semver.max.major, - target.os.version_range.semver.max.minor, - target.os.version_range.semver.max.patch, - }), - - .linux => try os_builtin_str_buffer.outStream().print( - \\ .linux = .{{ - \\ .range = .{{ - \\ .min = .{{ - \\ .major = {}, - \\ .minor = {}, - \\ .patch = {}, - \\ }}, - \\ .max = .{{ - \\ .major = {}, - \\ .minor = {}, - \\ .patch = {}, - \\ }}, - \\ }}, - \\ .glibc = .{{ - \\ .major = {}, - \\ .minor = {}, - \\ .patch = {}, - \\ }}, - \\ }}}}, - \\ - , .{ - target.os.version_range.linux.range.min.major, - target.os.version_range.linux.range.min.minor, - target.os.version_range.linux.range.min.patch, - - target.os.version_range.linux.range.max.major, - target.os.version_range.linux.range.max.minor, - target.os.version_range.linux.range.max.patch, - - target.os.version_range.linux.glibc.major, - target.os.version_range.linux.glibc.minor, - target.os.version_range.linux.glibc.patch, - }), - - .windows => try os_builtin_str_buffer.outStream().print( - \\ .windows = .{{ - \\ .min = {s}, - \\ .max = {s}, - \\ }}}}, - \\ - , .{ - target.os.version_range.windows.min, - target.os.version_range.windows.max, - }), - } - try os_builtin_str_buffer.appendSlice("};\n"); - - try cache_hash.appendSlice( - os_builtin_str_buffer.span()[os_builtin_str_ver_start_index..os_builtin_str_buffer.len()], - ); - - const glibc_or_darwin_version = blk: { - if (target.isGnuLibC()) { - const stage1_glibc = try std.heap.c_allocator.create(Stage2SemVer); - const stage2_glibc = target.os.version_range.linux.glibc; - stage1_glibc.* = .{ - .major = stage2_glibc.major, - .minor = stage2_glibc.minor, - .patch = stage2_glibc.patch, - }; - break :blk stage1_glibc; - } else if (target.isDarwin()) { - const stage1_semver = try std.heap.c_allocator.create(Stage2SemVer); - const stage2_semver = target.os.version_range.semver.min; - stage1_semver.* = .{ - .major = stage2_semver.major, - .minor = stage2_semver.minor, - .patch = stage2_semver.patch, - }; - break :blk stage1_semver; - } else { - break :blk null; - } - }; - - const std_dl = target.standardDynamicLinkerPath(); - const std_dl_z = if (std_dl.get()) |dl| - (try mem.dupeZ(std.heap.c_allocator, u8, dl)).ptr - else - null; - - const cache_hash_slice = cache_hash.toOwnedSlice(); - const asm_features = asm_features_list.toOwnedSlice(); - self.* = .{ - .arch = @enumToInt(target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch - .vendor = 0, - .os = @enumToInt(target.os.tag), - .abi = @enumToInt(target.abi), - .llvm_cpu_name = if (target.cpu.model.llvm_name) |s| s.ptr else null, - .llvm_cpu_features = llvm_features_buffer.toOwnedSlice().ptr, - .llvm_cpu_features_asm_ptr = asm_features.ptr, - .llvm_cpu_features_asm_len = asm_features.len, - .cpu_builtin_str = cpu_builtin_str_buffer.toOwnedSlice().ptr, - .os_builtin_str = os_builtin_str_buffer.toOwnedSlice().ptr, - .cache_hash = cache_hash_slice.ptr, - .cache_hash_len = cache_hash_slice.len, - .is_native_os = cross_target.isNativeOs(), - .is_native_cpu = cross_target.isNativeCpu(), - .glibc_or_darwin_version = glibc_or_darwin_version, - .dynamic_linker = dynamic_linker, - .standard_dynamic_linker_path = std_dl_z, - }; - } -}; - -fn enumInt(comptime Enum: type, int: c_int) Enum { - return @intToEnum(Enum, @intCast(@TagType(Enum), int)); -} - -fn crossTargetToTarget(cross_target: CrossTarget, dynamic_linker_ptr: *?[*:0]u8) !Target { - var info = try std.zig.system.NativeTargetInfo.detect(std.heap.c_allocator, cross_target); - if (info.cpu_detection_unimplemented) { - // TODO We want to just use detected_info.target but implementing - // CPU model & feature detection is todo so here we rely on LLVM. - const llvm = @import("llvm.zig"); - const llvm_cpu_name = llvm.GetHostCPUName(); - const llvm_cpu_features = llvm.GetNativeFeatures(); - const arch = std.Target.current.cpu.arch; - info.target.cpu = try detectNativeCpuWithLLVM(arch, llvm_cpu_name, llvm_cpu_features); - cross_target.updateCpuFeatures(&info.target.cpu.features); - info.target.cpu.arch = cross_target.getCpuArch(); - } - if (info.dynamic_linker.get()) |dl| { - dynamic_linker_ptr.* = try mem.dupeZ(std.heap.c_allocator, u8, dl); - } else { - dynamic_linker_ptr.* = null; - } - return info.target; -} - -// ABI warning -const Stage2SemVer = extern struct { - major: u32, - minor: u32, - patch: u32, -}; - -// ABI warning -const Stage2NativePaths = extern struct { - include_dirs_ptr: [*][*:0]u8, - include_dirs_len: usize, - lib_dirs_ptr: [*][*:0]u8, - lib_dirs_len: usize, - rpaths_ptr: [*][*:0]u8, - rpaths_len: usize, - warnings_ptr: [*][*:0]u8, - warnings_len: usize, -}; -// ABI warning -export fn stage2_detect_native_paths(stage1_paths: *Stage2NativePaths) Error { - stage2DetectNativePaths(stage1_paths) catch |err| switch (err) { - error.OutOfMemory => return .OutOfMemory, - }; - return .None; -} - -fn stage2DetectNativePaths(stage1_paths: *Stage2NativePaths) !void { - var paths = try std.zig.system.NativePaths.detect(std.heap.c_allocator); - errdefer paths.deinit(); - - try convertSlice(paths.include_dirs.span(), &stage1_paths.include_dirs_ptr, &stage1_paths.include_dirs_len); - try convertSlice(paths.lib_dirs.span(), &stage1_paths.lib_dirs_ptr, &stage1_paths.lib_dirs_len); - try convertSlice(paths.rpaths.span(), &stage1_paths.rpaths_ptr, &stage1_paths.rpaths_len); - try convertSlice(paths.warnings.span(), &stage1_paths.warnings_ptr, &stage1_paths.warnings_len); -} - -fn convertSlice(slice: [][:0]u8, ptr: *[*][*:0]u8, len: *usize) !void { - len.* = slice.len; - const new_slice = try std.heap.c_allocator.alloc([*:0]u8, slice.len); - for (slice) |item, i| { - new_slice[i] = item.ptr; - } - ptr.* = new_slice.ptr; -} - -export const stage2_is_zig0 = false; diff --git a/src/all_types.hpp b/src/all_types.hpp index 1fa04f2b79..7b8ad57483 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -10,7 +10,6 @@ #include "list.hpp" #include "buffer.hpp" -#include "cache_hash.hpp" #include "zig_llvm.h" #include "hash_map.hpp" #include "errmsg.hpp" @@ -1639,8 +1638,6 @@ struct ZigType { size_t abi_size; // Number of bits of information in this type. Known after ResolveStatusSizeKnown. size_t size_in_bits; - - bool gen_h_loop_flag; }; enum FnAnalState { @@ -1976,67 +1973,16 @@ struct TimeEvent { const char *name; }; -enum BuildMode { - BuildModeDebug, - BuildModeFastRelease, - BuildModeSafeRelease, - BuildModeSmallRelease, -}; - -enum CodeModel { - CodeModelDefault, - CodeModelTiny, - CodeModelSmall, - CodeModelKernel, - CodeModelMedium, - CodeModelLarge, -}; - -struct LinkLib { - Buf *name; - Buf *path; - ZigList symbols; // the list of symbols that we depend on from this lib - bool provided_explicitly; -}; - -enum ValgrindSupport { - ValgrindSupportAuto, - ValgrindSupportDisabled, - ValgrindSupportEnabled, -}; - -enum WantPIC { - WantPICAuto, - WantPICDisabled, - WantPICEnabled, -}; - -enum WantStackCheck { - WantStackCheckAuto, - WantStackCheckDisabled, - WantStackCheckEnabled, -}; - -enum WantCSanitize { - WantCSanitizeAuto, - WantCSanitizeDisabled, - WantCSanitizeEnabled, -}; - -enum OptionalBool { - OptionalBoolNull, - OptionalBoolFalse, - OptionalBoolTrue, -}; - struct CFile { ZigList args; const char *source_path; const char *preprocessor_only_basename; }; -// When adding fields, check if they should be added to the hash computation in build_with_cache struct CodeGen { + // Other code depends on this being first. + ZigStage1 stage1; + // arena allocator destroyed just prior to codegen emit heap::ArenaAllocator *pass1_arena; @@ -2048,8 +1994,6 @@ struct CodeGen { ZigLLVMDIBuilder *dbuilder; ZigLLVMDICompileUnit *compile_unit; ZigLLVMDIFile *compile_unit_file; - LinkLib *libc_link_lib; - LinkLib *libcpp_link_lib; LLVMTargetDataRef target_data_ref; LLVMTargetMachineRef target_machine; ZigLLVMDIFile *dummy_di_file; @@ -2104,7 +2048,6 @@ struct CodeGen { ZigList inline_fns; ZigList test_fns; ZigList errors_by_index; - ZigList caches_to_release; size_t largest_err_name_len; ZigList type_resolve_stack; @@ -2173,7 +2116,6 @@ struct CodeGen { Buf llvm_triple_str; Buf global_asm; Buf o_file_output_path; - Buf bin_file_output_path; Buf asm_file_output_path; Buf llvm_ir_file_output_path; Buf *cache_dir; @@ -2184,7 +2126,7 @@ struct CodeGen { const char **libc_include_dir_list; size_t libc_include_dir_len; - Buf *zig_c_headers_dir; // Cannot be overridden; derived from zig_lib_dir. + Buf *builtin_zig_path; Buf *zig_std_special_dir; // Cannot be overridden; derived from zig_lib_dir. IrInstSrc *invalid_inst_src; @@ -2206,18 +2148,9 @@ struct CodeGen { Stage2ProgressNode *main_progress_node; Stage2ProgressNode *sub_progress_node; - WantPIC want_pic; - WantStackCheck want_stack_check; - WantCSanitize want_sanitize_c; - CacheHash cache_hash; ErrColor err_color; uint32_t next_unresolved_index; unsigned pointer_size_bytes; - uint32_t target_os_index; - uint32_t target_arch_index; - uint32_t target_sub_arch_index; - uint32_t target_abi_index; - uint32_t target_oformat_index; bool is_big_endian; bool have_c_main; bool have_winmain; @@ -2226,9 +2159,6 @@ struct CodeGen { bool have_wwinmain_crt_startup; bool have_dllmain_crt_startup; bool have_err_ret_tracing; - bool link_eh_frame_hdr; - bool c_want_stdint; - bool c_want_stdbool; bool verbose_tokenize; bool verbose_ast; bool verbose_link; @@ -2239,61 +2169,24 @@ struct CodeGen { bool verbose_llvm_cpu_features; bool error_during_imports; bool generate_error_name_table; - bool enable_cache; // mutually exclusive with output_dir bool enable_time_report; bool enable_stack_report; - bool system_linker_hack; bool reported_bad_link_libc_error; - bool is_dynamic; // shared library rather than static library. dynamic musl rather than static musl. bool need_frame_size_prefix_data; - bool disable_c_depfile; - - //////////////////////////// Participates in Input Parameter Cache Hash - /////// Note: there is a separate cache hash for builtin.zig, when adding fields, - /////// consider if they need to go into both. - ZigList link_libs_list; - // add -framework [name] args to linker - ZigList darwin_frameworks; - // add -rpath [name] args to linker - ZigList rpath_list; - ZigList forbidden_libs; - ZigList link_objects; - ZigList assembly_files; - ZigList c_source_files; - ZigList lib_dirs; - ZigList framework_dirs; - - Stage2LibCInstallation *libc; - - bool is_versioned; - size_t version_major; - size_t version_minor; - size_t version_patch; - const char *linker_script; - size_t stack_size_override; + bool link_libc; + bool link_libcpp; BuildMode build_mode; - OutType out_type; const ZigTarget *zig_target; TargetSubsystem subsystem; // careful using this directly; see detect_subsystem - ValgrindSupport valgrind_support; CodeModel code_model; - OptionalBool linker_gc_sections; - OptionalBool linker_allow_shlib_undefined; - OptionalBool linker_bind_global_refs_locally; bool strip_debug_symbols; bool is_test_build; bool is_single_threaded; - bool want_single_threaded; - bool linker_rdynamic; - bool each_lib_rpath; - bool is_dummy_so; - bool disable_gen_h; - bool bundle_compiler_rt; bool have_pic; - bool have_dynamic_link; // this is whether the final thing will be dynamically linked. see also is_dynamic + bool link_mode_dynamic; + bool dll_export_fns; bool have_stack_probing; - bool have_sanitize_c; bool function_sections; bool enable_dump_analysis; bool enable_doc_generation; @@ -2301,23 +2194,13 @@ struct CodeGen { bool emit_asm; bool emit_llvm_ir; bool test_is_evented; - bool linker_z_nodelete; - bool linker_z_defs; + bool valgrind_enabled; Buf *root_out_name; Buf *test_filter; Buf *test_name_prefix; Buf *zig_lib_dir; Buf *zig_std_dir; - Buf *version_script_path; - Buf *override_soname; - Buf *linker_optimization; - - const char **llvm_argv; - size_t llvm_argv_len; - - const char **clang_argv; - size_t clang_argv_len; }; struct ZigVar { diff --git a/src/analyze.cpp b/src/analyze.cpp index b97423da73..ea19d81995 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -3496,7 +3496,7 @@ void add_var_export(CodeGen *g, ZigVar *var, const char *symbol_name, GlobalLink } void add_fn_export(CodeGen *g, ZigFn *fn_table_entry, const char *symbol_name, GlobalLinkageId linkage, CallingConvention cc) { - if (cc == CallingConventionC && strcmp(symbol_name, "main") == 0 && g->libc_link_lib != nullptr) { + if (cc == CallingConventionC && strcmp(symbol_name, "main") == 0 && g->link_libc) { g->have_c_main = true; } else if (cc == CallingConventionStdcall && g->zig_target->os == OsWindows) { if (strcmp(symbol_name, "WinMain") == 0) { @@ -7863,40 +7863,6 @@ const char *type_id_name(ZigTypeId id) { zig_unreachable(); } -LinkLib *create_link_lib(Buf *name) { - LinkLib *link_lib = heap::c_allocator.create(); - link_lib->name = name; - return link_lib; -} - -LinkLib *add_link_lib(CodeGen *g, Buf *name) { - bool is_libc = buf_eql_str(name, "c"); - bool is_libcpp = buf_eql_str(name, "c++") || buf_eql_str(name, "c++abi"); - - if (is_libc && g->libc_link_lib != nullptr) - return g->libc_link_lib; - - if (is_libcpp && g->libcpp_link_lib != nullptr) - return g->libcpp_link_lib; - - for (size_t i = 0; i < g->link_libs_list.length; i += 1) { - LinkLib *existing_lib = g->link_libs_list.at(i); - if (buf_eql_buf(existing_lib->name, name)) { - return existing_lib; - } - } - - LinkLib *link_lib = create_link_lib(name); - g->link_libs_list.append(link_lib); - - if (is_libc) - g->libc_link_lib = link_lib; - if (is_libcpp) - g->libcpp_link_lib = link_lib; - - return link_lib; -} - ZigType *get_align_amt_type(CodeGen *g) { if (g->align_amt_type == nullptr) { // according to LLVM the maximum alignment is 1 << 29. @@ -8017,12 +7983,13 @@ not_integer: return ErrorNone; } -Error file_fetch(CodeGen *g, Buf *resolved_path, Buf *contents) { - if (g->enable_cache) { - return cache_add_file_fetch(&g->cache_hash, resolved_path, contents); - } else { - return os_fetch_file_path(resolved_path, contents); - } +Error file_fetch(CodeGen *g, Buf *resolved_path, Buf *contents_buf) { + size_t len; + const char *contents = stage2_fetch_file(&g->stage1, buf_ptr(resolved_path), buf_len(resolved_path), &len); + if (contents == nullptr) + return ErrorNoMem; + buf_init_from_mem(contents_buf, contents, len); + return ErrorNone; } static X64CABIClass type_windows_abi_x86_64_class(CodeGen *g, ZigType *ty, size_t ty_size) { diff --git a/src/analyze.hpp b/src/analyze.hpp index 0df1a4ba91..07601e6dea 100644 --- a/src/analyze.hpp +++ b/src/analyze.hpp @@ -200,8 +200,6 @@ ZigTypeId type_id_at_index(size_t index); size_t type_id_len(); size_t type_id_index(ZigType *entry); ZigType *get_generic_fn_type(CodeGen *g, FnTypeId *fn_type_id); -LinkLib *create_link_lib(Buf *name); -LinkLib *add_link_lib(CodeGen *codegen, Buf *lib); bool optional_value_is_null(ZigValue *val); uint32_t get_abi_alignment(CodeGen *g, ZigType *type_entry); @@ -256,7 +254,6 @@ Error ensure_const_val_repr(IrAnalyze *ira, CodeGen *codegen, AstNode *source_no void typecheck_panic_fn(CodeGen *g, TldFn *tld_fn, ZigFn *panic_fn); Buf *type_bare_name(ZigType *t); Buf *type_h_name(ZigType *t); -Error create_c_object_cache(CodeGen *g, CacheHash **out_cache_hash, bool verbose); LLVMTypeRef get_llvm_type(CodeGen *g, ZigType *type); ZigLLVMDIType *get_llvm_di_type(CodeGen *g, ZigType *type); diff --git a/src/blake2.h b/src/blake2.h deleted file mode 100644 index 6420c5367a..0000000000 --- a/src/blake2.h +++ /dev/null @@ -1,196 +0,0 @@ -/* - BLAKE2 reference source code package - reference C implementations - - Copyright 2012, Samuel Neves . You may use this under the - terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at - your option. The terms of these licenses can be found at: - - - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 - - OpenSSL license : https://www.openssl.org/source/license.html - - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 - - More information about the BLAKE2 hash function can be found at - https://blake2.net. -*/ -#ifndef BLAKE2_H -#define BLAKE2_H - -#include -#include - -#if defined(_MSC_VER) -#define BLAKE2_PACKED(x) __pragma(pack(push, 1)) x __pragma(pack(pop)) -#else -#define BLAKE2_PACKED(x) x __attribute__((packed)) -#endif - -#if defined(__cplusplus) -extern "C" { -#endif - - enum blake2s_constant - { - BLAKE2S_BLOCKBYTES = 64, - BLAKE2S_OUTBYTES = 32, - BLAKE2S_KEYBYTES = 32, - BLAKE2S_SALTBYTES = 8, - BLAKE2S_PERSONALBYTES = 8 - }; - - enum blake2b_constant - { - BLAKE2B_BLOCKBYTES = 128, - BLAKE2B_OUTBYTES = 64, - BLAKE2B_KEYBYTES = 64, - BLAKE2B_SALTBYTES = 16, - BLAKE2B_PERSONALBYTES = 16 - }; - - typedef struct blake2s_state__ - { - uint32_t h[8]; - uint32_t t[2]; - uint32_t f[2]; - uint8_t buf[BLAKE2S_BLOCKBYTES]; - size_t buflen; - size_t outlen; - uint8_t last_node; - } blake2s_state; - - typedef struct blake2b_state__ - { - uint64_t h[8]; - uint64_t t[2]; - uint64_t f[2]; - uint8_t buf[BLAKE2B_BLOCKBYTES]; - size_t buflen; - size_t outlen; - uint8_t last_node; - } blake2b_state; - - typedef struct blake2sp_state__ - { - blake2s_state S[8][1]; - blake2s_state R[1]; - uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; - size_t buflen; - size_t outlen; - } blake2sp_state; - - typedef struct blake2bp_state__ - { - blake2b_state S[4][1]; - blake2b_state R[1]; - uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; - size_t buflen; - size_t outlen; - } blake2bp_state; - - - BLAKE2_PACKED(struct blake2s_param__ - { - uint8_t digest_length; /* 1 */ - uint8_t key_length; /* 2 */ - uint8_t fanout; /* 3 */ - uint8_t depth; /* 4 */ - uint32_t leaf_length; /* 8 */ - uint32_t node_offset; /* 12 */ - uint16_t xof_length; /* 14 */ - uint8_t node_depth; /* 15 */ - uint8_t inner_length; /* 16 */ - /* uint8_t reserved[0]; */ - uint8_t salt[BLAKE2S_SALTBYTES]; /* 24 */ - uint8_t personal[BLAKE2S_PERSONALBYTES]; /* 32 */ - }); - - typedef struct blake2s_param__ blake2s_param; - - BLAKE2_PACKED(struct blake2b_param__ - { - uint8_t digest_length; /* 1 */ - uint8_t key_length; /* 2 */ - uint8_t fanout; /* 3 */ - uint8_t depth; /* 4 */ - uint32_t leaf_length; /* 8 */ - uint32_t node_offset; /* 12 */ - uint32_t xof_length; /* 16 */ - uint8_t node_depth; /* 17 */ - uint8_t inner_length; /* 18 */ - uint8_t reserved[14]; /* 32 */ - uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ - uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ - }); - - typedef struct blake2b_param__ blake2b_param; - - typedef struct blake2xs_state__ - { - blake2s_state S[1]; - blake2s_param P[1]; - } blake2xs_state; - - typedef struct blake2xb_state__ - { - blake2b_state S[1]; - blake2b_param P[1]; - } blake2xb_state; - - /* Padded structs result in a compile-time error */ - enum { - BLAKE2_DUMMY_1 = 1/(sizeof(blake2s_param) == BLAKE2S_OUTBYTES), - BLAKE2_DUMMY_2 = 1/(sizeof(blake2b_param) == BLAKE2B_OUTBYTES) - }; - - /* Streaming API */ - int blake2s_init( blake2s_state *S, size_t outlen ); - int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); - int blake2s_update( blake2s_state *S, const void *in, size_t inlen ); - int blake2s_final( blake2s_state *S, void *out, size_t outlen ); - - int blake2b_init( blake2b_state *S, size_t outlen ); - int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); - int blake2b_update( blake2b_state *S, const void *in, size_t inlen ); - int blake2b_final( blake2b_state *S, void *out, size_t outlen ); - - int blake2sp_init( blake2sp_state *S, size_t outlen ); - int blake2sp_init_key( blake2sp_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2sp_update( blake2sp_state *S, const void *in, size_t inlen ); - int blake2sp_final( blake2sp_state *S, void *out, size_t outlen ); - - int blake2bp_init( blake2bp_state *S, size_t outlen ); - int blake2bp_init_key( blake2bp_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2bp_update( blake2bp_state *S, const void *in, size_t inlen ); - int blake2bp_final( blake2bp_state *S, void *out, size_t outlen ); - - /* Variable output length API */ - int blake2xs_init( blake2xs_state *S, const size_t outlen ); - int blake2xs_init_key( blake2xs_state *S, const size_t outlen, const void *key, size_t keylen ); - int blake2xs_update( blake2xs_state *S, const void *in, size_t inlen ); - int blake2xs_final(blake2xs_state *S, void *out, size_t outlen); - - int blake2xb_init( blake2xb_state *S, const size_t outlen ); - int blake2xb_init_key( blake2xb_state *S, const size_t outlen, const void *key, size_t keylen ); - int blake2xb_update( blake2xb_state *S, const void *in, size_t inlen ); - int blake2xb_final(blake2xb_state *S, void *out, size_t outlen); - - /* Simple API */ - int blake2s( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - - int blake2sp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - int blake2bp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - - int blake2xs( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - int blake2xb( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - - /* This is simply an alias for blake2b */ - int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - -#if defined(__cplusplus) -} -#endif - -#endif - diff --git a/src/blake2b.c b/src/blake2b.c deleted file mode 100644 index 600f951b9b..0000000000 --- a/src/blake2b.c +++ /dev/null @@ -1,539 +0,0 @@ -/* - BLAKE2 reference source code package - reference C implementations - - Copyright 2012, Samuel Neves . You may use this under the - terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at - your option. The terms of these licenses can be found at: - - - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 - - OpenSSL license : https://www.openssl.org/source/license.html - - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 - - More information about the BLAKE2 hash function can be found at - https://blake2.net. -*/ - -#include -#include -#include - -#include "blake2.h" -/* - BLAKE2 reference source code package - reference C implementations - - Copyright 2012, Samuel Neves . You may use this under the - terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at - your option. The terms of these licenses can be found at: - - - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 - - OpenSSL license : https://www.openssl.org/source/license.html - - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 - - More information about the BLAKE2 hash function can be found at - https://blake2.net. -*/ -#ifndef BLAKE2_IMPL_H -#define BLAKE2_IMPL_H - -#include -#include - -#if !defined(__cplusplus) && (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L) - #if defined(_MSC_VER) - #define BLAKE2_INLINE __inline - #elif defined(__GNUC__) - #define BLAKE2_INLINE __inline__ - #else - #define BLAKE2_INLINE - #endif -#else - #define BLAKE2_INLINE inline -#endif - -static BLAKE2_INLINE uint32_t load32( const void *src ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - uint32_t w; - memcpy(&w, src, sizeof w); - return w; -#else - const uint8_t *p = ( const uint8_t * )src; - return (( uint32_t )( p[0] ) << 0) | - (( uint32_t )( p[1] ) << 8) | - (( uint32_t )( p[2] ) << 16) | - (( uint32_t )( p[3] ) << 24) ; -#endif -} - -static BLAKE2_INLINE uint64_t load64( const void *src ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - uint64_t w; - memcpy(&w, src, sizeof w); - return w; -#else - const uint8_t *p = ( const uint8_t * )src; - return (( uint64_t )( p[0] ) << 0) | - (( uint64_t )( p[1] ) << 8) | - (( uint64_t )( p[2] ) << 16) | - (( uint64_t )( p[3] ) << 24) | - (( uint64_t )( p[4] ) << 32) | - (( uint64_t )( p[5] ) << 40) | - (( uint64_t )( p[6] ) << 48) | - (( uint64_t )( p[7] ) << 56) ; -#endif -} - -static BLAKE2_INLINE uint16_t load16( const void *src ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - uint16_t w; - memcpy(&w, src, sizeof w); - return w; -#else - const uint8_t *p = ( const uint8_t * )src; - return ( uint16_t )((( uint32_t )( p[0] ) << 0) | - (( uint32_t )( p[1] ) << 8)); -#endif -} - -static BLAKE2_INLINE void store16( void *dst, uint16_t w ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - memcpy(dst, &w, sizeof w); -#else - uint8_t *p = ( uint8_t * )dst; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; -#endif -} - -static BLAKE2_INLINE void store32( void *dst, uint32_t w ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - memcpy(dst, &w, sizeof w); -#else - uint8_t *p = ( uint8_t * )dst; - p[0] = (uint8_t)(w >> 0); - p[1] = (uint8_t)(w >> 8); - p[2] = (uint8_t)(w >> 16); - p[3] = (uint8_t)(w >> 24); -#endif -} - -static BLAKE2_INLINE void store64( void *dst, uint64_t w ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - memcpy(dst, &w, sizeof w); -#else - uint8_t *p = ( uint8_t * )dst; - p[0] = (uint8_t)(w >> 0); - p[1] = (uint8_t)(w >> 8); - p[2] = (uint8_t)(w >> 16); - p[3] = (uint8_t)(w >> 24); - p[4] = (uint8_t)(w >> 32); - p[5] = (uint8_t)(w >> 40); - p[6] = (uint8_t)(w >> 48); - p[7] = (uint8_t)(w >> 56); -#endif -} - -static BLAKE2_INLINE uint64_t load48( const void *src ) -{ - const uint8_t *p = ( const uint8_t * )src; - return (( uint64_t )( p[0] ) << 0) | - (( uint64_t )( p[1] ) << 8) | - (( uint64_t )( p[2] ) << 16) | - (( uint64_t )( p[3] ) << 24) | - (( uint64_t )( p[4] ) << 32) | - (( uint64_t )( p[5] ) << 40) ; -} - -static BLAKE2_INLINE void store48( void *dst, uint64_t w ) -{ - uint8_t *p = ( uint8_t * )dst; - p[0] = (uint8_t)(w >> 0); - p[1] = (uint8_t)(w >> 8); - p[2] = (uint8_t)(w >> 16); - p[3] = (uint8_t)(w >> 24); - p[4] = (uint8_t)(w >> 32); - p[5] = (uint8_t)(w >> 40); -} - -static BLAKE2_INLINE uint32_t rotr32( const uint32_t w, const unsigned c ) -{ - return ( w >> c ) | ( w << ( 32 - c ) ); -} - -static BLAKE2_INLINE uint64_t rotr64( const uint64_t w, const unsigned c ) -{ - return ( w >> c ) | ( w << ( 64 - c ) ); -} - -/* prevents compiler optimizing out memset() */ -static BLAKE2_INLINE void secure_zero_memory(void *v, size_t n) -{ - static void *(*const volatile memset_v)(void *, int, size_t) = &memset; - memset_v(v, 0, n); -} - -#endif - -static const uint64_t blake2b_IV[8] = -{ - 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, - 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, - 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, - 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL -}; - -static const uint8_t blake2b_sigma[12][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } -}; - - -static void blake2b_set_lastnode( blake2b_state *S ) -{ - S->f[1] = (uint64_t)-1; -} - -/* Some helper functions, not necessarily useful */ -static int blake2b_is_lastblock( const blake2b_state *S ) -{ - return S->f[0] != 0; -} - -static void blake2b_set_lastblock( blake2b_state *S ) -{ - if( S->last_node ) blake2b_set_lastnode( S ); - - S->f[0] = (uint64_t)-1; -} - -static void blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) -{ - S->t[0] += inc; - S->t[1] += ( S->t[0] < inc ); -} - -static void blake2b_init0( blake2b_state *S ) -{ - size_t i; - memset( S, 0, sizeof( blake2b_state ) ); - - for( i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; -} - -/* init xors IV with input parameter block */ -int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) -{ - const uint8_t *p = ( const uint8_t * )( P ); - size_t i; - - blake2b_init0( S ); - - /* IV XOR ParamBlock */ - for( i = 0; i < 8; ++i ) - S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); - - S->outlen = P->digest_length; - return 0; -} - - - -int blake2b_init( blake2b_state *S, size_t outlen ) -{ - blake2b_param P[1]; - - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - P->digest_length = (uint8_t)outlen; - P->key_length = 0; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store32( &P->node_offset, 0 ); - store32( &P->xof_length, 0 ); - P->node_depth = 0; - P->inner_length = 0; - memset( P->reserved, 0, sizeof( P->reserved ) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - return blake2b_init_param( S, P ); -} - - -int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) -{ - blake2b_param P[1]; - - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; - - P->digest_length = (uint8_t)outlen; - P->key_length = (uint8_t)keylen; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store32( &P->node_offset, 0 ); - store32( &P->xof_length, 0 ); - P->node_depth = 0; - P->inner_length = 0; - memset( P->reserved, 0, sizeof( P->reserved ) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - - if( blake2b_init_param( S, P ) < 0 ) return -1; - - { - uint8_t block[BLAKE2B_BLOCKBYTES]; - memset( block, 0, BLAKE2B_BLOCKBYTES ); - memcpy( block, key, keylen ); - blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); - secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ - } - return 0; -} - -#define G(r,i,a,b,c,d) \ - do { \ - a = a + b + m[blake2b_sigma[r][2*i+0]]; \ - d = rotr64(d ^ a, 32); \ - c = c + d; \ - b = rotr64(b ^ c, 24); \ - a = a + b + m[blake2b_sigma[r][2*i+1]]; \ - d = rotr64(d ^ a, 16); \ - c = c + d; \ - b = rotr64(b ^ c, 63); \ - } while(0) - -#define ROUND(r) \ - do { \ - G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ - G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ - G(r,2,v[ 2],v[ 6],v[10],v[14]); \ - G(r,3,v[ 3],v[ 7],v[11],v[15]); \ - G(r,4,v[ 0],v[ 5],v[10],v[15]); \ - G(r,5,v[ 1],v[ 6],v[11],v[12]); \ - G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ - G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ - } while(0) - -static void blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) -{ - uint64_t m[16]; - uint64_t v[16]; - size_t i; - - for( i = 0; i < 16; ++i ) { - m[i] = load64( block + i * sizeof( m[i] ) ); - } - - for( i = 0; i < 8; ++i ) { - v[i] = S->h[i]; - } - - v[ 8] = blake2b_IV[0]; - v[ 9] = blake2b_IV[1]; - v[10] = blake2b_IV[2]; - v[11] = blake2b_IV[3]; - v[12] = blake2b_IV[4] ^ S->t[0]; - v[13] = blake2b_IV[5] ^ S->t[1]; - v[14] = blake2b_IV[6] ^ S->f[0]; - v[15] = blake2b_IV[7] ^ S->f[1]; - - ROUND( 0 ); - ROUND( 1 ); - ROUND( 2 ); - ROUND( 3 ); - ROUND( 4 ); - ROUND( 5 ); - ROUND( 6 ); - ROUND( 7 ); - ROUND( 8 ); - ROUND( 9 ); - ROUND( 10 ); - ROUND( 11 ); - - for( i = 0; i < 8; ++i ) { - S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; - } -} - -#undef G -#undef ROUND - -int blake2b_update( blake2b_state *S, const void *pin, size_t inlen ) -{ - const unsigned char * in = (const unsigned char *)pin; - if( inlen > 0 ) - { - size_t left = S->buflen; - size_t fill = BLAKE2B_BLOCKBYTES - left; - if( inlen > fill ) - { - S->buflen = 0; - memcpy( S->buf + left, in, fill ); /* Fill buffer */ - blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); - blake2b_compress( S, S->buf ); /* Compress */ - in += fill; inlen -= fill; - while(inlen > BLAKE2B_BLOCKBYTES) { - blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); - blake2b_compress( S, in ); - in += BLAKE2B_BLOCKBYTES; - inlen -= BLAKE2B_BLOCKBYTES; - } - } - memcpy( S->buf + S->buflen, in, inlen ); - S->buflen += inlen; - } - return 0; -} - -int blake2b_final( blake2b_state *S, void *out, size_t outlen ) -{ - uint8_t buffer[BLAKE2B_OUTBYTES] = {0}; - size_t i; - - if( out == NULL || outlen < S->outlen ) - return -1; - - if( blake2b_is_lastblock( S ) ) - return -1; - - blake2b_increment_counter( S, S->buflen ); - blake2b_set_lastblock( S ); - memset( S->buf + S->buflen, 0, BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ - blake2b_compress( S, S->buf ); - - for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ - store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); - - memcpy( out, buffer, S->outlen ); - secure_zero_memory(buffer, sizeof(buffer)); - return 0; -} - -/* inlen, at least, should be uint64_t. Others can be size_t. */ -int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) -{ - blake2b_state S[1]; - - /* Verify parameters */ - if ( NULL == in && inlen > 0 ) return -1; - - if ( NULL == out ) return -1; - - if( NULL == key && keylen > 0 ) return -1; - - if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; - - if( keylen > BLAKE2B_KEYBYTES ) return -1; - - if( keylen > 0 ) - { - if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; - } - else - { - if( blake2b_init( S, outlen ) < 0 ) return -1; - } - - blake2b_update( S, ( const uint8_t * )in, inlen ); - blake2b_final( S, out, outlen ); - return 0; -} - -int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) { - return blake2b(out, outlen, in, inlen, key, keylen); -} - -#if defined(SUPERCOP) -int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) -{ - return blake2b( out, BLAKE2B_OUTBYTES, in, inlen, NULL, 0 ); -} -#endif - -#if defined(BLAKE2B_SELFTEST) -#include -#include "blake2-kat.h" -int main( void ) -{ - uint8_t key[BLAKE2B_KEYBYTES]; - uint8_t buf[BLAKE2_KAT_LENGTH]; - size_t i, step; - - for( i = 0; i < BLAKE2B_KEYBYTES; ++i ) - key[i] = ( uint8_t )i; - - for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) - buf[i] = ( uint8_t )i; - - /* Test simple API */ - for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) - { - uint8_t hash[BLAKE2B_OUTBYTES]; - blake2b( hash, BLAKE2B_OUTBYTES, buf, i, key, BLAKE2B_KEYBYTES ); - - if( 0 != memcmp( hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES ) ) - { - goto fail; - } - } - - /* Test streaming API */ - for(step = 1; step < BLAKE2B_BLOCKBYTES; ++step) { - for (i = 0; i < BLAKE2_KAT_LENGTH; ++i) { - uint8_t hash[BLAKE2B_OUTBYTES]; - blake2b_state S; - uint8_t * p = buf; - size_t mlen = i; - int err = 0; - - if( (err = blake2b_init_key(&S, BLAKE2B_OUTBYTES, key, BLAKE2B_KEYBYTES)) < 0 ) { - goto fail; - } - - while (mlen >= step) { - if ( (err = blake2b_update(&S, p, step)) < 0 ) { - goto fail; - } - mlen -= step; - p += step; - } - if ( (err = blake2b_update(&S, p, mlen)) < 0) { - goto fail; - } - if ( (err = blake2b_final(&S, hash, BLAKE2B_OUTBYTES)) < 0) { - goto fail; - } - - if (0 != memcmp(hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES)) { - goto fail; - } - } - } - - puts( "ok" ); - return 0; -fail: - puts("error"); - return -1; -} -#endif - diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp deleted file mode 100644 index c12d8f29ef..0000000000 --- a/src/cache_hash.cpp +++ /dev/null @@ -1,595 +0,0 @@ -/* - * Copyright (c) 2018 Andrew Kelley - * - * This file is part of zig, which is MIT licensed. - * See http://opensource.org/licenses/MIT - */ - -#include "stage2.h" -#include "cache_hash.hpp" -#include "all_types.hpp" -#include "buffer.hpp" -#include "os.hpp" - -#include - -void cache_init(CacheHash *ch, Buf *manifest_dir) { - int rc = blake2b_init(&ch->blake, 48); - assert(rc == 0); - ch->files = {}; - ch->manifest_dir = manifest_dir; - ch->manifest_file_path = nullptr; - ch->manifest_dirty = false; - ch->force_check_manifest = false; - ch->b64_digest = BUF_INIT; -} - -void cache_mem(CacheHash *ch, const char *ptr, size_t len) { - assert(ch->manifest_file_path == nullptr); - assert(ptr != nullptr); - blake2b_update(&ch->blake, ptr, len); -} - -void cache_slice(CacheHash *ch, Slice slice) { - // mix the length into the hash so that two juxtaposed cached slices can't collide - cache_usize(ch, slice.len); - cache_mem(ch, slice.ptr, slice.len); -} - -void cache_str(CacheHash *ch, const char *ptr) { - // + 1 to include the null byte - cache_mem(ch, ptr, strlen(ptr) + 1); -} - -void cache_int(CacheHash *ch, int x) { - assert(ch->manifest_file_path == nullptr); - // + 1 to include the null byte - uint8_t buf[sizeof(int) + 1]; - memcpy(buf, &x, sizeof(int)); - buf[sizeof(int)] = 0; - blake2b_update(&ch->blake, buf, sizeof(int) + 1); -} - -void cache_usize(CacheHash *ch, size_t x) { - assert(ch->manifest_file_path == nullptr); - // + 1 to include the null byte - uint8_t buf[sizeof(size_t) + 1]; - memcpy(buf, &x, sizeof(size_t)); - buf[sizeof(size_t)] = 0; - blake2b_update(&ch->blake, buf, sizeof(size_t) + 1); -} - -void cache_bool(CacheHash *ch, bool x) { - assert(ch->manifest_file_path == nullptr); - blake2b_update(&ch->blake, &x, 1); -} - -void cache_buf(CacheHash *ch, Buf *buf) { - assert(ch->manifest_file_path == nullptr); - assert(buf != nullptr); - // + 1 to include the null byte - blake2b_update(&ch->blake, buf_ptr(buf), buf_len(buf) + 1); -} - -void cache_buf_opt(CacheHash *ch, Buf *buf) { - assert(ch->manifest_file_path == nullptr); - if (buf == nullptr) { - cache_str(ch, ""); - cache_str(ch, ""); - } else { - cache_buf(ch, buf); - } -} - -void cache_list_of_link_lib(CacheHash *ch, LinkLib **ptr, size_t len) { - assert(ch->manifest_file_path == nullptr); - for (size_t i = 0; i < len; i += 1) { - LinkLib *lib = ptr[i]; - if (lib->provided_explicitly) { - cache_buf(ch, lib->name); - } - } - cache_str(ch, ""); -} - -void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len) { - assert(ch->manifest_file_path == nullptr); - for (size_t i = 0; i < len; i += 1) { - Buf *buf = ptr[i]; - cache_buf(ch, buf); - } - cache_str(ch, ""); -} - -void cache_list_of_file(CacheHash *ch, Buf **ptr, size_t len) { - assert(ch->manifest_file_path == nullptr); - - for (size_t i = 0; i < len; i += 1) { - Buf *buf = ptr[i]; - cache_file(ch, buf); - } - cache_str(ch, ""); -} - -void cache_list_of_str(CacheHash *ch, const char **ptr, size_t len) { - assert(ch->manifest_file_path == nullptr); - - for (size_t i = 0; i < len; i += 1) { - const char *s = ptr[i]; - cache_str(ch, s); - } - cache_str(ch, ""); -} - -void cache_file(CacheHash *ch, Buf *file_path) { - assert(ch->manifest_file_path == nullptr); - assert(file_path != nullptr); - Buf *resolved_path = buf_alloc(); - *resolved_path = os_path_resolve(&file_path, 1); - CacheHashFile *chf = ch->files.add_one(); - chf->path = resolved_path; - cache_buf(ch, resolved_path); -} - -void cache_file_opt(CacheHash *ch, Buf *file_path) { - assert(ch->manifest_file_path == nullptr); - if (file_path == nullptr) { - cache_str(ch, ""); - cache_str(ch, ""); - } else { - cache_file(ch, file_path); - } -} - -// Ported from std/base64.zig -static uint8_t base64_fs_alphabet[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; -static void base64_encode(Slice dest, Slice source) { - size_t dest_len = ((source.len + 2) / 3) * 4; - assert(dest.len == dest_len); - - size_t i = 0; - size_t out_index = 0; - for (; i + 2 < source.len; i += 3) { - dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; - out_index += 1; - - dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; - out_index += 1; - - dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i + 1] & 0xf) << 2) | ((source.ptr[i + 2] & 0xc0) >> 6)]; - out_index += 1; - - dest.ptr[out_index] = base64_fs_alphabet[source.ptr[i + 2] & 0x3f]; - out_index += 1; - } - - // Assert that we never need pad characters. - assert(i == source.len); -} - -// Ported from std/base64.zig -static Error base64_decode(Slice dest, Slice source) { - if (source.len % 4 != 0) - return ErrorInvalidFormat; - if (dest.len != (source.len / 4) * 3) - return ErrorInvalidFormat; - - // In Zig this is comptime computed. In C++ it's not worth it to do that. - uint8_t char_to_index[256]; - bool char_in_alphabet[256] = {0}; - for (size_t i = 0; i < 64; i += 1) { - uint8_t c = base64_fs_alphabet[i]; - assert(!char_in_alphabet[c]); - char_in_alphabet[c] = true; - char_to_index[c] = i; - } - - size_t src_cursor = 0; - size_t dest_cursor = 0; - - for (;src_cursor < source.len; src_cursor += 4) { - if (!char_in_alphabet[source.ptr[src_cursor + 0]]) return ErrorInvalidFormat; - if (!char_in_alphabet[source.ptr[src_cursor + 1]]) return ErrorInvalidFormat; - if (!char_in_alphabet[source.ptr[src_cursor + 2]]) return ErrorInvalidFormat; - if (!char_in_alphabet[source.ptr[src_cursor + 3]]) return ErrorInvalidFormat; - dest.ptr[dest_cursor + 0] = (char_to_index[source.ptr[src_cursor + 0]] << 2) | (char_to_index[source.ptr[src_cursor + 1]] >> 4); - dest.ptr[dest_cursor + 1] = (char_to_index[source.ptr[src_cursor + 1]] << 4) | (char_to_index[source.ptr[src_cursor + 2]] >> 2); - dest.ptr[dest_cursor + 2] = (char_to_index[source.ptr[src_cursor + 2]] << 6) | (char_to_index[source.ptr[src_cursor + 3]]); - dest_cursor += 3; - } - - assert(src_cursor == source.len); - assert(dest_cursor == dest.len); - return ErrorNone; -} - -static Error hash_file(uint8_t *digest, OsFile handle, Buf *contents) { - Error err; - - if (contents) { - buf_resize(contents, 0); - } - - blake2b_state blake; - int rc = blake2b_init(&blake, 48); - assert(rc == 0); - - for (;;) { - uint8_t buf[4096]; - size_t amt = 4096; - if ((err = os_file_read(handle, buf, &amt))) - return err; - if (amt == 0) { - rc = blake2b_final(&blake, digest, 48); - assert(rc == 0); - return ErrorNone; - } - blake2b_update(&blake, buf, amt); - if (contents) { - buf_append_mem(contents, (char*)buf, amt); - } - } -} - -// If the wall clock time, rounded to the same precision as the -// mtime, is equal to the mtime, then we cannot rely on this mtime -// yet. We will instead save an mtime value that indicates the hash -// must be unconditionally computed. -static bool is_problematic_timestamp(const OsTimeStamp *fs_clock) { - OsTimeStamp wall_clock = os_timestamp_calendar(); - // First make all the least significant zero bits in the fs_clock, also zero bits in the wall clock. - if (fs_clock->nsec == 0) { - wall_clock.nsec = 0; - if (fs_clock->sec == 0) { - wall_clock.sec = 0; - } else { - wall_clock.sec &= (-1ull) << ctzll(fs_clock->sec); - } - } else { - wall_clock.nsec &= (-1ull) << ctzll(fs_clock->nsec); - } - return wall_clock.nsec == fs_clock->nsec && wall_clock.sec == fs_clock->sec; -} - -static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf, Buf *contents) { - Error err; - - assert(chf->path != nullptr); - - OsFile this_file; - if ((err = os_file_open_r(chf->path, &this_file, &chf->attr))) - return err; - - if (is_problematic_timestamp(&chf->attr.mtime)) { - chf->attr.mtime.sec = 0; - chf->attr.mtime.nsec = 0; - chf->attr.inode = 0; - } - - if ((err = hash_file(chf->bin_digest, this_file, contents))) { - os_file_close(&this_file); - return err; - } - os_file_close(&this_file); - - blake2b_update(&ch->blake, chf->bin_digest, 48); - - return ErrorNone; -} - -Error cache_hit(CacheHash *ch, Buf *out_digest) { - Error err; - - uint8_t bin_digest[48]; - int rc = blake2b_final(&ch->blake, bin_digest, 48); - assert(rc == 0); - - buf_resize(&ch->b64_digest, 64); - base64_encode(buf_to_slice(&ch->b64_digest), {bin_digest, 48}); - - if (ch->files.length == 0 && !ch->force_check_manifest) { - buf_resize(out_digest, 64); - base64_encode(buf_to_slice(out_digest), {bin_digest, 48}); - return ErrorNone; - } - - rc = blake2b_init(&ch->blake, 48); - assert(rc == 0); - blake2b_update(&ch->blake, bin_digest, 48); - - ch->manifest_file_path = buf_alloc(); - os_path_join(ch->manifest_dir, &ch->b64_digest, ch->manifest_file_path); - - buf_append_str(ch->manifest_file_path, ".txt"); - - if ((err = os_make_path(ch->manifest_dir))) - return err; - - if ((err = os_file_open_lock_rw(ch->manifest_file_path, &ch->manifest_file))) - return err; - - Buf line_buf = BUF_INIT; - buf_resize(&line_buf, 512); - if ((err = os_file_read_all(ch->manifest_file, &line_buf))) { - os_file_close(&ch->manifest_file); - return err; - } - - size_t input_file_count = ch->files.length; - bool any_file_changed = false; - Error return_code = ErrorNone; - size_t file_i = 0; - SplitIterator line_it = memSplit(buf_to_slice(&line_buf), str("\n")); - for (;; file_i += 1) { - Optional> opt_line = SplitIterator_next(&line_it); - - CacheHashFile *chf; - if (file_i < input_file_count) { - chf = &ch->files.at(file_i); - } else if (any_file_changed) { - // cache miss. - // keep the manifest file open with the rw lock - // reset the hash - rc = blake2b_init(&ch->blake, 48); - assert(rc == 0); - blake2b_update(&ch->blake, bin_digest, 48); - ch->files.resize(input_file_count); - // bring the hash up to the input file hashes - for (file_i = 0; file_i < input_file_count; file_i += 1) { - blake2b_update(&ch->blake, ch->files.at(file_i).bin_digest, 48); - } - // caller can notice that out_digest is unmodified. - return return_code; - } else if (!opt_line.is_some) { - break; - } else { - chf = ch->files.add_one(); - chf->path = nullptr; - } - - if (!opt_line.is_some) - break; - - SplitIterator it = memSplit(opt_line.value, str(" ")); - - Optional> opt_inode = SplitIterator_next(&it); - if (!opt_inode.is_some) { - return_code = ErrorInvalidFormat; - break; - } - chf->attr.inode = strtoull((const char *)opt_inode.value.ptr, nullptr, 10); - - Optional> opt_mtime_sec = SplitIterator_next(&it); - if (!opt_mtime_sec.is_some) { - return_code = ErrorInvalidFormat; - break; - } - chf->attr.mtime.sec = strtoull((const char *)opt_mtime_sec.value.ptr, nullptr, 10); - - Optional> opt_mtime_nsec = SplitIterator_next(&it); - if (!opt_mtime_nsec.is_some) { - return_code = ErrorInvalidFormat; - break; - } - chf->attr.mtime.nsec = strtoull((const char *)opt_mtime_nsec.value.ptr, nullptr, 10); - - Optional> opt_digest = SplitIterator_next(&it); - if (!opt_digest.is_some) { - return_code = ErrorInvalidFormat; - break; - } - if ((err = base64_decode({chf->bin_digest, 48}, opt_digest.value))) { - return_code = ErrorInvalidFormat; - break; - } - - Slice file_path = SplitIterator_rest(&it); - if (file_path.len == 0) { - return_code = ErrorInvalidFormat; - break; - } - Buf *this_path = buf_create_from_slice(file_path); - if (chf->path != nullptr && !buf_eql_buf(this_path, chf->path)) { - return_code = ErrorInvalidFormat; - break; - } - chf->path = this_path; - - // if the mtime matches we can trust the digest - OsFile this_file; - OsFileAttr actual_attr; - if ((err = os_file_open_r(chf->path, &this_file, &actual_attr))) { - fprintf(stderr, "Unable to open %s\n: %s", buf_ptr(chf->path), err_str(err)); - os_file_close(&ch->manifest_file); - return ErrorCacheUnavailable; - } - if (chf->attr.mtime.sec == actual_attr.mtime.sec && - chf->attr.mtime.nsec == actual_attr.mtime.nsec && - chf->attr.inode == actual_attr.inode) - { - os_file_close(&this_file); - } else { - // we have to recompute the digest. - // later we'll rewrite the manifest with the new mtime/digest values - ch->manifest_dirty = true; - chf->attr = actual_attr; - - if (is_problematic_timestamp(&actual_attr.mtime)) { - chf->attr.mtime.sec = 0; - chf->attr.mtime.nsec = 0; - chf->attr.inode = 0; - } - - uint8_t actual_digest[48]; - if ((err = hash_file(actual_digest, this_file, nullptr))) { - os_file_close(&this_file); - os_file_close(&ch->manifest_file); - return err; - } - os_file_close(&this_file); - if (memcmp(chf->bin_digest, actual_digest, 48) != 0) { - memcpy(chf->bin_digest, actual_digest, 48); - // keep going until we have the input file digests - any_file_changed = true; - } - } - if (!any_file_changed) { - blake2b_update(&ch->blake, chf->bin_digest, 48); - } - } - if (file_i < input_file_count || file_i == 0 || return_code != ErrorNone) { - // manifest file is empty or missing entries, so this is a cache miss - ch->manifest_dirty = true; - for (; file_i < input_file_count; file_i += 1) { - CacheHashFile *chf = &ch->files.at(file_i); - if ((err = populate_file_hash(ch, chf, nullptr))) { - fprintf(stderr, "Unable to hash %s: %s\n", buf_ptr(chf->path), err_str(err)); - os_file_close(&ch->manifest_file); - return ErrorCacheUnavailable; - } - } - if (return_code != ErrorNone && return_code != ErrorInvalidFormat) { - os_file_close(&ch->manifest_file); - } - return return_code; - } - // Cache Hit - return cache_final(ch, out_digest); -} - -Error cache_add_file_fetch(CacheHash *ch, Buf *resolved_path, Buf *contents) { - Error err; - - assert(ch->manifest_file_path != nullptr); - CacheHashFile *chf = ch->files.add_one(); - chf->path = resolved_path; - if ((err = populate_file_hash(ch, chf, contents))) { - os_file_close(&ch->manifest_file); - return err; - } - - return ErrorNone; -} - -Error cache_add_file(CacheHash *ch, Buf *path) { - Buf *resolved_path = buf_alloc(); - *resolved_path = os_path_resolve(&path, 1); - return cache_add_file_fetch(ch, resolved_path, nullptr); -} - -Error cache_add_dep_file(CacheHash *ch, Buf *dep_file_path, bool verbose) { - Error err; - Buf *contents = buf_alloc(); - if ((err = os_fetch_file_path(dep_file_path, contents))) { - if (err == ErrorFileNotFound) - return err; - if (verbose) { - fprintf(stderr, "%s: unable to read .d file: %s\n", err_str(err), buf_ptr(dep_file_path)); - } - return ErrorReadingDepFile; - } - auto it = stage2_DepTokenizer_init(buf_ptr(contents), buf_len(contents)); - // skip first token: target - { - auto result = stage2_DepTokenizer_next(&it); - switch (result.type_id) { - case stage2_DepNextResult::error: - if (verbose) { - fprintf(stderr, "%s: failed processing .d file: %s\n", result.textz, buf_ptr(dep_file_path)); - } - err = ErrorInvalidDepFile; - goto finish; - case stage2_DepNextResult::null: - err = ErrorNone; - goto finish; - case stage2_DepNextResult::target: - case stage2_DepNextResult::prereq: - err = ErrorNone; - break; - } - } - // Process 0+ preqreqs. - // clang is invoked in single-source mode so we never get more targets. - for (;;) { - auto result = stage2_DepTokenizer_next(&it); - switch (result.type_id) { - case stage2_DepNextResult::error: - if (verbose) { - fprintf(stderr, "%s: failed processing .d file: %s\n", result.textz, buf_ptr(dep_file_path)); - } - err = ErrorInvalidDepFile; - goto finish; - case stage2_DepNextResult::null: - case stage2_DepNextResult::target: - err = ErrorNone; - goto finish; - case stage2_DepNextResult::prereq: - break; - } - auto textbuf = buf_alloc(); - buf_init_from_str(textbuf, result.textz); - if ((err = cache_add_file(ch, textbuf))) { - if (verbose) { - fprintf(stderr, "unable to add %s to cache: %s\n", result.textz, err_str(err)); - fprintf(stderr, "when processing .d file: %s\n", buf_ptr(dep_file_path)); - } - goto finish; - } - } - - finish: - stage2_DepTokenizer_deinit(&it); - return err; -} - -static Error write_manifest_file(CacheHash *ch) { - Error err; - Buf contents = BUF_INIT; - buf_resize(&contents, 0); - uint8_t encoded_digest[65]; - encoded_digest[64] = 0; - for (size_t i = 0; i < ch->files.length; i += 1) { - CacheHashFile *chf = &ch->files.at(i); - base64_encode({encoded_digest, 64}, {chf->bin_digest, 48}); - buf_appendf(&contents, "%" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %s %s\n", - chf->attr.inode, chf->attr.mtime.sec, chf->attr.mtime.nsec, encoded_digest, buf_ptr(chf->path)); - } - if ((err = os_file_overwrite(ch->manifest_file, &contents))) - return err; - - return ErrorNone; -} - -Error cache_final(CacheHash *ch, Buf *out_digest) { - assert(ch->manifest_file_path != nullptr); - - // We don't close the manifest file yet, because we want to - // keep it locked until the API user is done using it. - // We also don't write out the manifest yet, because until - // cache_release is called we still might be working on creating - // the artifacts to cache. - - uint8_t bin_digest[48]; - int rc = blake2b_final(&ch->blake, bin_digest, 48); - assert(rc == 0); - buf_resize(out_digest, 64); - base64_encode(buf_to_slice(out_digest), {bin_digest, 48}); - - return ErrorNone; -} - -void cache_release(CacheHash *ch) { - assert(ch->manifest_file_path != nullptr); - - Error err; - - if (ch->manifest_dirty) { - if ((err = write_manifest_file(ch))) { - fprintf(stderr, "Warning: Unable to write cache file '%s': %s\n", - buf_ptr(ch->manifest_file_path), err_str(err)); - } - } - - os_file_close(&ch->manifest_file); -} - diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp deleted file mode 100644 index ba2434076a..0000000000 --- a/src/cache_hash.hpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2018 Andrew Kelley - * - * This file is part of zig, which is MIT licensed. - * See http://opensource.org/licenses/MIT - */ - -#ifndef ZIG_CACHE_HASH_HPP -#define ZIG_CACHE_HASH_HPP - -#include "blake2.h" -#include "os.hpp" - -struct LinkLib; - -struct CacheHashFile { - Buf *path; - OsFileAttr attr; - uint8_t bin_digest[48]; - Buf *contents; -}; - -struct CacheHash { - blake2b_state blake; - ZigList files; - Buf *manifest_dir; - Buf *manifest_file_path; - Buf b64_digest; - OsFile manifest_file; - bool manifest_dirty; - bool force_check_manifest; -}; - -// Always call this first to set up. -void cache_init(CacheHash *ch, Buf *manifest_dir); - -// Next, use the hash population functions to add the initial parameters. -void cache_mem(CacheHash *ch, const char *ptr, size_t len); -void cache_slice(CacheHash *ch, Slice slice); -void cache_str(CacheHash *ch, const char *ptr); -void cache_int(CacheHash *ch, int x); -void cache_bool(CacheHash *ch, bool x); -void cache_usize(CacheHash *ch, size_t x); -void cache_buf(CacheHash *ch, Buf *buf); -void cache_buf_opt(CacheHash *ch, Buf *buf); -void cache_list_of_link_lib(CacheHash *ch, LinkLib **ptr, size_t len); -void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len); -void cache_list_of_file(CacheHash *ch, Buf **ptr, size_t len); -void cache_list_of_str(CacheHash *ch, const char **ptr, size_t len); -void cache_file(CacheHash *ch, Buf *path); -void cache_file_opt(CacheHash *ch, Buf *path); - -// Then call cache_hit when you're ready to see if you can skip the next step. -// out_b64_digest will be left unchanged if it was a cache miss. -// If you got a cache hit, the next step is cache_release. -// From this point on, there is a lock on the input params. Release -// the lock with cache_release. -// Set force_check_manifest if you plan to add files later, but have not -// added any files before calling cache_hit. CacheHash::b64_digest becomes -// available for use after this call, even in the case of a miss, and it -// is a hash of the input parameters only. -// If this function returns ErrorInvalidFormat, that error may be treated -// as a cache miss. -Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest); - -// If you did not get a cache hit, call this function for every file -// that is depended on, and then finish with cache_final. -Error ATTRIBUTE_MUST_USE cache_add_file(CacheHash *ch, Buf *path); -// This opens a file created by -MD -MF args to Clang -Error ATTRIBUTE_MUST_USE cache_add_dep_file(CacheHash *ch, Buf *path, bool verbose); - -// This variant of cache_add_file returns the file contents. -// Also the file path argument must be already resolved. -Error ATTRIBUTE_MUST_USE cache_add_file_fetch(CacheHash *ch, Buf *resolved_path, Buf *contents); - -// out_b64_digest will be the same thing that cache_hit returns if you got a cache hit -Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_b64_digest); - -// Until this function is called, no one will be able to get a lock on your input params. -void cache_release(CacheHash *ch); - - -#endif diff --git a/src/codegen.cpp b/src/codegen.cpp index ce6eeb1def..07728ef06e 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -8,7 +8,6 @@ #include "analyze.hpp" #include "ast_render.hpp" #include "codegen.hpp" -#include "compiler.hpp" #include "config.h" #include "errmsg.hpp" #include "error.hpp" @@ -21,7 +20,6 @@ #include "stage2.h" #include "dump_analysis.hpp" #include "softfloat.hpp" -#include "mem_profile.hpp" #include #include @@ -72,39 +70,6 @@ static const char *symbols_that_llvm_depends_on[] = { // TODO probably all of compiler-rt needs to go here }; -void codegen_set_clang_argv(CodeGen *g, const char **args, size_t len) { - g->clang_argv = args; - g->clang_argv_len = len; -} - -void codegen_set_llvm_argv(CodeGen *g, const char **args, size_t len) { - g->llvm_argv = args; - g->llvm_argv_len = len; -} - -void codegen_set_test_filter(CodeGen *g, Buf *filter) { - g->test_filter = filter; -} - -void codegen_set_test_name_prefix(CodeGen *g, Buf *prefix) { - g->test_name_prefix = prefix; -} - -void codegen_set_lib_version(CodeGen *g, bool is_versioned, size_t major, size_t minor, size_t patch) { - g->is_versioned = is_versioned; - g->version_major = major; - g->version_minor = minor; - g->version_patch = patch; -} - -void codegen_set_each_lib_rpath(CodeGen *g, bool each_lib_rpath) { - g->each_lib_rpath = each_lib_rpath; -} - -void codegen_set_errmsg_color(CodeGen *g, ErrColor err_color) { - g->err_color = err_color; -} - void codegen_set_strip(CodeGen *g, bool strip) { g->strip_debug_symbols = strip; if (!target_has_debug_info(g->zig_target)) { @@ -112,39 +77,6 @@ void codegen_set_strip(CodeGen *g, bool strip) { } } -void codegen_set_out_name(CodeGen *g, Buf *out_name) { - g->root_out_name = out_name; -} - -void codegen_add_lib_dir(CodeGen *g, const char *dir) { - g->lib_dirs.append(dir); -} - -void codegen_add_rpath(CodeGen *g, const char *name) { - g->rpath_list.append(buf_create_from_str(name)); -} - -LinkLib *codegen_add_link_lib(CodeGen *g, Buf *name) { - return add_link_lib(g, name); -} - -void codegen_add_forbidden_lib(CodeGen *codegen, Buf *lib) { - codegen->forbidden_libs.append(lib); -} - -void codegen_add_framework(CodeGen *g, const char *framework) { - g->darwin_frameworks.append(buf_create_from_str(framework)); -} - -void codegen_set_rdynamic(CodeGen *g, bool rdynamic) { - g->linker_rdynamic = rdynamic; -} - -void codegen_set_linker_script(CodeGen *g, const char *linker_script) { - g->linker_script = linker_script; -} - - static void render_const_val(CodeGen *g, ZigValue *const_val, const char *name); static void render_const_val_global(CodeGen *g, ZigValue *const_val, const char *name); static LLVMValueRef gen_const_val(CodeGen *g, ZigValue *const_val, const char *name); @@ -155,7 +87,6 @@ static LLVMValueRef build_alloca(CodeGen *g, ZigType *type_entry, const char *na static LLVMValueRef gen_await_early_return(CodeGen *g, IrInstGen *source_instr, LLVMValueRef target_frame_ptr, ZigType *result_type, ZigType *ptr_result_type, LLVMValueRef result_loc, bool non_async); -static Error get_tmp_filename(CodeGen *g, Buf *out, Buf *suffix); static void addLLVMAttr(LLVMValueRef val, LLVMAttributeIndex attr_index, const char *attr_name) { unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name, strlen(attr_name)); @@ -389,7 +320,7 @@ static uint32_t get_err_ret_trace_arg_index(CodeGen *g, ZigFn *fn_table_entry) { } static void maybe_export_dll(CodeGen *g, LLVMValueRef global_value, GlobalLinkageId linkage) { - if (linkage != GlobalLinkageIdInternal && g->zig_target->os == OsWindows && g->is_dynamic) { + if (linkage != GlobalLinkageIdInternal && g->zig_target->os == OsWindows && g->dll_export_fns) { LLVMSetDLLStorageClass(global_value, LLVMDLLExportStorageClass); } } @@ -541,7 +472,7 @@ static LLVMValueRef make_fn_llvm_value(CodeGen *g, ZigFn *fn) { g->build_mode != BuildModeSmallRelease && !fn->def_scope->safety_off; if (want_fn_safety) { - if (g->libc_link_lib != nullptr) { + if (g->link_libc) { addLLVMFnAttr(llvm_fn, "sspstrong"); addLLVMFnAttrStr(llvm_fn, "stack-protector-buffer-size", "4"); } @@ -3859,20 +3790,6 @@ static LLVMValueRef gen_valgrind_client_request(CodeGen *g, LLVMValueRef default zig_unreachable(); } -static bool want_valgrind_support(CodeGen *g) { - if (!target_has_valgrind_support(g->zig_target)) - return false; - switch (g->valgrind_support) { - case ValgrindSupportDisabled: - return false; - case ValgrindSupportEnabled: - return true; - case ValgrindSupportAuto: - return g->build_mode == BuildModeDebug; - } - zig_unreachable(); -} - static void gen_valgrind_undef(CodeGen *g, LLVMValueRef dest_ptr, LLVMValueRef byte_count) { static const uint32_t VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545; ZigType *usize = g->builtin_types.entry_usize; @@ -3895,7 +3812,7 @@ static void gen_undef_init(CodeGen *g, uint32_t ptr_align_bytes, ZigType *value_ LLVMValueRef byte_count = LLVMConstInt(usize->llvm_type, size_bytes, false); ZigLLVMBuildMemSet(g->builder, dest_ptr, fill_char, byte_count, ptr_align_bytes, false); // then tell valgrind that the memory is undefined even though we just memset it - if (want_valgrind_support(g)) { + if (g->valgrind_enabled) { gen_valgrind_undef(g, dest_ptr, byte_count); } } @@ -5598,7 +5515,7 @@ static LLVMValueRef ir_render_memset(CodeGen *g, IrExecutableGen *executable, Ir ZigLLVMBuildMemSet(g->builder, dest_ptr_casted, fill_char, len_val, get_ptr_align(g, ptr_type), ptr_type->data.pointer.is_volatile); - if (val_is_undef && want_valgrind_support(g)) { + if (val_is_undef && g->valgrind_enabled) { gen_valgrind_undef(g, dest_ptr_casted, len_val); } return nullptr; @@ -8350,13 +8267,6 @@ static void zig_llvm_emit_output(CodeGen *g) { exit(1); } - if (g->emit_bin) { - g->link_objects.append(&g->o_file_output_path); - if (g->bundle_compiler_rt && (g->out_type == OutTypeObj || (g->out_type == OutTypeLib && !g->is_dynamic))) { - zig_link_add_compiler_rt(g, g->sub_progress_node); - } - } - LLVMDisposeModule(g->module); g->module = nullptr; LLVMDisposeTargetData(g->target_data_ref); @@ -8751,75 +8661,12 @@ static const char *subsystem_to_str(TargetSubsystem subsystem) { zig_unreachable(); } -static bool detect_dynamic_link(CodeGen *g) { - if (g->is_dynamic) - return true; - if (g->zig_target->os == OsFreestanding) - return false; - if (target_os_requires_libc(g->zig_target->os)) - return true; - if (g->libc_link_lib != nullptr && target_is_glibc(g->zig_target)) - return true; - // If there are no dynamic libraries then we can disable dynamic linking. - for (size_t i = 0; i < g->link_libs_list.length; i += 1) { - LinkLib *link_lib = g->link_libs_list.at(i); - if (target_is_libc_lib_name(g->zig_target, buf_ptr(link_lib->name))) - continue; - if (target_is_libcpp_lib_name(g->zig_target, buf_ptr(link_lib->name))) - continue; - return true; - } - return false; -} - -static bool detect_pic(CodeGen *g) { - if (target_requires_pic(g->zig_target, g->libc_link_lib != nullptr)) - return true; - switch (g->want_pic) { - case WantPICDisabled: - return false; - case WantPICEnabled: - return true; - case WantPICAuto: - return g->have_dynamic_link; - } - zig_unreachable(); -} - -static bool detect_stack_probing(CodeGen *g) { - if (!target_supports_stack_probing(g->zig_target)) - return false; - switch (g->want_stack_check) { - case WantStackCheckDisabled: - return false; - case WantStackCheckEnabled: - return true; - case WantStackCheckAuto: - return g->build_mode == BuildModeSafeRelease || g->build_mode == BuildModeDebug; - } - zig_unreachable(); -} - -static bool detect_sanitize_c(CodeGen *g) { - if (!target_supports_sanitize_c(g->zig_target)) - return false; - switch (g->want_sanitize_c) { - case WantCSanitizeDisabled: - return false; - case WantCSanitizeEnabled: - return true; - case WantCSanitizeAuto: - return g->build_mode == BuildModeSafeRelease || g->build_mode == BuildModeDebug; - } - zig_unreachable(); -} - // Returns TargetSubsystemAuto to mean "no subsystem" TargetSubsystem detect_subsystem(CodeGen *g) { if (g->subsystem != TargetSubsystemAuto) return g->subsystem; if (g->zig_target->os == OsWindows) { - if (g->have_dllmain_crt_startup || (g->out_type == OutTypeLib && g->is_dynamic)) + if (g->have_dllmain_crt_startup) return TargetSubsystemAuto; if (g->have_c_main || g->is_test_build || g->have_winmain_crt_startup || g->have_wwinmain_crt_startup) return TargetSubsystemConsole; @@ -8831,15 +8678,6 @@ TargetSubsystem detect_subsystem(CodeGen *g) { return TargetSubsystemAuto; } -static bool detect_single_threaded(CodeGen *g) { - if (g->want_single_threaded) - return true; - if (target_is_single_threaded(g->zig_target)) { - return true; - } - return false; -} - static bool detect_err_ret_tracing(CodeGen *g) { return !g->strip_debug_symbols && g->build_mode != BuildModeFastRelease && @@ -8847,30 +8685,30 @@ static bool detect_err_ret_tracing(CodeGen *g) { } static LLVMCodeModel to_llvm_code_model(CodeGen *g) { - switch (g->code_model) { - case CodeModelDefault: - return LLVMCodeModelDefault; - case CodeModelTiny: - return LLVMCodeModelTiny; - case CodeModelSmall: - return LLVMCodeModelSmall; - case CodeModelKernel: - return LLVMCodeModelKernel; - case CodeModelMedium: - return LLVMCodeModelMedium; - case CodeModelLarge: - return LLVMCodeModelLarge; - } + switch (g->code_model) { + case CodeModelDefault: + return LLVMCodeModelDefault; + case CodeModelTiny: + return LLVMCodeModelTiny; + case CodeModelSmall: + return LLVMCodeModelSmall; + case CodeModelKernel: + return LLVMCodeModelKernel; + case CodeModelMedium: + return LLVMCodeModelMedium; + case CodeModelLarge: + return LLVMCodeModelLarge; + } - zig_unreachable(); + zig_unreachable(); } Buf *codegen_generate_builtin_source(CodeGen *g) { - g->have_dynamic_link = detect_dynamic_link(g); - g->have_pic = detect_pic(g); - g->have_stack_probing = detect_stack_probing(g); - g->have_sanitize_c = detect_sanitize_c(g); - g->is_single_threaded = detect_single_threaded(g); + // Note that this only runs when zig0 is building the self-hosted zig compiler code, + // so it makes a few assumption that are always true for that case. Once we have + // built the stage2 zig components then zig is in charge of generating the builtin.zig + // file. + g->have_err_ret_tracing = detect_err_ret_tracing(g); Buf *contents = buf_alloc(); @@ -8884,7 +8722,6 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { const char *name = target_os_name(os_type); if (os_type == g->zig_target->os) { - g->target_os_index = i; cur_os = name; } } @@ -8898,7 +8735,6 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { ZigLLVM_ArchType arch = target_arch_enum(arch_i); const char *arch_name = target_arch_name(arch); if (arch == g->zig_target->arch) { - g->target_arch_index = arch_i; cur_arch = arch_name; } } @@ -8913,7 +8749,6 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { const char *name = target_abi_name(abi); if (abi == g->zig_target->abi) { - g->target_abi_index = i; cur_abi = name; } } @@ -8929,7 +8764,6 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { ZigLLVM_ObjectFormatType target_oformat = target_object_format(g->zig_target); if (oformat == target_oformat) { - g->target_oformat_index = i; cur_obj_fmt = name; } } @@ -8979,23 +8813,9 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { const char *endian_str = g->is_big_endian ? "Endian.Big" : "Endian.Little"; buf_appendf(contents, "pub const endian = %s;\n", endian_str); } - const char *out_type = nullptr; - switch (g->out_type) { - case OutTypeExe: - out_type = "Exe"; - break; - case OutTypeLib: - out_type = "Lib"; - break; - case OutTypeObj: - case OutTypeUnknown: // This happens when running the `zig builtin` command. - out_type = "Obj"; - break; - } - buf_appendf(contents, "pub const output_mode = OutputMode.%s;\n", out_type); - const char *link_type = g->have_dynamic_link ? "Dynamic" : "Static"; - buf_appendf(contents, "pub const link_mode = LinkMode.%s;\n", link_type); - buf_appendf(contents, "pub const is_test = %s;\n", bool_to_str(g->is_test_build)); + buf_appendf(contents, "pub const output_mode = OutputMode.Obj;\n"); + buf_appendf(contents, "pub const link_mode = LinkMode.Static;\n"); + buf_appendf(contents, "pub const is_test = false;\n"); buf_appendf(contents, "pub const single_threaded = %s;\n", bool_to_str(g->is_single_threaded)); buf_append_str(contents, "/// Deprecated: use `std.Target.cpu.arch`\n"); buf_appendf(contents, "pub const arch = Arch.%s;\n", cur_arch); @@ -9018,40 +8838,13 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { } buf_appendf(contents, "pub const object_format = ObjectFormat.%s;\n", cur_obj_fmt); buf_appendf(contents, "pub const mode = %s;\n", build_mode_to_str(g->build_mode)); - buf_appendf(contents, "pub const link_libc = %s;\n", bool_to_str(g->libc_link_lib != nullptr)); - buf_appendf(contents, "pub const link_libcpp = %s;\n", bool_to_str(g->libcpp_link_lib != nullptr)); + buf_appendf(contents, "pub const link_libc = %s;\n", bool_to_str(g->link_libc)); + buf_appendf(contents, "pub const link_libcpp = %s;\n", bool_to_str(g->link_libcpp)); buf_appendf(contents, "pub const have_error_return_tracing = %s;\n", bool_to_str(g->have_err_ret_tracing)); - buf_appendf(contents, "pub const valgrind_support = %s;\n", bool_to_str(want_valgrind_support(g))); + buf_appendf(contents, "pub const valgrind_support = false;\n"); buf_appendf(contents, "pub const position_independent_code = %s;\n", bool_to_str(g->have_pic)); buf_appendf(contents, "pub const strip_debug_info = %s;\n", bool_to_str(g->strip_debug_symbols)); - - { - const char *code_model; - switch (g->code_model) { - case CodeModelDefault: - code_model = "default"; - break; - case CodeModelTiny: - code_model = "tiny"; - break; - case CodeModelSmall: - code_model = "small"; - break; - case CodeModelKernel: - code_model = "kernel"; - break; - case CodeModelMedium: - code_model = "medium"; - break; - case CodeModelLarge: - code_model = "large"; - break; - default: - zig_unreachable(); - } - - buf_appendf(contents, "pub const code_model = CodeModel.%s;\n", code_model); - } + buf_appendf(contents, "pub const code_model = CodeModel.default;\n"); { TargetSubsystem detected_subsystem = detect_subsystem(g); @@ -9060,15 +8853,6 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { } } - if (g->is_test_build) { - buf_appendf(contents, - "pub var test_functions: []TestFn = undefined; // overwritten later\n" - ); - - buf_appendf(contents, "pub const test_io_mode = %s;\n", - g->test_is_evented ? ".evented" : ".blocking"); - } - return contents; } @@ -9077,95 +8861,35 @@ static ZigPackage *create_test_runner_pkg(CodeGen *g) { } static Error define_builtin_compile_vars(CodeGen *g) { + Error err; + if (g->std_package == nullptr) return ErrorNone; - Error err; - - Buf *manifest_dir = buf_alloc(); - os_path_join(get_global_cache_dir(), buf_create_from_str("builtin"), manifest_dir); - - CacheHash cache_hash; - cache_init(&cache_hash, manifest_dir); - - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) - return err; - - // Only a few things affect builtin.zig - cache_buf(&cache_hash, compiler_id); - cache_int(&cache_hash, g->build_mode); - cache_bool(&cache_hash, g->strip_debug_symbols); - cache_int(&cache_hash, g->out_type); - cache_bool(&cache_hash, detect_dynamic_link(g)); - cache_bool(&cache_hash, g->is_test_build); - cache_bool(&cache_hash, g->is_single_threaded); - cache_bool(&cache_hash, g->test_is_evented); - cache_int(&cache_hash, g->code_model); - cache_int(&cache_hash, g->zig_target->is_native_os); - cache_int(&cache_hash, g->zig_target->is_native_cpu); - cache_int(&cache_hash, g->zig_target->arch); - cache_int(&cache_hash, g->zig_target->vendor); - cache_int(&cache_hash, g->zig_target->os); - cache_int(&cache_hash, g->zig_target->abi); - if (g->zig_target->cache_hash != nullptr) { - cache_mem(&cache_hash, g->zig_target->cache_hash, g->zig_target->cache_hash_len); - } - if (g->zig_target->glibc_or_darwin_version != nullptr) { - cache_int(&cache_hash, g->zig_target->glibc_or_darwin_version->major); - cache_int(&cache_hash, g->zig_target->glibc_or_darwin_version->minor); - cache_int(&cache_hash, g->zig_target->glibc_or_darwin_version->patch); - } - cache_bool(&cache_hash, g->have_err_ret_tracing); - cache_bool(&cache_hash, g->libc_link_lib != nullptr); - cache_bool(&cache_hash, g->libcpp_link_lib != nullptr); - cache_bool(&cache_hash, g->valgrind_support); - cache_bool(&cache_hash, g->link_eh_frame_hdr); - cache_int(&cache_hash, detect_subsystem(g)); - - Buf digest = BUF_INIT; - buf_resize(&digest, 0); - if ((err = cache_hit(&cache_hash, &digest))) { - // Treat an invalid format error as a cache miss. - if (err != ErrorInvalidFormat) - return err; - } - - // We should always get a cache hit because there are no - // files in the input hash. - assert(buf_len(&digest) != 0); - - Buf *this_dir = buf_alloc(); - os_path_join(manifest_dir, &digest, this_dir); - - if ((err = os_make_path(this_dir))) - return err; - const char *builtin_zig_basename = "builtin.zig"; - Buf *builtin_zig_path = buf_alloc(); - os_path_join(this_dir, buf_create_from_str(builtin_zig_basename), builtin_zig_path); - bool hit; - if ((err = os_file_exists(builtin_zig_path, &hit))) - return err; Buf *contents; - if (hit) { - contents = buf_alloc(); - if ((err = os_fetch_file_path(builtin_zig_path, contents))) { - fprintf(stderr, "Unable to open '%s': %s\n", buf_ptr(builtin_zig_path), err_str(err)); + if (g->builtin_zig_path == nullptr) { + // Then this is zig0 building stage2. We can make many assumptions about the compilation. + g->builtin_zig_path = buf_alloc(); + os_path_join(g->output_dir, buf_create_from_str(builtin_zig_basename), g->builtin_zig_path); + + contents = codegen_generate_builtin_source(g); + if ((err = os_write_file(g->builtin_zig_path, contents))) { + fprintf(stderr, "Unable to write file '%s': %s\n", buf_ptr(g->builtin_zig_path), err_str(err)); exit(1); } } else { - contents = codegen_generate_builtin_source(g); - if ((err = os_write_file(builtin_zig_path, contents))) { - fprintf(stderr, "Unable to write file '%s': %s\n", buf_ptr(builtin_zig_path), err_str(err)); + contents = buf_alloc(); + if ((err = os_fetch_file_path(g->builtin_zig_path, contents))) { + fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(g->builtin_zig_path), err_str(err)); exit(1); } } assert(g->main_pkg); assert(g->std_package); - g->compile_var_package = new_package(buf_ptr(this_dir), builtin_zig_basename, "builtin"); + g->compile_var_package = new_package(buf_ptr(g->output_dir), builtin_zig_basename, "builtin"); if (g->is_test_build) { if (g->test_runner_package == nullptr) { g->test_runner_package = create_test_runner_pkg(g); @@ -9180,7 +8904,7 @@ static Error define_builtin_compile_vars(CodeGen *g) { g->std_package->package_table.put(buf_create_from_str("builtin"), g->compile_var_package); g->std_package->package_table.put(buf_create_from_str("std"), g->std_package); g->std_package->package_table.put(buf_create_from_str("root"), g->root_pkg); - g->compile_var_import = add_source_file(g, g->compile_var_package, builtin_zig_path, contents, + g->compile_var_import = add_source_file(g, g->compile_var_package, g->builtin_zig_path, contents, SourceKindPkgMain); return ErrorNone; @@ -9190,17 +8914,8 @@ static void init(CodeGen *g) { if (g->module) return; - g->have_dynamic_link = detect_dynamic_link(g); - g->have_pic = detect_pic(g); - g->have_stack_probing = detect_stack_probing(g); - g->have_sanitize_c = detect_sanitize_c(g); - g->is_single_threaded = detect_single_threaded(g); g->have_err_ret_tracing = detect_err_ret_tracing(g); - if (target_is_single_threaded(g->zig_target)) { - g->is_single_threaded = true; - } - assert(g->root_out_name); g->module = LLVMModuleCreateWithName(buf_ptr(g->root_out_name)); @@ -9230,7 +8945,7 @@ static void init(CodeGen *g) { LLVMRelocMode reloc_mode; if (g->have_pic) { reloc_mode = LLVMRelocPIC; - } else if (g->have_dynamic_link) { + } else if (g->link_mode_dynamic) { reloc_mode = LLVMRelocDynamicNoPic; } else { reloc_mode = LLVMRelocStatic; @@ -9336,446 +9051,6 @@ static void init(CodeGen *g) { } } -static void detect_libc(CodeGen *g) { - Error err; - - if (g->libc != nullptr || g->libc_link_lib == nullptr) - return; - - if (target_can_build_libc(g->zig_target)) { - const char *generic_name = target_libc_generic_name(g->zig_target); - const char *arch_name = target_arch_name(g->zig_target->arch); - const char *abi_name = target_abi_name(g->zig_target->abi); - if (target_is_musl(g->zig_target)) { - // musl has some overrides. its headers are ABI-agnostic and so they all have the "musl" ABI name. - abi_name = "musl"; - // some architectures are handled by the same set of headers - arch_name = target_arch_musl_name(g->zig_target->arch); - } - Buf *arch_include_dir = buf_sprintf("%s" OS_SEP "libc" OS_SEP "include" OS_SEP "%s-%s-%s", - buf_ptr(g->zig_lib_dir), arch_name, target_os_name(g->zig_target->os), abi_name); - Buf *generic_include_dir = buf_sprintf("%s" OS_SEP "libc" OS_SEP "include" OS_SEP "generic-%s", - buf_ptr(g->zig_lib_dir), generic_name); - Buf *arch_os_include_dir = buf_sprintf("%s" OS_SEP "libc" OS_SEP "include" OS_SEP "%s-%s-any", - buf_ptr(g->zig_lib_dir), target_arch_name(g->zig_target->arch), target_os_name(g->zig_target->os)); - Buf *generic_os_include_dir = buf_sprintf("%s" OS_SEP "libc" OS_SEP "include" OS_SEP "any-%s-any", - buf_ptr(g->zig_lib_dir), target_os_name(g->zig_target->os)); - - g->libc_include_dir_len = 4; - g->libc_include_dir_list = heap::c_allocator.allocate(g->libc_include_dir_len); - g->libc_include_dir_list[0] = buf_ptr(arch_include_dir); - g->libc_include_dir_list[1] = buf_ptr(generic_include_dir); - g->libc_include_dir_list[2] = buf_ptr(arch_os_include_dir); - g->libc_include_dir_list[3] = buf_ptr(generic_os_include_dir); - return; - } - - if (g->zig_target->is_native_os) { - g->libc = heap::c_allocator.create(); - - if ((err = stage2_libc_find_native(g->libc))) { - fprintf(stderr, - "Unable to link against libc: Unable to find libc installation: %s\n" - "See `zig libc --help` for more details.\n", err_str(err)); - exit(1); - } - - bool want_sys_dir = !mem_eql_mem(g->libc->include_dir, g->libc->include_dir_len, - g->libc->sys_include_dir, g->libc->sys_include_dir_len); - size_t want_um_and_shared_dirs = (g->zig_target->os == OsWindows) ? 2 : 0; - size_t dir_count = 1 + want_sys_dir + want_um_and_shared_dirs; - g->libc_include_dir_len = 0; - g->libc_include_dir_list = heap::c_allocator.allocate(dir_count); - - g->libc_include_dir_list[g->libc_include_dir_len] = buf_ptr(buf_create_from_mem( - g->libc->include_dir, g->libc->include_dir_len)); - g->libc_include_dir_len += 1; - - if (want_sys_dir) { - g->libc_include_dir_list[g->libc_include_dir_len] = buf_ptr(buf_create_from_mem( - g->libc->sys_include_dir, g->libc->sys_include_dir_len)); - g->libc_include_dir_len += 1; - } - - if (want_um_and_shared_dirs != 0) { - Buf *include_dir_parent = buf_alloc(); - os_path_join(buf_create_from_mem(g->libc->include_dir, g->libc->include_dir_len), - buf_create_from_str(".."), include_dir_parent); - - Buf *buff1 = buf_alloc(); - os_path_join(include_dir_parent, buf_create_from_str("um"), buff1); - g->libc_include_dir_list[g->libc_include_dir_len] = buf_ptr(buff1); - g->libc_include_dir_len += 1; - - Buf *buff2 = buf_alloc(); - os_path_join(include_dir_parent, buf_create_from_str("shared"), buff2); - g->libc_include_dir_list[g->libc_include_dir_len] = buf_ptr(buff2); - g->libc_include_dir_len += 1; - } - assert(g->libc_include_dir_len == dir_count); - } else if ((g->out_type == OutTypeExe || (g->out_type == OutTypeLib && g->is_dynamic)) && - !target_os_is_darwin(g->zig_target->os)) - { - Buf triple_buf = BUF_INIT; - target_triple_zig(&triple_buf, g->zig_target); - fprintf(stderr, - "Zig is unable to provide a libc for the chosen target '%s'.\n" - "The target is non-native, so Zig also cannot use the native libc installation.\n" - "Choose a target which has a libc available (see `zig targets`), or\n" - "provide a libc installation text file (see `zig libc --help`).\n", buf_ptr(&triple_buf)); - exit(1); - } -} - -// does not add the "cc" arg -void add_cc_args(CodeGen *g, ZigList &args, const char *out_dep_path, - bool translate_c, FileExt source_kind) -{ - if (translate_c) { - args.append("-x"); - args.append("c"); - } - - args.append("-nostdinc"); - if (source_kind == FileExtCpp) { - args.append("-nostdinc++"); - } - args.append("-fno-spell-checking"); - - if (g->function_sections) { - args.append("-ffunction-sections"); - } - - if (!translate_c) { - switch (g->err_color) { - case ErrColorAuto: - break; - case ErrColorOff: - args.append("-fno-color-diagnostics"); - args.append("-fno-caret-diagnostics"); - break; - case ErrColorOn: - args.append("-fcolor-diagnostics"); - args.append("-fcaret-diagnostics"); - break; - } - } - - for (size_t i = 0; i < g->framework_dirs.length; i += 1) { - args.append("-iframework"); - args.append(g->framework_dirs.at(i)); - } - - if (g->libcpp_link_lib != nullptr) { - const char *libcxx_include_path = buf_ptr(buf_sprintf("%s" OS_SEP "libcxx" OS_SEP "include", - buf_ptr(g->zig_lib_dir))); - - const char *libcxxabi_include_path = buf_ptr(buf_sprintf("%s" OS_SEP "libcxxabi" OS_SEP "include", - buf_ptr(g->zig_lib_dir))); - - args.append("-isystem"); - args.append(libcxx_include_path); - - args.append("-isystem"); - args.append(libcxxabi_include_path); - - if (target_abi_is_musl(g->zig_target->abi)) { - args.append("-D_LIBCPP_HAS_MUSL_LIBC"); - } - args.append("-D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS"); - args.append("-D_LIBCXXABI_DISABLE_VISIBILITY_ANNOTATIONS"); - } - - args.append("-target"); - args.append(buf_ptr(&g->llvm_triple_str)); - - switch (source_kind) { - case FileExtC: - case FileExtCpp: - case FileExtHeader: - // According to Rich Felker libc headers are supposed to go before C language headers. - // However as noted by @dimenus, appending libc headers before c_headers breaks intrinsics - // and other compiler specific items. - args.append("-isystem"); - args.append(buf_ptr(g->zig_c_headers_dir)); - - for (size_t i = 0; i < g->libc_include_dir_len; i += 1) { - const char *include_dir = g->libc_include_dir_list[i]; - args.append("-isystem"); - args.append(include_dir); - } - - if (g->zig_target->llvm_cpu_name != nullptr) { - args.append("-Xclang"); - args.append("-target-cpu"); - args.append("-Xclang"); - args.append(g->zig_target->llvm_cpu_name); - } - if (g->zig_target->llvm_cpu_features != nullptr) { - // https://github.com/ziglang/zig/issues/5017 - SplitIterator it = memSplit(str(g->zig_target->llvm_cpu_features), str(",")); - Optional> flag = SplitIterator_next(&it); - while (flag.is_some) { - args.append("-Xclang"); - args.append("-target-feature"); - args.append("-Xclang"); - args.append(buf_ptr(buf_create_from_slice(flag.value))); - flag = SplitIterator_next(&it); - } - } - if (translate_c) { - // this gives us access to preprocessing entities, presumably at - // the cost of performance - args.append("-Xclang"); - args.append("-detailed-preprocessing-record"); - } - if (out_dep_path != nullptr) { - args.append("-MD"); - args.append("-MV"); - args.append("-MF"); - args.append(out_dep_path); - } - break; - case FileExtAsm: - case FileExtLLVMIr: - case FileExtLLVMBitCode: - case FileExtUnknown: - break; - } - for (size_t i = 0; i < g->zig_target->llvm_cpu_features_asm_len; i += 1) { - args.append(g->zig_target->llvm_cpu_features_asm_ptr[i]); - } - - if (g->zig_target->os == OsFreestanding) { - args.append("-ffreestanding"); - } - - // windows.h has files such as pshpack1.h which do #pragma packing, triggering a clang warning. - // So for this target, we disable this warning. - if (g->zig_target->os == OsWindows && target_abi_is_gnu(g->zig_target->abi)) { - args.append("-Wno-pragma-pack"); - } - - if (!g->strip_debug_symbols) { - args.append("-g"); - } - - if (codegen_have_frame_pointer(g)) { - args.append("-fno-omit-frame-pointer"); - } else { - args.append("-fomit-frame-pointer"); - } - - if (g->have_sanitize_c) { - args.append("-fsanitize=undefined"); - args.append("-fsanitize-trap=undefined"); - } - - switch (g->build_mode) { - case BuildModeDebug: - // windows c runtime requires -D_DEBUG if using debug libraries - args.append("-D_DEBUG"); - args.append("-Og"); - - if (g->libc_link_lib != nullptr) { - args.append("-fstack-protector-strong"); - args.append("--param"); - args.append("ssp-buffer-size=4"); - } else { - args.append("-fno-stack-protector"); - } - break; - case BuildModeSafeRelease: - // See the comment in the BuildModeFastRelease case for why we pass -O2 rather - // than -O3 here. - args.append("-O2"); - if (g->libc_link_lib != nullptr) { - args.append("-D_FORTIFY_SOURCE=2"); - args.append("-fstack-protector-strong"); - args.append("--param"); - args.append("ssp-buffer-size=4"); - } else { - args.append("-fno-stack-protector"); - } - break; - case BuildModeFastRelease: - args.append("-DNDEBUG"); - // Here we pass -O2 rather than -O3 because, although we do the equivalent of - // -O3 in Zig code, the justification for the difference here is that Zig - // has better detection and prevention of undefined behavior, so -O3 is safer for - // Zig code than it is for C code. Also, C programmers are used to their code - // running in -O2 and thus the -O3 path has been tested less. - args.append("-O2"); - args.append("-fno-stack-protector"); - break; - case BuildModeSmallRelease: - args.append("-DNDEBUG"); - args.append("-Os"); - args.append("-fno-stack-protector"); - break; - } - - if (target_supports_fpic(g->zig_target) && g->have_pic) { - args.append("-fPIC"); - } - - for (size_t arg_i = 0; arg_i < g->clang_argv_len; arg_i += 1) { - args.append(g->clang_argv[arg_i]); - } - -} - -void codegen_translate_c(CodeGen *g, Buf *full_path) { - Error err; - - Buf *src_basename = buf_alloc(); - Buf *src_dirname = buf_alloc(); - os_path_split(full_path, src_dirname, src_basename); - - Buf noextname = BUF_INIT; - os_path_extname(src_basename, &noextname, nullptr); - - Buf *zig_basename = buf_sprintf("%s.zig", buf_ptr(&noextname)); - - detect_libc(g); - - Buf cache_digest = BUF_INIT; - buf_resize(&cache_digest, 0); - - CacheHash *cache_hash = nullptr; - if (g->enable_cache) { - if ((err = create_c_object_cache(g, &cache_hash, true))) { - // Already printed error; verbose = true - exit(1); - } - cache_file(cache_hash, full_path); - // to distinguish from generating a C object - cache_buf(cache_hash, buf_create_from_str("translate-c")); - - if ((err = cache_hit(cache_hash, &cache_digest))) { - if (err != ErrorInvalidFormat) { - fprintf(stderr, "unable to check cache: %s\n", err_str(err)); - exit(1); - } - } - if (cache_hash->manifest_file_path != nullptr) { - g->caches_to_release.append(cache_hash); - } - } - - if (g->enable_cache && buf_len(&cache_digest) != 0) { - // cache hit - Buf *cached_path = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR OS_SEP "%s" OS_SEP "%s", - buf_ptr(g->cache_dir), buf_ptr(&cache_digest), buf_ptr(zig_basename)); - fprintf(stdout, "%s\n", buf_ptr(cached_path)); - return; - } - - // cache miss or cache disabled - init(g); - - Buf *out_dep_path = nullptr; - const char *out_dep_path_cstr = nullptr; - - if (g->enable_cache) { - buf_alloc();// we can't know the digest until we do the C compiler invocation, so we - // need a tmp filename. - out_dep_path = buf_alloc(); - if ((err = get_tmp_filename(g, out_dep_path, buf_sprintf("%s.d", buf_ptr(zig_basename))))) { - fprintf(stderr, "unable to create tmp dir: %s\n", err_str(err)); - exit(1); - } - out_dep_path_cstr = buf_ptr(out_dep_path); - } - - ZigList clang_argv = {0}; - add_cc_args(g, clang_argv, out_dep_path_cstr, true, FileExtC); - - clang_argv.append(buf_ptr(full_path)); - - if (g->verbose_cc) { - fprintf(stderr, "clang"); - for (size_t i = 0; i < clang_argv.length; i += 1) { - fprintf(stderr, " %s", clang_argv.at(i)); - } - fprintf(stderr, "\n"); - } - - clang_argv.append(nullptr); // to make the [start...end] argument work - - const char *resources_path = buf_ptr(g->zig_c_headers_dir); - Stage2ErrorMsg *errors_ptr; - size_t errors_len; - Stage2Ast *ast; - - err = stage2_translate_c(&ast, &errors_ptr, &errors_len, - &clang_argv.at(0), &clang_argv.last(), resources_path); - - if (err == ErrorCCompileErrors && errors_len > 0) { - for (size_t i = 0; i < errors_len; i += 1) { - Stage2ErrorMsg *clang_err = &errors_ptr[i]; - - ErrorMsg *err_msg = err_msg_create_with_offset( - clang_err->filename_ptr ? - buf_create_from_mem(clang_err->filename_ptr, clang_err->filename_len) : nullptr, - clang_err->line, clang_err->column, clang_err->offset, clang_err->source, - buf_create_from_mem(clang_err->msg_ptr, clang_err->msg_len)); - print_err_msg(err_msg, g->err_color); - } - exit(1); - } - - if (err) { - fprintf(stderr, "unable to parse C file: %s\n", err_str(err)); - exit(1); - } - - if (!g->enable_cache) { - stage2_render_ast(ast, stdout); - return; - } - - // add the files depended on to the cache system - if ((err = cache_add_dep_file(cache_hash, out_dep_path, true))) { - // Don't treat the absence of the .d file as a fatal error, the - // compiler may not produce one eg. when compiling .s files - if (err != ErrorFileNotFound) { - fprintf(stderr, "Failed to add C source dependencies to cache: %s\n", err_str(err)); - exit(1); - } - } - if (err != ErrorFileNotFound) { - os_delete_file(out_dep_path); - } - - if ((err = cache_final(cache_hash, &cache_digest))) { - fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); - exit(1); - } - - Buf *artifact_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR OS_SEP "%s", - buf_ptr(g->cache_dir), buf_ptr(&cache_digest)); - - if ((err = os_make_path(artifact_dir))) { - fprintf(stderr, "Unable to make dir: %s\n", err_str(err)); - exit(1); - } - - Buf *cached_path = buf_sprintf("%s" OS_SEP "%s", buf_ptr(artifact_dir), buf_ptr(zig_basename)); - - FILE *out_file = fopen(buf_ptr(cached_path), "wb"); - if (out_file == nullptr) { - fprintf(stderr, "Unable to open output file: %s\n", strerror(errno)); - exit(1); - } - stage2_render_ast(ast, out_file); - if (fclose(out_file) != 0) { - fprintf(stderr, "Unable to write to output file: %s\n", strerror(errno)); - exit(1); - } - fprintf(stdout, "%s\n", buf_ptr(cached_path)); -} - static void update_test_functions_builtin_decl(CodeGen *g) { Error err; @@ -9875,49 +9150,46 @@ static void gen_root_source(CodeGen *g) { assert(root_import_alias == g->root_import); assert(g->root_out_name); - assert(g->out_type != OutTypeUnknown); - if (!g->is_dummy_so) { - // Zig has lazy top level definitions. Here we semantically analyze the panic function. - Buf *import_target_path; - Buf full_path = BUF_INIT; - ZigType *std_import; - if ((err = analyze_import(g, g->root_import, buf_create_from_str("std"), &std_import, - &import_target_path, &full_path))) - { - if (err == ErrorFileNotFound) { - fprintf(stderr, "unable to find '%s'", buf_ptr(import_target_path)); - } else { - fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(&full_path), err_str(err)); - } - exit(1); + // Zig has lazy top level definitions. Here we semantically analyze the panic function. + Buf *import_target_path; + Buf full_path = BUF_INIT; + ZigType *std_import; + if ((err = analyze_import(g, g->root_import, buf_create_from_str("std"), &std_import, + &import_target_path, &full_path))) + { + if (err == ErrorFileNotFound) { + fprintf(stderr, "unable to find '%s'", buf_ptr(import_target_path)); + } else { + fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(&full_path), err_str(err)); } - - Tld *builtin_tld = find_decl(g, &get_container_scope(std_import)->base, - buf_create_from_str("builtin")); - assert(builtin_tld != nullptr); - resolve_top_level_decl(g, builtin_tld, nullptr, false); - report_errors_and_maybe_exit(g); - assert(builtin_tld->id == TldIdVar); - TldVar *builtin_tld_var = (TldVar*)builtin_tld; - ZigValue *builtin_val = builtin_tld_var->var->const_value; - assert(builtin_val->type->id == ZigTypeIdMetaType); - ZigType *builtin_type = builtin_val->data.x_type; - - Tld *panic_tld = find_decl(g, &get_container_scope(builtin_type)->base, - buf_create_from_str("panic")); - assert(panic_tld != nullptr); - resolve_top_level_decl(g, panic_tld, nullptr, false); - report_errors_and_maybe_exit(g); - assert(panic_tld->id == TldIdVar); - TldVar *panic_tld_var = (TldVar*)panic_tld; - ZigValue *panic_fn_val = panic_tld_var->var->const_value; - assert(panic_fn_val->type->id == ZigTypeIdFn); - assert(panic_fn_val->data.x_ptr.special == ConstPtrSpecialFunction); - g->panic_fn = panic_fn_val->data.x_ptr.data.fn.fn_entry; - assert(g->panic_fn != nullptr); + exit(1); } + Tld *builtin_tld = find_decl(g, &get_container_scope(std_import)->base, + buf_create_from_str("builtin")); + assert(builtin_tld != nullptr); + resolve_top_level_decl(g, builtin_tld, nullptr, false); + report_errors_and_maybe_exit(g); + assert(builtin_tld->id == TldIdVar); + TldVar *builtin_tld_var = (TldVar*)builtin_tld; + ZigValue *builtin_val = builtin_tld_var->var->const_value; + assert(builtin_val->type->id == ZigTypeIdMetaType); + ZigType *builtin_type = builtin_val->data.x_type; + + Tld *panic_tld = find_decl(g, &get_container_scope(builtin_type)->base, + buf_create_from_str("panic")); + assert(panic_tld != nullptr); + resolve_top_level_decl(g, panic_tld, nullptr, false); + report_errors_and_maybe_exit(g); + assert(panic_tld->id == TldIdVar); + TldVar *panic_tld_var = (TldVar*)panic_tld; + ZigValue *panic_fn_val = panic_tld_var->var->const_value; + assert(panic_fn_val->type->id == ZigTypeIdFn); + assert(panic_fn_val->data.x_ptr.special == ConstPtrSpecialFunction); + g->panic_fn = panic_fn_val->data.x_ptr.data.fn.fn_entry; + assert(g->panic_fn != nullptr); + if (!g->error_during_imports) { semantic_analyze(g); } @@ -9934,781 +9206,6 @@ static void gen_root_source(CodeGen *g) { } -static void print_zig_cc_cmd(ZigList *args) { - for (size_t arg_i = 0; arg_i < args->length; arg_i += 1) { - const char *space_str = (arg_i == 0) ? "" : " "; - fprintf(stderr, "%s%s", space_str, args->at(arg_i)); - } - fprintf(stderr, "\n"); -} - -// Caller should delete the file when done or rename it into a better location. -static Error get_tmp_filename(CodeGen *g, Buf *out, Buf *suffix) { - Error err; - buf_resize(out, 0); - os_path_join(g->cache_dir, buf_create_from_str("tmp" OS_SEP), out); - if ((err = os_make_path(out))) { - return err; - } - const char base64[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"; - assert(array_length(base64) == 64 + 1); - for (size_t i = 0; i < 12; i += 1) { - buf_append_char(out, base64[rand() % 64]); - } - buf_append_char(out, '-'); - buf_append_buf(out, suffix); - return ErrorNone; -} - -Error create_c_object_cache(CodeGen *g, CacheHash **out_cache_hash, bool verbose) { - Error err; - CacheHash *cache_hash = heap::c_allocator.create(); - Buf *manifest_dir = buf_sprintf("%s" OS_SEP CACHE_HASH_SUBDIR, buf_ptr(g->cache_dir)); - cache_init(cache_hash, manifest_dir); - - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) { - if (verbose) { - fprintf(stderr, "unable to get compiler id: %s\n", err_str(err)); - } - return err; - } - cache_buf(cache_hash, compiler_id); - cache_int(cache_hash, g->err_color); - cache_list_of_str(cache_hash, g->framework_dirs.items, g->framework_dirs.length); - cache_bool(cache_hash, g->libcpp_link_lib != nullptr); - cache_buf(cache_hash, g->zig_lib_dir); - cache_buf(cache_hash, g->zig_c_headers_dir); - cache_list_of_str(cache_hash, g->libc_include_dir_list, g->libc_include_dir_len); - cache_int(cache_hash, g->zig_target->is_native_os); - cache_int(cache_hash, g->zig_target->is_native_cpu); - cache_int(cache_hash, g->zig_target->arch); - cache_int(cache_hash, g->zig_target->vendor); - cache_int(cache_hash, g->zig_target->os); - cache_int(cache_hash, g->zig_target->abi); - cache_bool(cache_hash, g->strip_debug_symbols); - cache_int(cache_hash, g->build_mode); - cache_bool(cache_hash, g->have_pic); - cache_bool(cache_hash, g->have_sanitize_c); - cache_bool(cache_hash, want_valgrind_support(g)); - cache_bool(cache_hash, g->function_sections); - cache_int(cache_hash, g->code_model); - cache_bool(cache_hash, codegen_have_frame_pointer(g)); - cache_bool(cache_hash, g->libc_link_lib); - if (g->zig_target->cache_hash != nullptr) { - cache_mem(cache_hash, g->zig_target->cache_hash, g->zig_target->cache_hash_len); - } - - for (size_t arg_i = 0; arg_i < g->clang_argv_len; arg_i += 1) { - cache_str(cache_hash, g->clang_argv[arg_i]); - } - - *out_cache_hash = cache_hash; - return ErrorNone; -} - -static bool need_llvm_module(CodeGen *g) { - return buf_len(&g->main_pkg->root_src_path) != 0; -} - -// before gen_c_objects -static bool main_output_dir_is_just_one_c_object_pre(CodeGen *g) { - return g->enable_cache && g->c_source_files.length == 1 && !need_llvm_module(g) && - g->out_type == OutTypeObj && g->link_objects.length == 0; -} - -// after gen_c_objects -static bool main_output_dir_is_just_one_c_object_post(CodeGen *g) { - return g->enable_cache && g->link_objects.length == 1 && !need_llvm_module(g) && g->out_type == OutTypeObj; -} - -// returns true if it was a cache miss -static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) { - Error err; - - Buf *artifact_dir; - Buf *o_final_path; - - Buf *o_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR, buf_ptr(g->cache_dir)); - - Buf *c_source_file = buf_create_from_str(c_file->source_path); - Buf *c_source_basename = buf_alloc(); - os_path_split(c_source_file, nullptr, c_source_basename); - - Stage2ProgressNode *child_prog_node = stage2_progress_start(g->sub_progress_node, buf_ptr(c_source_basename), - buf_len(c_source_basename), 0); - - Buf *final_o_basename = buf_alloc(); - if (c_file->preprocessor_only_basename == nullptr) { - // We special case when doing build-obj for just one C file - if (main_output_dir_is_just_one_c_object_pre(g)) { - buf_init_from_buf(final_o_basename, g->root_out_name); - } else { - os_path_extname(c_source_basename, final_o_basename, nullptr); - } - buf_append_str(final_o_basename, target_o_file_ext(g->zig_target)); - } else { - buf_init_from_str(final_o_basename, c_file->preprocessor_only_basename); - } - - CacheHash *cache_hash; - if ((err = create_c_object_cache(g, &cache_hash, true))) { - // Already printed error; verbose = true - exit(1); - } - cache_file(cache_hash, c_source_file); - - // Note: not directory args, just args that always have a file next - static const char *file_args[] = { - "-include", - }; - for (size_t arg_i = 0; arg_i < c_file->args.length; arg_i += 1) { - const char *arg = c_file->args.at(arg_i); - cache_str(cache_hash, arg); - for (size_t file_arg_i = 0; file_arg_i < array_length(file_args); file_arg_i += 1) { - if (strcmp(arg, file_args[file_arg_i]) == 0 && arg_i + 1 < c_file->args.length) { - arg_i += 1; - cache_file(cache_hash, buf_create_from_str(c_file->args.at(arg_i))); - } - } - } - - Buf digest = BUF_INIT; - buf_resize(&digest, 0); - if ((err = cache_hit(cache_hash, &digest))) { - if (err != ErrorInvalidFormat) { - if (err == ErrorCacheUnavailable) { - // already printed error - } else { - fprintf(stderr, "unable to check cache when compiling C object: %s\n", err_str(err)); - } - exit(1); - } - } - bool is_cache_miss = g->disable_c_depfile || (buf_len(&digest) == 0); - if (is_cache_miss) { - // we can't know the digest until we do the C compiler invocation, so we - // need a tmp filename. - Buf *out_obj_path = buf_alloc(); - if ((err = get_tmp_filename(g, out_obj_path, final_o_basename))) { - fprintf(stderr, "unable to create tmp dir: %s\n", err_str(err)); - exit(1); - } - - Termination term; - ZigList args = {}; - args.append(buf_ptr(self_exe_path)); - args.append("clang"); - - if (c_file->preprocessor_only_basename == nullptr) { - args.append("-c"); - } - - Buf *out_dep_path = g->disable_c_depfile ? nullptr : buf_sprintf("%s.d", buf_ptr(out_obj_path)); - const char *out_dep_path_cstr = (out_dep_path == nullptr) ? nullptr : buf_ptr(out_dep_path); - FileExt ext = classify_file_ext(buf_ptr(c_source_basename), buf_len(c_source_basename)); - add_cc_args(g, args, out_dep_path_cstr, false, ext); - - args.append("-o"); - args.append(buf_ptr(out_obj_path)); - - args.append(buf_ptr(c_source_file)); - - for (size_t arg_i = 0; arg_i < c_file->args.length; arg_i += 1) { - args.append(c_file->args.at(arg_i)); - } - - if (g->verbose_cc) { - print_zig_cc_cmd(&args); - } - os_spawn_process(args, &term); - if (term.how != TerminationIdClean || term.code != 0) { - fprintf(stderr, "\nThe following command failed:\n"); - print_zig_cc_cmd(&args); - exit(1); - } - - if (out_dep_path != nullptr) { - // add the files depended on to the cache system - if ((err = cache_add_dep_file(cache_hash, out_dep_path, true))) { - // Don't treat the absence of the .d file as a fatal error, the - // compiler may not produce one eg. when compiling .s files - if (err != ErrorFileNotFound) { - fprintf(stderr, "Failed to add C source dependencies to cache: %s\n", err_str(err)); - exit(1); - } - } - if (err != ErrorFileNotFound) { - os_delete_file(out_dep_path); - } - - if ((err = cache_final(cache_hash, &digest))) { - fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); - exit(1); - } - } - artifact_dir = buf_alloc(); - os_path_join(o_dir, &digest, artifact_dir); - if ((err = os_make_path(artifact_dir))) { - fprintf(stderr, "Unable to create output directory '%s': %s", - buf_ptr(artifact_dir), err_str(err)); - exit(1); - } - o_final_path = buf_alloc(); - os_path_join(artifact_dir, final_o_basename, o_final_path); - if ((err = os_rename(out_obj_path, o_final_path))) { - fprintf(stderr, "Unable to rename object: %s\n", err_str(err)); - exit(1); - } - } else { - // cache hit - artifact_dir = buf_alloc(); - os_path_join(o_dir, &digest, artifact_dir); - o_final_path = buf_alloc(); - os_path_join(artifact_dir, final_o_basename, o_final_path); - } - - g->c_artifact_dir = artifact_dir; - g->link_objects.append(o_final_path); - g->caches_to_release.append(cache_hash); - - stage2_progress_end(child_prog_node); -} - -// returns true if we had any cache misses -static void gen_c_objects(CodeGen *g) { - Error err; - - if (g->c_source_files.length == 0) - return; - - Buf *self_exe_path = buf_alloc(); - if ((err = os_self_exe_path(self_exe_path))) { - fprintf(stderr, "Unable to get self exe path: %s\n", err_str(err)); - exit(1); - } - - codegen_add_time_event(g, "Compile C Objects"); - const char *c_prog_name = "Compile C Objects"; - codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, c_prog_name, strlen(c_prog_name), - g->c_source_files.length)); - - for (size_t c_file_i = 0; c_file_i < g->c_source_files.length; c_file_i += 1) { - CFile *c_file = g->c_source_files.at(c_file_i); - gen_c_object(g, self_exe_path, c_file); - } -} - -void codegen_add_object(CodeGen *g, Buf *object_path) { - g->link_objects.append(object_path); -} - -// Must be coordinated with with CIntType enum -static const char *c_int_type_names[] = { - "short", - "unsigned short", - "int", - "unsigned int", - "long", - "unsigned long", - "long long", - "unsigned long long", -}; - -struct GenH { - ZigList types_to_declare; -}; - -static void prepend_c_type_to_decl_list(CodeGen *g, GenH *gen_h, ZigType *type_entry) { - if (type_entry->gen_h_loop_flag) - return; - type_entry->gen_h_loop_flag = true; - - switch (type_entry->id) { - case ZigTypeIdInvalid: - case ZigTypeIdMetaType: - case ZigTypeIdComptimeFloat: - case ZigTypeIdComptimeInt: - case ZigTypeIdEnumLiteral: - case ZigTypeIdUndefined: - case ZigTypeIdNull: - case ZigTypeIdBoundFn: - case ZigTypeIdErrorUnion: - case ZigTypeIdErrorSet: - case ZigTypeIdFnFrame: - case ZigTypeIdAnyFrame: - zig_unreachable(); - case ZigTypeIdVoid: - case ZigTypeIdUnreachable: - return; - case ZigTypeIdBool: - g->c_want_stdbool = true; - return; - case ZigTypeIdInt: - g->c_want_stdint = true; - return; - case ZigTypeIdFloat: - return; - case ZigTypeIdOpaque: - gen_h->types_to_declare.append(type_entry); - return; - case ZigTypeIdStruct: - if(type_entry->data.structure.layout == ContainerLayoutExtern) { - for (uint32_t i = 0; i < type_entry->data.structure.src_field_count; i += 1) { - TypeStructField *field = type_entry->data.structure.fields[i]; - prepend_c_type_to_decl_list(g, gen_h, field->type_entry); - } - } - gen_h->types_to_declare.append(type_entry); - return; - case ZigTypeIdUnion: - for (uint32_t i = 0; i < type_entry->data.unionation.src_field_count; i += 1) { - TypeUnionField *field = &type_entry->data.unionation.fields[i]; - prepend_c_type_to_decl_list(g, gen_h, field->type_entry); - } - gen_h->types_to_declare.append(type_entry); - return; - case ZigTypeIdEnum: - prepend_c_type_to_decl_list(g, gen_h, type_entry->data.enumeration.tag_int_type); - gen_h->types_to_declare.append(type_entry); - return; - case ZigTypeIdPointer: - prepend_c_type_to_decl_list(g, gen_h, type_entry->data.pointer.child_type); - return; - case ZigTypeIdArray: - prepend_c_type_to_decl_list(g, gen_h, type_entry->data.array.child_type); - return; - case ZigTypeIdVector: - prepend_c_type_to_decl_list(g, gen_h, type_entry->data.vector.elem_type); - return; - case ZigTypeIdOptional: - prepend_c_type_to_decl_list(g, gen_h, type_entry->data.maybe.child_type); - return; - case ZigTypeIdFn: - for (size_t i = 0; i < type_entry->data.fn.fn_type_id.param_count; i += 1) { - prepend_c_type_to_decl_list(g, gen_h, type_entry->data.fn.fn_type_id.param_info[i].type); - } - prepend_c_type_to_decl_list(g, gen_h, type_entry->data.fn.fn_type_id.return_type); - return; - } -} - -static void get_c_type(CodeGen *g, GenH *gen_h, ZigType *type_entry, Buf *out_buf) { - assert(type_entry); - - for (size_t i = 0; i < array_length(c_int_type_names); i += 1) { - if (type_entry == g->builtin_types.entry_c_int[i]) { - buf_init_from_str(out_buf, c_int_type_names[i]); - return; - } - } - if (type_entry == g->builtin_types.entry_c_longdouble) { - buf_init_from_str(out_buf, "long double"); - return; - } - if (type_entry == g->builtin_types.entry_c_void) { - buf_init_from_str(out_buf, "void"); - return; - } - if (type_entry == g->builtin_types.entry_isize) { - g->c_want_stdint = true; - buf_init_from_str(out_buf, "intptr_t"); - return; - } - if (type_entry == g->builtin_types.entry_usize) { - g->c_want_stdint = true; - buf_init_from_str(out_buf, "uintptr_t"); - return; - } - - prepend_c_type_to_decl_list(g, gen_h, type_entry); - - switch (type_entry->id) { - case ZigTypeIdVoid: - buf_init_from_str(out_buf, "void"); - break; - case ZigTypeIdBool: - buf_init_from_str(out_buf, "bool"); - break; - case ZigTypeIdUnreachable: - buf_init_from_str(out_buf, "__attribute__((__noreturn__)) void"); - break; - case ZigTypeIdFloat: - switch (type_entry->data.floating.bit_count) { - case 32: - buf_init_from_str(out_buf, "float"); - break; - case 64: - buf_init_from_str(out_buf, "double"); - break; - case 80: - buf_init_from_str(out_buf, "__float80"); - break; - case 128: - buf_init_from_str(out_buf, "__float128"); - break; - default: - zig_unreachable(); - } - break; - case ZigTypeIdInt: - buf_resize(out_buf, 0); - buf_appendf(out_buf, "%sint%" PRIu32 "_t", - type_entry->data.integral.is_signed ? "" : "u", - type_entry->data.integral.bit_count); - break; - case ZigTypeIdPointer: - { - Buf child_buf = BUF_INIT; - ZigType *child_type = type_entry->data.pointer.child_type; - get_c_type(g, gen_h, child_type, &child_buf); - - const char *const_str = type_entry->data.pointer.is_const ? "const " : ""; - buf_resize(out_buf, 0); - buf_appendf(out_buf, "%s%s *", const_str, buf_ptr(&child_buf)); - break; - } - case ZigTypeIdOptional: - { - ZigType *child_type = type_entry->data.maybe.child_type; - if (!type_has_bits(g, child_type)) { - buf_init_from_str(out_buf, "bool"); - return; - } else if (type_is_nonnull_ptr(g, child_type)) { - return get_c_type(g, gen_h, child_type, out_buf); - } else { - zig_unreachable(); - } - } - case ZigTypeIdStruct: - case ZigTypeIdOpaque: - { - buf_init_from_str(out_buf, "struct "); - buf_append_buf(out_buf, type_h_name(type_entry)); - return; - } - case ZigTypeIdUnion: - { - buf_init_from_str(out_buf, "union "); - buf_append_buf(out_buf, type_h_name(type_entry)); - return; - } - case ZigTypeIdEnum: - { - buf_init_from_str(out_buf, "enum "); - buf_append_buf(out_buf, type_h_name(type_entry)); - return; - } - case ZigTypeIdArray: - { - ZigTypeArray *array_data = &type_entry->data.array; - - Buf *child_buf = buf_alloc(); - get_c_type(g, gen_h, array_data->child_type, child_buf); - - buf_resize(out_buf, 0); - buf_appendf(out_buf, "%s", buf_ptr(child_buf)); - return; - } - case ZigTypeIdVector: - zig_panic("TODO implement get_c_type for vector types"); - case ZigTypeIdErrorUnion: - case ZigTypeIdErrorSet: - case ZigTypeIdFn: - zig_panic("TODO implement get_c_type for more types"); - case ZigTypeIdInvalid: - case ZigTypeIdMetaType: - case ZigTypeIdBoundFn: - case ZigTypeIdComptimeFloat: - case ZigTypeIdComptimeInt: - case ZigTypeIdEnumLiteral: - case ZigTypeIdUndefined: - case ZigTypeIdNull: - case ZigTypeIdFnFrame: - case ZigTypeIdAnyFrame: - zig_unreachable(); - } -} - -static const char *preprocessor_alphabet1 = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; -static const char *preprocessor_alphabet2 = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - -static bool need_to_preprocessor_mangle(Buf *src) { - for (size_t i = 0; i < buf_len(src); i += 1) { - const char *alphabet = (i == 0) ? preprocessor_alphabet1 : preprocessor_alphabet2; - uint8_t byte = buf_ptr(src)[i]; - if (strchr(alphabet, byte) == nullptr) { - return true; - } - } - return false; -} - -static Buf *preprocessor_mangle(Buf *src) { - if (!need_to_preprocessor_mangle(src)) { - return buf_create_from_buf(src); - } - Buf *result = buf_alloc(); - for (size_t i = 0; i < buf_len(src); i += 1) { - const char *alphabet = (i == 0) ? preprocessor_alphabet1 : preprocessor_alphabet2; - uint8_t byte = buf_ptr(src)[i]; - if (strchr(alphabet, byte) == nullptr) { - // perform escape - buf_appendf(result, "_%02x_", byte); - } else { - buf_append_char(result, byte); - } - } - return result; -} - -static void gen_h_file_types(CodeGen* g, GenH* gen_h, Buf* out_buf) { - for (size_t type_i = 0; type_i < gen_h->types_to_declare.length; type_i += 1) { - ZigType *type_entry = gen_h->types_to_declare.at(type_i); - switch (type_entry->id) { - case ZigTypeIdInvalid: - case ZigTypeIdMetaType: - case ZigTypeIdVoid: - case ZigTypeIdBool: - case ZigTypeIdUnreachable: - case ZigTypeIdInt: - case ZigTypeIdFloat: - case ZigTypeIdPointer: - case ZigTypeIdComptimeFloat: - case ZigTypeIdComptimeInt: - case ZigTypeIdEnumLiteral: - case ZigTypeIdArray: - case ZigTypeIdUndefined: - case ZigTypeIdNull: - case ZigTypeIdErrorUnion: - case ZigTypeIdErrorSet: - case ZigTypeIdBoundFn: - case ZigTypeIdOptional: - case ZigTypeIdFn: - case ZigTypeIdVector: - case ZigTypeIdFnFrame: - case ZigTypeIdAnyFrame: - zig_unreachable(); - - case ZigTypeIdEnum: - if (type_entry->data.enumeration.layout == ContainerLayoutExtern) { - buf_appendf(out_buf, "enum %s {\n", buf_ptr(type_h_name(type_entry))); - for (uint32_t field_i = 0; field_i < type_entry->data.enumeration.src_field_count; field_i += 1) { - TypeEnumField *enum_field = &type_entry->data.enumeration.fields[field_i]; - Buf *value_buf = buf_alloc(); - bigint_append_buf(value_buf, &enum_field->value, 10); - buf_appendf(out_buf, " %s = %s", buf_ptr(enum_field->name), buf_ptr(value_buf)); - if (field_i != type_entry->data.enumeration.src_field_count - 1) { - buf_appendf(out_buf, ","); - } - buf_appendf(out_buf, "\n"); - } - buf_appendf(out_buf, "};\n\n"); - } else { - buf_appendf(out_buf, "enum %s;\n\n", buf_ptr(type_h_name(type_entry))); - } - break; - case ZigTypeIdStruct: - if (type_entry->data.structure.layout == ContainerLayoutExtern) { - buf_appendf(out_buf, "struct %s {\n", buf_ptr(type_h_name(type_entry))); - for (uint32_t field_i = 0; field_i < type_entry->data.structure.src_field_count; field_i += 1) { - TypeStructField *struct_field = type_entry->data.structure.fields[field_i]; - - Buf *type_name_buf = buf_alloc(); - get_c_type(g, gen_h, struct_field->type_entry, type_name_buf); - - if (struct_field->type_entry->id == ZigTypeIdArray) { - buf_appendf(out_buf, " %s %s[%" ZIG_PRI_u64 "];\n", buf_ptr(type_name_buf), - buf_ptr(struct_field->name), - struct_field->type_entry->data.array.len); - } else { - buf_appendf(out_buf, " %s %s;\n", buf_ptr(type_name_buf), buf_ptr(struct_field->name)); - } - - } - buf_appendf(out_buf, "};\n\n"); - } else { - buf_appendf(out_buf, "struct %s;\n\n", buf_ptr(type_h_name(type_entry))); - } - break; - case ZigTypeIdUnion: - if (type_entry->data.unionation.layout == ContainerLayoutExtern) { - buf_appendf(out_buf, "union %s {\n", buf_ptr(type_h_name(type_entry))); - for (uint32_t field_i = 0; field_i < type_entry->data.unionation.src_field_count; field_i += 1) { - TypeUnionField *union_field = &type_entry->data.unionation.fields[field_i]; - - Buf *type_name_buf = buf_alloc(); - get_c_type(g, gen_h, union_field->type_entry, type_name_buf); - buf_appendf(out_buf, " %s %s;\n", buf_ptr(type_name_buf), buf_ptr(union_field->name)); - } - buf_appendf(out_buf, "};\n\n"); - } else { - buf_appendf(out_buf, "union %s;\n\n", buf_ptr(type_h_name(type_entry))); - } - break; - case ZigTypeIdOpaque: - buf_appendf(out_buf, "struct %s;\n\n", buf_ptr(type_h_name(type_entry))); - break; - } - } -} - -static void gen_h_file_functions(CodeGen* g, GenH* gen_h, Buf* out_buf, Buf* export_macro) { - for (size_t fn_def_i = 0; fn_def_i < g->fn_defs.length; fn_def_i += 1) { - ZigFn *fn_table_entry = g->fn_defs.at(fn_def_i); - - if (fn_table_entry->export_list.length == 0) - continue; - - FnTypeId *fn_type_id = &fn_table_entry->type_entry->data.fn.fn_type_id; - - Buf return_type_c = BUF_INIT; - get_c_type(g, gen_h, fn_type_id->return_type, &return_type_c); - - Buf *symbol_name; - if (fn_table_entry->export_list.length == 0) { - symbol_name = &fn_table_entry->symbol_name; - } else { - GlobalExport *fn_export = &fn_table_entry->export_list.items[0]; - symbol_name = &fn_export->name; - } - - if (export_macro != nullptr) { - buf_appendf(out_buf, "%s %s %s(", - buf_ptr(export_macro), - buf_ptr(&return_type_c), - buf_ptr(symbol_name)); - } else { - buf_appendf(out_buf, "%s %s(", - buf_ptr(&return_type_c), - buf_ptr(symbol_name)); - } - - Buf param_type_c = BUF_INIT; - if (fn_type_id->param_count > 0) { - for (size_t param_i = 0; param_i < fn_type_id->param_count; param_i += 1) { - FnTypeParamInfo *param_info = &fn_type_id->param_info[param_i]; - AstNode *param_decl_node = get_param_decl_node(fn_table_entry, param_i); - Buf *param_name = param_decl_node->data.param_decl.name; - - const char *comma_str = (param_i == 0) ? "" : ", "; - const char *restrict_str = param_info->is_noalias ? "restrict" : ""; - get_c_type(g, gen_h, param_info->type, ¶m_type_c); - - if (param_info->type->id == ZigTypeIdArray) { - // Arrays decay to pointers - buf_appendf(out_buf, "%s%s%s %s[]", comma_str, buf_ptr(¶m_type_c), - restrict_str, buf_ptr(param_name)); - } else { - buf_appendf(out_buf, "%s%s%s %s", comma_str, buf_ptr(¶m_type_c), - restrict_str, buf_ptr(param_name)); - } - } - buf_appendf(out_buf, ")"); - } else { - buf_appendf(out_buf, "void)"); - } - - buf_appendf(out_buf, ";\n"); - } -} - -static void gen_h_file_variables(CodeGen* g, GenH* gen_h, Buf* h_buf, Buf* export_macro) { - for (size_t exp_var_i = 0; exp_var_i < g->global_vars.length; exp_var_i += 1) { - ZigVar* var = g->global_vars.at(exp_var_i)->var; - if (var->export_list.length == 0) - continue; - - Buf var_type_c = BUF_INIT; - get_c_type(g, gen_h, var->var_type, &var_type_c); - - if (export_macro != nullptr) { - buf_appendf(h_buf, "extern %s %s %s;\n", - buf_ptr(export_macro), - buf_ptr(&var_type_c), - var->name); - } else { - buf_appendf(h_buf, "extern %s %s;\n", - buf_ptr(&var_type_c), - var->name); - } - } -} - -static void gen_h_file(CodeGen *g) { - GenH gen_h_data = {0}; - GenH *gen_h = &gen_h_data; - - assert(!g->is_test_build); - assert(!g->disable_gen_h); - - Buf *out_h_path = buf_sprintf("%s" OS_SEP "%s.h", buf_ptr(g->output_dir), buf_ptr(g->root_out_name)); - - FILE *out_h = fopen(buf_ptr(out_h_path), "wb"); - if (!out_h) - zig_panic("unable to open %s: %s\n", buf_ptr(out_h_path), strerror(errno)); - - Buf *export_macro = nullptr; - if (g->is_dynamic) { - export_macro = preprocessor_mangle(buf_sprintf("%s_EXPORT", buf_ptr(g->root_out_name))); - buf_upcase(export_macro); - } - - Buf fns_buf = BUF_INIT; - buf_resize(&fns_buf, 0); - gen_h_file_functions(g, gen_h, &fns_buf, export_macro); - - Buf vars_buf = BUF_INIT; - buf_resize(&vars_buf, 0); - gen_h_file_variables(g, gen_h, &vars_buf, export_macro); - - // Types will be populated by exported functions and variables so it has to run last. - Buf types_buf = BUF_INIT; - buf_resize(&types_buf, 0); - gen_h_file_types(g, gen_h, &types_buf); - - Buf *ifdef_dance_name = preprocessor_mangle(buf_sprintf("%s_H", buf_ptr(g->root_out_name))); - buf_upcase(ifdef_dance_name); - - fprintf(out_h, "#ifndef %s\n", buf_ptr(ifdef_dance_name)); - fprintf(out_h, "#define %s\n\n", buf_ptr(ifdef_dance_name)); - - if (g->c_want_stdbool) - fprintf(out_h, "#include \n"); - if (g->c_want_stdint) - fprintf(out_h, "#include \n"); - - fprintf(out_h, "\n"); - - if (g->is_dynamic) { - fprintf(out_h, "#if defined(_WIN32)\n"); - fprintf(out_h, "#define %s __declspec(dllimport)\n", buf_ptr(export_macro)); - fprintf(out_h, "#else\n"); - fprintf(out_h, "#define %s __attribute__((visibility (\"default\")))\n", - buf_ptr(export_macro)); - fprintf(out_h, "#endif\n"); - fprintf(out_h, "\n"); - } - - fprintf(out_h, "#ifdef __cplusplus\n"); - fprintf(out_h, "extern \"C\" {\n"); - fprintf(out_h, "#endif\n"); - fprintf(out_h, "\n"); - - fprintf(out_h, "%s", buf_ptr(&types_buf)); - fprintf(out_h, "%s\n", buf_ptr(&fns_buf)); - fprintf(out_h, "%s\n", buf_ptr(&vars_buf)); - - fprintf(out_h, "#ifdef __cplusplus\n"); - fprintf(out_h, "} // extern \"C\"\n"); - fprintf(out_h, "#endif\n\n"); - - fprintf(out_h, "#endif // %s\n", buf_ptr(ifdef_dance_name)); - - if (fclose(out_h)) - zig_panic("unable to close h file: %s", strerror(errno)); -} - void codegen_print_timing_report(CodeGen *g, FILE *f) { double start_time = g->timing_events.at(0).time; double end_time = g->timing_events.last().time; @@ -10733,174 +9230,14 @@ void codegen_add_time_event(CodeGen *g, const char *name) { g->timing_events.append({seconds, name}); } -static void add_cache_pkg(CodeGen *g, CacheHash *ch, ZigPackage *pkg) { - if (buf_len(&pkg->root_src_path) == 0) - return; - pkg->added_to_cache = true; - - Buf *rel_full_path = buf_alloc(); - os_path_join(&pkg->root_src_dir, &pkg->root_src_path, rel_full_path); - cache_file(ch, rel_full_path); - - auto it = pkg->package_table.entry_iterator(); - for (;;) { - auto *entry = it.next(); - if (!entry) - break; - - if (!pkg->added_to_cache) { - cache_buf(ch, entry->key); - add_cache_pkg(g, ch, entry->value); - } - } -} - -// Called before init() -// is_cache_hit takes into account gen_c_objects -static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { - Error err; - - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) - return err; - - CacheHash *ch = &g->cache_hash; - cache_init(ch, manifest_dir); - - add_cache_pkg(g, ch, g->main_pkg); - if (g->linker_script != nullptr) { - cache_file(ch, buf_create_from_str(g->linker_script)); - } - cache_buf(ch, compiler_id); - cache_buf(ch, g->root_out_name); - cache_buf(ch, g->zig_lib_dir); - cache_buf(ch, g->zig_std_dir); - cache_list_of_link_lib(ch, g->link_libs_list.items, g->link_libs_list.length); - cache_list_of_buf(ch, g->darwin_frameworks.items, g->darwin_frameworks.length); - cache_list_of_buf(ch, g->rpath_list.items, g->rpath_list.length); - cache_list_of_buf(ch, g->forbidden_libs.items, g->forbidden_libs.length); - cache_int(ch, g->build_mode); - cache_int(ch, g->out_type); - cache_bool(ch, g->zig_target->is_native_os); - cache_bool(ch, g->zig_target->is_native_cpu); - cache_int(ch, g->zig_target->arch); - cache_int(ch, g->zig_target->vendor); - cache_int(ch, g->zig_target->os); - cache_int(ch, g->zig_target->abi); - if (g->zig_target->cache_hash != nullptr) { - cache_mem(ch, g->zig_target->cache_hash, g->zig_target->cache_hash_len); - } - if (g->zig_target->glibc_or_darwin_version != nullptr) { - cache_int(ch, g->zig_target->glibc_or_darwin_version->major); - cache_int(ch, g->zig_target->glibc_or_darwin_version->minor); - cache_int(ch, g->zig_target->glibc_or_darwin_version->patch); - } - if (g->zig_target->dynamic_linker != nullptr) { - cache_str(ch, g->zig_target->dynamic_linker); - } - cache_int(ch, detect_subsystem(g)); - cache_bool(ch, g->strip_debug_symbols); - cache_bool(ch, g->is_test_build); - if (g->is_test_build) { - cache_buf_opt(ch, g->test_filter); - cache_buf_opt(ch, g->test_name_prefix); - cache_bool(ch, g->test_is_evented); - } - cache_bool(ch, g->link_eh_frame_hdr); - cache_bool(ch, g->is_single_threaded); - cache_bool(ch, g->linker_rdynamic); - cache_bool(ch, g->each_lib_rpath); - cache_bool(ch, g->disable_gen_h); - cache_bool(ch, g->bundle_compiler_rt); - cache_bool(ch, want_valgrind_support(g)); - cache_bool(ch, g->have_pic); - cache_bool(ch, g->have_dynamic_link); - cache_bool(ch, g->have_stack_probing); - cache_bool(ch, g->have_sanitize_c); - cache_bool(ch, g->is_dummy_so); - cache_bool(ch, g->function_sections); - cache_bool(ch, g->enable_dump_analysis); - cache_bool(ch, g->enable_doc_generation); - cache_bool(ch, g->emit_bin); - cache_bool(ch, g->emit_llvm_ir); - cache_bool(ch, g->emit_asm); - cache_bool(ch, g->is_versioned); - cache_usize(ch, g->version_major); - cache_usize(ch, g->version_minor); - cache_usize(ch, g->version_patch); - cache_list_of_str(ch, g->llvm_argv, g->llvm_argv_len); - cache_list_of_str(ch, g->clang_argv, g->clang_argv_len); - cache_list_of_str(ch, g->lib_dirs.items, g->lib_dirs.length); - cache_list_of_str(ch, g->framework_dirs.items, g->framework_dirs.length); - if (g->libc) { - cache_slice(ch, Slice{g->libc->include_dir, g->libc->include_dir_len}); - cache_slice(ch, Slice{g->libc->sys_include_dir, g->libc->sys_include_dir_len}); - cache_slice(ch, Slice{g->libc->crt_dir, g->libc->crt_dir_len}); - cache_slice(ch, Slice{g->libc->msvc_lib_dir, g->libc->msvc_lib_dir_len}); - cache_slice(ch, Slice{g->libc->kernel32_lib_dir, g->libc->kernel32_lib_dir_len}); - } - cache_buf_opt(ch, g->version_script_path); - cache_buf_opt(ch, g->override_soname); - cache_buf_opt(ch, g->linker_optimization); - cache_int(ch, g->linker_gc_sections); - cache_int(ch, g->linker_allow_shlib_undefined); - cache_int(ch, g->linker_bind_global_refs_locally); - cache_bool(ch, g->linker_z_nodelete); - cache_bool(ch, g->linker_z_defs); - cache_usize(ch, g->stack_size_override); - - // gen_c_objects appends objects to g->link_objects which we want to include in the hash - gen_c_objects(g); - cache_list_of_file(ch, g->link_objects.items, g->link_objects.length); - - buf_resize(digest, 0); - if ((err = cache_hit(ch, digest))) { - if (err != ErrorInvalidFormat) - return err; - } - - if (ch->manifest_file_path != nullptr) { - g->caches_to_release.append(ch); - } - - return ErrorNone; -} - static void resolve_out_paths(CodeGen *g) { assert(g->output_dir != nullptr); assert(g->root_out_name != nullptr); if (g->emit_bin) { - Buf *out_basename = buf_create_from_buf(g->root_out_name); Buf *o_basename = buf_create_from_buf(g->root_out_name); - switch (g->out_type) { - case OutTypeUnknown: - zig_unreachable(); - case OutTypeObj: - if (need_llvm_module(g) && g->link_objects.length != 0 && !g->enable_cache && - buf_eql_buf(o_basename, out_basename)) - { - // make it not collide with main output object - buf_append_str(o_basename, ".root"); - } - buf_append_str(o_basename, target_o_file_ext(g->zig_target)); - buf_append_str(out_basename, target_o_file_ext(g->zig_target)); - break; - case OutTypeExe: - buf_append_str(o_basename, target_o_file_ext(g->zig_target)); - buf_append_str(out_basename, target_exe_file_ext(g->zig_target)); - break; - case OutTypeLib: - buf_append_str(o_basename, target_o_file_ext(g->zig_target)); - buf_resize(out_basename, 0); - buf_append_str(out_basename, target_lib_file_prefix(g->zig_target)); - buf_append_buf(out_basename, g->root_out_name); - buf_append_str(out_basename, target_lib_file_ext(g->zig_target, !g->is_dynamic, g->is_versioned, - g->version_major, g->version_minor, g->version_patch)); - break; - } + buf_append_str(o_basename, target_o_file_ext(g->zig_target)); os_path_join(g->output_dir, o_basename, &g->o_file_output_path); - os_path_join(g->output_dir, out_basename, &g->bin_file_output_path); } if (g->emit_asm) { Buf *asm_basename = buf_create_from_buf(g->root_out_name); @@ -10971,144 +9308,46 @@ static void output_type_information(CodeGen *g) { } } -static void init_output_dir(CodeGen *g, Buf *digest) { - if (main_output_dir_is_just_one_c_object_post(g)) { - g->output_dir = buf_alloc(); - os_path_dirname(g->link_objects.at(0), g->output_dir); - } else { - g->output_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR OS_SEP "%s", - buf_ptr(g->cache_dir), buf_ptr(digest)); - } -} +void codegen_build_object(CodeGen *g) { + assert(g->output_dir != nullptr); -void codegen_build_and_link(CodeGen *g) { - Error err; - assert(g->out_type != OutTypeUnknown); - - if (!g->enable_cache) { - if (g->output_dir == nullptr) { - g->output_dir = buf_create_from_str("."); - } else if ((err = os_make_path(g->output_dir))) { - fprintf(stderr, "Unable to create output directory: %s\n", err_str(err)); - exit(1); - } - } - - g->have_dynamic_link = detect_dynamic_link(g); - g->have_pic = detect_pic(g); - g->is_single_threaded = detect_single_threaded(g); g->have_err_ret_tracing = detect_err_ret_tracing(g); - g->have_sanitize_c = detect_sanitize_c(g); - detect_libc(g); - Buf digest = BUF_INIT; - if (g->enable_cache) { - Buf *manifest_dir = buf_alloc(); - os_path_join(g->cache_dir, buf_create_from_str(CACHE_HASH_SUBDIR), manifest_dir); + init(g); - if ((err = check_cache(g, manifest_dir, &digest))) { - if (err == ErrorCacheUnavailable) { - // message already printed - } else if (err == ErrorNotDir) { - fprintf(stderr, "Unable to check cache: %s is not a directory\n", - buf_ptr(manifest_dir)); - } else { - fprintf(stderr, "Unable to check cache: %s: %s\n", buf_ptr(manifest_dir), err_str(err)); - } - exit(1); - } - } else { - // There is a call to this in check_cache - gen_c_objects(g); + codegen_add_time_event(g, "Semantic Analysis"); + const char *progress_name = "Semantic Analysis"; + codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, + progress_name, strlen(progress_name), 0)); + + gen_root_source(g); + + resolve_out_paths(g); + + if (g->enable_dump_analysis || g->enable_doc_generation) { + output_type_information(g); } - if (g->enable_cache && buf_len(&digest) != 0) { - init_output_dir(g, &digest); - resolve_out_paths(g); - } else { - if (need_llvm_module(g)) { - init(g); - - codegen_add_time_event(g, "Semantic Analysis"); - const char *progress_name = "Semantic Analysis"; - codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, - progress_name, strlen(progress_name), 0)); - - gen_root_source(g); - - } - if (g->enable_cache) { - if (buf_len(&digest) == 0) { - if ((err = cache_final(&g->cache_hash, &digest))) { - fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); - exit(1); - } - } - init_output_dir(g, &digest); - - if ((err = os_make_path(g->output_dir))) { - fprintf(stderr, "Unable to create output directory: %s\n", err_str(err)); - exit(1); - } - } - resolve_out_paths(g); - - if (g->enable_dump_analysis || g->enable_doc_generation) { - output_type_information(g); - } - - if (need_llvm_module(g)) { - codegen_add_time_event(g, "Code Generation"); - { - const char *progress_name = "Code Generation"; - codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, - progress_name, strlen(progress_name), 0)); - } - - do_code_gen(g); - codegen_add_time_event(g, "LLVM Emit Output"); - { - const char *progress_name = "LLVM Emit Output"; - codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, - progress_name, strlen(progress_name), 0)); - } - zig_llvm_emit_output(g); - - if (!g->disable_gen_h && (g->out_type == OutTypeObj || g->out_type == OutTypeLib)) { - codegen_add_time_event(g, "Generate .h"); - { - const char *progress_name = "Generate .h"; - codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, - progress_name, strlen(progress_name), 0)); - } - gen_h_file(g); - } - } - - // If we're outputting assembly or llvm IR we skip linking. - // If we're making a library or executable we must link. - // If there is more than one object, we have to link them (with -r). - // Finally, if we didn't make an object from zig source, and we don't have caching enabled, - // then we have an object from C source that we must copy to the output dir which we do with a -r link. - if (g->emit_bin && - (g->out_type != OutTypeObj || g->link_objects.length > 1 || - (!need_llvm_module(g) && !g->enable_cache))) - { - codegen_link(g); - } + codegen_add_time_event(g, "Code Generation"); + { + const char *progress_name = "Code Generation"; + codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, + progress_name, strlen(progress_name), 0)); } - codegen_release_caches(g); + do_code_gen(g); + codegen_add_time_event(g, "LLVM Emit Output"); + { + const char *progress_name = "LLVM Emit Output"; + codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, + progress_name, strlen(progress_name), 0)); + } + zig_llvm_emit_output(g); + codegen_add_time_event(g, "Done"); codegen_switch_sub_prog_node(g, nullptr); } -void codegen_release_caches(CodeGen *g) { - while (g->caches_to_release.length != 0) { - cache_release(g->caches_to_release.pop()); - } -} - ZigPackage *codegen_create_package(CodeGen *g, const char *root_src_dir, const char *root_src_path, const char *pkg_path) { @@ -11125,43 +9364,17 @@ ZigPackage *codegen_create_package(CodeGen *g, const char *root_src_dir, const c return pkg; } -CodeGen *create_child_codegen(CodeGen *parent_gen, Buf *root_src_path, OutType out_type, - Stage2LibCInstallation *libc, const char *name, Stage2ProgressNode *parent_progress_node) -{ - Stage2ProgressNode *child_progress_node = stage2_progress_start( - parent_progress_node ? parent_progress_node : parent_gen->sub_progress_node, - name, strlen(name), 0); - - CodeGen *child_gen = codegen_create(nullptr, root_src_path, parent_gen->zig_target, out_type, - parent_gen->build_mode, parent_gen->zig_lib_dir, libc, get_global_cache_dir(), false, child_progress_node); - child_gen->root_out_name = buf_create_from_str(name); - child_gen->disable_gen_h = true; - child_gen->want_stack_check = WantStackCheckDisabled; - child_gen->want_sanitize_c = WantCSanitizeDisabled; - child_gen->verbose_tokenize = parent_gen->verbose_tokenize; - child_gen->verbose_ast = parent_gen->verbose_ast; - child_gen->verbose_link = parent_gen->verbose_link; - child_gen->verbose_ir = parent_gen->verbose_ir; - child_gen->verbose_llvm_ir = parent_gen->verbose_llvm_ir; - child_gen->verbose_cimport = parent_gen->verbose_cimport; - child_gen->verbose_cc = parent_gen->verbose_cc; - child_gen->verbose_llvm_cpu_features = parent_gen->verbose_llvm_cpu_features; - child_gen->llvm_argv = parent_gen->llvm_argv; - - codegen_set_strip(child_gen, parent_gen->strip_debug_symbols); - child_gen->want_pic = parent_gen->have_pic ? WantPICEnabled : WantPICDisabled; - child_gen->valgrind_support = ValgrindSupportDisabled; - - codegen_set_errmsg_color(child_gen, parent_gen->err_color); - - child_gen->enable_cache = true; - - return child_gen; +void codegen_destroy(CodeGen *g) { + if (g->pass1_arena != nullptr) { + g->pass1_arena->destruct(&heap::c_allocator); + g->pass1_arena = nullptr; + } + heap::c_allocator.destroy(g); } CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget *target, - OutType out_type, BuildMode build_mode, Buf *override_lib_dir, - Stage2LibCInstallation *libc, Buf *cache_dir, bool is_test_build, Stage2ProgressNode *progress_node) + BuildMode build_mode, Buf *override_lib_dir, + bool is_test_build, Stage2ProgressNode *progress_node) { CodeGen *g = heap::c_allocator.create(); g->emit_bin = true; @@ -11176,24 +9389,15 @@ CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget } g->subsystem = TargetSubsystemAuto; - g->libc = libc; g->zig_target = target; - g->cache_dir = cache_dir; - if (override_lib_dir == nullptr) { - g->zig_lib_dir = get_zig_lib_dir(); - } else { - g->zig_lib_dir = override_lib_dir; - } + assert(override_lib_dir != nullptr); + g->zig_lib_dir = override_lib_dir; g->zig_std_dir = buf_alloc(); os_path_join(g->zig_lib_dir, buf_create_from_str("std"), g->zig_std_dir); - g->zig_c_headers_dir = buf_alloc(); - os_path_join(g->zig_lib_dir, buf_create_from_str("include"), g->zig_c_headers_dir); - g->build_mode = build_mode; - g->out_type = out_type; g->import_table.init(32); g->builtin_fn_table.init(32); g->primitive_type_table.init(32); @@ -11256,18 +9460,6 @@ CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget g->zig_std_special_dir = buf_alloc(); os_path_join(g->zig_std_dir, buf_sprintf("special"), g->zig_std_special_dir); - assert(target != nullptr); - if (!target->is_native_os) { - g->each_lib_rpath = false; - } else { - g->each_lib_rpath = true; - } - - if (target_os_requires_libc(g->zig_target->os)) { - g->libc_link_lib = create_link_lib(buf_create_from_str("c")); - g->link_libs_list.append(g->libc_link_lib); - } - target_triple_llvm(&g->llvm_triple_str, g->zig_target); g->pointer_size_bytes = target_arch_pointer_bit_width(g->zig_target->arch) / 8; @@ -11302,36 +9494,21 @@ void codegen_switch_sub_prog_node(CodeGen *g, Stage2ProgressNode *node) { } ZigValue *CodeGen::Intern::for_undefined() { -#ifdef ZIG_ENABLE_MEM_PROFILE - mem::intern_counters.x_undefined += 1; -#endif return &this->x_undefined; } ZigValue *CodeGen::Intern::for_void() { -#ifdef ZIG_ENABLE_MEM_PROFILE - mem::intern_counters.x_void += 1; -#endif return &this->x_void; } ZigValue *CodeGen::Intern::for_null() { -#ifdef ZIG_ENABLE_MEM_PROFILE - mem::intern_counters.x_null += 1; -#endif return &this->x_null; } ZigValue *CodeGen::Intern::for_unreachable() { -#ifdef ZIG_ENABLE_MEM_PROFILE - mem::intern_counters.x_unreachable += 1; -#endif return &this->x_unreachable; } ZigValue *CodeGen::Intern::for_zero_byte() { -#ifdef ZIG_ENABLE_MEM_PROFILE - mem::intern_counters.zero_byte += 1; -#endif return &this->zero_byte; } diff --git a/src/codegen.hpp b/src/codegen.hpp index 3139071d52..5b4317c992 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -16,47 +16,20 @@ #include CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget *target, - OutType out_type, BuildMode build_mode, Buf *zig_lib_dir, - Stage2LibCInstallation *libc, Buf *cache_dir, bool is_test_build, Stage2ProgressNode *progress_node); + BuildMode build_mode, Buf *zig_lib_dir, + bool is_test_build, Stage2ProgressNode *progress_node); -CodeGen *create_child_codegen(CodeGen *parent_gen, Buf *root_src_path, OutType out_type, - Stage2LibCInstallation *libc, const char *name, Stage2ProgressNode *progress_node); +void codegen_build_object(CodeGen *g); +void codegen_destroy(CodeGen *); -void codegen_set_clang_argv(CodeGen *codegen, const char **args, size_t len); -void codegen_set_llvm_argv(CodeGen *codegen, const char **args, size_t len); -void codegen_set_each_lib_rpath(CodeGen *codegen, bool each_lib_rpath); - -void codegen_set_strip(CodeGen *codegen, bool strip); -void codegen_set_errmsg_color(CodeGen *codegen, ErrColor err_color); -void codegen_set_out_name(CodeGen *codegen, Buf *out_name); -void codegen_add_lib_dir(CodeGen *codegen, const char *dir); -void codegen_add_forbidden_lib(CodeGen *codegen, Buf *lib); -LinkLib *codegen_add_link_lib(CodeGen *codegen, Buf *lib); -void codegen_add_framework(CodeGen *codegen, const char *name); -void codegen_add_rpath(CodeGen *codegen, const char *name); -void codegen_set_rdynamic(CodeGen *g, bool rdynamic); -void codegen_set_linker_script(CodeGen *g, const char *linker_script); -void codegen_set_test_filter(CodeGen *g, Buf *filter); -void codegen_set_test_name_prefix(CodeGen *g, Buf *prefix); -void codegen_set_lib_version(CodeGen *g, bool is_versioned, size_t major, size_t minor, size_t patch); void codegen_add_time_event(CodeGen *g, const char *name); void codegen_print_timing_report(CodeGen *g, FILE *f); -void codegen_link(CodeGen *g); -void zig_link_add_compiler_rt(CodeGen *g, Stage2ProgressNode *progress_node); -void codegen_build_and_link(CodeGen *g); ZigPackage *codegen_create_package(CodeGen *g, const char *root_src_dir, const char *root_src_path, const char *pkg_path); -void codegen_add_assembly(CodeGen *g, Buf *path); -void codegen_add_object(CodeGen *g, Buf *object_path); - -void codegen_translate_c(CodeGen *g, Buf *full_path); - -Buf *codegen_generate_builtin_source(CodeGen *g); TargetSubsystem detect_subsystem(CodeGen *g); -void codegen_release_caches(CodeGen *codegen); bool codegen_fn_has_err_ret_tracing_arg(CodeGen *g, ZigType *return_type); bool codegen_fn_has_err_ret_tracing_stack(CodeGen *g, ZigFn *fn, bool is_async); diff --git a/src/compiler.cpp b/src/compiler.cpp deleted file mode 100644 index 6c477a1506..0000000000 --- a/src/compiler.cpp +++ /dev/null @@ -1,196 +0,0 @@ -#include "cache_hash.hpp" -#include "os.hpp" -#include "compiler.hpp" - -#include - -Error get_compiler_id(Buf **result) { - static Buf saved_compiler_id = BUF_INIT; - - if (saved_compiler_id.list.length != 0) { - *result = &saved_compiler_id; - return ErrorNone; - } - - Error err; - Buf *manifest_dir = buf_alloc(); - os_path_join(get_global_cache_dir(), buf_create_from_str("exe"), manifest_dir); - - CacheHash cache_hash; - CacheHash *ch = &cache_hash; - cache_init(ch, manifest_dir); - Buf self_exe_path = BUF_INIT; - if ((err = os_self_exe_path(&self_exe_path))) - return err; - - cache_file(ch, &self_exe_path); - - buf_resize(&saved_compiler_id, 0); - if ((err = cache_hit(ch, &saved_compiler_id))) { - if (err != ErrorInvalidFormat) - return err; - } - if (buf_len(&saved_compiler_id) != 0) { - cache_release(ch); - *result = &saved_compiler_id; - return ErrorNone; - } - ZigList lib_paths = {}; - if ((err = os_self_exe_shared_libs(lib_paths))) - return err; - #if defined(ZIG_OS_DARWIN) - // only add the self exe path on mac os - Buf *lib_path = lib_paths.at(0); - if ((err = cache_add_file(ch, lib_path))) - return err; - #else - for (size_t i = 0; i < lib_paths.length; i += 1) { - Buf *lib_path = lib_paths.at(i); - if ((err = cache_add_file(ch, lib_path))) - return err; - } - #endif - - if ((err = cache_final(ch, &saved_compiler_id))) - return err; - - cache_release(ch); - - *result = &saved_compiler_id; - return ErrorNone; -} - -static bool test_zig_install_prefix(Buf *test_path, Buf *out_zig_lib_dir) { - { - Buf *test_zig_dir = buf_sprintf("%s" OS_SEP "lib" OS_SEP "zig", buf_ptr(test_path)); - Buf *test_index_file = buf_sprintf("%s" OS_SEP "std" OS_SEP "std.zig", buf_ptr(test_zig_dir)); - int err; - bool exists; - if ((err = os_file_exists(test_index_file, &exists))) { - exists = false; - } - if (exists) { - buf_init_from_buf(out_zig_lib_dir, test_zig_dir); - return true; - } - } - - // Also try without "zig" - { - Buf *test_zig_dir = buf_sprintf("%s" OS_SEP "lib", buf_ptr(test_path)); - Buf *test_index_file = buf_sprintf("%s" OS_SEP "std" OS_SEP "std.zig", buf_ptr(test_zig_dir)); - int err; - bool exists; - if ((err = os_file_exists(test_index_file, &exists))) { - exists = false; - } - if (exists) { - buf_init_from_buf(out_zig_lib_dir, test_zig_dir); - return true; - } - } - - return false; -} - -static int find_zig_lib_dir(Buf *out_path) { - int err; - - Buf self_exe_path = BUF_INIT; - buf_resize(&self_exe_path, 0); - if (!(err = os_self_exe_path(&self_exe_path))) { - Buf *cur_path = &self_exe_path; - - for (;;) { - Buf *test_dir = buf_alloc(); - os_path_dirname(cur_path, test_dir); - - if (buf_eql_buf(test_dir, cur_path)) { - break; - } - - if (test_zig_install_prefix(test_dir, out_path)) { - return 0; - } - - cur_path = test_dir; - } - } - - return ErrorFileNotFound; -} - -Buf *get_zig_lib_dir(void) { - static Buf saved_lib_dir = BUF_INIT; - if (saved_lib_dir.list.length != 0) - return &saved_lib_dir; - buf_resize(&saved_lib_dir, 0); - - int err; - if ((err = find_zig_lib_dir(&saved_lib_dir))) { - fprintf(stderr, "Unable to find zig lib directory\n"); - exit(EXIT_FAILURE); - } - return &saved_lib_dir; -} - -Buf *get_zig_std_dir(Buf *zig_lib_dir) { - static Buf saved_std_dir = BUF_INIT; - if (saved_std_dir.list.length != 0) - return &saved_std_dir; - buf_resize(&saved_std_dir, 0); - - os_path_join(zig_lib_dir, buf_create_from_str("std"), &saved_std_dir); - - return &saved_std_dir; -} - -Buf *get_zig_special_dir(Buf *zig_lib_dir) { - static Buf saved_special_dir = BUF_INIT; - if (saved_special_dir.list.length != 0) - return &saved_special_dir; - buf_resize(&saved_special_dir, 0); - - os_path_join(get_zig_std_dir(zig_lib_dir), buf_sprintf("special"), &saved_special_dir); - - return &saved_special_dir; -} - -Buf *get_global_cache_dir(void) { - static Buf saved_global_cache_dir = BUF_INIT; - if (saved_global_cache_dir.list.length != 0) - return &saved_global_cache_dir; - buf_resize(&saved_global_cache_dir, 0); - - Buf app_data_dir = BUF_INIT; - Error err; - if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { - fprintf(stderr, "Unable to get application data dir: %s\n", err_str(err)); - exit(1); - } - os_path_join(&app_data_dir, buf_create_from_str("stage1"), &saved_global_cache_dir); - buf_deinit(&app_data_dir); - return &saved_global_cache_dir; -} - -FileExt classify_file_ext(const char *filename_ptr, size_t filename_len) { - if (mem_ends_with_str(filename_ptr, filename_len, ".c")) { - return FileExtC; - } else if (mem_ends_with_str(filename_ptr, filename_len, ".C") || - mem_ends_with_str(filename_ptr, filename_len, ".cc") || - mem_ends_with_str(filename_ptr, filename_len, ".cpp") || - mem_ends_with_str(filename_ptr, filename_len, ".cxx")) - { - return FileExtCpp; - } else if (mem_ends_with_str(filename_ptr, filename_len, ".ll")) { - return FileExtLLVMIr; - } else if (mem_ends_with_str(filename_ptr, filename_len, ".bc")) { - return FileExtLLVMBitCode; - } else if (mem_ends_with_str(filename_ptr, filename_len, ".s") || - mem_ends_with_str(filename_ptr, filename_len, ".S")) - { - return FileExtAsm; - } - // TODO look for .so, .so.X, .so.X.Y, .so.X.Y.Z - return FileExtUnknown; -} diff --git a/src/compiler.hpp b/src/compiler.hpp deleted file mode 100644 index ae2e6e9c5e..0000000000 --- a/src/compiler.hpp +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2018 Andrew Kelley - * - * This file is part of zig, which is MIT licensed. - * See http://opensource.org/licenses/MIT - */ - -#ifndef ZIG_COMPILER_HPP -#define ZIG_COMPILER_HPP - -#include "all_types.hpp" - -Error get_compiler_id(Buf **result); - -Buf *get_zig_lib_dir(void); -Buf *get_zig_special_dir(Buf *zig_lib_dir); -Buf *get_zig_std_dir(Buf *zig_lib_dir); - -Buf *get_global_cache_dir(void); - - -FileExt classify_file_ext(const char *filename_ptr, size_t filename_len); - -#endif diff --git a/src/config.h.in b/src/config.h.in index 2ec6c25b38..8c147e7d65 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -22,6 +22,4 @@ #define ZIG_LLVM_CONFIG_EXE "@LLVM_CONFIG_EXE@" #define ZIG_DIA_GUIDS_LIB "@ZIG_DIA_GUIDS_LIB_ESCAPED@" -#cmakedefine ZIG_ENABLE_MEM_PROFILE - #endif diff --git a/src/config.zig.in b/src/config.zig.in index a44149111e..9e574bc1e8 100644 --- a/src/config.zig.in +++ b/src/config.zig.in @@ -3,3 +3,4 @@ pub const version: []const u8 = "@ZIG_VERSION@"; pub const log_scopes: []const []const u8 = &[_][]const u8{}; pub const zir_dumps: []const []const u8 = &[_][]const u8{}; pub const enable_tracy = false; +pub const is_stage1 = true; diff --git a/src/dump_analysis.cpp b/src/dump_analysis.cpp index 2f41341b14..df0d6f3ca2 100644 --- a/src/dump_analysis.cpp +++ b/src/dump_analysis.cpp @@ -6,11 +6,11 @@ */ #include "dump_analysis.hpp" -#include "compiler.hpp" #include "analyze.hpp" #include "config.h" #include "ir.hpp" #include "codegen.hpp" +#include "os.hpp" enum JsonWriterState { JsonWriterStateInvalid, @@ -1173,7 +1173,6 @@ static void anal_dump_fn(AnalDumpCtx *ctx, ZigFn *fn) { } void zig_print_analysis_dump(CodeGen *g, FILE *f, const char *one_indent, const char *nl) { - Error err; AnalDumpCtx ctx = {}; ctx.g = g; JsonWriter *jw = &ctx.jw; @@ -1199,15 +1198,6 @@ void zig_print_analysis_dump(CodeGen *g, FILE *f, const char *one_indent, const jw_object_field(jw, "params"); jw_begin_object(jw); { - jw_object_field(jw, "zigId"); - - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) { - fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); - exit(1); - } - jw_string(jw, buf_ptr(compiler_id)); - jw_object_field(jw, "zigVersion"); jw_string(jw, ZIG_VERSION_STRING); diff --git a/src/empty.cpp b/src/empty.cpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/errmsg.hpp b/src/errmsg.hpp index e8b2f5872d..73cbd4e0d9 100644 --- a/src/errmsg.hpp +++ b/src/errmsg.hpp @@ -10,12 +10,7 @@ #include "buffer.hpp" #include "list.hpp" - -enum ErrColor { - ErrColorAuto, - ErrColorOff, - ErrColorOn, -}; +#include "stage1.h" struct ErrorMsg { size_t line_start; diff --git a/src/glibc.cpp b/src/glibc.cpp deleted file mode 100644 index 62f5604ba7..0000000000 --- a/src/glibc.cpp +++ /dev/null @@ -1,392 +0,0 @@ -/* - * Copyright (c) 2019 Andrew Kelley - * - * This file is part of zig, which is MIT licensed. - * See http://opensource.org/licenses/MIT - */ - -#include "glibc.hpp" -#include "compiler.hpp" -#include "cache_hash.hpp" -#include "codegen.hpp" - -static const ZigGLibCLib glibc_libs[] = { - {"c", 6}, - {"m", 6}, - {"pthread", 0}, - {"dl", 2}, - {"rt", 1}, - {"ld", 2}, - {"util", 1}, -}; - -Error glibc_load_metadata(ZigGLibCAbi **out_result, Buf *zig_lib_dir, bool verbose) { - Error err; - - ZigGLibCAbi *glibc_abi = heap::c_allocator.create(); - glibc_abi->vers_txt_path = buf_sprintf("%s" OS_SEP "libc" OS_SEP "glibc" OS_SEP "vers.txt", buf_ptr(zig_lib_dir)); - glibc_abi->fns_txt_path = buf_sprintf("%s" OS_SEP "libc" OS_SEP "glibc" OS_SEP "fns.txt", buf_ptr(zig_lib_dir)); - glibc_abi->abi_txt_path = buf_sprintf("%s" OS_SEP "libc" OS_SEP "glibc" OS_SEP "abi.txt", buf_ptr(zig_lib_dir)); - glibc_abi->version_table.init(16); - - Buf *vers_txt_contents = buf_alloc(); - if ((err = os_fetch_file_path(glibc_abi->vers_txt_path, vers_txt_contents))) { - if (verbose) { - fprintf(stderr, "Unable to read %s: %s\n", buf_ptr(glibc_abi->vers_txt_path), err_str(err)); - } - return err; - } - Buf *fns_txt_contents = buf_alloc(); - if ((err = os_fetch_file_path(glibc_abi->fns_txt_path, fns_txt_contents))) { - if (verbose) { - fprintf(stderr, "Unable to read %s: %s\n", buf_ptr(glibc_abi->fns_txt_path), err_str(err)); - } - return err; - } - Buf *abi_txt_contents = buf_alloc(); - if ((err = os_fetch_file_path(glibc_abi->abi_txt_path, abi_txt_contents))) { - if (verbose) { - fprintf(stderr, "Unable to read %s: %s\n", buf_ptr(glibc_abi->abi_txt_path), err_str(err)); - } - return err; - } - - { - SplitIterator it = memSplit(buf_to_slice(vers_txt_contents), str("\r\n")); - for (;;) { - Optional> opt_component = SplitIterator_next(&it); - if (!opt_component.is_some) break; - Buf *ver_buf = buf_create_from_slice(opt_component.value); - Stage2SemVer *this_ver = glibc_abi->all_versions.add_one(); - if ((err = target_parse_glibc_version(this_ver, buf_ptr(ver_buf)))) { - if (verbose) { - fprintf(stderr, "Unable to parse glibc version '%s': %s\n", buf_ptr(ver_buf), err_str(err)); - } - return err; - } - } - } - { - SplitIterator it = memSplit(buf_to_slice(fns_txt_contents), str("\r\n")); - for (;;) { - Optional> opt_component = SplitIterator_next(&it); - if (!opt_component.is_some) break; - SplitIterator line_it = memSplit(opt_component.value, str(" ")); - Optional> opt_fn_name = SplitIterator_next(&line_it); - if (!opt_fn_name.is_some) { - if (verbose) { - fprintf(stderr, "%s: Expected function name\n", buf_ptr(glibc_abi->fns_txt_path)); - } - return ErrorInvalidFormat; - } - Optional> opt_lib_name = SplitIterator_next(&line_it); - if (!opt_lib_name.is_some) { - if (verbose) { - fprintf(stderr, "%s: Expected lib name\n", buf_ptr(glibc_abi->fns_txt_path)); - } - return ErrorInvalidFormat; - } - - Buf *this_fn_name = buf_create_from_slice(opt_fn_name.value); - Buf *this_lib_name = buf_create_from_slice(opt_lib_name.value); - glibc_abi->all_functions.append({ this_fn_name, glibc_lib_find(buf_ptr(this_lib_name)) }); - } - } - { - SplitIterator it = memSplit(buf_to_slice(abi_txt_contents), str("\r\n")); - ZigGLibCVerList *ver_list_base = nullptr; - int line_num = 0; - for (;;) { - if (ver_list_base == nullptr) { - line_num += 1; - Optional> opt_line = SplitIterator_next_separate(&it); - if (!opt_line.is_some) break; - - ver_list_base = heap::c_allocator.allocate(glibc_abi->all_functions.length); - SplitIterator line_it = memSplit(opt_line.value, str(" ")); - for (;;) { - ZigTarget *target = heap::c_allocator.create(); - Optional> opt_target = SplitIterator_next(&line_it); - if (!opt_target.is_some) break; - - SplitIterator component_it = memSplit(opt_target.value, str("-")); - Optional> opt_arch = SplitIterator_next(&component_it); - assert(opt_arch.is_some); - Optional> opt_os = SplitIterator_next(&component_it); - assert(opt_os.is_some); // it's always "linux" so we ignore it - Optional> opt_abi = SplitIterator_next(&component_it); - assert(opt_abi.is_some); - - - err = target_parse_arch(&target->arch, (char*)opt_arch.value.ptr, opt_arch.value.len); - assert(err == ErrorNone); - - target->os = OsLinux; - - err = target_parse_abi(&target->abi, (char*)opt_abi.value.ptr, opt_abi.value.len); - if (err != ErrorNone) { - fprintf(stderr, "Error parsing %s:%d: %s\n", buf_ptr(glibc_abi->abi_txt_path), - line_num, err_str(err)); - fprintf(stderr, "arch: '%.*s', os: '%.*s', abi: '%.*s'\n", - (int)opt_arch.value.len, (const char*)opt_arch.value.ptr, - (int)opt_os.value.len, (const char*)opt_os.value.ptr, - (int)opt_abi.value.len, (const char*)opt_abi.value.ptr); - fprintf(stderr, "parsed from target: '%.*s'\n", - (int)opt_target.value.len, (const char*)opt_target.value.ptr); - fprintf(stderr, "parsed from line:\n%.*s\n", (int)opt_line.value.len, opt_line.value.ptr); - fprintf(stderr, "Zig installation appears to be corrupted.\n"); - exit(1); - } - - glibc_abi->version_table.put(target, ver_list_base); - } - continue; - } - for (size_t fn_i = 0; fn_i < glibc_abi->all_functions.length; fn_i += 1) { - ZigGLibCVerList *ver_list = &ver_list_base[fn_i]; - line_num += 1; - Optional> opt_line = SplitIterator_next_separate(&it); - assert(opt_line.is_some); - - SplitIterator line_it = memSplit(opt_line.value, str(" ")); - for (;;) { - Optional> opt_ver = SplitIterator_next(&line_it); - if (!opt_ver.is_some) break; - assert(ver_list->len < 8); // increase the array len in the type - - unsigned long ver_index = strtoul(buf_ptr(buf_create_from_slice(opt_ver.value)), nullptr, 10); - assert(ver_index < 255); // use a bigger integer in the type - ver_list->versions[ver_list->len] = ver_index; - ver_list->len += 1; - } - } - ver_list_base = nullptr; - } - } - - *out_result = glibc_abi; - return ErrorNone; -} - -Error glibc_build_dummies_and_maps(CodeGen *g, const ZigGLibCAbi *glibc_abi, const ZigTarget *target, - Buf **out_dir, bool verbose, Stage2ProgressNode *progress_node) -{ - Error err; - - Buf *cache_dir = get_global_cache_dir(); - CacheHash *cache_hash = heap::c_allocator.create(); - Buf *manifest_dir = buf_sprintf("%s" OS_SEP CACHE_HASH_SUBDIR, buf_ptr(cache_dir)); - cache_init(cache_hash, manifest_dir); - - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) { - if (verbose) { - fprintf(stderr, "unable to get compiler id: %s\n", err_str(err)); - } - return err; - } - cache_buf(cache_hash, compiler_id); - cache_int(cache_hash, target->arch); - cache_int(cache_hash, target->abi); - cache_int(cache_hash, target->glibc_or_darwin_version->major); - cache_int(cache_hash, target->glibc_or_darwin_version->minor); - cache_int(cache_hash, target->glibc_or_darwin_version->patch); - - Buf digest = BUF_INIT; - buf_resize(&digest, 0); - if ((err = cache_hit(cache_hash, &digest))) { - // Treat an invalid format error as a cache miss. - if (err != ErrorInvalidFormat) - return err; - } - // We should always get a cache hit because there are no - // files in the input hash. - assert(buf_len(&digest) != 0); - - Buf *dummy_dir = buf_alloc(); - os_path_join(manifest_dir, &digest, dummy_dir); - - if ((err = os_make_path(dummy_dir))) - return err; - - Buf *test_if_exists_path = buf_alloc(); - os_path_join(dummy_dir, buf_create_from_str("ok"), test_if_exists_path); - - bool hit; - if ((err = os_file_exists(test_if_exists_path, &hit))) - return err; - - if (hit) { - *out_dir = dummy_dir; - return ErrorNone; - } - - - ZigGLibCVerList *ver_list_base = glibc_abi->version_table.get(target); - - uint8_t target_ver_index = 0; - for (;target_ver_index < glibc_abi->all_versions.length; target_ver_index += 1) { - const Stage2SemVer *this_ver = &glibc_abi->all_versions.at(target_ver_index); - if (this_ver->major == target->glibc_or_darwin_version->major && - this_ver->minor == target->glibc_or_darwin_version->minor && - this_ver->patch == target->glibc_or_darwin_version->patch) - { - break; - } - } - if (target_ver_index == glibc_abi->all_versions.length) { - if (verbose) { - fprintf(stderr, "Unrecognized glibc version: %d.%d.%d\n", - target->glibc_or_darwin_version->major, - target->glibc_or_darwin_version->minor, - target->glibc_or_darwin_version->patch); - } - return ErrorUnknownABI; - } - - Buf *map_file_path = buf_sprintf("%s" OS_SEP "all.map", buf_ptr(dummy_dir)); - Buf *map_contents = buf_alloc(); - - for (uint8_t ver_i = 0; ver_i < glibc_abi->all_versions.length; ver_i += 1) { - const Stage2SemVer *ver = &glibc_abi->all_versions.at(ver_i); - if (ver->patch == 0) { - buf_appendf(map_contents, "GLIBC_%d.%d { };\n", ver->major, ver->minor); - } else { - buf_appendf(map_contents, "GLIBC_%d.%d.%d { };\n", ver->major, ver->minor, ver->patch); - } - } - - if ((err = os_write_file(map_file_path, map_contents))) { - if (verbose) { - fprintf(stderr, "unable to write %s: %s", buf_ptr(map_file_path), err_str(err)); - } - return err; - } - - - for (size_t lib_i = 0; lib_i < array_length(glibc_libs); lib_i += 1) { - const ZigGLibCLib *lib = &glibc_libs[lib_i]; - Buf *zig_file_path = buf_sprintf("%s" OS_SEP "%s.zig", buf_ptr(dummy_dir), lib->name); - Buf *zig_body = buf_alloc(); - Buf *zig_footer = buf_alloc(); - - buf_appendf(zig_body, "comptime {\n"); - buf_appendf(zig_body, " asm (\n"); - - for (size_t fn_i = 0; fn_i < glibc_abi->all_functions.length; fn_i += 1) { - const ZigGLibCFn *libc_fn = &glibc_abi->all_functions.at(fn_i); - if (libc_fn->lib != lib) continue; - ZigGLibCVerList *ver_list = &ver_list_base[fn_i]; - // Pick the default symbol version: - // - If there are no versions, don't emit it - // - Take the greatest one <= than the target one - // - If none of them is <= than the - // specified one don't pick any default version - if (ver_list->len == 0) continue; - uint8_t chosen_def_ver_index = 255; - for (uint8_t ver_i = 0; ver_i < ver_list->len; ver_i += 1) { - uint8_t ver_index = ver_list->versions[ver_i]; - if ((chosen_def_ver_index == 255 || ver_index > chosen_def_ver_index) && - target_ver_index >= ver_index) - { - chosen_def_ver_index = ver_index; - } - } - for (uint8_t ver_i = 0; ver_i < ver_list->len; ver_i += 1) { - uint8_t ver_index = ver_list->versions[ver_i]; - - Buf *stub_name; - const Stage2SemVer *ver = &glibc_abi->all_versions.at(ver_index); - const char *sym_name = buf_ptr(libc_fn->name); - if (ver->patch == 0) { - stub_name = buf_sprintf("%s_%d_%d", sym_name, ver->major, ver->minor); - } else { - stub_name = buf_sprintf("%s_%d_%d_%d", sym_name, ver->major, ver->minor, ver->patch); - } - - buf_appendf(zig_footer, "export fn %s() void {}\n", buf_ptr(stub_name)); - - // Default symbol version definition vs normal symbol version definition - const char *at_sign_str = (chosen_def_ver_index != 255 && - ver_index == chosen_def_ver_index) ? "@@" : "@"; - if (ver->patch == 0) { - buf_appendf(zig_body, " \\\\ .symver %s, %s%sGLIBC_%d.%d\n", - buf_ptr(stub_name), sym_name, at_sign_str, ver->major, ver->minor); - } else { - buf_appendf(zig_body, " \\\\ .symver %s, %s%sGLIBC_%d.%d.%d\n", - buf_ptr(stub_name), sym_name, at_sign_str, ver->major, ver->minor, ver->patch); - } - // Hide the stub to keep the symbol table clean - buf_appendf(zig_body, " \\\\ .hidden %s\n", buf_ptr(stub_name)); - } - } - - buf_appendf(zig_body, " );\n"); - buf_appendf(zig_body, "}\n"); - buf_append_buf(zig_body, zig_footer); - - if ((err = os_write_file(zig_file_path, zig_body))) { - if (verbose) { - fprintf(stderr, "unable to write %s: %s", buf_ptr(zig_file_path), err_str(err)); - } - return err; - } - - bool is_ld = (strcmp(lib->name, "ld") == 0); - - CodeGen *child_gen = create_child_codegen(g, zig_file_path, OutTypeLib, nullptr, lib->name, progress_node); - codegen_set_lib_version(child_gen, true, lib->sover, 0, 0); - child_gen->is_dynamic = true; - child_gen->is_dummy_so = true; - child_gen->version_script_path = map_file_path; - child_gen->enable_cache = false; - child_gen->output_dir = dummy_dir; - if (is_ld) { - assert(g->zig_target->standard_dynamic_linker_path != nullptr); - Buf *ld_basename = buf_alloc(); - os_path_split(buf_create_from_str(g->zig_target->standard_dynamic_linker_path), - nullptr, ld_basename); - child_gen->override_soname = ld_basename; - } - codegen_build_and_link(child_gen); - } - - if ((err = os_write_file(test_if_exists_path, buf_alloc()))) { - if (verbose) { - fprintf(stderr, "unable to write %s: %s", buf_ptr(test_if_exists_path), err_str(err)); - } - return err; - } - *out_dir = dummy_dir; - return ErrorNone; -} - -uint32_t hash_glibc_target(const ZigTarget *x) { - return x->arch * (uint32_t)3250106448 + - x->os * (uint32_t)542534372 + - x->abi * (uint32_t)59162639; -} - -bool eql_glibc_target(const ZigTarget *a, const ZigTarget *b) { - return a->arch == b->arch && - a->os == b->os && - a->abi == b->abi; -} - -size_t glibc_lib_count(void) { - return array_length(glibc_libs); -} - -const ZigGLibCLib *glibc_lib_enum(size_t index) { - assert(index < array_length(glibc_libs)); - return &glibc_libs[index]; -} - -const ZigGLibCLib *glibc_lib_find(const char *name) { - for (size_t i = 0; i < array_length(glibc_libs); i += 1) { - if (strcmp(glibc_libs[i].name, name) == 0) { - return &glibc_libs[i]; - } - } - return nullptr; -} diff --git a/src/glibc.hpp b/src/glibc.hpp deleted file mode 100644 index c04dcb4629..0000000000 --- a/src/glibc.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2019 Andrew Kelley - * - * This file is part of zig, which is MIT licensed. - * See http://opensource.org/licenses/MIT - */ - -#ifndef ZIG_GLIBC_HPP -#define ZIG_GLIBC_HPP - -#include "all_types.hpp" - -struct ZigGLibCLib { - const char *name; - uint8_t sover; -}; - -struct ZigGLibCFn { - Buf *name; - const ZigGLibCLib *lib; -}; - -struct ZigGLibCVerList { - uint8_t versions[8]; // 8 is just the max number, we know statically it's big enough - uint8_t len; -}; - -uint32_t hash_glibc_target(const ZigTarget *x); -bool eql_glibc_target(const ZigTarget *a, const ZigTarget *b); - -struct ZigGLibCAbi { - Buf *abi_txt_path; - Buf *vers_txt_path; - Buf *fns_txt_path; - ZigList all_versions; - ZigList all_functions; - // The value is a pointer to all_functions.length items and each item is an index - // into all_functions. - HashMap version_table; -}; - -Error glibc_load_metadata(ZigGLibCAbi **out_result, Buf *zig_lib_dir, bool verbose); -Error glibc_build_dummies_and_maps(CodeGen *codegen, const ZigGLibCAbi *glibc_abi, const ZigTarget *target, - Buf **out_dir, bool verbose, Stage2ProgressNode *progress_node); - -size_t glibc_lib_count(void); -const ZigGLibCLib *glibc_lib_enum(size_t index); -const ZigGLibCLib *glibc_lib_find(const char *name); - -#endif diff --git a/src/heap.cpp b/src/heap.cpp index 79c44d13dc..7e7a171bde 100644 --- a/src/heap.cpp +++ b/src/heap.cpp @@ -10,7 +10,6 @@ #include "config.h" #include "heap.hpp" -#include "mem_profile.hpp" namespace heap { @@ -48,21 +47,9 @@ void BootstrapAllocator::internal_deallocate(const mem::TypeInfo &info, void *pt mem::os::free(ptr); } -void CAllocator::init(const char *name) { -#ifdef ZIG_ENABLE_MEM_PROFILE - this->profile = bootstrap_allocator.create(); - this->profile->init(name, "CAllocator"); -#endif -} +void CAllocator::init(const char *name) { } -void CAllocator::deinit() { -#ifdef ZIG_ENABLE_MEM_PROFILE - assert(this->profile); - this->profile->deinit(); - bootstrap_allocator.destroy(this->profile); - this->profile = nullptr; -#endif -} +void CAllocator::deinit() { } CAllocator *CAllocator::construct(mem::Allocator *allocator, const char *name) { auto p = new(allocator->create()) CAllocator(); @@ -75,23 +62,11 @@ void CAllocator::destruct(mem::Allocator *allocator) { allocator->destroy(this); } -#ifdef ZIG_ENABLE_MEM_PROFILE -void CAllocator::print_report(FILE *file) { - this->profile->print_report(file); -} -#endif - void *CAllocator::internal_allocate(const mem::TypeInfo &info, size_t count) { -#ifdef ZIG_ENABLE_MEM_PROFILE - this->profile->record_alloc(info, count); -#endif return mem::os::calloc(count, info.size); } void *CAllocator::internal_allocate_nonzero(const mem::TypeInfo &info, size_t count) { -#ifdef ZIG_ENABLE_MEM_PROFILE - this->profile->record_alloc(info, count); -#endif return mem::os::malloc(count * info.size); } @@ -103,17 +78,10 @@ void *CAllocator::internal_reallocate(const mem::TypeInfo &info, void *old_ptr, } void *CAllocator::internal_reallocate_nonzero(const mem::TypeInfo &info, void *old_ptr, size_t old_count, size_t new_count) { -#ifdef ZIG_ENABLE_MEM_PROFILE - this->profile->record_dealloc(info, old_count); - this->profile->record_alloc(info, new_count); -#endif return mem::os::realloc(old_ptr, new_count * info.size); } void CAllocator::internal_deallocate(const mem::TypeInfo &info, void *ptr, size_t count) { -#ifdef ZIG_ENABLE_MEM_PROFILE - this->profile->record_dealloc(info, count); -#endif mem::os::free(ptr); } @@ -249,10 +217,6 @@ void ArenaAllocator::Impl::track_object(Object object) { } void ArenaAllocator::init(Allocator *backing, const char *name) { -#ifdef ZIG_ENABLE_MEM_PROFILE - this->profile = bootstrap_allocator.create(); - this->profile->init(name, "ArenaAllocator"); -#endif this->impl = bootstrap_allocator.create(); { auto &r = *this->impl; @@ -309,13 +273,6 @@ void ArenaAllocator::deinit() { t = prev; } } - -#ifdef ZIG_ENABLE_MEM_PROFILE - assert(this->profile); - this->profile->deinit(); - bootstrap_allocator.destroy(this->profile); - this->profile = nullptr; -#endif } ArenaAllocator *ArenaAllocator::construct(mem::Allocator *allocator, mem::Allocator *backing, const char *name) { @@ -329,23 +286,11 @@ void ArenaAllocator::destruct(mem::Allocator *allocator) { allocator->destroy(this); } -#ifdef ZIG_ENABLE_MEM_PROFILE -void ArenaAllocator::print_report(FILE *file) { - this->profile->print_report(file); -} -#endif - void *ArenaAllocator::internal_allocate(const mem::TypeInfo &info, size_t count) { -#ifdef ZIG_ENABLE_MEM_PROFILE - this->profile->record_alloc(info, count); -#endif return this->impl->allocate(info, count); } void *ArenaAllocator::internal_allocate_nonzero(const mem::TypeInfo &info, size_t count) { -#ifdef ZIG_ENABLE_MEM_PROFILE - this->profile->record_alloc(info, count); -#endif return this->impl->allocate(info, count); } @@ -354,17 +299,10 @@ void *ArenaAllocator::internal_reallocate(const mem::TypeInfo &info, void *old_p } void *ArenaAllocator::internal_reallocate_nonzero(const mem::TypeInfo &info, void *old_ptr, size_t old_count, size_t new_count) { -#ifdef ZIG_ENABLE_MEM_PROFILE - this->profile->record_dealloc(info, old_count); - this->profile->record_alloc(info, new_count); -#endif return this->impl->reallocate(info, old_ptr, old_count, new_count); } void ArenaAllocator::internal_deallocate(const mem::TypeInfo &info, void *ptr, size_t count) { -#ifdef ZIG_ENABLE_MEM_PROFILE - this->profile->record_dealloc(info, count); -#endif // noop } diff --git a/src/heap.hpp b/src/heap.hpp index e22ec42967..ec5c81026d 100644 --- a/src/heap.hpp +++ b/src/heap.hpp @@ -12,12 +12,6 @@ #include "util_base.hpp" #include "mem.hpp" -#ifdef ZIG_ENABLE_MEM_PROFILE -namespace mem { - struct Profile; -} -#endif - namespace heap { struct BootstrapAllocator final : mem::Allocator { @@ -40,9 +34,6 @@ struct CAllocator final : mem::Allocator { static CAllocator *construct(mem::Allocator *allocator, const char *name); void destruct(mem::Allocator *allocator) final; -#ifdef ZIG_ENABLE_MEM_PROFILE - void print_report(FILE *file = nullptr); -#endif private: ATTRIBUTE_RETURNS_NOALIAS void *internal_allocate(const mem::TypeInfo &info, size_t count) final; @@ -51,9 +42,6 @@ private: void *internal_reallocate_nonzero(const mem::TypeInfo &info, void *old_ptr, size_t old_count, size_t new_count) final; void internal_deallocate(const mem::TypeInfo &info, void *ptr, size_t count) final; -#ifdef ZIG_ENABLE_MEM_PROFILE - mem::Profile *profile; -#endif }; // @@ -71,9 +59,6 @@ struct ArenaAllocator final : mem::Allocator { static ArenaAllocator *construct(mem::Allocator *allocator, mem::Allocator *backing, const char *name); void destruct(mem::Allocator *allocator) final; -#ifdef ZIG_ENABLE_MEM_PROFILE - void print_report(FILE *file = nullptr); -#endif private: ATTRIBUTE_RETURNS_NOALIAS void *internal_allocate(const mem::TypeInfo &info, size_t count) final; @@ -82,10 +67,6 @@ private: void *internal_reallocate_nonzero(const mem::TypeInfo &info, void *old_ptr, size_t old_count, size_t new_count) final; void internal_deallocate(const mem::TypeInfo &info, void *ptr, size_t count) final; -#ifdef ZIG_ENABLE_MEM_PROFILE - mem::Profile *profile; -#endif - struct Impl; Impl *impl; }; diff --git a/src/install_files.h b/src/install_files.h deleted file mode 100644 index 8e7431145e..0000000000 --- a/src/install_files.h +++ /dev/null @@ -1,1907 +0,0 @@ -#ifndef ZIG_INSTALL_FILES_H -#define ZIG_INSTALL_FILES_H -static const char *ZIG_MUSL_SRC_FILES[] = { -"musl/src/aio/aio.c", -"musl/src/aio/aio_suspend.c", -"musl/src/aio/lio_listio.c", -"musl/src/complex/__cexp.c", -"musl/src/complex/__cexpf.c", -"musl/src/complex/cabs.c", -"musl/src/complex/cabsf.c", -"musl/src/complex/cabsl.c", -"musl/src/complex/cacos.c", -"musl/src/complex/cacosf.c", -"musl/src/complex/cacosh.c", -"musl/src/complex/cacoshf.c", -"musl/src/complex/cacoshl.c", -"musl/src/complex/cacosl.c", -"musl/src/complex/carg.c", -"musl/src/complex/cargf.c", -"musl/src/complex/cargl.c", -"musl/src/complex/casin.c", -"musl/src/complex/casinf.c", -"musl/src/complex/casinh.c", -"musl/src/complex/casinhf.c", -"musl/src/complex/casinhl.c", -"musl/src/complex/casinl.c", -"musl/src/complex/catan.c", -"musl/src/complex/catanf.c", -"musl/src/complex/catanh.c", -"musl/src/complex/catanhf.c", -"musl/src/complex/catanhl.c", -"musl/src/complex/catanl.c", -"musl/src/complex/ccos.c", -"musl/src/complex/ccosf.c", -"musl/src/complex/ccosh.c", -"musl/src/complex/ccoshf.c", -"musl/src/complex/ccoshl.c", -"musl/src/complex/ccosl.c", -"musl/src/complex/cexp.c", -"musl/src/complex/cexpf.c", -"musl/src/complex/cexpl.c", -"musl/src/complex/cimag.c", -"musl/src/complex/cimagf.c", -"musl/src/complex/cimagl.c", -"musl/src/complex/clog.c", -"musl/src/complex/clogf.c", -"musl/src/complex/clogl.c", -"musl/src/complex/conj.c", -"musl/src/complex/conjf.c", -"musl/src/complex/conjl.c", -"musl/src/complex/cpow.c", -"musl/src/complex/cpowf.c", -"musl/src/complex/cpowl.c", -"musl/src/complex/cproj.c", -"musl/src/complex/cprojf.c", -"musl/src/complex/cprojl.c", -"musl/src/complex/creal.c", -"musl/src/complex/crealf.c", -"musl/src/complex/creall.c", -"musl/src/complex/csin.c", -"musl/src/complex/csinf.c", -"musl/src/complex/csinh.c", -"musl/src/complex/csinhf.c", -"musl/src/complex/csinhl.c", -"musl/src/complex/csinl.c", -"musl/src/complex/csqrt.c", -"musl/src/complex/csqrtf.c", -"musl/src/complex/csqrtl.c", -"musl/src/complex/ctan.c", -"musl/src/complex/ctanf.c", -"musl/src/complex/ctanh.c", -"musl/src/complex/ctanhf.c", -"musl/src/complex/ctanhl.c", -"musl/src/complex/ctanl.c", -"musl/src/conf/confstr.c", -"musl/src/conf/fpathconf.c", -"musl/src/conf/legacy.c", -"musl/src/conf/pathconf.c", -"musl/src/conf/sysconf.c", -"musl/src/crypt/crypt.c", -"musl/src/crypt/crypt_blowfish.c", -"musl/src/crypt/crypt_des.c", -"musl/src/crypt/crypt_md5.c", -"musl/src/crypt/crypt_r.c", -"musl/src/crypt/crypt_sha256.c", -"musl/src/crypt/crypt_sha512.c", -"musl/src/crypt/encrypt.c", -"musl/src/ctype/__ctype_b_loc.c", -"musl/src/ctype/__ctype_get_mb_cur_max.c", -"musl/src/ctype/__ctype_tolower_loc.c", -"musl/src/ctype/__ctype_toupper_loc.c", -"musl/src/ctype/isalnum.c", -"musl/src/ctype/isalpha.c", -"musl/src/ctype/isascii.c", -"musl/src/ctype/isblank.c", -"musl/src/ctype/iscntrl.c", -"musl/src/ctype/isdigit.c", -"musl/src/ctype/isgraph.c", -"musl/src/ctype/islower.c", -"musl/src/ctype/isprint.c", -"musl/src/ctype/ispunct.c", -"musl/src/ctype/isspace.c", -"musl/src/ctype/isupper.c", -"musl/src/ctype/iswalnum.c", -"musl/src/ctype/iswalpha.c", -"musl/src/ctype/iswblank.c", -"musl/src/ctype/iswcntrl.c", -"musl/src/ctype/iswctype.c", -"musl/src/ctype/iswdigit.c", -"musl/src/ctype/iswgraph.c", -"musl/src/ctype/iswlower.c", -"musl/src/ctype/iswprint.c", -"musl/src/ctype/iswpunct.c", -"musl/src/ctype/iswspace.c", -"musl/src/ctype/iswupper.c", -"musl/src/ctype/iswxdigit.c", -"musl/src/ctype/isxdigit.c", -"musl/src/ctype/toascii.c", -"musl/src/ctype/tolower.c", -"musl/src/ctype/toupper.c", -"musl/src/ctype/towctrans.c", -"musl/src/ctype/wcswidth.c", -"musl/src/ctype/wctrans.c", -"musl/src/ctype/wcwidth.c", -"musl/src/dirent/alphasort.c", -"musl/src/dirent/closedir.c", -"musl/src/dirent/dirfd.c", -"musl/src/dirent/fdopendir.c", -"musl/src/dirent/opendir.c", -"musl/src/dirent/readdir.c", -"musl/src/dirent/readdir_r.c", -"musl/src/dirent/rewinddir.c", -"musl/src/dirent/scandir.c", -"musl/src/dirent/seekdir.c", -"musl/src/dirent/telldir.c", -"musl/src/dirent/versionsort.c", -"musl/src/env/__environ.c", -"musl/src/env/__init_tls.c", -"musl/src/env/__libc_start_main.c", -"musl/src/env/__reset_tls.c", -"musl/src/env/__stack_chk_fail.c", -"musl/src/env/clearenv.c", -"musl/src/env/getenv.c", -"musl/src/env/putenv.c", -"musl/src/env/secure_getenv.c", -"musl/src/env/setenv.c", -"musl/src/env/unsetenv.c", -"musl/src/errno/__errno_location.c", -"musl/src/errno/strerror.c", -"musl/src/exit/_Exit.c", -"musl/src/exit/abort.c", -"musl/src/exit/arm/__aeabi_atexit.c", -"musl/src/exit/assert.c", -"musl/src/exit/at_quick_exit.c", -"musl/src/exit/atexit.c", -"musl/src/exit/exit.c", -"musl/src/exit/quick_exit.c", -"musl/src/fcntl/creat.c", -"musl/src/fcntl/fcntl.c", -"musl/src/fcntl/open.c", -"musl/src/fcntl/openat.c", -"musl/src/fcntl/posix_fadvise.c", -"musl/src/fcntl/posix_fallocate.c", -"musl/src/fenv/__flt_rounds.c", -"musl/src/fenv/aarch64/fenv.s", -"musl/src/fenv/arm/fenv-hf.S", -"musl/src/fenv/arm/fenv.c", -"musl/src/fenv/fegetexceptflag.c", -"musl/src/fenv/feholdexcept.c", -"musl/src/fenv/fenv.c", -"musl/src/fenv/fesetexceptflag.c", -"musl/src/fenv/fesetround.c", -"musl/src/fenv/feupdateenv.c", -"musl/src/fenv/i386/fenv.s", -"musl/src/fenv/m68k/fenv.c", -"musl/src/fenv/mips/fenv-sf.c", -"musl/src/fenv/mips/fenv.S", -"musl/src/fenv/mips64/fenv-sf.c", -"musl/src/fenv/mips64/fenv.S", -"musl/src/fenv/mipsn32/fenv-sf.c", -"musl/src/fenv/mipsn32/fenv.S", -"musl/src/fenv/powerpc/fenv-sf.c", -"musl/src/fenv/powerpc/fenv.S", -"musl/src/fenv/powerpc64/fenv.c", -"musl/src/fenv/riscv64/fenv-sf.c", -"musl/src/fenv/riscv64/fenv.S", -"musl/src/fenv/s390x/fenv.c", -"musl/src/fenv/sh/fenv-nofpu.c", -"musl/src/fenv/sh/fenv.S", -"musl/src/fenv/x32/fenv.s", -"musl/src/fenv/x86_64/fenv.s", -"musl/src/internal/defsysinfo.c", -"musl/src/internal/floatscan.c", -"musl/src/internal/i386/defsysinfo.s", -"musl/src/internal/intscan.c", -"musl/src/internal/libc.c", -"musl/src/internal/procfdname.c", -"musl/src/internal/sh/__shcall.c", -"musl/src/internal/shgetc.c", -"musl/src/internal/syscall_ret.c", -"musl/src/internal/vdso.c", -"musl/src/internal/version.c", -"musl/src/ipc/ftok.c", -"musl/src/ipc/msgctl.c", -"musl/src/ipc/msgget.c", -"musl/src/ipc/msgrcv.c", -"musl/src/ipc/msgsnd.c", -"musl/src/ipc/semctl.c", -"musl/src/ipc/semget.c", -"musl/src/ipc/semop.c", -"musl/src/ipc/semtimedop.c", -"musl/src/ipc/shmat.c", -"musl/src/ipc/shmctl.c", -"musl/src/ipc/shmdt.c", -"musl/src/ipc/shmget.c", -"musl/src/ldso/__dlsym.c", -"musl/src/ldso/aarch64/dlsym.s", -"musl/src/ldso/aarch64/tlsdesc.s", -"musl/src/ldso/arm/dlsym.s", -"musl/src/ldso/arm/dlsym_time64.S", -"musl/src/ldso/arm/find_exidx.c", -"musl/src/ldso/arm/tlsdesc.S", -"musl/src/ldso/dl_iterate_phdr.c", -"musl/src/ldso/dladdr.c", -"musl/src/ldso/dlclose.c", -"musl/src/ldso/dlerror.c", -"musl/src/ldso/dlinfo.c", -"musl/src/ldso/dlopen.c", -"musl/src/ldso/dlsym.c", -"musl/src/ldso/i386/dlsym.s", -"musl/src/ldso/i386/dlsym_time64.S", -"musl/src/ldso/i386/tlsdesc.s", -"musl/src/ldso/m68k/dlsym.s", -"musl/src/ldso/m68k/dlsym_time64.S", -"musl/src/ldso/microblaze/dlsym.s", -"musl/src/ldso/microblaze/dlsym_time64.S", -"musl/src/ldso/mips/dlsym.s", -"musl/src/ldso/mips/dlsym_time64.S", -"musl/src/ldso/mips64/dlsym.s", -"musl/src/ldso/mipsn32/dlsym.s", -"musl/src/ldso/mipsn32/dlsym_time64.S", -"musl/src/ldso/or1k/dlsym.s", -"musl/src/ldso/or1k/dlsym_time64.S", -"musl/src/ldso/powerpc/dlsym.s", -"musl/src/ldso/powerpc/dlsym_time64.S", -"musl/src/ldso/powerpc64/dlsym.s", -"musl/src/ldso/riscv64/dlsym.s", -"musl/src/ldso/s390x/dlsym.s", -"musl/src/ldso/sh/dlsym.s", -"musl/src/ldso/sh/dlsym_time64.S", -"musl/src/ldso/tlsdesc.c", -"musl/src/ldso/x32/dlsym.s", -"musl/src/ldso/x86_64/dlsym.s", -"musl/src/ldso/x86_64/tlsdesc.s", -"musl/src/legacy/cuserid.c", -"musl/src/legacy/daemon.c", -"musl/src/legacy/err.c", -"musl/src/legacy/euidaccess.c", -"musl/src/legacy/ftw.c", -"musl/src/legacy/futimes.c", -"musl/src/legacy/getdtablesize.c", -"musl/src/legacy/getloadavg.c", -"musl/src/legacy/getpagesize.c", -"musl/src/legacy/getpass.c", -"musl/src/legacy/getusershell.c", -"musl/src/legacy/isastream.c", -"musl/src/legacy/lutimes.c", -"musl/src/legacy/ulimit.c", -"musl/src/legacy/utmpx.c", -"musl/src/legacy/valloc.c", -"musl/src/linux/adjtime.c", -"musl/src/linux/adjtimex.c", -"musl/src/linux/arch_prctl.c", -"musl/src/linux/brk.c", -"musl/src/linux/cache.c", -"musl/src/linux/cap.c", -"musl/src/linux/chroot.c", -"musl/src/linux/clock_adjtime.c", -"musl/src/linux/clone.c", -"musl/src/linux/copy_file_range.c", -"musl/src/linux/epoll.c", -"musl/src/linux/eventfd.c", -"musl/src/linux/fallocate.c", -"musl/src/linux/fanotify.c", -"musl/src/linux/flock.c", -"musl/src/linux/getdents.c", -"musl/src/linux/getrandom.c", -"musl/src/linux/inotify.c", -"musl/src/linux/ioperm.c", -"musl/src/linux/iopl.c", -"musl/src/linux/klogctl.c", -"musl/src/linux/membarrier.c", -"musl/src/linux/memfd_create.c", -"musl/src/linux/mlock2.c", -"musl/src/linux/module.c", -"musl/src/linux/mount.c", -"musl/src/linux/name_to_handle_at.c", -"musl/src/linux/open_by_handle_at.c", -"musl/src/linux/personality.c", -"musl/src/linux/pivot_root.c", -"musl/src/linux/ppoll.c", -"musl/src/linux/prctl.c", -"musl/src/linux/prlimit.c", -"musl/src/linux/process_vm.c", -"musl/src/linux/ptrace.c", -"musl/src/linux/quotactl.c", -"musl/src/linux/readahead.c", -"musl/src/linux/reboot.c", -"musl/src/linux/remap_file_pages.c", -"musl/src/linux/sbrk.c", -"musl/src/linux/sendfile.c", -"musl/src/linux/setfsgid.c", -"musl/src/linux/setfsuid.c", -"musl/src/linux/setgroups.c", -"musl/src/linux/sethostname.c", -"musl/src/linux/setns.c", -"musl/src/linux/settimeofday.c", -"musl/src/linux/signalfd.c", -"musl/src/linux/splice.c", -"musl/src/linux/stime.c", -"musl/src/linux/swap.c", -"musl/src/linux/sync_file_range.c", -"musl/src/linux/syncfs.c", -"musl/src/linux/sysinfo.c", -"musl/src/linux/tee.c", -"musl/src/linux/timerfd.c", -"musl/src/linux/unshare.c", -"musl/src/linux/utimes.c", -"musl/src/linux/vhangup.c", -"musl/src/linux/vmsplice.c", -"musl/src/linux/wait3.c", -"musl/src/linux/wait4.c", -"musl/src/linux/x32/sysinfo.c", -"musl/src/linux/xattr.c", -"musl/src/locale/__lctrans.c", -"musl/src/locale/__mo_lookup.c", -"musl/src/locale/bind_textdomain_codeset.c", -"musl/src/locale/c_locale.c", -"musl/src/locale/catclose.c", -"musl/src/locale/catgets.c", -"musl/src/locale/catopen.c", -"musl/src/locale/dcngettext.c", -"musl/src/locale/duplocale.c", -"musl/src/locale/freelocale.c", -"musl/src/locale/iconv.c", -"musl/src/locale/iconv_close.c", -"musl/src/locale/langinfo.c", -"musl/src/locale/locale_map.c", -"musl/src/locale/localeconv.c", -"musl/src/locale/newlocale.c", -"musl/src/locale/pleval.c", -"musl/src/locale/setlocale.c", -"musl/src/locale/strcoll.c", -"musl/src/locale/strfmon.c", -"musl/src/locale/strxfrm.c", -"musl/src/locale/textdomain.c", -"musl/src/locale/uselocale.c", -"musl/src/locale/wcscoll.c", -"musl/src/locale/wcsxfrm.c", -"musl/src/malloc/aligned_alloc.c", -"musl/src/malloc/expand_heap.c", -"musl/src/malloc/lite_malloc.c", -"musl/src/malloc/malloc.c", -"musl/src/malloc/malloc_usable_size.c", -"musl/src/malloc/memalign.c", -"musl/src/malloc/posix_memalign.c", -"musl/src/math/__cos.c", -"musl/src/math/__cosdf.c", -"musl/src/math/__cosl.c", -"musl/src/math/__expo2.c", -"musl/src/math/__expo2f.c", -"musl/src/math/__fpclassify.c", -"musl/src/math/__fpclassifyf.c", -"musl/src/math/__fpclassifyl.c", -"musl/src/math/__invtrigl.c", -"musl/src/math/__math_divzero.c", -"musl/src/math/__math_divzerof.c", -"musl/src/math/__math_invalid.c", -"musl/src/math/__math_invalidf.c", -"musl/src/math/__math_oflow.c", -"musl/src/math/__math_oflowf.c", -"musl/src/math/__math_uflow.c", -"musl/src/math/__math_uflowf.c", -"musl/src/math/__math_xflow.c", -"musl/src/math/__math_xflowf.c", -"musl/src/math/__polevll.c", -"musl/src/math/__rem_pio2.c", -"musl/src/math/__rem_pio2_large.c", -"musl/src/math/__rem_pio2f.c", -"musl/src/math/__rem_pio2l.c", -"musl/src/math/__signbit.c", -"musl/src/math/__signbitf.c", -"musl/src/math/__signbitl.c", -"musl/src/math/__sin.c", -"musl/src/math/__sindf.c", -"musl/src/math/__sinl.c", -"musl/src/math/__tan.c", -"musl/src/math/__tandf.c", -"musl/src/math/__tanl.c", -"musl/src/math/aarch64/ceil.c", -"musl/src/math/aarch64/ceilf.c", -"musl/src/math/aarch64/fabs.c", -"musl/src/math/aarch64/fabsf.c", -"musl/src/math/aarch64/floor.c", -"musl/src/math/aarch64/floorf.c", -"musl/src/math/aarch64/fma.c", -"musl/src/math/aarch64/fmaf.c", -"musl/src/math/aarch64/fmax.c", -"musl/src/math/aarch64/fmaxf.c", -"musl/src/math/aarch64/fmin.c", -"musl/src/math/aarch64/fminf.c", -"musl/src/math/aarch64/llrint.c", -"musl/src/math/aarch64/llrintf.c", -"musl/src/math/aarch64/llround.c", -"musl/src/math/aarch64/llroundf.c", -"musl/src/math/aarch64/lrint.c", -"musl/src/math/aarch64/lrintf.c", -"musl/src/math/aarch64/lround.c", -"musl/src/math/aarch64/lroundf.c", -"musl/src/math/aarch64/nearbyint.c", -"musl/src/math/aarch64/nearbyintf.c", -"musl/src/math/aarch64/rint.c", -"musl/src/math/aarch64/rintf.c", -"musl/src/math/aarch64/round.c", -"musl/src/math/aarch64/roundf.c", -"musl/src/math/aarch64/sqrt.c", -"musl/src/math/aarch64/sqrtf.c", -"musl/src/math/aarch64/trunc.c", -"musl/src/math/aarch64/truncf.c", -"musl/src/math/acos.c", -"musl/src/math/acosf.c", -"musl/src/math/acosh.c", -"musl/src/math/acoshf.c", -"musl/src/math/acoshl.c", -"musl/src/math/acosl.c", -"musl/src/math/arm/fabs.c", -"musl/src/math/arm/fabsf.c", -"musl/src/math/arm/fma.c", -"musl/src/math/arm/fmaf.c", -"musl/src/math/arm/sqrt.c", -"musl/src/math/arm/sqrtf.c", -"musl/src/math/asin.c", -"musl/src/math/asinf.c", -"musl/src/math/asinh.c", -"musl/src/math/asinhf.c", -"musl/src/math/asinhl.c", -"musl/src/math/asinl.c", -"musl/src/math/atan.c", -"musl/src/math/atan2.c", -"musl/src/math/atan2f.c", -"musl/src/math/atan2l.c", -"musl/src/math/atanf.c", -"musl/src/math/atanh.c", -"musl/src/math/atanhf.c", -"musl/src/math/atanhl.c", -"musl/src/math/atanl.c", -"musl/src/math/cbrt.c", -"musl/src/math/cbrtf.c", -"musl/src/math/cbrtl.c", -"musl/src/math/ceil.c", -"musl/src/math/ceilf.c", -"musl/src/math/ceill.c", -"musl/src/math/copysign.c", -"musl/src/math/copysignf.c", -"musl/src/math/copysignl.c", -"musl/src/math/cos.c", -"musl/src/math/cosf.c", -"musl/src/math/cosh.c", -"musl/src/math/coshf.c", -"musl/src/math/coshl.c", -"musl/src/math/cosl.c", -"musl/src/math/erf.c", -"musl/src/math/erff.c", -"musl/src/math/erfl.c", -"musl/src/math/exp.c", -"musl/src/math/exp10.c", -"musl/src/math/exp10f.c", -"musl/src/math/exp10l.c", -"musl/src/math/exp2.c", -"musl/src/math/exp2f.c", -"musl/src/math/exp2f_data.c", -"musl/src/math/exp2l.c", -"musl/src/math/exp_data.c", -"musl/src/math/expf.c", -"musl/src/math/expl.c", -"musl/src/math/expm1.c", -"musl/src/math/expm1f.c", -"musl/src/math/expm1l.c", -"musl/src/math/fabs.c", -"musl/src/math/fabsf.c", -"musl/src/math/fabsl.c", -"musl/src/math/fdim.c", -"musl/src/math/fdimf.c", -"musl/src/math/fdiml.c", -"musl/src/math/finite.c", -"musl/src/math/finitef.c", -"musl/src/math/floor.c", -"musl/src/math/floorf.c", -"musl/src/math/floorl.c", -"musl/src/math/fma.c", -"musl/src/math/fmaf.c", -"musl/src/math/fmal.c", -"musl/src/math/fmax.c", -"musl/src/math/fmaxf.c", -"musl/src/math/fmaxl.c", -"musl/src/math/fmin.c", -"musl/src/math/fminf.c", -"musl/src/math/fminl.c", -"musl/src/math/fmod.c", -"musl/src/math/fmodf.c", -"musl/src/math/fmodl.c", -"musl/src/math/frexp.c", -"musl/src/math/frexpf.c", -"musl/src/math/frexpl.c", -"musl/src/math/hypot.c", -"musl/src/math/hypotf.c", -"musl/src/math/hypotl.c", -"musl/src/math/i386/__invtrigl.s", -"musl/src/math/i386/acos.s", -"musl/src/math/i386/acosf.s", -"musl/src/math/i386/acosl.s", -"musl/src/math/i386/asin.s", -"musl/src/math/i386/asinf.s", -"musl/src/math/i386/asinl.s", -"musl/src/math/i386/atan.s", -"musl/src/math/i386/atan2.s", -"musl/src/math/i386/atan2f.s", -"musl/src/math/i386/atan2l.s", -"musl/src/math/i386/atanf.s", -"musl/src/math/i386/atanl.s", -"musl/src/math/i386/ceil.s", -"musl/src/math/i386/ceilf.s", -"musl/src/math/i386/ceill.s", -"musl/src/math/i386/exp2l.s", -"musl/src/math/i386/exp_ld.s", -"musl/src/math/i386/expl.s", -"musl/src/math/i386/expm1l.s", -"musl/src/math/i386/fabs.s", -"musl/src/math/i386/fabsf.s", -"musl/src/math/i386/fabsl.s", -"musl/src/math/i386/floor.s", -"musl/src/math/i386/floorf.s", -"musl/src/math/i386/floorl.s", -"musl/src/math/i386/fmod.s", -"musl/src/math/i386/fmodf.s", -"musl/src/math/i386/fmodl.s", -"musl/src/math/i386/hypot.s", -"musl/src/math/i386/hypotf.s", -"musl/src/math/i386/ldexp.s", -"musl/src/math/i386/ldexpf.s", -"musl/src/math/i386/ldexpl.s", -"musl/src/math/i386/llrint.s", -"musl/src/math/i386/llrintf.s", -"musl/src/math/i386/llrintl.s", -"musl/src/math/i386/log.s", -"musl/src/math/i386/log10.s", -"musl/src/math/i386/log10f.s", -"musl/src/math/i386/log10l.s", -"musl/src/math/i386/log1p.s", -"musl/src/math/i386/log1pf.s", -"musl/src/math/i386/log1pl.s", -"musl/src/math/i386/log2.s", -"musl/src/math/i386/log2f.s", -"musl/src/math/i386/log2l.s", -"musl/src/math/i386/logf.s", -"musl/src/math/i386/logl.s", -"musl/src/math/i386/lrint.s", -"musl/src/math/i386/lrintf.s", -"musl/src/math/i386/lrintl.s", -"musl/src/math/i386/remainder.s", -"musl/src/math/i386/remainderf.s", -"musl/src/math/i386/remainderl.s", -"musl/src/math/i386/remquo.s", -"musl/src/math/i386/remquof.s", -"musl/src/math/i386/remquol.s", -"musl/src/math/i386/rint.s", -"musl/src/math/i386/rintf.s", -"musl/src/math/i386/rintl.s", -"musl/src/math/i386/scalbln.s", -"musl/src/math/i386/scalblnf.s", -"musl/src/math/i386/scalblnl.s", -"musl/src/math/i386/scalbn.s", -"musl/src/math/i386/scalbnf.s", -"musl/src/math/i386/scalbnl.s", -"musl/src/math/i386/sqrt.s", -"musl/src/math/i386/sqrtf.s", -"musl/src/math/i386/sqrtl.s", -"musl/src/math/i386/trunc.s", -"musl/src/math/i386/truncf.s", -"musl/src/math/i386/truncl.s", -"musl/src/math/ilogb.c", -"musl/src/math/ilogbf.c", -"musl/src/math/ilogbl.c", -"musl/src/math/j0.c", -"musl/src/math/j0f.c", -"musl/src/math/j1.c", -"musl/src/math/j1f.c", -"musl/src/math/jn.c", -"musl/src/math/jnf.c", -"musl/src/math/ldexp.c", -"musl/src/math/ldexpf.c", -"musl/src/math/ldexpl.c", -"musl/src/math/lgamma.c", -"musl/src/math/lgamma_r.c", -"musl/src/math/lgammaf.c", -"musl/src/math/lgammaf_r.c", -"musl/src/math/lgammal.c", -"musl/src/math/llrint.c", -"musl/src/math/llrintf.c", -"musl/src/math/llrintl.c", -"musl/src/math/llround.c", -"musl/src/math/llroundf.c", -"musl/src/math/llroundl.c", -"musl/src/math/log.c", -"musl/src/math/log10.c", -"musl/src/math/log10f.c", -"musl/src/math/log10l.c", -"musl/src/math/log1p.c", -"musl/src/math/log1pf.c", -"musl/src/math/log1pl.c", -"musl/src/math/log2.c", -"musl/src/math/log2_data.c", -"musl/src/math/log2f.c", -"musl/src/math/log2f_data.c", -"musl/src/math/log2l.c", -"musl/src/math/log_data.c", -"musl/src/math/logb.c", -"musl/src/math/logbf.c", -"musl/src/math/logbl.c", -"musl/src/math/logf.c", -"musl/src/math/logf_data.c", -"musl/src/math/logl.c", -"musl/src/math/lrint.c", -"musl/src/math/lrintf.c", -"musl/src/math/lrintl.c", -"musl/src/math/lround.c", -"musl/src/math/lroundf.c", -"musl/src/math/lroundl.c", -"musl/src/math/mips/fabs.c", -"musl/src/math/mips/fabsf.c", -"musl/src/math/mips/sqrt.c", -"musl/src/math/mips/sqrtf.c", -"musl/src/math/modf.c", -"musl/src/math/modff.c", -"musl/src/math/modfl.c", -"musl/src/math/nan.c", -"musl/src/math/nanf.c", -"musl/src/math/nanl.c", -"musl/src/math/nearbyint.c", -"musl/src/math/nearbyintf.c", -"musl/src/math/nearbyintl.c", -"musl/src/math/nextafter.c", -"musl/src/math/nextafterf.c", -"musl/src/math/nextafterl.c", -"musl/src/math/nexttoward.c", -"musl/src/math/nexttowardf.c", -"musl/src/math/nexttowardl.c", -"musl/src/math/pow.c", -"musl/src/math/pow_data.c", -"musl/src/math/powerpc/fabs.c", -"musl/src/math/powerpc/fabsf.c", -"musl/src/math/powerpc/fma.c", -"musl/src/math/powerpc/fmaf.c", -"musl/src/math/powerpc/sqrt.c", -"musl/src/math/powerpc/sqrtf.c", -"musl/src/math/powerpc64/ceil.c", -"musl/src/math/powerpc64/ceilf.c", -"musl/src/math/powerpc64/fabs.c", -"musl/src/math/powerpc64/fabsf.c", -"musl/src/math/powerpc64/floor.c", -"musl/src/math/powerpc64/floorf.c", -"musl/src/math/powerpc64/fma.c", -"musl/src/math/powerpc64/fmaf.c", -"musl/src/math/powerpc64/fmax.c", -"musl/src/math/powerpc64/fmaxf.c", -"musl/src/math/powerpc64/fmin.c", -"musl/src/math/powerpc64/fminf.c", -"musl/src/math/powerpc64/lrint.c", -"musl/src/math/powerpc64/lrintf.c", -"musl/src/math/powerpc64/lround.c", -"musl/src/math/powerpc64/lroundf.c", -"musl/src/math/powerpc64/round.c", -"musl/src/math/powerpc64/roundf.c", -"musl/src/math/powerpc64/sqrt.c", -"musl/src/math/powerpc64/sqrtf.c", -"musl/src/math/powerpc64/trunc.c", -"musl/src/math/powerpc64/truncf.c", -"musl/src/math/powf.c", -"musl/src/math/powf_data.c", -"musl/src/math/powl.c", -"musl/src/math/remainder.c", -"musl/src/math/remainderf.c", -"musl/src/math/remainderl.c", -"musl/src/math/remquo.c", -"musl/src/math/remquof.c", -"musl/src/math/remquol.c", -"musl/src/math/rint.c", -"musl/src/math/rintf.c", -"musl/src/math/rintl.c", -"musl/src/math/riscv64/copysign.c", -"musl/src/math/riscv64/copysignf.c", -"musl/src/math/riscv64/fabs.c", -"musl/src/math/riscv64/fabsf.c", -"musl/src/math/riscv64/fma.c", -"musl/src/math/riscv64/fmaf.c", -"musl/src/math/riscv64/fmax.c", -"musl/src/math/riscv64/fmaxf.c", -"musl/src/math/riscv64/fmin.c", -"musl/src/math/riscv64/fminf.c", -"musl/src/math/riscv64/sqrt.c", -"musl/src/math/riscv64/sqrtf.c", -"musl/src/math/round.c", -"musl/src/math/roundf.c", -"musl/src/math/roundl.c", -"musl/src/math/s390x/ceil.c", -"musl/src/math/s390x/ceilf.c", -"musl/src/math/s390x/ceill.c", -"musl/src/math/s390x/fabs.c", -"musl/src/math/s390x/fabsf.c", -"musl/src/math/s390x/fabsl.c", -"musl/src/math/s390x/floor.c", -"musl/src/math/s390x/floorf.c", -"musl/src/math/s390x/floorl.c", -"musl/src/math/s390x/fma.c", -"musl/src/math/s390x/fmaf.c", -"musl/src/math/s390x/nearbyint.c", -"musl/src/math/s390x/nearbyintf.c", -"musl/src/math/s390x/nearbyintl.c", -"musl/src/math/s390x/rint.c", -"musl/src/math/s390x/rintf.c", -"musl/src/math/s390x/rintl.c", -"musl/src/math/s390x/round.c", -"musl/src/math/s390x/roundf.c", -"musl/src/math/s390x/roundl.c", -"musl/src/math/s390x/sqrt.c", -"musl/src/math/s390x/sqrtf.c", -"musl/src/math/s390x/sqrtl.c", -"musl/src/math/s390x/trunc.c", -"musl/src/math/s390x/truncf.c", -"musl/src/math/s390x/truncl.c", -"musl/src/math/scalb.c", -"musl/src/math/scalbf.c", -"musl/src/math/scalbln.c", -"musl/src/math/scalblnf.c", -"musl/src/math/scalblnl.c", -"musl/src/math/scalbn.c", -"musl/src/math/scalbnf.c", -"musl/src/math/scalbnl.c", -"musl/src/math/signgam.c", -"musl/src/math/significand.c", -"musl/src/math/significandf.c", -"musl/src/math/sin.c", -"musl/src/math/sincos.c", -"musl/src/math/sincosf.c", -"musl/src/math/sincosl.c", -"musl/src/math/sinf.c", -"musl/src/math/sinh.c", -"musl/src/math/sinhf.c", -"musl/src/math/sinhl.c", -"musl/src/math/sinl.c", -"musl/src/math/sqrt.c", -"musl/src/math/sqrtf.c", -"musl/src/math/sqrtl.c", -"musl/src/math/tan.c", -"musl/src/math/tanf.c", -"musl/src/math/tanh.c", -"musl/src/math/tanhf.c", -"musl/src/math/tanhl.c", -"musl/src/math/tanl.c", -"musl/src/math/tgamma.c", -"musl/src/math/tgammaf.c", -"musl/src/math/tgammal.c", -"musl/src/math/trunc.c", -"musl/src/math/truncf.c", -"musl/src/math/truncl.c", -"musl/src/math/x32/__invtrigl.s", -"musl/src/math/x32/acosl.s", -"musl/src/math/x32/asinl.s", -"musl/src/math/x32/atan2l.s", -"musl/src/math/x32/atanl.s", -"musl/src/math/x32/ceill.s", -"musl/src/math/x32/exp2l.s", -"musl/src/math/x32/expl.s", -"musl/src/math/x32/expm1l.s", -"musl/src/math/x32/fabs.s", -"musl/src/math/x32/fabsf.s", -"musl/src/math/x32/fabsl.s", -"musl/src/math/x32/floorl.s", -"musl/src/math/x32/fma.c", -"musl/src/math/x32/fmaf.c", -"musl/src/math/x32/fmodl.s", -"musl/src/math/x32/llrint.s", -"musl/src/math/x32/llrintf.s", -"musl/src/math/x32/llrintl.s", -"musl/src/math/x32/log10l.s", -"musl/src/math/x32/log1pl.s", -"musl/src/math/x32/log2l.s", -"musl/src/math/x32/logl.s", -"musl/src/math/x32/lrint.s", -"musl/src/math/x32/lrintf.s", -"musl/src/math/x32/lrintl.s", -"musl/src/math/x32/remainderl.s", -"musl/src/math/x32/rintl.s", -"musl/src/math/x32/sqrt.s", -"musl/src/math/x32/sqrtf.s", -"musl/src/math/x32/sqrtl.s", -"musl/src/math/x32/truncl.s", -"musl/src/math/x86_64/__invtrigl.s", -"musl/src/math/x86_64/acosl.s", -"musl/src/math/x86_64/asinl.s", -"musl/src/math/x86_64/atan2l.s", -"musl/src/math/x86_64/atanl.s", -"musl/src/math/x86_64/ceill.s", -"musl/src/math/x86_64/exp2l.s", -"musl/src/math/x86_64/expl.s", -"musl/src/math/x86_64/expm1l.s", -"musl/src/math/x86_64/fabs.s", -"musl/src/math/x86_64/fabsf.s", -"musl/src/math/x86_64/fabsl.s", -"musl/src/math/x86_64/floorl.s", -"musl/src/math/x86_64/fma.c", -"musl/src/math/x86_64/fmaf.c", -"musl/src/math/x86_64/fmodl.s", -"musl/src/math/x86_64/llrint.s", -"musl/src/math/x86_64/llrintf.s", -"musl/src/math/x86_64/llrintl.s", -"musl/src/math/x86_64/log10l.s", -"musl/src/math/x86_64/log1pl.s", -"musl/src/math/x86_64/log2l.s", -"musl/src/math/x86_64/logl.s", -"musl/src/math/x86_64/lrint.s", -"musl/src/math/x86_64/lrintf.s", -"musl/src/math/x86_64/lrintl.s", -"musl/src/math/x86_64/remainderl.s", -"musl/src/math/x86_64/rintl.s", -"musl/src/math/x86_64/sqrt.s", -"musl/src/math/x86_64/sqrtf.s", -"musl/src/math/x86_64/sqrtl.s", -"musl/src/math/x86_64/truncl.s", -"musl/src/misc/a64l.c", -"musl/src/misc/basename.c", -"musl/src/misc/dirname.c", -"musl/src/misc/ffs.c", -"musl/src/misc/ffsl.c", -"musl/src/misc/ffsll.c", -"musl/src/misc/fmtmsg.c", -"musl/src/misc/forkpty.c", -"musl/src/misc/get_current_dir_name.c", -"musl/src/misc/getauxval.c", -"musl/src/misc/getdomainname.c", -"musl/src/misc/getentropy.c", -"musl/src/misc/gethostid.c", -"musl/src/misc/getopt.c", -"musl/src/misc/getopt_long.c", -"musl/src/misc/getpriority.c", -"musl/src/misc/getresgid.c", -"musl/src/misc/getresuid.c", -"musl/src/misc/getrlimit.c", -"musl/src/misc/getrusage.c", -"musl/src/misc/getsubopt.c", -"musl/src/misc/initgroups.c", -"musl/src/misc/ioctl.c", -"musl/src/misc/issetugid.c", -"musl/src/misc/lockf.c", -"musl/src/misc/login_tty.c", -"musl/src/misc/mntent.c", -"musl/src/misc/nftw.c", -"musl/src/misc/openpty.c", -"musl/src/misc/ptsname.c", -"musl/src/misc/pty.c", -"musl/src/misc/realpath.c", -"musl/src/misc/setdomainname.c", -"musl/src/misc/setpriority.c", -"musl/src/misc/setrlimit.c", -"musl/src/misc/syscall.c", -"musl/src/misc/syslog.c", -"musl/src/misc/uname.c", -"musl/src/misc/wordexp.c", -"musl/src/mman/madvise.c", -"musl/src/mman/mincore.c", -"musl/src/mman/mlock.c", -"musl/src/mman/mlockall.c", -"musl/src/mman/mmap.c", -"musl/src/mman/mprotect.c", -"musl/src/mman/mremap.c", -"musl/src/mman/msync.c", -"musl/src/mman/munlock.c", -"musl/src/mman/munlockall.c", -"musl/src/mman/munmap.c", -"musl/src/mman/posix_madvise.c", -"musl/src/mman/shm_open.c", -"musl/src/mq/mq_close.c", -"musl/src/mq/mq_getattr.c", -"musl/src/mq/mq_notify.c", -"musl/src/mq/mq_open.c", -"musl/src/mq/mq_receive.c", -"musl/src/mq/mq_send.c", -"musl/src/mq/mq_setattr.c", -"musl/src/mq/mq_timedreceive.c", -"musl/src/mq/mq_timedsend.c", -"musl/src/mq/mq_unlink.c", -"musl/src/multibyte/btowc.c", -"musl/src/multibyte/c16rtomb.c", -"musl/src/multibyte/c32rtomb.c", -"musl/src/multibyte/internal.c", -"musl/src/multibyte/mblen.c", -"musl/src/multibyte/mbrlen.c", -"musl/src/multibyte/mbrtoc16.c", -"musl/src/multibyte/mbrtoc32.c", -"musl/src/multibyte/mbrtowc.c", -"musl/src/multibyte/mbsinit.c", -"musl/src/multibyte/mbsnrtowcs.c", -"musl/src/multibyte/mbsrtowcs.c", -"musl/src/multibyte/mbstowcs.c", -"musl/src/multibyte/mbtowc.c", -"musl/src/multibyte/wcrtomb.c", -"musl/src/multibyte/wcsnrtombs.c", -"musl/src/multibyte/wcsrtombs.c", -"musl/src/multibyte/wcstombs.c", -"musl/src/multibyte/wctob.c", -"musl/src/multibyte/wctomb.c", -"musl/src/network/accept.c", -"musl/src/network/accept4.c", -"musl/src/network/bind.c", -"musl/src/network/connect.c", -"musl/src/network/dn_comp.c", -"musl/src/network/dn_expand.c", -"musl/src/network/dn_skipname.c", -"musl/src/network/dns_parse.c", -"musl/src/network/ent.c", -"musl/src/network/ether.c", -"musl/src/network/freeaddrinfo.c", -"musl/src/network/gai_strerror.c", -"musl/src/network/getaddrinfo.c", -"musl/src/network/gethostbyaddr.c", -"musl/src/network/gethostbyaddr_r.c", -"musl/src/network/gethostbyname.c", -"musl/src/network/gethostbyname2.c", -"musl/src/network/gethostbyname2_r.c", -"musl/src/network/gethostbyname_r.c", -"musl/src/network/getifaddrs.c", -"musl/src/network/getnameinfo.c", -"musl/src/network/getpeername.c", -"musl/src/network/getservbyname.c", -"musl/src/network/getservbyname_r.c", -"musl/src/network/getservbyport.c", -"musl/src/network/getservbyport_r.c", -"musl/src/network/getsockname.c", -"musl/src/network/getsockopt.c", -"musl/src/network/h_errno.c", -"musl/src/network/herror.c", -"musl/src/network/hstrerror.c", -"musl/src/network/htonl.c", -"musl/src/network/htons.c", -"musl/src/network/if_freenameindex.c", -"musl/src/network/if_indextoname.c", -"musl/src/network/if_nameindex.c", -"musl/src/network/if_nametoindex.c", -"musl/src/network/in6addr_any.c", -"musl/src/network/in6addr_loopback.c", -"musl/src/network/inet_addr.c", -"musl/src/network/inet_aton.c", -"musl/src/network/inet_legacy.c", -"musl/src/network/inet_ntoa.c", -"musl/src/network/inet_ntop.c", -"musl/src/network/inet_pton.c", -"musl/src/network/listen.c", -"musl/src/network/lookup_ipliteral.c", -"musl/src/network/lookup_name.c", -"musl/src/network/lookup_serv.c", -"musl/src/network/netlink.c", -"musl/src/network/netname.c", -"musl/src/network/ns_parse.c", -"musl/src/network/ntohl.c", -"musl/src/network/ntohs.c", -"musl/src/network/proto.c", -"musl/src/network/recv.c", -"musl/src/network/recvfrom.c", -"musl/src/network/recvmmsg.c", -"musl/src/network/recvmsg.c", -"musl/src/network/res_init.c", -"musl/src/network/res_mkquery.c", -"musl/src/network/res_msend.c", -"musl/src/network/res_query.c", -"musl/src/network/res_querydomain.c", -"musl/src/network/res_send.c", -"musl/src/network/res_state.c", -"musl/src/network/resolvconf.c", -"musl/src/network/send.c", -"musl/src/network/sendmmsg.c", -"musl/src/network/sendmsg.c", -"musl/src/network/sendto.c", -"musl/src/network/serv.c", -"musl/src/network/setsockopt.c", -"musl/src/network/shutdown.c", -"musl/src/network/sockatmark.c", -"musl/src/network/socket.c", -"musl/src/network/socketpair.c", -"musl/src/passwd/fgetgrent.c", -"musl/src/passwd/fgetpwent.c", -"musl/src/passwd/fgetspent.c", -"musl/src/passwd/getgr_a.c", -"musl/src/passwd/getgr_r.c", -"musl/src/passwd/getgrent.c", -"musl/src/passwd/getgrent_a.c", -"musl/src/passwd/getgrouplist.c", -"musl/src/passwd/getpw_a.c", -"musl/src/passwd/getpw_r.c", -"musl/src/passwd/getpwent.c", -"musl/src/passwd/getpwent_a.c", -"musl/src/passwd/getspent.c", -"musl/src/passwd/getspnam.c", -"musl/src/passwd/getspnam_r.c", -"musl/src/passwd/lckpwdf.c", -"musl/src/passwd/nscd_query.c", -"musl/src/passwd/putgrent.c", -"musl/src/passwd/putpwent.c", -"musl/src/passwd/putspent.c", -"musl/src/prng/__rand48_step.c", -"musl/src/prng/__seed48.c", -"musl/src/prng/drand48.c", -"musl/src/prng/lcong48.c", -"musl/src/prng/lrand48.c", -"musl/src/prng/mrand48.c", -"musl/src/prng/rand.c", -"musl/src/prng/rand_r.c", -"musl/src/prng/random.c", -"musl/src/prng/seed48.c", -"musl/src/prng/srand48.c", -"musl/src/process/arm/vfork.s", -"musl/src/process/execl.c", -"musl/src/process/execle.c", -"musl/src/process/execlp.c", -"musl/src/process/execv.c", -"musl/src/process/execve.c", -"musl/src/process/execvp.c", -"musl/src/process/fexecve.c", -"musl/src/process/fork.c", -"musl/src/process/i386/vfork.s", -"musl/src/process/posix_spawn.c", -"musl/src/process/posix_spawn_file_actions_addchdir.c", -"musl/src/process/posix_spawn_file_actions_addclose.c", -"musl/src/process/posix_spawn_file_actions_adddup2.c", -"musl/src/process/posix_spawn_file_actions_addfchdir.c", -"musl/src/process/posix_spawn_file_actions_addopen.c", -"musl/src/process/posix_spawn_file_actions_destroy.c", -"musl/src/process/posix_spawn_file_actions_init.c", -"musl/src/process/posix_spawnattr_destroy.c", -"musl/src/process/posix_spawnattr_getflags.c", -"musl/src/process/posix_spawnattr_getpgroup.c", -"musl/src/process/posix_spawnattr_getsigdefault.c", -"musl/src/process/posix_spawnattr_getsigmask.c", -"musl/src/process/posix_spawnattr_init.c", -"musl/src/process/posix_spawnattr_sched.c", -"musl/src/process/posix_spawnattr_setflags.c", -"musl/src/process/posix_spawnattr_setpgroup.c", -"musl/src/process/posix_spawnattr_setsigdefault.c", -"musl/src/process/posix_spawnattr_setsigmask.c", -"musl/src/process/posix_spawnp.c", -"musl/src/process/s390x/vfork.s", -"musl/src/process/sh/vfork.s", -"musl/src/process/system.c", -"musl/src/process/vfork.c", -"musl/src/process/wait.c", -"musl/src/process/waitid.c", -"musl/src/process/waitpid.c", -"musl/src/process/x32/vfork.s", -"musl/src/process/x86_64/vfork.s", -"musl/src/regex/fnmatch.c", -"musl/src/regex/glob.c", -"musl/src/regex/regcomp.c", -"musl/src/regex/regerror.c", -"musl/src/regex/regexec.c", -"musl/src/regex/tre-mem.c", -"musl/src/sched/affinity.c", -"musl/src/sched/sched_cpucount.c", -"musl/src/sched/sched_get_priority_max.c", -"musl/src/sched/sched_getcpu.c", -"musl/src/sched/sched_getparam.c", -"musl/src/sched/sched_getscheduler.c", -"musl/src/sched/sched_rr_get_interval.c", -"musl/src/sched/sched_setparam.c", -"musl/src/sched/sched_setscheduler.c", -"musl/src/sched/sched_yield.c", -"musl/src/search/hsearch.c", -"musl/src/search/insque.c", -"musl/src/search/lsearch.c", -"musl/src/search/tdelete.c", -"musl/src/search/tdestroy.c", -"musl/src/search/tfind.c", -"musl/src/search/tsearch.c", -"musl/src/search/twalk.c", -"musl/src/select/poll.c", -"musl/src/select/pselect.c", -"musl/src/select/select.c", -"musl/src/setjmp/aarch64/longjmp.s", -"musl/src/setjmp/aarch64/setjmp.s", -"musl/src/setjmp/arm/longjmp.S", -"musl/src/setjmp/arm/setjmp.S", -"musl/src/setjmp/i386/longjmp.s", -"musl/src/setjmp/i386/setjmp.s", -"musl/src/setjmp/longjmp.c", -"musl/src/setjmp/m68k/longjmp.s", -"musl/src/setjmp/m68k/setjmp.s", -"musl/src/setjmp/microblaze/longjmp.s", -"musl/src/setjmp/microblaze/setjmp.s", -"musl/src/setjmp/mips/longjmp.S", -"musl/src/setjmp/mips/setjmp.S", -"musl/src/setjmp/mips64/longjmp.S", -"musl/src/setjmp/mips64/setjmp.S", -"musl/src/setjmp/mipsn32/longjmp.S", -"musl/src/setjmp/mipsn32/setjmp.S", -"musl/src/setjmp/or1k/longjmp.s", -"musl/src/setjmp/or1k/setjmp.s", -"musl/src/setjmp/powerpc/longjmp.S", -"musl/src/setjmp/powerpc/setjmp.S", -"musl/src/setjmp/powerpc64/longjmp.s", -"musl/src/setjmp/powerpc64/setjmp.s", -"musl/src/setjmp/riscv64/longjmp.S", -"musl/src/setjmp/riscv64/setjmp.S", -"musl/src/setjmp/s390x/longjmp.s", -"musl/src/setjmp/s390x/setjmp.s", -"musl/src/setjmp/setjmp.c", -"musl/src/setjmp/sh/longjmp.S", -"musl/src/setjmp/sh/setjmp.S", -"musl/src/setjmp/x32/longjmp.s", -"musl/src/setjmp/x32/setjmp.s", -"musl/src/setjmp/x86_64/longjmp.s", -"musl/src/setjmp/x86_64/setjmp.s", -"musl/src/signal/aarch64/restore.s", -"musl/src/signal/aarch64/sigsetjmp.s", -"musl/src/signal/arm/restore.s", -"musl/src/signal/arm/sigsetjmp.s", -"musl/src/signal/block.c", -"musl/src/signal/getitimer.c", -"musl/src/signal/i386/restore.s", -"musl/src/signal/i386/sigsetjmp.s", -"musl/src/signal/kill.c", -"musl/src/signal/killpg.c", -"musl/src/signal/m68k/sigsetjmp.s", -"musl/src/signal/microblaze/restore.s", -"musl/src/signal/microblaze/sigsetjmp.s", -"musl/src/signal/mips/restore.s", -"musl/src/signal/mips/sigsetjmp.s", -"musl/src/signal/mips64/restore.s", -"musl/src/signal/mips64/sigsetjmp.s", -"musl/src/signal/mipsn32/restore.s", -"musl/src/signal/mipsn32/sigsetjmp.s", -"musl/src/signal/or1k/sigsetjmp.s", -"musl/src/signal/powerpc/restore.s", -"musl/src/signal/powerpc/sigsetjmp.s", -"musl/src/signal/powerpc64/restore.s", -"musl/src/signal/powerpc64/sigsetjmp.s", -"musl/src/signal/psiginfo.c", -"musl/src/signal/psignal.c", -"musl/src/signal/raise.c", -"musl/src/signal/restore.c", -"musl/src/signal/riscv64/restore.s", -"musl/src/signal/riscv64/sigsetjmp.s", -"musl/src/signal/s390x/restore.s", -"musl/src/signal/s390x/sigsetjmp.s", -"musl/src/signal/setitimer.c", -"musl/src/signal/sh/restore.s", -"musl/src/signal/sh/sigsetjmp.s", -"musl/src/signal/sigaction.c", -"musl/src/signal/sigaddset.c", -"musl/src/signal/sigaltstack.c", -"musl/src/signal/sigandset.c", -"musl/src/signal/sigdelset.c", -"musl/src/signal/sigemptyset.c", -"musl/src/signal/sigfillset.c", -"musl/src/signal/sighold.c", -"musl/src/signal/sigignore.c", -"musl/src/signal/siginterrupt.c", -"musl/src/signal/sigisemptyset.c", -"musl/src/signal/sigismember.c", -"musl/src/signal/siglongjmp.c", -"musl/src/signal/signal.c", -"musl/src/signal/sigorset.c", -"musl/src/signal/sigpause.c", -"musl/src/signal/sigpending.c", -"musl/src/signal/sigprocmask.c", -"musl/src/signal/sigqueue.c", -"musl/src/signal/sigrelse.c", -"musl/src/signal/sigrtmax.c", -"musl/src/signal/sigrtmin.c", -"musl/src/signal/sigset.c", -"musl/src/signal/sigsetjmp.c", -"musl/src/signal/sigsetjmp_tail.c", -"musl/src/signal/sigsuspend.c", -"musl/src/signal/sigtimedwait.c", -"musl/src/signal/sigwait.c", -"musl/src/signal/sigwaitinfo.c", -"musl/src/signal/x32/getitimer.c", -"musl/src/signal/x32/restore.s", -"musl/src/signal/x32/setitimer.c", -"musl/src/signal/x32/sigsetjmp.s", -"musl/src/signal/x86_64/restore.s", -"musl/src/signal/x86_64/sigsetjmp.s", -"musl/src/stat/__xstat.c", -"musl/src/stat/chmod.c", -"musl/src/stat/fchmod.c", -"musl/src/stat/fchmodat.c", -"musl/src/stat/fstat.c", -"musl/src/stat/fstatat.c", -"musl/src/stat/futimens.c", -"musl/src/stat/futimesat.c", -"musl/src/stat/lchmod.c", -"musl/src/stat/lstat.c", -"musl/src/stat/mkdir.c", -"musl/src/stat/mkdirat.c", -"musl/src/stat/mkfifo.c", -"musl/src/stat/mkfifoat.c", -"musl/src/stat/mknod.c", -"musl/src/stat/mknodat.c", -"musl/src/stat/stat.c", -"musl/src/stat/statvfs.c", -"musl/src/stat/umask.c", -"musl/src/stat/utimensat.c", -"musl/src/stdio/__fclose_ca.c", -"musl/src/stdio/__fdopen.c", -"musl/src/stdio/__fmodeflags.c", -"musl/src/stdio/__fopen_rb_ca.c", -"musl/src/stdio/__lockfile.c", -"musl/src/stdio/__overflow.c", -"musl/src/stdio/__stdio_close.c", -"musl/src/stdio/__stdio_exit.c", -"musl/src/stdio/__stdio_read.c", -"musl/src/stdio/__stdio_seek.c", -"musl/src/stdio/__stdio_write.c", -"musl/src/stdio/__stdout_write.c", -"musl/src/stdio/__string_read.c", -"musl/src/stdio/__toread.c", -"musl/src/stdio/__towrite.c", -"musl/src/stdio/__uflow.c", -"musl/src/stdio/asprintf.c", -"musl/src/stdio/clearerr.c", -"musl/src/stdio/dprintf.c", -"musl/src/stdio/ext.c", -"musl/src/stdio/ext2.c", -"musl/src/stdio/fclose.c", -"musl/src/stdio/feof.c", -"musl/src/stdio/ferror.c", -"musl/src/stdio/fflush.c", -"musl/src/stdio/fgetc.c", -"musl/src/stdio/fgetln.c", -"musl/src/stdio/fgetpos.c", -"musl/src/stdio/fgets.c", -"musl/src/stdio/fgetwc.c", -"musl/src/stdio/fgetws.c", -"musl/src/stdio/fileno.c", -"musl/src/stdio/flockfile.c", -"musl/src/stdio/fmemopen.c", -"musl/src/stdio/fopen.c", -"musl/src/stdio/fopencookie.c", -"musl/src/stdio/fprintf.c", -"musl/src/stdio/fputc.c", -"musl/src/stdio/fputs.c", -"musl/src/stdio/fputwc.c", -"musl/src/stdio/fputws.c", -"musl/src/stdio/fread.c", -"musl/src/stdio/freopen.c", -"musl/src/stdio/fscanf.c", -"musl/src/stdio/fseek.c", -"musl/src/stdio/fsetpos.c", -"musl/src/stdio/ftell.c", -"musl/src/stdio/ftrylockfile.c", -"musl/src/stdio/funlockfile.c", -"musl/src/stdio/fwide.c", -"musl/src/stdio/fwprintf.c", -"musl/src/stdio/fwrite.c", -"musl/src/stdio/fwscanf.c", -"musl/src/stdio/getc.c", -"musl/src/stdio/getc_unlocked.c", -"musl/src/stdio/getchar.c", -"musl/src/stdio/getchar_unlocked.c", -"musl/src/stdio/getdelim.c", -"musl/src/stdio/getline.c", -"musl/src/stdio/gets.c", -"musl/src/stdio/getw.c", -"musl/src/stdio/getwc.c", -"musl/src/stdio/getwchar.c", -"musl/src/stdio/ofl.c", -"musl/src/stdio/ofl_add.c", -"musl/src/stdio/open_memstream.c", -"musl/src/stdio/open_wmemstream.c", -"musl/src/stdio/pclose.c", -"musl/src/stdio/perror.c", -"musl/src/stdio/popen.c", -"musl/src/stdio/printf.c", -"musl/src/stdio/putc.c", -"musl/src/stdio/putc_unlocked.c", -"musl/src/stdio/putchar.c", -"musl/src/stdio/putchar_unlocked.c", -"musl/src/stdio/puts.c", -"musl/src/stdio/putw.c", -"musl/src/stdio/putwc.c", -"musl/src/stdio/putwchar.c", -"musl/src/stdio/remove.c", -"musl/src/stdio/rename.c", -"musl/src/stdio/rewind.c", -"musl/src/stdio/scanf.c", -"musl/src/stdio/setbuf.c", -"musl/src/stdio/setbuffer.c", -"musl/src/stdio/setlinebuf.c", -"musl/src/stdio/setvbuf.c", -"musl/src/stdio/snprintf.c", -"musl/src/stdio/sprintf.c", -"musl/src/stdio/sscanf.c", -"musl/src/stdio/stderr.c", -"musl/src/stdio/stdin.c", -"musl/src/stdio/stdout.c", -"musl/src/stdio/swprintf.c", -"musl/src/stdio/swscanf.c", -"musl/src/stdio/tempnam.c", -"musl/src/stdio/tmpfile.c", -"musl/src/stdio/tmpnam.c", -"musl/src/stdio/ungetc.c", -"musl/src/stdio/ungetwc.c", -"musl/src/stdio/vasprintf.c", -"musl/src/stdio/vdprintf.c", -"musl/src/stdio/vfprintf.c", -"musl/src/stdio/vfscanf.c", -"musl/src/stdio/vfwprintf.c", -"musl/src/stdio/vfwscanf.c", -"musl/src/stdio/vprintf.c", -"musl/src/stdio/vscanf.c", -"musl/src/stdio/vsnprintf.c", -"musl/src/stdio/vsprintf.c", -"musl/src/stdio/vsscanf.c", -"musl/src/stdio/vswprintf.c", -"musl/src/stdio/vswscanf.c", -"musl/src/stdio/vwprintf.c", -"musl/src/stdio/vwscanf.c", -"musl/src/stdio/wprintf.c", -"musl/src/stdio/wscanf.c", -"musl/src/stdlib/abs.c", -"musl/src/stdlib/atof.c", -"musl/src/stdlib/atoi.c", -"musl/src/stdlib/atol.c", -"musl/src/stdlib/atoll.c", -"musl/src/stdlib/bsearch.c", -"musl/src/stdlib/div.c", -"musl/src/stdlib/ecvt.c", -"musl/src/stdlib/fcvt.c", -"musl/src/stdlib/gcvt.c", -"musl/src/stdlib/imaxabs.c", -"musl/src/stdlib/imaxdiv.c", -"musl/src/stdlib/labs.c", -"musl/src/stdlib/ldiv.c", -"musl/src/stdlib/llabs.c", -"musl/src/stdlib/lldiv.c", -"musl/src/stdlib/qsort.c", -"musl/src/stdlib/strtod.c", -"musl/src/stdlib/strtol.c", -"musl/src/stdlib/wcstod.c", -"musl/src/stdlib/wcstol.c", -"musl/src/string/arm/__aeabi_memcpy.s", -"musl/src/string/arm/__aeabi_memset.s", -"musl/src/string/arm/memcpy.c", -"musl/src/string/arm/memcpy_le.S", -"musl/src/string/bcmp.c", -"musl/src/string/bcopy.c", -"musl/src/string/bzero.c", -"musl/src/string/explicit_bzero.c", -"musl/src/string/i386/memcpy.s", -"musl/src/string/i386/memmove.s", -"musl/src/string/i386/memset.s", -"musl/src/string/index.c", -"musl/src/string/memccpy.c", -"musl/src/string/memchr.c", -"musl/src/string/memcmp.c", -"musl/src/string/memcpy.c", -"musl/src/string/memmem.c", -"musl/src/string/memmove.c", -"musl/src/string/mempcpy.c", -"musl/src/string/memrchr.c", -"musl/src/string/memset.c", -"musl/src/string/rindex.c", -"musl/src/string/stpcpy.c", -"musl/src/string/stpncpy.c", -"musl/src/string/strcasecmp.c", -"musl/src/string/strcasestr.c", -"musl/src/string/strcat.c", -"musl/src/string/strchr.c", -"musl/src/string/strchrnul.c", -"musl/src/string/strcmp.c", -"musl/src/string/strcpy.c", -"musl/src/string/strcspn.c", -"musl/src/string/strdup.c", -"musl/src/string/strerror_r.c", -"musl/src/string/strlcat.c", -"musl/src/string/strlcpy.c", -"musl/src/string/strlen.c", -"musl/src/string/strncasecmp.c", -"musl/src/string/strncat.c", -"musl/src/string/strncmp.c", -"musl/src/string/strncpy.c", -"musl/src/string/strndup.c", -"musl/src/string/strnlen.c", -"musl/src/string/strpbrk.c", -"musl/src/string/strrchr.c", -"musl/src/string/strsep.c", -"musl/src/string/strsignal.c", -"musl/src/string/strspn.c", -"musl/src/string/strstr.c", -"musl/src/string/strtok.c", -"musl/src/string/strtok_r.c", -"musl/src/string/strverscmp.c", -"musl/src/string/swab.c", -"musl/src/string/wcpcpy.c", -"musl/src/string/wcpncpy.c", -"musl/src/string/wcscasecmp.c", -"musl/src/string/wcscasecmp_l.c", -"musl/src/string/wcscat.c", -"musl/src/string/wcschr.c", -"musl/src/string/wcscmp.c", -"musl/src/string/wcscpy.c", -"musl/src/string/wcscspn.c", -"musl/src/string/wcsdup.c", -"musl/src/string/wcslen.c", -"musl/src/string/wcsncasecmp.c", -"musl/src/string/wcsncasecmp_l.c", -"musl/src/string/wcsncat.c", -"musl/src/string/wcsncmp.c", -"musl/src/string/wcsncpy.c", -"musl/src/string/wcsnlen.c", -"musl/src/string/wcspbrk.c", -"musl/src/string/wcsrchr.c", -"musl/src/string/wcsspn.c", -"musl/src/string/wcsstr.c", -"musl/src/string/wcstok.c", -"musl/src/string/wcswcs.c", -"musl/src/string/wmemchr.c", -"musl/src/string/wmemcmp.c", -"musl/src/string/wmemcpy.c", -"musl/src/string/wmemmove.c", -"musl/src/string/wmemset.c", -"musl/src/string/x86_64/memcpy.s", -"musl/src/string/x86_64/memmove.s", -"musl/src/string/x86_64/memset.s", -"musl/src/temp/__randname.c", -"musl/src/temp/mkdtemp.c", -"musl/src/temp/mkostemp.c", -"musl/src/temp/mkostemps.c", -"musl/src/temp/mkstemp.c", -"musl/src/temp/mkstemps.c", -"musl/src/temp/mktemp.c", -"musl/src/termios/cfgetospeed.c", -"musl/src/termios/cfmakeraw.c", -"musl/src/termios/cfsetospeed.c", -"musl/src/termios/tcdrain.c", -"musl/src/termios/tcflow.c", -"musl/src/termios/tcflush.c", -"musl/src/termios/tcgetattr.c", -"musl/src/termios/tcgetsid.c", -"musl/src/termios/tcsendbreak.c", -"musl/src/termios/tcsetattr.c", -"musl/src/thread/__lock.c", -"musl/src/thread/__set_thread_area.c", -"musl/src/thread/__syscall_cp.c", -"musl/src/thread/__timedwait.c", -"musl/src/thread/__tls_get_addr.c", -"musl/src/thread/__unmapself.c", -"musl/src/thread/__wait.c", -"musl/src/thread/aarch64/__set_thread_area.s", -"musl/src/thread/aarch64/__unmapself.s", -"musl/src/thread/aarch64/clone.s", -"musl/src/thread/aarch64/syscall_cp.s", -"musl/src/thread/arm/__aeabi_read_tp.s", -"musl/src/thread/arm/__set_thread_area.c", -"musl/src/thread/arm/__unmapself.s", -"musl/src/thread/arm/atomics.s", -"musl/src/thread/arm/clone.s", -"musl/src/thread/arm/syscall_cp.s", -"musl/src/thread/call_once.c", -"musl/src/thread/clone.c", -"musl/src/thread/cnd_broadcast.c", -"musl/src/thread/cnd_destroy.c", -"musl/src/thread/cnd_init.c", -"musl/src/thread/cnd_signal.c", -"musl/src/thread/cnd_timedwait.c", -"musl/src/thread/cnd_wait.c", -"musl/src/thread/default_attr.c", -"musl/src/thread/i386/__set_thread_area.s", -"musl/src/thread/i386/__unmapself.s", -"musl/src/thread/i386/clone.s", -"musl/src/thread/i386/syscall_cp.s", -"musl/src/thread/i386/tls.s", -"musl/src/thread/lock_ptc.c", -"musl/src/thread/m68k/__m68k_read_tp.s", -"musl/src/thread/m68k/clone.s", -"musl/src/thread/m68k/syscall_cp.s", -"musl/src/thread/microblaze/__set_thread_area.s", -"musl/src/thread/microblaze/__unmapself.s", -"musl/src/thread/microblaze/clone.s", -"musl/src/thread/microblaze/syscall_cp.s", -"musl/src/thread/mips/__unmapself.s", -"musl/src/thread/mips/clone.s", -"musl/src/thread/mips/syscall_cp.s", -"musl/src/thread/mips64/__unmapself.s", -"musl/src/thread/mips64/clone.s", -"musl/src/thread/mips64/syscall_cp.s", -"musl/src/thread/mipsn32/__unmapself.s", -"musl/src/thread/mipsn32/clone.s", -"musl/src/thread/mipsn32/syscall_cp.s", -"musl/src/thread/mtx_destroy.c", -"musl/src/thread/mtx_init.c", -"musl/src/thread/mtx_lock.c", -"musl/src/thread/mtx_timedlock.c", -"musl/src/thread/mtx_trylock.c", -"musl/src/thread/mtx_unlock.c", -"musl/src/thread/or1k/__set_thread_area.s", -"musl/src/thread/or1k/__unmapself.s", -"musl/src/thread/or1k/clone.s", -"musl/src/thread/or1k/syscall_cp.s", -"musl/src/thread/powerpc/__set_thread_area.s", -"musl/src/thread/powerpc/__unmapself.s", -"musl/src/thread/powerpc/clone.s", -"musl/src/thread/powerpc/syscall_cp.s", -"musl/src/thread/powerpc64/__set_thread_area.s", -"musl/src/thread/powerpc64/__unmapself.s", -"musl/src/thread/powerpc64/clone.s", -"musl/src/thread/powerpc64/syscall_cp.s", -"musl/src/thread/pthread_atfork.c", -"musl/src/thread/pthread_attr_destroy.c", -"musl/src/thread/pthread_attr_get.c", -"musl/src/thread/pthread_attr_init.c", -"musl/src/thread/pthread_attr_setdetachstate.c", -"musl/src/thread/pthread_attr_setguardsize.c", -"musl/src/thread/pthread_attr_setinheritsched.c", -"musl/src/thread/pthread_attr_setschedparam.c", -"musl/src/thread/pthread_attr_setschedpolicy.c", -"musl/src/thread/pthread_attr_setscope.c", -"musl/src/thread/pthread_attr_setstack.c", -"musl/src/thread/pthread_attr_setstacksize.c", -"musl/src/thread/pthread_barrier_destroy.c", -"musl/src/thread/pthread_barrier_init.c", -"musl/src/thread/pthread_barrier_wait.c", -"musl/src/thread/pthread_barrierattr_destroy.c", -"musl/src/thread/pthread_barrierattr_init.c", -"musl/src/thread/pthread_barrierattr_setpshared.c", -"musl/src/thread/pthread_cancel.c", -"musl/src/thread/pthread_cleanup_push.c", -"musl/src/thread/pthread_cond_broadcast.c", -"musl/src/thread/pthread_cond_destroy.c", -"musl/src/thread/pthread_cond_init.c", -"musl/src/thread/pthread_cond_signal.c", -"musl/src/thread/pthread_cond_timedwait.c", -"musl/src/thread/pthread_cond_wait.c", -"musl/src/thread/pthread_condattr_destroy.c", -"musl/src/thread/pthread_condattr_init.c", -"musl/src/thread/pthread_condattr_setclock.c", -"musl/src/thread/pthread_condattr_setpshared.c", -"musl/src/thread/pthread_create.c", -"musl/src/thread/pthread_detach.c", -"musl/src/thread/pthread_equal.c", -"musl/src/thread/pthread_getattr_np.c", -"musl/src/thread/pthread_getconcurrency.c", -"musl/src/thread/pthread_getcpuclockid.c", -"musl/src/thread/pthread_getschedparam.c", -"musl/src/thread/pthread_getspecific.c", -"musl/src/thread/pthread_join.c", -"musl/src/thread/pthread_key_create.c", -"musl/src/thread/pthread_kill.c", -"musl/src/thread/pthread_mutex_consistent.c", -"musl/src/thread/pthread_mutex_destroy.c", -"musl/src/thread/pthread_mutex_getprioceiling.c", -"musl/src/thread/pthread_mutex_init.c", -"musl/src/thread/pthread_mutex_lock.c", -"musl/src/thread/pthread_mutex_setprioceiling.c", -"musl/src/thread/pthread_mutex_timedlock.c", -"musl/src/thread/pthread_mutex_trylock.c", -"musl/src/thread/pthread_mutex_unlock.c", -"musl/src/thread/pthread_mutexattr_destroy.c", -"musl/src/thread/pthread_mutexattr_init.c", -"musl/src/thread/pthread_mutexattr_setprotocol.c", -"musl/src/thread/pthread_mutexattr_setpshared.c", -"musl/src/thread/pthread_mutexattr_setrobust.c", -"musl/src/thread/pthread_mutexattr_settype.c", -"musl/src/thread/pthread_once.c", -"musl/src/thread/pthread_rwlock_destroy.c", -"musl/src/thread/pthread_rwlock_init.c", -"musl/src/thread/pthread_rwlock_rdlock.c", -"musl/src/thread/pthread_rwlock_timedrdlock.c", -"musl/src/thread/pthread_rwlock_timedwrlock.c", -"musl/src/thread/pthread_rwlock_tryrdlock.c", -"musl/src/thread/pthread_rwlock_trywrlock.c", -"musl/src/thread/pthread_rwlock_unlock.c", -"musl/src/thread/pthread_rwlock_wrlock.c", -"musl/src/thread/pthread_rwlockattr_destroy.c", -"musl/src/thread/pthread_rwlockattr_init.c", -"musl/src/thread/pthread_rwlockattr_setpshared.c", -"musl/src/thread/pthread_self.c", -"musl/src/thread/pthread_setattr_default_np.c", -"musl/src/thread/pthread_setcancelstate.c", -"musl/src/thread/pthread_setcanceltype.c", -"musl/src/thread/pthread_setconcurrency.c", -"musl/src/thread/pthread_setname_np.c", -"musl/src/thread/pthread_setschedparam.c", -"musl/src/thread/pthread_setschedprio.c", -"musl/src/thread/pthread_setspecific.c", -"musl/src/thread/pthread_sigmask.c", -"musl/src/thread/pthread_spin_destroy.c", -"musl/src/thread/pthread_spin_init.c", -"musl/src/thread/pthread_spin_lock.c", -"musl/src/thread/pthread_spin_trylock.c", -"musl/src/thread/pthread_spin_unlock.c", -"musl/src/thread/pthread_testcancel.c", -"musl/src/thread/riscv64/__set_thread_area.s", -"musl/src/thread/riscv64/__unmapself.s", -"musl/src/thread/riscv64/clone.s", -"musl/src/thread/riscv64/syscall_cp.s", -"musl/src/thread/s390x/__set_thread_area.s", -"musl/src/thread/s390x/__tls_get_offset.s", -"musl/src/thread/s390x/__unmapself.s", -"musl/src/thread/s390x/clone.s", -"musl/src/thread/s390x/syscall_cp.s", -"musl/src/thread/sem_destroy.c", -"musl/src/thread/sem_getvalue.c", -"musl/src/thread/sem_init.c", -"musl/src/thread/sem_open.c", -"musl/src/thread/sem_post.c", -"musl/src/thread/sem_timedwait.c", -"musl/src/thread/sem_trywait.c", -"musl/src/thread/sem_unlink.c", -"musl/src/thread/sem_wait.c", -"musl/src/thread/sh/__set_thread_area.c", -"musl/src/thread/sh/__unmapself.c", -"musl/src/thread/sh/__unmapself_mmu.s", -"musl/src/thread/sh/atomics.s", -"musl/src/thread/sh/clone.s", -"musl/src/thread/sh/syscall_cp.s", -"musl/src/thread/synccall.c", -"musl/src/thread/syscall_cp.c", -"musl/src/thread/thrd_create.c", -"musl/src/thread/thrd_exit.c", -"musl/src/thread/thrd_join.c", -"musl/src/thread/thrd_sleep.c", -"musl/src/thread/thrd_yield.c", -"musl/src/thread/tls.c", -"musl/src/thread/tss_create.c", -"musl/src/thread/tss_delete.c", -"musl/src/thread/tss_set.c", -"musl/src/thread/vmlock.c", -"musl/src/thread/x32/__set_thread_area.s", -"musl/src/thread/x32/__unmapself.s", -"musl/src/thread/x32/clone.s", -"musl/src/thread/x32/syscall_cp.s", -"musl/src/thread/x86_64/__set_thread_area.s", -"musl/src/thread/x86_64/__unmapself.s", -"musl/src/thread/x86_64/clone.s", -"musl/src/thread/x86_64/syscall_cp.s", -"musl/src/time/__map_file.c", -"musl/src/time/__month_to_secs.c", -"musl/src/time/__secs_to_tm.c", -"musl/src/time/__tm_to_secs.c", -"musl/src/time/__tz.c", -"musl/src/time/__year_to_secs.c", -"musl/src/time/asctime.c", -"musl/src/time/asctime_r.c", -"musl/src/time/clock.c", -"musl/src/time/clock_getcpuclockid.c", -"musl/src/time/clock_getres.c", -"musl/src/time/clock_gettime.c", -"musl/src/time/clock_nanosleep.c", -"musl/src/time/clock_settime.c", -"musl/src/time/ctime.c", -"musl/src/time/ctime_r.c", -"musl/src/time/difftime.c", -"musl/src/time/ftime.c", -"musl/src/time/getdate.c", -"musl/src/time/gettimeofday.c", -"musl/src/time/gmtime.c", -"musl/src/time/gmtime_r.c", -"musl/src/time/localtime.c", -"musl/src/time/localtime_r.c", -"musl/src/time/mktime.c", -"musl/src/time/nanosleep.c", -"musl/src/time/strftime.c", -"musl/src/time/strptime.c", -"musl/src/time/time.c", -"musl/src/time/timegm.c", -"musl/src/time/timer_create.c", -"musl/src/time/timer_delete.c", -"musl/src/time/timer_getoverrun.c", -"musl/src/time/timer_gettime.c", -"musl/src/time/timer_settime.c", -"musl/src/time/times.c", -"musl/src/time/timespec_get.c", -"musl/src/time/utime.c", -"musl/src/time/wcsftime.c", -"musl/src/unistd/_exit.c", -"musl/src/unistd/access.c", -"musl/src/unistd/acct.c", -"musl/src/unistd/alarm.c", -"musl/src/unistd/chdir.c", -"musl/src/unistd/chown.c", -"musl/src/unistd/close.c", -"musl/src/unistd/ctermid.c", -"musl/src/unistd/dup.c", -"musl/src/unistd/dup2.c", -"musl/src/unistd/dup3.c", -"musl/src/unistd/faccessat.c", -"musl/src/unistd/fchdir.c", -"musl/src/unistd/fchown.c", -"musl/src/unistd/fchownat.c", -"musl/src/unistd/fdatasync.c", -"musl/src/unistd/fsync.c", -"musl/src/unistd/ftruncate.c", -"musl/src/unistd/getcwd.c", -"musl/src/unistd/getegid.c", -"musl/src/unistd/geteuid.c", -"musl/src/unistd/getgid.c", -"musl/src/unistd/getgroups.c", -"musl/src/unistd/gethostname.c", -"musl/src/unistd/getlogin.c", -"musl/src/unistd/getlogin_r.c", -"musl/src/unistd/getpgid.c", -"musl/src/unistd/getpgrp.c", -"musl/src/unistd/getpid.c", -"musl/src/unistd/getppid.c", -"musl/src/unistd/getsid.c", -"musl/src/unistd/getuid.c", -"musl/src/unistd/isatty.c", -"musl/src/unistd/lchown.c", -"musl/src/unistd/link.c", -"musl/src/unistd/linkat.c", -"musl/src/unistd/lseek.c", -"musl/src/unistd/mips/pipe.s", -"musl/src/unistd/mips64/pipe.s", -"musl/src/unistd/mipsn32/lseek.c", -"musl/src/unistd/mipsn32/pipe.s", -"musl/src/unistd/nice.c", -"musl/src/unistd/pause.c", -"musl/src/unistd/pipe.c", -"musl/src/unistd/pipe2.c", -"musl/src/unistd/posix_close.c", -"musl/src/unistd/pread.c", -"musl/src/unistd/preadv.c", -"musl/src/unistd/pwrite.c", -"musl/src/unistd/pwritev.c", -"musl/src/unistd/read.c", -"musl/src/unistd/readlink.c", -"musl/src/unistd/readlinkat.c", -"musl/src/unistd/readv.c", -"musl/src/unistd/renameat.c", -"musl/src/unistd/rmdir.c", -"musl/src/unistd/setegid.c", -"musl/src/unistd/seteuid.c", -"musl/src/unistd/setgid.c", -"musl/src/unistd/setpgid.c", -"musl/src/unistd/setpgrp.c", -"musl/src/unistd/setregid.c", -"musl/src/unistd/setresgid.c", -"musl/src/unistd/setresuid.c", -"musl/src/unistd/setreuid.c", -"musl/src/unistd/setsid.c", -"musl/src/unistd/setuid.c", -"musl/src/unistd/setxid.c", -"musl/src/unistd/sh/pipe.s", -"musl/src/unistd/sleep.c", -"musl/src/unistd/symlink.c", -"musl/src/unistd/symlinkat.c", -"musl/src/unistd/sync.c", -"musl/src/unistd/tcgetpgrp.c", -"musl/src/unistd/tcsetpgrp.c", -"musl/src/unistd/truncate.c", -"musl/src/unistd/ttyname.c", -"musl/src/unistd/ttyname_r.c", -"musl/src/unistd/ualarm.c", -"musl/src/unistd/unlink.c", -"musl/src/unistd/unlinkat.c", -"musl/src/unistd/usleep.c", -"musl/src/unistd/write.c", -"musl/src/unistd/writev.c", -"musl/src/unistd/x32/lseek.c", -}; -static const char *ZIG_MUSL_COMPAT_TIME32_FILES[] = { -"musl/compat/time32/__xstat.c", -"musl/compat/time32/adjtime32.c", -"musl/compat/time32/adjtimex_time32.c", -"musl/compat/time32/aio_suspend_time32.c", -"musl/compat/time32/clock_adjtime32.c", -"musl/compat/time32/clock_getres_time32.c", -"musl/compat/time32/clock_gettime32.c", -"musl/compat/time32/clock_nanosleep_time32.c", -"musl/compat/time32/clock_settime32.c", -"musl/compat/time32/cnd_timedwait_time32.c", -"musl/compat/time32/ctime32.c", -"musl/compat/time32/ctime32_r.c", -"musl/compat/time32/difftime32.c", -"musl/compat/time32/fstat_time32.c", -"musl/compat/time32/fstatat_time32.c", -"musl/compat/time32/ftime32.c", -"musl/compat/time32/futimens_time32.c", -"musl/compat/time32/futimes_time32.c", -"musl/compat/time32/futimesat_time32.c", -"musl/compat/time32/getitimer_time32.c", -"musl/compat/time32/getrusage_time32.c", -"musl/compat/time32/gettimeofday_time32.c", -"musl/compat/time32/gmtime32.c", -"musl/compat/time32/gmtime32_r.c", -"musl/compat/time32/localtime32.c", -"musl/compat/time32/localtime32_r.c", -"musl/compat/time32/lstat_time32.c", -"musl/compat/time32/lutimes_time32.c", -"musl/compat/time32/mktime32.c", -"musl/compat/time32/mq_timedreceive_time32.c", -"musl/compat/time32/mq_timedsend_time32.c", -"musl/compat/time32/mtx_timedlock_time32.c", -"musl/compat/time32/nanosleep_time32.c", -"musl/compat/time32/ppoll_time32.c", -"musl/compat/time32/pselect_time32.c", -"musl/compat/time32/pthread_cond_timedwait_time32.c", -"musl/compat/time32/pthread_mutex_timedlock_time32.c", -"musl/compat/time32/pthread_rwlock_timedrdlock_time32.c", -"musl/compat/time32/pthread_rwlock_timedwrlock_time32.c", -"musl/compat/time32/pthread_timedjoin_np_time32.c", -"musl/compat/time32/recvmmsg_time32.c", -"musl/compat/time32/sched_rr_get_interval_time32.c", -"musl/compat/time32/select_time32.c", -"musl/compat/time32/sem_timedwait_time32.c", -"musl/compat/time32/semtimedop_time32.c", -"musl/compat/time32/setitimer_time32.c", -"musl/compat/time32/settimeofday_time32.c", -"musl/compat/time32/sigtimedwait_time32.c", -"musl/compat/time32/stat_time32.c", -"musl/compat/time32/stime32.c", -"musl/compat/time32/thrd_sleep_time32.c", -"musl/compat/time32/time32.c", -"musl/compat/time32/time32gm.c", -"musl/compat/time32/timer_gettime32.c", -"musl/compat/time32/timer_settime32.c", -"musl/compat/time32/timerfd_gettime32.c", -"musl/compat/time32/timerfd_settime32.c", -"musl/compat/time32/timespec_get_time32.c", -"musl/compat/time32/utime_time32.c", -"musl/compat/time32/utimensat_time32.c", -"musl/compat/time32/utimes_time32.c", -"musl/compat/time32/wait3_time32.c", -"musl/compat/time32/wait4_time32.c", -}; -static const char *ZIG_LIBCXXABI_FILES[] = { -"src/abort_message.cpp", -"src/cxa_aux_runtime.cpp", -"src/cxa_default_handlers.cpp", -"src/cxa_demangle.cpp", -"src/cxa_exception.cpp", -"src/cxa_exception_storage.cpp", -"src/cxa_guard.cpp", -"src/cxa_handlers.cpp", -"src/cxa_noexception.cpp", -"src/cxa_personality.cpp", -"src/cxa_thread_atexit.cpp", -"src/cxa_unexpected.cpp", -"src/cxa_vector.cpp", -"src/cxa_virtual.cpp", -"src/fallback_malloc.cpp", -"src/private_typeinfo.cpp", -"src/stdlib_exception.cpp", -"src/stdlib_stdexcept.cpp", -"src/stdlib_typeinfo.cpp", -}; -static const char *ZIG_LIBCXX_FILES[] = { -"src/algorithm.cpp", -"src/any.cpp", -"src/bind.cpp", -"src/charconv.cpp", -"src/chrono.cpp", -"src/condition_variable.cpp", -"src/condition_variable_destructor.cpp", -"src/debug.cpp", -"src/exception.cpp", -"src/experimental/memory_resource.cpp", -"src/filesystem/directory_iterator.cpp", -"src/filesystem/operations.cpp", -"src/functional.cpp", -"src/future.cpp", -"src/hash.cpp", -"src/ios.cpp", -"src/iostream.cpp", -"src/locale.cpp", -"src/memory.cpp", -"src/mutex.cpp", -"src/mutex_destructor.cpp", -"src/new.cpp", -"src/optional.cpp", -"src/random.cpp", -"src/regex.cpp", -"src/shared_mutex.cpp", -"src/stdexcept.cpp", -"src/string.cpp", -"src/strstream.cpp", -"src/support/solaris/xlocale.cpp", -"src/support/win32/locale_win32.cpp", -"src/support/win32/support.cpp", -"src/support/win32/thread_win32.cpp", -"src/system_error.cpp", -"src/thread.cpp", -"src/typeinfo.cpp", -"src/utility.cpp", -"src/valarray.cpp", -"src/variant.cpp", -"src/vector.cpp", -}; -#endif diff --git a/src/ir.cpp b/src/ir.cpp index 6cb5d8bc2d..50076f9a86 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -22570,38 +22570,12 @@ static IrInstGen *ir_analyze_container_field_ptr(IrAnalyze *ira, Buf *field_name } static void add_link_lib_symbol(IrAnalyze *ira, Buf *lib_name, Buf *symbol_name, AstNode *source_node) { - bool is_libc = target_is_libc_lib_name(ira->codegen->zig_target, buf_ptr(lib_name)); - if (is_libc && ira->codegen->libc_link_lib == nullptr && !ira->codegen->reported_bad_link_libc_error) { - ir_add_error_node(ira, source_node, - buf_sprintf("dependency on library c must be explicitly specified in the build command")); + const char *msg = stage2_add_link_lib(&ira->codegen->stage1, buf_ptr(lib_name), buf_len(lib_name), + buf_ptr(symbol_name), buf_len(symbol_name)); + if (msg != nullptr) { + ir_add_error_node(ira, source_node, buf_create_from_str(msg)); ira->codegen->reported_bad_link_libc_error = true; } - - LinkLib *link_lib = add_link_lib(ira->codegen, lib_name); - for (size_t i = 0; i < link_lib->symbols.length; i += 1) { - Buf *existing_symbol_name = link_lib->symbols.at(i); - if (buf_eql_buf(existing_symbol_name, symbol_name)) { - return; - } - } - - if (!is_libc && !target_is_wasm(ira->codegen->zig_target) && !ira->codegen->have_pic && !ira->codegen->reported_bad_link_libc_error) { - ErrorMsg *msg = ir_add_error_node(ira, source_node, - buf_sprintf("dependency on dynamic library '%s' requires enabling Position Independent Code", - buf_ptr(lib_name))); - add_error_note(ira->codegen, msg, source_node, - buf_sprintf("fixed by `--library %s` or `-fPIC`", buf_ptr(lib_name))); - ira->codegen->reported_bad_link_libc_error = true; - } - - for (size_t i = 0; i < ira->codegen->forbidden_libs.length; i += 1) { - Buf *forbidden_lib_name = ira->codegen->forbidden_libs.at(i); - if (buf_eql_buf(lib_name, forbidden_lib_name)) { - ir_add_error_node(ira, source_node, - buf_sprintf("linking against forbidden library '%s'", buf_ptr(symbol_name))); - } - } - link_lib->symbols.append(symbol_name); } static IrInstGen *ir_error_dependency_loop(IrAnalyze *ira, IrInst* source_instr) { @@ -26355,13 +26329,6 @@ static IrInstGen *ir_analyze_instruction_type_name(IrAnalyze *ira, IrInstSrcType return result; } -static void ir_cimport_cache_paths(Buf *cache_dir, Buf *tmp_c_file_digest, Buf *out_zig_dir, Buf *out_zig_path) { - buf_resize(out_zig_dir, 0); - buf_resize(out_zig_path, 0); - buf_appendf(out_zig_dir, "%s" OS_SEP "o" OS_SEP "%s", - buf_ptr(cache_dir), buf_ptr(tmp_c_file_digest)); - buf_appendf(out_zig_path, "%s" OS_SEP "cimport.zig", buf_ptr(out_zig_dir)); -} static IrInstGen *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstSrcCImport *instruction) { Error err; AstNode *node = instruction->base.base.source_node; @@ -26393,145 +26360,7 @@ static IrInstGen *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstSrcCImpo cimport_pkg->package_table.put(buf_create_from_str("std"), ira->codegen->std_package); buf_init_from_buf(&cimport_pkg->pkg_path, namespace_name); - CacheHash *cache_hash; - if ((err = create_c_object_cache(ira->codegen, &cache_hash, false))) { - ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to create cache: %s", err_str(err))); - return ira->codegen->invalid_inst_gen; - } - cache_buf(cache_hash, &cimport_scope->buf); - - // Set this because we're not adding any files before checking for a hit. - cache_hash->force_check_manifest = true; - - Buf tmp_c_file_digest = BUF_INIT; - buf_resize(&tmp_c_file_digest, 0); - if ((err = cache_hit(cache_hash, &tmp_c_file_digest))) { - if (err != ErrorInvalidFormat) { - ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to check cache: %s", err_str(err))); - return ira->codegen->invalid_inst_gen; - } - } - ira->codegen->caches_to_release.append(cache_hash); - - Buf *out_zig_dir = buf_alloc(); - Buf *out_zig_path = buf_alloc(); - if (buf_len(&tmp_c_file_digest) == 0 || cache_hash->files.length == 0) { - // Cache Miss - Buf *tmp_c_file_dir = buf_sprintf("%s" OS_SEP "o" OS_SEP "%s", - buf_ptr(ira->codegen->cache_dir), buf_ptr(&cache_hash->b64_digest)); - Buf *resolve_paths[] = { - tmp_c_file_dir, - buf_create_from_str("cimport.h"), - }; - Buf tmp_c_file_path = os_path_resolve(resolve_paths, 2); - - if ((err = os_make_path(tmp_c_file_dir))) { - ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to make dir: %s", err_str(err))); - return ira->codegen->invalid_inst_gen; - } - - if ((err = os_write_file(&tmp_c_file_path, &cimport_scope->buf))) { - ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to write .h file: %s", err_str(err))); - return ira->codegen->invalid_inst_gen; - } - if (ira->codegen->verbose_cimport) { - fprintf(stderr, "@cImport source: %s\n", buf_ptr(&tmp_c_file_path)); - } - - Buf *tmp_dep_file = buf_sprintf("%s.d", buf_ptr(&tmp_c_file_path)); - - ZigList clang_argv = {0}; - - add_cc_args(ira->codegen, clang_argv, buf_ptr(tmp_dep_file), true, FileExtC); - - clang_argv.append(buf_ptr(&tmp_c_file_path)); - - if (ira->codegen->verbose_cc) { - fprintf(stderr, "clang"); - for (size_t i = 0; i < clang_argv.length; i += 1) { - fprintf(stderr, " %s", clang_argv.at(i)); - } - fprintf(stderr, "\n"); - } - - clang_argv.append(nullptr); // to make the [start...end] argument work - - Stage2ErrorMsg *errors_ptr; - size_t errors_len; - Stage2Ast *ast; - - const char *resources_path = buf_ptr(ira->codegen->zig_c_headers_dir); - - if ((err = stage2_translate_c(&ast, &errors_ptr, &errors_len, - &clang_argv.at(0), &clang_argv.last(), resources_path))) - { - if (err != ErrorCCompileErrors) { - ir_add_error_node(ira, node, buf_sprintf("C import failed: %s", err_str(err))); - return ira->codegen->invalid_inst_gen; - } - - ErrorMsg *parent_err_msg = ir_add_error_node(ira, node, buf_sprintf("C import failed")); - if (ira->codegen->libc_link_lib == nullptr) { - add_error_note(ira->codegen, parent_err_msg, node, - buf_sprintf("libc headers not available; compilation does not link against libc")); - } - for (size_t i = 0; i < errors_len; i += 1) { - Stage2ErrorMsg *clang_err = &errors_ptr[i]; - // Clang can emit "too many errors, stopping now", in which case `source` and `filename_ptr` are null - if (clang_err->source && clang_err->filename_ptr) { - ErrorMsg *err_msg = err_msg_create_with_offset( - clang_err->filename_ptr ? - buf_create_from_mem(clang_err->filename_ptr, clang_err->filename_len) : buf_alloc(), - clang_err->line, clang_err->column, clang_err->offset, clang_err->source, - buf_create_from_mem(clang_err->msg_ptr, clang_err->msg_len)); - err_msg_add_note(parent_err_msg, err_msg); - } - } - - return ira->codegen->invalid_inst_gen; - } - if (ira->codegen->verbose_cimport) { - fprintf(stderr, "@cImport .d file: %s\n", buf_ptr(tmp_dep_file)); - } - - if ((err = cache_add_dep_file(cache_hash, tmp_dep_file, false))) { - ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to parse .d file: %s", err_str(err))); - return ira->codegen->invalid_inst_gen; - } - if ((err = cache_final(cache_hash, &tmp_c_file_digest))) { - ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to finalize cache: %s", err_str(err))); - return ira->codegen->invalid_inst_gen; - } - - ir_cimport_cache_paths(ira->codegen->cache_dir, &tmp_c_file_digest, out_zig_dir, out_zig_path); - if ((err = os_make_path(out_zig_dir))) { - ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to make output dir: %s", err_str(err))); - return ira->codegen->invalid_inst_gen; - } - FILE *out_file = fopen(buf_ptr(out_zig_path), "wb"); - if (out_file == nullptr) { - ir_add_error_node(ira, node, - buf_sprintf("C import failed: unable to open output file: %s", strerror(errno))); - return ira->codegen->invalid_inst_gen; - } - stage2_render_ast(ast, out_file); - if (fclose(out_file) != 0) { - ir_add_error_node(ira, node, - buf_sprintf("C import failed: unable to write to output file: %s", strerror(errno))); - return ira->codegen->invalid_inst_gen; - } - - if (ira->codegen->verbose_cimport) { - fprintf(stderr, "@cImport output: %s\n", buf_ptr(out_zig_path)); - } - - } else { - // Cache Hit - ir_cimport_cache_paths(ira->codegen->cache_dir, &tmp_c_file_digest, out_zig_dir, out_zig_path); - if (ira->codegen->verbose_cimport) { - fprintf(stderr, "@cImport cache hit: %s\n", buf_ptr(out_zig_path)); - } - } + Buf *out_zig_path = buf_create_from_str(stage2_cimport(&ira->codegen->stage1)); Buf *import_code = buf_alloc(); if ((err = file_fetch(ira->codegen, out_zig_path, import_code))) { diff --git a/src/link.cpp b/src/link.cpp deleted file mode 100644 index 3983b48b42..0000000000 --- a/src/link.cpp +++ /dev/null @@ -1,2985 +0,0 @@ -/* - * Copyright (c) 2015 Andrew Kelley - * - * This file is part of zig, which is MIT licensed. - * See http://opensource.org/licenses/MIT - */ - -#include "os.hpp" -#include "config.h" -#include "codegen.hpp" -#include "analyze.hpp" -#include "compiler.hpp" -#include "install_files.h" -#include "glibc.hpp" - -static const char *msvcrt_common_src[] = { - "misc" OS_SEP "_create_locale.c", - "misc" OS_SEP "_free_locale.c", - "misc" OS_SEP "onexit_table.c", - "misc" OS_SEP "register_tls_atexit.c", - "stdio" OS_SEP "acrt_iob_func.c", - "misc" OS_SEP "_configthreadlocale.c", - "misc" OS_SEP "_get_current_locale.c", - "misc" OS_SEP "invalid_parameter_handler.c", - "misc" OS_SEP "output_format.c", - "misc" OS_SEP "purecall.c", - "secapi" OS_SEP "_access_s.c", - "secapi" OS_SEP "_cgets_s.c", - "secapi" OS_SEP "_cgetws_s.c", - "secapi" OS_SEP "_chsize_s.c", - "secapi" OS_SEP "_controlfp_s.c", - "secapi" OS_SEP "_cprintf_s.c", - "secapi" OS_SEP "_cprintf_s_l.c", - "secapi" OS_SEP "_ctime32_s.c", - "secapi" OS_SEP "_ctime64_s.c", - "secapi" OS_SEP "_cwprintf_s.c", - "secapi" OS_SEP "_cwprintf_s_l.c", - "secapi" OS_SEP "_gmtime32_s.c", - "secapi" OS_SEP "_gmtime64_s.c", - "secapi" OS_SEP "_localtime32_s.c", - "secapi" OS_SEP "_localtime64_s.c", - "secapi" OS_SEP "_mktemp_s.c", - "secapi" OS_SEP "_sopen_s.c", - "secapi" OS_SEP "_strdate_s.c", - "secapi" OS_SEP "_strtime_s.c", - "secapi" OS_SEP "_umask_s.c", - "secapi" OS_SEP "_vcprintf_s.c", - "secapi" OS_SEP "_vcprintf_s_l.c", - "secapi" OS_SEP "_vcwprintf_s.c", - "secapi" OS_SEP "_vcwprintf_s_l.c", - "secapi" OS_SEP "_vscprintf_p.c", - "secapi" OS_SEP "_vscwprintf_p.c", - "secapi" OS_SEP "_vswprintf_p.c", - "secapi" OS_SEP "_waccess_s.c", - "secapi" OS_SEP "_wasctime_s.c", - "secapi" OS_SEP "_wctime32_s.c", - "secapi" OS_SEP "_wctime64_s.c", - "secapi" OS_SEP "_wstrtime_s.c", - "secapi" OS_SEP "_wmktemp_s.c", - "secapi" OS_SEP "_wstrdate_s.c", - "secapi" OS_SEP "asctime_s.c", - "secapi" OS_SEP "memcpy_s.c", - "secapi" OS_SEP "memmove_s.c", - "secapi" OS_SEP "rand_s.c", - "secapi" OS_SEP "sprintf_s.c", - "secapi" OS_SEP "strerror_s.c", - "secapi" OS_SEP "vsprintf_s.c", - "secapi" OS_SEP "wmemcpy_s.c", - "secapi" OS_SEP "wmemmove_s.c", - "stdio" OS_SEP "mingw_lock.c", -}; - -static const char *msvcrt_i386_src[] = { - "misc" OS_SEP "lc_locale_func.c", - "misc" OS_SEP "___mb_cur_max_func.c", -}; - -static const char *msvcrt_other_src[] = { - "misc" OS_SEP "__p___argv.c", - "misc" OS_SEP "__p__acmdln.c", - "misc" OS_SEP "__p__fmode.c", - "misc" OS_SEP "__p__wcmdln.c", -}; - -static const char *mingwex_generic_src[] = { - "complex" OS_SEP "_cabs.c", - "complex" OS_SEP "cabs.c", - "complex" OS_SEP "cabsf.c", - "complex" OS_SEP "cabsl.c", - "complex" OS_SEP "cacos.c", - "complex" OS_SEP "cacosf.c", - "complex" OS_SEP "cacosl.c", - "complex" OS_SEP "carg.c", - "complex" OS_SEP "cargf.c", - "complex" OS_SEP "cargl.c", - "complex" OS_SEP "casin.c", - "complex" OS_SEP "casinf.c", - "complex" OS_SEP "casinl.c", - "complex" OS_SEP "catan.c", - "complex" OS_SEP "catanf.c", - "complex" OS_SEP "catanl.c", - "complex" OS_SEP "ccos.c", - "complex" OS_SEP "ccosf.c", - "complex" OS_SEP "ccosl.c", - "complex" OS_SEP "cexp.c", - "complex" OS_SEP "cexpf.c", - "complex" OS_SEP "cexpl.c", - "complex" OS_SEP "cimag.c", - "complex" OS_SEP "cimagf.c", - "complex" OS_SEP "cimagl.c", - "complex" OS_SEP "clog.c", - "complex" OS_SEP "clog10.c", - "complex" OS_SEP "clog10f.c", - "complex" OS_SEP "clog10l.c", - "complex" OS_SEP "clogf.c", - "complex" OS_SEP "clogl.c", - "complex" OS_SEP "conj.c", - "complex" OS_SEP "conjf.c", - "complex" OS_SEP "conjl.c", - "complex" OS_SEP "cpow.c", - "complex" OS_SEP "cpowf.c", - "complex" OS_SEP "cpowl.c", - "complex" OS_SEP "cproj.c", - "complex" OS_SEP "cprojf.c", - "complex" OS_SEP "cprojl.c", - "complex" OS_SEP "creal.c", - "complex" OS_SEP "crealf.c", - "complex" OS_SEP "creall.c", - "complex" OS_SEP "csin.c", - "complex" OS_SEP "csinf.c", - "complex" OS_SEP "csinl.c", - "complex" OS_SEP "csqrt.c", - "complex" OS_SEP "csqrtf.c", - "complex" OS_SEP "csqrtl.c", - "complex" OS_SEP "ctan.c", - "complex" OS_SEP "ctanf.c", - "complex" OS_SEP "ctanl.c", - "crt" OS_SEP "dllentry.c", - "crt" OS_SEP "dllmain.c", - "gdtoa" OS_SEP "arithchk.c", - "gdtoa" OS_SEP "dmisc.c", - "gdtoa" OS_SEP "dtoa.c", - "gdtoa" OS_SEP "g__fmt.c", - "gdtoa" OS_SEP "g_dfmt.c", - "gdtoa" OS_SEP "g_ffmt.c", - "gdtoa" OS_SEP "g_xfmt.c", - "gdtoa" OS_SEP "gdtoa.c", - "gdtoa" OS_SEP "gethex.c", - "gdtoa" OS_SEP "gmisc.c", - "gdtoa" OS_SEP "hd_init.c", - "gdtoa" OS_SEP "hexnan.c", - "gdtoa" OS_SEP "misc.c", - "gdtoa" OS_SEP "qnan.c", - "gdtoa" OS_SEP "smisc.c", - "gdtoa" OS_SEP "strtodg.c", - "gdtoa" OS_SEP "strtodnrp.c", - "gdtoa" OS_SEP "strtof.c", - "gdtoa" OS_SEP "strtopx.c", - "gdtoa" OS_SEP "sum.c", - "gdtoa" OS_SEP "ulp.c", - "math" OS_SEP "abs64.c", - "math" OS_SEP "cbrt.c", - "math" OS_SEP "cbrtf.c", - "math" OS_SEP "cbrtl.c", - "math" OS_SEP "cephes_emath.c", - "math" OS_SEP "copysign.c", - "math" OS_SEP "copysignf.c", - "math" OS_SEP "coshf.c", - "math" OS_SEP "coshl.c", - "math" OS_SEP "erfl.c", - "math" OS_SEP "expf.c", - "math" OS_SEP "fabs.c", - "math" OS_SEP "fabsf.c", - "math" OS_SEP "fabsl.c", - "math" OS_SEP "fdim.c", - "math" OS_SEP "fdimf.c", - "math" OS_SEP "fdiml.c", - "math" OS_SEP "fma.c", - "math" OS_SEP "fmaf.c", - "math" OS_SEP "fmal.c", - "math" OS_SEP "fmax.c", - "math" OS_SEP "fmaxf.c", - "math" OS_SEP "fmaxl.c", - "math" OS_SEP "fmin.c", - "math" OS_SEP "fminf.c", - "math" OS_SEP "fminl.c", - "math" OS_SEP "fp_consts.c", - "math" OS_SEP "fp_constsf.c", - "math" OS_SEP "fp_constsl.c", - "math" OS_SEP "fpclassify.c", - "math" OS_SEP "fpclassifyf.c", - "math" OS_SEP "fpclassifyl.c", - "math" OS_SEP "frexpf.c", - "math" OS_SEP "hypot.c", - "math" OS_SEP "hypotf.c", - "math" OS_SEP "hypotl.c", - "math" OS_SEP "isnan.c", - "math" OS_SEP "isnanf.c", - "math" OS_SEP "isnanl.c", - "math" OS_SEP "ldexpf.c", - "math" OS_SEP "lgamma.c", - "math" OS_SEP "lgammaf.c", - "math" OS_SEP "lgammal.c", - "math" OS_SEP "llrint.c", - "math" OS_SEP "llrintf.c", - "math" OS_SEP "llrintl.c", - "math" OS_SEP "llround.c", - "math" OS_SEP "llroundf.c", - "math" OS_SEP "llroundl.c", - "math" OS_SEP "log10f.c", - "math" OS_SEP "logf.c", - "math" OS_SEP "lrint.c", - "math" OS_SEP "lrintf.c", - "math" OS_SEP "lrintl.c", - "math" OS_SEP "lround.c", - "math" OS_SEP "lroundf.c", - "math" OS_SEP "lroundl.c", - "math" OS_SEP "modf.c", - "math" OS_SEP "modff.c", - "math" OS_SEP "modfl.c", - "math" OS_SEP "nextafterf.c", - "math" OS_SEP "nextafterl.c", - "math" OS_SEP "nexttoward.c", - "math" OS_SEP "nexttowardf.c", - "math" OS_SEP "powf.c", - "math" OS_SEP "powi.c", - "math" OS_SEP "powif.c", - "math" OS_SEP "powil.c", - "math" OS_SEP "rint.c", - "math" OS_SEP "rintf.c", - "math" OS_SEP "rintl.c", - "math" OS_SEP "round.c", - "math" OS_SEP "roundf.c", - "math" OS_SEP "roundl.c", - "math" OS_SEP "s_erf.c", - "math" OS_SEP "sf_erf.c", - "math" OS_SEP "signbit.c", - "math" OS_SEP "signbitf.c", - "math" OS_SEP "signbitl.c", - "math" OS_SEP "signgam.c", - "math" OS_SEP "sinhf.c", - "math" OS_SEP "sinhl.c", - "math" OS_SEP "sqrt.c", - "math" OS_SEP "sqrtf.c", - "math" OS_SEP "sqrtl.c", - "math" OS_SEP "tanhf.c", - "math" OS_SEP "tanhl.c", - "math" OS_SEP "tgamma.c", - "math" OS_SEP "tgammaf.c", - "math" OS_SEP "tgammal.c", - "math" OS_SEP "truncl.c", - "misc" OS_SEP "alarm.c", - "misc" OS_SEP "basename.c", - "misc" OS_SEP "btowc.c", - "misc" OS_SEP "delay-f.c", - "misc" OS_SEP "delay-n.c", - "misc" OS_SEP "delayimp.c", - "misc" OS_SEP "dirent.c", - "misc" OS_SEP "dirname.c", - "misc" OS_SEP "feclearexcept.c", - "misc" OS_SEP "fegetenv.c", - "misc" OS_SEP "fegetexceptflag.c", - "misc" OS_SEP "fegetround.c", - "misc" OS_SEP "feholdexcept.c", - "misc" OS_SEP "feraiseexcept.c", - "misc" OS_SEP "fesetenv.c", - "misc" OS_SEP "fesetexceptflag.c", - "misc" OS_SEP "fesetround.c", - "misc" OS_SEP "fetestexcept.c", - "misc" OS_SEP "feupdateenv.c", - "misc" OS_SEP "ftruncate.c", - "misc" OS_SEP "ftw.c", - "misc" OS_SEP "ftw64.c", - "misc" OS_SEP "fwide.c", - "misc" OS_SEP "getlogin.c", - "misc" OS_SEP "getopt.c", - "misc" OS_SEP "gettimeofday.c", - "misc" OS_SEP "imaxabs.c", - "misc" OS_SEP "imaxdiv.c", - "misc" OS_SEP "isblank.c", - "misc" OS_SEP "iswblank.c", - "misc" OS_SEP "mbrtowc.c", - "misc" OS_SEP "mbsinit.c", - "misc" OS_SEP "mempcpy.c", - "misc" OS_SEP "mingw-aligned-malloc.c", - "misc" OS_SEP "mingw-fseek.c", - "misc" OS_SEP "mingw_getsp.S", - "misc" OS_SEP "mingw_matherr.c", - "misc" OS_SEP "mingw_mbwc_convert.c", - "misc" OS_SEP "mingw_usleep.c", - "misc" OS_SEP "mingw_wcstod.c", - "misc" OS_SEP "mingw_wcstof.c", - "misc" OS_SEP "mingw_wcstold.c", - "misc" OS_SEP "mkstemp.c", - "misc" OS_SEP "seterrno.c", - "misc" OS_SEP "sleep.c", - "misc" OS_SEP "strnlen.c", - "misc" OS_SEP "strsafe.c", - "misc" OS_SEP "strtoimax.c", - "misc" OS_SEP "strtold.c", - "misc" OS_SEP "strtoumax.c", - "misc" OS_SEP "tdelete.c", - "misc" OS_SEP "tfind.c", - "misc" OS_SEP "tsearch.c", - "misc" OS_SEP "twalk.c", - "misc" OS_SEP "uchar_c16rtomb.c", - "misc" OS_SEP "uchar_c32rtomb.c", - "misc" OS_SEP "uchar_mbrtoc16.c", - "misc" OS_SEP "uchar_mbrtoc32.c", - "misc" OS_SEP "wassert.c", - "misc" OS_SEP "wcrtomb.c", - "misc" OS_SEP "wcsnlen.c", - "misc" OS_SEP "wcstof.c", - "misc" OS_SEP "wcstoimax.c", - "misc" OS_SEP "wcstold.c", - "misc" OS_SEP "wcstoumax.c", - "misc" OS_SEP "wctob.c", - "misc" OS_SEP "wctrans.c", - "misc" OS_SEP "wctype.c", - "misc" OS_SEP "wdirent.c", - "misc" OS_SEP "winbs_uint64.c", - "misc" OS_SEP "winbs_ulong.c", - "misc" OS_SEP "winbs_ushort.c", - "misc" OS_SEP "wmemchr.c", - "misc" OS_SEP "wmemcmp.c", - "misc" OS_SEP "wmemcpy.c", - "misc" OS_SEP "wmemmove.c", - "misc" OS_SEP "wmempcpy.c", - "misc" OS_SEP "wmemset.c", - "stdio" OS_SEP "_Exit.c", - "stdio" OS_SEP "_findfirst64i32.c", - "stdio" OS_SEP "_findnext64i32.c", - "stdio" OS_SEP "_fstat.c", - "stdio" OS_SEP "_fstat64i32.c", - "stdio" OS_SEP "_ftime.c", - "stdio" OS_SEP "_getc_nolock.c", - "stdio" OS_SEP "_getwc_nolock.c", - "stdio" OS_SEP "_putc_nolock.c", - "stdio" OS_SEP "_putwc_nolock.c", - "stdio" OS_SEP "_stat.c", - "stdio" OS_SEP "_stat64i32.c", - "stdio" OS_SEP "_wfindfirst64i32.c", - "stdio" OS_SEP "_wfindnext64i32.c", - "stdio" OS_SEP "_wstat.c", - "stdio" OS_SEP "_wstat64i32.c", - "stdio" OS_SEP "asprintf.c", - "stdio" OS_SEP "atoll.c", - "stdio" OS_SEP "fgetpos64.c", - "stdio" OS_SEP "fopen64.c", - "stdio" OS_SEP "fseeko32.c", - "stdio" OS_SEP "fseeko64.c", - "stdio" OS_SEP "fsetpos64.c", - "stdio" OS_SEP "ftello.c", - "stdio" OS_SEP "ftello64.c", - "stdio" OS_SEP "ftruncate64.c", - "stdio" OS_SEP "lltoa.c", - "stdio" OS_SEP "lltow.c", - "stdio" OS_SEP "lseek64.c", - "stdio" OS_SEP "mingw_asprintf.c", - "stdio" OS_SEP "mingw_fprintf.c", - "stdio" OS_SEP "mingw_fprintfw.c", - "stdio" OS_SEP "mingw_fscanf.c", - "stdio" OS_SEP "mingw_fwscanf.c", - "stdio" OS_SEP "mingw_pformat.c", - "stdio" OS_SEP "mingw_pformatw.c", - "stdio" OS_SEP "mingw_printf.c", - "stdio" OS_SEP "mingw_printfw.c", - "stdio" OS_SEP "mingw_scanf.c", - "stdio" OS_SEP "mingw_snprintf.c", - "stdio" OS_SEP "mingw_snprintfw.c", - "stdio" OS_SEP "mingw_sprintf.c", - "stdio" OS_SEP "mingw_sprintfw.c", - "stdio" OS_SEP "mingw_sscanf.c", - "stdio" OS_SEP "mingw_swscanf.c", - "stdio" OS_SEP "mingw_vasprintf.c", - "stdio" OS_SEP "mingw_vfprintf.c", - "stdio" OS_SEP "mingw_vfprintfw.c", - "stdio" OS_SEP "mingw_vfscanf.c", - "stdio" OS_SEP "mingw_vprintf.c", - "stdio" OS_SEP "mingw_vprintfw.c", - "stdio" OS_SEP "mingw_vsnprintf.c", - "stdio" OS_SEP "mingw_vsnprintfw.c", - "stdio" OS_SEP "mingw_vsprintf.c", - "stdio" OS_SEP "mingw_vsprintfw.c", - "stdio" OS_SEP "mingw_wscanf.c", - "stdio" OS_SEP "mingw_wvfscanf.c", - "stdio" OS_SEP "scanf.S", - "stdio" OS_SEP "snprintf.c", - "stdio" OS_SEP "snwprintf.c", - "stdio" OS_SEP "strtof.c", - "stdio" OS_SEP "strtok_r.c", - "stdio" OS_SEP "truncate.c", - "stdio" OS_SEP "ulltoa.c", - "stdio" OS_SEP "ulltow.c", - "stdio" OS_SEP "vasprintf.c", - "stdio" OS_SEP "vfscanf.c", - "stdio" OS_SEP "vfscanf2.S", - "stdio" OS_SEP "vfwscanf.c", - "stdio" OS_SEP "vfwscanf2.S", - "stdio" OS_SEP "vscanf.c", - "stdio" OS_SEP "vscanf2.S", - "stdio" OS_SEP "vsnprintf.c", - "stdio" OS_SEP "vsnwprintf.c", - "stdio" OS_SEP "vsscanf.c", - "stdio" OS_SEP "vsscanf2.S", - "stdio" OS_SEP "vswscanf.c", - "stdio" OS_SEP "vswscanf2.S", - "stdio" OS_SEP "vwscanf.c", - "stdio" OS_SEP "vwscanf2.S", - "stdio" OS_SEP "wtoll.c", -}; - -static const char *mingwex_x86_src[] = { - "math" OS_SEP "x86" OS_SEP "acosf.c", - "math" OS_SEP "x86" OS_SEP "acosh.c", - "math" OS_SEP "x86" OS_SEP "acoshf.c", - "math" OS_SEP "x86" OS_SEP "acoshl.c", - "math" OS_SEP "x86" OS_SEP "acosl.c", - "math" OS_SEP "x86" OS_SEP "asinf.c", - "math" OS_SEP "x86" OS_SEP "asinh.c", - "math" OS_SEP "x86" OS_SEP "asinhf.c", - "math" OS_SEP "x86" OS_SEP "asinhl.c", - "math" OS_SEP "x86" OS_SEP "asinl.c", - "math" OS_SEP "x86" OS_SEP "atan2.c", - "math" OS_SEP "x86" OS_SEP "atan2f.c", - "math" OS_SEP "x86" OS_SEP "atan2l.c", - "math" OS_SEP "x86" OS_SEP "atanf.c", - "math" OS_SEP "x86" OS_SEP "atanh.c", - "math" OS_SEP "x86" OS_SEP "atanhf.c", - "math" OS_SEP "x86" OS_SEP "atanhl.c", - "math" OS_SEP "x86" OS_SEP "atanl.c", - "math" OS_SEP "x86" OS_SEP "ceilf.S", - "math" OS_SEP "x86" OS_SEP "ceill.S", - "math" OS_SEP "x86" OS_SEP "ceil.S", - "math" OS_SEP "x86" OS_SEP "_chgsignl.S", - "math" OS_SEP "x86" OS_SEP "copysignl.S", - "math" OS_SEP "x86" OS_SEP "cos.c", - "math" OS_SEP "x86" OS_SEP "cosf.c", - "math" OS_SEP "x86" OS_SEP "cosl.c", - "math" OS_SEP "x86" OS_SEP "cosl_internal.S", - "math" OS_SEP "x86" OS_SEP "cossin.c", - "math" OS_SEP "x86" OS_SEP "exp2f.S", - "math" OS_SEP "x86" OS_SEP "exp2l.S", - "math" OS_SEP "x86" OS_SEP "exp2.S", - "math" OS_SEP "x86" OS_SEP "exp.c", - "math" OS_SEP "x86" OS_SEP "expl.c", - "math" OS_SEP "x86" OS_SEP "expm1.c", - "math" OS_SEP "x86" OS_SEP "expm1f.c", - "math" OS_SEP "x86" OS_SEP "expm1l.c", - "math" OS_SEP "x86" OS_SEP "floorf.S", - "math" OS_SEP "x86" OS_SEP "floorl.S", - "math" OS_SEP "x86" OS_SEP "floor.S", - "math" OS_SEP "x86" OS_SEP "fmod.c", - "math" OS_SEP "x86" OS_SEP "fmodf.c", - "math" OS_SEP "x86" OS_SEP "fmodl.c", - "math" OS_SEP "x86" OS_SEP "fucom.c", - "math" OS_SEP "x86" OS_SEP "ilogbf.S", - "math" OS_SEP "x86" OS_SEP "ilogbl.S", - "math" OS_SEP "x86" OS_SEP "ilogb.S", - "math" OS_SEP "x86" OS_SEP "internal_logl.S", - "math" OS_SEP "x86" OS_SEP "ldexp.c", - "math" OS_SEP "x86" OS_SEP "ldexpl.c", - "math" OS_SEP "x86" OS_SEP "log10l.S", - "math" OS_SEP "x86" OS_SEP "log1pf.S", - "math" OS_SEP "x86" OS_SEP "log1pl.S", - "math" OS_SEP "x86" OS_SEP "log1p.S", - "math" OS_SEP "x86" OS_SEP "log2f.S", - "math" OS_SEP "x86" OS_SEP "log2l.S", - "math" OS_SEP "x86" OS_SEP "log2.S", - "math" OS_SEP "x86" OS_SEP "logb.c", - "math" OS_SEP "x86" OS_SEP "logbf.c", - "math" OS_SEP "x86" OS_SEP "logbl.c", - "math" OS_SEP "x86" OS_SEP "log.c", - "math" OS_SEP "x86" OS_SEP "logl.c", - "math" OS_SEP "x86" OS_SEP "nearbyintf.S", - "math" OS_SEP "x86" OS_SEP "nearbyintl.S", - "math" OS_SEP "x86" OS_SEP "nearbyint.S", - "math" OS_SEP "x86" OS_SEP "pow.c", - "math" OS_SEP "x86" OS_SEP "powl.c", - "math" OS_SEP "x86" OS_SEP "remainderf.S", - "math" OS_SEP "x86" OS_SEP "remainderl.S", - "math" OS_SEP "x86" OS_SEP "remainder.S", - "math" OS_SEP "x86" OS_SEP "remquof.S", - "math" OS_SEP "x86" OS_SEP "remquol.S", - "math" OS_SEP "x86" OS_SEP "remquo.S", - "math" OS_SEP "x86" OS_SEP "scalbnf.S", - "math" OS_SEP "x86" OS_SEP "scalbnl.S", - "math" OS_SEP "x86" OS_SEP "scalbn.S", - "math" OS_SEP "x86" OS_SEP "sin.c", - "math" OS_SEP "x86" OS_SEP "sinf.c", - "math" OS_SEP "x86" OS_SEP "sinl.c", - "math" OS_SEP "x86" OS_SEP "sinl_internal.S", - "math" OS_SEP "x86" OS_SEP "tanf.c", - "math" OS_SEP "x86" OS_SEP "tanl.S", - "math" OS_SEP "x86" OS_SEP "truncf.S", - "math" OS_SEP "x86" OS_SEP "trunc.S", -}; - -static const char *mingwex_arm32_src[] = { - "math" OS_SEP "arm" OS_SEP "_chgsignl.S", - "math" OS_SEP "arm" OS_SEP "exp2.c", - "math" OS_SEP "arm" OS_SEP "nearbyint.S", - "math" OS_SEP "arm" OS_SEP "nearbyintf.S", - "math" OS_SEP "arm" OS_SEP "nearbyintl.S", - "math" OS_SEP "arm" OS_SEP "trunc.S", - "math" OS_SEP "arm" OS_SEP "truncf.S", -}; - -static const char *mingwex_arm64_src[] = { - "math" OS_SEP "arm64" OS_SEP "_chgsignl.S", - "math" OS_SEP "arm64" OS_SEP "exp2f.S", - "math" OS_SEP "arm64" OS_SEP "exp2.S", - "math" OS_SEP "arm64" OS_SEP "nearbyintf.S", - "math" OS_SEP "arm64" OS_SEP "nearbyintl.S", - "math" OS_SEP "arm64" OS_SEP "nearbyint.S", - "math" OS_SEP "arm64" OS_SEP "truncf.S", - "math" OS_SEP "arm64" OS_SEP "trunc.S", -}; - -static const char *mingw_uuid_src[] = { - "libsrc/ativscp-uuid.c", - "libsrc/atsmedia-uuid.c", - "libsrc/bth-uuid.c", - "libsrc/cguid-uuid.c", - "libsrc/comcat-uuid.c", - "libsrc/devguid.c", - "libsrc/docobj-uuid.c", - "libsrc/dxva-uuid.c", - "libsrc/exdisp-uuid.c", - "libsrc/extras-uuid.c", - "libsrc/fwp-uuid.c", - "libsrc/guid_nul.c", - "libsrc/hlguids-uuid.c", - "libsrc/hlink-uuid.c", - "libsrc/mlang-uuid.c", - "libsrc/msctf-uuid.c", - "libsrc/mshtmhst-uuid.c", - "libsrc/mshtml-uuid.c", - "libsrc/msxml-uuid.c", - "libsrc/netcon-uuid.c", - "libsrc/ntddkbd-uuid.c", - "libsrc/ntddmou-uuid.c", - "libsrc/ntddpar-uuid.c", - "libsrc/ntddscsi-uuid.c", - "libsrc/ntddser-uuid.c", - "libsrc/ntddstor-uuid.c", - "libsrc/ntddvdeo-uuid.c", - "libsrc/oaidl-uuid.c", - "libsrc/objidl-uuid.c", - "libsrc/objsafe-uuid.c", - "libsrc/ocidl-uuid.c", - "libsrc/oleacc-uuid.c", - "libsrc/olectlid-uuid.c", - "libsrc/oleidl-uuid.c", - "libsrc/power-uuid.c", - "libsrc/powrprof-uuid.c", - "libsrc/uianimation-uuid.c", - "libsrc/usbcamdi-uuid.c", - "libsrc/usbiodef-uuid.c", - "libsrc/uuid.c", - "libsrc/vds-uuid.c", - "libsrc/virtdisk-uuid.c", - "libsrc/wia-uuid.c", -}; - -struct MinGWDef { - const char *name; - bool always_link; -}; -static const MinGWDef mingw_def_list[] = { - {"advapi32",true}, - {"bcrypt", false}, - {"comctl32",false}, - {"comdlg32",false}, - {"crypt32", false}, - {"cryptnet",false}, - {"gdi32", false}, - {"imm32", false}, - {"kernel32",true}, - {"lz32", false}, - {"mpr", false}, - {"msvcrt", true}, - {"mswsock", false}, - {"ncrypt", false}, - {"netapi32",false}, - {"ntdll", true}, - {"ole32", false}, - {"oleaut32",false}, - {"opengl32",false}, - {"psapi", false}, - {"rpcns4", false}, - {"rpcrt4", false}, - {"scarddlg",false}, - {"setupapi",false}, - {"shell32", true}, - {"shlwapi", false}, - {"urlmon", false}, - {"user32", true}, - {"version", false}, - {"winmm", false}, - {"winscard",false}, - {"winspool",false}, - {"wintrust",false}, - {"ws2_32", false}, -}; - -struct LinkJob { - CodeGen *codegen; - ZigList args; - bool link_in_crt; - HashMap rpath_table; - Stage2ProgressNode *build_dep_prog_node; -}; - -static const char *build_libc_object(CodeGen *parent_gen, const char *name, CFile *c_file, - Stage2ProgressNode *progress_node) -{ - CodeGen *child_gen = create_child_codegen(parent_gen, nullptr, OutTypeObj, nullptr, name, progress_node); - child_gen->root_out_name = buf_create_from_str(name); - ZigList c_source_files = {0}; - c_source_files.append(c_file); - child_gen->c_source_files = c_source_files; - codegen_build_and_link(child_gen); - return buf_ptr(&child_gen->bin_file_output_path); -} - -static const char *path_from_zig_lib(CodeGen *g, const char *dir, const char *subpath) { - Buf *dir1 = buf_alloc(); - os_path_join(g->zig_lib_dir, buf_create_from_str(dir), dir1); - Buf *result = buf_alloc(); - os_path_join(dir1, buf_create_from_str(subpath), result); - return buf_ptr(result); -} - -static const char *path_from_libc(CodeGen *g, const char *subpath) { - return path_from_zig_lib(g, "libc", subpath); -} - -static const char *path_from_libunwind(CodeGen *g, const char *subpath) { - return path_from_zig_lib(g, "libunwind", subpath); -} - -static const char *build_libunwind(CodeGen *parent, Stage2ProgressNode *progress_node) { - CodeGen *child_gen = create_child_codegen(parent, nullptr, OutTypeLib, nullptr, "unwind", progress_node); - LinkLib *new_link_lib = codegen_add_link_lib(child_gen, buf_create_from_str("c")); - new_link_lib->provided_explicitly = false; - enum SrcKind { - SrcCpp, - SrcC, - SrcAsm, - }; - static const struct { - const char *path; - SrcKind kind; - } unwind_src[] = { - {"src" OS_SEP "libunwind.cpp", SrcCpp}, - {"src" OS_SEP "Unwind-EHABI.cpp", SrcCpp}, - {"src" OS_SEP "Unwind-seh.cpp", SrcCpp}, - - {"src" OS_SEP "UnwindLevel1.c", SrcC}, - {"src" OS_SEP "UnwindLevel1-gcc-ext.c", SrcC}, - {"src" OS_SEP "Unwind-sjlj.c", SrcC}, - - {"src" OS_SEP "UnwindRegistersRestore.S", SrcAsm}, - {"src" OS_SEP "UnwindRegistersSave.S", SrcAsm}, - }; - ZigList c_source_files = {0}; - for (size_t i = 0; i < array_length(unwind_src); i += 1) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = path_from_libunwind(parent, unwind_src[i].path); - switch (unwind_src[i].kind) { - case SrcC: - c_file->args.append("-std=c99"); - break; - case SrcCpp: - c_file->args.append("-fno-rtti"); - c_file->args.append("-I"); - c_file->args.append(path_from_zig_lib(parent, "libcxx", "include")); - break; - case SrcAsm: - break; - } - c_file->args.append("-I"); - c_file->args.append(path_from_libunwind(parent, "include")); - if (target_supports_fpic(parent->zig_target)) { - c_file->args.append("-fPIC"); - } - c_file->args.append("-D_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS"); - c_file->args.append("-Wa,--noexecstack"); - - // This is intentionally always defined because the macro definition means, should it only - // build for the target specified by compiler defines. Since we pass -target the compiler - // defines will be correct. - c_file->args.append("-D_LIBUNWIND_IS_NATIVE_ONLY"); - - if (parent->build_mode == BuildModeDebug) { - c_file->args.append("-D_DEBUG"); - } - if (parent->is_single_threaded) { - c_file->args.append("-D_LIBUNWIND_HAS_NO_THREADS"); - } - c_file->args.append("-Wno-bitwise-conditional-parentheses"); - c_source_files.append(c_file); - } - child_gen->c_source_files = c_source_files; - codegen_build_and_link(child_gen); - return buf_ptr(&child_gen->bin_file_output_path); -} - -static void mingw_add_cc_args(CodeGen *parent, CFile *c_file) { - c_file->args.append("-DHAVE_CONFIG_H"); - - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "mingw" OS_SEP "include", - buf_ptr(parent->zig_lib_dir)))); - - c_file->args.append("-isystem"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "include" OS_SEP "any-windows-any", - buf_ptr(parent->zig_lib_dir)))); - - if (target_is_arm(parent->zig_target) && - target_arch_pointer_bit_width(parent->zig_target->arch) == 32) - { - c_file->args.append("-mfpu=vfp"); - } - - c_file->args.append("-std=gnu11"); - c_file->args.append("-D_CRTBLD"); - c_file->args.append("-D_WIN32_WINNT=0x0f00"); - c_file->args.append("-D__MSVCRT_VERSION__=0x700"); -} - -static void glibc_add_include_dirs_arch(CFile *c_file, ZigLLVM_ArchType arch, const char *nptl, const char *dir) { - bool is_x86 = arch == ZigLLVM_x86 || arch == ZigLLVM_x86_64; - bool is_aarch64 = arch == ZigLLVM_aarch64 || arch == ZigLLVM_aarch64_be; - bool is_mips = arch == ZigLLVM_mips || arch == ZigLLVM_mipsel || - arch == ZigLLVM_mips64el || arch == ZigLLVM_mips64; - bool is_arm = arch == ZigLLVM_arm || arch == ZigLLVM_armeb; - bool is_ppc = arch == ZigLLVM_ppc || arch == ZigLLVM_ppc64 || arch == ZigLLVM_ppc64le; - bool is_riscv = arch == ZigLLVM_riscv32 || arch == ZigLLVM_riscv64; - bool is_sparc = arch == ZigLLVM_sparc || arch == ZigLLVM_sparcel || arch == ZigLLVM_sparcv9; - bool is_64 = target_arch_pointer_bit_width(arch) == 64; - - if (is_x86) { - if (arch == ZigLLVM_x86_64) { - if (nptl != nullptr) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "x86_64" OS_SEP "%s", dir, nptl))); - } else { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "x86_64", dir))); - } - } else if (arch == ZigLLVM_x86) { - if (nptl != nullptr) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "i386" OS_SEP "%s", dir, nptl))); - } else { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "i386", dir))); - } - } - if (nptl != nullptr) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "x86" OS_SEP "%s", dir, nptl))); - } else { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "x86", dir))); - } - } else if (is_arm) { - if (nptl != nullptr) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "arm" OS_SEP "%s", dir, nptl))); - } else { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "arm", dir))); - } - } else if (is_mips) { - if (nptl != nullptr) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "mips" OS_SEP "%s", dir, nptl))); - } else { - if (is_64) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "mips" OS_SEP "mips64", dir))); - } else { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "mips" OS_SEP "mips32", dir))); - } - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "mips", dir))); - } - } else if (is_sparc) { - if (nptl != nullptr) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "sparc" OS_SEP "%s", dir, nptl))); - } else { - if (is_64) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "sparc" OS_SEP "sparc64", dir))); - } else { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "sparc" OS_SEP "sparc32", dir))); - } - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "sparc", dir))); - } - } else if (is_aarch64) { - if (nptl != nullptr) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "aarch64" OS_SEP "%s", dir, nptl))); - } else { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "aarch64", dir))); - } - } else if (is_ppc) { - if (nptl != nullptr) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "powerpc" OS_SEP "%s", dir, nptl))); - } else { - if (is_64) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "powerpc" OS_SEP "powerpc64", dir))); - } else { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "powerpc" OS_SEP "powerpc32", dir))); - } - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "powerpc", dir))); - } - } else if (is_riscv) { - if (nptl != nullptr) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "riscv" OS_SEP "%s", dir, nptl))); - } else { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "riscv", dir))); - } - } -} - -static void glibc_add_include_dirs(CodeGen *parent, CFile *c_file) { - ZigLLVM_ArchType arch = parent->zig_target->arch; - const char *nptl = (parent->zig_target->os == OsLinux) ? "nptl" : "htl"; - const char *glibc = path_from_libc(parent, "glibc"); - - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "include", glibc))); - - if (parent->zig_target->os == OsLinux) { - glibc_add_include_dirs_arch(c_file, arch, nullptr, - path_from_libc(parent, "glibc" OS_SEP "sysdeps" OS_SEP "unix" OS_SEP "sysv" OS_SEP "linux")); - } - - if (nptl != nullptr) { - glibc_add_include_dirs_arch(c_file, arch, nptl, path_from_libc(parent, "glibc" OS_SEP "sysdeps")); - } - - if (parent->zig_target->os == OsLinux) { - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "sysdeps" OS_SEP - "unix" OS_SEP "sysv" OS_SEP "linux" OS_SEP "generic")); - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "sysdeps" OS_SEP - "unix" OS_SEP "sysv" OS_SEP "linux" OS_SEP "include")); - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "sysdeps" OS_SEP - "unix" OS_SEP "sysv" OS_SEP "linux")); - } - if (nptl != nullptr) { - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "sysdeps" OS_SEP "%s", glibc, nptl))); - } - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "sysdeps" OS_SEP "pthread")); - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "sysdeps" OS_SEP "unix" OS_SEP "sysv")); - - glibc_add_include_dirs_arch(c_file, arch, nullptr, - path_from_libc(parent, "glibc" OS_SEP "sysdeps" OS_SEP "unix")); - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "sysdeps" OS_SEP "unix")); - - glibc_add_include_dirs_arch(c_file, arch, nullptr, path_from_libc(parent, "glibc" OS_SEP "sysdeps")); - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "sysdeps" OS_SEP "generic")); - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "glibc")); - - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "include" OS_SEP "%s-%s-%s", - buf_ptr(parent->zig_lib_dir), target_arch_name(parent->zig_target->arch), - target_os_name(parent->zig_target->os), target_abi_name(parent->zig_target->abi)))); - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "include" OS_SEP "generic-glibc")); - - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "include" OS_SEP "%s-linux-any", - buf_ptr(parent->zig_lib_dir), target_arch_name(parent->zig_target->arch)))); - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "include" OS_SEP "any-linux-any")); -} - -static const char *glibc_start_asm_path(CodeGen *parent, const char *file) { - ZigLLVM_ArchType arch = parent->zig_target->arch; - bool is_aarch64 = arch == ZigLLVM_aarch64 || arch == ZigLLVM_aarch64_be; - bool is_mips = arch == ZigLLVM_mips || arch == ZigLLVM_mipsel || - arch == ZigLLVM_mips64el || arch == ZigLLVM_mips64; - bool is_arm = arch == ZigLLVM_arm || arch == ZigLLVM_armeb; - bool is_ppc = arch == ZigLLVM_ppc || arch == ZigLLVM_ppc64 || arch == ZigLLVM_ppc64le; - bool is_riscv = arch == ZigLLVM_riscv32 || arch == ZigLLVM_riscv64; - bool is_sparc = arch == ZigLLVM_sparc || arch == ZigLLVM_sparcel || arch == ZigLLVM_sparcv9; - bool is_64 = target_arch_pointer_bit_width(arch) == 64; - - Buf result = BUF_INIT; - buf_resize(&result, 0); - buf_append_buf(&result, parent->zig_lib_dir); - buf_append_str(&result, OS_SEP "libc" OS_SEP "glibc" OS_SEP "sysdeps" OS_SEP); - if (is_sparc) { - if (is_64) { - buf_append_str(&result, "sparc" OS_SEP "sparc64"); - } else { - buf_append_str(&result, "sparc" OS_SEP "sparc32"); - } - } else if (is_arm) { - buf_append_str(&result, "arm"); - } else if (is_mips) { - buf_append_str(&result, "mips"); - } else if (arch == ZigLLVM_x86_64) { - buf_append_str(&result, "x86_64"); - } else if (arch == ZigLLVM_x86) { - buf_append_str(&result, "i386"); - } else if (is_aarch64) { - buf_append_str(&result, "aarch64"); - } else if (is_riscv) { - buf_append_str(&result, "riscv"); - } else if (is_ppc) { - if (is_64) { - buf_append_str(&result, "powerpc" OS_SEP "powerpc64"); - } else { - buf_append_str(&result, "powerpc" OS_SEP "powerpc32"); - } - } - - buf_append_str(&result, OS_SEP); - buf_append_str(&result, file); - return buf_ptr(&result); -} - -static const char *musl_start_asm_path(CodeGen *parent, const char *file) { - Buf *result = buf_sprintf("%s" OS_SEP "libc" OS_SEP "musl" OS_SEP "crt" OS_SEP "%s" OS_SEP "%s", - buf_ptr(parent->zig_lib_dir), target_arch_musl_name(parent->zig_target->arch), file); - return buf_ptr(result); -} - -static void musl_add_cc_args(CodeGen *parent, CFile *c_file, bool want_O3) { - c_file->args.append("-std=c99"); - c_file->args.append("-ffreestanding"); - // Musl adds these args to builds with gcc but clang does not support them. - //c_file->args.append("-fexcess-precision=standard"); - //c_file->args.append("-frounding-math"); - c_file->args.append("-Wa,--noexecstack"); - c_file->args.append("-D_XOPEN_SOURCE=700"); - - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "musl" OS_SEP "arch" OS_SEP "%s", - buf_ptr(parent->zig_lib_dir), target_arch_musl_name(parent->zig_target->arch)))); - - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "musl" OS_SEP "arch" OS_SEP "generic", - buf_ptr(parent->zig_lib_dir)))); - - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "musl" OS_SEP "src" OS_SEP "include", - buf_ptr(parent->zig_lib_dir)))); - - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "musl" OS_SEP "src" OS_SEP "internal", - buf_ptr(parent->zig_lib_dir)))); - - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "musl" OS_SEP "include", - buf_ptr(parent->zig_lib_dir)))); - - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf( - "%s" OS_SEP "libc" OS_SEP "include" OS_SEP "%s-%s-musl", - buf_ptr(parent->zig_lib_dir), - target_arch_musl_name(parent->zig_target->arch), - target_os_name(parent->zig_target->os)))); - - c_file->args.append("-I"); - c_file->args.append(buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "include" OS_SEP "generic-musl", - buf_ptr(parent->zig_lib_dir)))); - - if (want_O3) - c_file->args.append("-O3"); - else - c_file->args.append("-Os"); - - c_file->args.append("-fomit-frame-pointer"); - c_file->args.append("-fno-unwind-tables"); - c_file->args.append("-fno-asynchronous-unwind-tables"); - c_file->args.append("-ffunction-sections"); - c_file->args.append("-fdata-sections"); -} - -static const char *musl_arch_names[] = { - "aarch64", - "arm", - "generic", - "i386", - "m68k", - "microblaze", - "mips", - "mips64", - "mipsn32", - "or1k", - "powerpc", - "powerpc64", - "riscv64", - "s390x", - "sh", - "x32", - "x86_64", -}; - -static bool is_musl_arch_name(const char *name) { - for (size_t i = 0; i < array_length(musl_arch_names); i += 1) { - if (strcmp(name, musl_arch_names[i]) == 0) - return true; - } - return false; -} - -enum MuslSrc { - MuslSrcAsm, - MuslSrcNormal, - MuslSrcO3, -}; - -static void add_musl_src_file(HashMap &source_table, - const char *file_path) -{ - Buf *src_file = buf_create_from_str(file_path); - - MuslSrc src_kind; - if (buf_ends_with_str(src_file, ".c")) { - bool want_O3 = buf_starts_with_str(src_file, "musl/src/malloc/") || - buf_starts_with_str(src_file, "musl/src/string/") || - buf_starts_with_str(src_file, "musl/src/internal/"); - src_kind = want_O3 ? MuslSrcO3 : MuslSrcNormal; - } else if (buf_ends_with_str(src_file, ".s") || buf_ends_with_str(src_file, ".S")) { - src_kind = MuslSrcAsm; - } else { - zig_unreachable(); - } - if (ZIG_OS_SEP_CHAR != '/') { - buf_replace(src_file, '/', ZIG_OS_SEP_CHAR); - } - source_table.put_unique(src_file, src_kind); -} - -static const char *build_musl(CodeGen *parent, Stage2ProgressNode *progress_node) { - CodeGen *child_gen = create_child_codegen(parent, nullptr, OutTypeLib, nullptr, "c", progress_node); - - // When there is a src//foo.* then it should substitute for src/foo.* - // Even a .s file can substitute for a .c file. - - const char *target_musl_arch_name = target_arch_musl_name(parent->zig_target->arch); - - HashMap source_table = {}; - source_table.init(2000); - - for (size_t i = 0; i < array_length(ZIG_MUSL_SRC_FILES); i += 1) { - add_musl_src_file(source_table, ZIG_MUSL_SRC_FILES[i]); - } - - static const char *time32_compat_arch_list[] = {"arm", "i386", "mips", "powerpc"}; - for (size_t arch_i = 0; arch_i < array_length(time32_compat_arch_list); arch_i += 1) { - if (strcmp(target_musl_arch_name, time32_compat_arch_list[arch_i]) == 0) { - for (size_t i = 0; i < array_length(ZIG_MUSL_COMPAT_TIME32_FILES); i += 1) { - add_musl_src_file(source_table, ZIG_MUSL_COMPAT_TIME32_FILES[i]); - } - } - } - - - ZigList c_source_files = {0}; - - Buf dirname = BUF_INIT; - Buf basename = BUF_INIT; - Buf noextbasename = BUF_INIT; - Buf dirbasename = BUF_INIT; - Buf before_arch_dir = BUF_INIT; - - auto source_it = source_table.entry_iterator(); - for (;;) { - auto *entry = source_it.next(); - if (!entry) break; - - Buf *src_file = entry->key; - MuslSrc src_kind = entry->value; - - os_path_split(src_file, &dirname, &basename); - os_path_extname(&basename, &noextbasename, nullptr); - os_path_split(&dirname, &before_arch_dir, &dirbasename); - - bool is_arch_specific = false; - // Architecture-specific implementations are under a / folder. - if (is_musl_arch_name(buf_ptr(&dirbasename))) { - // Not the architecture we're compiling for. - if (strcmp(buf_ptr(&dirbasename), target_musl_arch_name) != 0) - continue; - is_arch_specific = true; - } - - if (!is_arch_specific) { - Buf override_path = BUF_INIT; - - // Look for an arch specific override. - buf_resize(&override_path, 0); - buf_appendf(&override_path, "%s" OS_SEP "%s" OS_SEP "%s.s", - buf_ptr(&dirname), target_musl_arch_name, buf_ptr(&noextbasename)); - if (source_table.maybe_get(&override_path) != nullptr) - continue; - - buf_resize(&override_path, 0); - buf_appendf(&override_path, "%s" OS_SEP "%s" OS_SEP "%s.S", - buf_ptr(&dirname), target_musl_arch_name, buf_ptr(&noextbasename)); - if (source_table.maybe_get(&override_path) != nullptr) - continue; - - buf_resize(&override_path, 0); - buf_appendf(&override_path, "%s" OS_SEP "%s" OS_SEP "%s.c", - buf_ptr(&dirname), target_musl_arch_name, buf_ptr(&noextbasename)); - if (source_table.maybe_get(&override_path) != nullptr) - continue; - } - - Buf *full_path = buf_sprintf("%s" OS_SEP "libc" OS_SEP "%s", - buf_ptr(parent->zig_lib_dir), buf_ptr(src_file)); - - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = buf_ptr(full_path); - - musl_add_cc_args(parent, c_file, src_kind == MuslSrcO3); - c_file->args.append("-Qunused-arguments"); - c_file->args.append("-w"); // disable all warnings - - c_source_files.append(c_file); - } - - child_gen->c_source_files = c_source_files; - codegen_build_and_link(child_gen); - return buf_ptr(&child_gen->bin_file_output_path); -} - -static const char *build_libcxxabi(CodeGen *parent, Stage2ProgressNode *progress_node) { - CodeGen *child_gen = create_child_codegen(parent, nullptr, OutTypeLib, nullptr, "c++abi", progress_node); - codegen_add_link_lib(child_gen, buf_create_from_str("c")); - - ZigList c_source_files = {0}; - - const char *cxxabi_include_path = buf_ptr(buf_sprintf("%s" OS_SEP "libcxxabi" OS_SEP "include", - buf_ptr(parent->zig_lib_dir))); - const char *cxx_include_path = buf_ptr(buf_sprintf("%s" OS_SEP "libcxx" OS_SEP "include", - buf_ptr(parent->zig_lib_dir))); - - for (size_t i = 0; i < array_length(ZIG_LIBCXXABI_FILES); i += 1) { - const char *rel_src_path = ZIG_LIBCXXABI_FILES[i]; - - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = buf_ptr(buf_sprintf("%s" OS_SEP "libcxxabi" OS_SEP "%s", - buf_ptr(parent->zig_lib_dir), rel_src_path)); - - c_file->args.append("-DHAVE___CXA_THREAD_ATEXIT_IMPL"); - c_file->args.append("-D_LIBCPP_DISABLE_EXTERN_TEMPLATE"); - c_file->args.append("-D_LIBCPP_ENABLE_CXX17_REMOVED_UNEXPECTED_FUNCTIONS"); - c_file->args.append("-D_LIBCXXABI_BUILDING_LIBRARY"); - c_file->args.append("-D_LIBCXXABI_DISABLE_VISIBILITY_ANNOTATIONS"); - c_file->args.append("-D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS"); - - if (target_abi_is_musl(parent->zig_target->abi)) { - c_file->args.append("-D_LIBCPP_HAS_MUSL_LIBC"); - } - - c_file->args.append("-I"); - c_file->args.append(cxxabi_include_path); - - c_file->args.append("-I"); - c_file->args.append(cxx_include_path); - - c_file->args.append("-O3"); - c_file->args.append("-DNDEBUG"); - if (target_supports_fpic(parent->zig_target)) { - c_file->args.append("-fPIC"); - } - c_file->args.append("-nostdinc++"); - c_file->args.append("-fstrict-aliasing"); - c_file->args.append("-funwind-tables"); - c_file->args.append("-D_DEBUG"); - c_file->args.append("-UNDEBUG"); - c_file->args.append("-std=c++11"); - - c_source_files.append(c_file); - } - - - child_gen->c_source_files = c_source_files; - codegen_build_and_link(child_gen); - return buf_ptr(&child_gen->bin_file_output_path); -} - -static const char *build_libcxx(CodeGen *parent, Stage2ProgressNode *progress_node) { - CodeGen *child_gen = create_child_codegen(parent, nullptr, OutTypeLib, nullptr, "c++", progress_node); - codegen_add_link_lib(child_gen, buf_create_from_str("c")); - - ZigList c_source_files = {0}; - - const char *cxxabi_include_path = buf_ptr(buf_sprintf("%s" OS_SEP "libcxxabi" OS_SEP "include", - buf_ptr(parent->zig_lib_dir))); - const char *cxx_include_path = buf_ptr(buf_sprintf("%s" OS_SEP "libcxx" OS_SEP "include", - buf_ptr(parent->zig_lib_dir))); - - for (size_t i = 0; i < array_length(ZIG_LIBCXX_FILES); i += 1) { - const char *rel_src_path = ZIG_LIBCXX_FILES[i]; - - Buf *src_path_buf = buf_create_from_str(rel_src_path); - if (parent->zig_target->os == OsWindows) { - // filesystem stuff isn't supported on Windows - if (buf_starts_with_str(src_path_buf, "src/filesystem/")) { - continue; - } - } else { - if (buf_starts_with_str(src_path_buf, "src/support/win32/")) { - continue; - } - } - - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = buf_ptr(buf_sprintf("%s" OS_SEP "libcxx" OS_SEP "%s", - buf_ptr(parent->zig_lib_dir), rel_src_path)); - - c_file->args.append("-DNDEBUG"); - c_file->args.append("-D_LIBCPP_BUILDING_LIBRARY"); - c_file->args.append("-D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER"); - c_file->args.append("-DLIBCXX_BUILDING_LIBCXXABI"); - c_file->args.append("-D_LIBCXXABI_DISABLE_VISIBILITY_ANNOTATIONS"); - c_file->args.append("-D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS"); - - if (target_abi_is_musl(parent->zig_target->abi)) { - c_file->args.append("-D_LIBCPP_HAS_MUSL_LIBC"); - } - - c_file->args.append("-I"); - c_file->args.append(cxx_include_path); - - c_file->args.append("-I"); - c_file->args.append(cxxabi_include_path); - - c_file->args.append("-O3"); - c_file->args.append("-DNDEBUG"); - if (target_supports_fpic(parent->zig_target)) { - c_file->args.append("-fPIC"); - } - c_file->args.append("-nostdinc++"); - c_file->args.append("-fvisibility-inlines-hidden"); - c_file->args.append("-std=c++14"); - c_file->args.append("-Wno-user-defined-literals"); - - c_source_files.append(c_file); - } - - - child_gen->c_source_files = c_source_files; - codegen_build_and_link(child_gen); - return buf_ptr(&child_gen->bin_file_output_path); -} - -static void add_msvcrt_os_dep(CodeGen *parent, CodeGen *child_gen, const char *src_path) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "mingw" OS_SEP "%s", - buf_ptr(parent->zig_lib_dir), src_path)); - c_file->args.append("-DHAVE_CONFIG_H"); - c_file->args.append("-D__LIBMSVCRT__"); - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "mingw" OS_SEP "include")); - - c_file->args.append("-std=gnu99"); - c_file->args.append("-D_CRTBLD"); - c_file->args.append("-D_WIN32_WINNT=0x0f00"); - c_file->args.append("-D__MSVCRT_VERSION__=0x700"); - - c_file->args.append("-isystem"); - c_file->args.append(path_from_libc(parent, "include" OS_SEP "any-windows-any")); - - c_file->args.append("-g"); - c_file->args.append("-O2"); - - child_gen->c_source_files.append(c_file); -} - -static void add_mingwex_dep(CodeGen *parent, CodeGen *child_gen, const char *src_path) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "mingw" OS_SEP "%s", - buf_ptr(parent->zig_lib_dir), src_path)); - c_file->args.append("-DHAVE_CONFIG_H"); - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "mingw")); - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "mingw" OS_SEP "include")); - - c_file->args.append("-std=gnu99"); - c_file->args.append("-D_CRTBLD"); - c_file->args.append("-D_WIN32_WINNT=0x0f00"); - c_file->args.append("-D__MSVCRT_VERSION__=0x700"); - c_file->args.append("-g"); - c_file->args.append("-O2"); - - c_file->args.append("-isystem"); - c_file->args.append(path_from_libc(parent, "include" OS_SEP "any-windows-any")); - - child_gen->c_source_files.append(c_file); -} - -static void add_mingw_uuid_dep(CodeGen *parent, CodeGen *child_gen, const char *src_path) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = buf_ptr(buf_sprintf("%s" OS_SEP "libc" OS_SEP "mingw" OS_SEP "%s", - buf_ptr(parent->zig_lib_dir), src_path)); - c_file->args.append("-DHAVE_CONFIG_H"); - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "mingw")); - - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "mingw" OS_SEP "include")); - - c_file->args.append("-std=gnu99"); - c_file->args.append("-D_CRTBLD"); - c_file->args.append("-D_WIN32_WINNT=0x0f00"); - c_file->args.append("-D__MSVCRT_VERSION__=0x700"); - c_file->args.append("-g"); - c_file->args.append("-O2"); - - c_file->args.append("-isystem"); - c_file->args.append(path_from_libc(parent, "include" OS_SEP "any-windows-any")); - - child_gen->c_source_files.append(c_file); -} - -static const char *get_libc_crt_file(CodeGen *parent, const char *file, Stage2ProgressNode *progress_node) { - if (parent->libc == nullptr && parent->zig_target->os == OsWindows) { - if (strcmp(file, "crt2.o") == 0) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = buf_ptr(buf_sprintf( - "%s" OS_SEP "libc" OS_SEP "mingw" OS_SEP "crt" OS_SEP "crtexe.c", buf_ptr(parent->zig_lib_dir))); - mingw_add_cc_args(parent, c_file); - c_file->args.append("-U__CRTDLL__"); - c_file->args.append("-D__MSVCRT__"); - // Uncomment these 3 things for crtu - //c_file->args.append("-DUNICODE"); - //c_file->args.append("-D_UNICODE"); - //c_file->args.append("-DWPRFLAG=1"); - return build_libc_object(parent, "crt2", c_file, progress_node); - } else if (strcmp(file, "dllcrt2.o") == 0) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = buf_ptr(buf_sprintf( - "%s" OS_SEP "libc" OS_SEP "mingw" OS_SEP "crt" OS_SEP "crtdll.c", buf_ptr(parent->zig_lib_dir))); - mingw_add_cc_args(parent, c_file); - c_file->args.append("-U__CRTDLL__"); - c_file->args.append("-D__MSVCRT__"); - return build_libc_object(parent, "dllcrt2", c_file, progress_node); - } else if (strcmp(file, "mingw32.lib") == 0) { - CodeGen *child_gen = create_child_codegen(parent, nullptr, OutTypeLib, nullptr, "mingw32", progress_node); - - static const char *deps[] = { - "mingw" OS_SEP "crt" OS_SEP "crt0_c.c", - "mingw" OS_SEP "crt" OS_SEP "dll_argv.c", - "mingw" OS_SEP "crt" OS_SEP "gccmain.c", - "mingw" OS_SEP "crt" OS_SEP "natstart.c", - "mingw" OS_SEP "crt" OS_SEP "pseudo-reloc-list.c", - "mingw" OS_SEP "crt" OS_SEP "wildcard.c", - "mingw" OS_SEP "crt" OS_SEP "charmax.c", - "mingw" OS_SEP "crt" OS_SEP "crt0_w.c", - "mingw" OS_SEP "crt" OS_SEP "dllargv.c", - "mingw" OS_SEP "crt" OS_SEP "gs_support.c", - "mingw" OS_SEP "crt" OS_SEP "_newmode.c", - "mingw" OS_SEP "crt" OS_SEP "tlssup.c", - "mingw" OS_SEP "crt" OS_SEP "xncommod.c", - "mingw" OS_SEP "crt" OS_SEP "cinitexe.c", - "mingw" OS_SEP "crt" OS_SEP "merr.c", - "mingw" OS_SEP "crt" OS_SEP "usermatherr.c", - "mingw" OS_SEP "crt" OS_SEP "pesect.c", - "mingw" OS_SEP "crt" OS_SEP "udllargc.c", - "mingw" OS_SEP "crt" OS_SEP "xthdloc.c", - "mingw" OS_SEP "crt" OS_SEP "CRT_fp10.c", - "mingw" OS_SEP "crt" OS_SEP "mingw_helpers.c", - "mingw" OS_SEP "crt" OS_SEP "pseudo-reloc.c", - "mingw" OS_SEP "crt" OS_SEP "udll_argv.c", - "mingw" OS_SEP "crt" OS_SEP "xtxtmode.c", - "mingw" OS_SEP "crt" OS_SEP "crt_handler.c", - "mingw" OS_SEP "crt" OS_SEP "tlsthrd.c", - "mingw" OS_SEP "crt" OS_SEP "tlsmthread.c", - "mingw" OS_SEP "crt" OS_SEP "tlsmcrt.c", - "mingw" OS_SEP "crt" OS_SEP "cxa_atexit.c", - }; - for (size_t i = 0; i < array_length(deps); i += 1) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = path_from_libc(parent, deps[i]); - c_file->args.append("-DHAVE_CONFIG_H"); - c_file->args.append("-D_SYSCRT=1"); - c_file->args.append("-DCRTDLL=1"); - - c_file->args.append("-isystem"); - c_file->args.append(path_from_libc(parent, "include" OS_SEP "any-windows-any")); - - c_file->args.append("-isystem"); - c_file->args.append(path_from_libc(parent, "mingw" OS_SEP "include")); - - c_file->args.append("-std=gnu99"); - c_file->args.append("-D_CRTBLD"); - c_file->args.append("-D_WIN32_WINNT=0x0f00"); - c_file->args.append("-D__MSVCRT_VERSION__=0x700"); - c_file->args.append("-g"); - c_file->args.append("-O2"); - - child_gen->c_source_files.append(c_file); - } - codegen_build_and_link(child_gen); - return buf_ptr(&child_gen->bin_file_output_path); - } else if (strcmp(file, "msvcrt-os.lib") == 0) { - CodeGen *child_gen = create_child_codegen(parent, nullptr, OutTypeLib, nullptr, "msvcrt-os", progress_node); - - for (size_t i = 0; i < array_length(msvcrt_common_src); i += 1) { - add_msvcrt_os_dep(parent, child_gen, msvcrt_common_src[i]); - } - if (parent->zig_target->arch == ZigLLVM_x86) { - for (size_t i = 0; i < array_length(msvcrt_i386_src); i += 1) { - add_msvcrt_os_dep(parent, child_gen, msvcrt_i386_src[i]); - } - } else { - for (size_t i = 0; i < array_length(msvcrt_other_src); i += 1) { - add_msvcrt_os_dep(parent, child_gen, msvcrt_other_src[i]); - } - } - codegen_build_and_link(child_gen); - return buf_ptr(&child_gen->bin_file_output_path); - } else if (strcmp(file, "mingwex.lib") == 0) { - CodeGen *child_gen = create_child_codegen(parent, nullptr, OutTypeLib, nullptr, "mingwex", progress_node); - - for (size_t i = 0; i < array_length(mingwex_generic_src); i += 1) { - add_mingwex_dep(parent, child_gen, mingwex_generic_src[i]); - } - if (parent->zig_target->arch == ZigLLVM_x86 || parent->zig_target->arch == ZigLLVM_x86_64) { - for (size_t i = 0; i < array_length(mingwex_x86_src); i += 1) { - add_mingwex_dep(parent, child_gen, mingwex_x86_src[i]); - } - } else if (target_is_arm(parent->zig_target)) { - if (target_arch_pointer_bit_width(parent->zig_target->arch) == 32) { - for (size_t i = 0; i < array_length(mingwex_arm32_src); i += 1) { - add_mingwex_dep(parent, child_gen, mingwex_arm32_src[i]); - } - } else { - for (size_t i = 0; i < array_length(mingwex_arm64_src); i += 1) { - add_mingwex_dep(parent, child_gen, mingwex_arm64_src[i]); - } - } - } else { - zig_unreachable(); - } - codegen_build_and_link(child_gen); - return buf_ptr(&child_gen->bin_file_output_path); - } else if (strcmp(file, "uuid.lib") == 0) { - CodeGen *child_gen = create_child_codegen(parent, nullptr, OutTypeLib, nullptr, "uuid", progress_node); - for (size_t i = 0; i < array_length(mingw_uuid_src); i += 1) { - add_mingw_uuid_dep(parent, child_gen, mingw_uuid_src[i]); - } - codegen_build_and_link(child_gen); - return buf_ptr(&child_gen->bin_file_output_path); - } else { - zig_unreachable(); - } - } else if (parent->libc == nullptr && target_is_glibc(parent->zig_target)) { - if (strcmp(file, "crti.o") == 0) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = glibc_start_asm_path(parent, "crti.S"); - glibc_add_include_dirs(parent, c_file); - c_file->args.append("-D_LIBC_REENTRANT"); - c_file->args.append("-include"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-modules.h")); - c_file->args.append("-DMODULE_NAME=libc"); - c_file->args.append("-Wno-nonportable-include-path"); - c_file->args.append("-include"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-symbols.h")); - c_file->args.append("-DTOP_NAMESPACE=glibc"); - c_file->args.append("-DASSEMBLER"); - c_file->args.append("-g"); - c_file->args.append("-Wa,--noexecstack"); - return build_libc_object(parent, "crti", c_file, progress_node); - } else if (strcmp(file, "crtn.o") == 0) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = glibc_start_asm_path(parent, "crtn.S"); - glibc_add_include_dirs(parent, c_file); - c_file->args.append("-D_LIBC_REENTRANT"); - c_file->args.append("-DMODULE_NAME=libc"); - c_file->args.append("-DTOP_NAMESPACE=glibc"); - c_file->args.append("-DASSEMBLER"); - c_file->args.append("-g"); - c_file->args.append("-Wa,--noexecstack"); - return build_libc_object(parent, "crtn", c_file, progress_node); - } else if (strcmp(file, "start.os") == 0) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = glibc_start_asm_path(parent, "start.S"); - glibc_add_include_dirs(parent, c_file); - c_file->args.append("-D_LIBC_REENTRANT"); - c_file->args.append("-include"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-modules.h")); - c_file->args.append("-DMODULE_NAME=libc"); - c_file->args.append("-Wno-nonportable-include-path"); - c_file->args.append("-include"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-symbols.h")); - c_file->args.append("-DPIC"); - c_file->args.append("-DSHARED"); - c_file->args.append("-DTOP_NAMESPACE=glibc"); - c_file->args.append("-DASSEMBLER"); - c_file->args.append("-g"); - c_file->args.append("-Wa,--noexecstack"); - return build_libc_object(parent, "start", c_file, progress_node); - } else if (strcmp(file, "abi-note.o") == 0) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = path_from_libc(parent, "glibc" OS_SEP "csu" OS_SEP "abi-note.S"); - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "csu")); - glibc_add_include_dirs(parent, c_file); - c_file->args.append("-D_LIBC_REENTRANT"); - c_file->args.append("-DMODULE_NAME=libc"); - c_file->args.append("-DTOP_NAMESPACE=glibc"); - c_file->args.append("-DASSEMBLER"); - c_file->args.append("-g"); - c_file->args.append("-Wa,--noexecstack"); - return build_libc_object(parent, "abi-note", c_file, progress_node); - } else if (strcmp(file, "Scrt1.o") == 0) { - const char *start_os = get_libc_crt_file(parent, "start.os", progress_node); - const char *abi_note_o = get_libc_crt_file(parent, "abi-note.o", progress_node); - CodeGen *child_gen = create_child_codegen(parent, nullptr, OutTypeObj, nullptr, "Scrt1", progress_node); - codegen_add_object(child_gen, buf_create_from_str(start_os)); - codegen_add_object(child_gen, buf_create_from_str(abi_note_o)); - codegen_build_and_link(child_gen); - return buf_ptr(&child_gen->bin_file_output_path); - } else if (strcmp(file, "libc_nonshared.a") == 0) { - CodeGen *child_gen = create_child_codegen(parent, nullptr, OutTypeLib, nullptr, "c_nonshared", progress_node); - { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = path_from_libc(parent, "glibc" OS_SEP "csu" OS_SEP "elf-init.c"); - c_file->args.append("-std=gnu11"); - c_file->args.append("-fgnu89-inline"); - c_file->args.append("-g"); - c_file->args.append("-O2"); - c_file->args.append("-fmerge-all-constants"); - c_file->args.append("-fno-stack-protector"); - c_file->args.append("-fmath-errno"); - c_file->args.append("-fno-stack-protector"); - c_file->args.append("-I"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "csu")); - glibc_add_include_dirs(parent, c_file); - c_file->args.append("-DSTACK_PROTECTOR_LEVEL=0"); - c_file->args.append("-fPIC"); - c_file->args.append("-fno-stack-protector"); - c_file->args.append("-ftls-model=initial-exec"); - c_file->args.append("-D_LIBC_REENTRANT"); - c_file->args.append("-include"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-modules.h")); - c_file->args.append("-DMODULE_NAME=libc"); - c_file->args.append("-Wno-nonportable-include-path"); - c_file->args.append("-include"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-symbols.h")); - c_file->args.append("-DPIC"); - c_file->args.append("-DLIBC_NONSHARED=1"); - c_file->args.append("-DTOP_NAMESPACE=glibc"); - codegen_add_object(child_gen, buf_create_from_str( - build_libc_object(parent, "elf-init", c_file, progress_node))); - } - static const struct { - const char *name; - const char *path; - } deps[] = { - {"atexit", "glibc" OS_SEP "stdlib" OS_SEP "atexit.c"}, - {"at_quick_exit", "glibc" OS_SEP "stdlib" OS_SEP "at_quick_exit.c"}, - {"stat", "glibc" OS_SEP "io" OS_SEP "stat.c"}, - {"fstat", "glibc" OS_SEP "io" OS_SEP "fstat.c"}, - {"lstat", "glibc" OS_SEP "io" OS_SEP "lstat.c"}, - {"stat64", "glibc" OS_SEP "io" OS_SEP "stat64.c"}, - {"fstat64", "glibc" OS_SEP "io" OS_SEP "fstat64.c"}, - {"lstat64", "glibc" OS_SEP "io" OS_SEP "lstat64.c"}, - {"fstatat", "glibc" OS_SEP "io" OS_SEP "fstatat.c"}, - {"fstatat64", "glibc" OS_SEP "io" OS_SEP "fstatat64.c"}, - {"mknod", "glibc" OS_SEP "io" OS_SEP "mknod.c"}, - {"mknodat", "glibc" OS_SEP "io" OS_SEP "mknodat.c"}, - {"pthread_atfork", "glibc" OS_SEP "nptl" OS_SEP "pthread_atfork.c"}, - {"stack_chk_fail_local", "glibc" OS_SEP "debug" OS_SEP "stack_chk_fail_local.c"}, - }; - for (size_t i = 0; i < array_length(deps); i += 1) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = path_from_libc(parent, deps[i].path); - c_file->args.append("-std=gnu11"); - c_file->args.append("-fgnu89-inline"); - c_file->args.append("-g"); - c_file->args.append("-O2"); - c_file->args.append("-fmerge-all-constants"); - c_file->args.append("-fno-stack-protector"); - c_file->args.append("-fmath-errno"); - c_file->args.append("-ftls-model=initial-exec"); - c_file->args.append("-Wno-ignored-attributes"); - glibc_add_include_dirs(parent, c_file); - c_file->args.append("-D_LIBC_REENTRANT"); - c_file->args.append("-include"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-modules.h")); - c_file->args.append("-DMODULE_NAME=libc"); - c_file->args.append("-Wno-nonportable-include-path"); - c_file->args.append("-include"); - c_file->args.append(path_from_libc(parent, "glibc" OS_SEP "include" OS_SEP "libc-symbols.h")); - c_file->args.append("-DPIC"); - c_file->args.append("-DLIBC_NONSHARED=1"); - c_file->args.append("-DTOP_NAMESPACE=glibc"); - codegen_add_object(child_gen, buf_create_from_str( - build_libc_object(parent, deps[i].name, c_file, progress_node))); - } - codegen_build_and_link(child_gen); - return buf_ptr(&child_gen->bin_file_output_path); - } else { - zig_unreachable(); - } - } else if (parent->libc == nullptr && target_is_musl(parent->zig_target)) { - if (strcmp(file, "crti.o") == 0) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = musl_start_asm_path(parent, "crti.s"); - musl_add_cc_args(parent, c_file, false); - c_file->args.append("-Qunused-arguments"); - return build_libc_object(parent, "crti", c_file, progress_node); - } else if (strcmp(file, "crtn.o") == 0) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = musl_start_asm_path(parent, "crtn.s"); - c_file->args.append("-Qunused-arguments"); - musl_add_cc_args(parent, c_file, false); - return build_libc_object(parent, "crtn", c_file, progress_node); - } else if (strcmp(file, "crt1.o") == 0) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = path_from_libc(parent, "musl" OS_SEP "crt" OS_SEP "crt1.c"); - musl_add_cc_args(parent, c_file, false); - c_file->args.append("-fno-stack-protector"); - c_file->args.append("-DCRT"); - return build_libc_object(parent, "crt1", c_file, progress_node); - } else if (strcmp(file, "Scrt1.o") == 0) { - CFile *c_file = heap::c_allocator.create(); - c_file->source_path = path_from_libc(parent, "musl" OS_SEP "crt" OS_SEP "Scrt1.c"); - musl_add_cc_args(parent, c_file, false); - c_file->args.append("-fPIC"); - c_file->args.append("-fno-stack-protector"); - c_file->args.append("-DCRT"); - return build_libc_object(parent, "Scrt1", c_file, progress_node); - } else { - zig_unreachable(); - } - } else { - assert(parent->libc != nullptr); - Buf *out_buf = buf_alloc(); - os_path_join(buf_create_from_mem(parent->libc->crt_dir, parent->libc->crt_dir_len), - buf_create_from_str(file), out_buf); - return buf_ptr(out_buf); - } -} - -static Buf *build_a_raw(CodeGen *parent_gen, const char *aname, Buf *full_path, OutType child_out_type, - Stage2ProgressNode *progress_node) -{ - CodeGen *child_gen = create_child_codegen(parent_gen, full_path, child_out_type, parent_gen->libc, aname, - progress_node); - - // This is so that compiler_rt and libc.zig libraries know whether they - // will eventually be linked with libc. They make different decisions - // about what to export depending on whether libc is linked. - if (parent_gen->libc_link_lib != nullptr) { - LinkLib *new_link_lib = codegen_add_link_lib(child_gen, parent_gen->libc_link_lib->name); - new_link_lib->provided_explicitly = parent_gen->libc_link_lib->provided_explicitly; - } - - // Override the inherited build mode parameter - if (!parent_gen->is_test_build) { - switch (parent_gen->build_mode) { - case BuildModeDebug: - case BuildModeFastRelease: - case BuildModeSafeRelease: - child_gen->build_mode = BuildModeFastRelease; - break; - case BuildModeSmallRelease: - break; - } - } - - child_gen->function_sections = true; - child_gen->want_stack_check = WantStackCheckDisabled; - - codegen_build_and_link(child_gen); - return &child_gen->bin_file_output_path; -} - -static Buf *build_compiler_rt(CodeGen *parent_gen, OutType child_out_type, Stage2ProgressNode *progress_node) { - Buf *full_path = buf_alloc(); - os_path_join(parent_gen->zig_std_special_dir, buf_create_from_str("compiler_rt.zig"), full_path); - - return build_a_raw(parent_gen, "compiler_rt", full_path, child_out_type, progress_node); -} - -static Buf *build_c(CodeGen *parent_gen, OutType child_out_type, Stage2ProgressNode *progress_node) { - Buf *full_path = buf_alloc(); - os_path_join(parent_gen->zig_std_special_dir, buf_create_from_str("c.zig"), full_path); - - return build_a_raw(parent_gen, "c", full_path, child_out_type, progress_node); -} - -static const char *get_darwin_arch_string(const ZigTarget *t) { - switch (t->arch) { - case ZigLLVM_aarch64: - return "arm64"; - case ZigLLVM_thumb: - case ZigLLVM_arm: - return "arm"; - case ZigLLVM_ppc: - return "ppc"; - case ZigLLVM_ppc64: - return "ppc64"; - case ZigLLVM_ppc64le: - return "ppc64le"; - default: - return ZigLLVMGetArchTypeName(t->arch); - } -} - - -static const char *getLDMOption(const ZigTarget *t) { - switch (t->arch) { - case ZigLLVM_x86: - return "elf_i386"; - case ZigLLVM_aarch64: - return "aarch64linux"; - case ZigLLVM_aarch64_be: - return "aarch64_be_linux"; - case ZigLLVM_arm: - case ZigLLVM_thumb: - return "armelf_linux_eabi"; - case ZigLLVM_armeb: - case ZigLLVM_thumbeb: - return "armebelf_linux_eabi"; - case ZigLLVM_ppc: - return "elf32ppclinux"; - case ZigLLVM_ppc64: - return "elf64ppc"; - case ZigLLVM_ppc64le: - return "elf64lppc"; - case ZigLLVM_sparc: - case ZigLLVM_sparcel: - return "elf32_sparc"; - case ZigLLVM_sparcv9: - return "elf64_sparc"; - case ZigLLVM_mips: - return "elf32btsmip"; - case ZigLLVM_mipsel: - return "elf32ltsmip"; - case ZigLLVM_mips64: - return "elf64btsmip"; - case ZigLLVM_mips64el: - return "elf64ltsmip"; - case ZigLLVM_systemz: - return "elf64_s390"; - case ZigLLVM_x86_64: - if (t->abi == ZigLLVM_GNUX32) { - return "elf32_x86_64"; - } - // Any target elf will use the freebsd osabi if suffixed with "_fbsd". - if (t->os == OsFreeBSD) { - return "elf_x86_64_fbsd"; - } - return "elf_x86_64"; - case ZigLLVM_riscv32: - return "elf32lriscv"; - case ZigLLVM_riscv64: - return "elf64lriscv"; - default: - zig_unreachable(); - } -} - -static void add_rpath(LinkJob *lj, Buf *rpath) { - if (lj->rpath_table.maybe_get(rpath) != nullptr) - return; - - lj->args.append("-rpath"); - lj->args.append(buf_ptr(rpath)); - - lj->rpath_table.put(rpath, true); -} - -static void add_glibc_libs(LinkJob *lj) { - Error err; - ZigGLibCAbi *glibc_abi; - if ((err = glibc_load_metadata(&glibc_abi, lj->codegen->zig_lib_dir, true))) { - fprintf(stderr, "%s\n", err_str(err)); - exit(1); - } - - Buf *artifact_dir; - if ((err = glibc_build_dummies_and_maps(lj->codegen, glibc_abi, lj->codegen->zig_target, - &artifact_dir, true, lj->build_dep_prog_node))) - { - fprintf(stderr, "%s\n", err_str(err)); - exit(1); - } - - size_t lib_count = glibc_lib_count(); - for (size_t i = 0; i < lib_count; i += 1) { - const ZigGLibCLib *lib = glibc_lib_enum(i); - Buf *so_path = buf_sprintf("%s" OS_SEP "lib%s.so.%d.0.0", buf_ptr(artifact_dir), lib->name, lib->sover); - lj->args.append(buf_ptr(so_path)); - } -} - -static void construct_linker_job_elf(LinkJob *lj) { - CodeGen *g = lj->codegen; - - lj->args.append("-error-limit=0"); - - if (g->out_type == OutTypeExe) { - lj->args.append("-z"); - size_t stack_size = (g->stack_size_override == 0) ? 16777216 : g->stack_size_override; - lj->args.append(buf_ptr(buf_sprintf("stack-size=%" ZIG_PRI_usize, stack_size))); - } - - if (g->linker_script) { - lj->args.append("-T"); - lj->args.append(g->linker_script); - } - - switch (g->linker_gc_sections) { - case OptionalBoolNull: - if (g->out_type != OutTypeObj) { - lj->args.append("--gc-sections"); - } - break; - case OptionalBoolTrue: - lj->args.append("--gc-sections"); - break; - case OptionalBoolFalse: - break; - } - - if (g->link_eh_frame_hdr) { - lj->args.append("--eh-frame-hdr"); - } - - if (g->linker_rdynamic) { - lj->args.append("--export-dynamic"); - } - - if (g->linker_optimization != nullptr) { - lj->args.append(buf_ptr(g->linker_optimization)); - } - - if (g->linker_z_nodelete) { - lj->args.append("-z"); - lj->args.append("nodelete"); - } - if (g->linker_z_defs) { - lj->args.append("-z"); - lj->args.append("defs"); - } - - lj->args.append("-m"); - lj->args.append(getLDMOption(g->zig_target)); - - bool is_lib = g->out_type == OutTypeLib; - bool is_dyn_lib = g->is_dynamic && is_lib; - if (!g->have_dynamic_link) { - if (g->zig_target->arch == ZigLLVM_arm || g->zig_target->arch == ZigLLVM_armeb || - g->zig_target->arch == ZigLLVM_thumb || g->zig_target->arch == ZigLLVM_thumbeb) - { - lj->args.append("-Bstatic"); - } else { - lj->args.append("-static"); - } - } else if (is_dyn_lib) { - lj->args.append("-shared"); - } - - if (target_requires_pie(g->zig_target) && g->out_type == OutTypeExe) { - lj->args.append("-pie"); - } - - assert(buf_len(&g->bin_file_output_path) != 0); - lj->args.append("-o"); - lj->args.append(buf_ptr(&g->bin_file_output_path)); - - if (lj->link_in_crt) { - const char *crt1o; - if (g->zig_target->os == OsNetBSD) { - crt1o = "crt0.o"; - } else if (target_is_android(g->zig_target)) { - if (g->have_dynamic_link) { - crt1o = "crtbegin_dynamic.o"; - } else { - crt1o = "crtbegin_static.o"; - } - } else if (!g->have_dynamic_link) { - crt1o = "crt1.o"; - } else { - crt1o = "Scrt1.o"; - } - lj->args.append(get_libc_crt_file(g, crt1o, lj->build_dep_prog_node)); - if (target_libc_needs_crti_crtn(g->zig_target)) { - lj->args.append(get_libc_crt_file(g, "crti.o", lj->build_dep_prog_node)); - } - } - - for (size_t i = 0; i < g->rpath_list.length; i += 1) { - Buf *rpath = g->rpath_list.at(i); - add_rpath(lj, rpath); - } - if (g->each_lib_rpath) { - for (size_t i = 0; i < g->lib_dirs.length; i += 1) { - const char *lib_dir = g->lib_dirs.at(i); - for (size_t i = 0; i < g->link_libs_list.length; i += 1) { - LinkLib *link_lib = g->link_libs_list.at(i); - if (buf_eql_str(link_lib->name, "c")) { - continue; - } - bool does_exist; - Buf *test_path = buf_sprintf("%s/lib%s.so", lib_dir, buf_ptr(link_lib->name)); - if (os_file_exists(test_path, &does_exist) != ErrorNone) { - zig_panic("link: unable to check if file exists: %s", buf_ptr(test_path)); - } - if (does_exist) { - add_rpath(lj, buf_create_from_str(lib_dir)); - break; - } - } - } - } - - for (size_t i = 0; i < g->lib_dirs.length; i += 1) { - const char *lib_dir = g->lib_dirs.at(i); - lj->args.append("-L"); - lj->args.append(lib_dir); - } - - if (g->libc_link_lib != nullptr) { - if (g->libc != nullptr) { - lj->args.append("-L"); - lj->args.append(buf_ptr(buf_create_from_mem(g->libc->crt_dir, g->libc->crt_dir_len))); - } - - if (g->have_dynamic_link && (is_dyn_lib || g->out_type == OutTypeExe)) { - assert(g->zig_target->dynamic_linker != nullptr); - lj->args.append("-dynamic-linker"); - lj->args.append(g->zig_target->dynamic_linker); - } - } - - if (is_dyn_lib) { - Buf *soname = (g->override_soname == nullptr) ? - buf_sprintf("lib%s.so.%" ZIG_PRI_usize, buf_ptr(g->root_out_name), g->version_major) : - g->override_soname; - lj->args.append("-soname"); - lj->args.append(buf_ptr(soname)); - - if (g->version_script_path != nullptr) { - lj->args.append("-version-script"); - lj->args.append(buf_ptr(g->version_script_path)); - } - } - - // .o files - for (size_t i = 0; i < g->link_objects.length; i += 1) { - lj->args.append((const char *)buf_ptr(g->link_objects.at(i))); - } - - if (!g->is_dummy_so && (g->out_type == OutTypeExe || is_dyn_lib)) { - if (g->libc_link_lib == nullptr) { - Buf *libc_a_path = build_c(g, OutTypeLib, lj->build_dep_prog_node); - lj->args.append(buf_ptr(libc_a_path)); - } - - Buf *compiler_rt_o_path = build_compiler_rt(g, OutTypeLib, lj->build_dep_prog_node); - lj->args.append(buf_ptr(compiler_rt_o_path)); - } - - // libraries - for (size_t i = 0; i < g->link_libs_list.length; i += 1) { - LinkLib *link_lib = g->link_libs_list.at(i); - if (buf_eql_str(link_lib->name, "c")) { - // libc is linked specially - continue; - } - if (target_is_libcpp_lib_name(g->zig_target, buf_ptr(link_lib->name))) { - // libc++ is linked specially - continue; - } - if (g->libc == nullptr && target_is_libc_lib_name(g->zig_target, buf_ptr(link_lib->name))) { - // these libraries are always linked below when targeting glibc - continue; - } - Buf *arg; - if (buf_starts_with_str(link_lib->name, "/") || buf_ends_with_str(link_lib->name, ".a") || - buf_ends_with_str(link_lib->name, ".so")) - { - arg = link_lib->name; - } else { - arg = buf_sprintf("-l%s", buf_ptr(link_lib->name)); - } - lj->args.append(buf_ptr(arg)); - } - - // libc++ dep - if (g->libcpp_link_lib != nullptr && g->out_type != OutTypeObj) { - lj->args.append(build_libcxxabi(g, lj->build_dep_prog_node)); - lj->args.append(build_libcxx(g, lj->build_dep_prog_node)); - } - - // libc dep - if (g->libc_link_lib != nullptr && g->out_type != OutTypeObj) { - if (g->libc != nullptr) { - if (!g->have_dynamic_link) { - lj->args.append("--start-group"); - lj->args.append("-lc"); - lj->args.append("-lm"); - lj->args.append("--end-group"); - } else { - lj->args.append("-lc"); - lj->args.append("-lm"); - } - - if (g->zig_target->os == OsFreeBSD || - g->zig_target->os == OsNetBSD) - { - lj->args.append("-lpthread"); - } - } else if (target_is_glibc(g->zig_target)) { - lj->args.append(build_libunwind(g, lj->build_dep_prog_node)); - add_glibc_libs(lj); - lj->args.append(get_libc_crt_file(g, "libc_nonshared.a", lj->build_dep_prog_node)); - } else if (target_is_musl(g->zig_target)) { - lj->args.append(build_libunwind(g, lj->build_dep_prog_node)); - lj->args.append(build_musl(g, lj->build_dep_prog_node)); - } else if (g->libcpp_link_lib != nullptr) { - lj->args.append(build_libunwind(g, lj->build_dep_prog_node)); - } else { - zig_unreachable(); - } - } - - // crt end - if (lj->link_in_crt) { - if (target_is_android(g->zig_target)) { - lj->args.append(get_libc_crt_file(g, "crtend_android.o", lj->build_dep_prog_node)); - } else if (target_libc_needs_crti_crtn(g->zig_target)) { - lj->args.append(get_libc_crt_file(g, "crtn.o", lj->build_dep_prog_node)); - } - } - - switch (g->linker_allow_shlib_undefined) { - case OptionalBoolNull: - if (!g->zig_target->is_native_os) { - lj->args.append("--allow-shlib-undefined"); - } - break; - case OptionalBoolFalse: - break; - case OptionalBoolTrue: - lj->args.append("--allow-shlib-undefined"); - break; - } - switch (g->linker_bind_global_refs_locally) { - case OptionalBoolNull: - case OptionalBoolFalse: - break; - case OptionalBoolTrue: - lj->args.append("-Bsymbolic"); - break; - } -} - -static void construct_linker_job_wasm(LinkJob *lj) { - CodeGen *g = lj->codegen; - - lj->args.append("-error-limit=0"); - // Increase the default stack size to a more reasonable value of 1MB instead of - // the default of 1 Wasm page being 64KB, unless overriden by the user. - size_t stack_size = (g->stack_size_override == 0) ? 1048576 : g->stack_size_override; - lj->args.append("-z"); - lj->args.append(buf_ptr(buf_sprintf("stack-size=%" ZIG_PRI_usize, stack_size))); - - // put stack before globals so that stack overflow results in segfault immediately before corrupting globals - // see https://github.com/ziglang/zig/issues/4496 - lj->args.append("--stack-first"); - - if (g->out_type != OutTypeExe) { - lj->args.append("--no-entry"); // So lld doesn't look for _start. - - // If there are any C source files we cannot rely on individual exports. - if (g->c_source_files.length != 0) { - lj->args.append("--export-all"); - } else { - auto export_it = g->exported_symbol_names.entry_iterator(); - decltype(g->exported_symbol_names)::Entry *curr_entry = nullptr; - while ((curr_entry = export_it.next()) != nullptr) { - Buf *arg = buf_sprintf("--export=%s", buf_ptr(curr_entry->key)); - lj->args.append(buf_ptr(arg)); - } - } - } - lj->args.append("--allow-undefined"); - lj->args.append("-o"); - lj->args.append(buf_ptr(&g->bin_file_output_path)); - - // .o files - for (size_t i = 0; i < g->link_objects.length; i += 1) { - lj->args.append((const char *)buf_ptr(g->link_objects.at(i))); - } - - if (g->out_type != OutTypeObj) { - Buf *libc_o_path = build_c(g, OutTypeObj, lj->build_dep_prog_node); - lj->args.append(buf_ptr(libc_o_path)); - - Buf *compiler_rt_o_path = build_compiler_rt(g, OutTypeObj, lj->build_dep_prog_node); - lj->args.append(buf_ptr(compiler_rt_o_path)); - } -} - -static void coff_append_machine_arg(CodeGen *g, ZigList *list) { - if (g->zig_target->arch == ZigLLVM_x86) { - list->append("-MACHINE:X86"); - } else if (g->zig_target->arch == ZigLLVM_x86_64) { - list->append("-MACHINE:X64"); - } else if (target_is_arm(g->zig_target)) { - if (target_arch_pointer_bit_width(g->zig_target->arch) == 32) { - list->append("-MACHINE:ARM"); - } else { - list->append("-MACHINE:ARM64"); - } - } -} - -static void link_diag_callback(void *context, const char *ptr, size_t len) { - Buf *diag = reinterpret_cast(context); - buf_append_mem(diag, ptr, len); -} - -static bool zig_lld_link(ZigLLVM_ObjectFormatType oformat, const char **args, size_t arg_count, - Buf *diag) -{ - Buf *stdout_diag = buf_alloc(); - buf_resize(diag, 0); - bool result = ZigLLDLink(oformat, args, arg_count, link_diag_callback, stdout_diag, diag); - buf_destroy(stdout_diag); - return result; -} - -static void add_uefi_link_args(LinkJob *lj) { - lj->args.append("-BASE:0"); - lj->args.append("-ENTRY:EfiMain"); - lj->args.append("-OPT:REF"); - lj->args.append("-SAFESEH:NO"); - lj->args.append("-MERGE:.rdata=.data"); - lj->args.append("-ALIGN:32"); - lj->args.append("-NODEFAULTLIB"); - lj->args.append("-SECTION:.xdata,D"); -} - -static void add_msvc_link_args(LinkJob *lj, bool is_library) { - CodeGen *g = lj->codegen; - - bool is_dynamic = g->is_dynamic; - const char *lib_str = is_dynamic ? "" : "lib"; - const char *d_str = (g->build_mode == BuildModeDebug) ? "d" : ""; - - if (!is_dynamic) { - Buf *cmt_lib_name = buf_sprintf("libcmt%s.lib", d_str); - lj->args.append(buf_ptr(cmt_lib_name)); - } else { - Buf *msvcrt_lib_name = buf_sprintf("msvcrt%s.lib", d_str); - lj->args.append(buf_ptr(msvcrt_lib_name)); - } - - Buf *vcruntime_lib_name = buf_sprintf("%svcruntime%s.lib", lib_str, d_str); - lj->args.append(buf_ptr(vcruntime_lib_name)); - - Buf *crt_lib_name = buf_sprintf("%sucrt%s.lib", lib_str, d_str); - lj->args.append(buf_ptr(crt_lib_name)); - - //Visual C++ 2015 Conformance Changes - //https://msdn.microsoft.com/en-us/library/bb531344.aspx - lj->args.append("legacy_stdio_definitions.lib"); - - // msvcrt depends on kernel32 and ntdll - lj->args.append("kernel32.lib"); - lj->args.append("ntdll.lib"); -} - -static void print_zig_cc_cmd(ZigList *args) { - for (size_t arg_i = 0; arg_i < args->length; arg_i += 1) { - const char *space_str = (arg_i == 0) ? "" : " "; - fprintf(stderr, "%s%s", space_str, args->at(arg_i)); - } - fprintf(stderr, "\n"); -} - -static const char *get_def_lib(CodeGen *parent, const char *name, Buf *def_in_file) { - Error err; - - Buf *self_exe_path = buf_alloc(); - if ((err = os_self_exe_path(self_exe_path))) { - fprintf(stderr, "Unable to get self exe path: %s\n", err_str(err)); - exit(1); - } - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) { - fprintf(stderr, "Unable to get compiler id: %s\n", err_str(err)); - exit(1); - } - - Buf *cache_dir = get_global_cache_dir(); - Buf *o_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR, buf_ptr(cache_dir)); - Buf *manifest_dir = buf_sprintf("%s" OS_SEP CACHE_HASH_SUBDIR, buf_ptr(cache_dir)); - - Buf *def_include_dir = buf_sprintf("%s" OS_SEP "libc" OS_SEP "mingw" OS_SEP "def-include", - buf_ptr(parent->zig_lib_dir)); - - CacheHash *cache_hash = heap::c_allocator.create(); - cache_init(cache_hash, manifest_dir); - - cache_buf(cache_hash, compiler_id); - cache_file(cache_hash, def_in_file); - cache_buf(cache_hash, def_include_dir); - cache_int(cache_hash, parent->zig_target->arch); - - Buf digest = BUF_INIT; - buf_resize(&digest, 0); - if ((err = cache_hit(cache_hash, &digest))) { - if (err != ErrorInvalidFormat) { - if (err == ErrorCacheUnavailable) { - // already printed error - } else { - fprintf(stderr, "unable to check cache when processing .def.in file: %s\n", err_str(err)); - } - exit(1); - } - } - - Buf *artifact_dir; - Buf *lib_final_path; - Buf *final_lib_basename = buf_sprintf("%s.lib", name); - - bool is_cache_miss = (buf_len(&digest) == 0); - if (is_cache_miss) { - if ((err = cache_final(cache_hash, &digest))) { - fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); - exit(1); - } - artifact_dir = buf_alloc(); - os_path_join(o_dir, &digest, artifact_dir); - if ((err = os_make_path(artifact_dir))) { - fprintf(stderr, "Unable to create output directory '%s': %s", - buf_ptr(artifact_dir), err_str(err)); - exit(1); - } - Buf *final_def_basename = buf_sprintf("%s.def", name); - Buf *def_final_path = buf_alloc(); - os_path_join(artifact_dir, final_def_basename, def_final_path); - - ZigList args = {}; - args.append(buf_ptr(self_exe_path)); - args.append("clang"); - args.append("-x"); - args.append("c"); - args.append(buf_ptr(def_in_file)); - args.append("-Wp,-w"); - args.append("-undef"); - args.append("-P"); - args.append("-I"); - args.append(buf_ptr(def_include_dir)); - if (target_is_arm(parent->zig_target)) { - if (target_arch_pointer_bit_width(parent->zig_target->arch) == 32) { - args.append("-DDEF_ARM32"); - } else { - args.append("-DDEF_ARM64"); - } - } else if (parent->zig_target->arch == ZigLLVM_x86) { - args.append("-DDEF_I386"); - } else if (parent->zig_target->arch == ZigLLVM_x86_64) { - args.append("-DDEF_X64"); - } else { - zig_unreachable(); - } - args.append("-E"); - args.append("-o"); - args.append(buf_ptr(def_final_path)); - - if (parent->verbose_cc) { - print_zig_cc_cmd(&args); - } - Termination term; - os_spawn_process(args, &term); - if (term.how != TerminationIdClean || term.code != 0) { - fprintf(stderr, "\nThe following command failed:\n"); - print_zig_cc_cmd(&args); - exit(1); - } - - lib_final_path = buf_alloc(); - os_path_join(artifact_dir, final_lib_basename, lib_final_path); - - if (ZigLLVMWriteImportLibrary(buf_ptr(def_final_path), - parent->zig_target->arch, - buf_ptr(lib_final_path), - /* kill_at */ true)) - { - zig_panic("link: could not emit %s", buf_ptr(lib_final_path)); - } - } else { - // cache hit - artifact_dir = buf_alloc(); - os_path_join(o_dir, &digest, artifact_dir); - lib_final_path = buf_alloc(); - os_path_join(artifact_dir, final_lib_basename, lib_final_path); - } - parent->caches_to_release.append(cache_hash); - - return buf_ptr(lib_final_path); -} - -static bool is_linking_system_lib(CodeGen *g, const char *name) { - for (size_t lib_i = 0; lib_i < g->link_libs_list.length; lib_i += 1) { - LinkLib *link_lib = g->link_libs_list.at(lib_i); - if (buf_eql_str(link_lib->name, name)) { - return true; - } - } - return false; -} - -static Error find_mingw_lib_def(LinkJob *lj, const char *name, Buf *out_path) { - CodeGen *g = lj->codegen; - Buf override_path = BUF_INIT; - Error err; - - char const *lib_path = nullptr; - if (g->zig_target->arch == ZigLLVM_x86) { - lib_path = "lib32"; - } else if (g->zig_target->arch == ZigLLVM_x86_64) { - lib_path = "lib64"; - } else if (target_is_arm(g->zig_target)) { - const bool is_32 = target_arch_pointer_bit_width(g->zig_target->arch) == 32; - lib_path = is_32 ? "libarm32" : "libarm64"; - } else { - zig_unreachable(); - } - - // Try the archtecture-specific path first - buf_resize(&override_path, 0); - buf_appendf(&override_path, "%s" OS_SEP "libc" OS_SEP "mingw" OS_SEP "%s" OS_SEP "%s.def", buf_ptr(g->zig_lib_dir), lib_path, name); - - bool does_exist; - if ((err = os_file_exists(&override_path, &does_exist)) != ErrorNone) { - return err; - } - - if (!does_exist) { - // Try the generic version - buf_resize(&override_path, 0); - buf_appendf(&override_path, "%s" OS_SEP "libc" OS_SEP "mingw" OS_SEP "lib-common" OS_SEP "%s.def", buf_ptr(g->zig_lib_dir), name); - - if ((err = os_file_exists(&override_path, &does_exist)) != ErrorNone) { - return err; - } - } - - if (!does_exist) { - // Try the generic version and preprocess it - buf_resize(&override_path, 0); - buf_appendf(&override_path, "%s" OS_SEP "libc" OS_SEP "mingw" OS_SEP "lib-common" OS_SEP "%s.def.in", buf_ptr(g->zig_lib_dir), name); - - if ((err = os_file_exists(&override_path, &does_exist)) != ErrorNone) { - return err; - } - } - - if (!does_exist) { - return ErrorFileNotFound; - } - - buf_init_from_buf(out_path, &override_path); - return ErrorNone; -} - -static void add_mingw_link_args(LinkJob *lj, bool is_library) { - CodeGen *g = lj->codegen; - - lj->args.append("-lldmingw"); - - bool is_dll = g->out_type == OutTypeLib && g->is_dynamic; - - if (g->zig_target->arch == ZigLLVM_x86) { - lj->args.append("-ALTERNATENAME:__image_base__=___ImageBase"); - } else { - lj->args.append("-ALTERNATENAME:__image_base__=__ImageBase"); - } - - if (is_dll) { - lj->args.append(get_libc_crt_file(g, "dllcrt2.o", lj->build_dep_prog_node)); - } else { - lj->args.append(get_libc_crt_file(g, "crt2.o", lj->build_dep_prog_node)); - } - - lj->args.append(get_libc_crt_file(g, "mingw32.lib", lj->build_dep_prog_node)); - lj->args.append(get_libc_crt_file(g, "mingwex.lib", lj->build_dep_prog_node)); - lj->args.append(get_libc_crt_file(g, "msvcrt-os.lib", lj->build_dep_prog_node)); - - for (size_t def_i = 0; def_i < array_length(mingw_def_list); def_i += 1) { - const char *name = mingw_def_list[def_i].name; - const bool always_link = mingw_def_list[def_i].always_link; - - if (always_link || is_linking_system_lib(g, name)) { - Buf lib_path = BUF_INIT; - Error err = find_mingw_lib_def(lj, name, &lib_path); - - if (err == ErrorFileNotFound) { - zig_panic("link: could not find .def file to build %s\n", name); - } else if (err != ErrorNone) { - zig_panic("link: unable to check if .def file for %s exists: %s", - name, err_str(err)); - } - - lj->args.append(get_def_lib(g, name, &lib_path)); - } - } -} - -static void add_win_link_args(LinkJob *lj, bool is_library, bool *have_windows_dll_import_libs) { - if (lj->link_in_crt) { - if (target_abi_is_gnu(lj->codegen->zig_target->abi)) { - *have_windows_dll_import_libs = true; - add_mingw_link_args(lj, is_library); - } else { - add_msvc_link_args(lj, is_library); - } - } else { - lj->args.append("-NODEFAULTLIB"); - if (!is_library) { - if (lj->codegen->have_winmain) { - lj->args.append("-ENTRY:WinMain"); - } else if (lj->codegen->have_wwinmain) { - lj->args.append("-ENTRY:wWinMain"); - } else if (lj->codegen->have_wwinmain_crt_startup) { - lj->args.append("-ENTRY:wWinMainCRTStartup"); - } else { - lj->args.append("-ENTRY:WinMainCRTStartup"); - } - } - } -} - -static bool is_mingw_link_lib(Buf *name) { - for (size_t def_i = 0; def_i < array_length(mingw_def_list); def_i += 1) { - if (buf_eql_str_ignore_case(name, mingw_def_list[def_i].name)) { - return true; - } - } - return false; -} -static void construct_linker_job_coff(LinkJob *lj) { - Error err; - CodeGen *g = lj->codegen; - - lj->args.append("-ERRORLIMIT:0"); - - lj->args.append("-NOLOGO"); - - if (!g->strip_debug_symbols) { - lj->args.append("-DEBUG"); - } - - if (g->out_type == OutTypeExe) { - // TODO compile time stack upper bound detection - size_t stack_size = (g->stack_size_override == 0) ? 16777216 : g->stack_size_override; - lj->args.append(buf_ptr(buf_sprintf("-STACK:%" ZIG_PRI_usize, stack_size))); - } - - coff_append_machine_arg(g, &lj->args); - - bool is_library = g->out_type == OutTypeLib; - if (is_library && g->is_dynamic) { - lj->args.append("-DLL"); - } - - lj->args.append(buf_ptr(buf_sprintf("-OUT:%s", buf_ptr(&g->bin_file_output_path)))); - - if (g->libc_link_lib != nullptr && g->libc != nullptr) { - Buf *buff0 = buf_create_from_str("-LIBPATH:"); - buf_append_mem(buff0, g->libc->crt_dir, g->libc->crt_dir_len); - lj->args.append(buf_ptr(buff0)); - - if (target_abi_is_gnu(g->zig_target->abi)) { - Buf *buff1 = buf_create_from_str("-LIBPATH:"); - buf_append_mem(buff1, g->libc->sys_include_dir, g->libc->sys_include_dir_len); - lj->args.append(buf_ptr(buff1)); - - Buf *buff2 = buf_create_from_str("-LIBPATH:"); - buf_append_mem(buff2, g->libc->include_dir, g->libc->include_dir_len); - lj->args.append(buf_ptr(buff2)); - } else { - Buf *buff1 = buf_create_from_str("-LIBPATH:"); - buf_append_mem(buff1, g->libc->msvc_lib_dir, g->libc->msvc_lib_dir_len); - lj->args.append(buf_ptr(buff1)); - - Buf *buff2 = buf_create_from_str("-LIBPATH:"); - buf_append_mem(buff2, g->libc->kernel32_lib_dir, g->libc->kernel32_lib_dir_len); - lj->args.append(buf_ptr(buff2)); - } - } - - for (size_t i = 0; i < g->lib_dirs.length; i += 1) { - const char *lib_dir = g->lib_dirs.at(i); - lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", lib_dir))); - } - - for (size_t i = 0; i < g->link_objects.length; i += 1) { - lj->args.append((const char *)buf_ptr(g->link_objects.at(i))); - } - - bool have_windows_dll_import_libs = false; - switch (detect_subsystem(g)) { - case TargetSubsystemAuto: - if (g->zig_target->os == OsUefi) { - add_uefi_link_args(lj); - } else { - add_win_link_args(lj, is_library, &have_windows_dll_import_libs); - } - break; - case TargetSubsystemConsole: - lj->args.append("-SUBSYSTEM:console"); - add_win_link_args(lj, is_library, &have_windows_dll_import_libs); - break; - case TargetSubsystemEfiApplication: - lj->args.append("-SUBSYSTEM:efi_application"); - add_uefi_link_args(lj); - break; - case TargetSubsystemEfiBootServiceDriver: - lj->args.append("-SUBSYSTEM:efi_boot_service_driver"); - add_uefi_link_args(lj); - break; - case TargetSubsystemEfiRom: - lj->args.append("-SUBSYSTEM:efi_rom"); - add_uefi_link_args(lj); - break; - case TargetSubsystemEfiRuntimeDriver: - lj->args.append("-SUBSYSTEM:efi_runtime_driver"); - add_uefi_link_args(lj); - break; - case TargetSubsystemNative: - lj->args.append("-SUBSYSTEM:native"); - add_win_link_args(lj, is_library, &have_windows_dll_import_libs); - break; - case TargetSubsystemPosix: - lj->args.append("-SUBSYSTEM:posix"); - add_win_link_args(lj, is_library, &have_windows_dll_import_libs); - break; - case TargetSubsystemWindows: - lj->args.append("-SUBSYSTEM:windows"); - add_win_link_args(lj, is_library, &have_windows_dll_import_libs); - break; - } - - // libc++ dep - if (g->libcpp_link_lib != nullptr && g->out_type != OutTypeObj) { - lj->args.append(build_libcxxabi(g, lj->build_dep_prog_node)); - lj->args.append(build_libcxx(g, lj->build_dep_prog_node)); - lj->args.append(build_libunwind(g, lj->build_dep_prog_node)); - } - - if (g->out_type == OutTypeExe || (g->out_type == OutTypeLib && g->is_dynamic)) { - if (g->libc_link_lib == nullptr && !g->is_dummy_so) { - Buf *libc_a_path = build_c(g, OutTypeLib, lj->build_dep_prog_node); - lj->args.append(buf_ptr(libc_a_path)); - } - - // msvc compiler_rt is missing some stuff, so we still build it and rely on weak linkage - Buf *compiler_rt_o_path = build_compiler_rt(g, OutTypeLib, lj->build_dep_prog_node); - lj->args.append(buf_ptr(compiler_rt_o_path)); - } - - for (size_t lib_i = 0; lib_i < g->link_libs_list.length; lib_i += 1) { - LinkLib *link_lib = g->link_libs_list.at(lib_i); - if (buf_eql_str(link_lib->name, "c")) { - continue; - } - if (target_is_libcpp_lib_name(g->zig_target, buf_ptr(link_lib->name))) { - // libc++ is linked specially - continue; - } - if (g->libc == nullptr && target_is_libc_lib_name(g->zig_target, buf_ptr(link_lib->name))) { - // these libraries are always linked below when targeting glibc - continue; - } - bool is_sys_lib = is_mingw_link_lib(link_lib->name); - if (have_windows_dll_import_libs && is_sys_lib) { - continue; - } - // If we're linking in the CRT or the libs are provided explictly we don't want to generate def/libs - if ((lj->link_in_crt && is_sys_lib) || link_lib->provided_explicitly) { - if (target_abi_is_gnu(lj->codegen->zig_target->abi)) { - if (buf_eql_str(link_lib->name, "uuid")) { - // mingw-w64 provides this lib - lj->args.append(get_libc_crt_file(g, "uuid.lib", lj->build_dep_prog_node)); - } else { - Buf* lib_name = buf_sprintf("lib%s.a", buf_ptr(link_lib->name)); - lj->args.append(buf_ptr(lib_name)); - } - } else { - Buf* lib_name = buf_sprintf("%s.lib", buf_ptr(link_lib->name)); - lj->args.append(buf_ptr(lib_name)); - } - continue; - } - - // This library may be a system one and we may have a suitable .lib file - - // Normalize the library name to lower case, the FS may be - // case-sensitive - char *name = strdup(buf_ptr(link_lib->name)); - assert(name != nullptr); - for (char *ch = name; *ch; ++ch) *ch = tolower(*ch); - - Buf lib_path = BUF_INIT; - err = find_mingw_lib_def(lj, name, &lib_path); - - if (err == ErrorFileNotFound) { - zig_panic("link: could not find .def file to build %s\n", name); - } else if (err != ErrorNone) { - zig_panic("link: unable to check if .def file for %s exists: %s", - name, err_str(err)); - } - - lj->args.append(get_def_lib(g, name, &lib_path)); - - mem::os::free(name); - } -} - -static void construct_linker_job_macho(LinkJob *lj) { - CodeGen *g = lj->codegen; - - lj->args.append("-error-limit"); - lj->args.append("0"); - lj->args.append("-demangle"); - - switch (g->linker_gc_sections) { - case OptionalBoolNull: - // TODO why do we not follow the same logic of elf here? - break; - case OptionalBoolTrue: - lj->args.append("--gc-sections"); - break; - case OptionalBoolFalse: - break; - } - - if (g->linker_rdynamic) { - lj->args.append("-export_dynamic"); - } - - if (g->linker_optimization != nullptr) { - lj->args.append(buf_ptr(g->linker_optimization)); - } - - if (g->linker_z_nodelete) { - lj->args.append("-z"); - lj->args.append("nodelete"); - } - if (g->linker_z_defs) { - lj->args.append("-z"); - lj->args.append("defs"); - } - - bool is_lib = g->out_type == OutTypeLib; - bool is_dyn_lib = g->is_dynamic && is_lib; - if (is_lib && !g->is_dynamic) { - lj->args.append("-static"); - } else { - lj->args.append("-dynamic"); - } - - if (is_dyn_lib) { - lj->args.append("-dylib"); - - Buf *compat_vers = buf_sprintf("%" ZIG_PRI_usize ".0.0", g->version_major); - lj->args.append("-compatibility_version"); - lj->args.append(buf_ptr(compat_vers)); - - Buf *cur_vers = buf_sprintf("%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize, - g->version_major, g->version_minor, g->version_patch); - lj->args.append("-current_version"); - lj->args.append(buf_ptr(cur_vers)); - - // TODO getting an error when running an executable when doing this rpath thing - //Buf *dylib_install_name = buf_sprintf("@rpath/lib%s.%" ZIG_PRI_usize ".dylib", - // buf_ptr(g->root_out_name), g->version_major); - //lj->args.append("-install_name"); - //lj->args.append(buf_ptr(dylib_install_name)); - - assert(buf_len(&g->bin_file_output_path) != 0); - } - - lj->args.append("-arch"); - lj->args.append(get_darwin_arch_string(g->zig_target)); - - if (g->zig_target->glibc_or_darwin_version != nullptr) { - if (g->zig_target->os == OsMacOSX) { - lj->args.append("-macosx_version_min"); - } else if (g->zig_target->os == OsIOS) { - if (g->zig_target->arch == ZigLLVM_x86 || g->zig_target->arch == ZigLLVM_x86_64) { - lj->args.append("-ios_simulator_version_min"); - } else { - lj->args.append("-iphoneos_version_min"); - } - } - - Buf *version_string = buf_sprintf("%d.%d.%d", - g->zig_target->glibc_or_darwin_version->major, - g->zig_target->glibc_or_darwin_version->minor, - g->zig_target->glibc_or_darwin_version->patch); - lj->args.append(buf_ptr(version_string)); - - lj->args.append("-sdk_version"); - lj->args.append(buf_ptr(version_string)); - } else if (stage2_is_zig0 && g->zig_target->os == OsMacOSX) { - // running `zig0`; `-pie` requires versions >= 10.5; select 10.13 - lj->args.append("-macosx_version_min"); - lj->args.append("10.13"); - lj->args.append("-sdk_version"); - lj->args.append("10.13"); - } - - if (g->out_type == OutTypeExe) { - lj->args.append("-pie"); - } - - lj->args.append("-o"); - lj->args.append(buf_ptr(&g->bin_file_output_path)); - - for (size_t i = 0; i < g->rpath_list.length; i += 1) { - Buf *rpath = g->rpath_list.at(i); - add_rpath(lj, rpath); - } - if (is_dyn_lib) { - add_rpath(lj, &g->bin_file_output_path); - } - - if (is_dyn_lib) { - if (g->system_linker_hack) { - lj->args.append("-headerpad_max_install_names"); - } - } - - for (size_t i = 0; i < g->lib_dirs.length; i += 1) { - const char *lib_dir = g->lib_dirs.at(i); - lj->args.append("-L"); - lj->args.append(lib_dir); - } - - // .o files - for (size_t i = 0; i < g->link_objects.length; i += 1) { - lj->args.append((const char *)buf_ptr(g->link_objects.at(i))); - } - - // compiler_rt on darwin is missing some stuff, so we still build it and rely on LinkOnce - if (g->out_type == OutTypeExe || is_dyn_lib) { - Buf *compiler_rt_o_path = build_compiler_rt(g, OutTypeLib, lj->build_dep_prog_node); - lj->args.append(buf_ptr(compiler_rt_o_path)); - } - - // libraries - for (size_t lib_i = 0; lib_i < g->link_libs_list.length; lib_i += 1) { - LinkLib *link_lib = g->link_libs_list.at(lib_i); - if (buf_eql_str(link_lib->name, "c")) { - // libc is linked specially - continue; - } - if (target_is_libcpp_lib_name(g->zig_target, buf_ptr(link_lib->name))) { - // libc++ is linked specially - continue; - } - if (g->zig_target->is_native_os && target_is_libc_lib_name(g->zig_target, buf_ptr(link_lib->name))) { - // libSystem is linked specially - continue; - } - - Buf *arg; - if (buf_starts_with_str(link_lib->name, "/") || buf_ends_with_str(link_lib->name, ".a") || - buf_ends_with_str(link_lib->name, ".dylib")) - { - arg = link_lib->name; - } else { - arg = buf_sprintf("-l%s", buf_ptr(link_lib->name)); - } - lj->args.append(buf_ptr(arg)); - } - - // libc++ dep - if (g->libcpp_link_lib != nullptr && g->out_type != OutTypeObj) { - lj->args.append(build_libcxxabi(g, lj->build_dep_prog_node)); - lj->args.append(build_libcxx(g, lj->build_dep_prog_node)); - } - - // libc dep - if (g->zig_target->is_native_os || stage2_is_zig0) { - // on Darwin, libSystem has libc in it, but also you have to use it - // to make syscalls because the syscall numbers are not documented - // and change between versions. - // so we always link against libSystem - lj->args.append("-lSystem"); - } - - for (size_t i = 0; i < g->framework_dirs.length; i += 1) { - const char *framework_dir = g->framework_dirs.at(i); - lj->args.append("-F"); - lj->args.append(framework_dir); - } - - for (size_t i = 0; i < g->darwin_frameworks.length; i += 1) { - lj->args.append("-framework"); - lj->args.append(buf_ptr(g->darwin_frameworks.at(i))); - } - - switch (g->linker_allow_shlib_undefined) { - case OptionalBoolNull: - if (!g->zig_target->is_native_os && !stage2_is_zig0) { - // TODO https://github.com/ziglang/zig/issues/5059 - lj->args.append("-undefined"); - lj->args.append("dynamic_lookup"); - } - break; - case OptionalBoolFalse: - break; - case OptionalBoolTrue: - lj->args.append("-undefined"); - lj->args.append("dynamic_lookup"); - break; - } - switch (g->linker_bind_global_refs_locally) { - case OptionalBoolNull: - case OptionalBoolFalse: - break; - case OptionalBoolTrue: - lj->args.append("-Bsymbolic"); - break; - } -} - -static void construct_linker_job(LinkJob *lj) { - switch (target_object_format(lj->codegen->zig_target)) { - case ZigLLVM_UnknownObjectFormat: - case ZigLLVM_XCOFF: - zig_unreachable(); - - case ZigLLVM_COFF: - return construct_linker_job_coff(lj); - case ZigLLVM_ELF: - return construct_linker_job_elf(lj); - case ZigLLVM_MachO: - return construct_linker_job_macho(lj); - case ZigLLVM_Wasm: - return construct_linker_job_wasm(lj); - } -} - -void zig_link_add_compiler_rt(CodeGen *g, Stage2ProgressNode *progress_node) { - Buf *compiler_rt_o_path = build_compiler_rt(g, OutTypeObj, progress_node); - g->link_objects.append(compiler_rt_o_path); -} - -void codegen_link(CodeGen *g) { - codegen_add_time_event(g, "Build Dependencies"); - LinkJob lj = {0}; - - { - const char *progress_name = "Build Dependencies"; - codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, - progress_name, strlen(progress_name), 0)); - lj.build_dep_prog_node = g->sub_progress_node; - } - - - // even though we're calling LLD as a library it thinks the first - // argument is its own exe name - lj.args.append("lld"); - - lj.rpath_table.init(4); - lj.codegen = g; - - if (g->out_type == OutTypeObj) { - lj.args.append("-r"); - } - - if (g->out_type == OutTypeLib && !g->is_dynamic && !target_is_wasm(g->zig_target)) { - ZigList file_names = {}; - for (size_t i = 0; i < g->link_objects.length; i += 1) { - file_names.append(buf_ptr(g->link_objects.at(i))); - } - ZigLLVM_OSType os_type = get_llvm_os_type(g->zig_target->os); - codegen_add_time_event(g, "LLVM Link"); - { - const char *progress_name = "Link"; - codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, - progress_name, strlen(progress_name), 0)); - } - if (g->verbose_link) { - fprintf(stderr, "ar rcs %s", buf_ptr(&g->bin_file_output_path)); - for (size_t i = 0; i < file_names.length; i += 1) { - fprintf(stderr, " %s", file_names.at(i)); - } - fprintf(stderr, "\n"); - } - if (ZigLLVMWriteArchive(buf_ptr(&g->bin_file_output_path), file_names.items, file_names.length, os_type)) { - fprintf(stderr, "Unable to write archive '%s'\n", buf_ptr(&g->bin_file_output_path)); - exit(1); - } - return; - } - - lj.link_in_crt = (g->libc_link_lib != nullptr && g->out_type == OutTypeExe); - - construct_linker_job(&lj); - - if (g->verbose_link) { - for (size_t i = 0; i < lj.args.length; i += 1) { - const char *space = (i != 0) ? " " : ""; - fprintf(stderr, "%s%s", space, lj.args.at(i)); - } - fprintf(stderr, "\n"); - } - - Buf diag = BUF_INIT; - - codegen_add_time_event(g, "LLVM Link"); - { - const char *progress_name = "Link"; - codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, - progress_name, strlen(progress_name), 0)); - } - if (g->system_linker_hack && g->zig_target->os == OsMacOSX) { - Termination term; - ZigList args = {}; - args.append("ld"); - for (size_t i = 1; i < lj.args.length; i += 1) { - args.append(lj.args.at(i)); - } - os_spawn_process(args, &term); - if (term.how != TerminationIdClean || term.code != 0) { - exit(1); - } - } else if (!zig_lld_link(target_object_format(g->zig_target), lj.args.items, lj.args.length, &diag)) { - fprintf(stderr, "%s\n", buf_ptr(&diag)); - exit(1); - } -} - diff --git a/src/main.cpp b/src/main.cpp deleted file mode 100644 index 003ba6b7c6..0000000000 --- a/src/main.cpp +++ /dev/null @@ -1,1474 +0,0 @@ -/* - * Copyright (c) 2015 Andrew Kelley - * - * This file is part of zig, which is MIT licensed. - * See http://opensource.org/licenses/MIT - */ - -#include "ast_render.hpp" -#include "buffer.hpp" -#include "codegen.hpp" -#include "compiler.hpp" -#include "config.h" -#include "error.hpp" -#include "heap.hpp" -#include "os.hpp" -#include "target.hpp" -#include "stage2.h" -#include "glibc.hpp" -#include "dump_analysis.hpp" -#include "mem_profile.hpp" - -#include - -static int print_error_usage(const char *arg0) { - fprintf(stderr, "See `%s --help` for detailed usage information\n", arg0); - return EXIT_FAILURE; -} - -static int print_full_usage(const char *arg0, FILE *file, int return_code) { - fprintf(file, - "Usage: %s [command] [options]\n" - "\n" - "Commands:\n" - " build build project from build.zig\n" - " build-exe [source] create executable from source or object files\n" - " build-lib [source] create library from source or object files\n" - " build-obj [source] create object from source or assembly\n" - " builtin show the source code of @import(\"builtin\")\n" - " cc use Zig as a drop-in C compiler\n" - " c++ use Zig as a drop-in C++ compiler\n" - " env print lib path, std path, compiler id and version\n" - " fmt parse files and render in canonical zig format\n" - " id print the base64-encoded compiler id\n" - " init-exe initialize a `zig build` application in the cwd\n" - " init-lib initialize a `zig build` library in the cwd\n" - " libc [paths_file] Display native libc paths file or validate one\n" - " run [source] [-- [args]] create executable and run immediately\n" - " translate-c [source] convert c code to zig code\n" - " targets list available compilation targets\n" - " test [source] create and run a test build\n" - " version print version number and exit\n" - " zen print zen of zig and exit\n" - "\n" - "Compile Options:\n" - " --c-source [options] [file] compile C source code\n" - " --cache-dir [path] override the local cache directory\n" - " --cache [auto|off|on] build in cache, print output path to stdout\n" - " --color [auto|off|on] enable or disable colored error messages\n" - " --disable-valgrind omit valgrind client requests in debug builds\n" - " --eh-frame-hdr enable C++ exception handling by passing --eh-frame-hdr to linker\n" - " --enable-valgrind include valgrind client requests release builds\n" - " -fstack-check enable stack probing in unsafe builds\n" - " -fno-stack-check disable stack probing in safe builds\n" - " -fsanitize-c enable C undefined behavior detection in unsafe builds\n" - " -fno-sanitize-c disable C undefined behavior detection in safe builds\n" - " --emit [asm|bin|llvm-ir] (deprecated) emit a specific file format as compilation output\n" - " -fPIC enable Position Independent Code\n" - " -fno-PIC disable Position Independent Code\n" - " -ftime-report print timing diagnostics\n" - " -fstack-report print stack size diagnostics\n" - " -fmem-report print memory usage diagnostics\n" - " -fdump-analysis write analysis.json file with type information\n" - " -femit-docs create a docs/ dir with html documentation\n" - " -fno-emit-docs do not produce docs/ dir with html documentation\n" - " -femit-bin (default) output machine code\n" - " -fno-emit-bin do not output machine code\n" - " -femit-asm output .s (assembly code)\n" - " -fno-emit-asm (default) do not output .s (assembly code)\n" - " -femit-llvm-ir produce a .ll file with LLVM IR\n" - " -fno-emit-llvm-ir (default) do not produce a .ll file with LLVM IR\n" - " -femit-h generate a C header file (.h)\n" - " -fno-emit-h (default) do not generate a C header file (.h)\n" - " --libc [file] Provide a file which specifies libc paths\n" - " --name [name] override output name\n" - " --output-dir [dir] override output directory (defaults to cwd)\n" - " --pkg-begin [name] [path] make pkg available to import and push current pkg\n" - " --pkg-end pop current pkg\n" - " --main-pkg-path set the directory of the root package\n" - " --release-fast build with optimizations on and safety off\n" - " --release-safe build with optimizations on and safety on\n" - " --release-small build with size optimizations on and safety off\n" - " --single-threaded source may assume it is only used single-threaded\n" - " -dynamic create a shared library (.so; .dll; .dylib)\n" - " --strip exclude debug symbols\n" - " -target [name] -- see the targets command\n" - " --verbose-tokenize enable compiler debug output for tokenization\n" - " --verbose-ast enable compiler debug output for AST parsing\n" - " --verbose-link enable compiler debug output for linking\n" - " --verbose-ir enable compiler debug output for Zig IR\n" - " --verbose-llvm-ir enable compiler debug output for LLVM IR\n" - " --verbose-cimport enable compiler debug output for C imports\n" - " --verbose-cc enable compiler debug output for C compilation\n" - " --verbose-llvm-cpu-features enable compiler debug output for LLVM CPU features\n" - " -dirafter [dir] add directory to AFTER include search path\n" - " -isystem [dir] add directory to SYSTEM include search path\n" - " -I[dir] add directory to include search path\n" - " -mllvm [arg] (unsupported) forward an arg to LLVM's option processing\n" - " --override-lib-dir [arg] override path to Zig lib directory\n" - " -ffunction-sections places each function in a separate section\n" - " -D[macro]=[value] define C [macro] to [value] (1 if [value] omitted)\n" - " -mcpu [cpu] specify target CPU and feature set\n" - " -code-model [default|tiny| set target code model\n" - " small|kernel|\n" - " medium|large]\n" - "\n" - "Link Options:\n" - " --bundle-compiler-rt for static libraries, include compiler-rt symbols\n" - " --dynamic-linker [path] set the path to ld.so\n" - " --each-lib-rpath add rpath for each used dynamic library\n" - " --library [lib] link against lib\n" - " --forbid-library [lib] make it an error to link against lib\n" - " --library-path [dir] add a directory to the library search path\n" - " --linker-script [path] use a custom linker script\n" - " --version-script [path] provide a version .map file\n" - " --object [obj] add object file to build\n" - " -L[dir] alias for --library-path\n" - " -l[lib] alias for --library\n" - " -rdynamic add all symbols to the dynamic symbol table\n" - " -rpath [path] add directory to the runtime library search path\n" - " --stack [size] (linux, windows, Wasm) override default stack size\n" - " --subsystem [subsystem] (windows) /SUBSYSTEM: to the linker\n" - " -F[dir] (darwin) add search path for frameworks\n" - " -framework [name] (darwin) link against framework\n" - " --ver-major [ver] dynamic library semver major version\n" - " --ver-minor [ver] dynamic library semver minor version\n" - " --ver-patch [ver] dynamic library semver patch version\n" - " -Bsymbolic bind global references locally\n" - "\n" - "Test Options:\n" - " --test-filter [text] skip tests that do not match filter\n" - " --test-name-prefix [text] add prefix to all tests\n" - " --test-cmd [arg] specify test execution command one arg at a time\n" - " --test-cmd-bin appends test binary path to test cmd args\n" - " --test-evented-io runs the test in evented I/O mode\n" - , arg0); - return return_code; -} - -static int print_libc_usage(const char *arg0, FILE *file, int return_code) { - fprintf(file, - "Usage: %s libc\n" - "\n" - "Detect the native libc installation and print the resulting paths to stdout.\n" - "You can save this into a file and then edit the paths to create a cross\n" - "compilation libc kit. Then you can pass `--libc [file]` for Zig to use it.\n" - "\n" - "When compiling natively and no `--libc` argument provided, Zig will create\n" - "`%s/native_libc.txt`\n" - "so that it does not have to detect libc on every invocation. You can remove\n" - "this file to have Zig re-detect the native libc.\n" - "\n\n" - "Usage: %s libc [file]\n" - "\n" - "Parse a libc installation text file and validate it.\n" - , arg0, buf_ptr(get_global_cache_dir()), arg0); - return return_code; -} - -enum Cmd { - CmdNone, - CmdBuild, - CmdBuiltin, - CmdRun, - CmdTargets, - CmdTest, - CmdTranslateC, - CmdVersion, - CmdZen, - CmdLibC, -}; - -static const char *default_zig_cache_name = "zig-cache"; - -struct CliPkg { - const char *name; - const char *path; - ZigList children; - CliPkg *parent; -}; - -static void add_package(CodeGen *g, CliPkg *cli_pkg, ZigPackage *pkg) { - for (size_t i = 0; i < cli_pkg->children.length; i += 1) { - CliPkg *child_cli_pkg = cli_pkg->children.at(i); - - Buf *dirname = buf_alloc(); - Buf *basename = buf_alloc(); - os_path_split(buf_create_from_str(child_cli_pkg->path), dirname, basename); - - ZigPackage *child_pkg = codegen_create_package(g, buf_ptr(dirname), buf_ptr(basename), - buf_ptr(buf_sprintf("%s.%s", buf_ptr(&pkg->pkg_path), child_cli_pkg->name))); - auto entry = pkg->package_table.put_unique(buf_create_from_str(child_cli_pkg->name), child_pkg); - if (entry) { - ZigPackage *existing_pkg = entry->value; - Buf *full_path = buf_alloc(); - os_path_join(&existing_pkg->root_src_dir, &existing_pkg->root_src_path, full_path); - fprintf(stderr, "Unable to add package '%s'->'%s': already exists as '%s'\n", - child_cli_pkg->name, child_cli_pkg->path, buf_ptr(full_path)); - exit(EXIT_FAILURE); - } - - add_package(g, child_cli_pkg, child_pkg); - } -} - -enum CacheOpt { - CacheOptAuto, - CacheOptOn, - CacheOptOff, -}; - -static bool get_cache_opt(CacheOpt opt, bool default_value) { - switch (opt) { - case CacheOptAuto: - return default_value; - case CacheOptOn: - return true; - case CacheOptOff: - return false; - } - zig_unreachable(); -} - -static int zig_error_no_build_file(void) { - fprintf(stderr, - "No 'build.zig' file found, in the current directory or any parent directories.\n" - "Initialize a 'build.zig' template file with `zig init-lib` or `zig init-exe`,\n" - "or see `zig --help` for more options.\n" - ); - return EXIT_FAILURE; -} - -static bool str_starts_with(const char *s1, const char *s2) { - size_t s2_len = strlen(s2); - if (strlen(s1) < s2_len) { - return false; - } - return memcmp(s1, s2, s2_len) == 0; -} - -extern "C" int ZigClang_main(int argc, char **argv); - -#ifdef ZIG_ENABLE_MEM_PROFILE -bool mem_report = false; -#endif - -int main_exit(Stage2ProgressNode *root_progress_node, int exit_code) { - if (root_progress_node != nullptr) { - stage2_progress_end(root_progress_node); - } - return exit_code; -} - -static int main0(int argc, char **argv) { - char *arg0 = argv[0]; - Error err; - - if (argc >= 2 && (strcmp(argv[1], "clang") == 0 || - strcmp(argv[1], "-cc1") == 0 || strcmp(argv[1], "-cc1as") == 0)) - { - return ZigClang_main(argc, argv); - } - - if (argc == 2 && strcmp(argv[1], "id") == 0) { - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) { - fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); - return EXIT_FAILURE; - } - printf("%s\n", buf_ptr(compiler_id)); - return EXIT_SUCCESS; - } - - enum InitKind { - InitKindNone, - InitKindExe, - InitKindLib, - }; - InitKind init_kind = InitKindNone; - if (argc >= 2) { - const char *init_cmd = argv[1]; - if (strcmp(init_cmd, "init-exe") == 0) { - init_kind = InitKindExe; - } else if (strcmp(init_cmd, "init-lib") == 0) { - init_kind = InitKindLib; - } - if (init_kind != InitKindNone) { - if (argc >= 3) { - fprintf(stderr, "Unexpected extra argument: %s\n", argv[2]); - return print_error_usage(arg0); - } - Buf *cmd_template_path = buf_alloc(); - os_path_join(get_zig_special_dir(get_zig_lib_dir()), buf_create_from_str(init_cmd), cmd_template_path); - Buf *build_zig_path = buf_alloc(); - os_path_join(cmd_template_path, buf_create_from_str("build.zig"), build_zig_path); - Buf *src_dir_path = buf_alloc(); - os_path_join(cmd_template_path, buf_create_from_str("src"), src_dir_path); - Buf *main_zig_path = buf_alloc(); - os_path_join(src_dir_path, buf_create_from_str("main.zig"), main_zig_path); - - Buf *cwd = buf_alloc(); - if ((err = os_get_cwd(cwd))) { - fprintf(stderr, "Unable to get cwd: %s\n", err_str(err)); - return EXIT_FAILURE; - } - Buf *cwd_basename = buf_alloc(); - os_path_split(cwd, nullptr, cwd_basename); - - Buf *build_zig_contents = buf_alloc(); - if ((err = os_fetch_file_path(build_zig_path, build_zig_contents))) { - fprintf(stderr, "Unable to read %s: %s\n", buf_ptr(build_zig_path), err_str(err)); - return EXIT_FAILURE; - } - Buf *modified_build_zig_contents = buf_alloc(); - for (size_t i = 0; i < buf_len(build_zig_contents); i += 1) { - char c = buf_ptr(build_zig_contents)[i]; - if (c == '$') { - buf_append_buf(modified_build_zig_contents, cwd_basename); - } else { - buf_append_char(modified_build_zig_contents, c); - } - } - - Buf *main_zig_contents = buf_alloc(); - if ((err = os_fetch_file_path(main_zig_path, main_zig_contents))) { - fprintf(stderr, "Unable to read %s: %s\n", buf_ptr(main_zig_path), err_str(err)); - return EXIT_FAILURE; - } - - Buf *out_build_zig_path = buf_create_from_str("build.zig"); - Buf *out_src_dir_path = buf_create_from_str("src"); - Buf *out_main_zig_path = buf_alloc(); - os_path_join(out_src_dir_path, buf_create_from_str("main.zig"), out_main_zig_path); - - bool already_exists; - if ((err = os_file_exists(out_build_zig_path, &already_exists))) { - fprintf(stderr, "Unable test existence of %s: %s\n", buf_ptr(out_build_zig_path), err_str(err)); - return EXIT_FAILURE; - } - if (already_exists) { - fprintf(stderr, "This file would be overwritten: %s\n", buf_ptr(out_build_zig_path)); - return EXIT_FAILURE; - } - - if ((err = os_make_dir(out_src_dir_path))) { - fprintf(stderr, "Unable to make directory: %s: %s\n", buf_ptr(out_src_dir_path), err_str(err)); - return EXIT_FAILURE; - } - if ((err = os_write_file(out_build_zig_path, modified_build_zig_contents))) { - fprintf(stderr, "Unable to write file: %s: %s\n", buf_ptr(out_build_zig_path), err_str(err)); - return EXIT_FAILURE; - } - if ((err = os_write_file(out_main_zig_path, main_zig_contents))) { - fprintf(stderr, "Unable to write file: %s: %s\n", buf_ptr(out_main_zig_path), err_str(err)); - return EXIT_FAILURE; - } - fprintf(stderr, "Created %s\n", buf_ptr(out_build_zig_path)); - fprintf(stderr, "Created %s\n", buf_ptr(out_main_zig_path)); - if (init_kind == InitKindExe) { - fprintf(stderr, "\nNext, try `zig build --help` or `zig build run`\n"); - } else if (init_kind == InitKindLib) { - fprintf(stderr, "\nNext, try `zig build --help` or `zig build test`\n"); - } else { - zig_unreachable(); - } - - return EXIT_SUCCESS; - } - } - - Cmd cmd = CmdNone; - const char *in_file = nullptr; - Buf *output_dir = nullptr; - bool strip = false; - bool is_dynamic = false; - OutType out_type = OutTypeUnknown; - const char *out_name = nullptr; - bool verbose_tokenize = false; - bool verbose_ast = false; - bool verbose_link = false; - bool verbose_ir = false; - bool verbose_llvm_ir = false; - bool verbose_cimport = false; - bool verbose_cc = false; - bool verbose_llvm_cpu_features = false; - bool link_eh_frame_hdr = false; - ErrColor color = ErrColorAuto; - CacheOpt enable_cache = CacheOptAuto; - const char *dynamic_linker = nullptr; - const char *libc_txt = nullptr; - ZigList clang_argv = {0}; - ZigList lib_dirs = {0}; - ZigList link_libs = {0}; - ZigList forbidden_link_libs = {0}; - ZigList framework_dirs = {0}; - ZigList frameworks = {0}; - bool have_libc = false; - const char *target_string = nullptr; - bool rdynamic = false; - const char *linker_script = nullptr; - Buf *version_script = nullptr; - ZigList rpath_list = {0}; - bool each_lib_rpath = false; - ZigList objects = {0}; - ZigList c_source_files = {0}; - const char *test_filter = nullptr; - const char *test_name_prefix = nullptr; - bool test_evented_io = false; - bool is_versioned = false; - size_t ver_major = 0; - size_t ver_minor = 0; - size_t ver_patch = 0; - bool timing_info = false; - bool stack_report = false; - bool enable_dump_analysis = false; - bool enable_doc_generation = false; - bool emit_bin = true; - const char *emit_bin_override_path = nullptr; - bool emit_asm = false; - bool emit_llvm_ir = false; - bool emit_h = false; - const char *cache_dir = nullptr; - CliPkg *cur_pkg = heap::c_allocator.create(); - BuildMode build_mode = BuildModeDebug; - ZigList test_exec_args = {0}; - int runtime_args_start = -1; - bool system_linker_hack = false; - TargetSubsystem subsystem = TargetSubsystemAuto; - bool want_single_threaded = false; - bool bundle_compiler_rt = false; - Buf *override_lib_dir = nullptr; - Buf *main_pkg_path = nullptr; - ValgrindSupport valgrind_support = ValgrindSupportAuto; - WantPIC want_pic = WantPICAuto; - WantStackCheck want_stack_check = WantStackCheckAuto; - WantCSanitize want_sanitize_c = WantCSanitizeAuto; - bool function_sections = false; - const char *mcpu = nullptr; - CodeModel code_model = CodeModelDefault; - bool want_native_include_dirs = false; - OptionalBool linker_bind_global_refs_locally = OptionalBoolNull; - size_t stack_size_override = 0; - - ZigList llvm_argv = {0}; - llvm_argv.append("zig (LLVM option parsing)"); - - if (argc >= 2 && strcmp(argv[1], "build") == 0) { - Buf zig_exe_path_buf = BUF_INIT; - if ((err = os_self_exe_path(&zig_exe_path_buf))) { - fprintf(stderr, "Unable to determine path to zig's own executable\n"); - return EXIT_FAILURE; - } - const char *zig_exe_path = buf_ptr(&zig_exe_path_buf); - const char *build_file = nullptr; - - init_all_targets(); - - ZigList args = {0}; - args.append(NULL); // placeholder - args.append(zig_exe_path); - args.append(NULL); // placeholder - args.append(NULL); // placeholder - for (int i = 2; i < argc; i += 1) { - if (strcmp(argv[i], "--help") == 0) { - args.append(argv[i]); - } else if (i + 1 < argc && strcmp(argv[i], "--build-file") == 0) { - build_file = argv[i + 1]; - i += 1; - } else if (i + 1 < argc && strcmp(argv[i], "--cache-dir") == 0) { - cache_dir = argv[i + 1]; - i += 1; - } else if (i + 1 < argc && strcmp(argv[i], "--override-lib-dir") == 0) { - override_lib_dir = buf_create_from_str(argv[i + 1]); - i += 1; - - args.append("--override-lib-dir"); - args.append(buf_ptr(override_lib_dir)); - } else { - args.append(argv[i]); - } - } - - Buf *zig_lib_dir = (override_lib_dir == nullptr) ? get_zig_lib_dir() : override_lib_dir; - - Buf *build_runner_path = buf_alloc(); - os_path_join(get_zig_special_dir(zig_lib_dir), buf_create_from_str("build_runner.zig"), build_runner_path); - - ZigTarget target; - if ((err = target_parse_triple(&target, "native", nullptr, nullptr))) { - fprintf(stderr, "Unable to get native target: %s\n", err_str(err)); - return EXIT_FAILURE; - } - - Buf *build_file_buf = buf_create_from_str((build_file != nullptr) ? build_file : "build.zig"); - Buf build_file_abs = os_path_resolve(&build_file_buf, 1); - Buf build_file_basename = BUF_INIT; - Buf build_file_dirname = BUF_INIT; - os_path_split(&build_file_abs, &build_file_dirname, &build_file_basename); - - for (;;) { - bool build_file_exists; - if ((err = os_file_exists(&build_file_abs, &build_file_exists))) { - fprintf(stderr, "unable to check existence of '%s': %s\n", buf_ptr(&build_file_abs), err_str(err)); - return 1; - } - if (build_file_exists) - break; - - if (build_file != nullptr) { - // they asked for a specific build file path. only look for that one - return zig_error_no_build_file(); - } - - Buf *next_dir = buf_alloc(); - os_path_dirname(&build_file_dirname, next_dir); - if (buf_eql_buf(&build_file_dirname, next_dir)) { - // no more parent directories to search, give up - return zig_error_no_build_file(); - } - os_path_join(next_dir, &build_file_basename, &build_file_abs); - buf_init_from_buf(&build_file_dirname, next_dir); - } - - Buf full_cache_dir = BUF_INIT; - if (cache_dir == nullptr) { - os_path_join(&build_file_dirname, buf_create_from_str(default_zig_cache_name), &full_cache_dir); - } else { - Buf *cache_dir_buf = buf_create_from_str(cache_dir); - full_cache_dir = os_path_resolve(&cache_dir_buf, 1); - } - Stage2ProgressNode *root_progress_node = stage2_progress_start_root(stage2_progress_create(), "", 0, 0); - - CodeGen *g = codegen_create(main_pkg_path, build_runner_path, &target, OutTypeExe, - BuildModeDebug, override_lib_dir, nullptr, &full_cache_dir, false, root_progress_node); - g->valgrind_support = valgrind_support; - g->enable_time_report = timing_info; - codegen_set_out_name(g, buf_create_from_str("build")); - - args.items[2] = buf_ptr(&build_file_dirname); - args.items[3] = buf_ptr(&full_cache_dir); - - ZigPackage *build_pkg = codegen_create_package(g, buf_ptr(&build_file_dirname), - buf_ptr(&build_file_basename), "std.special"); - g->main_pkg->package_table.put(buf_create_from_str("@build"), build_pkg); - g->enable_cache = get_cache_opt(enable_cache, true); - codegen_build_and_link(g); - if (root_progress_node != nullptr) { - stage2_progress_end(root_progress_node); - root_progress_node = nullptr; - } - - Termination term; - args.items[0] = buf_ptr(&g->bin_file_output_path); - os_spawn_process(args, &term); - if (term.how != TerminationIdClean || term.code != 0) { - fprintf(stderr, "\nBuild failed. The following command failed:\n"); - const char *prefix = ""; - for (size_t i = 0; i < args.length; i += 1) { - fprintf(stderr, "%s%s", prefix, args.at(i)); - prefix = " "; - } - fprintf(stderr, "\n"); - } - return (term.how == TerminationIdClean) ? term.code : -1; - } else if (argc >= 2 && strcmp(argv[1], "fmt") == 0) { - return stage2_fmt(argc, argv); - } else if (argc >= 2 && strcmp(argv[1], "env") == 0) { - return stage2_env(argc, argv); - } else if (argc >= 2 && strcmp(argv[1], "cc") == 0) { - return stage2_cc(argc, argv, false); - } else if (argc >= 2 && strcmp(argv[1], "c++") == 0) { - return stage2_cc(argc, argv, true); - } else for (int i = 1; i < argc; i += 1) { - char *arg = argv[i]; - - if (arg[0] == '-') { - if (strcmp(arg, "--") == 0) { - if (cmd == CmdRun) { - runtime_args_start = i + 1; - break; // rest of the args are for the program - } else { - fprintf(stderr, "Unexpected end-of-parameter mark: %s\n", arg); - } - } else if (strcmp(arg, "--release-fast") == 0) { - build_mode = BuildModeFastRelease; - } else if (strcmp(arg, "--release-safe") == 0) { - build_mode = BuildModeSafeRelease; - } else if (strcmp(arg, "--release-small") == 0) { - build_mode = BuildModeSmallRelease; - } else if (strcmp(arg, "--help") == 0) { - if (cmd == CmdLibC) { - return print_libc_usage(arg0, stdout, EXIT_SUCCESS); - } else { - return print_full_usage(arg0, stdout, EXIT_SUCCESS); - } - } else if (strcmp(arg, "--strip") == 0) { - strip = true; - } else if (strcmp(arg, "-dynamic") == 0) { - is_dynamic = true; - } else if (strcmp(arg, "--verbose-tokenize") == 0) { - verbose_tokenize = true; - } else if (strcmp(arg, "--verbose-ast") == 0) { - verbose_ast = true; - } else if (strcmp(arg, "--verbose-link") == 0) { - verbose_link = true; - } else if (strcmp(arg, "--verbose-ir") == 0) { - verbose_ir = true; - } else if (strcmp(arg, "--verbose-llvm-ir") == 0) { - verbose_llvm_ir = true; - } else if (strcmp(arg, "--verbose-cimport") == 0) { - verbose_cimport = true; - } else if (strcmp(arg, "--verbose-cc") == 0) { - verbose_cc = true; - } else if (strcmp(arg, "--verbose-llvm-cpu-features") == 0) { - verbose_llvm_cpu_features = true; - } else if (strcmp(arg, "-rdynamic") == 0) { - rdynamic = true; - } else if (strcmp(arg, "--each-lib-rpath") == 0) { - each_lib_rpath = true; - } else if (strcmp(arg, "-ftime-report") == 0) { - timing_info = true; - } else if (strcmp(arg, "-fstack-report") == 0) { - stack_report = true; - } else if (strcmp(arg, "-fmem-report") == 0) { -#ifdef ZIG_ENABLE_MEM_PROFILE - mem_report = true; - mem::report_print = true; -#else - fprintf(stderr, "-fmem-report requires configuring with -DZIG_ENABLE_MEM_PROFILE=ON\n"); - return print_error_usage(arg0); -#endif - } else if (strcmp(arg, "-fdump-analysis") == 0) { - enable_dump_analysis = true; - } else if (strcmp(arg, "-femit-docs") == 0) { - enable_doc_generation = true; - } else if (strcmp(arg, "--enable-valgrind") == 0) { - valgrind_support = ValgrindSupportEnabled; - } else if (strcmp(arg, "--disable-valgrind") == 0) { - valgrind_support = ValgrindSupportDisabled; - } else if (strcmp(arg, "--eh-frame-hdr") == 0) { - link_eh_frame_hdr = true; - } else if (strcmp(arg, "-fPIC") == 0) { - want_pic = WantPICEnabled; - } else if (strcmp(arg, "-fno-PIC") == 0) { - want_pic = WantPICDisabled; - } else if (strcmp(arg, "-fstack-check") == 0) { - want_stack_check = WantStackCheckEnabled; - } else if (strcmp(arg, "-fno-stack-check") == 0) { - want_stack_check = WantStackCheckDisabled; - } else if (strcmp(arg, "-fsanitize-c") == 0) { - want_sanitize_c = WantCSanitizeEnabled; - } else if (strcmp(arg, "-fno-sanitize-c") == 0) { - want_sanitize_c = WantCSanitizeDisabled; - } else if (strcmp(arg, "--system-linker-hack") == 0) { - system_linker_hack = true; - } else if (strcmp(arg, "--single-threaded") == 0) { - want_single_threaded = true;; - } else if (strcmp(arg, "--bundle-compiler-rt") == 0) { - bundle_compiler_rt = true; - } else if (strcmp(arg, "-Bsymbolic") == 0) { - linker_bind_global_refs_locally = OptionalBoolTrue; - } else if (strcmp(arg, "--test-cmd-bin") == 0) { - test_exec_args.append(nullptr); - } else if (arg[1] == 'D' && arg[2] != 0) { - clang_argv.append("-D"); - clang_argv.append(&arg[2]); - } else if (arg[1] == 'L' && arg[2] != 0) { - // alias for --library-path - lib_dirs.append(&arg[2]); - } else if (arg[1] == 'l' && arg[2] != 0) { - // alias for --library - const char *l = &arg[2]; - if (strcmp(l, "c") == 0) { - have_libc = true; - link_libs.append("c"); - } else if (strcmp(l, "c++") == 0 || strcmp(l, "stdc++") == 0) { - link_libs.append("c++"); - } else { - link_libs.append(l); - } - } else if (arg[1] == 'I' && arg[2] != 0) { - clang_argv.append("-I"); - clang_argv.append(&arg[2]); - } else if (arg[1] == 'F' && arg[2] != 0) { - framework_dirs.append(&arg[2]); - } else if (strcmp(arg, "--pkg-begin") == 0) { - if (i + 2 >= argc) { - fprintf(stderr, "Expected 2 arguments after --pkg-begin\n"); - return print_error_usage(arg0); - } - CliPkg *new_cur_pkg = heap::c_allocator.create(); - i += 1; - new_cur_pkg->name = argv[i]; - i += 1; - new_cur_pkg->path = argv[i]; - new_cur_pkg->parent = cur_pkg; - cur_pkg->children.append(new_cur_pkg); - cur_pkg = new_cur_pkg; - } else if (strcmp(arg, "--pkg-end") == 0) { - if (cur_pkg->parent == nullptr) { - fprintf(stderr, "Encountered --pkg-end with no matching --pkg-begin\n"); - return EXIT_FAILURE; - } - cur_pkg = cur_pkg->parent; - } else if (strcmp(arg, "-ffunction-sections") == 0) { - function_sections = true; - } else if (strcmp(arg, "--test-evented-io") == 0) { - test_evented_io = true; - } else if (strcmp(arg, "-femit-bin") == 0) { - emit_bin = true; - } else if (strcmp(arg, "-fno-emit-bin") == 0) { - emit_bin = false; - } else if (strcmp(arg, "-femit-asm") == 0) { - emit_asm = true; - } else if (strcmp(arg, "-fno-emit-asm") == 0) { - emit_asm = false; - } else if (strcmp(arg, "-femit-llvm-ir") == 0) { - emit_llvm_ir = true; - } else if (strcmp(arg, "-fno-emit-llvm-ir") == 0) { - emit_llvm_ir = false; - } else if (strcmp(arg, "-femit-h") == 0) { - emit_h = true; - } else if (strcmp(arg, "-fno-emit-h") == 0 || strcmp(arg, "--disable-gen-h") == 0) { - // the --disable-gen-h is there to support godbolt. once they upgrade to -fno-emit-h then we can remove this - emit_h = false; - } else if (str_starts_with(arg, "-mcpu=")) { - mcpu = arg + strlen("-mcpu="); - } else if (i + 1 >= argc) { - fprintf(stderr, "Expected another argument after %s\n", arg); - return print_error_usage(arg0); - } else { - i += 1; - if (strcmp(arg, "--output-dir") == 0) { - output_dir = buf_create_from_str(argv[i]); - } else if (strcmp(arg, "--color") == 0) { - if (strcmp(argv[i], "auto") == 0) { - color = ErrColorAuto; - } else if (strcmp(argv[i], "on") == 0) { - color = ErrColorOn; - } else if (strcmp(argv[i], "off") == 0) { - color = ErrColorOff; - } else { - fprintf(stderr, "--color options are 'auto', 'on', or 'off'\n"); - return print_error_usage(arg0); - } - } else if (strcmp(arg, "--cache") == 0) { - if (strcmp(argv[i], "auto") == 0) { - enable_cache = CacheOptAuto; - } else if (strcmp(argv[i], "on") == 0) { - enable_cache = CacheOptOn; - } else if (strcmp(argv[i], "off") == 0) { - enable_cache = CacheOptOff; - } else { - fprintf(stderr, "--cache options are 'auto', 'on', or 'off'\n"); - return print_error_usage(arg0); - } - } else if (strcmp(arg, "--emit") == 0) { - if (strcmp(argv[i], "asm") == 0) { - emit_asm = true; - emit_bin = false; - } else if (strcmp(argv[i], "bin") == 0) { - emit_bin = true; - } else if (strcmp(argv[i], "llvm-ir") == 0) { - emit_llvm_ir = true; - emit_bin = false; - } else { - fprintf(stderr, "--emit options are 'asm', 'bin', or 'llvm-ir'\n"); - return print_error_usage(arg0); - } - } else if (strcmp(arg, "--name") == 0) { - out_name = argv[i]; - } else if (strcmp(arg, "--dynamic-linker") == 0) { - dynamic_linker = argv[i]; - } else if (strcmp(arg, "--libc") == 0) { - libc_txt = argv[i]; - } else if (strcmp(arg, "-D") == 0) { - clang_argv.append("-D"); - clang_argv.append(argv[i]); - } else if (strcmp(arg, "-isystem") == 0) { - clang_argv.append("-isystem"); - clang_argv.append(argv[i]); - } else if (strcmp(arg, "-I") == 0) { - clang_argv.append("-I"); - clang_argv.append(argv[i]); - } else if (strcmp(arg, "-dirafter") == 0) { - clang_argv.append("-dirafter"); - clang_argv.append(argv[i]); - } else if (strcmp(arg, "-mllvm") == 0) { - clang_argv.append("-mllvm"); - clang_argv.append(argv[i]); - - llvm_argv.append(argv[i]); - } else if (strcmp(arg, "-code-model") == 0) { - if (strcmp(argv[i], "default") == 0) { - code_model = CodeModelDefault; - } else if (strcmp(argv[i], "tiny") == 0) { - code_model = CodeModelTiny; - } else if (strcmp(argv[i], "small") == 0) { - code_model = CodeModelSmall; - } else if (strcmp(argv[i], "kernel") == 0) { - code_model = CodeModelKernel; - } else if (strcmp(argv[i], "medium") == 0) { - code_model = CodeModelMedium; - } else if (strcmp(argv[i], "large") == 0) { - code_model = CodeModelLarge; - } else { - fprintf(stderr, "-code-model options are 'default', 'tiny', 'small', 'kernel', 'medium', or 'large'\n"); - return print_error_usage(arg0); - } - } else if (strcmp(arg, "--override-lib-dir") == 0) { - override_lib_dir = buf_create_from_str(argv[i]); - } else if (strcmp(arg, "--main-pkg-path") == 0) { - main_pkg_path = buf_create_from_str(argv[i]); - } else if (strcmp(arg, "--library-path") == 0 || strcmp(arg, "-L") == 0) { - lib_dirs.append(argv[i]); - } else if (strcmp(arg, "-F") == 0) { - framework_dirs.append(argv[i]); - } else if (strcmp(arg, "--library") == 0 || strcmp(arg, "-l") == 0) { - if (strcmp(argv[i], "c") == 0) { - have_libc = true; - link_libs.append("c"); - } else if (strcmp(argv[i], "c++") == 0 || strcmp(argv[i], "stdc++") == 0) { - link_libs.append("c++"); - } else { - link_libs.append(argv[i]); - } - } else if (strcmp(arg, "--forbid-library") == 0) { - forbidden_link_libs.append(argv[i]); - } else if (strcmp(arg, "--object") == 0) { - objects.append(argv[i]); - } else if (strcmp(arg, "--c-source") == 0) { - CFile *c_file = heap::c_allocator.create(); - for (;;) { - if (argv[i][0] == '-') { - c_file->args.append(argv[i]); - i += 1; - if (i < argc) { - continue; - } - - break; - } else { - c_file->source_path = argv[i]; - c_source_files.append(c_file); - break; - } - } - } else if (strcmp(arg, "--cache-dir") == 0) { - cache_dir = argv[i]; - } else if (strcmp(arg, "-target") == 0) { - target_string = argv[i]; - } else if (strcmp(arg, "-framework") == 0) { - frameworks.append(argv[i]); - } else if (strcmp(arg, "--linker-script") == 0) { - linker_script = argv[i]; - } else if (strcmp(arg, "--version-script") == 0) { - version_script = buf_create_from_str(argv[i]); - } else if (strcmp(arg, "-rpath") == 0) { - rpath_list.append(argv[i]); - } else if (strcmp(arg, "--test-filter") == 0) { - test_filter = argv[i]; - } else if (strcmp(arg, "--test-name-prefix") == 0) { - test_name_prefix = argv[i]; - } else if (strcmp(arg, "--ver-major") == 0) { - is_versioned = true; - ver_major = atoi(argv[i]); - } else if (strcmp(arg, "--ver-minor") == 0) { - is_versioned = true; - ver_minor = atoi(argv[i]); - } else if (strcmp(arg, "--ver-patch") == 0) { - is_versioned = true; - ver_patch = atoi(argv[i]); - } else if (strcmp(arg, "--test-cmd") == 0) { - test_exec_args.append(argv[i]); - } else if (strcmp(arg, "--stack") == 0) { - stack_size_override = atoi(argv[i]); - } else if (strcmp(arg, "--subsystem") == 0) { - if (strcmp(argv[i], "console") == 0) { - subsystem = TargetSubsystemConsole; - } else if (strcmp(argv[i], "windows") == 0) { - subsystem = TargetSubsystemWindows; - } else if (strcmp(argv[i], "posix") == 0) { - subsystem = TargetSubsystemPosix; - } else if (strcmp(argv[i], "native") == 0) { - subsystem = TargetSubsystemNative; - } else if (strcmp(argv[i], "efi_application") == 0) { - subsystem = TargetSubsystemEfiApplication; - } else if (strcmp(argv[i], "efi_boot_service_driver") == 0) { - subsystem = TargetSubsystemEfiBootServiceDriver; - } else if (strcmp(argv[i], "efi_rom") == 0) { - subsystem = TargetSubsystemEfiRom; - } else if (strcmp(argv[i], "efi_runtime_driver") == 0) { - subsystem = TargetSubsystemEfiRuntimeDriver; - } else { - fprintf(stderr, "invalid: --subsystem %s\n" - "Options are:\n" - " console\n" - " windows\n" - " posix\n" - " native\n" - " efi_application\n" - " efi_boot_service_driver\n" - " efi_rom\n" - " efi_runtime_driver\n" - , argv[i]); - return EXIT_FAILURE; - } - } else if (strcmp(arg, "-mcpu") == 0) { - mcpu = argv[i]; - } else { - fprintf(stderr, "Invalid argument: %s\n", arg); - return print_error_usage(arg0); - } - } - } else if (cmd == CmdNone) { - if (strcmp(arg, "build-exe") == 0) { - cmd = CmdBuild; - out_type = OutTypeExe; - } else if (strcmp(arg, "build-obj") == 0) { - cmd = CmdBuild; - out_type = OutTypeObj; - } else if (strcmp(arg, "build-lib") == 0) { - cmd = CmdBuild; - out_type = OutTypeLib; - } else if (strcmp(arg, "run") == 0) { - cmd = CmdRun; - out_type = OutTypeExe; - } else if (strcmp(arg, "version") == 0) { - cmd = CmdVersion; - } else if (strcmp(arg, "zen") == 0) { - cmd = CmdZen; - } else if (strcmp(arg, "libc") == 0) { - cmd = CmdLibC; - } else if (strcmp(arg, "translate-c") == 0) { - cmd = CmdTranslateC; - } else if (strcmp(arg, "test") == 0) { - cmd = CmdTest; - out_type = OutTypeExe; - } else if (strcmp(arg, "targets") == 0) { - cmd = CmdTargets; - } else if (strcmp(arg, "builtin") == 0) { - cmd = CmdBuiltin; - } else { - fprintf(stderr, "Unrecognized command: %s\n", arg); - return print_error_usage(arg0); - } - } else { - switch (cmd) { - case CmdBuild: - case CmdRun: - case CmdTranslateC: - case CmdTest: - case CmdLibC: - if (!in_file) { - in_file = arg; - } else { - fprintf(stderr, "Unexpected extra parameter: %s\n", arg); - return print_error_usage(arg0); - } - break; - case CmdBuiltin: - case CmdVersion: - case CmdZen: - case CmdTargets: - fprintf(stderr, "Unexpected extra parameter: %s\n", arg); - return print_error_usage(arg0); - case CmdNone: - zig_unreachable(); - } - } - } - - if (cur_pkg->parent != nullptr) { - fprintf(stderr, "Unmatched --pkg-begin\n"); - return EXIT_FAILURE; - } - - Stage2Progress *progress = stage2_progress_create(); - Stage2ProgressNode *root_progress_node = stage2_progress_start_root(progress, "", 0, 0); - if (color == ErrColorOff) stage2_progress_disable_tty(progress); - - init_all_targets(); - - ZigTarget target; - if ((err = target_parse_triple(&target, target_string, mcpu, dynamic_linker))) { - fprintf(stderr, "invalid target: %s\n" - "See `%s targets` to display valid targets.\n", err_str(err), arg0); - return print_error_usage(arg0); - } - - Buf zig_triple_buf = BUF_INIT; - target_triple_zig(&zig_triple_buf, &target); - - // If both output_dir and enable_cache are provided, and doing build-lib, we - // will just do a file copy at the end. This helps when bootstrapping zig from zig0 - // because we want to pass something like this: - // zig0 build-lib --cache on --output-dir ${CMAKE_BINARY_DIR} - // And we don't have access to `zig0 build` because that would require detecting native libc - // on systems where we are not able to build a libc from source for them. - // But that's the only reason this works, so otherwise we give an error here. - Buf *final_output_dir_step = nullptr; - if (output_dir != nullptr && enable_cache == CacheOptOn) { - if (cmd == CmdBuild && out_type == OutTypeLib) { - final_output_dir_step = output_dir; - output_dir = nullptr; - } else { - fprintf(stderr, "`--output-dir` is incompatible with --cache on.\n"); - return print_error_usage(arg0); - } - } - - if (target_requires_pic(&target, have_libc) && want_pic == WantPICDisabled) { - fprintf(stderr, "`--disable-pic` is incompatible with target '%s'\n", buf_ptr(&zig_triple_buf)); - return print_error_usage(arg0); - } - - if ((emit_asm || emit_llvm_ir) && in_file == nullptr) { - fprintf(stderr, "A root source file is required when using `-femit-asm` or `-femit-llvm-ir`\n"); - return print_error_usage(arg0); - } - - if (llvm_argv.length > 1) { - llvm_argv.append(nullptr); - ZigLLVMParseCommandLineOptions(llvm_argv.length - 1, llvm_argv.items); - } - - switch (cmd) { - case CmdLibC: { - if (in_file) { - Stage2LibCInstallation libc; - if ((err = stage2_libc_parse(&libc, in_file))) { - fprintf(stderr, "unable to parse libc file: %s\n", err_str(err)); - return main_exit(root_progress_node, EXIT_FAILURE); - } - return main_exit(root_progress_node, EXIT_SUCCESS); - } - Stage2LibCInstallation libc; - if ((err = stage2_libc_find_native(&libc))) { - fprintf(stderr, "unable to find native libc file: %s\n", err_str(err)); - return main_exit(root_progress_node, EXIT_FAILURE); - } - if ((err = stage2_libc_render(&libc, stdout))) { - fprintf(stderr, "unable to print libc file: %s\n", err_str(err)); - return main_exit(root_progress_node, EXIT_FAILURE); - } - return main_exit(root_progress_node, EXIT_SUCCESS); - } - case CmdBuiltin: { - CodeGen *g = codegen_create(main_pkg_path, nullptr, &target, - out_type, build_mode, override_lib_dir, nullptr, nullptr, false, root_progress_node); - codegen_set_strip(g, strip); - for (size_t i = 0; i < link_libs.length; i += 1) { - LinkLib *link_lib = codegen_add_link_lib(g, buf_create_from_str(link_libs.at(i))); - link_lib->provided_explicitly = true; - } - g->subsystem = subsystem; - g->valgrind_support = valgrind_support; - g->want_pic = want_pic; - g->want_stack_check = want_stack_check; - g->want_sanitize_c = want_sanitize_c; - g->want_single_threaded = want_single_threaded; - g->test_is_evented = test_evented_io; - Buf *builtin_source = codegen_generate_builtin_source(g); - if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) { - fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout))); - return main_exit(root_progress_node, EXIT_FAILURE); - } - return main_exit(root_progress_node, EXIT_SUCCESS); - } - case CmdRun: - case CmdBuild: - case CmdTranslateC: - case CmdTest: - { - if (cmd == CmdBuild && !in_file && objects.length == 0 && - c_source_files.length == 0) - { - fprintf(stderr, - "Expected at least one of these things:\n" - " * Zig root source file argument\n" - " * --object argument\n" - " * --c-source argument\n"); - return print_error_usage(arg0); - } else if ((cmd == CmdTranslateC || - cmd == CmdTest || cmd == CmdRun) && !in_file) - { - fprintf(stderr, "Expected source file argument.\n"); - return print_error_usage(arg0); - } else if (cmd == CmdRun && !emit_bin) { - fprintf(stderr, "Cannot run without emitting a binary file.\n"); - return print_error_usage(arg0); - } - - bool any_system_lib_dependencies = false; - for (size_t i = 0; i < link_libs.length; i += 1) { - if (!target_is_libc_lib_name(&target, link_libs.at(i)) && - !target_is_libcpp_lib_name(&target, link_libs.at(i))) - { - any_system_lib_dependencies = true; - break; - } - } - - if (target.is_native_os && (any_system_lib_dependencies || want_native_include_dirs)) { - Error err; - Stage2NativePaths paths; - if ((err = stage2_detect_native_paths(&paths))) { - fprintf(stderr, "unable to detect native system paths: %s\n", err_str(err)); - exit(1); - } - for (size_t i = 0; i < paths.warnings_len; i += 1) { - const char *warning = paths.warnings_ptr[i]; - fprintf(stderr, "warning: %s\n", warning); - } - for (size_t i = 0; i < paths.include_dirs_len; i += 1) { - const char *include_dir = paths.include_dirs_ptr[i]; - clang_argv.append("-isystem"); - clang_argv.append(include_dir); - } - for (size_t i = 0; i < paths.lib_dirs_len; i += 1) { - const char *lib_dir = paths.lib_dirs_ptr[i]; - lib_dirs.append(lib_dir); - } - for (size_t i = 0; i < paths.rpaths_len; i += 1) { - const char *rpath = paths.rpaths_ptr[i]; - rpath_list.append(rpath); - } - } - - - assert(cmd != CmdBuild || out_type != OutTypeUnknown); - - bool need_name = (cmd == CmdBuild || cmd == CmdTranslateC); - - if (cmd == CmdRun) { - out_name = "run"; - } - - Buf *in_file_buf = nullptr; - - Buf *buf_out_name = (cmd == CmdTest) ? buf_create_from_str("test") : - (out_name == nullptr) ? nullptr : buf_create_from_str(out_name); - - if (in_file) { - in_file_buf = buf_create_from_str(in_file); - - if (need_name && buf_out_name == nullptr) { - Buf basename = BUF_INIT; - os_path_split(in_file_buf, nullptr, &basename); - buf_out_name = buf_alloc(); - os_path_extname(&basename, buf_out_name, nullptr); - } - } - - if (need_name && buf_out_name == nullptr && c_source_files.length == 1) { - Buf basename = BUF_INIT; - os_path_split(buf_create_from_str(c_source_files.at(0)->source_path), nullptr, &basename); - buf_out_name = buf_alloc(); - os_path_extname(&basename, buf_out_name, nullptr); - } - if (need_name && buf_out_name == nullptr && objects.length == 1) { - Buf basename = BUF_INIT; - os_path_split(buf_create_from_str(objects.at(0)), nullptr, &basename); - buf_out_name = buf_alloc(); - os_path_extname(&basename, buf_out_name, nullptr); - } - if (need_name && buf_out_name == nullptr && emit_bin_override_path != nullptr) { - Buf basename = BUF_INIT; - os_path_split(buf_create_from_str(emit_bin_override_path), nullptr, &basename); - buf_out_name = buf_alloc(); - os_path_extname(&basename, buf_out_name, nullptr); - } - - if (need_name && buf_out_name == nullptr) { - fprintf(stderr, "--name [name] not provided and unable to infer\n\n"); - return print_error_usage(arg0); - } - - Buf *zig_root_source_file = (cmd == CmdTranslateC) ? nullptr : in_file_buf; - - if (cmd == CmdRun && buf_out_name == nullptr) { - buf_out_name = buf_create_from_str("run"); - } - Stage2LibCInstallation *libc = nullptr; - if (libc_txt != nullptr) { - libc = heap::c_allocator.create(); - if ((err = stage2_libc_parse(libc, libc_txt))) { - fprintf(stderr, "Unable to parse --libc text file: %s\n", err_str(err)); - return main_exit(root_progress_node, EXIT_FAILURE); - } - } - Buf *cache_dir_buf; - if (cache_dir == nullptr) { - if (cmd == CmdRun) { - cache_dir_buf = get_global_cache_dir(); - } else { - cache_dir_buf = buf_create_from_str(default_zig_cache_name); - } - } else { - cache_dir_buf = buf_create_from_str(cache_dir); - } - CodeGen *g = codegen_create(main_pkg_path, zig_root_source_file, &target, out_type, build_mode, - override_lib_dir, libc, cache_dir_buf, cmd == CmdTest, root_progress_node); - if (llvm_argv.length >= 2) codegen_set_llvm_argv(g, llvm_argv.items + 1, llvm_argv.length - 2); - g->valgrind_support = valgrind_support; - g->link_eh_frame_hdr = link_eh_frame_hdr; - g->want_pic = want_pic; - g->want_stack_check = want_stack_check; - g->want_sanitize_c = want_sanitize_c; - g->subsystem = subsystem; - - g->enable_time_report = timing_info; - g->enable_stack_report = stack_report; - g->enable_dump_analysis = enable_dump_analysis; - g->enable_doc_generation = enable_doc_generation; - g->emit_bin = emit_bin; - g->emit_asm = emit_asm; - g->emit_llvm_ir = emit_llvm_ir; - - codegen_set_out_name(g, buf_out_name); - codegen_set_lib_version(g, is_versioned, ver_major, ver_minor, ver_patch); - g->want_single_threaded = want_single_threaded; - codegen_set_linker_script(g, linker_script); - g->version_script_path = version_script; - if (each_lib_rpath) - codegen_set_each_lib_rpath(g, each_lib_rpath); - - codegen_set_clang_argv(g, clang_argv.items, clang_argv.length); - - codegen_set_strip(g, strip); - g->is_dynamic = is_dynamic; - g->verbose_tokenize = verbose_tokenize; - g->verbose_ast = verbose_ast; - g->verbose_link = verbose_link; - g->verbose_ir = verbose_ir; - g->verbose_llvm_ir = verbose_llvm_ir; - g->verbose_cimport = verbose_cimport; - g->verbose_cc = verbose_cc; - g->verbose_llvm_cpu_features = verbose_llvm_cpu_features; - g->output_dir = output_dir; - g->disable_gen_h = !emit_h; - g->bundle_compiler_rt = bundle_compiler_rt; - codegen_set_errmsg_color(g, color); - g->system_linker_hack = system_linker_hack; - g->function_sections = function_sections; - g->code_model = code_model; - - g->linker_bind_global_refs_locally = linker_bind_global_refs_locally; - g->stack_size_override = stack_size_override; - - for (size_t i = 0; i < lib_dirs.length; i += 1) { - codegen_add_lib_dir(g, lib_dirs.at(i)); - } - for (size_t i = 0; i < framework_dirs.length; i += 1) { - g->framework_dirs.append(framework_dirs.at(i)); - } - for (size_t i = 0; i < link_libs.length; i += 1) { - LinkLib *link_lib = codegen_add_link_lib(g, buf_create_from_str(link_libs.at(i))); - link_lib->provided_explicitly = true; - } - for (size_t i = 0; i < forbidden_link_libs.length; i += 1) { - Buf *forbidden_link_lib = buf_create_from_str(forbidden_link_libs.at(i)); - codegen_add_forbidden_lib(g, forbidden_link_lib); - } - for (size_t i = 0; i < frameworks.length; i += 1) { - codegen_add_framework(g, frameworks.at(i)); - } - for (size_t i = 0; i < rpath_list.length; i += 1) { - codegen_add_rpath(g, rpath_list.at(i)); - } - - codegen_set_rdynamic(g, rdynamic); - - if (test_filter) { - codegen_set_test_filter(g, buf_create_from_str(test_filter)); - } - g->test_is_evented = test_evented_io; - - if (test_name_prefix) { - codegen_set_test_name_prefix(g, buf_create_from_str(test_name_prefix)); - } - - add_package(g, cur_pkg, g->main_pkg); - - if (cmd == CmdBuild || cmd == CmdRun || cmd == CmdTest) { - g->c_source_files = c_source_files; - for (size_t i = 0; i < objects.length; i += 1) { - codegen_add_object(g, buf_create_from_str(objects.at(i))); - } - } - - - if (cmd == CmdBuild || cmd == CmdRun) { - g->enable_cache = get_cache_opt(enable_cache, cmd == CmdRun); - codegen_build_and_link(g); - if (root_progress_node != nullptr) { - stage2_progress_end(root_progress_node); - root_progress_node = nullptr; - } - if (timing_info) - codegen_print_timing_report(g, stdout); - if (stack_report) - zig_print_stack_report(g, stdout); - - if (cmd == CmdRun) { -#ifdef ZIG_ENABLE_MEM_PROFILE - if (mem::report_print) - mem::print_report(); -#endif - - const char *exec_path = buf_ptr(&g->bin_file_output_path); - ZigList args = {0}; - - args.append(exec_path); - if (runtime_args_start != -1) { - for (int i = runtime_args_start; i < argc; ++i) { - args.append(argv[i]); - } - } - args.append(nullptr); - - os_execv(exec_path, args.items); - - args.pop(); - Termination term; - os_spawn_process(args, &term); - return term.code; - } else if (cmd == CmdBuild) { - if (emit_bin_override_path != nullptr) { -#if defined(ZIG_OS_WINDOWS) - buf_replace(g->output_dir, '/', '\\'); -#endif - Buf *dest_path = buf_create_from_str(emit_bin_override_path); - Buf *source_path = &g->bin_file_output_path; - if ((err = os_update_file(source_path, dest_path))) { - fprintf(stderr, "unable to copy %s to %s: %s\n", buf_ptr(source_path), - buf_ptr(dest_path), err_str(err)); - return main_exit(root_progress_node, EXIT_FAILURE); - } - } else if (g->enable_cache) { -#if defined(ZIG_OS_WINDOWS) - buf_replace(&g->bin_file_output_path, '/', '\\'); - buf_replace(g->output_dir, '/', '\\'); -#endif - if (final_output_dir_step != nullptr) { - Buf *dest_basename = buf_alloc(); - os_path_split(&g->bin_file_output_path, nullptr, dest_basename); - Buf *dest_path = buf_alloc(); - os_path_join(final_output_dir_step, dest_basename, dest_path); - - if ((err = os_update_file(&g->bin_file_output_path, dest_path))) { - fprintf(stderr, "unable to copy %s to %s: %s\n", buf_ptr(&g->bin_file_output_path), - buf_ptr(dest_path), err_str(err)); - return main_exit(root_progress_node, EXIT_FAILURE); - } - } else { - if (printf("%s\n", buf_ptr(g->output_dir)) < 0) - return main_exit(root_progress_node, EXIT_FAILURE); - } - } - return main_exit(root_progress_node, EXIT_SUCCESS); - } else { - zig_unreachable(); - } - } else if (cmd == CmdTranslateC) { - g->enable_cache = get_cache_opt(enable_cache, false); - codegen_translate_c(g, in_file_buf); - if (timing_info) - codegen_print_timing_report(g, stderr); - return main_exit(root_progress_node, EXIT_SUCCESS); - } else if (cmd == CmdTest) { - ZigTarget native; - if ((err = target_parse_triple(&native, "native", nullptr, nullptr))) { - fprintf(stderr, "Unable to get native target: %s\n", err_str(err)); - return EXIT_FAILURE; - } - - g->enable_cache = get_cache_opt(enable_cache, output_dir == nullptr); - codegen_build_and_link(g); - if (root_progress_node != nullptr) { - stage2_progress_end(root_progress_node); - root_progress_node = nullptr; - } - - if (timing_info) { - codegen_print_timing_report(g, stdout); - } - - if (stack_report) { - zig_print_stack_report(g, stdout); - } - - if (!g->emit_bin) { - fprintf(stderr, "Semantic analysis complete. No binary produced due to -fno-emit-bin.\n"); - return main_exit(root_progress_node, EXIT_SUCCESS); - } - - Buf *test_exe_path_unresolved = &g->bin_file_output_path; - Buf *test_exe_path = buf_alloc(); - *test_exe_path = os_path_resolve(&test_exe_path_unresolved, 1); - - if (!g->emit_bin) { - fprintf(stderr, "Created %s but skipping execution because no binary generated.\n", - buf_ptr(test_exe_path)); - return main_exit(root_progress_node, EXIT_SUCCESS); - } - - for (size_t i = 0; i < test_exec_args.length; i += 1) { - if (test_exec_args.items[i] == nullptr) { - test_exec_args.items[i] = buf_ptr(test_exe_path); - } - } - - if (!target_can_exec(&native, &target) && test_exec_args.length == 0) { - fprintf(stderr, "Created %s but skipping execution because it is non-native.\n", - buf_ptr(test_exe_path)); - return main_exit(root_progress_node, EXIT_SUCCESS); - } - - Termination term; - if (test_exec_args.length == 0) { - test_exec_args.append(buf_ptr(test_exe_path)); - } - os_spawn_process(test_exec_args, &term); - if (term.how != TerminationIdClean || term.code != 0) { - fprintf(stderr, "\nTests failed. Use the following command to reproduce the failure:\n"); - fprintf(stderr, "%s\n", buf_ptr(test_exe_path)); - } - return main_exit(root_progress_node, (term.how == TerminationIdClean) ? term.code : -1); - } else { - zig_unreachable(); - } - } - case CmdVersion: - printf("%s\n", ZIG_VERSION_STRING); - return main_exit(root_progress_node, EXIT_SUCCESS); - case CmdZen: { - const char *ptr; - size_t len; - stage2_zen(&ptr, &len); - fwrite(ptr, len, 1, stdout); - return main_exit(root_progress_node, EXIT_SUCCESS); - } - case CmdTargets: - return stage2_cmd_targets(target_string, mcpu, dynamic_linker); - case CmdNone: - return print_full_usage(arg0, stderr, EXIT_FAILURE); - } - zig_unreachable(); -} - -int main(int argc, char **argv) { - stage2_attach_segfault_handler(); - os_init(); - mem::init(); - - auto result = main0(argc, argv); - -#ifdef ZIG_ENABLE_MEM_PROFILE - if (mem::report_print) - mem::intern_counters.print_report(); -#endif - mem::deinit(); - return result; -} diff --git a/src/mem.cpp b/src/mem.cpp index 51ee9a27ee..48dbd791de 100644 --- a/src/mem.cpp +++ b/src/mem.cpp @@ -7,7 +7,6 @@ #include "config.h" #include "mem.hpp" -#include "mem_profile.hpp" #include "heap.hpp" namespace mem { @@ -22,16 +21,4 @@ void deinit() { heap::bootstrap_allocator_state.deinit(); } -#ifdef ZIG_ENABLE_MEM_PROFILE -void print_report(FILE *file) { - heap::c_allocator_state.print_report(file); - intern_counters.print_report(file); -} -#endif - -#ifdef ZIG_ENABLE_MEM_PROFILE -bool report_print = false; -FILE *report_file{nullptr}; -#endif - } // namespace mem diff --git a/src/mem.hpp b/src/mem.hpp index 9e262b7d53..3008febbde 100644 --- a/src/mem.hpp +++ b/src/mem.hpp @@ -135,15 +135,6 @@ protected: virtual void internal_deallocate(const TypeInfo &info, void *ptr, size_t count) = 0; }; -#ifdef ZIG_ENABLE_MEM_PROFILE -void print_report(FILE *file = nullptr); - -// global memory report flag -extern bool report_print; -// global memory report default destination -extern FILE *report_file; -#endif - } // namespace mem #endif diff --git a/src/mem_profile.cpp b/src/mem_profile.cpp deleted file mode 100644 index 13ba57f913..0000000000 --- a/src/mem_profile.cpp +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright (c) 2020 Andrew Kelley - * - * This file is part of zig, which is MIT licensed. - * See http://opensource.org/licenses/MIT - */ - -#include "config.h" - -#ifdef ZIG_ENABLE_MEM_PROFILE - -#include "mem.hpp" -#include "mem_list.hpp" -#include "mem_profile.hpp" -#include "heap.hpp" - -namespace mem { - -void Profile::init(const char *name, const char *kind) { - this->name = name; - this->kind = kind; - this->usage_table.init(heap::bootstrap_allocator, 1024); -} - -void Profile::deinit() { - assert(this->name != nullptr); - if (mem::report_print) - this->print_report(); - this->usage_table.deinit(heap::bootstrap_allocator); - this->name = nullptr; -} - -void Profile::record_alloc(const TypeInfo &info, size_t count) { - if (count == 0) return; - auto existing_entry = this->usage_table.put_unique( - heap::bootstrap_allocator, - UsageKey{info.name_ptr, info.name_len}, - Entry{info, 1, count, 0, 0} ); - if (existing_entry != nullptr) { - assert(existing_entry->value.info.size == info.size); // allocated name does not match type - existing_entry->value.alloc.calls += 1; - existing_entry->value.alloc.objects += count; - } -} - -void Profile::record_dealloc(const TypeInfo &info, size_t count) { - if (count == 0) return; - auto existing_entry = this->usage_table.maybe_get(UsageKey{info.name_ptr, info.name_len}); - if (existing_entry == nullptr) { - fprintf(stderr, "deallocated name '"); - for (size_t i = 0; i < info.name_len; ++i) - fputc(info.name_ptr[i], stderr); - zig_panic("' (size %zu) not found in allocated table; compromised memory usage stats", info.size); - } - if (existing_entry->value.info.size != info.size) { - fprintf(stderr, "deallocated name '"); - for (size_t i = 0; i < info.name_len; ++i) - fputc(info.name_ptr[i], stderr); - zig_panic("' does not match expected type size %zu", info.size); - } - assert(existing_entry->value.alloc.calls - existing_entry->value.dealloc.calls > 0); - assert(existing_entry->value.alloc.objects - existing_entry->value.dealloc.objects >= count); - existing_entry->value.dealloc.calls += 1; - existing_entry->value.dealloc.objects += count; -} - -static size_t entry_remain_total_bytes(const Profile::Entry *entry) { - return (entry->alloc.objects - entry->dealloc.objects) * entry->info.size; -} - -static int entry_compare(const void *a, const void *b) { - size_t total_a = entry_remain_total_bytes(*reinterpret_cast(a)); - size_t total_b = entry_remain_total_bytes(*reinterpret_cast(b)); - if (total_a > total_b) - return -1; - if (total_a < total_b) - return 1; - return 0; -}; - -void Profile::print_report(FILE *file) { - if (!file) { - file = report_file; - if (!file) - file = stderr; - } - fprintf(file, "\n--- MEMORY PROFILE REPORT [%s]: %s ---\n", this->kind, this->name); - - List list; - auto it = this->usage_table.entry_iterator(); - for (;;) { - auto entry = it.next(); - if (!entry) - break; - list.append(&heap::bootstrap_allocator, &entry->value); - } - - qsort(list.items, list.length, sizeof(const Entry *), entry_compare); - - size_t total_bytes_alloc = 0; - size_t total_bytes_dealloc = 0; - - size_t total_calls_alloc = 0; - size_t total_calls_dealloc = 0; - - for (size_t i = 0; i < list.length; i += 1) { - const Entry *entry = list.at(i); - fprintf(file, " "); - for (size_t j = 0; j < entry->info.name_len; ++j) - fputc(entry->info.name_ptr[j], file); - fprintf(file, ": %zu bytes each", entry->info.size); - - fprintf(file, ", alloc{ %zu calls, %zu objects, total ", entry->alloc.calls, entry->alloc.objects); - const auto alloc_num_bytes = entry->alloc.objects * entry->info.size; - zig_pretty_print_bytes(file, alloc_num_bytes); - - fprintf(file, " }, dealloc{ %zu calls, %zu objects, total ", entry->dealloc.calls, entry->dealloc.objects); - const auto dealloc_num_bytes = entry->dealloc.objects * entry->info.size; - zig_pretty_print_bytes(file, dealloc_num_bytes); - - fprintf(file, " }, remain{ %zu calls, %zu objects, total ", - entry->alloc.calls - entry->dealloc.calls, - entry->alloc.objects - entry->dealloc.objects ); - const auto remain_num_bytes = alloc_num_bytes - dealloc_num_bytes; - zig_pretty_print_bytes(file, remain_num_bytes); - - fprintf(file, " }\n"); - - total_bytes_alloc += alloc_num_bytes; - total_bytes_dealloc += dealloc_num_bytes; - - total_calls_alloc += entry->alloc.calls; - total_calls_dealloc += entry->dealloc.calls; - } - - fprintf(file, "\n Total bytes allocated: "); - zig_pretty_print_bytes(file, total_bytes_alloc); - fprintf(file, ", deallocated: "); - zig_pretty_print_bytes(file, total_bytes_dealloc); - fprintf(file, ", remaining: "); - zig_pretty_print_bytes(file, total_bytes_alloc - total_bytes_dealloc); - - fprintf(file, "\n Total calls alloc: %zu, dealloc: %zu, remain: %zu\n", - total_calls_alloc, total_calls_dealloc, (total_calls_alloc - total_calls_dealloc)); - - list.deinit(&heap::bootstrap_allocator); -} - -uint32_t Profile::usage_hash(UsageKey key) { - // FNV 32-bit hash - uint32_t h = 2166136261; - for (size_t i = 0; i < key.name_len; ++i) { - h = h ^ key.name_ptr[i]; - h = h * 16777619; - } - return h; -} - -bool Profile::usage_equal(UsageKey a, UsageKey b) { - return memcmp(a.name_ptr, b.name_ptr, a.name_len > b.name_len ? a.name_len : b.name_len) == 0; -} - -void InternCounters::print_report(FILE *file) { - if (!file) { - file = report_file; - if (!file) - file = stderr; - } - fprintf(file, "\n--- IR INTERNING REPORT ---\n"); - fprintf(file, " undefined: interned %zu times\n", intern_counters.x_undefined); - fprintf(file, " void: interned %zu times\n", intern_counters.x_void); - fprintf(file, " null: interned %zu times\n", intern_counters.x_null); - fprintf(file, " unreachable: interned %zu times\n", intern_counters.x_unreachable); - fprintf(file, " zero_byte: interned %zu times\n", intern_counters.zero_byte); -} - -InternCounters intern_counters; - -} // namespace mem - -#endif diff --git a/src/mem_profile.hpp b/src/mem_profile.hpp deleted file mode 100644 index 3b13b7680b..0000000000 --- a/src/mem_profile.hpp +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2020 Andrew Kelley - * - * This file is part of zig, which is MIT licensed. - * See http://opensource.org/licenses/MIT - */ - -#ifndef ZIG_MEM_PROFILE_HPP -#define ZIG_MEM_PROFILE_HPP - -#include "config.h" - -#ifdef ZIG_ENABLE_MEM_PROFILE - -#include - -#include "mem.hpp" -#include "mem_hash_map.hpp" -#include "util.hpp" - -namespace mem { - -struct Profile { - void init(const char *name, const char *kind); - void deinit(); - - void record_alloc(const TypeInfo &info, size_t count); - void record_dealloc(const TypeInfo &info, size_t count); - - void print_report(FILE *file = nullptr); - - struct Entry { - TypeInfo info; - - struct Use { - size_t calls; - size_t objects; - } alloc, dealloc; - }; - -private: - const char *name; - const char *kind; - - struct UsageKey { - const char *name_ptr; - size_t name_len; - }; - - static uint32_t usage_hash(UsageKey key); - static bool usage_equal(UsageKey a, UsageKey b); - - HashMap usage_table; -}; - -struct InternCounters { - size_t x_undefined; - size_t x_void; - size_t x_null; - size_t x_unreachable; - size_t zero_byte; - - void print_report(FILE *file = nullptr); -}; - -extern InternCounters intern_counters; - -} // namespace mem - -#endif -#endif diff --git a/src/mem_type_info.hpp b/src/mem_type_info.hpp index 8698992ca0..d8a7933268 100644 --- a/src/mem_type_info.hpp +++ b/src/mem_type_info.hpp @@ -10,18 +10,8 @@ #include "config.h" -#ifndef ZIG_TYPE_INFO_IMPLEMENTATION -# ifdef ZIG_ENABLE_MEM_PROFILE -# define ZIG_TYPE_INFO_IMPLEMENTATION 1 -# else -# define ZIG_TYPE_INFO_IMPLEMENTATION 0 -# endif -#endif - namespace mem { -#if ZIG_TYPE_INFO_IMPLEMENTATION == 0 - struct TypeInfo { size_t size; size_t alignment; @@ -32,105 +22,6 @@ struct TypeInfo { } }; -#elif ZIG_TYPE_INFO_IMPLEMENTATION == 1 - -// -// A non-portable way to get a human-readable type-name compatible with -// non-RTTI C++ compiler mode; eg. `-fno-rtti`. -// -// Minimum requirements are c++11 and a compiler that has a constant for the -// current function's decorated name whereby a template-type name can be -// computed. eg. `__PRETTY_FUNCTION__` or `__FUNCSIG__`. -// -// given the following snippet: -// -// | #include -// | -// | struct Top {}; -// | namespace mynamespace { -// | using custom = unsigned int; -// | struct Foo { -// | struct Bar {}; -// | }; -// | }; -// | -// | template -// | void foobar() { -// | #ifdef _MSC_VER -// | fprintf(stderr, "--> %s\n", __FUNCSIG__); -// | #else -// | fprintf(stderr, "--> %s\n", __PRETTY_FUNCTION__); -// | #endif -// | } -// | -// | int main() { -// | foobar(); -// | foobar(); -// | foobar(); -// | foobar(); -// | foobar(); -// | } -// -// gcc 9.2.0 produces: -// --> void foobar() [with T = Top] -// --> void foobar() [with T = unsigned int] -// --> void foobar() [with T = unsigned int] -// --> void foobar() [with T = mynamespace::Foo*] -// --> void foobar() [with T = mynamespace::Foo::Bar*] -// -// xcode 11.3.1/clang produces: -// --> void foobar() [T = Top] -// --> void foobar() [T = unsigned int] -// --> void foobar() [T = unsigned int] -// --> void foobar() [T = mynamespace::Foo *] -// --> void foobar() [T = mynamespace::Foo::Bar *] -// -// VStudio 2019 16.5.0/msvc produces: -// --> void __cdecl foobar(void) -// --> void __cdecl foobar(void) -// --> void __cdecl foobar(void) -// --> void __cdecl foobar(void) -// --> void __cdecl foobar(void) -// -struct TypeInfo { - const char *name_ptr; - size_t name_len; - size_t size; - size_t alignment; - - static constexpr TypeInfo to_type_info(const char *str, size_t start, size_t end, size_t size, size_t alignment) { - return TypeInfo{str + start, end - start, size, alignment}; - } - - static constexpr size_t index_of(const char *str, char c) { - return *str == c ? 0 : 1 + index_of(str + 1, c); - } - - template - static constexpr const char *decorated_name() { -#ifdef _MSC_VER - return __FUNCSIG__; -#else - return __PRETTY_FUNCTION__; -#endif - } - - static constexpr TypeInfo extract(const char *decorated, size_t size, size_t alignment) { -#ifdef _MSC_VER - return to_type_info(decorated, index_of(decorated, '<') + 1, index_of(decorated, '>'), size, alignment); -#else - return to_type_info(decorated, index_of(decorated, '=') + 2, index_of(decorated, ']'), size, alignment); -#endif - } - - template - static constexpr TypeInfo make() { - return TypeInfo::extract(TypeInfo::decorated_name(), sizeof(T), alignof(T)); - } -}; - -#endif // ZIG_TYPE_INFO_IMPLEMENTATION - } // namespace mem #endif diff --git a/src/stage1.cpp b/src/stage1.cpp new file mode 100644 index 0000000000..417076aa99 --- /dev/null +++ b/src/stage1.cpp @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2020 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#include "stage1.h" +#include "os.hpp" +#include "all_types.hpp" +#include "codegen.hpp" + +void zig_stage1_os_init(void) { + os_init(); + mem::init(); + init_all_targets(); +} + +struct ZigStage1 *zig_stage1_create(BuildMode optimize_mode, + const char *main_pkg_path_ptr, size_t main_pkg_path_len, + const char *root_src_path_ptr, size_t root_src_path_len, + const char *zig_lib_dir_ptr, size_t zig_lib_dir_len, + const ZigTarget *target, bool is_test_build, Stage2ProgressNode *progress_node) +{ + Buf *main_pkg_path = buf_create_from_mem(main_pkg_path_ptr, main_pkg_path_len); + Buf *root_src_path = buf_create_from_mem(root_src_path_ptr, root_src_path_len); + Buf *zig_lib_dir = buf_create_from_mem(zig_lib_dir_ptr, zig_lib_dir_len); + CodeGen *g = codegen_create(main_pkg_path, root_src_path, target, optimize_mode, + zig_lib_dir, is_test_build, progress_node); + return &g->stage1; +} + +void zig_stage1_destroy(struct ZigStage1 *stage1) { + CodeGen *codegen = reinterpret_cast(stage1); + codegen_destroy(codegen); +} + +static void add_package(CodeGen *g, ZigStage1Pkg *stage1_pkg, ZigPackage *pkg) { + for (size_t i = 0; i < stage1_pkg->children_len; i += 1) { + ZigStage1Pkg *child_cli_pkg = stage1_pkg->children_ptr[i]; + + Buf *dirname = buf_alloc(); + Buf *basename = buf_alloc(); + os_path_split(buf_create_from_mem(child_cli_pkg->path_ptr, child_cli_pkg->path_len), dirname, basename); + + ZigPackage *child_pkg = codegen_create_package(g, buf_ptr(dirname), buf_ptr(basename), + buf_ptr(buf_sprintf("%s.%.*s", buf_ptr(&pkg->pkg_path), + (int)child_cli_pkg->name_len, child_cli_pkg->name_ptr))); + auto entry = pkg->package_table.put_unique( + buf_create_from_mem(child_cli_pkg->name_ptr, child_cli_pkg->name_len), + child_pkg); + if (entry) { + ZigPackage *existing_pkg = entry->value; + Buf *full_path = buf_alloc(); + os_path_join(&existing_pkg->root_src_dir, &existing_pkg->root_src_path, full_path); + fprintf(stderr, "Unable to add package '%.*s'->'%.*s': already exists as '%s'\n", + (int)child_cli_pkg->name_len, child_cli_pkg->name_ptr, + (int)child_cli_pkg->path_len, child_cli_pkg->path_ptr, + buf_ptr(full_path)); + exit(EXIT_FAILURE); + } + + add_package(g, child_cli_pkg, child_pkg); + } +} + +void zig_stage1_build_object(struct ZigStage1 *stage1) { + CodeGen *g = reinterpret_cast(stage1); + + g->root_out_name = buf_create_from_mem(stage1->root_name_ptr, stage1->root_name_len); + g->zig_lib_dir = buf_create_from_mem(stage1->zig_lib_dir_ptr, stage1->zig_lib_dir_len); + g->zig_std_dir = buf_create_from_mem(stage1->zig_std_dir_ptr, stage1->zig_std_dir_len); + g->output_dir = buf_create_from_mem(stage1->output_dir_ptr, stage1->output_dir_len); + if (stage1->builtin_zig_path_len != 0) { + g->builtin_zig_path = buf_create_from_mem(stage1->builtin_zig_path_ptr, stage1->builtin_zig_path_len); + } + if (stage1->test_filter_len != 0) { + g->test_filter = buf_create_from_mem(stage1->test_filter_ptr, stage1->test_filter_len); + } + if (stage1->test_name_prefix_len != 0) { + g->test_name_prefix = buf_create_from_mem(stage1->test_name_prefix_ptr, stage1->test_name_prefix_len); + } + + g->link_mode_dynamic = stage1->link_mode_dynamic; + g->dll_export_fns = stage1->dll_export_fns; + g->have_pic = stage1->pic; + g->have_stack_probing = stage1->enable_stack_probing; + g->is_single_threaded = stage1->is_single_threaded; + g->valgrind_enabled = stage1->valgrind_enabled; + g->link_libc = stage1->link_libc; + g->link_libcpp = stage1->link_libcpp; + g->function_sections = stage1->function_sections; + + g->subsystem = stage1->subsystem; + + g->enable_time_report = stage1->enable_time_report; + g->enable_stack_report = stage1->enable_stack_report; + g->enable_dump_analysis = stage1->dump_analysis; + g->enable_doc_generation = stage1->enable_doc_generation; + g->emit_bin = stage1->emit_bin; + g->emit_asm = stage1->emit_asm; + g->emit_llvm_ir = stage1->emit_llvm_ir; + g->test_is_evented = stage1->test_is_evented; + + g->verbose_tokenize = stage1->verbose_tokenize; + g->verbose_ast = stage1->verbose_ast; + g->verbose_link = stage1->verbose_link; + g->verbose_ir = stage1->verbose_ir; + g->verbose_llvm_ir = stage1->verbose_llvm_ir; + g->verbose_cimport = stage1->verbose_cimport; + g->verbose_cc = stage1->verbose_cc; + g->verbose_llvm_cpu_features = stage1->verbose_llvm_cpu_features; + + g->err_color = stage1->err_color; + g->code_model = stage1->code_model; + + { + g->strip_debug_symbols = stage1->strip; + if (!target_has_debug_info(g->zig_target)) { + g->strip_debug_symbols = true; + } + } + + add_package(g, stage1->root_pkg, g->main_pkg); + + codegen_build_object(g); +} diff --git a/src/stage1.h b/src/stage1.h new file mode 100644 index 0000000000..b7144d8409 --- /dev/null +++ b/src/stage1.h @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2020 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +// This file deals with exposing stage1 C++ code to stage2 Zig code. + +#ifndef ZIG_STAGE1_H +#define ZIG_STAGE1_H + +#include "zig_llvm.h" + +#include + +#ifdef __cplusplus +#define ZIG_EXTERN_C extern "C" +#else +#define ZIG_EXTERN_C +#endif + +// ABI warning +enum ErrColor { + ErrColorAuto, + ErrColorOff, + ErrColorOn, +}; + +// ABI warning +enum CodeModel { + CodeModelDefault, + CodeModelTiny, + CodeModelSmall, + CodeModelKernel, + CodeModelMedium, + CodeModelLarge, +}; + +// ABI warning +enum TargetSubsystem { + TargetSubsystemConsole, + TargetSubsystemWindows, + TargetSubsystemPosix, + TargetSubsystemNative, + TargetSubsystemEfiApplication, + TargetSubsystemEfiBootServiceDriver, + TargetSubsystemEfiRom, + TargetSubsystemEfiRuntimeDriver, + + // This means Zig should infer the subsystem. + // It's last so that the indexes of other items can line up + // with the enum in builtin.zig. + TargetSubsystemAuto +}; + + +// ABI warning +// Synchronize with target.cpp::os_list +enum Os { + OsFreestanding, + OsAnanas, + OsCloudABI, + OsDragonFly, + OsFreeBSD, + OsFuchsia, + OsIOS, + OsKFreeBSD, + OsLinux, + OsLv2, // PS3 + OsMacOSX, + OsNetBSD, + OsOpenBSD, + OsSolaris, + OsWindows, + OsHaiku, + OsMinix, + OsRTEMS, + OsNaCl, // Native Client + OsCNK, // BG/P Compute-Node Kernel + OsAIX, + OsCUDA, // NVIDIA CUDA + OsNVCL, // NVIDIA OpenCL + OsAMDHSA, // AMD HSA Runtime + OsPS4, + OsELFIAMCU, + OsTvOS, // Apple tvOS + OsWatchOS, // Apple watchOS + OsMesa3D, + OsContiki, + OsAMDPAL, + OsHermitCore, + OsHurd, + OsWASI, + OsEmscripten, + OsUefi, + OsOther, +}; + +// ABI warning +struct ZigTarget { + enum ZigLLVM_ArchType arch; + enum ZigLLVM_VendorType vendor; + + enum ZigLLVM_EnvironmentType abi; + Os os; + + bool is_native_os; + bool is_native_cpu; + + const char *llvm_cpu_name; + const char *llvm_cpu_features; + const char *cpu_builtin_str; + const char *os_builtin_str; + const char *dynamic_linker; + + const char **llvm_cpu_features_asm_ptr; + size_t llvm_cpu_features_asm_len; +}; + +// ABI warning +struct Stage2Progress; +// ABI warning +struct Stage2ProgressNode; + +enum BuildMode { + BuildModeDebug, + BuildModeFastRelease, + BuildModeSafeRelease, + BuildModeSmallRelease, +}; + + +struct ZigStage1Pkg { + const char *name_ptr; + size_t name_len; + + const char *path_ptr; + size_t path_len; + + struct ZigStage1Pkg **children_ptr; + size_t children_len; + + struct ZigStage1Pkg *parent; +}; + +// This struct is used by both main.cpp and stage1.zig. +struct ZigStage1 { + const char *root_name_ptr; + size_t root_name_len; + + const char *output_dir_ptr; + size_t output_dir_len; + + const char *builtin_zig_path_ptr; + size_t builtin_zig_path_len; + + const char *test_filter_ptr; + size_t test_filter_len; + + const char *test_name_prefix_ptr; + size_t test_name_prefix_len; + + const char *zig_lib_dir_ptr; + size_t zig_lib_dir_len; + + const char *zig_std_dir_ptr; + size_t zig_std_dir_len; + + void *userdata; + struct ZigStage1Pkg *root_pkg; + + CodeModel code_model; + TargetSubsystem subsystem; + ErrColor err_color; + + bool pic; + bool link_libc; + bool link_libcpp; + bool strip; + bool is_single_threaded; + bool dll_export_fns; + bool link_mode_dynamic; + bool valgrind_enabled; + bool function_sections; + bool enable_stack_probing; + bool enable_time_report; + bool enable_stack_report; + bool dump_analysis; + bool enable_doc_generation; + bool emit_bin; + bool emit_asm; + bool emit_llvm_ir; + bool test_is_evented; + bool verbose_tokenize; + bool verbose_ast; + bool verbose_link; + bool verbose_ir; + bool verbose_llvm_ir; + bool verbose_cimport; + bool verbose_cc; + bool verbose_llvm_cpu_features; +}; + +ZIG_EXTERN_C void zig_stage1_os_init(void); + +ZIG_EXTERN_C struct ZigStage1 *zig_stage1_create(enum BuildMode optimize_mode, + const char *main_pkg_path_ptr, size_t main_pkg_path_len, + const char *root_src_path_ptr, size_t root_src_path_len, + const char *zig_lib_dir_ptr, size_t zig_lib_dir_len, + const ZigTarget *target, bool is_test_build, Stage2ProgressNode *progress_node); + +ZIG_EXTERN_C void zig_stage1_build_object(struct ZigStage1 *); + +ZIG_EXTERN_C void zig_stage1_destroy(struct ZigStage1 *); + +#endif diff --git a/src/stage2.cpp b/src/stage2.cpp index ad6a9f9d97..f8b078c237 100644 --- a/src/stage2.cpp +++ b/src/stage2.cpp @@ -5,40 +5,12 @@ #include "util.hpp" #include "zig_llvm.h" #include "target.hpp" +#include "buffer.hpp" +#include "os.hpp" #include #include #include -Error stage2_translate_c(struct Stage2Ast **out_ast, - struct Stage2ErrorMsg **out_errors_ptr, size_t *out_errors_len, - const char **args_begin, const char **args_end, const char *resources_path) -{ - const char *msg = "stage0 called stage2_translate_c"; - stage2_panic(msg, strlen(msg)); -} - -void stage2_free_clang_errors(struct Stage2ErrorMsg *ptr, size_t len) { - const char *msg = "stage0 called stage2_free_clang_errors"; - stage2_panic(msg, strlen(msg)); -} - -void stage2_zen(const char **ptr, size_t *len) { - const char *msg = "stage0 called stage2_zen"; - stage2_panic(msg, strlen(msg)); -} - -int stage2_env(int argc, char** argv) { - const char *msg = "stage0 called stage2_env"; - stage2_panic(msg, strlen(msg)); -} - -int stage2_cc(int argc, char** argv, bool is_cpp) { - const char *msg = "stage0 called stage2_cc"; - stage2_panic(msg, strlen(msg)); -} - -void stage2_attach_segfault_handler(void) { } - void stage2_panic(const char *ptr, size_t len) { fwrite(ptr, 1, len, stderr); fprintf(stderr, "\n"); @@ -46,32 +18,6 @@ void stage2_panic(const char *ptr, size_t len) { abort(); } -void stage2_render_ast(struct Stage2Ast *ast, FILE *output_file) { - const char *msg = "stage0 called stage2_render_ast"; - stage2_panic(msg, strlen(msg)); -} - -int stage2_fmt(int argc, char **argv) { - const char *msg = "stage0 called stage2_fmt"; - stage2_panic(msg, strlen(msg)); -} - -stage2_DepTokenizer stage2_DepTokenizer_init(const char *input, size_t len) { - const char *msg = "stage0 called stage2_DepTokenizer_init"; - stage2_panic(msg, strlen(msg)); -} - -void stage2_DepTokenizer_deinit(stage2_DepTokenizer *self) { - const char *msg = "stage0 called stage2_DepTokenizer_deinit"; - stage2_panic(msg, strlen(msg)); -} - -stage2_DepNextResult stage2_DepTokenizer_next(stage2_DepTokenizer *self) { - const char *msg = "stage0 called stage2_DepTokenizer_next"; - stage2_panic(msg, strlen(msg)); -} - - struct Stage2Progress { int trash; }; @@ -196,10 +142,6 @@ static void get_native_target(ZigTarget *target) { if (target->abi == ZigLLVM_UnknownEnvironment) { target->abi = target_default_abi(target->arch, target->os); } - if (target_is_glibc(target)) { - target->glibc_or_darwin_version = heap::c_allocator.create(); - target_init_default_glibc_version(target); - } } Error stage2_target_parse(struct ZigTarget *target, const char *zig_triple, const char *mcpu, @@ -217,13 +159,11 @@ Error stage2_target_parse(struct ZigTarget *target, const char *zig_triple, cons if (mcpu == nullptr) { target->llvm_cpu_name = ZigLLVMGetHostCPUName(); target->llvm_cpu_features = ZigLLVMGetNativeFeatures(); - target->cache_hash = "native\n\n"; } else if (strcmp(mcpu, "baseline") == 0) { target->is_native_os = false; target->is_native_cpu = false; target->llvm_cpu_name = ""; target->llvm_cpu_features = ""; - target->cache_hash = "baseline\n\n"; } else { const char *msg = "stage0 can't handle CPU/features in the target"; stage2_panic(msg, strlen(msg)); @@ -264,11 +204,8 @@ Error stage2_target_parse(struct ZigTarget *target, const char *zig_triple, cons const char *msg = "stage0 can't handle CPU/features in the target"; stage2_panic(msg, strlen(msg)); } - target->cache_hash = "\n\n"; } - target->cache_hash_len = strlen(target->cache_hash); - if (dynamic_linker != nullptr) { target->dynamic_linker = dynamic_linker; } @@ -276,49 +213,29 @@ Error stage2_target_parse(struct ZigTarget *target, const char *zig_triple, cons return ErrorNone; } -int stage2_cmd_targets(const char *zig_triple, const char *mcpu, const char *dynamic_linker) { - const char *msg = "stage0 called stage2_cmd_targets"; +const char *stage2_fetch_file(struct ZigStage1 *stage1, const char *path_ptr, size_t path_len, + size_t *result_len) +{ + Error err; + Buf contents_buf = BUF_INIT; + Buf path_buf = BUF_INIT; + + buf_init_from_mem(&path_buf, path_ptr, path_len); + if ((err = os_fetch_file_path(&path_buf, &contents_buf))) { + return nullptr; + } + *result_len = buf_len(&contents_buf); + return buf_ptr(&contents_buf); +} + +const char *stage2_cimport(struct ZigStage1 *stage1) { + const char *msg = "stage0 called stage2_cimport"; stage2_panic(msg, strlen(msg)); } -enum Error stage2_libc_parse(struct Stage2LibCInstallation *libc, const char *libc_file) { - libc->include_dir = "/dummy/include"; - libc->include_dir_len = strlen(libc->include_dir); - libc->sys_include_dir = "/dummy/sys/include"; - libc->sys_include_dir_len = strlen(libc->sys_include_dir); - libc->crt_dir = ""; - libc->crt_dir_len = strlen(libc->crt_dir); - libc->msvc_lib_dir = ""; - libc->msvc_lib_dir_len = strlen(libc->msvc_lib_dir); - libc->kernel32_lib_dir = ""; - libc->kernel32_lib_dir_len = strlen(libc->kernel32_lib_dir); - return ErrorNone; +const char *stage2_add_link_lib(struct ZigStage1 *stage1, + const char *lib_name_ptr, size_t lib_name_len, + const char *symbol_name_ptr, size_t symbol_name_len) +{ + return nullptr; } - -enum Error stage2_libc_render(struct Stage2LibCInstallation *self, FILE *file) { - const char *msg = "stage0 called stage2_libc_render"; - stage2_panic(msg, strlen(msg)); -} - -enum Error stage2_libc_find_native(struct Stage2LibCInstallation *libc) { - const char *msg = "stage0 called stage2_libc_find_native"; - stage2_panic(msg, strlen(msg)); -} - -enum Error stage2_detect_native_paths(struct Stage2NativePaths *native_paths) { - native_paths->include_dirs_ptr = nullptr; - native_paths->include_dirs_len = 0; - - native_paths->lib_dirs_ptr = nullptr; - native_paths->lib_dirs_len = 0; - - native_paths->rpaths_ptr = nullptr; - native_paths->rpaths_len = 0; - - native_paths->warnings_ptr = nullptr; - native_paths->warnings_len = 0; - - return ErrorNone; -} - -const bool stage2_is_zig0 = true; diff --git a/src/stage2.h b/src/stage2.h index 2e64189450..4bc92631dc 100644 --- a/src/stage2.h +++ b/src/stage2.h @@ -5,6 +5,8 @@ * See http://opensource.org/licenses/MIT */ +// This file deals with exposing stage2 Zig code to stage1 C++ code. + #ifndef ZIG_STAGE2_H #define ZIG_STAGE2_H @@ -12,7 +14,7 @@ #include #include -#include "zig_llvm.h" +#include "stage1.h" #ifdef __cplusplus #define ZIG_EXTERN_C extern "C" @@ -27,7 +29,7 @@ #endif // ABI warning: the types and declarations in this file must match both those in -// stage2.cpp and src-self-hosted/stage2.zig. +// stage2.cpp and src-self-hosted/stage1.zig. // ABI warning enum Error { @@ -124,74 +126,9 @@ struct Stage2ErrorMsg { unsigned offset; // byte offset into source }; -// ABI warning -struct Stage2Ast; - -// ABI warning -ZIG_EXTERN_C enum Error stage2_translate_c(struct Stage2Ast **out_ast, - struct Stage2ErrorMsg **out_errors_ptr, size_t *out_errors_len, - const char **args_begin, const char **args_end, const char *resources_path); - -// ABI warning -ZIG_EXTERN_C void stage2_free_clang_errors(struct Stage2ErrorMsg *ptr, size_t len); - -// ABI warning -ZIG_EXTERN_C void stage2_render_ast(struct Stage2Ast *ast, FILE *output_file); - -// ABI warning -ZIG_EXTERN_C void stage2_zen(const char **ptr, size_t *len); - -// ABI warning -ZIG_EXTERN_C int stage2_env(int argc, char **argv); - -// ABI warning -ZIG_EXTERN_C int stage2_cc(int argc, char **argv, bool is_cpp); - -// ABI warning -ZIG_EXTERN_C void stage2_attach_segfault_handler(void); - // ABI warning ZIG_EXTERN_C ZIG_ATTRIBUTE_NORETURN void stage2_panic(const char *ptr, size_t len); -// ABI warning -ZIG_EXTERN_C int stage2_fmt(int argc, char **argv); - -// ABI warning -struct stage2_DepTokenizer { - void *handle; -}; - -// ABI warning -struct stage2_DepNextResult { - enum TypeId { - error, - null, - target, - prereq, - }; - - TypeId type_id; - - // when ent == error --> error text - // when ent == null --> undefined - // when ent == target --> target pathname - // when ent == prereq --> prereq pathname - const char *textz; -}; - -// ABI warning -ZIG_EXTERN_C stage2_DepTokenizer stage2_DepTokenizer_init(const char *input, size_t len); - -// ABI warning -ZIG_EXTERN_C void stage2_DepTokenizer_deinit(stage2_DepTokenizer *self); - -// ABI warning -ZIG_EXTERN_C stage2_DepNextResult stage2_DepTokenizer_next(stage2_DepTokenizer *self); - -// ABI warning -struct Stage2Progress; -// ABI warning -struct Stage2ProgressNode; // ABI warning ZIG_EXTERN_C Stage2Progress *stage2_progress_create(void); // ABI warning @@ -212,69 +149,6 @@ ZIG_EXTERN_C void stage2_progress_complete_one(Stage2ProgressNode *node); ZIG_EXTERN_C void stage2_progress_update_node(Stage2ProgressNode *node, size_t completed_count, size_t estimated_total_items); -// ABI warning -struct Stage2LibCInstallation { - const char *include_dir; - size_t include_dir_len; - const char *sys_include_dir; - size_t sys_include_dir_len; - const char *crt_dir; - size_t crt_dir_len; - const char *msvc_lib_dir; - size_t msvc_lib_dir_len; - const char *kernel32_lib_dir; - size_t kernel32_lib_dir_len; -}; - -// ABI warning -ZIG_EXTERN_C enum Error stage2_libc_parse(struct Stage2LibCInstallation *libc, const char *libc_file); -// ABI warning -ZIG_EXTERN_C enum Error stage2_libc_render(struct Stage2LibCInstallation *self, FILE *file); -// ABI warning -ZIG_EXTERN_C enum Error stage2_libc_find_native(struct Stage2LibCInstallation *libc); - -// ABI warning -// Synchronize with target.cpp::os_list -enum Os { - OsFreestanding, - OsAnanas, - OsCloudABI, - OsDragonFly, - OsFreeBSD, - OsFuchsia, - OsIOS, - OsKFreeBSD, - OsLinux, - OsLv2, // PS3 - OsMacOSX, - OsNetBSD, - OsOpenBSD, - OsSolaris, - OsWindows, - OsHaiku, - OsMinix, - OsRTEMS, - OsNaCl, // Native Client - OsCNK, // BG/P Compute-Node Kernel - OsAIX, - OsCUDA, // NVIDIA CUDA - OsNVCL, // NVIDIA OpenCL - OsAMDHSA, // AMD HSA Runtime - OsPS4, - OsELFIAMCU, - OsTvOS, // Apple tvOS - OsWatchOS, // Apple watchOS - OsMesa3D, - OsContiki, - OsAMDPAL, - OsHermitCore, - OsHurd, - OsWASI, - OsEmscripten, - OsUefi, - OsOther, -}; - // ABI warning struct Stage2SemVer { uint32_t major; @@ -282,56 +156,20 @@ struct Stage2SemVer { uint32_t patch; }; -// ABI warning -struct ZigTarget { - enum ZigLLVM_ArchType arch; - enum ZigLLVM_VendorType vendor; - - enum ZigLLVM_EnvironmentType abi; - Os os; - - bool is_native_os; - bool is_native_cpu; - - // null means default. this is double-purposed to be darwin min version - struct Stage2SemVer *glibc_or_darwin_version; - - const char *llvm_cpu_name; - const char *llvm_cpu_features; - const char *cpu_builtin_str; - const char *cache_hash; - size_t cache_hash_len; - const char *os_builtin_str; - const char *dynamic_linker; - const char *standard_dynamic_linker_path; - - const char **llvm_cpu_features_asm_ptr; - size_t llvm_cpu_features_asm_len; -}; - // ABI warning ZIG_EXTERN_C enum Error stage2_target_parse(struct ZigTarget *target, const char *zig_triple, const char *mcpu, const char *dynamic_linker); // ABI warning -ZIG_EXTERN_C int stage2_cmd_targets(const char *zig_triple, const char *mcpu, const char *dynamic_linker); - +ZIG_EXTERN_C const char *stage2_fetch_file(struct ZigStage1 *stage1, const char *path_ptr, size_t path_len, + size_t *result_len); // ABI warning -struct Stage2NativePaths { - const char **include_dirs_ptr; - size_t include_dirs_len; - const char **lib_dirs_ptr; - size_t lib_dirs_len; - const char **rpaths_ptr; - size_t rpaths_len; - const char **warnings_ptr; - size_t warnings_len; -}; -// ABI warning -ZIG_EXTERN_C enum Error stage2_detect_native_paths(struct Stage2NativePaths *native_paths); +ZIG_EXTERN_C const char *stage2_cimport(struct ZigStage1 *stage1); // ABI warning -ZIG_EXTERN_C const bool stage2_is_zig0; +ZIG_EXTERN_C const char *stage2_add_link_lib(struct ZigStage1 *stage1, + const char *lib_name_ptr, size_t lib_name_len, + const char *symbol_name_ptr, size_t symbol_name_len); #endif diff --git a/src/target.cpp b/src/target.cpp index dff134a01d..f1cab8eded 100644 --- a/src/target.cpp +++ b/src/target.cpp @@ -10,8 +10,6 @@ #include "target.hpp" #include "util.hpp" #include "os.hpp" -#include "compiler.hpp" -#include "glibc.hpp" #include @@ -346,33 +344,6 @@ const char *target_abi_name(ZigLLVM_EnvironmentType abi) { return ZigLLVMGetEnvironmentTypeName(abi); } -Error target_parse_glibc_version(Stage2SemVer *glibc_ver, const char *text) { - glibc_ver->major = 2; - glibc_ver->minor = 0; - glibc_ver->patch = 0; - SplitIterator it = memSplit(str(text), str("GLIBC_.")); - { - Optional> opt_component = SplitIterator_next(&it); - if (!opt_component.is_some) return ErrorUnknownABI; - glibc_ver->major = strtoul(buf_ptr(buf_create_from_slice(opt_component.value)), nullptr, 10); - } - { - Optional> opt_component = SplitIterator_next(&it); - if (!opt_component.is_some) return ErrorNone; - glibc_ver->minor = strtoul(buf_ptr(buf_create_from_slice(opt_component.value)), nullptr, 10); - } - { - Optional> opt_component = SplitIterator_next(&it); - if (!opt_component.is_some) return ErrorNone; - glibc_ver->patch = strtoul(buf_ptr(buf_create_from_slice(opt_component.value)), nullptr, 10); - } - return ErrorNone; -} - -void target_init_default_glibc_version(ZigTarget *target) { - *target->glibc_or_darwin_version = {2, 17, 0}; -} - Error target_parse_arch(ZigLLVM_ArchType *out_arch, const char *arch_ptr, size_t arch_len) { *out_arch = ZigLLVM_UnknownArch; for (size_t arch_i = 0; arch_i < array_length(arch_list); arch_i += 1) { @@ -756,66 +727,6 @@ const char *target_llvm_ir_file_ext(const ZigTarget *target) { return ".ll"; } -const char *target_exe_file_ext(const ZigTarget *target) { - if (target->os == OsWindows) { - return ".exe"; - } else if (target->os == OsUefi) { - return ".efi"; - } else if (target_is_wasm(target)) { - return ".wasm"; - } else { - return ""; - } -} - -const char *target_lib_file_prefix(const ZigTarget *target) { - if ((target->os == OsWindows && !target_abi_is_gnu(target->abi)) || - target->os == OsUefi || - target_is_wasm(target)) - { - return ""; - } else { - return "lib"; - } -} - -const char *target_lib_file_ext(const ZigTarget *target, bool is_static, bool is_versioned, - size_t version_major, size_t version_minor, size_t version_patch) -{ - if (target_is_wasm(target)) { - return ".wasm"; - } - if (target->os == OsWindows || target->os == OsUefi) { - if (is_static) { - if (target->os == OsWindows && target_abi_is_gnu(target->abi)) { - return ".a"; - } else { - return ".lib"; - } - } else { - return ".dll"; - } - } else { - if (is_static) { - return ".a"; - } else if (target_os_is_darwin(target->os)) { - if (is_versioned) { - return buf_ptr(buf_sprintf(".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".dylib", - version_major, version_minor, version_patch)); - } else { - return ".dylib"; - } - } else { - if (is_versioned) { - return buf_ptr(buf_sprintf(".so.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize, - version_major, version_minor, version_patch)); - } else { - return ".so"; - } - } - } -} - bool target_is_android(const ZigTarget *target) { return target->abi == ZigLLVM_Android; } @@ -992,40 +903,6 @@ bool target_os_requires_libc(Os os) { return (target_os_is_darwin(os) || os == OsFreeBSD || os == OsNetBSD || os == OsDragonFly); } -bool target_supports_fpic(const ZigTarget *target) { - // This is not whether the target supports Position Independent Code, but whether the -fPIC - // C compiler argument is valid. - return target->os != OsWindows; -} - -bool target_supports_clang_march_native(const ZigTarget *target) { - // Whether clang supports -march=native on this target. - // Arguably it should always work, but in reality it gives: - // error: the clang compiler does not support '-march=native' - // If we move CPU detection logic into Zig itelf, we will not need this, - // instead we will always pass target features and CPU configuration explicitly. - return target->arch != ZigLLVM_aarch64 && - target->arch != ZigLLVM_aarch64_be; -} - -bool target_supports_stack_probing(const ZigTarget *target) { - return target->os != OsWindows && target->os != OsUefi && (target->arch == ZigLLVM_x86 || target->arch == ZigLLVM_x86_64); -} - -bool target_supports_sanitize_c(const ZigTarget *target) { - return true; -} - -bool target_requires_pic(const ZigTarget *target, bool linking_libc) { - // This function returns whether non-pic code is completely invalid on the given target. - return target_is_android(target) || target->os == OsWindows || target->os == OsUefi || target_os_requires_libc(target->os) || - (linking_libc && target_is_glibc(target)); -} - -bool target_requires_pie(const ZigTarget *target) { - return target_is_android(target); -} - bool target_is_glibc(const ZigTarget *target) { return target->os == OsLinux && target_abi_is_gnu(target->abi); } @@ -1038,10 +915,6 @@ bool target_is_wasm(const ZigTarget *target) { return target->arch == ZigLLVM_wasm32 || target->arch == ZigLLVM_wasm64; } -bool target_is_single_threaded(const ZigTarget *target) { - return target_is_wasm(target); -} - ZigLLVM_EnvironmentType target_default_abi(ZigLLVM_ArchType arch, Os os) { if (arch == ZigLLVM_wasm32 || arch == ZigLLVM_wasm64) { return ZigLLVM_Musl; diff --git a/src/target.hpp b/src/target.hpp index 5e44301fff..c9db4754f9 100644 --- a/src/target.hpp +++ b/src/target.hpp @@ -12,22 +12,6 @@ struct Buf; -enum TargetSubsystem { - TargetSubsystemConsole, - TargetSubsystemWindows, - TargetSubsystemPosix, - TargetSubsystemNative, - TargetSubsystemEfiApplication, - TargetSubsystemEfiBootServiceDriver, - TargetSubsystemEfiRom, - TargetSubsystemEfiRuntimeDriver, - - // This means Zig should infer the subsystem. - // It's last so that the indexes of other items can line up - // with the enum in builtin.zig. - TargetSubsystemAuto -}; - enum CIntType { CIntTypeShort, CIntTypeUShort, @@ -46,9 +30,6 @@ Error target_parse_arch(ZigLLVM_ArchType *arch, const char *arch_ptr, size_t arc Error target_parse_os(Os *os, const char *os_ptr, size_t os_len); Error target_parse_abi(ZigLLVM_EnvironmentType *abi, const char *abi_ptr, size_t abi_len); -Error target_parse_glibc_version(Stage2SemVer *out, const char *text); -void target_init_default_glibc_version(ZigTarget *target); - size_t target_arch_count(void); ZigLLVM_ArchType target_arch_enum(size_t index); const char *target_arch_name(ZigLLVM_ArchType arch); @@ -85,10 +66,6 @@ uint32_t target_c_type_size_in_bits(const ZigTarget *target, CIntType id); const char *target_o_file_ext(const ZigTarget *target); const char *target_asm_file_ext(const ZigTarget *target); const char *target_llvm_ir_file_ext(const ZigTarget *target); -const char *target_exe_file_ext(const ZigTarget *target); -const char *target_lib_file_prefix(const ZigTarget *target); -const char *target_lib_file_ext(const ZigTarget *target, bool is_static, bool is_versioned, - size_t version_major, size_t version_minor, size_t version_patch); bool target_can_exec(const ZigTarget *host_target, const ZigTarget *guest_target); ZigLLVM_OSType get_llvm_os_type(Os os_type); @@ -104,10 +81,6 @@ bool target_can_build_libc(const ZigTarget *target); const char *target_libc_generic_name(const ZigTarget *target); bool target_is_libc_lib_name(const ZigTarget *target, const char *name); bool target_is_libcpp_lib_name(const ZigTarget *target, const char *name); -bool target_supports_fpic(const ZigTarget *target); -bool target_supports_clang_march_native(const ZigTarget *target); -bool target_requires_pic(const ZigTarget *target, bool linking_libc); -bool target_requires_pie(const ZigTarget *target); bool target_abi_is_gnu(ZigLLVM_EnvironmentType abi); bool target_abi_is_musl(ZigLLVM_EnvironmentType abi); bool target_is_glibc(const ZigTarget *target); @@ -115,9 +88,6 @@ bool target_is_musl(const ZigTarget *target); bool target_is_wasm(const ZigTarget *target); bool target_is_riscv(const ZigTarget *target); bool target_is_android(const ZigTarget *target); -bool target_is_single_threaded(const ZigTarget *target); -bool target_supports_stack_probing(const ZigTarget *target); -bool target_supports_sanitize_c(const ZigTarget *target); bool target_has_debug_info(const ZigTarget *target); const char *target_arch_musl_name(ZigLLVM_ArchType arch); diff --git a/src/zig0.cpp b/src/zig0.cpp new file mode 100644 index 0000000000..ad90a17eb3 --- /dev/null +++ b/src/zig0.cpp @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2015 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +// This file is the entry point for zig0, which is *only* used to build +// stage2, the self-hosted compiler, into an object file, which is then +// linked by the same build system (cmake) that linked this binary. + +#include "stage1.h" +#include "heap.hpp" +#include "stage2.h" +#include "target.hpp" +#include "error.hpp" + +#include +#include + +static int print_error_usage(const char *arg0) { + fprintf(stderr, "See `%s --help` for detailed usage information\n", arg0); + return EXIT_FAILURE; +} + +static int print_full_usage(const char *arg0, FILE *file, int return_code) { + fprintf(file, + "Usage: %s [options] builds an object file\n" + "\n" + "Options:\n" + " --color [auto|off|on] enable or disable colored error messages\n" + " --name [name] override output name\n" + " --output-dir [dir] override output directory (defaults to cwd)\n" + " --pkg-begin [name] [path] make pkg available to import and push current pkg\n" + " --pkg-end pop current pkg\n" + " --main-pkg-path set the directory of the root package\n" + " --release-fast build with optimizations on and safety off\n" + " --release-safe build with optimizations on and safety on\n" + " --release-small build with size optimizations on and safety off\n" + " --single-threaded source may assume it is only used single-threaded\n" + " -dynamic create a shared library (.so; .dll; .dylib)\n" + " --strip exclude debug symbols\n" + " -target [name] -- see the targets command\n" + " -mcpu [cpu] specify target CPU and feature set\n" + " --verbose-tokenize enable compiler debug output for tokenization\n" + " --verbose-ast enable compiler debug output for AST parsing\n" + " --verbose-link enable compiler debug output for linking\n" + " --verbose-ir enable compiler debug output for Zig IR\n" + " --verbose-llvm-ir enable compiler debug output for LLVM IR\n" + " --verbose-cimport enable compiler debug output for C imports\n" + " --verbose-cc enable compiler debug output for C compilation\n" + " --verbose-llvm-cpu-features enable compiler debug output for LLVM CPU features\n" + "\n" + , arg0); + return return_code; +} + +static bool str_starts_with(const char *s1, const char *s2) { + size_t s2_len = strlen(s2); + if (strlen(s1) < s2_len) { + return false; + } + return memcmp(s1, s2, s2_len) == 0; +} + +int main_exit(Stage2ProgressNode *root_progress_node, int exit_code) { + if (root_progress_node != nullptr) { + stage2_progress_end(root_progress_node); + } + return exit_code; +} + +int main(int argc, char **argv) { + zig_stage1_os_init(); + + char *arg0 = argv[0]; + Error err; + + const char *in_file = nullptr; + const char *output_dir = nullptr; + bool strip = false; + const char *out_name = nullptr; + bool verbose_tokenize = false; + bool verbose_ast = false; + bool verbose_link = false; + bool verbose_ir = false; + bool verbose_llvm_ir = false; + bool verbose_cimport = false; + bool verbose_cc = false; + bool verbose_llvm_cpu_features = false; + ErrColor color = ErrColorAuto; + const char *dynamic_linker = nullptr; + bool link_libc = false; + bool link_libcpp = false; + const char *target_string = nullptr; + ZigStage1Pkg *cur_pkg = heap::c_allocator.create(); + BuildMode build_mode = BuildModeDebug; + TargetSubsystem subsystem = TargetSubsystemAuto; + const char *override_lib_dir = nullptr; + const char *main_pkg_path = nullptr; + const char *mcpu = nullptr; + + for (int i = 1; i < argc; i += 1) { + char *arg = argv[i]; + + if (arg[0] == '-') { + if (strcmp(arg, "--") == 0) { + fprintf(stderr, "Unexpected end-of-parameter mark: %s\n", arg); + } else if (strcmp(arg, "--release-fast") == 0) { + build_mode = BuildModeFastRelease; + } else if (strcmp(arg, "--release-safe") == 0) { + build_mode = BuildModeSafeRelease; + } else if (strcmp(arg, "--release-small") == 0) { + build_mode = BuildModeSmallRelease; + } else if (strcmp(arg, "--help") == 0) { + return print_full_usage(arg0, stdout, EXIT_SUCCESS); + } else if (strcmp(arg, "--strip") == 0) { + strip = true; + } else if (strcmp(arg, "--verbose-tokenize") == 0) { + verbose_tokenize = true; + } else if (strcmp(arg, "--verbose-ast") == 0) { + verbose_ast = true; + } else if (strcmp(arg, "--verbose-link") == 0) { + verbose_link = true; + } else if (strcmp(arg, "--verbose-ir") == 0) { + verbose_ir = true; + } else if (strcmp(arg, "--verbose-llvm-ir") == 0) { + verbose_llvm_ir = true; + } else if (strcmp(arg, "--verbose-cimport") == 0) { + verbose_cimport = true; + } else if (strcmp(arg, "--verbose-cc") == 0) { + verbose_cc = true; + } else if (strcmp(arg, "--verbose-llvm-cpu-features") == 0) { + verbose_llvm_cpu_features = true; + } else if (arg[1] == 'l' && arg[2] != 0) { + // alias for --library + const char *l = &arg[2]; + if (strcmp(l, "c") == 0) { + link_libc = true; + } else if (strcmp(l, "c++") == 0 || strcmp(l, "stdc++") == 0) { + link_libcpp = true; + } + } else if (strcmp(arg, "--pkg-begin") == 0) { + if (i + 2 >= argc) { + fprintf(stderr, "Expected 2 arguments after --pkg-begin\n"); + return print_error_usage(arg0); + } + ZigStage1Pkg *new_cur_pkg = heap::c_allocator.create(); + i += 1; + new_cur_pkg->name_ptr = argv[i]; + new_cur_pkg->name_len = strlen(argv[i]); + i += 1; + new_cur_pkg->path_ptr = argv[i]; + new_cur_pkg->path_len = strlen(argv[i]); + new_cur_pkg->parent = cur_pkg; + cur_pkg->children_ptr = heap::c_allocator.reallocate(cur_pkg->children_ptr, + cur_pkg->children_len, cur_pkg->children_len + 1); + cur_pkg->children_ptr[cur_pkg->children_len] = new_cur_pkg; + cur_pkg->children_len += 1; + + cur_pkg = new_cur_pkg; + } else if (strcmp(arg, "--pkg-end") == 0) { + if (cur_pkg->parent == nullptr) { + fprintf(stderr, "Encountered --pkg-end with no matching --pkg-begin\n"); + return EXIT_FAILURE; + } + cur_pkg = cur_pkg->parent; + } else if (str_starts_with(arg, "-mcpu=")) { + mcpu = arg + strlen("-mcpu="); + } else if (i + 1 >= argc) { + fprintf(stderr, "Expected another argument after %s\n", arg); + return print_error_usage(arg0); + } else { + i += 1; + if (strcmp(arg, "--output-dir") == 0) { + output_dir = argv[i]; + } else if (strcmp(arg, "--color") == 0) { + if (strcmp(argv[i], "auto") == 0) { + color = ErrColorAuto; + } else if (strcmp(argv[i], "on") == 0) { + color = ErrColorOn; + } else if (strcmp(argv[i], "off") == 0) { + color = ErrColorOff; + } else { + fprintf(stderr, "--color options are 'auto', 'on', or 'off'\n"); + return print_error_usage(arg0); + } + } else if (strcmp(arg, "--name") == 0) { + out_name = argv[i]; + } else if (strcmp(arg, "--dynamic-linker") == 0) { + dynamic_linker = argv[i]; + } else if (strcmp(arg, "--override-lib-dir") == 0) { + override_lib_dir = argv[i]; + } else if (strcmp(arg, "--main-pkg-path") == 0) { + main_pkg_path = argv[i]; + } else if (strcmp(arg, "--library") == 0 || strcmp(arg, "-l") == 0) { + if (strcmp(argv[i], "c") == 0) { + link_libc = true; + } else if (strcmp(argv[i], "c++") == 0 || strcmp(argv[i], "stdc++") == 0) { + link_libcpp = true; + } + } else if (strcmp(arg, "-target") == 0) { + target_string = argv[i]; + } else if (strcmp(arg, "--subsystem") == 0) { + if (strcmp(argv[i], "console") == 0) { + subsystem = TargetSubsystemConsole; + } else if (strcmp(argv[i], "windows") == 0) { + subsystem = TargetSubsystemWindows; + } else if (strcmp(argv[i], "posix") == 0) { + subsystem = TargetSubsystemPosix; + } else if (strcmp(argv[i], "native") == 0) { + subsystem = TargetSubsystemNative; + } else if (strcmp(argv[i], "efi_application") == 0) { + subsystem = TargetSubsystemEfiApplication; + } else if (strcmp(argv[i], "efi_boot_service_driver") == 0) { + subsystem = TargetSubsystemEfiBootServiceDriver; + } else if (strcmp(argv[i], "efi_rom") == 0) { + subsystem = TargetSubsystemEfiRom; + } else if (strcmp(argv[i], "efi_runtime_driver") == 0) { + subsystem = TargetSubsystemEfiRuntimeDriver; + } else { + fprintf(stderr, "invalid: --subsystem %s\n" + "Options are:\n" + " console\n" + " windows\n" + " posix\n" + " native\n" + " efi_application\n" + " efi_boot_service_driver\n" + " efi_rom\n" + " efi_runtime_driver\n" + , argv[i]); + return EXIT_FAILURE; + } + } else if (strcmp(arg, "-mcpu") == 0) { + mcpu = argv[i]; + } else { + fprintf(stderr, "Invalid argument: %s\n", arg); + return print_error_usage(arg0); + } + } + } else if (!in_file) { + in_file = arg; + } else { + fprintf(stderr, "Unexpected extra parameter: %s\n", arg); + return print_error_usage(arg0); + } + } + + if (cur_pkg->parent != nullptr) { + fprintf(stderr, "Unmatched --pkg-begin\n"); + return EXIT_FAILURE; + } + + Stage2Progress *progress = stage2_progress_create(); + Stage2ProgressNode *root_progress_node = stage2_progress_start_root(progress, "", 0, 0); + if (color == ErrColorOff) stage2_progress_disable_tty(progress); + + ZigTarget target; + if ((err = target_parse_triple(&target, target_string, mcpu, dynamic_linker))) { + fprintf(stderr, "invalid target: %s\n", err_str(err)); + return print_error_usage(arg0); + } + + if (in_file == nullptr) { + fprintf(stderr, "missing zig file\n"); + return print_error_usage(arg0); + } + + if (out_name == nullptr) { + fprintf(stderr, "missing --name\n"); + return print_error_usage(arg0); + } + + ZigStage1 *stage1 = zig_stage1_create(build_mode, + main_pkg_path, (main_pkg_path == nullptr) ? 0 : strlen(main_pkg_path), + in_file, strlen(in_file), + override_lib_dir, strlen(override_lib_dir), &target, false, + root_progress_node); + + stage1->root_name_ptr = out_name; + stage1->root_name_len = strlen(out_name); + stage1->strip = strip; + stage1->verbose_tokenize = verbose_tokenize; + stage1->verbose_ast = verbose_ast; + stage1->verbose_link = verbose_link; + stage1->verbose_ir = verbose_ir; + stage1->verbose_llvm_ir = verbose_llvm_ir; + stage1->verbose_cimport = verbose_cimport; + stage1->verbose_cc = verbose_cc; + stage1->verbose_llvm_cpu_features = verbose_llvm_cpu_features; + stage1->output_dir_ptr = output_dir; + stage1->output_dir_len = strlen(output_dir); + stage1->root_pkg = cur_pkg; + stage1->err_color = color; + stage1->link_libc = link_libc; + stage1->link_libcpp = link_libcpp; + stage1->subsystem = subsystem; + stage1->pic = true; + stage1->emit_bin = true; + + zig_stage1_build_object(stage1); + + zig_stage1_destroy(stage1); + + return main_exit(root_progress_node, EXIT_SUCCESS); +} From a9b18023a4c33a0563fca2aa20a239e3ee38927a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 17 Sep 2020 23:27:24 -0700 Subject: [PATCH 061/210] stage2: implement --show-builtin This takes the place of `zig builtin`. This is an improvement over the command because now the generated source will correctly show LinkMode and OutputMode, whereas before it was always stuck as Static and Obj, respectively. --- BRANCH_TODO | 14 +- src-self-hosted/Compilation.zig | 219 +++++++++++++++++++++++++++++--- src-self-hosted/glibc.zig | 8 +- src-self-hosted/libunwind.zig | 4 +- src-self-hosted/link.zig | 5 +- src-self-hosted/link/Elf.zig | 2 +- src-self-hosted/main.zig | 43 ++++--- src/all_types.hpp | 2 - src/stage1.cpp | 2 - src/stage1.h | 2 - src/zig0.cpp | 10 -- 11 files changed, 247 insertions(+), 64 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 56f4661259..369cd26437 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,9 +1,5 @@ - * `zig builtin` * `zig translate-c` * `zig test` - * `zig run` - * `zig init-lib` - * `zig init-exe` * `zig build` * `-ftime-report` * -fstack-report print stack size diagnostics\n" @@ -15,14 +11,14 @@ * -femit-llvm-ir produce a .ll file with LLVM IR\n" * -fno-emit-llvm-ir (default) do not produce a .ll file with LLVM IR\n" * --cache-dir [path] override the local cache directory\n" - * move main.cpp to stage2 * make sure zig cc works - using it as a preprocessor (-E) - try building some software * support rpaths in ELF linker code * build & link against compiler-rt - stage1 C++ code integration - * build & link againstn freestanding libc + * repair @cImport + * build & link against freestanding libc * add CLI support for a way to pass extra flags to c source files * capture lld stdout/stderr better * musl @@ -41,6 +37,9 @@ * implement -fno-emit-bin * audit the base cache hash * audit the CLI options for stage2 + * `zig init-lib` + * `zig init-exe` + * `zig run` * implement serialization/deserialization of incremental compilation metadata * incremental compilation - implement detection of which source files changed @@ -69,3 +68,6 @@ * rename src-self-hosted/ to src/ * improve Directory.join to only use 1 allocation in a clean way. * tracy builds with lc++ + * some kind of "zig identifier escape" function rather than unconditionally using @"" syntax + in builtin.zig + * rename Mode to OptimizeMode diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 05f76f08f3..514ad1ed06 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -29,7 +29,7 @@ c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{}, link_error_flags: link.File.ErrorFlags = .{}, -work_queue: std.fifo.LinearFifo(WorkItem, .Dynamic), +work_queue: std.fifo.LinearFifo(Job, .Dynamic), /// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator. failed_c_objects: std.AutoArrayHashMapUnmanaged(*CObject, *ErrorMsg) = .{}, @@ -43,8 +43,9 @@ sanitize_c: bool, /// This is `true` for `zig cc`, `zig c++`, and `zig translate-c`. clang_passthrough_mode: bool, /// Whether to print clang argvs to stdout. -debug_cc: bool, +verbose_cc: bool, disable_c_depfile: bool, +is_test: bool, c_source_files: []const CSourceFile, clang_argv: []const []const u8, @@ -56,16 +57,16 @@ zig_cache_directory: Directory, libc_include_dir_list: []const []const u8, rand: *std.rand.Random, -/// Populated when we build libc++.a. A WorkItem to build this is placed in the queue +/// Populated when we build libc++.a. A Job to build this is placed in the queue /// and resolved before calling linker.flush(). libcxx_static_lib: ?[]const u8 = null, -/// Populated when we build libc++abi.a. A WorkItem to build this is placed in the queue +/// Populated when we build libc++abi.a. A Job to build this is placed in the queue /// and resolved before calling linker.flush(). libcxxabi_static_lib: ?[]const u8 = null, -/// Populated when we build libunwind.a. A WorkItem to build this is placed in the queue +/// Populated when we build libunwind.a. A Job to build this is placed in the queue /// and resolved before calling linker.flush(). libunwind_static_lib: ?CRTFile = null, -/// Populated when we build c.a. A WorkItem to build this is placed in the queue +/// Populated when we build c.a. A Job to build this is placed in the queue /// and resolved before calling linker.flush(). libc_static_lib: ?[]const u8 = null, @@ -98,7 +99,7 @@ pub const CSourceFile = struct { extra_flags: []const []const u8 = &[0][]const u8{}, }; -const WorkItem = union(enum) { +const Job = union(enum) { /// Write the machine code for a Decl to the output file. codegen_decl: *Module.Decl, /// The Decl needs to be analyzed and possibly export itself. @@ -116,8 +117,11 @@ const WorkItem = union(enum) { glibc_crt_file: glibc.CRTFile, /// all of the glibc shared objects glibc_shared_objects, - + /// libunwind.a, usually needed when linking libc libunwind: void, + + /// Generate builtin.zig source code and write it into the correct place. + generate_builtin_zig: void, }; pub const CObject = struct { @@ -282,8 +286,9 @@ pub const InitOptions = struct { linker_z_nodelete: bool = false, linker_z_defs: bool = false, clang_passthrough_mode: bool = false, - debug_cc: bool = false, - debug_link: bool = false, + verbose_cc: bool = false, + verbose_link: bool = false, + is_test: bool = false, stack_size_override: ?u64 = null, self_exe_path: ?[]const u8 = null, version: ?std.builtin.Version = null, @@ -570,6 +575,11 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { break :blk link_artifact_directory; }; + const error_return_tracing = !options.strip and switch (options.optimize_mode) { + .Debug, .ReleaseSafe => true, + .ReleaseFast, .ReleaseSmall => false, + }; + const bin_file = try link.File.openPath(gpa, .{ .directory = bin_directory, .sub_path = emit_bin.basename, @@ -612,9 +622,10 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .valgrind = valgrind, .stack_check = stack_check, .single_threaded = single_threaded, - .debug_link = options.debug_link, + .verbose_link = options.verbose_link, .machine_code_model = options.machine_code_model, .dll_export_fns = dll_export_fns, + .error_return_tracing = error_return_tracing, }); errdefer bin_file.destroy(); @@ -624,7 +635,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .zig_lib_directory = options.zig_lib_directory, .zig_cache_directory = options.zig_cache_directory, .bin_file = bin_file, - .work_queue = std.fifo.LinearFifo(WorkItem, .Dynamic).init(gpa), + .work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa), .keep_source_files_loaded = options.keep_source_files_loaded, .use_clang = use_clang, .clang_argv = options.clang_argv, @@ -635,14 +646,19 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .sanitize_c = sanitize_c, .rand = options.rand, .clang_passthrough_mode = options.clang_passthrough_mode, - .debug_cc = options.debug_cc, + .verbose_cc = options.verbose_cc, .disable_c_depfile = options.disable_c_depfile, .owned_link_dir = owned_link_dir, + .is_test = options.is_test, }; break :comp comp; }; errdefer comp.destroy(); + if (comp.bin_file.options.module) |mod| { + try comp.work_queue.writeItem(.{ .generate_builtin_zig = {} }); + } + // Add a `CObject` for each `c_source_files`. try comp.c_object_table.ensureCapacity(gpa, options.c_source_files.len); for (options.c_source_files) |c_source_file| { @@ -659,7 +675,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { // If we need to build glibc for the target, add work items for it. // We go through the work queue so that building can be done in parallel. if (comp.wantBuildGLibCFromSource()) { - try comp.addBuildingGLibCWorkItems(); + try comp.addBuildingGLibCJobs(); } if (comp.wantBuildLibUnwindFromSource()) { try comp.work_queue.writeItem(.{ .libunwind = {} }); @@ -715,7 +731,7 @@ pub fn update(self: *Compilation) !void { defer tracy.end(); // For compiling C objects, we rely on the cache hash system to avoid duplicating work. - // Add a WorkItem for each C object. + // Add a Job for each C object. try self.work_queue.ensureUnusedCapacity(self.c_object_table.items().len); for (self.c_object_table.items()) |entry| { self.work_queue.writeItemAssumeCapacity(.{ .c_object = entry.key }); @@ -974,6 +990,13 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { fatal("unable to build libunwind: {}", .{@errorName(err)}); }; }, + .generate_builtin_zig => { + // This Job is only queued up if there is a zig module. + self.updateBuiltinZigFile(self.bin_file.options.module.?) catch |err| { + // TODO Expose this as a normal compile error rather than crashing here. + fatal("unable to update builtin.zig file: {}", .{@errorName(err)}); + }; + }, }; } @@ -1057,7 +1080,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { try argv.append(c_object.src.src_path); try argv.appendSlice(c_object.src.extra_flags); - if (comp.debug_cc) { + if (comp.verbose_cc) { for (argv.items[0 .. argv.items.len - 1]) |arg| { std.debug.print("{} ", .{arg}); } @@ -1616,8 +1639,8 @@ pub fn get_libc_crt_file(comp: *Compilation, arena: *Allocator, basename: []cons return full_path; } -fn addBuildingGLibCWorkItems(comp: *Compilation) !void { - try comp.work_queue.write(&[_]WorkItem{ +fn addBuildingGLibCJobs(comp: *Compilation) !void { + try comp.work_queue.write(&[_]Job{ .{ .glibc_crt_file = .crti_o }, .{ .glibc_crt_file = .crtn_o }, .{ .glibc_crt_file = .scrt1_o }, @@ -1646,3 +1669,163 @@ fn wantBuildLibUnwindFromSource(comp: *Compilation) bool { return comp.bin_file.options.link_libc and is_exe_or_dyn_lib and comp.bin_file.options.libc_installation == null; } + +fn updateBuiltinZigFile(comp: *Compilation, mod: *Module) !void { + const source = try comp.generateBuiltinZigSource(); + defer comp.gpa.free(source); + try mod.zig_cache_artifact_directory.handle.writeFile("builtin.zig", source); +} + +pub fn generateBuiltinZigSource(comp: *Compilation) ![]u8 { + var buffer = std.ArrayList(u8).init(comp.gpa); + defer buffer.deinit(); + + const target = comp.getTarget(); + const generic_arch_name = target.cpu.arch.genericName(); + + @setEvalBranchQuota(4000); + try buffer.writer().print( + \\usingnamespace @import("std").builtin; + \\/// Deprecated + \\pub const arch = std.Target.current.cpu.arch; + \\/// Deprecated + \\pub const endian = std.Target.current.cpu.arch.endian(); + \\pub const output_mode = OutputMode.{}; + \\pub const link_mode = LinkMode.{}; + \\pub const is_test = {}; + \\pub const single_threaded = {}; + \\pub const abi = Abi.{}; + \\pub const cpu: Cpu = Cpu{{ + \\ .arch = .{}, + \\ .model = &Target.{}.cpu.{}, + \\ .features = Target.{}.featureSet(&[_]Target.{}.Feature{{ + \\ + , .{ + @tagName(comp.bin_file.options.output_mode), + @tagName(comp.bin_file.options.link_mode), + comp.is_test, + comp.bin_file.options.single_threaded, + @tagName(target.abi), + @tagName(target.cpu.arch), + generic_arch_name, + target.cpu.model.name, + generic_arch_name, + generic_arch_name, + }); + + for (target.cpu.arch.allFeaturesList()) |feature, index_usize| { + const index = @intCast(std.Target.Cpu.Feature.Set.Index, index_usize); + const is_enabled = target.cpu.features.isEnabled(index); + if (is_enabled) { + // TODO some kind of "zig identifier escape" function rather than + // unconditionally using @"" syntax + try buffer.appendSlice(" .@\""); + try buffer.appendSlice(feature.name); + try buffer.appendSlice("\",\n"); + } + } + + try buffer.writer().print( + \\ }}), + \\}}; + \\pub const os = Os{{ + \\ .tag = .{}, + \\ .version_range = .{{ + , + .{@tagName(target.os.tag)}, + ); + + switch (target.os.getVersionRange()) { + .none => try buffer.appendSlice(" .none = {} }\n"), + .semver => |semver| try buffer.outStream().print( + \\ .semver = .{{ + \\ .min = .{{ + \\ .major = {}, + \\ .minor = {}, + \\ .patch = {}, + \\ }}, + \\ .max = .{{ + \\ .major = {}, + \\ .minor = {}, + \\ .patch = {}, + \\ }}, + \\ }}}}, + \\ + , .{ + semver.min.major, + semver.min.minor, + semver.min.patch, + + semver.max.major, + semver.max.minor, + semver.max.patch, + }), + .linux => |linux| try buffer.outStream().print( + \\ .linux = .{{ + \\ .range = .{{ + \\ .min = .{{ + \\ .major = {}, + \\ .minor = {}, + \\ .patch = {}, + \\ }}, + \\ .max = .{{ + \\ .major = {}, + \\ .minor = {}, + \\ .patch = {}, + \\ }}, + \\ }}, + \\ .glibc = .{{ + \\ .major = {}, + \\ .minor = {}, + \\ .patch = {}, + \\ }}, + \\ }}}}, + \\ + , .{ + linux.range.min.major, + linux.range.min.minor, + linux.range.min.patch, + + linux.range.max.major, + linux.range.max.minor, + linux.range.max.patch, + + linux.glibc.major, + linux.glibc.minor, + linux.glibc.patch, + }), + .windows => |windows| try buffer.outStream().print( + \\ .windows = .{{ + \\ .min = {s}, + \\ .max = {s}, + \\ }}}}, + \\ + , + .{ windows.min, windows.max }, + ), + } + try buffer.appendSlice("};\n"); + try buffer.writer().print( + \\pub const object_format = ObjectFormat.{}; + \\pub const mode = Mode.{}; + \\pub const link_libc = {}; + \\pub const link_libcpp = {}; + \\pub const have_error_return_tracing = {}; + \\pub const valgrind_support = {}; + \\pub const position_independent_code = {}; + \\pub const strip_debug_info = {}; + \\pub const code_model = CodeModel.{}; + \\ + , .{ + @tagName(comp.bin_file.options.object_format), + @tagName(comp.bin_file.options.optimize_mode), + comp.bin_file.options.link_libc, + comp.bin_file.options.link_libcpp, + comp.bin_file.options.error_return_tracing, + comp.bin_file.options.valgrind, + comp.bin_file.options.pic, + comp.bin_file.options.strip, + @tagName(comp.bin_file.options.machine_code_model), + }); + return buffer.toOwnedSlice(); +} diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index 5bda249080..5e34302c06 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -711,8 +711,8 @@ fn build_crt_file( .is_native_os = comp.bin_file.options.is_native_os, .self_exe_path = comp.self_exe_path, .c_source_files = c_source_files, - .debug_cc = comp.debug_cc, - .debug_link = comp.bin_file.options.debug_link, + .verbose_cc = comp.verbose_cc, + .verbose_link = comp.bin_file.options.verbose_link, .clang_passthrough_mode = comp.clang_passthrough_mode, }); defer sub_compilation.destroy(); @@ -987,8 +987,8 @@ fn buildSharedLib( .strip = comp.bin_file.options.strip, .is_native_os = false, .self_exe_path = comp.self_exe_path, - .debug_cc = comp.debug_cc, - .debug_link = comp.bin_file.options.debug_link, + .verbose_cc = comp.verbose_cc, + .verbose_link = comp.bin_file.options.verbose_link, .clang_passthrough_mode = comp.clang_passthrough_mode, .version = version, .version_script = map_file_path, diff --git a/src-self-hosted/libunwind.zig b/src-self-hosted/libunwind.zig index 19f77da2dc..3ba52573ce 100644 --- a/src-self-hosted/libunwind.zig +++ b/src-self-hosted/libunwind.zig @@ -107,8 +107,8 @@ pub fn buildStaticLib(comp: *Compilation) !void { .is_native_os = comp.bin_file.options.is_native_os, .self_exe_path = comp.self_exe_path, .c_source_files = &c_source_files, - .debug_cc = comp.debug_cc, - .debug_link = comp.bin_file.options.debug_link, + .verbose_cc = comp.verbose_cc, + .verbose_link = comp.bin_file.options.verbose_link, .clang_passthrough_mode = comp.clang_passthrough_mode, .link_libc = true, }); diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 069a90872e..1ad293b275 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -61,8 +61,9 @@ pub const Options = struct { valgrind: bool, stack_check: bool, single_threaded: bool, - debug_link: bool = false, + verbose_link: bool = false, dll_export_fns: bool, + error_return_tracing: bool, gc_sections: ?bool = null, allow_shlib_undefined: ?bool = null, linker_script: ?[]const u8 = null, @@ -441,7 +442,7 @@ pub const File = struct { base.options.sub_path; const full_out_path_z = try arena.dupeZ(u8, full_out_path); - if (base.options.debug_link) { + if (base.options.verbose_link) { std.debug.print("ar rcs {}", .{full_out_path_z}); for (object_files.items) |arg| { std.debug.print(" {}", .{arg}); diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 5c2c73d740..62a5e486d0 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -1569,7 +1569,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { try argv.append("-Bsymbolic"); } - if (self.base.options.debug_link) { + if (self.base.options.verbose_link) { for (argv.items[0 .. argv.items.len - 1]) |arg| { std.debug.print("{} ", .{arg}); } diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index db68c867b2..77ca2b398e 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -179,6 +179,7 @@ const usage_build_generic = \\ --color [auto|off|on] Enable or disable colored error messages \\ -femit-bin[=path] (default) output machine code \\ -fno-emit-bin Do not output machine code + \\ --show-builtin Output the source of @import("builtin") then exit \\ \\Compile Options: \\ -target [name] -- see the targets command @@ -233,8 +234,8 @@ const usage_build_generic = \\ \\Debug Options (Zig Compiler Development): \\ -ftime-report Print timing diagnostics - \\ --debug-link Verbose linker invocation - \\ --debug-cc Verbose C compiler invocation + \\ --verbose-link Display linker invocations + \\ --verbose-cc Display C compiler invocations \\ ; @@ -274,9 +275,10 @@ pub fn buildOutputType( var strip = false; var single_threaded = false; var watch = false; - var debug_link = false; - var debug_cc = false; + var verbose_link = false; + var verbose_cc = false; var time_report = false; + var show_builtin = false; var emit_bin: Emit = .yes_default_path; var emit_zir: Emit = .no; var target_arch_os_abi: []const u8 = "native"; @@ -531,6 +533,8 @@ pub fn buildOutputType( dll_export_fns = true; } else if (mem.eql(u8, arg, "-fno-dll-export-fns")) { dll_export_fns = false; + } else if (mem.eql(u8, arg, "--show-builtin")) { + show_builtin = true; } else if (mem.eql(u8, arg, "--strip")) { strip = true; } else if (mem.eql(u8, arg, "--single-threaded")) { @@ -539,10 +543,10 @@ pub fn buildOutputType( link_eh_frame_hdr = true; } else if (mem.eql(u8, arg, "-Bsymbolic")) { linker_bind_global_refs_locally = true; - } else if (mem.eql(u8, arg, "--debug-link")) { - debug_link = true; - } else if (mem.eql(u8, arg, "--debug-cc")) { - debug_cc = true; + } else if (mem.eql(u8, arg, "--verbose-link")) { + verbose_link = true; + } else if (mem.eql(u8, arg, "--verbose-cc")) { + verbose_cc = true; } else if (mem.startsWith(u8, arg, "-T")) { linker_script = arg[2..]; } else if (mem.startsWith(u8, arg, "-L")) { @@ -680,8 +684,8 @@ pub fn buildOutputType( }, .linker_script => linker_script = it.only_arg, .verbose_cmds => { - debug_cc = true; - debug_link = true; + verbose_cc = true; + verbose_link = true; }, .for_linker => try linker_args.append(it.only_arg), .linker_input_z => { @@ -873,6 +877,8 @@ pub fn buildOutputType( } else if (emit_bin == .yes) { const basename = fs.path.basename(emit_bin.yes); break :blk mem.split(basename, ".").next().?; + } else if (show_builtin) { + break :blk "builtin"; } else { fatal("--name [name] not provided and unable to infer", .{}); } @@ -1160,17 +1166,20 @@ pub fn buildOutputType( .clang_passthrough_mode = arg_mode != .build, .version = if (have_version) version else null, .libc_installation = if (libc_installation) |*lci| lci else null, - .debug_cc = debug_cc, - .debug_link = debug_link, + .verbose_cc = verbose_cc, + .verbose_link = verbose_link, .machine_code_model = machine_code_model, }) catch |err| { fatal("unable to create compilation: {}", .{@errorName(err)}); }; defer comp.destroy(); - const stdin = std.io.getStdIn().inStream(); - const stderr = std.io.getStdErr().outStream(); - var repl_buf: [1024]u8 = undefined; + if (show_builtin) { + const source = try comp.generateBuiltinZigSource(); + defer comp.gpa.free(source); + try std.io.getStdOut().writeAll(source); + return; + } try updateModule(gpa, comp, zir_out_path); @@ -1179,6 +1188,10 @@ pub fn buildOutputType( fatal("TODO: implement `zig cc` when using it as a preprocessor", .{}); } + const stdin = std.io.getStdIn().inStream(); + const stderr = std.io.getStdErr().outStream(); + var repl_buf: [1024]u8 = undefined; + while (watch) { try stderr.print("🦎 ", .{}); if (output_mode == .Exe) { diff --git a/src/all_types.hpp b/src/all_types.hpp index 7b8ad57483..75a89d272f 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -2161,11 +2161,9 @@ struct CodeGen { bool have_err_ret_tracing; bool verbose_tokenize; bool verbose_ast; - bool verbose_link; bool verbose_ir; bool verbose_llvm_ir; bool verbose_cimport; - bool verbose_cc; bool verbose_llvm_cpu_features; bool error_during_imports; bool generate_error_name_table; diff --git a/src/stage1.cpp b/src/stage1.cpp index 417076aa99..36d59c0a05 100644 --- a/src/stage1.cpp +++ b/src/stage1.cpp @@ -104,11 +104,9 @@ void zig_stage1_build_object(struct ZigStage1 *stage1) { g->verbose_tokenize = stage1->verbose_tokenize; g->verbose_ast = stage1->verbose_ast; - g->verbose_link = stage1->verbose_link; g->verbose_ir = stage1->verbose_ir; g->verbose_llvm_ir = stage1->verbose_llvm_ir; g->verbose_cimport = stage1->verbose_cimport; - g->verbose_cc = stage1->verbose_cc; g->verbose_llvm_cpu_features = stage1->verbose_llvm_cpu_features; g->err_color = stage1->err_color; diff --git a/src/stage1.h b/src/stage1.h index b7144d8409..5b0c4a4df6 100644 --- a/src/stage1.h +++ b/src/stage1.h @@ -194,11 +194,9 @@ struct ZigStage1 { bool test_is_evented; bool verbose_tokenize; bool verbose_ast; - bool verbose_link; bool verbose_ir; bool verbose_llvm_ir; bool verbose_cimport; - bool verbose_cc; bool verbose_llvm_cpu_features; }; diff --git a/src/zig0.cpp b/src/zig0.cpp index ad90a17eb3..62378ff68c 100644 --- a/src/zig0.cpp +++ b/src/zig0.cpp @@ -44,11 +44,9 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) { " -mcpu [cpu] specify target CPU and feature set\n" " --verbose-tokenize enable compiler debug output for tokenization\n" " --verbose-ast enable compiler debug output for AST parsing\n" - " --verbose-link enable compiler debug output for linking\n" " --verbose-ir enable compiler debug output for Zig IR\n" " --verbose-llvm-ir enable compiler debug output for LLVM IR\n" " --verbose-cimport enable compiler debug output for C imports\n" - " --verbose-cc enable compiler debug output for C compilation\n" " --verbose-llvm-cpu-features enable compiler debug output for LLVM CPU features\n" "\n" , arg0); @@ -82,11 +80,9 @@ int main(int argc, char **argv) { const char *out_name = nullptr; bool verbose_tokenize = false; bool verbose_ast = false; - bool verbose_link = false; bool verbose_ir = false; bool verbose_llvm_ir = false; bool verbose_cimport = false; - bool verbose_cc = false; bool verbose_llvm_cpu_features = false; ErrColor color = ErrColorAuto; const char *dynamic_linker = nullptr; @@ -120,16 +116,12 @@ int main(int argc, char **argv) { verbose_tokenize = true; } else if (strcmp(arg, "--verbose-ast") == 0) { verbose_ast = true; - } else if (strcmp(arg, "--verbose-link") == 0) { - verbose_link = true; } else if (strcmp(arg, "--verbose-ir") == 0) { verbose_ir = true; } else if (strcmp(arg, "--verbose-llvm-ir") == 0) { verbose_llvm_ir = true; } else if (strcmp(arg, "--verbose-cimport") == 0) { verbose_cimport = true; - } else if (strcmp(arg, "--verbose-cc") == 0) { - verbose_cc = true; } else if (strcmp(arg, "--verbose-llvm-cpu-features") == 0) { verbose_llvm_cpu_features = true; } else if (arg[1] == 'l' && arg[2] != 0) { @@ -283,11 +275,9 @@ int main(int argc, char **argv) { stage1->strip = strip; stage1->verbose_tokenize = verbose_tokenize; stage1->verbose_ast = verbose_ast; - stage1->verbose_link = verbose_link; stage1->verbose_ir = verbose_ir; stage1->verbose_llvm_ir = verbose_llvm_ir; stage1->verbose_cimport = verbose_cimport; - stage1->verbose_cc = verbose_cc; stage1->verbose_llvm_cpu_features = verbose_llvm_cpu_features; stage1->output_dir_ptr = output_dir; stage1->output_dir_len = strlen(output_dir); From dc79651e6a46dc050976feabbb1b5e25a68dcf3f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Sep 2020 01:33:32 -0700 Subject: [PATCH 062/210] stage2: add CLI for `zig translate-c` --- BRANCH_TODO | 1 - CMakeLists.txt | 26 ++++------- src-self-hosted/Compilation.zig | 20 +++++---- src-self-hosted/main.zig | 77 ++++++++++++++++++++++++++++++--- src-self-hosted/translate_c.zig | 4 +- 5 files changed, 93 insertions(+), 35 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 369cd26437..9455511dbf 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ - * `zig translate-c` * `zig test` * `zig build` * `-ftime-report` diff --git a/CMakeLists.txt b/CMakeLists.txt index c8746338d4..d55c76ee9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -392,15 +392,18 @@ if(ZIG_TEST_COVERAGE) set(EXE_LDFLAGS "${EXE_LDFLAGS} -fprofile-arcs -ftest-coverage") endif() -add_library(zig_cpp STATIC ${ZIG_CPP_SOURCES}) +add_library(zig_cpp STATIC ${ZIG_SOURCES} ${ZIG_CPP_SOURCES}) set_target_properties(zig_cpp PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} ) target_link_libraries(zig_cpp LINK_PUBLIC + opt_c_util + ${SOFTFLOAT_LIBRARIES} ${CLANG_LIBRARIES} ${LLD_LIBRARIES} ${LLVM_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} ) if(ZIG_WORKAROUND_POLLY_SO) target_link_libraries(zig_cpp LINK_PUBLIC "-Wl,${ZIG_WORKAROUND_POLLY_SO}") @@ -411,27 +414,16 @@ set_target_properties(opt_c_util PROPERTIES COMPILE_FLAGS "${OPTIMIZED_C_FLAGS}" ) -add_library(zigcompiler STATIC ${ZIG_SOURCES}) -set_target_properties(zigcompiler PROPERTIES - COMPILE_FLAGS ${EXE_CFLAGS} - LINK_FLAGS ${EXE_LDFLAGS} -) -target_link_libraries(zigcompiler LINK_PUBLIC - zig_cpp - opt_c_util - ${SOFTFLOAT_LIBRARIES} - ${CMAKE_THREAD_LIBS_INIT} -) if(NOT MSVC) - target_link_libraries(zigcompiler LINK_PUBLIC ${LIBXML2}) + target_link_libraries(zig_cpp LINK_PUBLIC ${LIBXML2}) endif() if(ZIG_DIA_GUIDS_LIB) - target_link_libraries(zigcompiler LINK_PUBLIC ${ZIG_DIA_GUIDS_LIB}) + target_link_libraries(zig_cpp LINK_PUBLIC ${ZIG_DIA_GUIDS_LIB}) endif() if(MSVC OR MINGW) - target_link_libraries(zigcompiler LINK_PUBLIC version) + target_link_libraries(zig_cpp LINK_PUBLIC version) endif() add_executable(zig0 ${ZIG0_SOURCES}) @@ -439,7 +431,7 @@ set_target_properties(zig0 PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} LINK_FLAGS ${EXE_LDFLAGS} ) -target_link_libraries(zig0 zigcompiler) +target_link_libraries(zig0 zig_cpp) if(MSVC) set(ZIG1_OBJECT "${CMAKE_BINARY_DIR}/zig1.obj") @@ -493,7 +485,7 @@ set_target_properties(zig PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} LINK_FLAGS ${EXE_LDFLAGS} ) -target_link_libraries(zig zigcompiler "${ZIG1_OBJECT}") +target_link_libraries(zig "${ZIG1_OBJECT}" zig_cpp) if(MSVC) target_link_libraries(zig ntdll.lib) elseif(MINGW) diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 514ad1ed06..ed46508485 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -1005,7 +1005,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { defer tracy.end(); if (!build_options.have_llvm) { - return comp.failCObj(c_object, "clang not available: compiler not built with LLVM extensions enabled", .{}); + return comp.failCObj(c_object, "clang not available: compiler built without LLVM extensions", .{}); } const self_exe_path = comp.self_exe_path orelse return comp.failCObj(c_object, "clang compilation disabled", .{}); @@ -1081,10 +1081,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { try argv.appendSlice(c_object.src.extra_flags); if (comp.verbose_cc) { - for (argv.items[0 .. argv.items.len - 1]) |arg| { - std.debug.print("{} ", .{arg}); - } - std.debug.print("{}\n", .{argv.items[argv.items.len - 1]}); + dump_argv(argv.items); } const child = try std.ChildProcess.init(argv.items, arena); @@ -1190,7 +1187,7 @@ fn tmpFilePath(comp: *Compilation, arena: *Allocator, suffix: []const u8) error{ } /// Add common C compiler args between translate-c and C object compilation. -fn addCCArgs( +pub fn addCCArgs( comp: *Compilation, arena: *Allocator, argv: *std.ArrayList([]const u8), @@ -1671,12 +1668,19 @@ fn wantBuildLibUnwindFromSource(comp: *Compilation) bool { } fn updateBuiltinZigFile(comp: *Compilation, mod: *Module) !void { - const source = try comp.generateBuiltinZigSource(); + const source = try comp.generateBuiltinZigSource(comp.gpa); defer comp.gpa.free(source); try mod.zig_cache_artifact_directory.handle.writeFile("builtin.zig", source); } -pub fn generateBuiltinZigSource(comp: *Compilation) ![]u8 { +pub fn dump_argv(argv: []const []const u8) void { + for (argv[0 .. argv.len - 1]) |arg| { + std.debug.print("{} ", .{arg}); + } + std.debug.print("{}\n", .{argv[argv.len - 1]}); +} + +pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8 { var buffer = std.ArrayList(u8).init(comp.gpa); defer buffer.deinit(); diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 77ca2b398e..f18ccdd803 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -15,6 +15,7 @@ const build_options = @import("build_options"); const warn = std.log.warn; const introspect = @import("introspect.zig"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; +const translate_c = @import("translate_c.zig"); pub fn fatal(comptime format: []const u8, args: anytype) noreturn { std.log.emerg(format, args); @@ -864,6 +865,10 @@ pub fn buildOutputType( } } + if (arg_mode == .translate_c and c_source_files.items.len != 1) { + fatal("translate-c expects exactly 1 source file (found {})", .{c_source_files.items.len}); + } + const root_name = if (provided_name) |n| n else blk: { if (root_src_file) |file| { const basename = fs.path.basename(file); @@ -1175,10 +1180,10 @@ pub fn buildOutputType( defer comp.destroy(); if (show_builtin) { - const source = try comp.generateBuiltinZigSource(); - defer comp.gpa.free(source); - try std.io.getStdOut().writeAll(source); - return; + return std.io.getStdOut().writeAll(try comp.generateBuiltinZigSource(arena)); + } + if (arg_mode == .translate_c) { + return cmdTranslateC(comp, arena); } try updateModule(gpa, comp, zir_out_path); @@ -1248,6 +1253,63 @@ fn updateModule(gpa: *Allocator, comp: *Compilation, zir_out_path: ?[]const u8) } } +fn cmdTranslateC(comp: *Compilation, arena: *Allocator) !void { + if (!build_options.have_llvm) + fatal("cannot translate-c: compiler built without LLVM extensions", .{}); + + assert(comp.c_source_files.len == 1); + + var argv = std.ArrayList([]const u8).init(arena); + + const c_source_file = comp.c_source_files[0]; + const file_ext = Compilation.classifyFileExt(c_source_file.src_path); + try comp.addCCArgs(arena, &argv, file_ext, true, null); + try argv.append(c_source_file.src_path); + + if (comp.verbose_cc) { + std.debug.print("clang ", .{}); + Compilation.dump_argv(argv.items); + } + + // Convert to null terminated args. + const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1); + new_argv_with_sentinel[argv.items.len] = null; + const new_argv = new_argv_with_sentinel[0..argv.items.len :null]; + for (argv.items) |arg, i| { + new_argv[i] = try arena.dupeZ(u8, arg); + } + + const c_headers_dir_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"include"}); + const c_headers_dir_path_z = try arena.dupeZ(u8, c_headers_dir_path); + var clang_errors: []translate_c.ClangErrMsg = &[0]translate_c.ClangErrMsg{}; + const tree = translate_c.translate( + comp.gpa, + new_argv.ptr, + new_argv.ptr + new_argv.len, + &clang_errors, + c_headers_dir_path_z, + ) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ASTUnitFailure => fatal("clang API returned errors but due to a clang bug, it is not exposing the errors for zig to see. For more details: https://github.com/ziglang/zig/issues/4455", .{}), + error.SemanticAnalyzeFail => { + for (clang_errors) |clang_err| { + std.debug.print("{}:{}:{}: {}\n", .{ + if (clang_err.filename_ptr) |p| p[0..clang_err.filename_len] else "(no file)", + clang_err.line + 1, + clang_err.column + 1, + clang_err.msg_ptr[0..clang_err.msg_len], + }); + } + process.exit(1); + }, + }; + defer tree.deinit(); + + var bos = io.bufferedOutStream(io.getStdOut().writer()); + _ = try std.zig.render(comp.gpa, bos.writer(), tree); + try bos.flush(); +} + pub const usage_libc = \\Usage: zig libc \\ @@ -1401,8 +1463,9 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { process.exit(code); } - const stdout = io.getStdOut().outStream(); - _ = try std.zig.render(gpa, stdout, tree); + var bos = io.bufferedOutStream(io.getStdOut().writer()); + _ = try std.zig.render(gpa, bos.writer(), tree); + try bos.flush(); return; } @@ -1644,7 +1707,7 @@ extern "c" fn ZigClang_main(argc: c_int, argv: [*:null]?[*:0]u8) c_int; /// TODO https://github.com/ziglang/zig/issues/3257 fn punt_to_clang(arena: *Allocator, args: []const []const u8) error{OutOfMemory} { if (!build_options.have_llvm) - fatal("`zig cc` and `zig c++` unavailable: compiler not built with LLVM extensions enabled", .{}); + fatal("`zig cc` and `zig c++` unavailable: compiler built without LLVM extensions", .{}); // Convert the args to the format Clang expects. const argv = try arena.alloc(?[*:0]u8, args.len + 1); for (args) |arg, i| { diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 1cdb5099e1..c5c5231ca2 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -1,5 +1,5 @@ -// This is the userland implementation of translate-c which is used by both stage1 -// and stage2. +//! This is the userland implementation of translate-c which is used by both stage1 +//! and stage2. const std = @import("std"); const assert = std.debug.assert; From 333b12a8f9beb5864fa05ce413c5b935e925ec14 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Sep 2020 01:58:16 -0700 Subject: [PATCH 063/210] std: start: use std.log instead of stderr When main returns an error code. --- lib/std/start.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/start.zig b/lib/std/start.zig index aea31a1531..71940b12ca 100644 --- a/lib/std/start.zig +++ b/lib/std/start.zig @@ -224,7 +224,7 @@ inline fn initEventLoopAndCallMain() u8 { if (std.event.Loop.instance) |loop| { if (!@hasDecl(root, "event_loop")) { loop.init() catch |err| { - std.debug.warn("error: {}\n", .{@errorName(err)}); + std.log.err("{}", .{@errorName(err)}); if (@errorReturnTrace()) |trace| { std.debug.dumpStackTrace(trace.*); } @@ -270,7 +270,7 @@ pub fn callMain() u8 { }, .ErrorUnion => { const result = root.main() catch |err| { - std.debug.warn("error: {}\n", .{@errorName(err)}); + std.log.err("{}", .{@errorName(err)}); if (@errorReturnTrace()) |trace| { std.debug.dumpStackTrace(trace.*); } From ea6181aaf6c3e22ced8b8ac202c5fc93a8e90674 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Fri, 11 Sep 2020 13:02:06 +1000 Subject: [PATCH 064/210] zig fmt: Add test for nesting if expressions --- lib/std/zig/parser_test.zig | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 6b8734c9d4..fe32a371e9 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3392,6 +3392,43 @@ test "zig fmt: Indent comma correctly after multiline string literals in arg lis ); } +test "zig fmt: Control flow statement as body of blockless if" { + try testCanonical( + \\pub fn main() void { + \\ const zoom_node = if (focused_node == layout_first) + \\ if (it.next()) { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ } else null + \\ else + \\ focused_node; + \\ + \\ const zoom_node = if (focused_node == layout_first) while (it.next()) |node| { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ } else null else + \\ focused_node; + \\ + \\ const zoom_node = if (focused_node == layout_first) + \\ if (it.next()) { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ } else null; + \\ + \\ const zoom_node = if (focused_node == layout_first) while (it.next()) |node| { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ }; + \\ + \\ const zoom_node = if (focused_node == layout_first) for (nodes) |node| { + \\ break node; + \\ }; + \\ + \\ const zoom_node = if (focused_node == layout_first) switch (nodes) { + \\ 0 => 0, + \\ } else + \\ focused_node; + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; From 9f0821e68836a495c726e0aae13e62c5235c5446 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Sun, 30 Aug 2020 15:02:05 +1000 Subject: [PATCH 065/210] zig fmt: Fix erroneously commented out code, add passing test case to close #5722 --- lib/std/zig/parser_test.zig | 15 +++++++++++++++ lib/std/zig/render.zig | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index fe32a371e9..50208582dc 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3429,6 +3429,21 @@ test "zig fmt: Control flow statement as body of blockless if" { ); } +test "zig fmt: " { + try testCanonical( + \\pub fn sendViewTags(self: Self) void { + \\ var it = ViewStack(View).iterator(self.output.views.first, std.math.maxInt(u32)); + \\ while (it.next()) |node| + \\ view_tags.append(node.view.current_tags) catch { + \\ c.wl_resource_post_no_memory(self.wl_resource); + \\ log.crit(.river_status, "out of memory", .{}); + \\ return; + \\ }; + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 4432d08787..522be107b0 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -1762,7 +1762,7 @@ fn renderExpression( } if (while_node.payload) |payload| { - const payload_space = Space.Space; //if (while_node.continue_expr != null) Space.Space else block_start_space; + const payload_space = if (while_node.continue_expr != null) Space.Space else block_start_space; try renderExpression(allocator, ais, tree, payload, payload_space); } From e1bd27119220c59211509f65c39fbb89c69b939b Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Sun, 30 Aug 2020 18:25:04 +1000 Subject: [PATCH 066/210] zig fmt: Allow trailing comments to do manual array formatting. close #5948 --- lib/std/zig/parser_test.zig | 44 ++++++- lib/std/zig/render.zig | 229 ++++++++++++++++++++++-------------- 2 files changed, 182 insertions(+), 91 deletions(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 50208582dc..20cafed5d3 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -1301,8 +1301,10 @@ test "zig fmt: array literal with hint" { \\const a = []u8{ \\ 1, 2, \\ 3, 4, - \\ 5, 6, // blah - \\ 7, 8, + \\ 5, + \\ 6, // blah + \\ 7, + \\ 8, \\}; \\const a = []u8{ \\ 1, 2, @@ -3444,6 +3446,44 @@ test "zig fmt: " { ); } +test "zig fmt: allow trailing line comments to do manual array formatting" { + try testCanonical( + \\fn foo() void { + \\ self.code.appendSliceAssumeCapacity(&[_]u8{ + \\ 0x55, // push rbp + \\ 0x48, 0x89, 0xe5, // mov rbp, rsp + \\ 0x48, 0x81, 0xec, // sub rsp, imm32 (with reloc) + \\ }); + \\ + \\ di_buf.appendAssumeCapacity(&[_]u8{ + \\ 1, DW.TAG_compile_unit, DW.CHILDREN_no, // header + \\ DW.AT_stmt_list, DW_FORM_data4, // form value pairs + \\ DW.AT_low_pc, DW_FORM_addr, + \\ DW.AT_high_pc, DW_FORM_addr, + \\ DW.AT_name, DW_FORM_strp, + \\ DW.AT_comp_dir, DW_FORM_strp, + \\ DW.AT_producer, DW_FORM_strp, + \\ DW.AT_language, DW_FORM_data2, + \\ 0, 0, // sentinel + \\ }); + \\ + \\ self.code.appendSliceAssumeCapacity(&[_]u8{ + \\ 0x55, // push rbp + \\ 0x48, 0x89, 0xe5, // mov rbp, rsp + \\ // How do we handle this? + \\ //0x48, 0x81, 0xec, // sub rsp, imm32 (with reloc) + \\ // Here's a blank line, should that be allowed? + \\ + \\ 0x48, 0x89, 0xe5, + \\ 0x33, 0x45, + \\ // Now the comment breaks a single line -- how do we handle this? + \\ 0x88, + \\ }); + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 522be107b0..3594cd5ca9 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -523,11 +523,11 @@ fn renderExpression( }; { - try ais.pushIndent(); + ais.pushIndent(); defer ais.popIndent(); try renderToken(tree, ais, infix_op_node.op_token, after_op_space); } - try ais.pushIndentOneShot(); + ais.pushIndentOneShot(); return renderExpression(allocator, ais, tree, infix_op_node.rhs, space); }, @@ -746,109 +746,130 @@ fn renderExpression( } // scan to find row size - const maybe_row_size: ?usize = blk: { - var count: usize = 1; - for (exprs) |expr, i| { - if (i + 1 < exprs.len) { - const expr_last_token = expr.lastToken() + 1; - const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, exprs[i + 1].firstToken()); - if (loc.line != 0) break :blk count; - count += 1; - } else { - const expr_last_token = expr.lastToken(); - const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, rtoken); - if (loc.line == 0) { - // all on one line - const src_has_trailing_comma = trailblk: { - const maybe_comma = tree.prevToken(rtoken); - break :trailblk tree.token_ids[maybe_comma] == .Comma; - }; - if (src_has_trailing_comma) { - break :blk 1; // force row size 1 - } else { - break :blk null; // no newlines - } - } - break :blk count; - } - } - unreachable; - }; - - if (maybe_row_size) |row_size| { - // A place to store the width of each expression and its column's maximum - var widths = try allocator.alloc(usize, exprs.len + row_size); - defer allocator.free(widths); - mem.set(usize, widths, 0); - - var expr_widths = widths[0 .. widths.len - row_size]; - var column_widths = widths[widths.len - row_size ..]; - - // Null ais for counting the printed length of each expression - var counting_stream = std.io.countingOutStream(std.io.null_out_stream); - var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, counting_stream.writer()); - - for (exprs) |expr, i| { - counting_stream.bytes_written = 0; - try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); - const width = @intCast(usize, counting_stream.bytes_written); - const col = i % row_size; - column_widths[col] = std.math.max(column_widths[col], width); - expr_widths[i] = width; - } - + if (rowSize(tree, exprs, rtoken, false) != null) { { ais.pushIndentNextLine(); defer ais.popIndent(); try renderToken(tree, ais, lbrace, Space.Newline); - var col: usize = 1; - for (exprs) |expr, i| { - if (i + 1 < exprs.len) { - const next_expr = exprs[i + 1]; - try renderExpression(allocator, ais, tree, expr, Space.None); + var expr_index: usize = 0; + while (rowSize(tree, exprs[expr_index..], rtoken, true)) |row_size| { + const row_exprs = exprs[expr_index..]; + // A place to store the width of each expression and its column's maximum + var widths = try allocator.alloc(usize, row_exprs.len + row_size); + defer allocator.free(widths); + mem.set(usize, widths, 0); - const comma = tree.nextToken(expr.*.lastToken()); + var expr_widths = widths[0 .. widths.len - row_size]; + var column_widths = widths[widths.len - row_size ..]; - if (col != row_size) { - try renderToken(tree, ais, comma, Space.Space); // , + // Null stream for counting the printed length of each expression + var counting_stream = std.io.countingOutStream(std.io.null_out_stream); + var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, counting_stream.writer()); - const padding = column_widths[i % row_size] - expr_widths[i]; - try ais.writer().writeByteNTimes(' ', padding); - - col += 1; - continue; + // Find next row with trailing comment (if any) to end the current section then + var section_end = sec_end: { + var this_line_first_expr: usize = 0; + var this_line_size = rowSize(tree, row_exprs, rtoken, true); + for (row_exprs) |expr, i| { + // Ignore comment on first line of this section + if (i == 0 or tree.tokensOnSameLine(row_exprs[0].firstToken(), expr.lastToken())) continue; + // Track start of line containing comment + if (!tree.tokensOnSameLine(row_exprs[this_line_first_expr].firstToken(), expr.lastToken())) { + this_line_first_expr = i; + this_line_size = rowSize(tree, row_exprs[this_line_first_expr..], rtoken, true); + } + if (expr.lastToken() + 2 < tree.token_ids.len) { + if (tree.token_ids[expr.lastToken() + 1] == .Comma and + tree.token_ids[expr.lastToken() + 2] == .LineComment and + tree.tokensOnSameLine(expr.lastToken(), expr.lastToken() + 2)) + { + var comment_token_loc = tree.token_locs[expr.lastToken() + 2]; + const comment_is_empty = mem.trimRight(u8, tree.tokenSliceLoc(comment_token_loc), " ").len == 2; + if (!comment_is_empty) { + // Found row ending in comment + break :sec_end i - this_line_size.? + 1; + } + } + } } - col = 1; + break :sec_end row_exprs.len; + }; + expr_index += section_end; - if (tree.token_ids[tree.nextToken(comma)] != .MultilineStringLiteralLine) { - try renderToken(tree, ais, comma, Space.Newline); // , - } else { - try renderToken(tree, ais, comma, Space.None); // , - } + const section_exprs = row_exprs[0..section_end]; - try renderExtraNewline(tree, ais, next_expr); - } else { - try renderExpression(allocator, ais, tree, expr, Space.Comma); // , + // Calculate size of columns in current section + for (section_exprs) |expr, i| { + counting_stream.bytes_written = 0; + try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); + const width = @intCast(usize, counting_stream.bytes_written); + const col = i % row_size; + column_widths[col] = std.math.max(column_widths[col], width); + expr_widths[i] = width; + } + + // Render exprs in current section + var col: usize = 1; + for (section_exprs) |expr, i| { + if (i + 1 < section_exprs.len) { + const next_expr = section_exprs[i + 1]; + try renderExpression(allocator, ais, tree, expr, Space.None); + + const comma = tree.nextToken(expr.*.lastToken()); + + if (col != row_size) { + try renderToken(tree, ais, comma, Space.Space); // , + + const padding = column_widths[i % row_size] - expr_widths[i]; + try ais.writer().writeByteNTimes(' ', padding); + + col += 1; + continue; + } + col = 1; + + if (tree.token_ids[tree.nextToken(comma)] != .MultilineStringLiteralLine) { + try renderToken(tree, ais, comma, Space.Newline); // , + } else { + try renderToken(tree, ais, comma, Space.None); // , + } + + try renderExtraNewline(tree, ais, next_expr); + } else { + const maybe_comma = tree.nextToken(expr.*.lastToken()); + if (tree.token_ids[maybe_comma] == .Comma) { + try renderExpression(allocator, ais, tree, expr, Space.None); // , + try renderToken(tree, ais, maybe_comma, Space.Newline); // , + } else { + try renderExpression(allocator, ais, tree, expr, Space.Comma); // , + } + } + } + + if (expr_index == exprs.len) { + break; } - } - } - return renderToken(tree, ais, rtoken, space); - } else { - try renderToken(tree, ais, lbrace, Space.Space); - for (exprs) |expr, i| { - if (i + 1 < exprs.len) { - const next_expr = exprs[i + 1]; - try renderExpression(allocator, ais, tree, expr, Space.None); - const comma = tree.nextToken(expr.*.lastToken()); - try renderToken(tree, ais, comma, Space.Space); // , - } else { - try renderExpression(allocator, ais, tree, expr, Space.Space); } } return renderToken(tree, ais, rtoken, space); } + + // Single line + try renderToken(tree, ais, lbrace, Space.Space); + for (exprs) |expr, i| { + if (i + 1 < exprs.len) { + const next_expr = exprs[i + 1]; + try renderExpression(allocator, ais, tree, expr, Space.None); + const comma = tree.nextToken(expr.*.lastToken()); + try renderToken(tree, ais, comma, Space.Space); // , + } else { + try renderExpression(allocator, ais, tree, expr, Space.Space); + } + } + + return renderToken(tree, ais, rtoken, space); }, .StructInitializer, .StructInitializerDot => { @@ -1879,7 +1900,7 @@ fn renderExpression( const after_rparen_space = if (if_node.payload == null) Space.Newline else Space.Space; { - try ais.pushIndent(); + ais.pushIndent(); defer ais.popIndent(); try renderToken(tree, ais, rparen, after_rparen_space); // ) } @@ -2567,3 +2588,33 @@ fn copyFixingWhitespace(ais: anytype, slice: []const u8) @TypeOf(ais.*).Error!vo else => try ais.writer().writeByte(byte), }; } + +fn rowSize(tree: *ast.Tree, exprs: []*ast.Node, rtoken: ast.TokenIndex, force: bool) ?usize { + var count: usize = 1; + for (exprs) |expr, i| { + if (i + 1 < exprs.len) { + const expr_last_token = expr.lastToken() + 1; + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, exprs[i + 1].firstToken()); + if (loc.line != 0) return count; + count += 1; + } else { + if (force) return count; + const expr_last_token = expr.lastToken(); + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, rtoken); + if (loc.line == 0) { + // all on one line + const src_has_trailing_comma = trailblk: { + const maybe_comma = tree.prevToken(rtoken); + break :trailblk tree.token_ids[maybe_comma] == .Comma; + }; + if (src_has_trailing_comma) { + return 1; // force row size 1 + } else { + return null; // no newlines + } + } + return count; + } + } + unreachable; +} From 291482a0310312fa3d84b3a967fa3f2d5b71b165 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Mon, 9 Mar 2020 14:04:31 +1100 Subject: [PATCH 067/210] zig fmt: Don't consider width of expressions containing multiline string literals when calculating padding for array initializers. fixes #3739 Changes some of the special casing for multiline string literals. --- lib/std/zig/ast.zig | 9 +++ lib/std/zig/parser_test.zig | 59 +++++++++++++++ lib/std/zig/render.zig | 145 +++++++++++++++++++++--------------- 3 files changed, 155 insertions(+), 58 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 404e8c413a..d8943adde0 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -823,6 +823,15 @@ pub const Node = struct { } } + pub fn findFirstWithId(self: *Node, id: Id) ?*Node { + if (self.id == id) return self; + var child_i: usize = 0; + while (self.iterate(child_i)) |child| : (child_i += 1) { + if (child.findFirstWithId(id)) |result| return result; + } + return null; + } + pub fn dump(self: *Node, indent: usize) void { { var i: usize = 0; diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 20cafed5d3..78443afe7a 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3484,6 +3484,65 @@ test "zig fmt: allow trailing line comments to do manual array formatting" { ); } +test "zig fmt: multiline string literals should play nice with array initializers" { + try testCanonical( + \\fn main() void { + \\ var a = .{.{.{.{.{.{.{.{ + \\ 0, + \\ }}}}}}}}; + \\ myFunc(.{ + \\ "aaaaaaa", "bbbbbb", "ccccc", + \\ "dddd", ("eee"), ("fff"), + \\ ("gggg"), + \\ // Line comment + \\ \\Multiline String Literals can be quite long + \\ , + \\ \\Multiline String Literals can be quite long + \\ \\Multiline String Literals can be quite long + \\ , + \\ \\Multiline String Literals can be quite long + \\ \\Multiline String Literals can be quite long + \\ \\Multiline String Literals can be quite long + \\ \\Multiline String Literals can be quite long + \\ , + \\ ( + \\ \\Multiline String Literals can be quite long + \\ ), + \\ .{ + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ }, + \\ .{( + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ )}, + \\ .{ "xxxxxxx", "xxx", ( + \\ \\ xxx + \\ ), "xxx", "xxx" }, + \\ .{ "xxxxxxx", "xxx", "xxx", "xxx" }, .{ "xxxxxxx", "xxx", "xxx", "xxx" }, + \\ "aaaaaaa", "bbbbbb", "ccccc", // - + \\ "dddd", ("eee"), ("fff"), + \\ .{ + \\ "xxx", "xxx", + \\ ( + \\ \\ xxx + \\ ), + \\ "xxxxxxxxxxxxxx", "xxx", + \\ }, + \\ .{ + \\ ( + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ ), + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ }, + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ }); + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 3594cd5ca9..b2687ada98 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -714,37 +714,24 @@ fn renderExpression( .node => |node| tree.nextToken(node.lastToken()), }; - if (exprs.len == 0) { - switch (lhs) { - .dot => |dot| try renderToken(tree, ais, dot, Space.None), - .node => |node| try renderExpression(allocator, ais, tree, node, Space.None), - } - - { - ais.pushIndent(); - defer ais.popIndent(); - try renderToken(tree, ais, lbrace, Space.None); - } - - return renderToken(tree, ais, rtoken, space); - } - if (exprs.len == 1 and tree.token_ids[exprs[0].*.lastToken() + 1] == .RBrace) { - const expr = exprs[0]; - - switch (lhs) { - .dot => |dot| try renderToken(tree, ais, dot, Space.None), - .node => |node| try renderExpression(allocator, ais, tree, node, Space.None), - } - try renderToken(tree, ais, lbrace, Space.None); - try renderExpression(allocator, ais, tree, expr, Space.None); - return renderToken(tree, ais, rtoken, space); - } - switch (lhs) { .dot => |dot| try renderToken(tree, ais, dot, Space.None), .node => |node| try renderExpression(allocator, ais, tree, node, Space.None), } + if (exprs.len == 0) { + try renderToken(tree, ais, lbrace, Space.None); + return renderToken(tree, ais, rtoken, space); + } + + if (exprs.len == 1 and exprs[0].tag != .MultilineStringLiteral and tree.token_ids[exprs[0].*.lastToken() + 1] == .RBrace) { + const expr = exprs[0]; + + try renderToken(tree, ais, lbrace, Space.None); + try renderExpression(allocator, ais, tree, expr, Space.None); + return renderToken(tree, ais, rtoken, space); + } + // scan to find row size if (rowSize(tree, exprs, rtoken, false) != null) { { @@ -763,11 +750,7 @@ fn renderExpression( var expr_widths = widths[0 .. widths.len - row_size]; var column_widths = widths[widths.len - row_size ..]; - // Null stream for counting the printed length of each expression - var counting_stream = std.io.countingOutStream(std.io.null_out_stream); - var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, counting_stream.writer()); - - // Find next row with trailing comment (if any) to end the current section then + // Find next row with trailing comment (if any) to end the current section var section_end = sec_end: { var this_line_first_expr: usize = 0; var this_line_size = rowSize(tree, row_exprs, rtoken, true); @@ -779,12 +762,15 @@ fn renderExpression( this_line_first_expr = i; this_line_size = rowSize(tree, row_exprs[this_line_first_expr..], rtoken, true); } - if (expr.lastToken() + 2 < tree.token_ids.len) { - if (tree.token_ids[expr.lastToken() + 1] == .Comma and - tree.token_ids[expr.lastToken() + 2] == .LineComment and - tree.tokensOnSameLine(expr.lastToken(), expr.lastToken() + 2)) + + const maybe_comma = expr.lastToken() + 1; + const maybe_comment = expr.lastToken() + 2; + if (maybe_comment < tree.token_ids.len) { + if (tree.token_ids[maybe_comma] == .Comma and + tree.token_ids[maybe_comment] == .LineComment and + tree.tokensOnSameLine(expr.lastToken(), maybe_comment)) { - var comment_token_loc = tree.token_locs[expr.lastToken() + 2]; + var comment_token_loc = tree.token_locs[maybe_comment]; const comment_is_empty = mem.trimRight(u8, tree.tokenSliceLoc(comment_token_loc), " ").len == 2; if (!comment_is_empty) { // Found row ending in comment @@ -799,18 +785,56 @@ fn renderExpression( const section_exprs = row_exprs[0..section_end]; + // Null stream for counting the printed length of each expression + var line_find_stream = std.io.findByteOutStream('\n', std.io.null_out_stream); + var counting_stream = std.io.countingOutStream(line_find_stream.writer()); + var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, counting_stream.writer()); + // Calculate size of columns in current section + var c: usize = 0; + var single_line = true; for (section_exprs) |expr, i| { - counting_stream.bytes_written = 0; - try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); - const width = @intCast(usize, counting_stream.bytes_written); - const col = i % row_size; - column_widths[col] = std.math.max(column_widths[col], width); - expr_widths[i] = width; + if (i + 1 < section_exprs.len) { + counting_stream.bytes_written = 0; + line_find_stream.byte_found = false; + try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); + const width = @intCast(usize, counting_stream.bytes_written); + expr_widths[i] = width; + + if (!line_find_stream.byte_found) { + const col = c % row_size; + column_widths[col] = std.math.max(column_widths[col], width); + + const expr_last_token = expr.*.lastToken() + 1; + const next_expr = section_exprs[i + 1]; + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].start, next_expr.*.firstToken()); + if (loc.line == 0) { + c += 1; + } else { + single_line = false; + c = 0; + } + } else { + single_line = false; + c = 0; + } + } else { + counting_stream.bytes_written = 0; + try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); + const width = @intCast(usize, counting_stream.bytes_written); + expr_widths[i] = width; + + if (!line_find_stream.byte_found) { + const col = c % row_size; + column_widths[col] = std.math.max(column_widths[col], width); + } + break; + } } // Render exprs in current section - var col: usize = 1; + c = 0; + var last_col_index: usize = row_size - 1; for (section_exprs) |expr, i| { if (i + 1 < section_exprs.len) { const next_expr = section_exprs[i + 1]; @@ -818,23 +842,28 @@ fn renderExpression( const comma = tree.nextToken(expr.*.lastToken()); - if (col != row_size) { + if (c != last_col_index) { + line_find_stream.byte_found = false; + try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); + try renderExpression(allocator, &auto_indenting_stream, tree, next_expr, Space.None); + if (!line_find_stream.byte_found) { + // Neither the current or next expression is multiline + try renderToken(tree, ais, comma, Space.Space); // , + assert(column_widths[c % row_size] >= expr_widths[i]); + const padding = column_widths[c % row_size] - expr_widths[i]; + try ais.writer().writeByteNTimes(' ', padding); + + c += 1; + continue; + } + } + if (single_line) { try renderToken(tree, ais, comma, Space.Space); // , - - const padding = column_widths[i % row_size] - expr_widths[i]; - try ais.writer().writeByteNTimes(' ', padding); - - col += 1; continue; } - col = 1; - - if (tree.token_ids[tree.nextToken(comma)] != .MultilineStringLiteralLine) { - try renderToken(tree, ais, comma, Space.Newline); // , - } else { - try renderToken(tree, ais, comma, Space.None); // , - } + c = 0; + try renderToken(tree, ais, comma, Space.Newline); // , try renderExtraNewline(tree, ais, next_expr); } else { const maybe_comma = tree.nextToken(expr.*.lastToken()); @@ -2594,13 +2623,13 @@ fn rowSize(tree: *ast.Tree, exprs: []*ast.Node, rtoken: ast.TokenIndex, force: b for (exprs) |expr, i| { if (i + 1 < exprs.len) { const expr_last_token = expr.lastToken() + 1; - const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, exprs[i + 1].firstToken()); + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].start, exprs[i + 1].firstToken()); if (loc.line != 0) return count; count += 1; } else { if (force) return count; const expr_last_token = expr.lastToken(); - const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, rtoken); + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].start, rtoken); if (loc.line == 0) { // all on one line const src_has_trailing_comma = trailblk: { From 206a8cf6709df51213252b03bf3ba4a9b8b52b6f Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Wed, 9 Sep 2020 21:45:05 +1000 Subject: [PATCH 068/210] zig fmt: fix comments and multiline literals in function args --- lib/std/zig/parser_test.zig | 40 +++++++++++++++++++++++ lib/std/zig/render.zig | 63 +++++++++++++++++++++---------------- 2 files changed, 76 insertions(+), 27 deletions(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 78443afe7a..d6dd9c1a73 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3543,6 +3543,46 @@ test "zig fmt: multiline string literals should play nice with array initializer ); } +test "zig fmt: use of comments and Multiline string literals may force the parameters over multiple lines" { + try testCanonical( + \\pub fn makeMemUndefined(qzz: []u8) i1 { + \\ cases.add( // fixed bug #2032 + \\ "compile diagnostic string for top level decl type", + \\ \\export fn entry() void { + \\ \\ var foo: u32 = @This(){}; + \\ \\} + \\ , &[_][]const u8{ + \\ "tmp.zig:2:27: error: type 'u32' does not support array initialization", + \\ }); + \\ @compileError( + \\ \\ unknown-length pointers and C pointers cannot be hashed deeply. + \\ \\ Consider providing your own hash function. + \\ \\ unknown-length pointers and C pointers cannot be hashed deeply. + \\ \\ Consider providing your own hash function. + \\ ); + \\ return @intCast(i1, doMemCheckClientRequestExpr(0, // default return + \\ .MakeMemUndefined, @ptrToInt(qzz.ptr), qzz.len, 0, 0, 0)); + \\} + \\ + \\// This looks like garbage don't do this + \\const rparen = tree.prevToken( + \\// the first token for the annotation expressions is the left + \\// parenthesis, hence the need for two prevToken + \\ if (fn_proto.getAlignExpr()) |align_expr| + \\ tree.prevToken(tree.prevToken(align_expr.firstToken())) + \\else if (fn_proto.getSectionExpr()) |section_expr| + \\ tree.prevToken(tree.prevToken(section_expr.firstToken())) + \\else if (fn_proto.getCallconvExpr()) |callconv_expr| + \\ tree.prevToken(tree.prevToken(callconv_expr.firstToken())) + \\else switch (fn_proto.return_type) { + \\ .Explicit => |node| node.firstToken(), + \\ .InferErrorSet => |node| tree.prevToken(node.firstToken()), + \\ .Invalid => unreachable, + \\}); + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index b2687ada98..319788546e 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -1058,21 +1058,22 @@ fn renderExpression( }; if (src_has_trailing_comma) { - try renderToken(tree, ais, lparen, Space.Newline); - - const params = call.params(); - for (params) |param_node, i| { + { ais.pushIndent(); defer ais.popIndent(); - if (i + 1 < params.len) { - const next_node = params[i + 1]; - try renderExpression(allocator, ais, tree, param_node, Space.None); - const comma = tree.nextToken(param_node.lastToken()); - try renderToken(tree, ais, comma, Space.Newline); // , - try renderExtraNewline(tree, ais, next_node); - } else { - try renderExpression(allocator, ais, tree, param_node, Space.Comma); + try renderToken(tree, ais, lparen, Space.Newline); // ( + const params = call.params(); + for (params) |param_node, i| { + if (i + 1 < params.len) { + const next_node = params[i + 1]; + try renderExpression(allocator, ais, tree, param_node, Space.None); + const comma = tree.nextToken(param_node.lastToken()); + try renderToken(tree, ais, comma, Space.Newline); // , + try renderExtraNewline(tree, ais, next_node); + } else { + try renderExpression(allocator, ais, tree, param_node, Space.Comma); + } } } return renderToken(tree, ais, call.rtoken, space); @@ -1082,7 +1083,10 @@ fn renderExpression( const params = call.params(); for (params) |param_node, i| { - if (param_node.*.tag == .MultilineStringLiteral) ais.pushIndentOneShot(); + const maybe_comment = param_node.firstToken() - 1; + if (param_node.*.tag == .MultilineStringLiteral or tree.token_ids[maybe_comment] == .LineComment) { + ais.pushIndentOneShot(); + } try renderExpression(allocator, ais, tree, param_node, Space.None); @@ -1092,7 +1096,7 @@ fn renderExpression( try renderToken(tree, ais, comma, Space.Space); } } - return renderToken(tree, ais, call.rtoken, space); + return renderToken(tree, ais, call.rtoken, space); // ) }, .ArrayAccess => { @@ -1497,6 +1501,10 @@ fn renderExpression( // render all on one line, no trailing comma const params = builtin_call.params(); for (params) |param_node, i| { + const maybe_comment = param_node.firstToken() - 1; + if (param_node.*.tag == .MultilineStringLiteral or tree.token_ids[maybe_comment] == .LineComment) { + ais.pushIndentOneShot(); + } try renderExpression(allocator, ais, tree, param_node, Space.None); if (i + 1 < params.len) { @@ -1548,19 +1556,20 @@ fn renderExpression( assert(tree.token_ids[lparen] == .LParen); const rparen = tree.prevToken( - // the first token for the annotation expressions is the left - // parenthesis, hence the need for two prevToken - if (fn_proto.getAlignExpr()) |align_expr| - tree.prevToken(tree.prevToken(align_expr.firstToken())) - else if (fn_proto.getSectionExpr()) |section_expr| - tree.prevToken(tree.prevToken(section_expr.firstToken())) - else if (fn_proto.getCallconvExpr()) |callconv_expr| - tree.prevToken(tree.prevToken(callconv_expr.firstToken())) - else switch (fn_proto.return_type) { - .Explicit => |node| node.firstToken(), - .InferErrorSet => |node| tree.prevToken(node.firstToken()), - .Invalid => unreachable, - }); + // the first token for the annotation expressions is the left + // parenthesis, hence the need for two prevToken + if (fn_proto.getAlignExpr()) |align_expr| + tree.prevToken(tree.prevToken(align_expr.firstToken())) + else if (fn_proto.getSectionExpr()) |section_expr| + tree.prevToken(tree.prevToken(section_expr.firstToken())) + else if (fn_proto.getCallconvExpr()) |callconv_expr| + tree.prevToken(tree.prevToken(callconv_expr.firstToken())) + else switch (fn_proto.return_type) { + .Explicit => |node| node.firstToken(), + .InferErrorSet => |node| tree.prevToken(node.firstToken()), + .Invalid => unreachable, + }, + ); assert(tree.token_ids[rparen] == .RParen); const src_params_trailing_comma = blk: { From c06674e701fdd74165778dd46b72cc469ba29140 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Wed, 9 Sep 2020 22:58:13 +1000 Subject: [PATCH 069/210] zig fmt: Small cleanup --- lib/std/zig/render.zig | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 319788546e..78f8ddb022 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -747,6 +747,10 @@ fn renderExpression( defer allocator.free(widths); mem.set(usize, widths, 0); + var expr_newlines = try allocator.alloc(bool, row_exprs.len); + defer allocator.free(expr_newlines); + mem.set(bool, expr_newlines, false); + var expr_widths = widths[0 .. widths.len - row_size]; var column_widths = widths[widths.len - row_size ..]; @@ -791,7 +795,7 @@ fn renderExpression( var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, counting_stream.writer()); // Calculate size of columns in current section - var c: usize = 0; + var column_counter: usize = 0; var single_line = true; for (section_exprs) |expr, i| { if (i + 1 < section_exprs.len) { @@ -800,40 +804,42 @@ fn renderExpression( try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); const width = @intCast(usize, counting_stream.bytes_written); expr_widths[i] = width; + expr_newlines[i] = line_find_stream.byte_found; if (!line_find_stream.byte_found) { - const col = c % row_size; - column_widths[col] = std.math.max(column_widths[col], width); + const column = column_counter % row_size; + column_widths[column] = std.math.max(column_widths[column], width); const expr_last_token = expr.*.lastToken() + 1; const next_expr = section_exprs[i + 1]; const loc = tree.tokenLocation(tree.token_locs[expr_last_token].start, next_expr.*.firstToken()); if (loc.line == 0) { - c += 1; + column_counter += 1; } else { single_line = false; - c = 0; + column_counter = 0; } } else { single_line = false; - c = 0; + column_counter = 0; } } else { counting_stream.bytes_written = 0; try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); const width = @intCast(usize, counting_stream.bytes_written); expr_widths[i] = width; + expr_newlines[i] = line_find_stream.byte_found; if (!line_find_stream.byte_found) { - const col = c % row_size; - column_widths[col] = std.math.max(column_widths[col], width); + const column = column_counter % row_size; + column_widths[column] = std.math.max(column_widths[column], width); } break; } } // Render exprs in current section - c = 0; + column_counter = 0; var last_col_index: usize = row_size - 1; for (section_exprs) |expr, i| { if (i + 1 < section_exprs.len) { @@ -842,18 +848,15 @@ fn renderExpression( const comma = tree.nextToken(expr.*.lastToken()); - if (c != last_col_index) { - line_find_stream.byte_found = false; - try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); - try renderExpression(allocator, &auto_indenting_stream, tree, next_expr, Space.None); - if (!line_find_stream.byte_found) { + if (column_counter != last_col_index) { + if (!expr_newlines[i] and !expr_newlines[i + 1]) { // Neither the current or next expression is multiline try renderToken(tree, ais, comma, Space.Space); // , - assert(column_widths[c % row_size] >= expr_widths[i]); - const padding = column_widths[c % row_size] - expr_widths[i]; + assert(column_widths[column_counter % row_size] >= expr_widths[i]); + const padding = column_widths[column_counter % row_size] - expr_widths[i]; try ais.writer().writeByteNTimes(' ', padding); - c += 1; + column_counter += 1; continue; } } @@ -862,7 +865,7 @@ fn renderExpression( continue; } - c = 0; + column_counter = 0; try renderToken(tree, ais, comma, Space.Newline); // , try renderExtraNewline(tree, ais, next_expr); } else { @@ -1091,7 +1094,6 @@ fn renderExpression( try renderExpression(allocator, ais, tree, param_node, Space.None); if (i + 1 < params.len) { - const next_param = params[i + 1]; const comma = tree.nextToken(param_node.lastToken()); try renderToken(tree, ais, comma, Space.Space); } From 40b6e86a999ce80b9f71c7a88df6186400f151ac Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Thu, 10 Sep 2020 20:32:40 +1000 Subject: [PATCH 070/210] zig fmt: fix #6171 --- lib/std/zig/parser_test.zig | 18 ++++++++++++++++++ lib/std/zig/render.zig | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index d6dd9c1a73..f3cfe811a4 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3583,6 +3583,24 @@ test "zig fmt: use of comments and Multiline string literals may force the param ); } +test "zig fmt: single argument trailing commas in @builtins()" { + try testCanonical( + \\pub fn foo(qzz: []u8) i1 { + \\ @panic( + \\ foo, + \\ ); + \\ panic( + \\ foo, + \\ ); + \\ @panic( + \\ foo, + \\ bar, + \\ ); + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 78f8ddb022..fbf2139b42 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -1489,7 +1489,7 @@ fn renderExpression( try renderToken(tree, ais, builtin_call.builtin_token, Space.None); // @name const src_params_trailing_comma = blk: { - if (builtin_call.params_len < 2) break :blk false; + if (builtin_call.params_len == 0) break :blk false; const last_node = builtin_call.params()[builtin_call.params_len - 1]; const maybe_comma = tree.nextToken(last_node.lastToken()); break :blk tree.token_ids[maybe_comma] == .Comma; From 1aacedf6e197ea212025dccad622894a44eb5461 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Thu, 10 Sep 2020 23:35:18 +1000 Subject: [PATCH 071/210] zig fmt: Fix regression in ArrayInitializers --- lib/std/zig/parser_test.zig | 41 ++++++++++++++++++++++++++++++++++--- lib/std/zig/render.zig | 39 +++++++++++++++-------------------- 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index f3cfe811a4..c7d64bc513 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3516,9 +3516,13 @@ test "zig fmt: multiline string literals should play nice with array initializer \\ .{( \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx \\ )}, - \\ .{ "xxxxxxx", "xxx", ( - \\ \\ xxx - \\ ), "xxx", "xxx" }, + \\ .{ + \\ "xxxxxxx", "xxx", + \\ ( + \\ \\ xxx + \\ ), + \\ "xxx", "xxx", + \\ }, \\ .{ "xxxxxxx", "xxx", "xxx", "xxx" }, .{ "xxxxxxx", "xxx", "xxx", "xxx" }, \\ "aaaaaaa", "bbbbbb", "ccccc", // - \\ "dddd", ("eee"), ("fff"), @@ -3601,6 +3605,37 @@ test "zig fmt: single argument trailing commas in @builtins()" { ); } +test "zig fmt: trailing comma should force multiline 1 column" { + try testTransform( + \\pub const UUID_NULL: uuid_t = [16]u8{0,0,0,0,}; + \\ + , + \\pub const UUID_NULL: uuid_t = [16]u8{ + \\ 0, + \\ 0, + \\ 0, + \\ 0, + \\}; + \\ + ); +} + +test "zig fmt: function params should align nicely" { + try testCanonical( + \\pub fn foo() void { + \\ cases.addRuntimeSafety("slicing operator with sentinel", + \\ \\const std = @import("std"); + \\ ++ check_panic_msg ++ + \\ \\pub fn main() void { + \\ \\ var buf = [4]u8{'a','b','c',0}; + \\ \\ const slice = buf[0..:0]; + \\ \\} + \\ ); + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index fbf2139b42..30f739aaef 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -733,14 +733,14 @@ fn renderExpression( } // scan to find row size - if (rowSize(tree, exprs, rtoken, false) != null) { + if (rowSize(tree, exprs, rtoken) != null) { { ais.pushIndentNextLine(); defer ais.popIndent(); try renderToken(tree, ais, lbrace, Space.Newline); var expr_index: usize = 0; - while (rowSize(tree, exprs[expr_index..], rtoken, true)) |row_size| { + while (rowSize(tree, exprs[expr_index..], rtoken)) |row_size| { const row_exprs = exprs[expr_index..]; // A place to store the width of each expression and its column's maximum var widths = try allocator.alloc(usize, row_exprs.len + row_size); @@ -757,14 +757,14 @@ fn renderExpression( // Find next row with trailing comment (if any) to end the current section var section_end = sec_end: { var this_line_first_expr: usize = 0; - var this_line_size = rowSize(tree, row_exprs, rtoken, true); + var this_line_size = rowSize(tree, row_exprs, rtoken); for (row_exprs) |expr, i| { // Ignore comment on first line of this section if (i == 0 or tree.tokensOnSameLine(row_exprs[0].firstToken(), expr.lastToken())) continue; // Track start of line containing comment if (!tree.tokensOnSameLine(row_exprs[this_line_first_expr].firstToken(), expr.lastToken())) { this_line_first_expr = i; - this_line_size = rowSize(tree, row_exprs[this_line_first_expr..], rtoken, true); + this_line_size = rowSize(tree, row_exprs[this_line_first_expr..], rtoken); } const maybe_comma = expr.lastToken() + 1; @@ -860,7 +860,7 @@ fn renderExpression( continue; } } - if (single_line) { + if (single_line and row_size != 1) { try renderToken(tree, ais, comma, Space.Space); // , continue; } @@ -1087,7 +1087,8 @@ fn renderExpression( const params = call.params(); for (params) |param_node, i| { const maybe_comment = param_node.firstToken() - 1; - if (param_node.*.tag == .MultilineStringLiteral or tree.token_ids[maybe_comment] == .LineComment) { + const maybe_multiline_string = param_node.firstToken(); + if (tree.token_ids[maybe_multiline_string] == .MultilineStringLiteralLine or tree.token_ids[maybe_comment] == .LineComment) { ais.pushIndentOneShot(); } @@ -2629,7 +2630,16 @@ fn copyFixingWhitespace(ais: anytype, slice: []const u8) @TypeOf(ais.*).Error!vo }; } -fn rowSize(tree: *ast.Tree, exprs: []*ast.Node, rtoken: ast.TokenIndex, force: bool) ?usize { +fn rowSize(tree: *ast.Tree, exprs: []*ast.Node, rtoken: ast.TokenIndex) ?usize { + const first_token = exprs[0].firstToken(); + const first_loc = tree.tokenLocation(tree.token_locs[first_token].start, rtoken); + if (first_loc.line == 0) { + const maybe_comma = tree.prevToken(rtoken); + if (tree.token_ids[maybe_comma] == .Comma) + return 1; + return null; // no newlines + } + var count: usize = 1; for (exprs) |expr, i| { if (i + 1 < exprs.len) { @@ -2638,21 +2648,6 @@ fn rowSize(tree: *ast.Tree, exprs: []*ast.Node, rtoken: ast.TokenIndex, force: b if (loc.line != 0) return count; count += 1; } else { - if (force) return count; - const expr_last_token = expr.lastToken(); - const loc = tree.tokenLocation(tree.token_locs[expr_last_token].start, rtoken); - if (loc.line == 0) { - // all on one line - const src_has_trailing_comma = trailblk: { - const maybe_comma = tree.prevToken(rtoken); - break :trailblk tree.token_ids[maybe_comma] == .Comma; - }; - if (src_has_trailing_comma) { - return 1; // force row size 1 - } else { - return null; // no newlines - } - } return count; } } From 4496a6c9cccbd6a9c82b5d4ca7f533b18ebeab32 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Tue, 15 Sep 2020 18:49:59 +1000 Subject: [PATCH 072/210] zig fmt: Special case un-indent comma after multiline string in param list --- lib/std/zig/parser_test.zig | 11 +++++++++-- lib/std/zig/render.zig | 7 +++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index c7d64bc513..994ad6d5d1 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -1374,7 +1374,7 @@ test "zig fmt: multiline string parameter in fn call with trailing comma" { \\ \\ZIG_C_HEADER_FILES {} \\ \\ZIG_DIA_GUIDS_LIB {} \\ \\ - \\ , + \\ , \\ std.cstr.toSliceConst(c.ZIG_CMAKE_BINARY_DIR), \\ std.cstr.toSliceConst(c.ZIG_CXX_COMPILER), \\ std.cstr.toSliceConst(c.ZIG_DIA_GUIDS_LIB), @@ -3385,10 +3385,17 @@ test "zig fmt: Indent comma correctly after multiline string literals in arg lis \\ \\------------ \\ \\xxxxxxxxxxxx \\ \\xxxxxxxxxxxx - \\ , + \\ , \\ g.GtkMessageType.GTK_MESSAGE_WARNING, \\ null, \\ ); + \\ + \\ z.display_message_dialog(*const [323:0]u8, + \\ \\Message Text + \\ \\------------ + \\ \\xxxxxxxxxxxx + \\ \\xxxxxxxxxxxx + \\ , g.GtkMessageType.GTK_MESSAGE_WARNING, null); \\} \\ ); diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 30f739aaef..67afbb77d9 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -1071,6 +1071,13 @@ fn renderExpression( if (i + 1 < params.len) { const next_node = params[i + 1]; try renderExpression(allocator, ais, tree, param_node, Space.None); + + // Unindent the comma for multiline string literals + const maybe_multiline_string = param_node.firstToken(); + const is_multiline_string = tree.token_ids[maybe_multiline_string] == .MultilineStringLiteralLine; + if (is_multiline_string) ais.popIndent(); + defer if (is_multiline_string) ais.pushIndent(); + const comma = tree.nextToken(param_node.lastToken()); try renderToken(tree, ais, comma, Space.Newline); // , try renderExtraNewline(tree, ais, next_node); From d08842887f4c566186ad3babacd9c155233663b3 Mon Sep 17 00:00:00 2001 From: Calle Englund Date: Fri, 18 Sep 2020 16:40:20 +0200 Subject: [PATCH 073/210] Workaround MacOS build failure due to #6087 --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index ceaecf5552..2b81669a30 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,7 @@ set(ZIG_PREFER_CLANG_CPP_DYLIB off CACHE BOOL "Try to link against -lclang-cpp") set(ZIG_WORKAROUND_4799 off CACHE BOOL "workaround for https://github.com/ziglang/zig/issues/4799") set(ZIG_WORKAROUND_POLLY_SO off CACHE STRING "workaround for https://github.com/ziglang/zig/issues/4799") set(ZIG_USE_CCACHE off CACHE BOOL "Use ccache if available") +set(ZIG_WORKAROUND_6087 off CACHE BOOL "workaround for https://github.com/ziglang/zig/issues/6087") if(CCACHE_PROGRAM AND ZIG_USE_CCACHE) SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}") @@ -90,6 +91,11 @@ if(APPLE AND ZIG_STATIC) list(APPEND LLVM_LIBRARIES "${ZLIB}") endif() +if(APPLE AND ZIG_WORKAROUND_6087) + list(REMOVE_ITEM LLVM_LIBRARIES "-llibxml2.tbd") + list(APPEND LLVM_LIBRARIES "-lxml2") +endif() + if(APPLE AND ZIG_WORKAROUND_4799) # eg: ${CMAKE_PREFIX_PATH} could be /usr/local/opt/llvm/ list(APPEND LLVM_LIBRARIES "-Wl,${CMAKE_PREFIX_PATH}/lib/libPolly.a" "-Wl,${CMAKE_PREFIX_PATH}/lib/libPollyPPCG.a" "-Wl,${CMAKE_PREFIX_PATH}/lib/libPollyISL.a") From 2ef68631cb7045c277b348777b1a064845b95cd8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Sep 2020 22:48:28 -0700 Subject: [PATCH 074/210] stage2 now supports using stage1 as a backend for compiling zig code * move stage2.cpp code into zig0.cpp for simplicity * add -ftime-report and some more CLI options to stage2 * stage2 compites the llvm cpu features string * classifyFileExt understands more file extensions * correction to generateBuiltinZigSource using the wrong allocator (thanks dbandstra!) * stage2 is now able to build hello.zig into hello.o using stage1 as a library however it fails linking due to missing compiler-rt * remove dead code * simplify zig0 builtin.zig source * fix not resolving builtin.zig source path causing duplicate imports * fix stage1.h not being valid C code * fix stage2.h not being valid C code --- BRANCH_TODO | 8 +- CMakeLists.txt | 1 - src-self-hosted/Compilation.zig | 247 +++++++++++++++++-- src-self-hosted/link.zig | 13 + src-self-hosted/link/Elf.zig | 15 +- src-self-hosted/main.zig | 114 ++++++--- src-self-hosted/stage1.zig | 412 +++++++++++--------------------- src/analyze.cpp | 2 +- src/codegen.cpp | 61 ++--- src/codegen.hpp | 3 +- src/stage1.cpp | 11 +- src/stage1.h | 25 +- src/stage2.cpp | 241 ------------------- src/stage2.h | 16 +- src/target.cpp | 7 +- src/target.hpp | 1 - src/zig0.cpp | 251 ++++++++++++++++++- 17 files changed, 750 insertions(+), 678 deletions(-) delete mode 100644 src/stage2.cpp diff --git a/BRANCH_TODO b/BRANCH_TODO index 9455511dbf..bee98d6d2b 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,3 +1,6 @@ + * build & link against compiler-rt + * build & link against freestanding libc + * Cache integration for stage1 zig code compilation * `zig test` * `zig build` * `-ftime-report` @@ -14,10 +17,7 @@ - using it as a preprocessor (-E) - try building some software * support rpaths in ELF linker code - * build & link against compiler-rt - - stage1 C++ code integration * repair @cImport - * build & link against freestanding libc * add CLI support for a way to pass extra flags to c source files * capture lld stdout/stderr better * musl @@ -35,10 +35,12 @@ * implement emit-h in stage2 * implement -fno-emit-bin * audit the base cache hash + * --main-pkg-path * audit the CLI options for stage2 * `zig init-lib` * `zig init-exe` * `zig run` + * restore error messages for stage2_add_link_lib * implement serialization/deserialization of incremental compilation metadata * incremental compilation - implement detection of which source files changed diff --git a/CMakeLists.txt b/CMakeLists.txt index d55c76ee9c..1a19c275d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -258,7 +258,6 @@ find_package(Threads) # This is our shim which will be replaced by stage1.zig. set(ZIG0_SOURCES "${CMAKE_SOURCE_DIR}/src/zig0.cpp" - "${CMAKE_SOURCE_DIR}/src/stage2.cpp" ) set(ZIG_SOURCES diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index ed46508485..f08c5ef82f 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -19,6 +19,7 @@ const libunwind = @import("libunwind.zig"); const fatal = @import("main.zig").fatal; const Module = @import("Module.zig"); const Cache = @import("Cache.zig"); +const stage1 = @import("stage1.zig"); /// General-purpose allocator. Used for both temporary and long-term storage. gpa: *Allocator, @@ -26,6 +27,7 @@ gpa: *Allocator, arena_state: std.heap.ArenaAllocator.State, bin_file: *link.File, c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{}, +stage1_module: ?*stage1.Module, link_error_flags: link.File.ErrorFlags = .{}, @@ -122,6 +124,8 @@ const Job = union(enum) { /// Generate builtin.zig source code and write it into the correct place. generate_builtin_zig: void, + /// Use stage1 C++ code to compile zig code into an object file. + stage1_module: void, }; pub const CObject = struct { @@ -274,6 +278,7 @@ pub const InitOptions = struct { strip: bool = false, single_threaded: bool = false, is_native_os: bool, + time_report: bool = false, link_eh_frame_hdr: bool = false, linker_script: ?[]const u8 = null, version_script: ?[]const u8 = null, @@ -288,12 +293,20 @@ pub const InitOptions = struct { clang_passthrough_mode: bool = false, verbose_cc: bool = false, verbose_link: bool = false, + verbose_tokenize: bool = false, + verbose_ast: bool = false, + verbose_ir: bool = false, + verbose_llvm_ir: bool = false, + verbose_cimport: bool = false, + verbose_llvm_cpu_features: bool = false, is_test: bool = false, stack_size_override: ?u64 = null, self_exe_path: ?[]const u8 = null, version: ?std.builtin.Version = null, libc_installation: ?*const LibCInstallation = null, machine_code_model: std.builtin.CodeModel = .default, + /// This is for stage1 and should be deleted upon completion of self-hosting. + color: @import("main.zig").Color = .Auto, }; pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { @@ -332,11 +345,27 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { { break :blk true; } + + if (build_options.is_stage1) { + // If stage1 generates an object file, self-hosted linker is not + // yet sophisticated enough to handle that. + break :blk options.root_pkg != null; + } + break :blk false; }; // Make a decision on whether to use LLVM or our own backend. const use_llvm = if (options.use_llvm) |explicit| explicit else blk: { + // If we have no zig code to compile, no need for LLVM. + if (options.root_pkg == null) + break :blk false; + + // If we are the stage1 compiler, we depend on the stage1 c++ llvm backend + // to compile zig code. + if (build_options.is_stage1) + break :blk true; + // We would want to prefer LLVM for release builds when it is available, however // we don't have an LLVM backend yet :) // We would also want to prefer LLVM for architectures that we don't have self-hosted support for too. @@ -580,6 +609,118 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .ReleaseFast, .ReleaseSmall => false, }; + const llvm_cpu_features: ?[*:0]const u8 = if (build_options.have_llvm and use_llvm) blk: { + var buf = std.ArrayList(u8).init(arena); + for (options.target.cpu.arch.allFeaturesList()) |feature, index_usize| { + const index = @intCast(Target.Cpu.Feature.Set.Index, index_usize); + const is_enabled = options.target.cpu.features.isEnabled(index); + + if (feature.llvm_name) |llvm_name| { + const plus_or_minus = "-+"[@boolToInt(is_enabled)]; + try buf.ensureCapacity(buf.items.len + 2 + llvm_name.len); + buf.appendAssumeCapacity(plus_or_minus); + buf.appendSliceAssumeCapacity(llvm_name); + buf.appendSliceAssumeCapacity(","); + } + } + assert(mem.endsWith(u8, buf.items, ",")); + buf.items[buf.items.len - 1] = 0; + buf.shrink(buf.items.len); + break :blk buf.items[0 .. buf.items.len - 1 :0].ptr; + } else null; + + const stage1_module: ?*stage1.Module = if (build_options.is_stage1 and use_llvm) blk: { + // Here we use the legacy stage1 C++ compiler to compile Zig code. + const stage2_target = try arena.create(stage1.Stage2Target); + stage2_target.* = .{ + .arch = @enumToInt(options.target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch + .os = @enumToInt(options.target.os.tag), + .abi = @enumToInt(options.target.abi), + .is_native_os = options.is_native_os, + .is_native_cpu = false, // Only true when bootstrapping the compiler. + .llvm_cpu_name = if (options.target.cpu.model.llvm_name) |s| s.ptr else null, + .llvm_cpu_features = llvm_cpu_features.?, + }; + const progress = try arena.create(std.Progress); + const main_progress_node = try progress.start("", 100); + if (options.color == .Off) progress.terminal = null; + + const mod = module.?; + const main_zig_file = mod.root_pkg.root_src_path; + const zig_lib_dir = options.zig_lib_directory.path.?; + const builtin_sub = &[_][]const u8{"builtin.zig"}; + const builtin_zig_path = try mod.zig_cache_artifact_directory.join(arena, builtin_sub); + + const stage1_module = stage1.create( + @enumToInt(options.optimize_mode), + undefined, + 0, // TODO --main-pkg-path + main_zig_file.ptr, + main_zig_file.len, + zig_lib_dir.ptr, + zig_lib_dir.len, + stage2_target, + options.is_test, + ) orelse return error.OutOfMemory; + + const output_dir = bin_directory.path orelse "."; + + const stage1_pkg = try arena.create(stage1.Pkg); + stage1_pkg.* = .{ + .name_ptr = undefined, + .name_len = 0, + .path_ptr = undefined, + .path_len = 0, + .children_ptr = undefined, + .children_len = 0, + .parent = null, + }; + + stage1_module.* = .{ + .root_name_ptr = root_name.ptr, + .root_name_len = root_name.len, + .output_dir_ptr = output_dir.ptr, + .output_dir_len = output_dir.len, + .builtin_zig_path_ptr = builtin_zig_path.ptr, + .builtin_zig_path_len = builtin_zig_path.len, + .test_filter_ptr = "", + .test_filter_len = 0, + .test_name_prefix_ptr = "", + .test_name_prefix_len = 0, + .userdata = @ptrToInt(comp), + .root_pkg = stage1_pkg, + .code_model = @enumToInt(options.machine_code_model), + .subsystem = stage1.TargetSubsystem.Auto, + .err_color = @enumToInt(options.color), + .pic = pic, + .link_libc = options.link_libc, + .link_libcpp = options.link_libcpp, + .strip = options.strip, + .is_single_threaded = single_threaded, + .dll_export_fns = dll_export_fns, + .link_mode_dynamic = link_mode == .Dynamic, + .valgrind_enabled = valgrind, + .function_sections = options.function_sections orelse false, + .enable_stack_probing = stack_check, + .enable_time_report = options.time_report, + .enable_stack_report = false, + .dump_analysis = false, + .enable_doc_generation = false, + .emit_bin = true, + .emit_asm = false, + .emit_llvm_ir = false, + .test_is_evented = false, + .verbose_tokenize = options.verbose_tokenize, + .verbose_ast = options.verbose_ast, + .verbose_ir = options.verbose_ir, + .verbose_llvm_ir = options.verbose_llvm_ir, + .verbose_cimport = options.verbose_cimport, + .verbose_llvm_cpu_features = options.verbose_llvm_cpu_features, + .main_progress_node = main_progress_node, + }; + break :blk stage1_module; + } else null; + const bin_file = try link.File.openPath(gpa, .{ .directory = bin_directory, .sub_path = emit_bin.basename, @@ -626,6 +767,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .machine_code_model = options.machine_code_model, .dll_export_fns = dll_export_fns, .error_return_tracing = error_return_tracing, + .llvm_cpu_features = llvm_cpu_features, }); errdefer bin_file.destroy(); @@ -635,6 +777,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .zig_lib_directory = options.zig_lib_directory, .zig_cache_directory = options.zig_cache_directory, .bin_file = bin_file, + .stage1_module = stage1_module, .work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa), .keep_source_files_loaded = options.keep_source_files_loaded, .use_clang = use_clang, @@ -681,6 +824,10 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { try comp.work_queue.writeItem(.{ .libunwind = {} }); } + if (comp.stage1_module) |module| { + try comp.work_queue.writeItem(.{ .stage1_module = {} }); + } + return comp; } @@ -689,6 +836,11 @@ pub fn destroy(self: *Compilation) void { self.bin_file.destroy(); if (optional_module) |module| module.deinit(); + if (self.stage1_module) |module| { + module.main_progress_node.?.end(); + module.destroy(); + } + const gpa = self.gpa; self.work_queue.deinit(); @@ -997,6 +1149,10 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { fatal("unable to update builtin.zig file: {}", .{@errorName(err)}); }; }, + .stage1_module => { + // This Job is only queued up if there is a zig module. + self.stage1_module.?.build_object(); + }, }; } @@ -1365,7 +1521,7 @@ pub fn addCCArgs( try argv.append("-fPIC"); } }, - .so, .assembly, .ll, .bc, .unknown => {}, + .shared_library, .assembly, .ll, .bc, .unknown, .static_library, .object, .zig, .zir => {}, } if (out_dep_path) |p| { try argv.appendSlice(&[_][]const u8{ "-MD", "-MV", "-MF", p }); @@ -1445,17 +1601,39 @@ pub const FileExt = enum { ll, bc, assembly, - so, + shared_library, + object, + static_library, + zig, + zir, unknown, pub fn clangSupportsDepFile(ext: FileExt) bool { return switch (ext) { .c, .cpp, .h => true, - .ll, .bc, .assembly, .so, .unknown => false, + + .ll, + .bc, + .assembly, + .shared_library, + .object, + .static_library, + .zig, + .zir, + .unknown, + => false, }; } }; +pub fn hasObjectExt(filename: []const u8) bool { + return mem.endsWith(u8, filename, ".o") or mem.endsWith(u8, filename, ".obj"); +} + +pub fn hasStaticLibraryExt(filename: []const u8) bool { + return mem.endsWith(u8, filename, ".a") or mem.endsWith(u8, filename, ".lib"); +} + pub fn hasCExt(filename: []const u8) bool { return mem.endsWith(u8, filename, ".c"); } @@ -1471,6 +1649,32 @@ pub fn hasAsmExt(filename: []const u8) bool { return mem.endsWith(u8, filename, ".s") or mem.endsWith(u8, filename, ".S"); } +pub fn hasSharedLibraryExt(filename: []const u8) bool { + if (mem.endsWith(u8, filename, ".so") or + mem.endsWith(u8, filename, ".dll") or + mem.endsWith(u8, filename, ".dylib")) + { + return true; + } + // Look for .so.X, .so.X.Y, .so.X.Y.Z + var it = mem.split(filename, "."); + _ = it.next().?; + var so_txt = it.next() orelse return false; + while (!mem.eql(u8, so_txt, "so")) { + so_txt = it.next() orelse return false; + } + const n1 = it.next() orelse return false; + const n2 = it.next(); + const n3 = it.next(); + + _ = std.fmt.parseInt(u32, n1, 10) catch return false; + if (n2) |x| _ = std.fmt.parseInt(u32, x, 10) catch return false; + if (n3) |x| _ = std.fmt.parseInt(u32, x, 10) catch return false; + if (it.next() != null) return false; + + return true; +} + pub fn classifyFileExt(filename: []const u8) FileExt { if (hasCExt(filename)) { return .c; @@ -1484,26 +1688,19 @@ pub fn classifyFileExt(filename: []const u8) FileExt { return .assembly; } else if (mem.endsWith(u8, filename, ".h")) { return .h; - } else if (mem.endsWith(u8, filename, ".so")) { - return .so; + } else if (mem.endsWith(u8, filename, ".zig")) { + return .zig; + } else if (mem.endsWith(u8, filename, ".zir")) { + return .zig; + } else if (hasSharedLibraryExt(filename)) { + return .shared_library; + } else if (hasStaticLibraryExt(filename)) { + return .static_library; + } else if (hasObjectExt(filename)) { + return .object; + } else { + return .unknown; } - // Look for .so.X, .so.X.Y, .so.X.Y.Z - var it = mem.split(filename, "."); - _ = it.next().?; - var so_txt = it.next() orelse return .unknown; - while (!mem.eql(u8, so_txt, "so")) { - so_txt = it.next() orelse return .unknown; - } - const n1 = it.next() orelse return .unknown; - const n2 = it.next(); - const n3 = it.next(); - - _ = std.fmt.parseInt(u32, n1, 10) catch return .unknown; - if (n2) |x| _ = std.fmt.parseInt(u32, x, 10) catch return .unknown; - if (n3) |x| _ = std.fmt.parseInt(u32, x, 10) catch return .unknown; - if (it.next() != null) return .unknown; - - return .so; } test "classifyFileExt" { @@ -1681,7 +1878,7 @@ pub fn dump_argv(argv: []const []const u8) void { } pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8 { - var buffer = std.ArrayList(u8).init(comp.gpa); + var buffer = std.ArrayList(u8).init(allocator); defer buffer.deinit(); const target = comp.getTarget(); @@ -1691,9 +1888,9 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8 try buffer.writer().print( \\usingnamespace @import("std").builtin; \\/// Deprecated - \\pub const arch = std.Target.current.cpu.arch; + \\pub const arch = Target.current.cpu.arch; \\/// Deprecated - \\pub const endian = std.Target.current.cpu.arch.endian(); + \\pub const endian = Target.current.cpu.arch.endian(); \\pub const output_mode = OutputMode.{}; \\pub const link_mode = LinkMode.{}; \\pub const is_test = {}; diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 1ad293b275..88a964f2ff 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -69,6 +69,7 @@ pub const Options = struct { linker_script: ?[]const u8 = null, version_script: ?[]const u8 = null, override_soname: ?[]const u8 = null, + llvm_cpu_features: ?[*:0]const u8 = null, /// Extra args passed directly to LLD. Ignored when not linking with LLD. extra_lld_args: []const []const u8 = &[0][]const u8, @@ -134,6 +135,18 @@ pub const File = struct { /// rewriting it. A malicious file is detected as incremental link failure /// and does not cause Illegal Behavior. This operation is not atomic. pub fn openPath(allocator: *Allocator, options: Options) !*File { + const use_stage1 = build_options.is_stage1 and options.use_llvm; + if (use_stage1) { + return switch (options.object_format) { + .coff, .pe => &(try Coff.createEmpty(allocator, options)).base, + .elf => &(try Elf.createEmpty(allocator, options)).base, + .macho => &(try MachO.createEmpty(allocator, options)).base, + .wasm => &(try Wasm.createEmpty(allocator, options)).base, + .c => unreachable, // Reported error earlier. + .hex => return error.HexObjectFormatUnimplemented, + .raw => return error.RawObjectFormatUnimplemented, + }; + } const use_lld = build_options.have_llvm and options.use_lld; // comptime known false when !have_llvm const sub_path = if (use_lld) blk: { if (options.module == null) { diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 62a5e486d0..5257b46279 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -1222,13 +1222,16 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { // If there is no Zig code to compile, then we should skip flushing the output file because it // will not be part of the linker line anyway. const module_obj_path: ?[]const u8 = if (self.base.options.module) |module| blk: { - try self.flushModule(comp); + const use_stage1 = build_options.is_stage1 and self.base.options.use_llvm; + if (use_stage1) { + const obj_basename = try std.fmt.allocPrint(arena, "{}.o", .{self.base.options.root_name}); + const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename}); + break :blk full_obj_path; + } + try self.flushModule(comp); const obj_basename = self.base.intermediary_basename.?; - const full_obj_path = if (directory.path) |dir_path| - try std.fs.path.join(arena, &[_][]const u8{dir_path, obj_basename}) - else - obj_basename; + const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename}); break :blk full_obj_path; } else null; @@ -1504,7 +1507,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { // (the check for that needs to be earlier), but they could be full paths to .so files, in which // case we want to avoid prepending "-l". const ext = Compilation.classifyFileExt(link_lib); - const arg = if (ext == .so) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib}); + const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib}); argv.appendAssumeCapacity(arg); } diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index f18ccdd803..6aa4be7c41 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -222,21 +222,27 @@ const usage_build_generic = \\ --libc [file] Provide a file which specifies libc paths \\ \\Link Options: - \\ -l[lib], --library [lib] Link against system library + \\ -l[lib], --library [lib] Link against system library \\ -L[d], --library-directory [d] Add a directory to the library search path - \\ -T[script] Use a custom linker script - \\ --dynamic-linker [path] Set the dynamic interpreter path (usually ld.so) - \\ --version [ver] Dynamic library semver - \\ -rdynamic Add all symbols to the dynamic symbol table - \\ -rpath [path] Add directory to the runtime library search path - \\ --eh-frame-hdr Enable C++ exception handling by passing --eh-frame-hdr to linker - \\ -dynamic Force output to be dynamically linked - \\ -static Force output to be statically linked + \\ -T[script] Use a custom linker script + \\ --dynamic-linker [path] Set the dynamic interpreter path (usually ld.so) + \\ --version [ver] Dynamic library semver + \\ -rdynamic Add all symbols to the dynamic symbol table + \\ -rpath [path] Add directory to the runtime library search path + \\ --eh-frame-hdr Enable C++ exception handling by passing --eh-frame-hdr to linker + \\ -dynamic Force output to be dynamically linked + \\ -static Force output to be statically linked \\ \\Debug Options (Zig Compiler Development): - \\ -ftime-report Print timing diagnostics - \\ --verbose-link Display linker invocations - \\ --verbose-cc Display C compiler invocations + \\ -ftime-report Print timing diagnostics + \\ --verbose-link Display linker invocations + \\ --verbose-cc Display C compiler invocations + \\ --verbose-tokenize Enable compiler debug output for tokenization + \\ --verbose-ast Enable compiler debug output for AST parsing + \\ --verbose-ir Enable compiler debug output for Zig IR + \\ --verbose-llvm-ir Enable compiler debug output for LLVM IR + \\ --verbose-cimport Enable compiler debug output for C imports + \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features \\ ; @@ -278,6 +284,12 @@ pub fn buildOutputType( var watch = false; var verbose_link = false; var verbose_cc = false; + var verbose_tokenize = false; + var verbose_ast = false; + var verbose_ir = false; + var verbose_llvm_ir = false; + var verbose_cimport = false; + var verbose_llvm_cpu_features = false; var time_report = false; var show_builtin = false; var emit_bin: Emit = .yes_default_path; @@ -548,6 +560,18 @@ pub fn buildOutputType( verbose_link = true; } else if (mem.eql(u8, arg, "--verbose-cc")) { verbose_cc = true; + } else if (mem.eql(u8, arg, "--verbose-tokenize")) { + verbose_tokenize = true; + } else if (mem.eql(u8, arg, "--verbose-ast")) { + verbose_ast = true; + } else if (mem.eql(u8, arg, "--verbose-ir")) { + verbose_ir = true; + } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { + verbose_llvm_ir = true; + } else if (mem.eql(u8, arg, "--verbose-cimport")) { + verbose_cimport = true; + } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { + verbose_llvm_cpu_features = true; } else if (mem.startsWith(u8, arg, "-T")) { linker_script = arg[2..]; } else if (mem.startsWith(u8, arg, "-L")) { @@ -563,28 +587,27 @@ pub fn buildOutputType( } else { fatal("unrecognized parameter: '{}'", .{arg}); } - } else if (mem.endsWith(u8, arg, ".o") or - mem.endsWith(u8, arg, ".obj") or - mem.endsWith(u8, arg, ".a") or - mem.endsWith(u8, arg, ".lib")) - { - try link_objects.append(arg); - } else if (Compilation.hasAsmExt(arg) or Compilation.hasCExt(arg) or Compilation.hasCppExt(arg)) { - // TODO a way to pass extra flags on the CLI - try c_source_files.append(.{ .src_path = arg }); - } else if (mem.endsWith(u8, arg, ".so") or - mem.endsWith(u8, arg, ".dylib") or - mem.endsWith(u8, arg, ".dll")) - { - fatal("linking against dynamic libraries not yet supported", .{}); - } else if (mem.endsWith(u8, arg, ".zig") or mem.endsWith(u8, arg, ".zir")) { - if (root_src_file) |other| { - fatal("found another zig file '{}' after root source file '{}'", .{ arg, other }); - } else { - root_src_file = arg; - } - } else { - fatal("unrecognized file extension of parameter '{}'", .{arg}); + } else switch (Compilation.classifyFileExt(arg)) { + .object, .static_library => { + try link_objects.append(arg); + }, + .assembly, .c, .cpp, .h, .ll, .bc => { + // TODO a way to pass extra flags on the CLI + try c_source_files.append(.{ .src_path = arg }); + }, + .shared_library => { + fatal("linking against dynamic libraries not yet supported", .{}); + }, + .zig, .zir => { + if (root_src_file) |other| { + fatal("found another zig file '{}' after root source file '{}'", .{ arg, other }); + } else { + root_src_file = arg; + } + }, + .unknown => { + fatal("unrecognized file extension of parameter '{}'", .{arg}); + }, } } } else { @@ -617,7 +640,16 @@ pub fn buildOutputType( const file_ext = Compilation.classifyFileExt(mem.spanZ(it.only_arg)); switch (file_ext) { .assembly, .c, .cpp, .ll, .bc, .h => try c_source_files.append(.{ .src_path = it.only_arg }), - .unknown, .so => try link_objects.append(it.only_arg), + .unknown, .shared_library, .object, .static_library => { + try link_objects.append(it.only_arg); + }, + .zig, .zir => { + if (root_src_file) |other| { + fatal("found another zig file '{}' after root source file '{}'", .{ it.only_arg, other }); + } else { + root_src_file = it.only_arg; + } + }, } }, .l => { @@ -1173,7 +1205,15 @@ pub fn buildOutputType( .libc_installation = if (libc_installation) |*lci| lci else null, .verbose_cc = verbose_cc, .verbose_link = verbose_link, + .verbose_tokenize = verbose_tokenize, + .verbose_ast = verbose_ast, + .verbose_ir = verbose_ir, + .verbose_llvm_ir = verbose_llvm_ir, + .verbose_cimport = verbose_cimport, + .verbose_llvm_cpu_features = verbose_llvm_cpu_features, .machine_code_model = machine_code_model, + .color = color, + .time_report = time_report, }) catch |err| { fatal("unable to create compilation: {}", .{@errorName(err)}); }; @@ -1193,6 +1233,10 @@ pub fn buildOutputType( fatal("TODO: implement `zig cc` when using it as a preprocessor", .{}); } + if (build_options.is_stage1 and comp.stage1_module != null and watch) { + std.log.warn("--watch is not recommended with the stage1 backend; it leaks memory and is not capable of incremental compilation", .{}); + } + const stdin = std.io.getStdIn().inStream(); const stderr = std.io.getStdErr().outStream(); var repl_buf: [1024]u8 = undefined; diff --git a/src-self-hosted/stage1.zig b/src-self-hosted/stage1.zig index c2ce99db22..d8d32f02cd 100644 --- a/src-self-hosted/stage1.zig +++ b/src-self-hosted/stage1.zig @@ -10,6 +10,7 @@ const stage2 = @import("main.zig"); const fatal = stage2.fatal; const CrossTarget = std.zig.CrossTarget; const Target = std.Target; +const Compilation = @import("Compilation.zig"); comptime { assert(std.builtin.link_libc); @@ -23,6 +24,8 @@ pub const log_level = stage2.log_level; pub export fn main(argc: c_int, argv: [*]const [*:0]const u8) c_int { std.debug.maybeEnableSegfaultHandler(); + zig_stage1_os_init(); + const gpa = std.heap.c_allocator; var arena_instance = std.heap.ArenaAllocator.init(gpa); defer arena_instance.deinit(); @@ -36,6 +39,106 @@ pub export fn main(argc: c_int, argv: [*]const [*:0]const u8) c_int { return 0; } +/// Matches stage2.Color; +pub const ErrColor = c_int; +/// Matches std.builtin.CodeModel +pub const CodeModel = c_int; +/// Matches std.Target.Os.Tag +pub const OS = c_int; +/// Matches std.builtin.BuildMode +pub const BuildMode = c_int; + +pub const TargetSubsystem = extern enum(c_int) { + Console, + Windows, + Posix, + Native, + EfiApplication, + EfiBootServiceDriver, + EfiRom, + EfiRuntimeDriver, + Auto, +}; + +pub const Pkg = extern struct { + name_ptr: [*]const u8, + name_len: usize, + path_ptr: [*]const u8, + path_len: usize, + children_ptr: [*]*Pkg, + children_len: usize, + parent: ?*Pkg, +}; + +pub const Module = extern struct { + root_name_ptr: [*]const u8, + root_name_len: usize, + output_dir_ptr: [*]const u8, + output_dir_len: usize, + builtin_zig_path_ptr: [*]const u8, + builtin_zig_path_len: usize, + test_filter_ptr: [*]const u8, + test_filter_len: usize, + test_name_prefix_ptr: [*]const u8, + test_name_prefix_len: usize, + userdata: usize, + root_pkg: *Pkg, + main_progress_node: ?*std.Progress.Node, + code_model: CodeModel, + subsystem: TargetSubsystem, + err_color: ErrColor, + pic: bool, + link_libc: bool, + link_libcpp: bool, + strip: bool, + is_single_threaded: bool, + dll_export_fns: bool, + link_mode_dynamic: bool, + valgrind_enabled: bool, + function_sections: bool, + enable_stack_probing: bool, + enable_time_report: bool, + enable_stack_report: bool, + dump_analysis: bool, + enable_doc_generation: bool, + emit_bin: bool, + emit_asm: bool, + emit_llvm_ir: bool, + test_is_evented: bool, + verbose_tokenize: bool, + verbose_ast: bool, + verbose_ir: bool, + verbose_llvm_ir: bool, + verbose_cimport: bool, + verbose_llvm_cpu_features: bool, + + pub fn build_object(mod: *Module) void { + zig_stage1_build_object(mod); + } + + pub fn destroy(mod: *Module) void { + zig_stage1_destroy(mod); + } +}; + +extern fn zig_stage1_os_init() void; + +pub const create = zig_stage1_create; +extern fn zig_stage1_create( + optimize_mode: BuildMode, + main_pkg_path_ptr: [*]const u8, + main_pkg_path_len: usize, + root_src_path_ptr: [*]const u8, + root_src_path_len: usize, + zig_lib_dir_ptr: [*c]const u8, + zig_lib_dir_len: usize, + target: [*c]const Stage2Target, + is_test_build: bool, +) ?*Module; + +extern fn zig_stage1_build_object(*Module) void; +extern fn zig_stage1_destroy(*Module) void; + // ABI warning export fn stage2_panic(ptr: [*]const u8, len: usize) void { @panic(ptr[0..len]); @@ -199,297 +302,50 @@ export fn stage2_progress_update_node(node: *std.Progress.Node, done_count: usiz } // ABI warning -const Stage2Target = extern struct { +pub const Stage2Target = extern struct { arch: c_int, - vendor: c_int, - + os: OS, abi: c_int, - os: c_int, is_native_os: bool, is_native_cpu: bool, llvm_cpu_name: ?[*:0]const u8, llvm_cpu_features: ?[*:0]const u8, - cpu_builtin_str: ?[*:0]const u8, - os_builtin_str: ?[*:0]const u8, - - dynamic_linker: ?[*:0]const u8, - - llvm_cpu_features_asm_ptr: [*]const [*:0]const u8, - llvm_cpu_features_asm_len: usize, - - fn fromTarget(self: *Stage2Target, cross_target: CrossTarget) !void { - const allocator = std.heap.c_allocator; - - var dynamic_linker: ?[*:0]u8 = null; - const target = try crossTargetToTarget(cross_target, &dynamic_linker); - - const generic_arch_name = target.cpu.arch.genericName(); - var cpu_builtin_str_buffer = try std.ArrayListSentineled(u8, 0).allocPrint(allocator, - \\Cpu{{ - \\ .arch = .{}, - \\ .model = &Target.{}.cpu.{}, - \\ .features = Target.{}.featureSet(&[_]Target.{}.Feature{{ - \\ - , .{ - @tagName(target.cpu.arch), - generic_arch_name, - target.cpu.model.name, - generic_arch_name, - generic_arch_name, - }); - defer cpu_builtin_str_buffer.deinit(); - - var llvm_features_buffer = try std.ArrayListSentineled(u8, 0).initSize(allocator, 0); - defer llvm_features_buffer.deinit(); - - // Unfortunately we have to do the work twice, because Clang does not support - // the same command line parameters for CPU features when assembling code as it does - // when compiling C code. - var asm_features_list = std.ArrayList([*:0]const u8).init(allocator); - defer asm_features_list.deinit(); - - for (target.cpu.arch.allFeaturesList()) |feature, index_usize| { - const index = @intCast(Target.Cpu.Feature.Set.Index, index_usize); - const is_enabled = target.cpu.features.isEnabled(index); - - if (feature.llvm_name) |llvm_name| { - const plus_or_minus = "-+"[@boolToInt(is_enabled)]; - try llvm_features_buffer.append(plus_or_minus); - try llvm_features_buffer.appendSlice(llvm_name); - try llvm_features_buffer.appendSlice(","); - } - - if (is_enabled) { - // TODO some kind of "zig identifier escape" function rather than - // unconditionally using @"" syntax - try cpu_builtin_str_buffer.appendSlice(" .@\""); - try cpu_builtin_str_buffer.appendSlice(feature.name); - try cpu_builtin_str_buffer.appendSlice("\",\n"); - } - } - - switch (target.cpu.arch) { - .riscv32, .riscv64 => { - if (Target.riscv.featureSetHas(target.cpu.features, .relax)) { - try asm_features_list.append("-mrelax"); - } else { - try asm_features_list.append("-mno-relax"); - } - }, - else => { - // TODO - // Argh, why doesn't the assembler accept the list of CPU features?! - // I don't see a way to do this other than hard coding everything. - }, - } - - try cpu_builtin_str_buffer.appendSlice( - \\ }), - \\}; - \\ - ); - - assert(mem.endsWith(u8, llvm_features_buffer.span(), ",")); - llvm_features_buffer.shrink(llvm_features_buffer.len() - 1); - - var os_builtin_str_buffer = try std.ArrayListSentineled(u8, 0).allocPrint(allocator, - \\Os{{ - \\ .tag = .{}, - \\ .version_range = .{{ - , .{@tagName(target.os.tag)}); - defer os_builtin_str_buffer.deinit(); - - // We'll re-use the OS version range builtin string for the cache hash. - const os_builtin_str_ver_start_index = os_builtin_str_buffer.len(); - - @setEvalBranchQuota(2000); - switch (target.os.tag) { - .freestanding, - .ananas, - .cloudabi, - .dragonfly, - .fuchsia, - .ios, - .kfreebsd, - .lv2, - .solaris, - .haiku, - .minix, - .rtems, - .nacl, - .cnk, - .aix, - .cuda, - .nvcl, - .amdhsa, - .ps4, - .elfiamcu, - .tvos, - .watchos, - .mesa3d, - .contiki, - .amdpal, - .hermit, - .hurd, - .wasi, - .emscripten, - .uefi, - .other, - => try os_builtin_str_buffer.appendSlice(" .none = {} }\n"), - - .freebsd, - .macosx, - .netbsd, - .openbsd, - => try os_builtin_str_buffer.outStream().print( - \\ .semver = .{{ - \\ .min = .{{ - \\ .major = {}, - \\ .minor = {}, - \\ .patch = {}, - \\ }}, - \\ .max = .{{ - \\ .major = {}, - \\ .minor = {}, - \\ .patch = {}, - \\ }}, - \\ }}}}, - \\ - , .{ - target.os.version_range.semver.min.major, - target.os.version_range.semver.min.minor, - target.os.version_range.semver.min.patch, - - target.os.version_range.semver.max.major, - target.os.version_range.semver.max.minor, - target.os.version_range.semver.max.patch, - }), - - .linux => try os_builtin_str_buffer.outStream().print( - \\ .linux = .{{ - \\ .range = .{{ - \\ .min = .{{ - \\ .major = {}, - \\ .minor = {}, - \\ .patch = {}, - \\ }}, - \\ .max = .{{ - \\ .major = {}, - \\ .minor = {}, - \\ .patch = {}, - \\ }}, - \\ }}, - \\ .glibc = .{{ - \\ .major = {}, - \\ .minor = {}, - \\ .patch = {}, - \\ }}, - \\ }}}}, - \\ - , .{ - target.os.version_range.linux.range.min.major, - target.os.version_range.linux.range.min.minor, - target.os.version_range.linux.range.min.patch, - - target.os.version_range.linux.range.max.major, - target.os.version_range.linux.range.max.minor, - target.os.version_range.linux.range.max.patch, - - target.os.version_range.linux.glibc.major, - target.os.version_range.linux.glibc.minor, - target.os.version_range.linux.glibc.patch, - }), - - .windows => try os_builtin_str_buffer.outStream().print( - \\ .windows = .{{ - \\ .min = {s}, - \\ .max = {s}, - \\ }}}}, - \\ - , .{ - target.os.version_range.windows.min, - target.os.version_range.windows.max, - }), - } - try os_builtin_str_buffer.appendSlice("};\n"); - - const glibc_or_darwin_version = blk: { - if (target.isGnuLibC()) { - const stage1_glibc = try std.heap.c_allocator.create(Stage2SemVer); - const stage2_glibc = target.os.version_range.linux.glibc; - stage1_glibc.* = .{ - .major = stage2_glibc.major, - .minor = stage2_glibc.minor, - .patch = stage2_glibc.patch, - }; - break :blk stage1_glibc; - } else if (target.isDarwin()) { - const stage1_semver = try std.heap.c_allocator.create(Stage2SemVer); - const stage2_semver = target.os.version_range.semver.min; - stage1_semver.* = .{ - .major = stage2_semver.major, - .minor = stage2_semver.minor, - .patch = stage2_semver.patch, - }; - break :blk stage1_semver; - } else { - break :blk null; - } - }; - - const std_dl = target.standardDynamicLinkerPath(); - const std_dl_z = if (std_dl.get()) |dl| - (try mem.dupeZ(std.heap.c_allocator, u8, dl)).ptr - else - null; - - const asm_features = asm_features_list.toOwnedSlice(); - self.* = .{ - .arch = @enumToInt(target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch - .vendor = 0, - .os = @enumToInt(target.os.tag), - .abi = @enumToInt(target.abi), - .llvm_cpu_name = if (target.cpu.model.llvm_name) |s| s.ptr else null, - .llvm_cpu_features = llvm_features_buffer.toOwnedSlice().ptr, - .llvm_cpu_features_asm_ptr = asm_features.ptr, - .llvm_cpu_features_asm_len = asm_features.len, - .cpu_builtin_str = cpu_builtin_str_buffer.toOwnedSlice().ptr, - .os_builtin_str = os_builtin_str_buffer.toOwnedSlice().ptr, - .is_native_os = cross_target.isNativeOs(), - .is_native_cpu = cross_target.isNativeCpu(), - .glibc_or_darwin_version = glibc_or_darwin_version, - .dynamic_linker = dynamic_linker, - .standard_dynamic_linker_path = std_dl_z, - }; - } }; -fn crossTargetToTarget(cross_target: CrossTarget, dynamic_linker_ptr: *?[*:0]u8) !Target { - var info = try std.zig.system.NativeTargetInfo.detect(std.heap.c_allocator, cross_target); - if (info.cpu_detection_unimplemented) { - // TODO We want to just use detected_info.target but implementing - // CPU model & feature detection is todo so here we rely on LLVM. - const llvm = @import("llvm.zig"); - const llvm_cpu_name = llvm.GetHostCPUName(); - const llvm_cpu_features = llvm.GetNativeFeatures(); - const arch = Target.current.cpu.arch; - info.target.cpu = try detectNativeCpuWithLLVM(arch, llvm_cpu_name, llvm_cpu_features); - cross_target.updateCpuFeatures(&info.target.cpu.features); - info.target.cpu.arch = cross_target.getCpuArch(); - } - if (info.dynamic_linker.get()) |dl| { - dynamic_linker_ptr.* = try mem.dupeZ(std.heap.c_allocator, u8, dl); - } else { - dynamic_linker_ptr.* = null; - } - return info.target; -} - // ABI warning const Stage2SemVer = extern struct { major: u32, minor: u32, patch: u32, }; + +// ABI warning +export fn stage2_cimport(stage1: *Module) [*:0]const u8 { + @panic("TODO implement stage2_cimport"); +} + +export fn stage2_add_link_lib( + stage1: *Module, + lib_name_ptr: [*c]const u8, + lib_name_len: usize, + symbol_name_ptr: [*c]const u8, + symbol_name_len: usize, +) ?[*:0]const u8 { + return null; // no error +} + +export fn stage2_fetch_file( + stage1: *Module, + path_ptr: [*]const u8, + path_len: usize, + result_len: *usize, +) ?[*]const u8 { + const comp = @intToPtr(*Compilation, stage1.userdata); + // TODO integrate this with cache hash + const file_path = path_ptr[0..path_len]; + const contents = std.fs.cwd().readFileAlloc(comp.gpa, file_path, std.math.maxInt(u32)) catch return null; + result_len.* = contents.len; + return contents.ptr; +} diff --git a/src/analyze.cpp b/src/analyze.cpp index ea19d81995..6c6f198e7a 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -7987,7 +7987,7 @@ Error file_fetch(CodeGen *g, Buf *resolved_path, Buf *contents_buf) { size_t len; const char *contents = stage2_fetch_file(&g->stage1, buf_ptr(resolved_path), buf_len(resolved_path), &len); if (contents == nullptr) - return ErrorNoMem; + return ErrorFileNotFound; buf_init_from_mem(contents_buf, contents, len); return ErrorNone; } diff --git a/src/codegen.cpp b/src/codegen.cpp index 07728ef06e..6768021410 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -8809,33 +8809,18 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { static_assert(TargetSubsystemEfiBootServiceDriver == 5, ""); static_assert(TargetSubsystemEfiRom == 6, ""); static_assert(TargetSubsystemEfiRuntimeDriver == 7, ""); - { - const char *endian_str = g->is_big_endian ? "Endian.Big" : "Endian.Little"; - buf_appendf(contents, "pub const endian = %s;\n", endian_str); - } + + buf_append_str(contents, "/// Deprecated: use `std.Target.current.cpu.arch`\n"); + buf_append_str(contents, "pub const arch = Target.current.cpu.arch;\n"); + buf_append_str(contents, "/// Deprecated: use `std.Target.current.cpu.arch.endian()`\n"); + buf_append_str(contents, "pub const endian = Target.current.cpu.arch.endian();\n"); buf_appendf(contents, "pub const output_mode = OutputMode.Obj;\n"); buf_appendf(contents, "pub const link_mode = LinkMode.Static;\n"); buf_appendf(contents, "pub const is_test = false;\n"); buf_appendf(contents, "pub const single_threaded = %s;\n", bool_to_str(g->is_single_threaded)); - buf_append_str(contents, "/// Deprecated: use `std.Target.cpu.arch`\n"); - buf_appendf(contents, "pub const arch = Arch.%s;\n", cur_arch); buf_appendf(contents, "pub const abi = Abi.%s;\n", cur_abi); - { - buf_append_str(contents, "pub const cpu: Cpu = "); - if (g->zig_target->cpu_builtin_str != nullptr) { - buf_append_str(contents, g->zig_target->cpu_builtin_str); - } else { - buf_appendf(contents, "Target.Cpu.baseline(.%s);\n", cur_arch); - } - } - { - buf_append_str(contents, "pub const os = "); - if (g->zig_target->os_builtin_str != nullptr) { - buf_append_str(contents, g->zig_target->os_builtin_str); - } else { - buf_appendf(contents, "Target.Os.Tag.defaultVersionRange(.%s);\n", cur_os); - } - } + buf_appendf(contents, "pub const cpu: Cpu = Target.Cpu.baseline(.%s);\n", cur_arch); + buf_appendf(contents, "pub const os = Target.Os.Tag.defaultVersionRange(.%s);\n", cur_os); buf_appendf(contents, "pub const object_format = ObjectFormat.%s;\n", cur_obj_fmt); buf_appendf(contents, "pub const mode = %s;\n", build_mode_to_str(g->build_mode)); buf_appendf(contents, "pub const link_libc = %s;\n", bool_to_str(g->link_libc)); @@ -8866,6 +8851,8 @@ static Error define_builtin_compile_vars(CodeGen *g) { if (g->std_package == nullptr) return ErrorNone; + assert(g->main_pkg); + const char *builtin_zig_basename = "builtin.zig"; Buf *contents; @@ -8879,17 +8866,22 @@ static Error define_builtin_compile_vars(CodeGen *g) { fprintf(stderr, "Unable to write file '%s': %s\n", buf_ptr(g->builtin_zig_path), err_str(err)); exit(1); } + + g->compile_var_package = new_package(buf_ptr(g->output_dir), builtin_zig_basename, "builtin"); } else { + Buf *resolve_paths[] = { g->builtin_zig_path, }; + *g->builtin_zig_path = os_path_resolve(resolve_paths, 1); + contents = buf_alloc(); if ((err = os_fetch_file_path(g->builtin_zig_path, contents))) { fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(g->builtin_zig_path), err_str(err)); exit(1); } + Buf builtin_dirname = BUF_INIT; + os_path_dirname(g->builtin_zig_path, &builtin_dirname); + g->compile_var_package = new_package(buf_ptr(&builtin_dirname), builtin_zig_basename, "builtin"); } - assert(g->main_pkg); - assert(g->std_package); - g->compile_var_package = new_package(buf_ptr(g->output_dir), builtin_zig_basename, "builtin"); if (g->is_test_build) { if (g->test_runner_package == nullptr) { g->test_runner_package = create_test_runner_pkg(g); @@ -8914,6 +8906,13 @@ static void init(CodeGen *g) { if (g->module) return; + codegen_add_time_event(g, "Initialize"); + { + const char *progress_name = "Initialize"; + codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, + progress_name, strlen(progress_name), 0)); + } + g->have_err_ret_tracing = detect_err_ret_tracing(g); assert(g->root_out_name); @@ -9374,19 +9373,11 @@ void codegen_destroy(CodeGen *g) { CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget *target, BuildMode build_mode, Buf *override_lib_dir, - bool is_test_build, Stage2ProgressNode *progress_node) + bool is_test_build) { CodeGen *g = heap::c_allocator.create(); g->emit_bin = true; g->pass1_arena = heap::ArenaAllocator::construct(&heap::c_allocator, &heap::c_allocator, "pass1"); - g->main_progress_node = progress_node; - - codegen_add_time_event(g, "Initialize"); - { - const char *progress_name = "Initialize"; - codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, - progress_name, strlen(progress_name), 0)); - } g->subsystem = TargetSubsystemAuto; g->zig_target = target; @@ -9440,7 +9431,7 @@ CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget Buf resolved_main_pkg_path = os_path_resolve(&main_pkg_path, 1); if (!buf_starts_with_buf(&resolved_root_src_path, &resolved_main_pkg_path)) { - fprintf(stderr, "Root source path '%s' outside main package path '%s'", + fprintf(stderr, "Root source path '%s' outside main package path '%s'\n", buf_ptr(root_src_path), buf_ptr(main_pkg_path)); exit(1); } diff --git a/src/codegen.hpp b/src/codegen.hpp index 5b4317c992..33b2f74757 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -16,8 +16,7 @@ #include CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget *target, - BuildMode build_mode, Buf *zig_lib_dir, - bool is_test_build, Stage2ProgressNode *progress_node); + BuildMode build_mode, Buf *zig_lib_dir, bool is_test_build); void codegen_build_object(CodeGen *g); void codegen_destroy(CodeGen *); diff --git a/src/stage1.cpp b/src/stage1.cpp index 36d59c0a05..b5b87c05d9 100644 --- a/src/stage1.cpp +++ b/src/stage1.cpp @@ -20,13 +20,14 @@ struct ZigStage1 *zig_stage1_create(BuildMode optimize_mode, const char *main_pkg_path_ptr, size_t main_pkg_path_len, const char *root_src_path_ptr, size_t root_src_path_len, const char *zig_lib_dir_ptr, size_t zig_lib_dir_len, - const ZigTarget *target, bool is_test_build, Stage2ProgressNode *progress_node) + const ZigTarget *target, bool is_test_build) { - Buf *main_pkg_path = buf_create_from_mem(main_pkg_path_ptr, main_pkg_path_len); + Buf *main_pkg_path = (main_pkg_path_len == 0) ? + nullptr : buf_create_from_mem(main_pkg_path_ptr, main_pkg_path_len); Buf *root_src_path = buf_create_from_mem(root_src_path_ptr, root_src_path_len); Buf *zig_lib_dir = buf_create_from_mem(zig_lib_dir_ptr, zig_lib_dir_len); CodeGen *g = codegen_create(main_pkg_path, root_src_path, target, optimize_mode, - zig_lib_dir, is_test_build, progress_node); + zig_lib_dir, is_test_build); return &g->stage1; } @@ -68,8 +69,6 @@ void zig_stage1_build_object(struct ZigStage1 *stage1) { CodeGen *g = reinterpret_cast(stage1); g->root_out_name = buf_create_from_mem(stage1->root_name_ptr, stage1->root_name_len); - g->zig_lib_dir = buf_create_from_mem(stage1->zig_lib_dir_ptr, stage1->zig_lib_dir_len); - g->zig_std_dir = buf_create_from_mem(stage1->zig_std_dir_ptr, stage1->zig_std_dir_len); g->output_dir = buf_create_from_mem(stage1->output_dir_ptr, stage1->output_dir_len); if (stage1->builtin_zig_path_len != 0) { g->builtin_zig_path = buf_create_from_mem(stage1->builtin_zig_path_ptr, stage1->builtin_zig_path_len); @@ -119,6 +118,8 @@ void zig_stage1_build_object(struct ZigStage1 *stage1) { } } + g->main_progress_node = stage1->main_progress_node; + add_package(g, stage1->root_pkg, g->main_pkg); codegen_build_object(g); diff --git a/src/stage1.h b/src/stage1.h index 5b0c4a4df6..c0bd0f70fc 100644 --- a/src/stage1.h +++ b/src/stage1.h @@ -100,22 +100,14 @@ enum Os { // ABI warning struct ZigTarget { enum ZigLLVM_ArchType arch; - enum ZigLLVM_VendorType vendor; - + enum Os os; enum ZigLLVM_EnvironmentType abi; - Os os; bool is_native_os; bool is_native_cpu; const char *llvm_cpu_name; const char *llvm_cpu_features; - const char *cpu_builtin_str; - const char *os_builtin_str; - const char *dynamic_linker; - - const char **llvm_cpu_features_asm_ptr; - size_t llvm_cpu_features_asm_len; }; // ABI warning @@ -161,18 +153,13 @@ struct ZigStage1 { const char *test_name_prefix_ptr; size_t test_name_prefix_len; - const char *zig_lib_dir_ptr; - size_t zig_lib_dir_len; - - const char *zig_std_dir_ptr; - size_t zig_std_dir_len; - void *userdata; struct ZigStage1Pkg *root_pkg; + struct Stage2ProgressNode *main_progress_node; - CodeModel code_model; - TargetSubsystem subsystem; - ErrColor err_color; + enum CodeModel code_model; + enum TargetSubsystem subsystem; + enum ErrColor err_color; bool pic; bool link_libc; @@ -206,7 +193,7 @@ ZIG_EXTERN_C struct ZigStage1 *zig_stage1_create(enum BuildMode optimize_mode, const char *main_pkg_path_ptr, size_t main_pkg_path_len, const char *root_src_path_ptr, size_t root_src_path_len, const char *zig_lib_dir_ptr, size_t zig_lib_dir_len, - const ZigTarget *target, bool is_test_build, Stage2ProgressNode *progress_node); + const struct ZigTarget *target, bool is_test_build); ZIG_EXTERN_C void zig_stage1_build_object(struct ZigStage1 *); diff --git a/src/stage2.cpp b/src/stage2.cpp deleted file mode 100644 index f8b078c237..0000000000 --- a/src/stage2.cpp +++ /dev/null @@ -1,241 +0,0 @@ -// This file is a shim for zig1. The real implementations of these are in -// src-self-hosted/stage1.zig - -#include "stage2.h" -#include "util.hpp" -#include "zig_llvm.h" -#include "target.hpp" -#include "buffer.hpp" -#include "os.hpp" -#include -#include -#include - -void stage2_panic(const char *ptr, size_t len) { - fwrite(ptr, 1, len, stderr); - fprintf(stderr, "\n"); - fflush(stderr); - abort(); -} - -struct Stage2Progress { - int trash; -}; - -struct Stage2ProgressNode { - int trash; -}; - -Stage2Progress *stage2_progress_create(void) { - return nullptr; -} - -void stage2_progress_destroy(Stage2Progress *progress) {} - -Stage2ProgressNode *stage2_progress_start_root(Stage2Progress *progress, - const char *name_ptr, size_t name_len, size_t estimated_total_items) -{ - return nullptr; -} -Stage2ProgressNode *stage2_progress_start(Stage2ProgressNode *node, - const char *name_ptr, size_t name_len, size_t estimated_total_items) -{ - return nullptr; -} -void stage2_progress_end(Stage2ProgressNode *node) {} -void stage2_progress_complete_one(Stage2ProgressNode *node) {} -void stage2_progress_disable_tty(Stage2Progress *progress) {} -void stage2_progress_update_node(Stage2ProgressNode *node, size_t completed_count, size_t estimated_total_items){} - -static Os get_zig_os_type(ZigLLVM_OSType os_type) { - switch (os_type) { - case ZigLLVM_UnknownOS: - return OsFreestanding; - case ZigLLVM_Ananas: - return OsAnanas; - case ZigLLVM_CloudABI: - return OsCloudABI; - case ZigLLVM_DragonFly: - return OsDragonFly; - case ZigLLVM_FreeBSD: - return OsFreeBSD; - case ZigLLVM_Fuchsia: - return OsFuchsia; - case ZigLLVM_IOS: - return OsIOS; - case ZigLLVM_KFreeBSD: - return OsKFreeBSD; - case ZigLLVM_Linux: - return OsLinux; - case ZigLLVM_Lv2: - return OsLv2; - case ZigLLVM_Darwin: - case ZigLLVM_MacOSX: - return OsMacOSX; - case ZigLLVM_NetBSD: - return OsNetBSD; - case ZigLLVM_OpenBSD: - return OsOpenBSD; - case ZigLLVM_Solaris: - return OsSolaris; - case ZigLLVM_Win32: - return OsWindows; - case ZigLLVM_Haiku: - return OsHaiku; - case ZigLLVM_Minix: - return OsMinix; - case ZigLLVM_RTEMS: - return OsRTEMS; - case ZigLLVM_NaCl: - return OsNaCl; - case ZigLLVM_CNK: - return OsCNK; - case ZigLLVM_AIX: - return OsAIX; - case ZigLLVM_CUDA: - return OsCUDA; - case ZigLLVM_NVCL: - return OsNVCL; - case ZigLLVM_AMDHSA: - return OsAMDHSA; - case ZigLLVM_PS4: - return OsPS4; - case ZigLLVM_ELFIAMCU: - return OsELFIAMCU; - case ZigLLVM_TvOS: - return OsTvOS; - case ZigLLVM_WatchOS: - return OsWatchOS; - case ZigLLVM_Mesa3D: - return OsMesa3D; - case ZigLLVM_Contiki: - return OsContiki; - case ZigLLVM_AMDPAL: - return OsAMDPAL; - case ZigLLVM_HermitCore: - return OsHermitCore; - case ZigLLVM_Hurd: - return OsHurd; - case ZigLLVM_WASI: - return OsWASI; - case ZigLLVM_Emscripten: - return OsEmscripten; - } - zig_unreachable(); -} - -static void get_native_target(ZigTarget *target) { - // first zero initialize - *target = {}; - - ZigLLVM_OSType os_type; - ZigLLVM_ObjectFormatType oformat; // ignored; based on arch/os - ZigLLVMGetNativeTarget( - &target->arch, - &target->vendor, - &os_type, - &target->abi, - &oformat); - target->os = get_zig_os_type(os_type); - target->is_native_os = true; - target->is_native_cpu = true; - if (target->abi == ZigLLVM_UnknownEnvironment) { - target->abi = target_default_abi(target->arch, target->os); - } -} - -Error stage2_target_parse(struct ZigTarget *target, const char *zig_triple, const char *mcpu, - const char *dynamic_linker) -{ - Error err; - - if (zig_triple != nullptr && strcmp(zig_triple, "native") == 0) { - zig_triple = nullptr; - } - - if (zig_triple == nullptr) { - get_native_target(target); - - if (mcpu == nullptr) { - target->llvm_cpu_name = ZigLLVMGetHostCPUName(); - target->llvm_cpu_features = ZigLLVMGetNativeFeatures(); - } else if (strcmp(mcpu, "baseline") == 0) { - target->is_native_os = false; - target->is_native_cpu = false; - target->llvm_cpu_name = ""; - target->llvm_cpu_features = ""; - } else { - const char *msg = "stage0 can't handle CPU/features in the target"; - stage2_panic(msg, strlen(msg)); - } - } else { - // first initialize all to zero - *target = {}; - - SplitIterator it = memSplit(str(zig_triple), str("-")); - - Optional> opt_archsub = SplitIterator_next(&it); - Optional> opt_os = SplitIterator_next(&it); - Optional> opt_abi = SplitIterator_next(&it); - - if (!opt_archsub.is_some) - return ErrorMissingArchitecture; - - if ((err = target_parse_arch(&target->arch, (char*)opt_archsub.value.ptr, opt_archsub.value.len))) { - return err; - } - - if (!opt_os.is_some) - return ErrorMissingOperatingSystem; - - if ((err = target_parse_os(&target->os, (char*)opt_os.value.ptr, opt_os.value.len))) { - return err; - } - - if (opt_abi.is_some) { - if ((err = target_parse_abi(&target->abi, (char*)opt_abi.value.ptr, opt_abi.value.len))) { - return err; - } - } else { - target->abi = target_default_abi(target->arch, target->os); - } - - if (mcpu != nullptr && strcmp(mcpu, "baseline") != 0) { - const char *msg = "stage0 can't handle CPU/features in the target"; - stage2_panic(msg, strlen(msg)); - } - } - - if (dynamic_linker != nullptr) { - target->dynamic_linker = dynamic_linker; - } - - return ErrorNone; -} - -const char *stage2_fetch_file(struct ZigStage1 *stage1, const char *path_ptr, size_t path_len, - size_t *result_len) -{ - Error err; - Buf contents_buf = BUF_INIT; - Buf path_buf = BUF_INIT; - - buf_init_from_mem(&path_buf, path_ptr, path_len); - if ((err = os_fetch_file_path(&path_buf, &contents_buf))) { - return nullptr; - } - *result_len = buf_len(&contents_buf); - return buf_ptr(&contents_buf); -} - -const char *stage2_cimport(struct ZigStage1 *stage1) { - const char *msg = "stage0 called stage2_cimport"; - stage2_panic(msg, strlen(msg)); -} - -const char *stage2_add_link_lib(struct ZigStage1 *stage1, - const char *lib_name_ptr, size_t lib_name_len, - const char *symbol_name_ptr, size_t symbol_name_len) -{ - return nullptr; -} diff --git a/src/stage2.h b/src/stage2.h index 4bc92631dc..104faa6ca8 100644 --- a/src/stage2.h +++ b/src/stage2.h @@ -130,23 +130,23 @@ struct Stage2ErrorMsg { ZIG_EXTERN_C ZIG_ATTRIBUTE_NORETURN void stage2_panic(const char *ptr, size_t len); // ABI warning -ZIG_EXTERN_C Stage2Progress *stage2_progress_create(void); +ZIG_EXTERN_C struct Stage2Progress *stage2_progress_create(void); // ABI warning -ZIG_EXTERN_C void stage2_progress_disable_tty(Stage2Progress *progress); +ZIG_EXTERN_C void stage2_progress_disable_tty(struct Stage2Progress *progress); // ABI warning -ZIG_EXTERN_C void stage2_progress_destroy(Stage2Progress *progress); +ZIG_EXTERN_C void stage2_progress_destroy(struct Stage2Progress *progress); // ABI warning -ZIG_EXTERN_C Stage2ProgressNode *stage2_progress_start_root(Stage2Progress *progress, +ZIG_EXTERN_C struct Stage2ProgressNode *stage2_progress_start_root(struct Stage2Progress *progress, const char *name_ptr, size_t name_len, size_t estimated_total_items); // ABI warning -ZIG_EXTERN_C Stage2ProgressNode *stage2_progress_start(Stage2ProgressNode *node, +ZIG_EXTERN_C struct Stage2ProgressNode *stage2_progress_start(struct Stage2ProgressNode *node, const char *name_ptr, size_t name_len, size_t estimated_total_items); // ABI warning -ZIG_EXTERN_C void stage2_progress_end(Stage2ProgressNode *node); +ZIG_EXTERN_C void stage2_progress_end(struct Stage2ProgressNode *node); // ABI warning -ZIG_EXTERN_C void stage2_progress_complete_one(Stage2ProgressNode *node); +ZIG_EXTERN_C void stage2_progress_complete_one(struct Stage2ProgressNode *node); // ABI warning -ZIG_EXTERN_C void stage2_progress_update_node(Stage2ProgressNode *node, +ZIG_EXTERN_C void stage2_progress_update_node(struct Stage2ProgressNode *node, size_t completed_count, size_t estimated_total_items); // ABI warning diff --git a/src/target.cpp b/src/target.cpp index f1cab8eded..433c988a01 100644 --- a/src/target.cpp +++ b/src/target.cpp @@ -380,10 +380,6 @@ Error target_parse_abi(ZigLLVM_EnvironmentType *out_abi, const char *abi_ptr, si return ErrorUnknownABI; } -Error target_parse_triple(ZigTarget *target, const char *triple, const char *mcpu, const char *dynamic_linker) { - return stage2_target_parse(target, triple, mcpu, dynamic_linker); -} - const char *target_arch_name(ZigLLVM_ArchType arch) { return ZigLLVMGetArchTypeName(arch); } @@ -408,7 +404,7 @@ void target_triple_llvm(Buf *triple, const ZigTarget *target) { buf_resize(triple, 0); buf_appendf(triple, "%s-%s-%s-%s", ZigLLVMGetArchTypeName(target->arch), - ZigLLVMGetVendorTypeName(target->vendor), + ZigLLVMGetVendorTypeName(ZigLLVM_UnknownVendor), ZigLLVMGetOSTypeName(get_llvm_os_type(target->os)), ZigLLVMGetEnvironmentTypeName(target->abi)); } @@ -1149,7 +1145,6 @@ void target_libc_enum(size_t index, ZigTarget *out_target) { out_target->arch = libcs_available[index].arch; out_target->os = libcs_available[index].os; out_target->abi = libcs_available[index].abi; - out_target->vendor = ZigLLVM_UnknownVendor; out_target->is_native_os = false; out_target->is_native_cpu = false; } diff --git a/src/target.hpp b/src/target.hpp index c9db4754f9..d34c2aeae5 100644 --- a/src/target.hpp +++ b/src/target.hpp @@ -25,7 +25,6 @@ enum CIntType { CIntTypeCount, }; -Error target_parse_triple(ZigTarget *target, const char *triple, const char *mcpu, const char *dynamic_linker); Error target_parse_arch(ZigLLVM_ArchType *arch, const char *arch_ptr, size_t arch_len); Error target_parse_os(Os *os, const char *os_ptr, size_t os_len); Error target_parse_abi(ZigLLVM_EnvironmentType *abi, const char *abi_ptr, size_t abi_len); diff --git a/src/zig0.cpp b/src/zig0.cpp index 62378ff68c..bd447abded 100644 --- a/src/zig0.cpp +++ b/src/zig0.cpp @@ -14,6 +14,9 @@ #include "stage2.h" #include "target.hpp" #include "error.hpp" +#include "util.hpp" +#include "buffer.hpp" +#include "os.hpp" #include #include @@ -33,7 +36,6 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) { " --output-dir [dir] override output directory (defaults to cwd)\n" " --pkg-begin [name] [path] make pkg available to import and push current pkg\n" " --pkg-end pop current pkg\n" - " --main-pkg-path set the directory of the root package\n" " --release-fast build with optimizations on and safety off\n" " --release-safe build with optimizations on and safety on\n" " --release-small build with size optimizations on and safety off\n" @@ -53,6 +55,170 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) { return return_code; } +static Os get_zig_os_type(ZigLLVM_OSType os_type) { + switch (os_type) { + case ZigLLVM_UnknownOS: + return OsFreestanding; + case ZigLLVM_Ananas: + return OsAnanas; + case ZigLLVM_CloudABI: + return OsCloudABI; + case ZigLLVM_DragonFly: + return OsDragonFly; + case ZigLLVM_FreeBSD: + return OsFreeBSD; + case ZigLLVM_Fuchsia: + return OsFuchsia; + case ZigLLVM_IOS: + return OsIOS; + case ZigLLVM_KFreeBSD: + return OsKFreeBSD; + case ZigLLVM_Linux: + return OsLinux; + case ZigLLVM_Lv2: + return OsLv2; + case ZigLLVM_Darwin: + case ZigLLVM_MacOSX: + return OsMacOSX; + case ZigLLVM_NetBSD: + return OsNetBSD; + case ZigLLVM_OpenBSD: + return OsOpenBSD; + case ZigLLVM_Solaris: + return OsSolaris; + case ZigLLVM_Win32: + return OsWindows; + case ZigLLVM_Haiku: + return OsHaiku; + case ZigLLVM_Minix: + return OsMinix; + case ZigLLVM_RTEMS: + return OsRTEMS; + case ZigLLVM_NaCl: + return OsNaCl; + case ZigLLVM_CNK: + return OsCNK; + case ZigLLVM_AIX: + return OsAIX; + case ZigLLVM_CUDA: + return OsCUDA; + case ZigLLVM_NVCL: + return OsNVCL; + case ZigLLVM_AMDHSA: + return OsAMDHSA; + case ZigLLVM_PS4: + return OsPS4; + case ZigLLVM_ELFIAMCU: + return OsELFIAMCU; + case ZigLLVM_TvOS: + return OsTvOS; + case ZigLLVM_WatchOS: + return OsWatchOS; + case ZigLLVM_Mesa3D: + return OsMesa3D; + case ZigLLVM_Contiki: + return OsContiki; + case ZigLLVM_AMDPAL: + return OsAMDPAL; + case ZigLLVM_HermitCore: + return OsHermitCore; + case ZigLLVM_Hurd: + return OsHurd; + case ZigLLVM_WASI: + return OsWASI; + case ZigLLVM_Emscripten: + return OsEmscripten; + } + zig_unreachable(); +} + +static void get_native_target(ZigTarget *target) { + // first zero initialize + *target = {}; + + ZigLLVM_OSType os_type; + ZigLLVM_ObjectFormatType oformat; // ignored; based on arch/os + ZigLLVM_VendorType trash; + ZigLLVMGetNativeTarget( + &target->arch, + &trash, + &os_type, + &target->abi, + &oformat); + target->os = get_zig_os_type(os_type); + target->is_native_os = true; + target->is_native_cpu = true; + if (target->abi == ZigLLVM_UnknownEnvironment) { + target->abi = target_default_abi(target->arch, target->os); + } +} + +static Error target_parse_triple(struct ZigTarget *target, const char *zig_triple, const char *mcpu, + const char *dynamic_linker) +{ + Error err; + + if (zig_triple != nullptr && strcmp(zig_triple, "native") == 0) { + zig_triple = nullptr; + } + + if (zig_triple == nullptr) { + get_native_target(target); + + if (mcpu == nullptr) { + target->llvm_cpu_name = ZigLLVMGetHostCPUName(); + target->llvm_cpu_features = ZigLLVMGetNativeFeatures(); + } else if (strcmp(mcpu, "baseline") == 0) { + target->is_native_os = false; + target->is_native_cpu = false; + target->llvm_cpu_name = ""; + target->llvm_cpu_features = ""; + } else { + const char *msg = "stage0 can't handle CPU/features in the target"; + stage2_panic(msg, strlen(msg)); + } + } else { + // first initialize all to zero + *target = {}; + + SplitIterator it = memSplit(str(zig_triple), str("-")); + + Optional> opt_archsub = SplitIterator_next(&it); + Optional> opt_os = SplitIterator_next(&it); + Optional> opt_abi = SplitIterator_next(&it); + + if (!opt_archsub.is_some) + return ErrorMissingArchitecture; + + if ((err = target_parse_arch(&target->arch, (char*)opt_archsub.value.ptr, opt_archsub.value.len))) { + return err; + } + + if (!opt_os.is_some) + return ErrorMissingOperatingSystem; + + if ((err = target_parse_os(&target->os, (char*)opt_os.value.ptr, opt_os.value.len))) { + return err; + } + + if (opt_abi.is_some) { + if ((err = target_parse_abi(&target->abi, (char*)opt_abi.value.ptr, opt_abi.value.len))) { + return err; + } + } else { + target->abi = target_default_abi(target->arch, target->os); + } + + if (mcpu != nullptr && strcmp(mcpu, "baseline") != 0) { + const char *msg = "stage0 can't handle CPU/features in the target"; + stage2_panic(msg, strlen(msg)); + } + } + + return ErrorNone; +} + + static bool str_starts_with(const char *s1, const char *s2) { size_t s2_len = strlen(s2); if (strlen(s1) < s2_len) { @@ -90,10 +256,9 @@ int main(int argc, char **argv) { bool link_libcpp = false; const char *target_string = nullptr; ZigStage1Pkg *cur_pkg = heap::c_allocator.create(); - BuildMode build_mode = BuildModeDebug; + BuildMode optimize_mode = BuildModeDebug; TargetSubsystem subsystem = TargetSubsystemAuto; const char *override_lib_dir = nullptr; - const char *main_pkg_path = nullptr; const char *mcpu = nullptr; for (int i = 1; i < argc; i += 1) { @@ -103,11 +268,11 @@ int main(int argc, char **argv) { if (strcmp(arg, "--") == 0) { fprintf(stderr, "Unexpected end-of-parameter mark: %s\n", arg); } else if (strcmp(arg, "--release-fast") == 0) { - build_mode = BuildModeFastRelease; + optimize_mode = BuildModeFastRelease; } else if (strcmp(arg, "--release-safe") == 0) { - build_mode = BuildModeSafeRelease; + optimize_mode = BuildModeSafeRelease; } else if (strcmp(arg, "--release-small") == 0) { - build_mode = BuildModeSmallRelease; + optimize_mode = BuildModeSmallRelease; } else if (strcmp(arg, "--help") == 0) { return print_full_usage(arg0, stdout, EXIT_SUCCESS); } else if (strcmp(arg, "--strip") == 0) { @@ -183,8 +348,6 @@ int main(int argc, char **argv) { dynamic_linker = argv[i]; } else if (strcmp(arg, "--override-lib-dir") == 0) { override_lib_dir = argv[i]; - } else if (strcmp(arg, "--main-pkg-path") == 0) { - main_pkg_path = argv[i]; } else if (strcmp(arg, "--library") == 0 || strcmp(arg, "-l") == 0) { if (strcmp(argv[i], "c") == 0) { link_libc = true; @@ -264,12 +427,13 @@ int main(int argc, char **argv) { return print_error_usage(arg0); } - ZigStage1 *stage1 = zig_stage1_create(build_mode, - main_pkg_path, (main_pkg_path == nullptr) ? 0 : strlen(main_pkg_path), + ZigStage1 *stage1 = zig_stage1_create(optimize_mode, + nullptr, 0, in_file, strlen(in_file), - override_lib_dir, strlen(override_lib_dir), &target, false, - root_progress_node); + override_lib_dir, strlen(override_lib_dir), + &target, false); + stage1->main_progress_node = root_progress_node; stage1->root_name_ptr = out_name; stage1->root_name_len = strlen(out_name); stage1->strip = strip; @@ -295,3 +459,66 @@ int main(int argc, char **argv) { return main_exit(root_progress_node, EXIT_SUCCESS); } + +void stage2_panic(const char *ptr, size_t len) { + fwrite(ptr, 1, len, stderr); + fprintf(stderr, "\n"); + fflush(stderr); + abort(); +} + +struct Stage2Progress { + int trash; +}; + +struct Stage2ProgressNode { + int trash; +}; + +Stage2Progress *stage2_progress_create(void) { + return nullptr; +} + +void stage2_progress_destroy(Stage2Progress *progress) {} + +Stage2ProgressNode *stage2_progress_start_root(Stage2Progress *progress, + const char *name_ptr, size_t name_len, size_t estimated_total_items) +{ + return nullptr; +} +Stage2ProgressNode *stage2_progress_start(Stage2ProgressNode *node, + const char *name_ptr, size_t name_len, size_t estimated_total_items) +{ + return nullptr; +} +void stage2_progress_end(Stage2ProgressNode *node) {} +void stage2_progress_complete_one(Stage2ProgressNode *node) {} +void stage2_progress_disable_tty(Stage2Progress *progress) {} +void stage2_progress_update_node(Stage2ProgressNode *node, size_t completed_count, size_t estimated_total_items){} + +const char *stage2_fetch_file(struct ZigStage1 *stage1, const char *path_ptr, size_t path_len, + size_t *result_len) +{ + Error err; + Buf contents_buf = BUF_INIT; + Buf path_buf = BUF_INIT; + + buf_init_from_mem(&path_buf, path_ptr, path_len); + if ((err = os_fetch_file_path(&path_buf, &contents_buf))) { + return nullptr; + } + *result_len = buf_len(&contents_buf); + return buf_ptr(&contents_buf); +} + +const char *stage2_cimport(struct ZigStage1 *stage1) { + const char *msg = "stage0 called stage2_cimport"; + stage2_panic(msg, strlen(msg)); +} + +const char *stage2_add_link_lib(struct ZigStage1 *stage1, + const char *lib_name_ptr, size_t lib_name_len, + const char *symbol_name_ptr, size_t symbol_name_len) +{ + return nullptr; +} From 302e1565237a0f54101e9c50af34039321dfbc29 Mon Sep 17 00:00:00 2001 From: Vexu Date: Sat, 19 Sep 2020 00:08:46 +0300 Subject: [PATCH 075/210] stage2: make DepTokenizer non-allocating --- src-self-hosted/DepTokenizer.zig | 371 ++++++++++++++----------------- 1 file changed, 166 insertions(+), 205 deletions(-) diff --git a/src-self-hosted/DepTokenizer.zig b/src-self-hosted/DepTokenizer.zig index 262e7d558b..a2d5de97d9 100644 --- a/src-self-hosted/DepTokenizer.zig +++ b/src-self-hosted/DepTokenizer.zig @@ -1,9 +1,7 @@ const Tokenizer = @This(); -arena: std.heap.ArenaAllocator, index: usize, bytes: []const u8, -error_text: []const u8, state: State, const std = @import("std"); @@ -12,11 +10,9 @@ const assert = std.debug.assert; pub fn init(allocator: *std.mem.Allocator, bytes: []const u8) Tokenizer { return Tokenizer{ - .arena = std.heap.ArenaAllocator.init(allocator), .index = 0, .bytes = bytes, - .error_text = "", - .state = State{ .lhs = {} }, + .state = .lhs, }; } @@ -24,339 +20,306 @@ pub fn deinit(self: *Tokenizer) void { self.arena.deinit(); } -pub fn next(self: *Tokenizer) Error!?Token { +pub fn next(self: *Tokenizer) ?Token { + var start = self.index; + var must_resolve = false; while (self.index < self.bytes.len) { const char = self.bytes[self.index]; - while (true) { switch (self.state) { .lhs => switch (char) { '\t', '\n', '\r', ' ' => { // silently ignore whitespace - break; // advance + self.index += 1; }, else => { - self.state = State{ .target = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) }; + start = self.index; + self.state = .target; }, }, - .target => |*target| switch (char) { + .target => switch (char) { '\t', '\n', '\r', ' ' => { - return self.errorIllegalChar(self.index, char, "invalid target", .{}); + return errorIllegalChar(.invalid_target, self.index, char); }, '$' => { - self.state = State{ .target_dollar_sign = target.* }; - break; // advance + self.state = .target_dollar_sign; + self.index += 1; }, '\\' => { - self.state = State{ .target_reverse_solidus = target.* }; - break; // advance + self.state = .target_reverse_solidus; + self.index += 1; }, ':' => { - self.state = State{ .target_colon = target.* }; - break; // advance + self.state = .target_colon; + self.index += 1; }, else => { - try target.append(char); - break; // advance + self.index += 1; }, }, - .target_reverse_solidus => |*target| switch (char) { + .target_reverse_solidus => switch (char) { '\t', '\n', '\r' => { - return self.errorIllegalChar(self.index, char, "bad target escape", .{}); + return errorIllegalChar(.bad_target_escape, self.index, char); }, ' ', '#', '\\' => { - try target.append(char); - self.state = State{ .target = target.* }; - break; // advance + must_resolve = true; + self.state = .target; + self.index += 1; }, '$' => { - try target.appendSlice(self.bytes[self.index - 1 .. self.index]); - self.state = State{ .target_dollar_sign = target.* }; - break; // advance + self.state = .target_dollar_sign; + self.index += 1; }, else => { - try target.appendSlice(self.bytes[self.index - 1 .. self.index + 1]); - self.state = State{ .target = target.* }; - break; // advance + self.state = .target; + self.index += 1; }, }, - .target_dollar_sign => |*target| switch (char) { + .target_dollar_sign => switch (char) { '$' => { - try target.append(char); - self.state = State{ .target = target.* }; - break; // advance + must_resolve = true; + self.state = .target; + self.index += 1; }, else => { - return self.errorIllegalChar(self.index, char, "expecting '$'", .{}); + return errorIllegalChar(.expected_dollar_sign, self.index, char); }, }, - .target_colon => |*target| switch (char) { + .target_colon => switch (char) { '\n', '\r' => { - const bytes = target.span(); + const bytes = self.bytes[start..self.index - 1]; if (bytes.len != 0) { - self.state = State{ .lhs = {} }; - return Token{ .id = .target, .bytes = bytes }; + self.state = .lhs; + return finishTarget(must_resolve, bytes); } // silently ignore null target - self.state = State{ .lhs = {} }; - continue; + self.state = .lhs; }, '\\' => { - self.state = State{ .target_colon_reverse_solidus = target.* }; - break; // advance + self.state = .target_colon_reverse_solidus; + self.index += 1; }, else => { - const bytes = target.span(); + const bytes = self.bytes[start..self.index - 1]; if (bytes.len != 0) { - self.state = State{ .rhs = {} }; - return Token{ .id = .target, .bytes = bytes }; + self.state = .rhs; + return finishTarget(must_resolve, bytes); } // silently ignore null target - self.state = State{ .lhs = {} }; - continue; + self.state = .lhs; }, }, - .target_colon_reverse_solidus => |*target| switch (char) { + .target_colon_reverse_solidus => switch (char) { '\n', '\r' => { - const bytes = target.span(); + const bytes = self.bytes[start .. self.index - 2]; if (bytes.len != 0) { - self.state = State{ .lhs = {} }; - return Token{ .id = .target, .bytes = bytes }; + self.state = .lhs; + return finishTarget(must_resolve, bytes); } // silently ignore null target - self.state = State{ .lhs = {} }; - continue; + self.state = .lhs; }, else => { - try target.appendSlice(self.bytes[self.index - 2 .. self.index + 1]); - self.state = State{ .target = target.* }; - break; + self.state = .target; }, }, .rhs => switch (char) { '\t', ' ' => { // silently ignore horizontal whitespace - break; // advance + self.index += 1; }, '\n', '\r' => { - self.state = State{ .lhs = {} }; - continue; + self.state = .lhs; }, '\\' => { - self.state = State{ .rhs_continuation = {} }; - break; // advance + self.state = .rhs_continuation; + self.index += 1; }, '"' => { - self.state = State{ .prereq_quote = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) }; - break; // advance + self.state = .prereq_quote; + self.index += 1; + start = self.index; }, else => { - self.state = State{ .prereq = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) }; + start = self.index; + self.state = .prereq; }, }, .rhs_continuation => switch (char) { '\n' => { - self.state = State{ .rhs = {} }; - break; // advance + self.state = .rhs; + self.index += 1; }, '\r' => { - self.state = State{ .rhs_continuation_linefeed = {} }; - break; // advance + self.state = .rhs_continuation_linefeed; + self.index += 1; }, else => { - return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{}); + return errorIllegalChar(.continuation_eol, self.index, char); }, }, .rhs_continuation_linefeed => switch (char) { '\n' => { - self.state = State{ .rhs = {} }; - break; // advance + self.state = .rhs; + self.index += 1; }, else => { - return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{}); + return errorIllegalChar(.continuation_eol, self.index, char); }, }, - .prereq_quote => |*prereq| switch (char) { + .prereq_quote => switch (char) { '"' => { - const bytes = prereq.span(); self.index += 1; - self.state = State{ .rhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; + self.state = .rhs; + return Token{ .prereq = self.bytes[start .. self.index - 1] }; }, else => { - try prereq.append(char); - break; // advance + self.index += 1; }, }, - .prereq => |*prereq| switch (char) { + .prereq => switch (char) { '\t', ' ' => { - const bytes = prereq.span(); - self.state = State{ .rhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; + self.state = .rhs; + return Token{ .prereq = self.bytes[start..self.index] }; }, '\n', '\r' => { - const bytes = prereq.span(); - self.state = State{ .lhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; + self.state = .lhs; + return Token{ .prereq = self.bytes[start..self.index] }; }, '\\' => { - self.state = State{ .prereq_continuation = prereq.* }; - break; // advance + self.state = .prereq_continuation; + self.index += 1; }, else => { - try prereq.append(char); - break; // advance + self.index += 1; }, }, - .prereq_continuation => |*prereq| switch (char) { + .prereq_continuation => switch (char) { '\n' => { - const bytes = prereq.span(); self.index += 1; - self.state = State{ .rhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; + self.state = .rhs; + return Token{ .prereq = self.bytes[start .. self.index - 2] }; }, '\r' => { - self.state = State{ .prereq_continuation_linefeed = prereq.* }; - break; // advance + self.state = .prereq_continuation_linefeed; + self.index += 1; }, else => { // not continuation - try prereq.appendSlice(self.bytes[self.index - 1 .. self.index + 1]); - self.state = State{ .prereq = prereq.* }; - break; // advance + self.state = .prereq; + self.index += 1; }, }, - .prereq_continuation_linefeed => |prereq| switch (char) { + .prereq_continuation_linefeed => switch (char) { '\n' => { - const bytes = prereq.span(); self.index += 1; - self.state = State{ .rhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; + self.state = .rhs; + return Token{ .prereq = self.bytes[start .. self.index - 1] }; }, else => { - return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{}); + return errorIllegalChar(.continuation_eol, self.index, char); }, }, } - } - self.index += 1; - } - - // eof, handle maybe incomplete token - if (self.index == 0) return null; - const idx = self.index - 1; + } else { switch (self.state) { .lhs, .rhs, .rhs_continuation, .rhs_continuation_linefeed, - => {}, - .target => |target| { - return self.errorPosition(idx, target.span(), "incomplete target", .{}); + => return null, + .target => { + return Token{ .incomplete_target = self.bytes[start..] }; }, .target_reverse_solidus, .target_dollar_sign, => { - const index = self.index - 1; - return self.errorIllegalChar(idx, self.bytes[idx], "incomplete escape", .{}); + const idx = self.index - 1; + return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]); }, - .target_colon => |target| { - const bytes = target.span(); + .target_colon => { + const bytes = self.bytes[start.. self.index - 1]; if (bytes.len != 0) { self.index += 1; - self.state = State{ .rhs = {} }; - return Token{ .id = .target, .bytes = bytes }; + self.state = .rhs; + return finishTarget(must_resolve, bytes); } // silently ignore null target - self.state = State{ .lhs = {} }; + self.state = .lhs; + return null; }, - .target_colon_reverse_solidus => |target| { - const bytes = target.span(); + .target_colon_reverse_solidus => { + const bytes = self.bytes[start..self.index - 2]; if (bytes.len != 0) { self.index += 1; - self.state = State{ .rhs = {} }; - return Token{ .id = .target, .bytes = bytes }; + self.state = .rhs; + return finishTarget(must_resolve, bytes); } // silently ignore null target - self.state = State{ .lhs = {} }; + self.state = .lhs; + return null; }, - .prereq_quote => |prereq| { - return self.errorPosition(idx, prereq.span(), "incomplete quoted prerequisite", .{}); + .prereq_quote => { + return Token{ .incomplete_quoted_prerequisite = self.bytes[start..] }; }, - .prereq => |prereq| { - const bytes = prereq.span(); - self.state = State{ .lhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; + .prereq => { + self.state = .lhs; + return Token{ .prereq = self.bytes[start..] }; }, - .prereq_continuation => |prereq| { - const bytes = prereq.span(); - self.state = State{ .lhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; + .prereq_continuation => { + self.state = .lhs; + return Token{ .prereq = self.bytes[start.. self.index - 1] }; }, - .prereq_continuation_linefeed => |prereq| { - const bytes = prereq.span(); - self.state = State{ .lhs = {} }; - return Token{ .id = .prereq, .bytes = bytes }; + .prereq_continuation_linefeed => { + self.state = .lhs; + return Token{ .prereq = self.bytes[start.. self.index - 2] }; }, } - return null; +} + unreachable; } -fn errorf(self: *Tokenizer, comptime fmt: []const u8, args: anytype) Error { - self.error_text = try std.fmt.allocPrintZ(&self.arena.allocator, fmt, args); - return Error.InvalidInput; +fn errorIllegalChar(comptime id: @TagType(Token), index: usize, char: u8) Token { + return @unionInit(Token, @tagName(id), .{ .index = index, .char = char }); } -fn errorPosition(self: *Tokenizer, position: usize, bytes: []const u8, comptime fmt: []const u8, args: anytype) Error { - var buffer = std.ArrayList(u8).init(&self.arena.allocator); - try buffer.outStream().print(fmt, args); - try buffer.appendSlice(" '"); - const out = buffer.writer(); - try printCharValues(out, bytes); - try buffer.appendSlice("'"); - try buffer.outStream().print(" at position {}", .{position - (bytes.len - 1)}); - try buffer.append(0); - self.error_text = buffer.items[0 .. buffer.items.len - 1 :0]; - return Error.InvalidInput; +fn finishTarget(must_resolve: bool, bytes: []const u8) Token { + return if (must_resolve) + .{ .target_must_resolve = bytes } + else + .{ .target = bytes }; } -fn errorIllegalChar(self: *Tokenizer, position: usize, char: u8, comptime fmt: []const u8, args: anytype) Error { - var buffer = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0); - try buffer.appendSlice("illegal char "); - try printUnderstandableChar(&buffer, char); - try buffer.outStream().print(" at position {}", .{position}); - if (fmt.len != 0) try buffer.outStream().print(": " ++ fmt, args); - self.error_text = buffer.span(); - return Error.InvalidInput; -} - -const Error = error{ - OutOfMemory, - InvalidInput, +const State = enum { + lhs, + target, + target_reverse_solidus, + target_dollar_sign, + target_colon, + target_colon_reverse_solidus, + rhs, + rhs_continuation, + rhs_continuation_linefeed, + prereq_quote, + prereq, + prereq_continuation, + prereq_continuation_linefeed, }; -const State = union(enum) { - lhs: void, - target: std.ArrayListSentineled(u8, 0), - target_reverse_solidus: std.ArrayListSentineled(u8, 0), - target_dollar_sign: std.ArrayListSentineled(u8, 0), - target_colon: std.ArrayListSentineled(u8, 0), - target_colon_reverse_solidus: std.ArrayListSentineled(u8, 0), - rhs: void, - rhs_continuation: void, - rhs_continuation_linefeed: void, - prereq_quote: std.ArrayListSentineled(u8, 0), - prereq: std.ArrayListSentineled(u8, 0), - prereq_continuation: std.ArrayListSentineled(u8, 0), - prereq_continuation_linefeed: std.ArrayListSentineled(u8, 0), -}; +pub const Token = union(enum) { + target: []const u8, + target_must_resolve: []const u8, + prereq: []const u8, + incomplete_quoted_prerequisite: []const u8, + incomplete_target: []const u8, + invalid_target: IndexAndChar, + bad_target_escape: IndexAndChar, + expected_dollar_sign: IndexAndChar, + continuation_eol: IndexAndChar, + incomplete_escape: IndexAndChar, -pub const Token = struct { - id: ID, - bytes: []const u8, - - pub const ID = enum { - target, - prereq, + pub const IndexAndChar = struct { + index: usize, + char: u8, }; }; @@ -845,26 +808,24 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void { var it = Tokenizer.init(arena, input); var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0); var i: usize = 0; - while (true) { - const r = it.next() catch |err| { - switch (err) { - Tokenizer.Error.InvalidInput => { - if (i != 0) try buffer.appendSlice("\n"); - try buffer.appendSlice("ERROR: "); - try buffer.appendSlice(it.error_text); - }, - else => return err, - } - break; - }; - const token = r orelse break; + while (it.next()) |token| { if (i != 0) try buffer.appendSlice("\n"); - try buffer.appendSlice(@tagName(token.id)); + switch (token) { + .target, .prereq => |bytes| { + try buffer.appendSlice(@tagName(token)); try buffer.appendSlice(" = {"); - for (token.bytes) |b| { + for (bytes) |b| { try buffer.append(printable_char_tab[b]); } try buffer.appendSlice("}"); + }, + .target_must_resolve => { + @panic("TODO"); + }, + else => { + @panic("TODO"); + }, + } i += 1; } const got: []const u8 = buffer.span(); From f27bc79121bd9971c9e3465fdba3cc7f7d6ff864 Mon Sep 17 00:00:00 2001 From: Vexu Date: Sat, 19 Sep 2020 00:28:17 +0300 Subject: [PATCH 076/210] stage2: DepTokenizer add target resolver --- src-self-hosted/DepTokenizer.zig | 333 ++++++++++++++++++------------- 1 file changed, 190 insertions(+), 143 deletions(-) diff --git a/src-self-hosted/DepTokenizer.zig b/src-self-hosted/DepTokenizer.zig index a2d5de97d9..3ced32ce71 100644 --- a/src-self-hosted/DepTokenizer.zig +++ b/src-self-hosted/DepTokenizer.zig @@ -25,256 +25,256 @@ pub fn next(self: *Tokenizer) ?Token { var must_resolve = false; while (self.index < self.bytes.len) { const char = self.bytes[self.index]; - switch (self.state) { - .lhs => switch (char) { - '\t', '\n', '\r', ' ' => { - // silently ignore whitespace + switch (self.state) { + .lhs => switch (char) { + '\t', '\n', '\r', ' ' => { + // silently ignore whitespace self.index += 1; - }, - else => { + }, + else => { start = self.index; self.state = .target; - }, }, + }, .target => switch (char) { - '\t', '\n', '\r', ' ' => { + '\t', '\n', '\r', ' ' => { return errorIllegalChar(.invalid_target, self.index, char); - }, - '$' => { + }, + '$' => { self.state = .target_dollar_sign; self.index += 1; - }, - '\\' => { + }, + '\\' => { self.state = .target_reverse_solidus; self.index += 1; - }, - ':' => { + }, + ':' => { self.state = .target_colon; self.index += 1; - }, - else => { - self.index += 1; - }, }, + else => { + self.index += 1; + }, + }, .target_reverse_solidus => switch (char) { - '\t', '\n', '\r' => { + '\t', '\n', '\r' => { return errorIllegalChar(.bad_target_escape, self.index, char); - }, - ' ', '#', '\\' => { + }, + ' ', '#', '\\' => { must_resolve = true; self.state = .target; self.index += 1; - }, - '$' => { + }, + '$' => { self.state = .target_dollar_sign; self.index += 1; - }, - else => { + }, + else => { self.state = .target; self.index += 1; - }, }, + }, .target_dollar_sign => switch (char) { - '$' => { + '$' => { must_resolve = true; self.state = .target; self.index += 1; - }, - else => { - return errorIllegalChar(.expected_dollar_sign, self.index, char); - }, }, + else => { + return errorIllegalChar(.expected_dollar_sign, self.index, char); + }, + }, .target_colon => switch (char) { - '\n', '\r' => { - const bytes = self.bytes[start..self.index - 1]; - if (bytes.len != 0) { + '\n', '\r' => { + const bytes = self.bytes[start .. self.index - 1]; + if (bytes.len != 0) { self.state = .lhs; return finishTarget(must_resolve, bytes); - } - // silently ignore null target + } + // silently ignore null target self.state = .lhs; - }, - '\\' => { + }, + '\\' => { self.state = .target_colon_reverse_solidus; self.index += 1; - }, - else => { - const bytes = self.bytes[start..self.index - 1]; - if (bytes.len != 0) { + }, + else => { + const bytes = self.bytes[start .. self.index - 1]; + if (bytes.len != 0) { self.state = .rhs; return finishTarget(must_resolve, bytes); - } - // silently ignore null target + } + // silently ignore null target self.state = .lhs; - }, }, + }, .target_colon_reverse_solidus => switch (char) { - '\n', '\r' => { + '\n', '\r' => { const bytes = self.bytes[start .. self.index - 2]; - if (bytes.len != 0) { + if (bytes.len != 0) { self.state = .lhs; return finishTarget(must_resolve, bytes); - } - // silently ignore null target + } + // silently ignore null target self.state = .lhs; - }, - else => { - self.state = .target; - }, }, - .rhs => switch (char) { - '\t', ' ' => { - // silently ignore horizontal whitespace + else => { + self.state = .target; + }, + }, + .rhs => switch (char) { + '\t', ' ' => { + // silently ignore horizontal whitespace self.index += 1; - }, - '\n', '\r' => { + }, + '\n', '\r' => { self.state = .lhs; - }, - '\\' => { + }, + '\\' => { self.state = .rhs_continuation; self.index += 1; - }, - '"' => { + }, + '"' => { self.state = .prereq_quote; self.index += 1; start = self.index; - }, - else => { + }, + else => { start = self.index; self.state = .prereq; - }, }, - .rhs_continuation => switch (char) { - '\n' => { + }, + .rhs_continuation => switch (char) { + '\n' => { self.state = .rhs; self.index += 1; - }, - '\r' => { + }, + '\r' => { self.state = .rhs_continuation_linefeed; self.index += 1; - }, - else => { - return errorIllegalChar(.continuation_eol, self.index, char); - }, }, - .rhs_continuation_linefeed => switch (char) { - '\n' => { + else => { + return errorIllegalChar(.continuation_eol, self.index, char); + }, + }, + .rhs_continuation_linefeed => switch (char) { + '\n' => { self.state = .rhs; self.index += 1; - }, - else => { - return errorIllegalChar(.continuation_eol, self.index, char); - }, }, + else => { + return errorIllegalChar(.continuation_eol, self.index, char); + }, + }, .prereq_quote => switch (char) { - '"' => { - self.index += 1; + '"' => { + self.index += 1; self.state = .rhs; return Token{ .prereq = self.bytes[start .. self.index - 1] }; - }, - else => { - self.index += 1; - }, }, + else => { + self.index += 1; + }, + }, .prereq => switch (char) { - '\t', ' ' => { + '\t', ' ' => { self.state = .rhs; return Token{ .prereq = self.bytes[start..self.index] }; - }, - '\n', '\r' => { + }, + '\n', '\r' => { self.state = .lhs; return Token{ .prereq = self.bytes[start..self.index] }; - }, - '\\' => { + }, + '\\' => { self.state = .prereq_continuation; self.index += 1; - }, - else => { - self.index += 1; - }, }, + else => { + self.index += 1; + }, + }, .prereq_continuation => switch (char) { - '\n' => { - self.index += 1; + '\n' => { + self.index += 1; self.state = .rhs; return Token{ .prereq = self.bytes[start .. self.index - 2] }; - }, - '\r' => { + }, + '\r' => { self.state = .prereq_continuation_linefeed; self.index += 1; - }, - else => { - // not continuation + }, + else => { + // not continuation self.state = .prereq; self.index += 1; - }, }, + }, .prereq_continuation_linefeed => switch (char) { - '\n' => { - self.index += 1; + '\n' => { + self.index += 1; self.state = .rhs; return Token{ .prereq = self.bytes[start .. self.index - 1] }; - }, - else => { - return errorIllegalChar(.continuation_eol, self.index, char); - }, }, - } + else => { + return errorIllegalChar(.continuation_eol, self.index, char); + }, + }, + } } else { - switch (self.state) { - .lhs, - .rhs, - .rhs_continuation, - .rhs_continuation_linefeed, + switch (self.state) { + .lhs, + .rhs, + .rhs_continuation, + .rhs_continuation_linefeed, => return null, .target => { return Token{ .incomplete_target = self.bytes[start..] }; - }, - .target_reverse_solidus, - .target_dollar_sign, - => { + }, + .target_reverse_solidus, + .target_dollar_sign, + => { const idx = self.index - 1; return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]); - }, + }, .target_colon => { - const bytes = self.bytes[start.. self.index - 1]; - if (bytes.len != 0) { - self.index += 1; + const bytes = self.bytes[start .. self.index - 1]; + if (bytes.len != 0) { + self.index += 1; self.state = .rhs; return finishTarget(must_resolve, bytes); - } - // silently ignore null target + } + // silently ignore null target self.state = .lhs; return null; - }, + }, .target_colon_reverse_solidus => { - const bytes = self.bytes[start..self.index - 2]; - if (bytes.len != 0) { - self.index += 1; + const bytes = self.bytes[start .. self.index - 2]; + if (bytes.len != 0) { + self.index += 1; self.state = .rhs; return finishTarget(must_resolve, bytes); - } - // silently ignore null target + } + // silently ignore null target self.state = .lhs; return null; - }, + }, .prereq_quote => { return Token{ .incomplete_quoted_prerequisite = self.bytes[start..] }; - }, + }, .prereq => { self.state = .lhs; return Token{ .prereq = self.bytes[start..] }; - }, + }, .prereq_continuation => { self.state = .lhs; - return Token{ .prereq = self.bytes[start.. self.index - 1] }; - }, + return Token{ .prereq = self.bytes[start .. self.index - 1] }; + }, .prereq_continuation_linefeed => { self.state = .lhs; - return Token{ .prereq = self.bytes[start.. self.index - 2] }; - }, + return Token{ .prereq = self.bytes[start .. self.index - 2] }; + }, + } } -} unreachable; } @@ -321,6 +321,46 @@ pub const Token = union(enum) { index: usize, char: u8, }; + + /// Resolve escapes in target. Only valid with .target_must_resolve. + pub fn resolve(self: Token, buf: *std.ArrayList(u8)) std.mem.Allocator.Error!void { + const bytes = self.target_must_resolve; // resolve called on incorrect token + + try buf.ensureCapacity(bytes.len); // cannot be longer than the unescaped string + var state: enum { start, escape, dollar } = .start; + for (bytes) |c| { + switch (state) { + .start => { + switch (c) { + '\\' => state = .escape, + '$' => state = .dollar, + else => buf.appendAssumeCapacity(c), + } + }, + .escape => { + switch (c) { + ' ', '#', '\\' => {}, + '$' => { + buf.appendAssumeCapacity('\\'); + state = .dollar; + continue; + }, + else => buf.appendAssumeCapacity('\\'), + } + buf.appendAssumeCapacity(c); + state = .start; + }, + .dollar => { + buf.appendAssumeCapacity('$'); + switch (c) { + '$' => {}, + else => buf.appendAssumeCapacity(c), + } + state = .start; + }, + } + } + } }; test "empty file" { @@ -807,20 +847,27 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void { var it = Tokenizer.init(arena, input); var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0); + var resolve_buf = std.ArrayList(u8).init(arena); var i: usize = 0; while (it.next()) |token| { if (i != 0) try buffer.appendSlice("\n"); switch (token) { .target, .prereq => |bytes| { try buffer.appendSlice(@tagName(token)); - try buffer.appendSlice(" = {"); + try buffer.appendSlice(" = {"); for (bytes) |b| { - try buffer.append(printable_char_tab[b]); - } - try buffer.appendSlice("}"); + try buffer.append(printable_char_tab[b]); + } + try buffer.appendSlice("}"); }, .target_must_resolve => { - @panic("TODO"); + try buffer.appendSlice("target = {"); + try token.resolve(&resolve_buf); + for (resolve_buf.items) |b| { + try buffer.append(printable_char_tab[b]); + } + resolve_buf.items.len = 0; + try buffer.appendSlice("}"); }, else => { @panic("TODO"); From b759308fb30b130ea5d084c3814b1820a589ceee Mon Sep 17 00:00:00 2001 From: Vexu Date: Sat, 19 Sep 2020 14:08:32 +0300 Subject: [PATCH 077/210] stage2: DepTokenizer print errors --- src-self-hosted/DepTokenizer.zig | 107 ++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 29 deletions(-) diff --git a/src-self-hosted/DepTokenizer.zig b/src-self-hosted/DepTokenizer.zig index 3ced32ce71..52cddd1606 100644 --- a/src-self-hosted/DepTokenizer.zig +++ b/src-self-hosted/DepTokenizer.zig @@ -228,7 +228,7 @@ pub fn next(self: *Tokenizer) ?Token { .rhs_continuation_linefeed, => return null, .target => { - return Token{ .incomplete_target = self.bytes[start..] }; + return errorPosition(.incomplete_target, start, self.bytes[start..]); }, .target_reverse_solidus, .target_dollar_sign, @@ -259,7 +259,7 @@ pub fn next(self: *Tokenizer) ?Token { return null; }, .prereq_quote => { - return Token{ .incomplete_quoted_prerequisite = self.bytes[start..] }; + return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]); }, .prereq => { self.state = .lhs; @@ -278,6 +278,10 @@ pub fn next(self: *Tokenizer) ?Token { unreachable; } +fn errorPosition(comptime id: @TagType(Token), index: usize, bytes: []const u8) Token { + return @unionInit(Token, @tagName(id), .{ .index = index, .bytes = bytes }); +} + fn errorIllegalChar(comptime id: @TagType(Token), index: usize, char: u8) Token { return @unionInit(Token, @tagName(id), .{ .index = index, .char = char }); } @@ -309,8 +313,10 @@ pub const Token = union(enum) { target: []const u8, target_must_resolve: []const u8, prereq: []const u8, - incomplete_quoted_prerequisite: []const u8, - incomplete_target: []const u8, + + incomplete_quoted_prerequisite: IndexAndBytes, + incomplete_target: IndexAndBytes, + invalid_target: IndexAndChar, bad_target_escape: IndexAndChar, expected_dollar_sign: IndexAndChar, @@ -322,11 +328,15 @@ pub const Token = union(enum) { char: u8, }; + pub const IndexAndBytes = struct { + index: usize, + bytes: []const u8, + }; + /// Resolve escapes in target. Only valid with .target_must_resolve. - pub fn resolve(self: Token, buf: *std.ArrayList(u8)) std.mem.Allocator.Error!void { + pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void { const bytes = self.target_must_resolve; // resolve called on incorrect token - try buf.ensureCapacity(bytes.len); // cannot be longer than the unescaped string var state: enum { start, escape, dollar } = .start; for (bytes) |c| { switch (state) { @@ -334,33 +344,74 @@ pub const Token = union(enum) { switch (c) { '\\' => state = .escape, '$' => state = .dollar, - else => buf.appendAssumeCapacity(c), + else => try writer.writeByte(c), } }, .escape => { switch (c) { ' ', '#', '\\' => {}, '$' => { - buf.appendAssumeCapacity('\\'); + try writer.writeByte('\\'); state = .dollar; continue; }, - else => buf.appendAssumeCapacity('\\'), + else => try writer.writeByte('\\'), } - buf.appendAssumeCapacity(c); + try writer.writeByte(c); state = .start; }, .dollar => { - buf.appendAssumeCapacity('$'); + try writer.writeByte('$'); switch (c) { '$' => {}, - else => buf.appendAssumeCapacity(c), + else => try writer.writeByte(c), } state = .start; }, } } } + + pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void { + switch (self) { + .target, .target_must_resolve, .prereq => unreachable, // not an error + .incomplete_quoted_prerequisite, + .incomplete_target, + => |index_and_bytes| { + try writer.print("{} '", .{self.errStr()}); + if (self == .incomplete_target) { + const tmp = Token{ .target_must_resolve = index_and_bytes.bytes }; + try tmp.resolve(writer); + } else { + try printCharValues(writer, index_and_bytes.bytes); + } + try writer.print("' at position {}", .{index_and_bytes.index}); + }, + .invalid_target, + .bad_target_escape, + .expected_dollar_sign, + .continuation_eol, + .incomplete_escape, + => |index_and_char| { + try writer.writeAll("illegal char "); + try printUnderstandableChar(writer, index_and_char.char); + try writer.print(" at position {}: {}", .{ index_and_char.index, self.errStr() }); + }, + } + } + + fn errStr(self: Token) []const u8 { + return switch (self) { + .target, .target_must_resolve, .prereq => unreachable, // not an error + .incomplete_quoted_prerequisite => "incomplete quoted prerequisite", + .incomplete_target => "incomplete target", + .invalid_target => "invalid target", + .bad_target_escape => "bad target escape", + .expected_dollar_sign => "expecting '$'", + .continuation_eol => "continuation expecting end-of-line", + .incomplete_escape => "incomplete escape", + }; + } }; test "empty file" { @@ -755,16 +806,16 @@ test "error incomplete target" { ); try depTokenizer("\\ foo.o", - \\ERROR: incomplete target ' foo.o' at position 1 + \\ERROR: incomplete target ' foo.o' at position 0 ); try depTokenizer("\\#foo.o", - \\ERROR: incomplete target '#foo.o' at position 1 + \\ERROR: incomplete target '#foo.o' at position 0 ); try depTokenizer("\\\\foo.o", - \\ERROR: incomplete target '\foo.o' at position 1 + \\ERROR: incomplete target '\foo.o' at position 0 ); try depTokenizer("$$foo.o", - \\ERROR: incomplete target '$foo.o' at position 1 + \\ERROR: incomplete target '$foo.o' at position 0 ); } @@ -862,7 +913,7 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void { }, .target_must_resolve => { try buffer.appendSlice("target = {"); - try token.resolve(&resolve_buf); + try token.resolve(resolve_buf.writer()); for (resolve_buf.items) |b| { try buffer.append(printable_char_tab[b]); } @@ -870,7 +921,9 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void { try buffer.appendSlice("}"); }, else => { - @panic("TODO"); + try buffer.appendSlice("ERROR: "); + try token.printError(buffer.outStream()); + break; }, } i += 1; @@ -1005,23 +1058,19 @@ fn printCharValues(out: anytype, bytes: []const u8) !void { } } -fn printUnderstandableChar(buffer: *std.ArrayListSentineled(u8, 0), char: u8) !void { +fn printUnderstandableChar(out: anytype, char: u8) !void { if (!std.ascii.isPrint(char) or char == ' ') { - try buffer.outStream().print("\\x{X:0>2}", .{char}); + try out.print("\\x{X:0>2}", .{char}); } else { - try buffer.appendSlice("'"); - try buffer.append(printable_char_tab[char]); - try buffer.appendSlice("'"); + try out.print("'{c}'", .{printable_char_tab[char]}); } } // zig fmt: off -const printable_char_tab: []const u8 = +const printable_char_tab: [256]u8 = ( "................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++ "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~." ++ "................................................................" ++ - "................................................................"; -// zig fmt: on -comptime { - assert(printable_char_tab.len == 256); -} + "................................................................" +).*; + From ced061fcbf6c00b91e93428456647b280ae58e1b Mon Sep 17 00:00:00 2001 From: Vexu Date: Sat, 19 Sep 2020 14:16:58 +0300 Subject: [PATCH 078/210] stage2: update uses of DepTokenizer --- src-self-hosted/Cache.zig | 38 ++++++++++++++------------------ src-self-hosted/DepTokenizer.zig | 18 +++------------ 2 files changed, 20 insertions(+), 36 deletions(-) diff --git a/src-self-hosted/Cache.zig b/src-self-hosted/Cache.zig index 125353ebe2..cbab24715e 100644 --- a/src-self-hosted/Cache.zig +++ b/src-self-hosted/Cache.zig @@ -460,35 +460,31 @@ pub const CacheHash = struct { const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max); defer self.cache.gpa.free(dep_file_contents); - const DepTokenizer = @import("DepTokenizer.zig"); - var it = DepTokenizer.init(self.cache.gpa, dep_file_contents); - defer it.deinit(); + var error_buf = std.ArrayList(u8).init(self.cache.gpa); + defer error_buf.deinit(); + + var it: @import("DepTokenizer.zig") = .{ .bytes = dep_file_contents }; // Skip first token: target. - { - const opt_result = it.next() catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.InvalidInput => { - std.log.err("failed parsing {}: {}: {}", .{ dep_file_basename, @errorName(err), it.error_text }); - return error.InvalidDepFile; - }, - }; - _ = opt_result orelse return; // Empty dep file OK. + switch (it.next() orelse return) { // Empty dep file OK. + .target, .target_must_resolve, .prereq => {}, + else => |err| { + try err.printError(error_buf.writer()); + std.log.err("failed parsing {}: {}", .{ dep_file_basename, error_buf.items }); + return error.InvalidDepFile; + }, } // Process 0+ preqreqs. // Clang is invoked in single-source mode so we never get more targets. while (true) { - const opt_result = it.next() catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.InvalidInput => { - std.log.err("failed parsing {}: {}: {}", .{ dep_file_basename, @errorName(err), it.error_text }); + switch (it.next() orelse return) { + .target, .target_must_resolve => return, + .prereq => |bytes| try self.addFilePost(bytes), + else => |err| { + try err.printError(error_buf.writer()); + std.log.err("failed parsing {}: {}", .{ dep_file_basename, error_buf.items }); return error.InvalidDepFile; }, - }; - const result = opt_result orelse return; - switch (result.id) { - .target => return, - .prereq => try self.addFilePost(result.bytes), } } } diff --git a/src-self-hosted/DepTokenizer.zig b/src-self-hosted/DepTokenizer.zig index 52cddd1606..cc2211a1aa 100644 --- a/src-self-hosted/DepTokenizer.zig +++ b/src-self-hosted/DepTokenizer.zig @@ -1,25 +1,13 @@ const Tokenizer = @This(); -index: usize, +index: usize = 0, bytes: []const u8, -state: State, +state: State = .lhs, const std = @import("std"); const testing = std.testing; const assert = std.debug.assert; -pub fn init(allocator: *std.mem.Allocator, bytes: []const u8) Tokenizer { - return Tokenizer{ - .index = 0, - .bytes = bytes, - .state = .lhs, - }; -} - -pub fn deinit(self: *Tokenizer) void { - self.arena.deinit(); -} - pub fn next(self: *Tokenizer) ?Token { var start = self.index; var must_resolve = false; @@ -896,7 +884,7 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void { const arena = &arena_allocator.allocator; defer arena_allocator.deinit(); - var it = Tokenizer.init(arena, input); + var it: Tokenizer = .{ .bytes = input }; var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0); var resolve_buf = std.ArrayList(u8).init(arena); var i: usize = 0; From 561ed38f12a8436d93962423e56e0f3949ea28d4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Sep 2020 23:24:07 -0700 Subject: [PATCH 079/210] ci: drop the cmake patch for linux hopefully they've solved the polly plugin thing by now on apt.llvm.org. --- ci/azure/linux_script | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/ci/azure/linux_script b/ci/azure/linux_script index fb4caf18c0..99647ee063 100755 --- a/ci/azure/linux_script +++ b/ci/azure/linux_script @@ -28,22 +28,6 @@ PATH=$PWD/$WASMTIME:$PATH # This will affect the cmake command below. git config core.abbrev 9 -# This patch is a workaround for -# https://bugs.llvm.org/show_bug.cgi?id=44870 / https://github.com/llvm/llvm-project/issues/191 -# It only applies to the apt.llvm.org packages. -patch <<'END_PATCH' ---- CMakeLists.txt -+++ CMakeLists.txt -@@ -369,6 +369,7 @@ target_link_libraries(zig_cpp LINK_PUBLIC - ${CLANG_LIBRARIES} - ${LLD_LIBRARIES} - ${LLVM_LIBRARIES} -+ "-Wl,/usr/lib/llvm-10/lib/LLVMPolly.so" - ) - - add_library(opt_c_util STATIC ${OPTIMIZED_C_SOURCES}) -END_PATCH - export CC=gcc-7 export CXX=g++-7 mkdir build From bd1465a3fedf2a377a9e350400107f32595097a0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Sep 2020 16:32:56 -0700 Subject: [PATCH 080/210] cmake: output better message when building self-hosted Thanks @bfredl! --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a19c275d3..6389ec7dbe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -463,6 +463,7 @@ if("${ZIG_TARGET_TRIPLE}" STREQUAL "native") COMMAND zig0 ${BUILD_ZIG1_ARGS} DEPENDS zig0 BYPRODUCTS "${ZIG1_OBJECT}" + COMMENT STATUS "Building self-hosted component ${ZIG1_OBJECT}" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ) set(ZIG_EXECUTABLE "${zig_BINARY_DIR}/zig") @@ -473,6 +474,7 @@ else() add_custom_target(zig_build_zig1 ALL COMMAND "${ZIG_EXECUTABLE}" ${BUILD_ZIG1_ARGS} BYPRODUCTS "${ZIG1_OBJECT}" + COMMENT STATUS "Building self-hosted component ${ZIG1_OBJECT}" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ) endif() From b8861a94ddd6d58c69b1c6d2874df27cc3f6f347 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Sep 2020 16:33:29 -0700 Subject: [PATCH 081/210] stage2: building compiler_rt and libc static archive with stage1 * add CLI support for verbose debugging options * implement building compiler_rt and libc static archive using stage1 C++ backend * add function_sections and link_libcpp to root cache hash. * cleanups to use the new Directory.join method. * Package supports being initialized directly and cleaned up create() method. * fix classifyFileExt incorrectly saying .zir is .zig. Thanks @Rocknest! * unify updateSubCompilation implementations --- BRANCH_TODO | 28 ++-- src-self-hosted/Compilation.zig | 288 ++++++++++++++++++++++++-------- src-self-hosted/Package.zig | 34 ++-- src-self-hosted/glibc.zig | 36 ++-- src-self-hosted/libunwind.zig | 28 +--- src-self-hosted/link.zig | 12 +- src-self-hosted/link/Elf.zig | 22 ++- src-self-hosted/main.zig | 10 +- 8 files changed, 294 insertions(+), 164 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index bee98d6d2b..35972cb3e2 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,6 +1,8 @@ + * Cache integration for stage1 zig code compilation * build & link against compiler-rt * build & link against freestanding libc - * Cache integration for stage1 zig code compilation + * resolve builtin.zig not working on windows & macos; try os_path_real + * build & link against libcxx and libcxxabi * `zig test` * `zig build` * `-ftime-report` @@ -16,32 +18,33 @@ * make sure zig cc works - using it as a preprocessor (-E) - try building some software + * implement proper parsing of LLD stderr/stdout and exposing compile errors + * implement proper parsing of clang stderr/stdout and exposing compile errors * support rpaths in ELF linker code * repair @cImport * add CLI support for a way to pass extra flags to c source files - * capture lld stdout/stderr better * musl * mingw-w64 * use global zig-cache dir for crt files * MachO LLD linking * COFF LLD linking * WASM LLD linking - * implement proper parsing of LLD stderr/stdout and exposing compile errors - * implement proper parsing of clang stderr/stdout and exposing compile errors - * implement proper compile errors for failing to build glibc crt files and shared libs - * improve the stage2 tests to support testing with LLVM extensions enabled - * multi-thread building C objects * support cross compiling stage2 with `zig build` - * implement emit-h in stage2 - * implement -fno-emit-bin - * audit the base cache hash * --main-pkg-path * audit the CLI options for stage2 * `zig init-lib` * `zig init-exe` * `zig run` * restore error messages for stage2_add_link_lib + * audit the base cache hash + * implement proper compile errors for failing to build glibc crt files and shared libs + * implement -fno-emit-bin + * improve the stage2 tests to support testing with LLVM extensions enabled + * rename src/ to src/stage1/ + * rename src-self-hosted/ to src/ + * implement emit-h in stage2 + * multi-thread building C objects * implement serialization/deserialization of incremental compilation metadata * incremental compilation - implement detection of which source files changed * improve the cache hash logic for c objects with respect to extra flags and file parameters @@ -59,16 +62,13 @@ * integrate target features into building assembly code * libc_installation.zig: make it look for msvc only if msvc abi is chosen * switch the default C ABI for windows to be mingw-w64 - * port windows_sdk.cpp to zig * change glibc log errors to normal exposed compile errors * update Package to use Compilation.Directory in create() - skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd) (maybe make it an explicit option and have main.zig disable it) - make it possible for Package to not openDir and reference already existing resources. - * rename src/ to src/stage1/ - * rename src-self-hosted/ to src/ * improve Directory.join to only use 1 allocation in a clean way. * tracy builds with lc++ * some kind of "zig identifier escape" function rather than unconditionally using @"" syntax in builtin.zig - * rename Mode to OptimizeMode + * rename std.builtin.Mode to std.builtin.OptimizeMode diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index f08c5ef82f..32edc92718 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -46,6 +46,12 @@ sanitize_c: bool, clang_passthrough_mode: bool, /// Whether to print clang argvs to stdout. verbose_cc: bool, +verbose_tokenize: bool, +verbose_ast: bool, +verbose_ir: bool, +verbose_llvm_ir: bool, +verbose_cimport: bool, +verbose_llvm_cpu_features: bool, disable_c_depfile: bool, is_test: bool, @@ -59,18 +65,21 @@ zig_cache_directory: Directory, libc_include_dir_list: []const []const u8, rand: *std.rand.Random, -/// Populated when we build libc++.a. A Job to build this is placed in the queue +/// Populated when we build the libc++ static library. A Job to build this is placed in the queue /// and resolved before calling linker.flush(). libcxx_static_lib: ?[]const u8 = null, -/// Populated when we build libc++abi.a. A Job to build this is placed in the queue +/// Populated when we build the libc++abi static library. A Job to build this is placed in the queue /// and resolved before calling linker.flush(). libcxxabi_static_lib: ?[]const u8 = null, -/// Populated when we build libunwind.a. A Job to build this is placed in the queue +/// Populated when we build the libunwind static library. A Job to build this is placed in the queue /// and resolved before calling linker.flush(). libunwind_static_lib: ?CRTFile = null, -/// Populated when we build c.a. A Job to build this is placed in the queue +/// Populated when we build the libc static library. A Job to build this is placed in the queue /// and resolved before calling linker.flush(). -libc_static_lib: ?[]const u8 = null, +libc_static_lib: ?CRTFile = null, +/// Populated when we build the libcompiler_rt static library. A Job to build this is placed in the queue +/// and resolved before calling linker.flush(). +compiler_rt_static_lib: ?CRTFile = null, glibc_so_files: ?glibc.BuiltSharedObjects = null, @@ -121,6 +130,11 @@ const Job = union(enum) { glibc_shared_objects, /// libunwind.a, usually needed when linking libc libunwind: void, + /// needed when producing a dynamic library or executable + libcompiler_rt: void, + /// needed when not linking libc and using LLVM for code generation because it generates + /// calls to, for example, memcpy and memset. + zig_libc: void, /// Generate builtin.zig source code and write it into the correct place. generate_builtin_zig: void, @@ -310,6 +324,15 @@ pub const InitOptions = struct { }; pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { + const is_dyn_lib = switch (options.output_mode) { + .Obj, .Exe => false, + .Lib => (options.link_mode orelse .Static) == .Dynamic, + }; + const is_exe_or_dyn_lib = switch (options.output_mode) { + .Obj => false, + .Lib => is_dyn_lib, + .Exe => true, + }; const comp: *Compilation = comp: { // For allocations that have the same lifetime as Compilation. This arena is used only during this // initialization and then is freed in deinit(). @@ -375,15 +398,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { return error.MachineCodeModelNotSupported; } - const is_dyn_lib = switch (options.output_mode) { - .Obj, .Exe => false, - .Lib => (options.link_mode orelse .Static) == .Dynamic, - }; - const is_exe_or_dyn_lib = switch (options.output_mode) { - .Obj => false, - .Lib => is_dyn_lib, - .Exe => true, - }; const must_dynamic_link = dl: { if (target_util.cannotDynamicLink(options.target)) break :dl false; @@ -461,6 +475,27 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { }; const single_threaded = options.single_threaded or target_util.isSingleThreaded(options.target); + const function_sections = options.function_sections orelse false; + + const llvm_cpu_features: ?[*:0]const u8 = if (build_options.have_llvm and use_llvm) blk: { + var buf = std.ArrayList(u8).init(arena); + for (options.target.cpu.arch.allFeaturesList()) |feature, index_usize| { + const index = @intCast(Target.Cpu.Feature.Set.Index, index_usize); + const is_enabled = options.target.cpu.features.isEnabled(index); + + if (feature.llvm_name) |llvm_name| { + const plus_or_minus = "-+"[@boolToInt(is_enabled)]; + try buf.ensureCapacity(buf.items.len + 2 + llvm_name.len); + buf.appendAssumeCapacity(plus_or_minus); + buf.appendSliceAssumeCapacity(llvm_name); + buf.appendSliceAssumeCapacity(","); + } + } + assert(mem.endsWith(u8, buf.items, ",")); + buf.items[buf.items.len - 1] = 0; + buf.shrink(buf.items.len); + break :blk buf.items[0 .. buf.items.len - 1 :0].ptr; + } else null; // We put everything into the cache hash that *cannot be modified during an incremental update*. // For example, one cannot change the target between updates, but one can change source files, @@ -489,8 +524,10 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { cache.hash.add(pic); cache.hash.add(stack_check); cache.hash.add(link_mode); + cache.hash.add(function_sections); cache.hash.add(options.strip); cache.hash.add(options.link_libc); + cache.hash.add(options.link_libcpp); cache.hash.add(options.output_mode); cache.hash.add(options.machine_code_model); // TODO audit this and make sure everything is in it @@ -596,10 +633,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { owned_link_dir = artifact_dir; const link_artifact_directory: Directory = .{ .handle = artifact_dir, - .path = if (options.zig_cache_directory.path) |p| - try std.fs.path.join(arena, &[_][]const u8{ p, artifact_sub_dir }) - else - artifact_sub_dir, + .path = try options.zig_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}), }; break :blk link_artifact_directory; }; @@ -609,26 +643,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .ReleaseFast, .ReleaseSmall => false, }; - const llvm_cpu_features: ?[*:0]const u8 = if (build_options.have_llvm and use_llvm) blk: { - var buf = std.ArrayList(u8).init(arena); - for (options.target.cpu.arch.allFeaturesList()) |feature, index_usize| { - const index = @intCast(Target.Cpu.Feature.Set.Index, index_usize); - const is_enabled = options.target.cpu.features.isEnabled(index); - - if (feature.llvm_name) |llvm_name| { - const plus_or_minus = "-+"[@boolToInt(is_enabled)]; - try buf.ensureCapacity(buf.items.len + 2 + llvm_name.len); - buf.appendAssumeCapacity(plus_or_minus); - buf.appendSliceAssumeCapacity(llvm_name); - buf.appendSliceAssumeCapacity(","); - } - } - assert(mem.endsWith(u8, buf.items, ",")); - buf.items[buf.items.len - 1] = 0; - buf.shrink(buf.items.len); - break :blk buf.items[0 .. buf.items.len - 1 :0].ptr; - } else null; - const stage1_module: ?*stage1.Module = if (build_options.is_stage1 and use_llvm) blk: { // Here we use the legacy stage1 C++ compiler to compile Zig code. const stage2_target = try arena.create(stage1.Stage2Target); @@ -646,7 +660,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { if (options.color == .Off) progress.terminal = null; const mod = module.?; - const main_zig_file = mod.root_pkg.root_src_path; + const main_zig_file = try mod.root_pkg.root_src_directory.join(arena, &[_][]const u8{ + mod.root_pkg.root_src_path, + }); const zig_lib_dir = options.zig_lib_directory.path.?; const builtin_sub = &[_][]const u8{"builtin.zig"}; const builtin_zig_path = try mod.zig_cache_artifact_directory.join(arena, builtin_sub); @@ -700,7 +716,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .dll_export_fns = dll_export_fns, .link_mode_dynamic = link_mode == .Dynamic, .valgrind_enabled = valgrind, - .function_sections = options.function_sections orelse false, + .function_sections = function_sections, .enable_stack_probing = stack_check, .enable_time_report = options.time_report, .enable_stack_report = false, @@ -790,6 +806,12 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .rand = options.rand, .clang_passthrough_mode = options.clang_passthrough_mode, .verbose_cc = options.verbose_cc, + .verbose_tokenize = options.verbose_tokenize, + .verbose_ast = options.verbose_ast, + .verbose_ir = options.verbose_ir, + .verbose_llvm_ir = options.verbose_llvm_ir, + .verbose_cimport = options.verbose_cimport, + .verbose_llvm_cpu_features = options.verbose_llvm_cpu_features, .disable_c_depfile = options.disable_c_depfile, .owned_link_dir = owned_link_dir, .is_test = options.is_test, @@ -823,10 +845,15 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { if (comp.wantBuildLibUnwindFromSource()) { try comp.work_queue.writeItem(.{ .libunwind = {} }); } - if (comp.stage1_module) |module| { try comp.work_queue.writeItem(.{ .stage1_module = {} }); } + if (is_exe_or_dyn_lib) { + try comp.work_queue.writeItem(.{ .libcompiler_rt = {} }); + if (!comp.bin_file.options.link_libc) { + try comp.work_queue.writeItem(.{ .zig_libc = {} }); + } + } return comp; } @@ -852,8 +879,14 @@ pub fn destroy(self: *Compilation) void { self.crt_files.deinit(gpa); } - if (self.libunwind_static_lib) |*unwind_crt_file| { - unwind_crt_file.deinit(gpa); + if (self.libunwind_static_lib) |*crt_file| { + crt_file.deinit(gpa); + } + if (self.compiler_rt_static_lib) |*crt_file| { + crt_file.deinit(gpa); + } + if (self.libc_static_lib) |*crt_file| { + crt_file.deinit(gpa); } for (self.c_object_table.items()) |entry| { @@ -889,41 +922,46 @@ pub fn update(self: *Compilation) !void { self.work_queue.writeItemAssumeCapacity(.{ .c_object = entry.key }); } - if (self.bin_file.options.module) |module| { - module.generation += 1; + const use_stage1 = build_options.is_stage1 and self.bin_file.options.use_llvm; + if (!use_stage1) { + if (self.bin_file.options.module) |module| { + module.generation += 1; - // TODO Detect which source files changed. - // Until then we simulate a full cache miss. Source files could have been loaded for any reason; - // to force a refresh we unload now. - if (module.root_scope.cast(Module.Scope.File)) |zig_file| { - zig_file.unload(module.gpa); - module.analyzeContainer(&zig_file.root_container) catch |err| switch (err) { - error.AnalysisFail => { - assert(self.totalErrorCount() != 0); - }, - else => |e| return e, - }; - } else if (module.root_scope.cast(Module.Scope.ZIRModule)) |zir_module| { - zir_module.unload(module.gpa); - module.analyzeRootZIRModule(zir_module) catch |err| switch (err) { - error.AnalysisFail => { - assert(self.totalErrorCount() != 0); - }, - else => |e| return e, - }; + // TODO Detect which source files changed. + // Until then we simulate a full cache miss. Source files could have been loaded for any reason; + // to force a refresh we unload now. + if (module.root_scope.cast(Module.Scope.File)) |zig_file| { + zig_file.unload(module.gpa); + module.analyzeContainer(&zig_file.root_container) catch |err| switch (err) { + error.AnalysisFail => { + assert(self.totalErrorCount() != 0); + }, + else => |e| return e, + }; + } else if (module.root_scope.cast(Module.Scope.ZIRModule)) |zir_module| { + zir_module.unload(module.gpa); + module.analyzeRootZIRModule(zir_module) catch |err| switch (err) { + error.AnalysisFail => { + assert(self.totalErrorCount() != 0); + }, + else => |e| return e, + }; + } } } try self.performAllTheWork(); - if (self.bin_file.options.module) |module| { - // Process the deletion set. - while (module.deletion_set.popOrNull()) |decl| { - if (decl.dependants.items().len != 0) { - decl.deletion_flag = false; - continue; + if (!use_stage1) { + if (self.bin_file.options.module) |module| { + // Process the deletion set. + while (module.deletion_set.popOrNull()) |decl| { + if (decl.dependants.items().len != 0) { + decl.deletion_flag = false; + continue; + } + try module.deleteDecl(decl); } - try module.deleteDecl(decl); } } @@ -1142,6 +1180,18 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { fatal("unable to build libunwind: {}", .{@errorName(err)}); }; }, + .libcompiler_rt => { + self.buildStaticLibFromZig("compiler_rt.zig", &self.compiler_rt_static_lib) catch |err| { + // TODO Expose this as a normal compile error rather than crashing here. + fatal("unable to build compiler_rt: {}", .{@errorName(err)}); + }; + }, + .zig_libc => { + self.buildStaticLibFromZig("c.zig", &self.libc_static_lib) catch |err| { + // TODO Expose this as a normal compile error rather than crashing here. + fatal("unable to build zig's multitarget libc: {}", .{@errorName(err)}); + }; + }, .generate_builtin_zig => { // This Job is only queued up if there is a zig module. self.updateBuiltinZigFile(self.bin_file.options.module.?) catch |err| { @@ -1691,7 +1741,7 @@ pub fn classifyFileExt(filename: []const u8) FileExt { } else if (mem.endsWith(u8, filename, ".zig")) { return .zig; } else if (mem.endsWith(u8, filename, ".zir")) { - return .zig; + return .zir; } else if (hasSharedLibraryExt(filename)) { return .shared_library; } else if (hasStaticLibraryExt(filename)) { @@ -2030,3 +2080,97 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8 }); return buffer.toOwnedSlice(); } + +pub fn updateSubCompilation(sub_compilation: *Compilation) !void { + try sub_compilation.update(); + + // Look for compilation errors in this sub_compilation + var errors = try sub_compilation.getAllErrorsAlloc(); + defer errors.deinit(sub_compilation.gpa); + + if (errors.list.len != 0) { + for (errors.list) |full_err_msg| { + std.log.err("{}:{}:{}: {}\n", .{ + full_err_msg.src_path, + full_err_msg.line + 1, + full_err_msg.column + 1, + full_err_msg.msg, + }); + } + return error.BuildingLibCObjectFailed; + } +} + +fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFile) !void { + const tracy = trace(@src()); + defer tracy.end(); + + const special_sub = "std" ++ std.fs.path.sep_str ++ "special"; + const special_path = try comp.zig_lib_directory.join(comp.gpa, &[_][]const u8{special_sub}); + defer comp.gpa.free(special_path); + + var special_dir = try comp.zig_lib_directory.handle.openDir(special_sub, .{}); + defer special_dir.close(); + + var root_pkg: Package = .{ + .root_src_directory = .{ + .path = special_path, + .handle = special_dir, + }, + .root_src_path = basename, + }; + + const emit_bin = Compilation.EmitLoc{ + .directory = null, // Put it in the cache directory. + .basename = basename, + }; + const optimize_mode: std.builtin.Mode = blk: { + if (comp.is_test) + break :blk comp.bin_file.options.optimize_mode; + switch (comp.bin_file.options.optimize_mode) { + .Debug, .ReleaseFast, .ReleaseSafe => break :blk .ReleaseFast, + .ReleaseSmall => break :blk .ReleaseSmall, + } + }; + const sub_compilation = try Compilation.create(comp.gpa, .{ + // TODO use the global cache directory here + .zig_cache_directory = comp.zig_cache_directory, + .zig_lib_directory = comp.zig_lib_directory, + .target = comp.getTarget(), + .root_name = mem.split(basename, ".").next().?, + .root_pkg = &root_pkg, + .output_mode = .Lib, + .rand = comp.rand, + .libc_installation = comp.bin_file.options.libc_installation, + .emit_bin = emit_bin, + .optimize_mode = optimize_mode, + .link_mode = .Static, + .function_sections = true, + .want_sanitize_c = false, + .want_stack_check = false, + .want_valgrind = false, + .want_pic = comp.bin_file.options.pic, + .emit_h = null, + .strip = comp.bin_file.options.strip, + .is_native_os = comp.bin_file.options.is_native_os, + .self_exe_path = comp.self_exe_path, + .verbose_cc = comp.verbose_cc, + .verbose_link = comp.bin_file.options.verbose_link, + .verbose_tokenize = comp.verbose_tokenize, + .verbose_ast = comp.verbose_ast, + .verbose_ir = comp.verbose_ir, + .verbose_llvm_ir = comp.verbose_llvm_ir, + .verbose_cimport = comp.verbose_cimport, + .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features, + .clang_passthrough_mode = comp.clang_passthrough_mode, + }); + defer sub_compilation.destroy(); + + try sub_compilation.updateSubCompilation(); + + assert(out.* == null); + out.* = Compilation.CRTFile{ + .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{basename}), + .lock = sub_compilation.bin_file.toOwnedLock(), + }; +} diff --git a/src-self-hosted/Package.zig b/src-self-hosted/Package.zig index 027aaafe7d..14be8b64d6 100644 --- a/src-self-hosted/Package.zig +++ b/src-self-hosted/Package.zig @@ -1,32 +1,41 @@ pub const Table = std.StringHashMapUnmanaged(*Package); root_src_directory: Compilation.Directory, -/// Relative to `root_src_directory`. -root_src_path: []u8, -table: Table, +/// Relative to `root_src_directory`. May contain path separators. +root_src_path: []const u8, +table: Table = .{}, + +const std = @import("std"); +const mem = std.mem; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const Package = @This(); +const Compilation = @import("Compilation.zig"); /// No references to `root_src_dir` and `root_src_path` are kept. pub fn create( gpa: *Allocator, - base_dir: std.fs.Dir, - /// Relative to `base_dir`. + base_directory: Compilation.Directory, + /// Relative to `base_directory`. root_src_dir: []const u8, /// Relative to `root_src_dir`. root_src_path: []const u8, ) !*Package { const ptr = try gpa.create(Package); errdefer gpa.destroy(ptr); + + const root_src_dir_path = try base_directory.join(gpa, &[_][]const u8{root_src_dir}); + errdefer gpa.free(root_src_dir_path); + const root_src_path_dupe = try mem.dupe(gpa, u8, root_src_path); errdefer gpa.free(root_src_path_dupe); - const root_src_dir_path = try mem.dupe(gpa, u8, root_src_dir); - errdefer gpa.free(root_src_dir_path); + ptr.* = .{ .root_src_directory = .{ .path = root_src_dir_path, - .handle = try base_dir.openDir(root_src_dir, .{}), + .handle = try base_directory.handle.openDir(root_src_dir, .{}), }, .root_src_path = root_src_path_dupe, - .table = .{}, }; return ptr; } @@ -50,10 +59,3 @@ pub fn add(pkg: *Package, gpa: *Allocator, name: []const u8, package: *Package) const name_dupe = try mem.dupe(gpa, u8, name); pkg.table.putAssumeCapacityNoClobber(name_dupe, package); } - -const std = @import("std"); -const mem = std.mem; -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; -const Package = @This(); -const Compilation = @import("Compilation.zig"); diff --git a/src-self-hosted/glibc.zig b/src-self-hosted/glibc.zig index 5e34302c06..9ede436f63 100644 --- a/src-self-hosted/glibc.zig +++ b/src-self-hosted/glibc.zig @@ -713,11 +713,17 @@ fn build_crt_file( .c_source_files = c_source_files, .verbose_cc = comp.verbose_cc, .verbose_link = comp.bin_file.options.verbose_link, + .verbose_tokenize = comp.verbose_tokenize, + .verbose_ast = comp.verbose_ast, + .verbose_ir = comp.verbose_ir, + .verbose_llvm_ir = comp.verbose_llvm_ir, + .verbose_cimport = comp.verbose_cimport, + .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features, .clang_passthrough_mode = comp.clang_passthrough_mode, }); defer sub_compilation.destroy(); - try updateSubCompilation(sub_compilation); + try sub_compilation.updateSubCompilation(); try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1); const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p| @@ -989,6 +995,12 @@ fn buildSharedLib( .self_exe_path = comp.self_exe_path, .verbose_cc = comp.verbose_cc, .verbose_link = comp.bin_file.options.verbose_link, + .verbose_tokenize = comp.verbose_tokenize, + .verbose_ast = comp.verbose_ast, + .verbose_ir = comp.verbose_ir, + .verbose_llvm_ir = comp.verbose_llvm_ir, + .verbose_cimport = comp.verbose_cimport, + .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features, .clang_passthrough_mode = comp.clang_passthrough_mode, .version = version, .version_script = map_file_path, @@ -997,25 +1009,5 @@ fn buildSharedLib( }); defer sub_compilation.destroy(); - try updateSubCompilation(sub_compilation); -} - -fn updateSubCompilation(sub_compilation: *Compilation) !void { - try sub_compilation.update(); - - // Look for compilation errors in this sub_compilation - var errors = try sub_compilation.getAllErrorsAlloc(); - defer errors.deinit(sub_compilation.gpa); - - if (errors.list.len != 0) { - for (errors.list) |full_err_msg| { - std.log.err("{}:{}:{}: {}\n", .{ - full_err_msg.src_path, - full_err_msg.line + 1, - full_err_msg.column + 1, - full_err_msg.msg, - }); - } - return error.BuildingLibCObjectFailed; - } + try sub_compilation.updateSubCompilation(); } diff --git a/src-self-hosted/libunwind.zig b/src-self-hosted/libunwind.zig index 3ba52573ce..0bb44808ff 100644 --- a/src-self-hosted/libunwind.zig +++ b/src-self-hosted/libunwind.zig @@ -109,12 +109,18 @@ pub fn buildStaticLib(comp: *Compilation) !void { .c_source_files = &c_source_files, .verbose_cc = comp.verbose_cc, .verbose_link = comp.bin_file.options.verbose_link, + .verbose_tokenize = comp.verbose_tokenize, + .verbose_ast = comp.verbose_ast, + .verbose_ir = comp.verbose_ir, + .verbose_llvm_ir = comp.verbose_llvm_ir, + .verbose_cimport = comp.verbose_cimport, + .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features, .clang_passthrough_mode = comp.clang_passthrough_mode, .link_libc = true, }); defer sub_compilation.destroy(); - try updateSubCompilation(sub_compilation); + try sub_compilation.updateSubCompilation(); assert(comp.libunwind_static_lib == null); comp.libunwind_static_lib = Compilation.CRTFile{ @@ -122,23 +128,3 @@ pub fn buildStaticLib(comp: *Compilation) !void { .lock = sub_compilation.bin_file.toOwnedLock(), }; } - -fn updateSubCompilation(sub_compilation: *Compilation) !void { - try sub_compilation.update(); - - // Look for compilation errors in this sub_compilation - var errors = try sub_compilation.getAllErrorsAlloc(); - defer errors.deinit(sub_compilation.gpa); - - if (errors.list.len != 0) { - for (errors.list) |full_err_msg| { - std.log.err("{}:{}:{}: {}\n", .{ - full_err_msg.src_path, - full_err_msg.line + 1, - full_err_msg.column + 1, - full_err_msg.msg, - }); - } - return error.BuildingLibCObjectFailed; - } -} diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 88a964f2ff..3ec81715e4 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -387,13 +387,15 @@ pub const File = struct { // If there is no Zig code to compile, then we should skip flushing the output file because it // will not be part of the linker line anyway. const module_obj_path: ?[]const u8 = if (base.options.module) |module| blk: { + const use_stage1 = build_options.is_stage1 and base.options.use_llvm; + if (use_stage1) { + const obj_basename = try std.fmt.allocPrint(arena, "{}.o", .{base.options.root_name}); + const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename}); + break :blk full_obj_path; + } try base.flushModule(comp); - const obj_basename = base.intermediary_basename.?; - const full_obj_path = if (directory.path) |dir_path| - try std.fs.path.join(arena, &[_][]const u8{ dir_path, obj_basename }) - else - obj_basename; + const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename}); break :blk full_obj_path; } else null; diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 5257b46279..0a44665176 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -1253,8 +1253,9 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { const is_lib = self.base.options.output_mode == .Lib; const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib; + const is_exe_or_dyn_lib = is_dyn_lib or self.base.options.output_mode == .Exe; const have_dynamic_linker = self.base.options.link_libc and - self.base.options.link_mode == .Dynamic and (is_dyn_lib or self.base.options.output_mode == .Exe); + self.base.options.link_mode == .Dynamic and is_exe_or_dyn_lib; try ch.addOptionalFile(self.base.options.linker_script); try ch.addOptionalFile(self.base.options.version_script); @@ -1489,16 +1490,13 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { try argv.append(p); } - // TODO compiler-rt and libc - //if (!g->is_dummy_so && (g->out_type == OutTypeExe || is_dyn_lib)) { - // if (g->libc_link_lib == nullptr) { - // Buf *libc_a_path = build_c(g, OutTypeLib, lj->build_dep_prog_node); - // try argv.append(buf_ptr(libc_a_path)); - // } - - // Buf *compiler_rt_o_path = build_compiler_rt(g, OutTypeLib, lj->build_dep_prog_node); - // try argv.append(buf_ptr(compiler_rt_o_path)); - //} + // compiler-rt and libc + if (is_exe_or_dyn_lib) { + if (!self.base.options.link_libc) { + try argv.append(comp.libc_static_lib.?.full_object_path); + } + try argv.append(comp.compiler_rt_static_lib.?.full_object_path); + } // Shared libraries. try argv.ensureCapacity(argv.items.len + self.base.options.system_libs.len); @@ -1545,7 +1543,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { try argv.append(try comp.get_libc_crt_file(arena, "libc_nonshared.a")); } else if (target.isMusl()) { try argv.append(comp.libunwind_static_lib.?.full_object_path); - try argv.append(comp.libc_static_lib.?); + try argv.append(comp.libc_static_lib.?.full_object_path); } else if (self.base.options.link_libcpp) { try argv.append(comp.libunwind_static_lib.?.full_object_path); } else { diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 6aa4be7c41..430a85b93e 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -1108,8 +1108,14 @@ pub fn buildOutputType( .yes => |p| p, }; - const root_pkg = if (root_src_file) |src_path| try Package.create(gpa, fs.cwd(), ".", src_path) else null; - defer if (root_pkg) |pkg| pkg.destroy(gpa); + var root_pkg_memory: Package = undefined; + const root_pkg: ?*Package = if (root_src_file) |src_path| blk: { + root_pkg_memory = .{ + .root_src_directory = .{ .path = null, .handle = fs.cwd() }, + .root_src_path = src_path, + }; + break :blk &root_pkg_memory; + } else null; const self_exe_path = try fs.selfExePathAlloc(arena); var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| { From 61c54b2d49bf087465e2c9a72a190e6ee5861027 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Sep 2020 18:21:49 -0700 Subject: [PATCH 082/210] Cache integration for stage1 zig code compilation --- BRANCH_TODO | 3 - src-self-hosted/Cache.zig | 2 +- src-self-hosted/Compilation.zig | 276 ++++++++++++++++++++------------ src-self-hosted/link/Elf.zig | 2 +- src-self-hosted/main.zig | 2 +- src-self-hosted/stage1.zig | 4 +- 6 files changed, 178 insertions(+), 111 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 35972cb3e2..920c6f04c6 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,6 +1,3 @@ - * Cache integration for stage1 zig code compilation - * build & link against compiler-rt - * build & link against freestanding libc * resolve builtin.zig not working on windows & macos; try os_path_real * build & link against libcxx and libcxxabi * `zig test` diff --git a/src-self-hosted/Cache.zig b/src-self-hosted/Cache.zig index cbab24715e..24c6ae3ac4 100644 --- a/src-self-hosted/Cache.zig +++ b/src-self-hosted/Cache.zig @@ -410,7 +410,7 @@ pub const CacheHash = struct { /// calculated. This is useful for processes that don't know the all the files that /// are depended on ahead of time. For example, a source file that can import other files /// will need to be recompiled if the imported file is changed. - pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 { + pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]const u8 { assert(self.manifest_file != null); const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path}); diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 32edc92718..29c6dc36cf 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -27,7 +27,8 @@ gpa: *Allocator, arena_state: std.heap.ArenaAllocator.State, bin_file: *link.File, c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{}, -stage1_module: ?*stage1.Module, +stage1_lock: ?Cache.Lock = null, +stage1_cache_hash: *Cache.CacheHash = undefined, link_error_flags: link.File.ErrorFlags = .{}, @@ -54,6 +55,7 @@ verbose_cimport: bool, verbose_llvm_cpu_features: bool, disable_c_depfile: bool, is_test: bool, +time_report: bool, c_source_files: []const CSourceFile, clang_argv: []const []const u8, @@ -91,6 +93,10 @@ crt_files: std.StringHashMapUnmanaged(CRTFile) = .{}, /// Keeping track of this possibly open resource so we can close it later. owned_link_dir: ?std.fs.Dir, +/// This is for stage1 and should be deleted upon completion of self-hosting. +/// Don't use this for anything other than stage1 compatibility. +color: @import("main.zig").Color = .Auto, + pub const InnerError = Module.InnerError; pub const CRTFile = struct { @@ -643,100 +649,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .ReleaseFast, .ReleaseSmall => false, }; - const stage1_module: ?*stage1.Module = if (build_options.is_stage1 and use_llvm) blk: { - // Here we use the legacy stage1 C++ compiler to compile Zig code. - const stage2_target = try arena.create(stage1.Stage2Target); - stage2_target.* = .{ - .arch = @enumToInt(options.target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch - .os = @enumToInt(options.target.os.tag), - .abi = @enumToInt(options.target.abi), - .is_native_os = options.is_native_os, - .is_native_cpu = false, // Only true when bootstrapping the compiler. - .llvm_cpu_name = if (options.target.cpu.model.llvm_name) |s| s.ptr else null, - .llvm_cpu_features = llvm_cpu_features.?, - }; - const progress = try arena.create(std.Progress); - const main_progress_node = try progress.start("", 100); - if (options.color == .Off) progress.terminal = null; - - const mod = module.?; - const main_zig_file = try mod.root_pkg.root_src_directory.join(arena, &[_][]const u8{ - mod.root_pkg.root_src_path, - }); - const zig_lib_dir = options.zig_lib_directory.path.?; - const builtin_sub = &[_][]const u8{"builtin.zig"}; - const builtin_zig_path = try mod.zig_cache_artifact_directory.join(arena, builtin_sub); - - const stage1_module = stage1.create( - @enumToInt(options.optimize_mode), - undefined, - 0, // TODO --main-pkg-path - main_zig_file.ptr, - main_zig_file.len, - zig_lib_dir.ptr, - zig_lib_dir.len, - stage2_target, - options.is_test, - ) orelse return error.OutOfMemory; - - const output_dir = bin_directory.path orelse "."; - - const stage1_pkg = try arena.create(stage1.Pkg); - stage1_pkg.* = .{ - .name_ptr = undefined, - .name_len = 0, - .path_ptr = undefined, - .path_len = 0, - .children_ptr = undefined, - .children_len = 0, - .parent = null, - }; - - stage1_module.* = .{ - .root_name_ptr = root_name.ptr, - .root_name_len = root_name.len, - .output_dir_ptr = output_dir.ptr, - .output_dir_len = output_dir.len, - .builtin_zig_path_ptr = builtin_zig_path.ptr, - .builtin_zig_path_len = builtin_zig_path.len, - .test_filter_ptr = "", - .test_filter_len = 0, - .test_name_prefix_ptr = "", - .test_name_prefix_len = 0, - .userdata = @ptrToInt(comp), - .root_pkg = stage1_pkg, - .code_model = @enumToInt(options.machine_code_model), - .subsystem = stage1.TargetSubsystem.Auto, - .err_color = @enumToInt(options.color), - .pic = pic, - .link_libc = options.link_libc, - .link_libcpp = options.link_libcpp, - .strip = options.strip, - .is_single_threaded = single_threaded, - .dll_export_fns = dll_export_fns, - .link_mode_dynamic = link_mode == .Dynamic, - .valgrind_enabled = valgrind, - .function_sections = function_sections, - .enable_stack_probing = stack_check, - .enable_time_report = options.time_report, - .enable_stack_report = false, - .dump_analysis = false, - .enable_doc_generation = false, - .emit_bin = true, - .emit_asm = false, - .emit_llvm_ir = false, - .test_is_evented = false, - .verbose_tokenize = options.verbose_tokenize, - .verbose_ast = options.verbose_ast, - .verbose_ir = options.verbose_ir, - .verbose_llvm_ir = options.verbose_llvm_ir, - .verbose_cimport = options.verbose_cimport, - .verbose_llvm_cpu_features = options.verbose_llvm_cpu_features, - .main_progress_node = main_progress_node, - }; - break :blk stage1_module; - } else null; - const bin_file = try link.File.openPath(gpa, .{ .directory = bin_directory, .sub_path = emit_bin.basename, @@ -793,7 +705,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .zig_lib_directory = options.zig_lib_directory, .zig_cache_directory = options.zig_cache_directory, .bin_file = bin_file, - .stage1_module = stage1_module, .work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa), .keep_source_files_loaded = options.keep_source_files_loaded, .use_clang = use_clang, @@ -815,6 +726,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .disable_c_depfile = options.disable_c_depfile, .owned_link_dir = owned_link_dir, .is_test = options.is_test, + .color = options.color, + .time_report = options.time_report, }; break :comp comp; }; @@ -845,7 +758,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { if (comp.wantBuildLibUnwindFromSource()) { try comp.work_queue.writeItem(.{ .libunwind = {} }); } - if (comp.stage1_module) |module| { + if (build_options.is_stage1 and comp.bin_file.options.use_llvm) { try comp.work_queue.writeItem(.{ .stage1_module = {} }); } if (is_exe_or_dyn_lib) { @@ -858,15 +771,19 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { return comp; } +fn releaseStage1Lock(comp: *Compilation) void { + if (comp.stage1_lock) |*lock| { + lock.release(); + comp.stage1_lock = null; + } +} + pub fn destroy(self: *Compilation) void { const optional_module = self.bin_file.options.module; self.bin_file.destroy(); if (optional_module) |module| module.deinit(); - if (self.stage1_module) |module| { - module.main_progress_node.?.end(); - module.destroy(); - } + self.releaseStage1Lock(); const gpa = self.gpa; self.work_queue.deinit(); @@ -1200,8 +1117,9 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { }; }, .stage1_module => { - // This Job is only queued up if there is a zig module. - self.stage1_module.?.build_object(); + self.updateStage1Module() catch |err| { + fatal("unable to build stage1 zig object: {}", .{@errorName(err)}); + }; }, }; } @@ -2174,3 +2092,155 @@ fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFil .lock = sub_compilation.bin_file.toOwnedLock(), }; } + +fn updateStage1Module(comp: *Compilation) !void { + const tracy = trace(@src()); + defer tracy.end(); + + var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); + defer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + // Here we use the legacy stage1 C++ compiler to compile Zig code. + const mod = comp.bin_file.options.module.?; + const directory = mod.zig_cache_artifact_directory; // Just an alias to make it shorter to type. + const main_zig_file = try mod.root_pkg.root_src_directory.join(arena, &[_][]const u8{ + mod.root_pkg.root_src_path, + }); + const zig_lib_dir = comp.zig_lib_directory.path.?; + const builtin_zig_path = try directory.join(arena, &[_][]const u8{"builtin.zig"}); + const target = comp.getTarget(); + const id_symlink_basename = "stage1.id"; + + // We are about to obtain this lock, so here we give other processes a chance first. + comp.releaseStage1Lock(); + + // Unlike with the self-hosted Zig module, stage1 does not support incremental compilation, + // so we input all the zig source files into the cache hash system. We're going to keep + // the artifact directory the same, however, so we take the same strategy as linking + // does where we have a file which specifies the hash of the output directory so that we can + // skip the expensive compilation step if the hash matches. + var ch = comp.cache_parent.obtain(); + defer ch.deinit(); + + _ = try ch.addFile(main_zig_file, null); + ch.hash.add(comp.bin_file.options.valgrind); + ch.hash.add(comp.bin_file.options.single_threaded); + ch.hash.add(target.os.getVersionRange()); + ch.hash.add(comp.bin_file.options.dll_export_fns); + ch.hash.add(comp.bin_file.options.function_sections); + + if (try ch.hit()) { + const digest = ch.final(); + + var prev_digest_buf: [digest.len]u8 = undefined; + const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { + // Handle this as a cache miss. + break :blk prev_digest_buf[0..0]; + }; + if (mem.eql(u8, prev_digest, &digest)) { + comp.stage1_lock = ch.toOwnedLock(); + return; + } + } + + const stage2_target = try arena.create(stage1.Stage2Target); + stage2_target.* = .{ + .arch = @enumToInt(target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch + .os = @enumToInt(target.os.tag), + .abi = @enumToInt(target.abi), + .is_native_os = comp.bin_file.options.is_native_os, + .is_native_cpu = false, // Only true when bootstrapping the compiler. + .llvm_cpu_name = if (target.cpu.model.llvm_name) |s| s.ptr else null, + .llvm_cpu_features = comp.bin_file.options.llvm_cpu_features.?, + }; + var progress: std.Progress = .{}; + var main_progress_node = try progress.start("", 100); + defer main_progress_node.end(); + if (comp.color == .Off) progress.terminal = null; + + comp.stage1_cache_hash = &ch; + + const stage1_module = stage1.create( + @enumToInt(comp.bin_file.options.optimize_mode), + undefined, + 0, // TODO --main-pkg-path + main_zig_file.ptr, + main_zig_file.len, + zig_lib_dir.ptr, + zig_lib_dir.len, + stage2_target, + comp.is_test, + ) orelse return error.OutOfMemory; + + const stage1_pkg = try arena.create(stage1.Pkg); + stage1_pkg.* = .{ + .name_ptr = undefined, + .name_len = 0, + .path_ptr = undefined, + .path_len = 0, + .children_ptr = undefined, + .children_len = 0, + .parent = null, + }; + const output_dir = comp.bin_file.options.directory.path orelse "."; + stage1_module.* = .{ + .root_name_ptr = comp.bin_file.options.root_name.ptr, + .root_name_len = comp.bin_file.options.root_name.len, + .output_dir_ptr = output_dir.ptr, + .output_dir_len = output_dir.len, + .builtin_zig_path_ptr = builtin_zig_path.ptr, + .builtin_zig_path_len = builtin_zig_path.len, + .test_filter_ptr = "", + .test_filter_len = 0, + .test_name_prefix_ptr = "", + .test_name_prefix_len = 0, + .userdata = @ptrToInt(comp), + .root_pkg = stage1_pkg, + .code_model = @enumToInt(comp.bin_file.options.machine_code_model), + .subsystem = stage1.TargetSubsystem.Auto, + .err_color = @enumToInt(comp.color), + .pic = comp.bin_file.options.pic, + .link_libc = comp.bin_file.options.link_libc, + .link_libcpp = comp.bin_file.options.link_libcpp, + .strip = comp.bin_file.options.strip, + .is_single_threaded = comp.bin_file.options.single_threaded, + .dll_export_fns = comp.bin_file.options.dll_export_fns, + .link_mode_dynamic = comp.bin_file.options.link_mode == .Dynamic, + .valgrind_enabled = comp.bin_file.options.valgrind, + .function_sections = comp.bin_file.options.function_sections, + .enable_stack_probing = comp.bin_file.options.stack_check, + .enable_time_report = comp.time_report, + .enable_stack_report = false, + .dump_analysis = false, + .enable_doc_generation = false, + .emit_bin = true, + .emit_asm = false, + .emit_llvm_ir = false, + .test_is_evented = false, + .verbose_tokenize = comp.verbose_tokenize, + .verbose_ast = comp.verbose_ast, + .verbose_ir = comp.verbose_ir, + .verbose_llvm_ir = comp.verbose_llvm_ir, + .verbose_cimport = comp.verbose_cimport, + .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features, + .main_progress_node = main_progress_node, + }; + stage1_module.build_object(); + stage1_module.destroy(); + + const digest = ch.final(); + + // Update the dangling symlink with the digest. If it fails we can continue; it only + // means that the next invocation will have an unnecessary cache miss. + directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { + std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); + }; + // Again failure here only means an unnecessary cache miss. + ch.writeManifest() catch |err| { + std.log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)}); + }; + // We hang on to this lock so that the output file path can be used without + // other processes clobbering it. + comp.stage1_lock = ch.toOwnedLock(); +} diff --git a/src-self-hosted/link/Elf.zig b/src-self-hosted/link/Elf.zig index 0a44665176..98deefb3bf 100644 --- a/src-self-hosted/link/Elf.zig +++ b/src-self-hosted/link/Elf.zig @@ -1590,7 +1590,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { const ok = llvm.Link(.ELF, new_argv.ptr, new_argv.len, append_diagnostic, 0, 0); if (!ok) return error.LLDReportedFailure; - // Update the dangling symlink "id.txt" with the digest. If it fails we can continue; it only + // Update the dangling symlink with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 430a85b93e..02ba72b51c 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -1239,7 +1239,7 @@ pub fn buildOutputType( fatal("TODO: implement `zig cc` when using it as a preprocessor", .{}); } - if (build_options.is_stage1 and comp.stage1_module != null and watch) { + if (build_options.is_stage1 and comp.stage1_lock != null and watch) { std.log.warn("--watch is not recommended with the stage1 backend; it leaks memory and is not capable of incremental compilation", .{}); } diff --git a/src-self-hosted/stage1.zig b/src-self-hosted/stage1.zig index d8d32f02cd..1ff7b4cf4c 100644 --- a/src-self-hosted/stage1.zig +++ b/src-self-hosted/stage1.zig @@ -343,9 +343,9 @@ export fn stage2_fetch_file( result_len: *usize, ) ?[*]const u8 { const comp = @intToPtr(*Compilation, stage1.userdata); - // TODO integrate this with cache hash const file_path = path_ptr[0..path_len]; - const contents = std.fs.cwd().readFileAlloc(comp.gpa, file_path, std.math.maxInt(u32)) catch return null; + const max_file_size = std.math.maxInt(u32); + const contents = comp.stage1_cache_hash.addFilePostFetch(file_path, max_file_size) catch return null; result_len.* = contents.len; return contents.ptr; } From b9f61d401502f5d221e72c0d0e3bf448b11dcd68 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Sep 2020 18:26:06 -0700 Subject: [PATCH 083/210] stage1: resolve builtin.zig path when bootstrapping --- BRANCH_TODO | 1 - src/codegen.cpp | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 920c6f04c6..ae725e118a 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ - * resolve builtin.zig not working on windows & macos; try os_path_real * build & link against libcxx and libcxxabi * `zig test` * `zig build` diff --git a/src/codegen.cpp b/src/codegen.cpp index 6768021410..0a8a1f7a95 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -8861,6 +8861,9 @@ static Error define_builtin_compile_vars(CodeGen *g) { g->builtin_zig_path = buf_alloc(); os_path_join(g->output_dir, buf_create_from_str(builtin_zig_basename), g->builtin_zig_path); + Buf *resolve_paths[] = { g->builtin_zig_path, }; + *g->builtin_zig_path = os_path_resolve(resolve_paths, 1); + contents = codegen_generate_builtin_source(g); if ((err = os_write_file(g->builtin_zig_path, contents))) { fprintf(stderr, "Unable to write file '%s': %s\n", buf_ptr(g->builtin_zig_path), err_str(err)); From 528832bd3a2e7b686ee84aef5887df740a6114db Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Sep 2020 18:38:55 -0700 Subject: [PATCH 084/210] rename src-self-hosted/ to src/ --- BRANCH_TODO | 2 - CMakeLists.txt | 53 +++++++++--------- build.zig | 4 +- {src-self-hosted => src}/Cache.zig | 0 {src-self-hosted => src}/Compilation.zig | 0 {src-self-hosted => src}/DepTokenizer.zig | 0 {src-self-hosted => src}/Module.zig | 0 {src-self-hosted => src}/Package.zig | 0 {src-self-hosted => src}/TypedValue.zig | 0 {src-self-hosted => src}/astgen.zig | 0 {src-self-hosted => src}/clang.zig | 0 {src-self-hosted => src}/clang_options.zig | 0 .../clang_options_data.zig | 0 {src-self-hosted => src}/codegen.zig | 0 {src-self-hosted => src}/codegen/arm.zig | 0 {src-self-hosted => src}/codegen/c.zig | 0 {src-self-hosted => src}/codegen/llvm.zig | 0 {src-self-hosted => src}/codegen/riscv64.zig | 0 {src-self-hosted => src}/codegen/spu-mk2.zig | 0 .../codegen/spu-mk2/interpreter.zig | 0 {src-self-hosted => src}/codegen/wasm.zig | 0 {src-self-hosted => src}/codegen/x86.zig | 0 {src-self-hosted => src}/codegen/x86_64.zig | 0 {src-self-hosted => src}/glibc.zig | 0 {src-self-hosted => src}/introspect.zig | 0 {src-self-hosted => src}/ir.zig | 0 .../libc_installation.zig | 0 {src-self-hosted => src}/libcxx.zig | 0 {src-self-hosted => src}/libunwind.zig | 0 {src-self-hosted => src}/link.zig | 0 {src-self-hosted => src}/link/C.zig | 0 {src-self-hosted => src}/link/Coff.zig | 0 {src-self-hosted => src}/link/Elf.zig | 0 {src-self-hosted => src}/link/MachO.zig | 0 {src-self-hosted => src}/link/Wasm.zig | 0 {src-self-hosted => src}/link/cbe.h | 0 {src-self-hosted => src}/link/msdos-stub.bin | Bin {src-self-hosted => src}/liveness.zig | 0 {src-self-hosted => src}/llvm.zig | 0 {src-self-hosted => src}/main.zig | 0 {src-self-hosted => src}/musl.zig | 0 {src-self-hosted => src}/print_env.zig | 0 {src-self-hosted => src}/print_targets.zig | 0 {src-self-hosted => src}/stage1.zig | 0 src/{ => stage1}/all_types.hpp | 0 src/{ => stage1}/analyze.cpp | 0 src/{ => stage1}/analyze.hpp | 0 src/{ => stage1}/ast_render.cpp | 0 src/{ => stage1}/ast_render.hpp | 0 src/{ => stage1}/bigfloat.cpp | 0 src/{ => stage1}/bigfloat.hpp | 0 src/{ => stage1}/bigint.cpp | 0 src/{ => stage1}/bigint.hpp | 0 src/{ => stage1}/buffer.cpp | 0 src/{ => stage1}/buffer.hpp | 0 src/{ => stage1}/codegen.cpp | 0 src/{ => stage1}/codegen.hpp | 0 src/{ => stage1}/config.h.in | 0 src/{ => stage1}/dump_analysis.cpp | 0 src/{ => stage1}/dump_analysis.hpp | 0 src/{ => stage1}/empty.cpp | 0 src/{ => stage1}/errmsg.cpp | 0 src/{ => stage1}/errmsg.hpp | 0 src/{ => stage1}/error.cpp | 0 src/{ => stage1}/error.hpp | 0 src/{ => stage1}/hash_map.hpp | 0 src/{ => stage1}/heap.cpp | 0 src/{ => stage1}/heap.hpp | 0 src/{ => stage1}/ir.cpp | 0 src/{ => stage1}/ir.hpp | 0 src/{ => stage1}/ir_print.cpp | 0 src/{ => stage1}/ir_print.hpp | 0 src/{ => stage1}/list.hpp | 0 src/{ => stage1}/mem.cpp | 0 src/{ => stage1}/mem.hpp | 0 src/{ => stage1}/mem_hash_map.hpp | 0 src/{ => stage1}/mem_list.hpp | 0 src/{ => stage1}/mem_type_info.hpp | 0 src/{ => stage1}/os.cpp | 0 src/{ => stage1}/os.hpp | 0 src/{ => stage1}/parse_f128.c | 0 src/{ => stage1}/parse_f128.h | 0 src/{ => stage1}/parser.cpp | 0 src/{ => stage1}/parser.hpp | 0 src/{ => stage1}/range_set.cpp | 0 src/{ => stage1}/range_set.hpp | 0 src/{ => stage1}/softfloat.hpp | 0 src/{ => stage1}/softfloat_ext.cpp | 0 src/{ => stage1}/softfloat_ext.hpp | 0 src/{ => stage1}/stage1.cpp | 0 src/{ => stage1}/stage1.h | 0 src/{ => stage1}/stage2.h | 0 src/{ => stage1}/target.cpp | 0 src/{ => stage1}/target.hpp | 0 src/{ => stage1}/tokenizer.cpp | 0 src/{ => stage1}/tokenizer.hpp | 0 src/{ => stage1}/util.cpp | 0 src/{ => stage1}/util.hpp | 0 src/{ => stage1}/util_base.hpp | 0 src/{ => stage1}/zig0.cpp | 0 {src-self-hosted => src}/target.zig | 0 {src-self-hosted => src}/test.zig | 0 {src-self-hosted => src}/tracy.zig | 0 {src-self-hosted => src}/translate_c.zig | 0 {src-self-hosted => src}/type.zig | 0 {src-self-hosted => src}/value.zig | 0 {src-self-hosted => src}/windows_sdk.zig | 0 {src-self-hosted => src}/zir.zig | 0 {src-self-hosted => src}/zir_sema.zig | 0 109 files changed, 29 insertions(+), 30 deletions(-) rename {src-self-hosted => src}/Cache.zig (100%) rename {src-self-hosted => src}/Compilation.zig (100%) rename {src-self-hosted => src}/DepTokenizer.zig (100%) rename {src-self-hosted => src}/Module.zig (100%) rename {src-self-hosted => src}/Package.zig (100%) rename {src-self-hosted => src}/TypedValue.zig (100%) rename {src-self-hosted => src}/astgen.zig (100%) rename {src-self-hosted => src}/clang.zig (100%) rename {src-self-hosted => src}/clang_options.zig (100%) rename {src-self-hosted => src}/clang_options_data.zig (100%) rename {src-self-hosted => src}/codegen.zig (100%) rename {src-self-hosted => src}/codegen/arm.zig (100%) rename {src-self-hosted => src}/codegen/c.zig (100%) rename {src-self-hosted => src}/codegen/llvm.zig (100%) rename {src-self-hosted => src}/codegen/riscv64.zig (100%) rename {src-self-hosted => src}/codegen/spu-mk2.zig (100%) rename {src-self-hosted => src}/codegen/spu-mk2/interpreter.zig (100%) rename {src-self-hosted => src}/codegen/wasm.zig (100%) rename {src-self-hosted => src}/codegen/x86.zig (100%) rename {src-self-hosted => src}/codegen/x86_64.zig (100%) rename {src-self-hosted => src}/glibc.zig (100%) rename {src-self-hosted => src}/introspect.zig (100%) rename {src-self-hosted => src}/ir.zig (100%) rename {src-self-hosted => src}/libc_installation.zig (100%) rename {src-self-hosted => src}/libcxx.zig (100%) rename {src-self-hosted => src}/libunwind.zig (100%) rename {src-self-hosted => src}/link.zig (100%) rename {src-self-hosted => src}/link/C.zig (100%) rename {src-self-hosted => src}/link/Coff.zig (100%) rename {src-self-hosted => src}/link/Elf.zig (100%) rename {src-self-hosted => src}/link/MachO.zig (100%) rename {src-self-hosted => src}/link/Wasm.zig (100%) rename {src-self-hosted => src}/link/cbe.h (100%) rename {src-self-hosted => src}/link/msdos-stub.bin (100%) rename {src-self-hosted => src}/liveness.zig (100%) rename {src-self-hosted => src}/llvm.zig (100%) rename {src-self-hosted => src}/main.zig (100%) rename {src-self-hosted => src}/musl.zig (100%) rename {src-self-hosted => src}/print_env.zig (100%) rename {src-self-hosted => src}/print_targets.zig (100%) rename {src-self-hosted => src}/stage1.zig (100%) rename src/{ => stage1}/all_types.hpp (100%) rename src/{ => stage1}/analyze.cpp (100%) rename src/{ => stage1}/analyze.hpp (100%) rename src/{ => stage1}/ast_render.cpp (100%) rename src/{ => stage1}/ast_render.hpp (100%) rename src/{ => stage1}/bigfloat.cpp (100%) rename src/{ => stage1}/bigfloat.hpp (100%) rename src/{ => stage1}/bigint.cpp (100%) rename src/{ => stage1}/bigint.hpp (100%) rename src/{ => stage1}/buffer.cpp (100%) rename src/{ => stage1}/buffer.hpp (100%) rename src/{ => stage1}/codegen.cpp (100%) rename src/{ => stage1}/codegen.hpp (100%) rename src/{ => stage1}/config.h.in (100%) rename src/{ => stage1}/dump_analysis.cpp (100%) rename src/{ => stage1}/dump_analysis.hpp (100%) rename src/{ => stage1}/empty.cpp (100%) rename src/{ => stage1}/errmsg.cpp (100%) rename src/{ => stage1}/errmsg.hpp (100%) rename src/{ => stage1}/error.cpp (100%) rename src/{ => stage1}/error.hpp (100%) rename src/{ => stage1}/hash_map.hpp (100%) rename src/{ => stage1}/heap.cpp (100%) rename src/{ => stage1}/heap.hpp (100%) rename src/{ => stage1}/ir.cpp (100%) rename src/{ => stage1}/ir.hpp (100%) rename src/{ => stage1}/ir_print.cpp (100%) rename src/{ => stage1}/ir_print.hpp (100%) rename src/{ => stage1}/list.hpp (100%) rename src/{ => stage1}/mem.cpp (100%) rename src/{ => stage1}/mem.hpp (100%) rename src/{ => stage1}/mem_hash_map.hpp (100%) rename src/{ => stage1}/mem_list.hpp (100%) rename src/{ => stage1}/mem_type_info.hpp (100%) rename src/{ => stage1}/os.cpp (100%) rename src/{ => stage1}/os.hpp (100%) rename src/{ => stage1}/parse_f128.c (100%) rename src/{ => stage1}/parse_f128.h (100%) rename src/{ => stage1}/parser.cpp (100%) rename src/{ => stage1}/parser.hpp (100%) rename src/{ => stage1}/range_set.cpp (100%) rename src/{ => stage1}/range_set.hpp (100%) rename src/{ => stage1}/softfloat.hpp (100%) rename src/{ => stage1}/softfloat_ext.cpp (100%) rename src/{ => stage1}/softfloat_ext.hpp (100%) rename src/{ => stage1}/stage1.cpp (100%) rename src/{ => stage1}/stage1.h (100%) rename src/{ => stage1}/stage2.h (100%) rename src/{ => stage1}/target.cpp (100%) rename src/{ => stage1}/target.hpp (100%) rename src/{ => stage1}/tokenizer.cpp (100%) rename src/{ => stage1}/tokenizer.hpp (100%) rename src/{ => stage1}/util.cpp (100%) rename src/{ => stage1}/util.hpp (100%) rename src/{ => stage1}/util_base.hpp (100%) rename src/{ => stage1}/zig0.cpp (100%) rename {src-self-hosted => src}/target.zig (100%) rename {src-self-hosted => src}/test.zig (100%) rename {src-self-hosted => src}/tracy.zig (100%) rename {src-self-hosted => src}/translate_c.zig (100%) rename {src-self-hosted => src}/type.zig (100%) rename {src-self-hosted => src}/value.zig (100%) rename {src-self-hosted => src}/windows_sdk.zig (100%) rename {src-self-hosted => src}/zir.zig (100%) rename {src-self-hosted => src}/zir_sema.zig (100%) diff --git a/BRANCH_TODO b/BRANCH_TODO index ae725e118a..56868a8f8f 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -37,8 +37,6 @@ * implement proper compile errors for failing to build glibc crt files and shared libs * implement -fno-emit-bin * improve the stage2 tests to support testing with LLVM extensions enabled - * rename src/ to src/stage1/ - * rename src-self-hosted/ to src/ * implement emit-h in stage2 * multi-thread building C objects * implement serialization/deserialization of incremental compilation metadata diff --git a/CMakeLists.txt b/CMakeLists.txt index 6389ec7dbe..e3035213a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -257,34 +257,34 @@ find_package(Threads) # This is our shim which will be replaced by stage1.zig. set(ZIG0_SOURCES - "${CMAKE_SOURCE_DIR}/src/zig0.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/zig0.cpp" ) set(ZIG_SOURCES - "${CMAKE_SOURCE_DIR}/src/analyze.cpp" - "${CMAKE_SOURCE_DIR}/src/ast_render.cpp" - "${CMAKE_SOURCE_DIR}/src/bigfloat.cpp" - "${CMAKE_SOURCE_DIR}/src/bigint.cpp" - "${CMAKE_SOURCE_DIR}/src/buffer.cpp" - "${CMAKE_SOURCE_DIR}/src/codegen.cpp" - "${CMAKE_SOURCE_DIR}/src/dump_analysis.cpp" - "${CMAKE_SOURCE_DIR}/src/errmsg.cpp" - "${CMAKE_SOURCE_DIR}/src/error.cpp" - "${CMAKE_SOURCE_DIR}/src/heap.cpp" - "${CMAKE_SOURCE_DIR}/src/ir.cpp" - "${CMAKE_SOURCE_DIR}/src/ir_print.cpp" - "${CMAKE_SOURCE_DIR}/src/mem.cpp" - "${CMAKE_SOURCE_DIR}/src/os.cpp" - "${CMAKE_SOURCE_DIR}/src/parser.cpp" - "${CMAKE_SOURCE_DIR}/src/range_set.cpp" - "${CMAKE_SOURCE_DIR}/src/stage1.cpp" - "${CMAKE_SOURCE_DIR}/src/target.cpp" - "${CMAKE_SOURCE_DIR}/src/tokenizer.cpp" - "${CMAKE_SOURCE_DIR}/src/util.cpp" - "${CMAKE_SOURCE_DIR}/src/softfloat_ext.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/analyze.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/ast_render.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/bigfloat.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/bigint.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/buffer.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/codegen.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/dump_analysis.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/errmsg.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/error.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/heap.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/ir.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/ir_print.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/mem.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/os.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/parser.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/range_set.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/stage1.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/target.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/tokenizer.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/util.cpp" + "${CMAKE_SOURCE_DIR}/src/stage1/softfloat_ext.cpp" ) set(OPTIMIZED_C_SOURCES - "${CMAKE_SOURCE_DIR}/src/parse_f128.c" + "${CMAKE_SOURCE_DIR}/src/stage1/parse_f128.c" ) set(ZIG_CPP_SOURCES # These are planned to stay even when we are self-hosted. @@ -314,7 +314,7 @@ set(ZIG_STD_DEST "${ZIG_LIB_DIR}/std") set(ZIG_CONFIG_H_OUT "${CMAKE_BINARY_DIR}/config.h") set(ZIG_CONFIG_ZIG_OUT "${CMAKE_BINARY_DIR}/config.zig") configure_file ( - "${CMAKE_SOURCE_DIR}/src/config.h.in" + "${CMAKE_SOURCE_DIR}/src/stage1/config.h.in" "${ZIG_CONFIG_H_OUT}" ) configure_file ( @@ -326,6 +326,7 @@ include_directories( ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} "${CMAKE_SOURCE_DIR}/src" + "${CMAKE_SOURCE_DIR}/src/stage1" ) # These have to go before the -Wno- flags @@ -444,7 +445,7 @@ else() endif() set(BUILD_ZIG1_ARGS - "src-self-hosted/stage1.zig" + "src/stage1.zig" -target "${ZIG_TARGET_TRIPLE}" "-mcpu=${ZIG_TARGET_MCPU}" --name zig1 @@ -480,7 +481,7 @@ else() endif() # cmake won't let us configure an executable without C sources. -add_executable(zig "${CMAKE_SOURCE_DIR}/src/empty.cpp") +add_executable(zig "${CMAKE_SOURCE_DIR}/src/stage1/empty.cpp") set_target_properties(zig PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} diff --git a/build.zig b/build.zig index 9e5f2425c2..331af1204c 100644 --- a/build.zig +++ b/build.zig @@ -38,7 +38,7 @@ pub fn build(b: *Builder) !void { const test_step = b.step("test", "Run all the tests"); - var test_stage2 = b.addTest("src-self-hosted/test.zig"); + var test_stage2 = b.addTest("src/test.zig"); test_stage2.setBuildMode(mode); test_stage2.addPackagePath("stage2_tests", "test/stage2/test.zig"); @@ -76,7 +76,7 @@ pub fn build(b: *Builder) !void { const tracy = b.option([]const u8, "tracy", "Enable Tracy integration. Supply path to Tracy source"); const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse enable_llvm; - var exe = b.addExecutable("zig", "src-self-hosted/main.zig"); + var exe = b.addExecutable("zig", "src/main.zig"); exe.install(); exe.setBuildMode(mode); exe.setTarget(target); diff --git a/src-self-hosted/Cache.zig b/src/Cache.zig similarity index 100% rename from src-self-hosted/Cache.zig rename to src/Cache.zig diff --git a/src-self-hosted/Compilation.zig b/src/Compilation.zig similarity index 100% rename from src-self-hosted/Compilation.zig rename to src/Compilation.zig diff --git a/src-self-hosted/DepTokenizer.zig b/src/DepTokenizer.zig similarity index 100% rename from src-self-hosted/DepTokenizer.zig rename to src/DepTokenizer.zig diff --git a/src-self-hosted/Module.zig b/src/Module.zig similarity index 100% rename from src-self-hosted/Module.zig rename to src/Module.zig diff --git a/src-self-hosted/Package.zig b/src/Package.zig similarity index 100% rename from src-self-hosted/Package.zig rename to src/Package.zig diff --git a/src-self-hosted/TypedValue.zig b/src/TypedValue.zig similarity index 100% rename from src-self-hosted/TypedValue.zig rename to src/TypedValue.zig diff --git a/src-self-hosted/astgen.zig b/src/astgen.zig similarity index 100% rename from src-self-hosted/astgen.zig rename to src/astgen.zig diff --git a/src-self-hosted/clang.zig b/src/clang.zig similarity index 100% rename from src-self-hosted/clang.zig rename to src/clang.zig diff --git a/src-self-hosted/clang_options.zig b/src/clang_options.zig similarity index 100% rename from src-self-hosted/clang_options.zig rename to src/clang_options.zig diff --git a/src-self-hosted/clang_options_data.zig b/src/clang_options_data.zig similarity index 100% rename from src-self-hosted/clang_options_data.zig rename to src/clang_options_data.zig diff --git a/src-self-hosted/codegen.zig b/src/codegen.zig similarity index 100% rename from src-self-hosted/codegen.zig rename to src/codegen.zig diff --git a/src-self-hosted/codegen/arm.zig b/src/codegen/arm.zig similarity index 100% rename from src-self-hosted/codegen/arm.zig rename to src/codegen/arm.zig diff --git a/src-self-hosted/codegen/c.zig b/src/codegen/c.zig similarity index 100% rename from src-self-hosted/codegen/c.zig rename to src/codegen/c.zig diff --git a/src-self-hosted/codegen/llvm.zig b/src/codegen/llvm.zig similarity index 100% rename from src-self-hosted/codegen/llvm.zig rename to src/codegen/llvm.zig diff --git a/src-self-hosted/codegen/riscv64.zig b/src/codegen/riscv64.zig similarity index 100% rename from src-self-hosted/codegen/riscv64.zig rename to src/codegen/riscv64.zig diff --git a/src-self-hosted/codegen/spu-mk2.zig b/src/codegen/spu-mk2.zig similarity index 100% rename from src-self-hosted/codegen/spu-mk2.zig rename to src/codegen/spu-mk2.zig diff --git a/src-self-hosted/codegen/spu-mk2/interpreter.zig b/src/codegen/spu-mk2/interpreter.zig similarity index 100% rename from src-self-hosted/codegen/spu-mk2/interpreter.zig rename to src/codegen/spu-mk2/interpreter.zig diff --git a/src-self-hosted/codegen/wasm.zig b/src/codegen/wasm.zig similarity index 100% rename from src-self-hosted/codegen/wasm.zig rename to src/codegen/wasm.zig diff --git a/src-self-hosted/codegen/x86.zig b/src/codegen/x86.zig similarity index 100% rename from src-self-hosted/codegen/x86.zig rename to src/codegen/x86.zig diff --git a/src-self-hosted/codegen/x86_64.zig b/src/codegen/x86_64.zig similarity index 100% rename from src-self-hosted/codegen/x86_64.zig rename to src/codegen/x86_64.zig diff --git a/src-self-hosted/glibc.zig b/src/glibc.zig similarity index 100% rename from src-self-hosted/glibc.zig rename to src/glibc.zig diff --git a/src-self-hosted/introspect.zig b/src/introspect.zig similarity index 100% rename from src-self-hosted/introspect.zig rename to src/introspect.zig diff --git a/src-self-hosted/ir.zig b/src/ir.zig similarity index 100% rename from src-self-hosted/ir.zig rename to src/ir.zig diff --git a/src-self-hosted/libc_installation.zig b/src/libc_installation.zig similarity index 100% rename from src-self-hosted/libc_installation.zig rename to src/libc_installation.zig diff --git a/src-self-hosted/libcxx.zig b/src/libcxx.zig similarity index 100% rename from src-self-hosted/libcxx.zig rename to src/libcxx.zig diff --git a/src-self-hosted/libunwind.zig b/src/libunwind.zig similarity index 100% rename from src-self-hosted/libunwind.zig rename to src/libunwind.zig diff --git a/src-self-hosted/link.zig b/src/link.zig similarity index 100% rename from src-self-hosted/link.zig rename to src/link.zig diff --git a/src-self-hosted/link/C.zig b/src/link/C.zig similarity index 100% rename from src-self-hosted/link/C.zig rename to src/link/C.zig diff --git a/src-self-hosted/link/Coff.zig b/src/link/Coff.zig similarity index 100% rename from src-self-hosted/link/Coff.zig rename to src/link/Coff.zig diff --git a/src-self-hosted/link/Elf.zig b/src/link/Elf.zig similarity index 100% rename from src-self-hosted/link/Elf.zig rename to src/link/Elf.zig diff --git a/src-self-hosted/link/MachO.zig b/src/link/MachO.zig similarity index 100% rename from src-self-hosted/link/MachO.zig rename to src/link/MachO.zig diff --git a/src-self-hosted/link/Wasm.zig b/src/link/Wasm.zig similarity index 100% rename from src-self-hosted/link/Wasm.zig rename to src/link/Wasm.zig diff --git a/src-self-hosted/link/cbe.h b/src/link/cbe.h similarity index 100% rename from src-self-hosted/link/cbe.h rename to src/link/cbe.h diff --git a/src-self-hosted/link/msdos-stub.bin b/src/link/msdos-stub.bin similarity index 100% rename from src-self-hosted/link/msdos-stub.bin rename to src/link/msdos-stub.bin diff --git a/src-self-hosted/liveness.zig b/src/liveness.zig similarity index 100% rename from src-self-hosted/liveness.zig rename to src/liveness.zig diff --git a/src-self-hosted/llvm.zig b/src/llvm.zig similarity index 100% rename from src-self-hosted/llvm.zig rename to src/llvm.zig diff --git a/src-self-hosted/main.zig b/src/main.zig similarity index 100% rename from src-self-hosted/main.zig rename to src/main.zig diff --git a/src-self-hosted/musl.zig b/src/musl.zig similarity index 100% rename from src-self-hosted/musl.zig rename to src/musl.zig diff --git a/src-self-hosted/print_env.zig b/src/print_env.zig similarity index 100% rename from src-self-hosted/print_env.zig rename to src/print_env.zig diff --git a/src-self-hosted/print_targets.zig b/src/print_targets.zig similarity index 100% rename from src-self-hosted/print_targets.zig rename to src/print_targets.zig diff --git a/src-self-hosted/stage1.zig b/src/stage1.zig similarity index 100% rename from src-self-hosted/stage1.zig rename to src/stage1.zig diff --git a/src/all_types.hpp b/src/stage1/all_types.hpp similarity index 100% rename from src/all_types.hpp rename to src/stage1/all_types.hpp diff --git a/src/analyze.cpp b/src/stage1/analyze.cpp similarity index 100% rename from src/analyze.cpp rename to src/stage1/analyze.cpp diff --git a/src/analyze.hpp b/src/stage1/analyze.hpp similarity index 100% rename from src/analyze.hpp rename to src/stage1/analyze.hpp diff --git a/src/ast_render.cpp b/src/stage1/ast_render.cpp similarity index 100% rename from src/ast_render.cpp rename to src/stage1/ast_render.cpp diff --git a/src/ast_render.hpp b/src/stage1/ast_render.hpp similarity index 100% rename from src/ast_render.hpp rename to src/stage1/ast_render.hpp diff --git a/src/bigfloat.cpp b/src/stage1/bigfloat.cpp similarity index 100% rename from src/bigfloat.cpp rename to src/stage1/bigfloat.cpp diff --git a/src/bigfloat.hpp b/src/stage1/bigfloat.hpp similarity index 100% rename from src/bigfloat.hpp rename to src/stage1/bigfloat.hpp diff --git a/src/bigint.cpp b/src/stage1/bigint.cpp similarity index 100% rename from src/bigint.cpp rename to src/stage1/bigint.cpp diff --git a/src/bigint.hpp b/src/stage1/bigint.hpp similarity index 100% rename from src/bigint.hpp rename to src/stage1/bigint.hpp diff --git a/src/buffer.cpp b/src/stage1/buffer.cpp similarity index 100% rename from src/buffer.cpp rename to src/stage1/buffer.cpp diff --git a/src/buffer.hpp b/src/stage1/buffer.hpp similarity index 100% rename from src/buffer.hpp rename to src/stage1/buffer.hpp diff --git a/src/codegen.cpp b/src/stage1/codegen.cpp similarity index 100% rename from src/codegen.cpp rename to src/stage1/codegen.cpp diff --git a/src/codegen.hpp b/src/stage1/codegen.hpp similarity index 100% rename from src/codegen.hpp rename to src/stage1/codegen.hpp diff --git a/src/config.h.in b/src/stage1/config.h.in similarity index 100% rename from src/config.h.in rename to src/stage1/config.h.in diff --git a/src/dump_analysis.cpp b/src/stage1/dump_analysis.cpp similarity index 100% rename from src/dump_analysis.cpp rename to src/stage1/dump_analysis.cpp diff --git a/src/dump_analysis.hpp b/src/stage1/dump_analysis.hpp similarity index 100% rename from src/dump_analysis.hpp rename to src/stage1/dump_analysis.hpp diff --git a/src/empty.cpp b/src/stage1/empty.cpp similarity index 100% rename from src/empty.cpp rename to src/stage1/empty.cpp diff --git a/src/errmsg.cpp b/src/stage1/errmsg.cpp similarity index 100% rename from src/errmsg.cpp rename to src/stage1/errmsg.cpp diff --git a/src/errmsg.hpp b/src/stage1/errmsg.hpp similarity index 100% rename from src/errmsg.hpp rename to src/stage1/errmsg.hpp diff --git a/src/error.cpp b/src/stage1/error.cpp similarity index 100% rename from src/error.cpp rename to src/stage1/error.cpp diff --git a/src/error.hpp b/src/stage1/error.hpp similarity index 100% rename from src/error.hpp rename to src/stage1/error.hpp diff --git a/src/hash_map.hpp b/src/stage1/hash_map.hpp similarity index 100% rename from src/hash_map.hpp rename to src/stage1/hash_map.hpp diff --git a/src/heap.cpp b/src/stage1/heap.cpp similarity index 100% rename from src/heap.cpp rename to src/stage1/heap.cpp diff --git a/src/heap.hpp b/src/stage1/heap.hpp similarity index 100% rename from src/heap.hpp rename to src/stage1/heap.hpp diff --git a/src/ir.cpp b/src/stage1/ir.cpp similarity index 100% rename from src/ir.cpp rename to src/stage1/ir.cpp diff --git a/src/ir.hpp b/src/stage1/ir.hpp similarity index 100% rename from src/ir.hpp rename to src/stage1/ir.hpp diff --git a/src/ir_print.cpp b/src/stage1/ir_print.cpp similarity index 100% rename from src/ir_print.cpp rename to src/stage1/ir_print.cpp diff --git a/src/ir_print.hpp b/src/stage1/ir_print.hpp similarity index 100% rename from src/ir_print.hpp rename to src/stage1/ir_print.hpp diff --git a/src/list.hpp b/src/stage1/list.hpp similarity index 100% rename from src/list.hpp rename to src/stage1/list.hpp diff --git a/src/mem.cpp b/src/stage1/mem.cpp similarity index 100% rename from src/mem.cpp rename to src/stage1/mem.cpp diff --git a/src/mem.hpp b/src/stage1/mem.hpp similarity index 100% rename from src/mem.hpp rename to src/stage1/mem.hpp diff --git a/src/mem_hash_map.hpp b/src/stage1/mem_hash_map.hpp similarity index 100% rename from src/mem_hash_map.hpp rename to src/stage1/mem_hash_map.hpp diff --git a/src/mem_list.hpp b/src/stage1/mem_list.hpp similarity index 100% rename from src/mem_list.hpp rename to src/stage1/mem_list.hpp diff --git a/src/mem_type_info.hpp b/src/stage1/mem_type_info.hpp similarity index 100% rename from src/mem_type_info.hpp rename to src/stage1/mem_type_info.hpp diff --git a/src/os.cpp b/src/stage1/os.cpp similarity index 100% rename from src/os.cpp rename to src/stage1/os.cpp diff --git a/src/os.hpp b/src/stage1/os.hpp similarity index 100% rename from src/os.hpp rename to src/stage1/os.hpp diff --git a/src/parse_f128.c b/src/stage1/parse_f128.c similarity index 100% rename from src/parse_f128.c rename to src/stage1/parse_f128.c diff --git a/src/parse_f128.h b/src/stage1/parse_f128.h similarity index 100% rename from src/parse_f128.h rename to src/stage1/parse_f128.h diff --git a/src/parser.cpp b/src/stage1/parser.cpp similarity index 100% rename from src/parser.cpp rename to src/stage1/parser.cpp diff --git a/src/parser.hpp b/src/stage1/parser.hpp similarity index 100% rename from src/parser.hpp rename to src/stage1/parser.hpp diff --git a/src/range_set.cpp b/src/stage1/range_set.cpp similarity index 100% rename from src/range_set.cpp rename to src/stage1/range_set.cpp diff --git a/src/range_set.hpp b/src/stage1/range_set.hpp similarity index 100% rename from src/range_set.hpp rename to src/stage1/range_set.hpp diff --git a/src/softfloat.hpp b/src/stage1/softfloat.hpp similarity index 100% rename from src/softfloat.hpp rename to src/stage1/softfloat.hpp diff --git a/src/softfloat_ext.cpp b/src/stage1/softfloat_ext.cpp similarity index 100% rename from src/softfloat_ext.cpp rename to src/stage1/softfloat_ext.cpp diff --git a/src/softfloat_ext.hpp b/src/stage1/softfloat_ext.hpp similarity index 100% rename from src/softfloat_ext.hpp rename to src/stage1/softfloat_ext.hpp diff --git a/src/stage1.cpp b/src/stage1/stage1.cpp similarity index 100% rename from src/stage1.cpp rename to src/stage1/stage1.cpp diff --git a/src/stage1.h b/src/stage1/stage1.h similarity index 100% rename from src/stage1.h rename to src/stage1/stage1.h diff --git a/src/stage2.h b/src/stage1/stage2.h similarity index 100% rename from src/stage2.h rename to src/stage1/stage2.h diff --git a/src/target.cpp b/src/stage1/target.cpp similarity index 100% rename from src/target.cpp rename to src/stage1/target.cpp diff --git a/src/target.hpp b/src/stage1/target.hpp similarity index 100% rename from src/target.hpp rename to src/stage1/target.hpp diff --git a/src/tokenizer.cpp b/src/stage1/tokenizer.cpp similarity index 100% rename from src/tokenizer.cpp rename to src/stage1/tokenizer.cpp diff --git a/src/tokenizer.hpp b/src/stage1/tokenizer.hpp similarity index 100% rename from src/tokenizer.hpp rename to src/stage1/tokenizer.hpp diff --git a/src/util.cpp b/src/stage1/util.cpp similarity index 100% rename from src/util.cpp rename to src/stage1/util.cpp diff --git a/src/util.hpp b/src/stage1/util.hpp similarity index 100% rename from src/util.hpp rename to src/stage1/util.hpp diff --git a/src/util_base.hpp b/src/stage1/util_base.hpp similarity index 100% rename from src/util_base.hpp rename to src/stage1/util_base.hpp diff --git a/src/zig0.cpp b/src/stage1/zig0.cpp similarity index 100% rename from src/zig0.cpp rename to src/stage1/zig0.cpp diff --git a/src-self-hosted/target.zig b/src/target.zig similarity index 100% rename from src-self-hosted/target.zig rename to src/target.zig diff --git a/src-self-hosted/test.zig b/src/test.zig similarity index 100% rename from src-self-hosted/test.zig rename to src/test.zig diff --git a/src-self-hosted/tracy.zig b/src/tracy.zig similarity index 100% rename from src-self-hosted/tracy.zig rename to src/tracy.zig diff --git a/src-self-hosted/translate_c.zig b/src/translate_c.zig similarity index 100% rename from src-self-hosted/translate_c.zig rename to src/translate_c.zig diff --git a/src-self-hosted/type.zig b/src/type.zig similarity index 100% rename from src-self-hosted/type.zig rename to src/type.zig diff --git a/src-self-hosted/value.zig b/src/value.zig similarity index 100% rename from src-self-hosted/value.zig rename to src/value.zig diff --git a/src-self-hosted/windows_sdk.zig b/src/windows_sdk.zig similarity index 100% rename from src-self-hosted/windows_sdk.zig rename to src/windows_sdk.zig diff --git a/src-self-hosted/zir.zig b/src/zir.zig similarity index 100% rename from src-self-hosted/zir.zig rename to src/zir.zig diff --git a/src-self-hosted/zir_sema.zig b/src/zir_sema.zig similarity index 100% rename from src-self-hosted/zir_sema.zig rename to src/zir_sema.zig From ead50ea6657d31632f5661f788b035a66c344d13 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Sep 2020 21:01:04 -0700 Subject: [PATCH 085/210] stage2: implement `zig run` and `zig test` --- BRANCH_TODO | 13 +- src/Compilation.zig | 45 +- src/main.zig | 1145 +++++++++++++++++++++++-------------------- 3 files changed, 671 insertions(+), 532 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 56868a8f8f..f30f67fc69 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,6 +1,9 @@ * build & link against libcxx and libcxxabi - * `zig test` * `zig build` + * repair @cImport + * make sure zig cc works + - using it as a preprocessor (-E) + - try building some software * `-ftime-report` * -fstack-report print stack size diagnostics\n" * -fdump-analysis write analysis.json file with type information\n" @@ -11,17 +14,14 @@ * -femit-llvm-ir produce a .ll file with LLVM IR\n" * -fno-emit-llvm-ir (default) do not produce a .ll file with LLVM IR\n" * --cache-dir [path] override the local cache directory\n" - * make sure zig cc works - - using it as a preprocessor (-E) - - try building some software * implement proper parsing of LLD stderr/stdout and exposing compile errors * implement proper parsing of clang stderr/stdout and exposing compile errors * support rpaths in ELF linker code - * repair @cImport * add CLI support for a way to pass extra flags to c source files * musl * mingw-w64 * use global zig-cache dir for crt files + * use global zig-cache dir for `zig run` executables but not `zig test` * MachO LLD linking * COFF LLD linking * WASM LLD linking @@ -30,9 +30,9 @@ * audit the CLI options for stage2 * `zig init-lib` * `zig init-exe` - * `zig run` * restore error messages for stage2_add_link_lib * audit the base cache hash + * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process. * implement proper compile errors for failing to build glibc crt files and shared libs * implement -fno-emit-bin @@ -66,3 +66,4 @@ * some kind of "zig identifier escape" function rather than unconditionally using @"" syntax in builtin.zig * rename std.builtin.Mode to std.builtin.OptimizeMode + * implement `zig run` and `zig test` when combined with `--watch` diff --git a/src/Compilation.zig b/src/Compilation.zig index 29c6dc36cf..725e212b96 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -97,6 +97,10 @@ owned_link_dir: ?std.fs.Dir, /// Don't use this for anything other than stage1 compatibility. color: @import("main.zig").Color = .Auto, +test_filter: ?[]const u8, +test_name_prefix: ?[]const u8, +test_evented_io: bool, + pub const InnerError = Module.InnerError; pub const CRTFile = struct { @@ -327,6 +331,9 @@ pub const InitOptions = struct { machine_code_model: std.builtin.CodeModel = .default, /// This is for stage1 and should be deleted upon completion of self-hosting. color: @import("main.zig").Color = .Auto, + test_filter: ?[]const u8 = null, + test_name_prefix: ?[]const u8 = null, + test_evented_io: bool = false, }; pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { @@ -554,6 +561,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { hash.add(single_threaded); hash.add(options.target.os.getVersionRange()); hash.add(dll_export_fns); + hash.add(options.is_test); const digest = hash.final(); const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); @@ -728,6 +736,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .is_test = options.is_test, .color = options.color, .time_report = options.time_report, + .test_filter = options.test_filter, + .test_name_prefix = options.test_name_prefix, + .test_evented_io = options.test_evented_io, }; break :comp comp; }; @@ -1996,6 +2007,25 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8 comp.bin_file.options.strip, @tagName(comp.bin_file.options.machine_code_model), }); + + if (comp.is_test) { + try buffer.appendSlice( + \\pub var test_functions: []TestFn = undefined; // overwritten later + \\ + ); + if (comp.test_evented_io) { + try buffer.appendSlice( + \\pub const test_io_mode = .evented; + \\ + ); + } else { + try buffer.appendSlice( + \\pub const test_io_mode = .blocking; + \\ + ); + } + } + return buffer.toOwnedSlice(); } @@ -2129,6 +2159,7 @@ fn updateStage1Module(comp: *Compilation) !void { ch.hash.add(target.os.getVersionRange()); ch.hash.add(comp.bin_file.options.dll_export_fns); ch.hash.add(comp.bin_file.options.function_sections); + ch.hash.add(comp.is_test); if (try ch.hit()) { const digest = ch.final(); @@ -2155,7 +2186,7 @@ fn updateStage1Module(comp: *Compilation) !void { .llvm_cpu_features = comp.bin_file.options.llvm_cpu_features.?, }; var progress: std.Progress = .{}; - var main_progress_node = try progress.start("", 100); + var main_progress_node = try progress.start("", null); defer main_progress_node.end(); if (comp.color == .Off) progress.terminal = null; @@ -2184,6 +2215,8 @@ fn updateStage1Module(comp: *Compilation) !void { .parent = null, }; const output_dir = comp.bin_file.options.directory.path orelse "."; + const test_filter = comp.test_filter orelse ""[0..0]; + const test_name_prefix = comp.test_name_prefix orelse ""[0..0]; stage1_module.* = .{ .root_name_ptr = comp.bin_file.options.root_name.ptr, .root_name_len = comp.bin_file.options.root_name.len, @@ -2191,10 +2224,10 @@ fn updateStage1Module(comp: *Compilation) !void { .output_dir_len = output_dir.len, .builtin_zig_path_ptr = builtin_zig_path.ptr, .builtin_zig_path_len = builtin_zig_path.len, - .test_filter_ptr = "", - .test_filter_len = 0, - .test_name_prefix_ptr = "", - .test_name_prefix_len = 0, + .test_filter_ptr = test_filter.ptr, + .test_filter_len = test_filter.len, + .test_name_prefix_ptr = test_name_prefix.ptr, + .test_name_prefix_len = test_name_prefix.len, .userdata = @ptrToInt(comp), .root_pkg = stage1_pkg, .code_model = @enumToInt(comp.bin_file.options.machine_code_model), @@ -2217,7 +2250,7 @@ fn updateStage1Module(comp: *Compilation) !void { .emit_bin = true, .emit_asm = false, .emit_llvm_ir = false, - .test_is_evented = false, + .test_is_evented = comp.test_evented_io, .verbose_tokenize = comp.verbose_tokenize, .verbose_ast = comp.verbose_ast, .verbose_ir = comp.verbose_ir, diff --git a/src/main.zig b/src/main.zig index 02ba72b51c..c35586e925 100644 --- a/src/main.zig +++ b/src/main.zig @@ -43,8 +43,10 @@ const usage = \\ env Print lib path, std path, compiler id and version \\ fmt Parse file and render in canonical zig format \\ libc Display native libc paths file or validate one + \\ run Create executable and run immediately \\ translate-c Convert C code to Zig code \\ targets List available compilation targets + \\ test Create and run a test build \\ version Print version number and exit \\ zen Print zen of zig and exit \\ @@ -120,6 +122,10 @@ pub fn mainArgs(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v return buildOutputType(gpa, arena, args, .{ .build = .Lib }); } else if (mem.eql(u8, cmd, "build-obj")) { return buildOutputType(gpa, arena, args, .{ .build = .Obj }); + } else if (mem.eql(u8, cmd, "test")) { + return buildOutputType(gpa, arena, args, .zig_test); + } else if (mem.eql(u8, cmd, "run")) { + return buildOutputType(gpa, arena, args, .run); } else if (mem.eql(u8, cmd, "cc")) { return buildOutputType(gpa, arena, args, .cc); } else if (mem.eql(u8, cmd, "c++")) { @@ -156,6 +162,8 @@ const usage_build_generic = \\Usage: zig build-exe [files] \\ zig build-lib [files] \\ zig build-obj [files] + \\ zig test [files] + \\ zig run [file] [-- [args]] \\ \\Supported file types: \\ .zig Zig source code @@ -233,6 +241,13 @@ const usage_build_generic = \\ -dynamic Force output to be dynamically linked \\ -static Force output to be statically linked \\ + \\Test Options: + \\ --test-filter [text] Skip tests that do not match filter + \\ --test-name-prefix [text] Add prefix to all tests + \\ --test-cmd [arg] Specify test execution command one arg at a time + \\ --test-cmd-bin Appends test binary path to test cmd args + \\ --test-evented-io Runs the test in evented I/O mode + \\ \\Debug Options (Zig Compiler Development): \\ -ftime-report Print timing diagnostics \\ --verbose-link Display linker invocations @@ -269,6 +284,8 @@ pub fn buildOutputType( cc, cpp, translate_c, + zig_test, + run, }, ) !void { var color: Color = .Auto; @@ -321,6 +338,7 @@ pub fn buildOutputType( var linker_bind_global_refs_locally: ?bool = null; var linker_z_nodelete = false; var linker_z_defs = false; + var test_evented_io = false; var stack_size_override: ?u64 = null; var use_llvm: ?bool = null; var use_lld: ?bool = null; @@ -328,6 +346,9 @@ pub fn buildOutputType( var link_eh_frame_hdr = false; var libc_paths_file: ?[]const u8 = null; var machine_code_model: std.builtin.CodeModel = .default; + var runtime_args_start: ?usize = null; + var test_filter: ?[]const u8 = null; + var test_name_prefix: ?[]const u8 = null; var system_libs = std.ArrayList([]const u8).init(gpa); defer system_libs.deinit(); @@ -356,545 +377,575 @@ pub fn buildOutputType( var frameworks = std.ArrayList([]const u8).init(gpa); defer frameworks.deinit(); - if (arg_mode == .build or arg_mode == .translate_c) { - output_mode = switch (arg_mode) { - .build => |m| m, - .translate_c => .Obj, - else => unreachable, - }; - switch (arg_mode) { - .build => switch (output_mode) { - .Exe => emit_h = .no, - .Obj, .Lib => emit_h = .yes_default_path, - }, - .translate_c => emit_h = .no, - else => unreachable, - } - const args = all_args[2..]; - var i: usize = 0; - while (i < args.len) : (i += 1) { - const arg = args[i]; - if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { - try io.getStdOut().writeAll(usage_build_generic); - process.exit(0); - } else if (mem.eql(u8, arg, "--color")) { - if (i + 1 >= args.len) { - fatal("expected [auto|on|off] after --color", .{}); - } - i += 1; - const next_arg = args[i]; - if (mem.eql(u8, next_arg, "auto")) { - color = .Auto; - } else if (mem.eql(u8, next_arg, "on")) { - color = .On; - } else if (mem.eql(u8, next_arg, "off")) { - color = .Off; + // null means replace with the test executable binary + var test_exec_args = std.ArrayList(?[]const u8).init(gpa); + defer test_exec_args.deinit(); + + switch (arg_mode) { + .build, .translate_c, .zig_test, .run => { + output_mode = switch (arg_mode) { + .build => |m| m, + .translate_c => .Obj, + .zig_test, .run => .Exe, + else => unreachable, + }; + switch (arg_mode) { + .build => switch (output_mode) { + .Exe => emit_h = .no, + .Obj, .Lib => emit_h = .yes_default_path, + }, + .translate_c, .zig_test, .run => emit_h = .no, + else => unreachable, + } + const args = all_args[2..]; + var i: usize = 0; + while (i < args.len) : (i += 1) { + const arg = args[i]; + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + try io.getStdOut().writeAll(usage_build_generic); + process.exit(0); + } else if (mem.eql(u8, arg, "--")) { + if (arg_mode == .run) { + runtime_args_start = i + 1; + } else { + fatal("unexpected end-of-parameter mark: --", .{}); + } + } else if (mem.eql(u8, arg, "--color")) { + if (i + 1 >= args.len) { + fatal("expected [auto|on|off] after --color", .{}); + } + i += 1; + const next_arg = args[i]; + if (mem.eql(u8, next_arg, "auto")) { + color = .Auto; + } else if (mem.eql(u8, next_arg, "on")) { + color = .On; + } else if (mem.eql(u8, next_arg, "off")) { + color = .Off; + } else { + fatal("expected [auto|on|off] after --color, found '{}'", .{next_arg}); + } + } else if (mem.eql(u8, arg, "--mode")) { + if (i + 1 >= args.len) { + fatal("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode", .{}); + } + i += 1; + const next_arg = args[i]; + if (mem.eql(u8, next_arg, "Debug")) { + build_mode = .Debug; + } else if (mem.eql(u8, next_arg, "ReleaseSafe")) { + build_mode = .ReleaseSafe; + } else if (mem.eql(u8, next_arg, "ReleaseFast")) { + build_mode = .ReleaseFast; + } else if (mem.eql(u8, next_arg, "ReleaseSmall")) { + build_mode = .ReleaseSmall; + } else { + fatal("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode, found '{}'", .{next_arg}); + } + } else if (mem.eql(u8, arg, "--stack")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + stack_size_override = std.fmt.parseInt(u64, args[i], 10) catch |err| { + fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); + }; + } else if (mem.eql(u8, arg, "--name")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + provided_name = args[i]; + } else if (mem.eql(u8, arg, "-rpath")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + try rpath_list.append(args[i]); + } else if (mem.eql(u8, arg, "--library-directory") or mem.eql(u8, arg, "-L")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + try lib_dirs.append(args[i]); + } else if (mem.eql(u8, arg, "-T")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + linker_script = args[i]; + } else if (mem.eql(u8, arg, "--version-script")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + version_script = args[i]; + } else if (mem.eql(u8, arg, "--library") or mem.eql(u8, arg, "-l")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + // We don't know whether this library is part of libc or libc++ until we resolve the target. + // So we simply append to the list for now. + i += 1; + try system_libs.append(args[i]); + } else if (mem.eql(u8, arg, "-D") or + mem.eql(u8, arg, "-isystem") or + mem.eql(u8, arg, "-I") or + mem.eql(u8, arg, "-dirafter")) + { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + try clang_argv.append(arg); + try clang_argv.append(args[i]); + } else if (mem.eql(u8, arg, "--version")) { + if (i + 1 >= args.len) { + fatal("expected parameter after --version", .{}); + } + i += 1; + version = std.builtin.Version.parse(args[i]) catch |err| { + fatal("unable to parse --version '{}': {}", .{ args[i], @errorName(err) }); + }; + have_version = true; + } else if (mem.eql(u8, arg, "-target")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + target_arch_os_abi = args[i]; + } else if (mem.eql(u8, arg, "-mcpu")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + target_mcpu = args[i]; + } else if (mem.eql(u8, arg, "-mcmodel")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + machine_code_model = parseCodeModel(args[i]); + } else if (mem.startsWith(u8, arg, "-ofmt=")) { + target_ofmt = arg["-ofmt=".len..]; + } else if (mem.startsWith(u8, arg, "-mcpu=")) { + target_mcpu = arg["-mcpu=".len..]; + } else if (mem.startsWith(u8, arg, "-mcmodel=")) { + machine_code_model = parseCodeModel(arg["-mcmodel=".len..]); + } else if (mem.eql(u8, arg, "--dynamic-linker")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + target_dynamic_linker = args[i]; + } else if (mem.eql(u8, arg, "--libc")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + libc_paths_file = args[i]; + } else if (mem.eql(u8, arg, "--test-filter")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + test_filter = args[i]; + } else if (mem.eql(u8, arg, "--test-name-prefix")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + test_name_prefix = args[i]; + } else if (mem.eql(u8, arg, "--test-cmd")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + try test_exec_args.append(args[i]); + } else if (mem.eql(u8, arg, "--test-cmd-bin")) { + try test_exec_args.append(null); + } else if (mem.eql(u8, arg, "--test-evented-io")) { + test_evented_io = true; + } else if (mem.eql(u8, arg, "--watch")) { + watch = true; + } else if (mem.eql(u8, arg, "-ftime-report")) { + time_report = true; + } else if (mem.eql(u8, arg, "-fPIC")) { + want_pic = true; + } else if (mem.eql(u8, arg, "-fno-PIC")) { + want_pic = false; + } else if (mem.eql(u8, arg, "-fstack-check")) { + want_stack_check = true; + } else if (mem.eql(u8, arg, "-fno-stack-check")) { + want_stack_check = false; + } else if (mem.eql(u8, arg, "-fsanitize-c")) { + want_sanitize_c = true; + } else if (mem.eql(u8, arg, "-fno-sanitize-c")) { + want_sanitize_c = false; + } else if (mem.eql(u8, arg, "-fvalgrind")) { + want_valgrind = true; + } else if (mem.eql(u8, arg, "-fno-valgrind")) { + want_valgrind = false; + } else if (mem.eql(u8, arg, "-fLLVM")) { + use_llvm = true; + } else if (mem.eql(u8, arg, "-fno-LLVM")) { + use_llvm = false; + } else if (mem.eql(u8, arg, "-fLLD")) { + use_lld = true; + } else if (mem.eql(u8, arg, "-fno-LLD")) { + use_lld = false; + } else if (mem.eql(u8, arg, "-fClang")) { + use_clang = true; + } else if (mem.eql(u8, arg, "-fno-Clang")) { + use_clang = false; + } else if (mem.eql(u8, arg, "-rdynamic")) { + rdynamic = true; + } else if (mem.eql(u8, arg, "-femit-bin")) { + emit_bin = .yes_default_path; + } else if (mem.startsWith(u8, arg, "-femit-bin=")) { + emit_bin = .{ .yes = arg["-femit-bin=".len..] }; + } else if (mem.eql(u8, arg, "-fno-emit-bin")) { + emit_bin = .no; + } else if (mem.eql(u8, arg, "-femit-zir")) { + emit_zir = .yes_default_path; + } else if (mem.startsWith(u8, arg, "-femit-zir=")) { + emit_zir = .{ .yes = arg["-femit-zir=".len..] }; + } else if (mem.eql(u8, arg, "-fno-emit-zir")) { + emit_zir = .no; + } else if (mem.eql(u8, arg, "-femit-h")) { + emit_h = .yes_default_path; + } else if (mem.startsWith(u8, arg, "-femit-h=")) { + emit_h = .{ .yes = arg["-femit-h=".len..] }; + } else if (mem.eql(u8, arg, "-fno-emit-h")) { + emit_h = .no; + } else if (mem.eql(u8, arg, "-dynamic")) { + link_mode = .Dynamic; + } else if (mem.eql(u8, arg, "-static")) { + link_mode = .Static; + } else if (mem.eql(u8, arg, "-fdll-export-fns")) { + dll_export_fns = true; + } else if (mem.eql(u8, arg, "-fno-dll-export-fns")) { + dll_export_fns = false; + } else if (mem.eql(u8, arg, "--show-builtin")) { + show_builtin = true; + } else if (mem.eql(u8, arg, "--strip")) { + strip = true; + } else if (mem.eql(u8, arg, "--single-threaded")) { + single_threaded = true; + } else if (mem.eql(u8, arg, "--eh-frame-hdr")) { + link_eh_frame_hdr = true; + } else if (mem.eql(u8, arg, "-Bsymbolic")) { + linker_bind_global_refs_locally = true; + } else if (mem.eql(u8, arg, "--verbose-link")) { + verbose_link = true; + } else if (mem.eql(u8, arg, "--verbose-cc")) { + verbose_cc = true; + } else if (mem.eql(u8, arg, "--verbose-tokenize")) { + verbose_tokenize = true; + } else if (mem.eql(u8, arg, "--verbose-ast")) { + verbose_ast = true; + } else if (mem.eql(u8, arg, "--verbose-ir")) { + verbose_ir = true; + } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { + verbose_llvm_ir = true; + } else if (mem.eql(u8, arg, "--verbose-cimport")) { + verbose_cimport = true; + } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { + verbose_llvm_cpu_features = true; + } else if (mem.startsWith(u8, arg, "-T")) { + linker_script = arg[2..]; + } else if (mem.startsWith(u8, arg, "-L")) { + try lib_dirs.append(arg[2..]); + } else if (mem.startsWith(u8, arg, "-l")) { + // We don't know whether this library is part of libc or libc++ until we resolve the target. + // So we simply append to the list for now. + try system_libs.append(arg[2..]); + } else if (mem.startsWith(u8, arg, "-D") or + mem.startsWith(u8, arg, "-I")) + { + try clang_argv.append(arg); } else { - fatal("expected [auto|on|off] after --color, found '{}'", .{next_arg}); - } - } else if (mem.eql(u8, arg, "--mode")) { - if (i + 1 >= args.len) { - fatal("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode", .{}); + fatal("unrecognized parameter: '{}'", .{arg}); } + } else switch (Compilation.classifyFileExt(arg)) { + .object, .static_library => { + try link_objects.append(arg); + }, + .assembly, .c, .cpp, .h, .ll, .bc => { + // TODO a way to pass extra flags on the CLI + try c_source_files.append(.{ .src_path = arg }); + }, + .shared_library => { + fatal("linking against dynamic libraries not yet supported", .{}); + }, + .zig, .zir => { + if (root_src_file) |other| { + fatal("found another zig file '{}' after root source file '{}'", .{ arg, other }); + } else { + root_src_file = arg; + } + }, + .unknown => { + fatal("unrecognized file extension of parameter '{}'", .{arg}); + }, + } + } + }, + .cc, .cpp => { + emit_h = .no; + strip = true; + ensure_libc_on_non_freestanding = true; + ensure_libcpp_on_non_freestanding = arg_mode == .cpp; + want_native_include_dirs = true; + + var c_arg = false; + var is_shared_lib = false; + var linker_args = std.ArrayList([]const u8).init(arena); + var it = ClangArgIterator.init(arena, all_args); + while (it.has_next) { + it.next() catch |err| { + fatal("unable to parse command line parameters: {}", .{@errorName(err)}); + }; + switch (it.zig_equivalent) { + .target => target_arch_os_abi = it.only_arg, // example: -target riscv64-linux-unknown + .o => { + // -o + emit_bin = .{ .yes = it.only_arg }; + enable_cache = true; + }, + .c => c_arg = true, // -c + .other => { + try clang_argv.appendSlice(it.other_args); + }, + .positional => { + const file_ext = Compilation.classifyFileExt(mem.spanZ(it.only_arg)); + switch (file_ext) { + .assembly, .c, .cpp, .ll, .bc, .h => try c_source_files.append(.{ .src_path = it.only_arg }), + .unknown, .shared_library, .object, .static_library => { + try link_objects.append(it.only_arg); + }, + .zig, .zir => { + if (root_src_file) |other| { + fatal("found another zig file '{}' after root source file '{}'", .{ it.only_arg, other }); + } else { + root_src_file = it.only_arg; + } + }, + } + }, + .l => { + // -l + // We don't know whether this library is part of libc or libc++ until we resolve the target. + // So we simply append to the list for now. + try system_libs.append(it.only_arg); + }, + .ignore => {}, + .driver_punt => { + // Never mind what we're doing, just pass the args directly. For example --help. + return punt_to_clang(arena, all_args); + }, + .pic => want_pic = true, + .no_pic => want_pic = false, + .nostdlib => ensure_libc_on_non_freestanding = false, + .nostdlib_cpp => ensure_libcpp_on_non_freestanding = false, + .shared => { + link_mode = .Dynamic; + is_shared_lib = true; + }, + .rdynamic => rdynamic = true, + .wl => { + var split_it = mem.split(it.only_arg, ","); + while (split_it.next()) |linker_arg| { + try linker_args.append(linker_arg); + } + }, + .pp_or_asm => { + // This handles both -E and -S. + only_pp_or_asm = true; + try clang_argv.appendSlice(it.other_args); + }, + .optimize => { + // Alright, what release mode do they want? + if (mem.eql(u8, it.only_arg, "Os")) { + build_mode = .ReleaseSmall; + } else if (mem.eql(u8, it.only_arg, "O2") or + mem.eql(u8, it.only_arg, "O3") or + mem.eql(u8, it.only_arg, "O4")) + { + build_mode = .ReleaseFast; + } else if (mem.eql(u8, it.only_arg, "Og") or + mem.eql(u8, it.only_arg, "O0")) + { + build_mode = .Debug; + } else { + try clang_argv.appendSlice(it.other_args); + } + }, + .debug => { + strip = false; + if (mem.eql(u8, it.only_arg, "-g")) { + // We handled with strip = false above. + } else { + try clang_argv.appendSlice(it.other_args); + } + }, + .sanitize => { + if (mem.eql(u8, it.only_arg, "undefined")) { + want_sanitize_c = true; + } else { + try clang_argv.appendSlice(it.other_args); + } + }, + .linker_script => linker_script = it.only_arg, + .verbose_cmds => { + verbose_cc = true; + verbose_link = true; + }, + .for_linker => try linker_args.append(it.only_arg), + .linker_input_z => { + try linker_args.append("-z"); + try linker_args.append(it.only_arg); + }, + .lib_dir => try lib_dirs.append(it.only_arg), + .mcpu => target_mcpu = it.only_arg, + .dep_file => { + disable_c_depfile = true; + try clang_argv.appendSlice(it.other_args); + }, + .framework_dir => try framework_dirs.append(it.only_arg), + .framework => try frameworks.append(it.only_arg), + .nostdlibinc => want_native_include_dirs = false, + } + } + // Parse linker args. + var i: usize = 0; + while (i < linker_args.items.len) : (i += 1) { + const arg = linker_args.items[i]; + if (mem.eql(u8, arg, "-soname")) { i += 1; - const next_arg = args[i]; - if (mem.eql(u8, next_arg, "Debug")) { - build_mode = .Debug; - } else if (mem.eql(u8, next_arg, "ReleaseSafe")) { - build_mode = .ReleaseSafe; - } else if (mem.eql(u8, next_arg, "ReleaseFast")) { - build_mode = .ReleaseFast; - } else if (mem.eql(u8, next_arg, "ReleaseSmall")) { - build_mode = .ReleaseSmall; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + const soname = linker_args.items[i]; + override_soname = soname; + // Use it as --name. + // Example: libsoundio.so.2 + var prefix: usize = 0; + if (mem.startsWith(u8, soname, "lib")) { + prefix = 3; + } + var end: usize = soname.len; + if (mem.endsWith(u8, soname, ".so")) { + end -= 3; } else { - fatal("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode, found '{}'", .{next_arg}); + var found_digit = false; + while (end > 0 and std.ascii.isDigit(soname[end - 1])) { + found_digit = true; + end -= 1; + } + if (found_digit and end > 0 and soname[end - 1] == '.') { + end -= 1; + } else { + end = soname.len; + } + if (mem.endsWith(u8, soname[prefix..end], ".so")) { + end -= 3; + } } - } else if (mem.eql(u8, arg, "--stack")) { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); - i += 1; - stack_size_override = std.fmt.parseInt(u64, args[i], 10) catch |err| { - fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); - }; - } else if (mem.eql(u8, arg, "--name")) { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); - i += 1; - provided_name = args[i]; + provided_name = soname[prefix..end]; } else if (mem.eql(u8, arg, "-rpath")) { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; - try rpath_list.append(args[i]); - } else if (mem.eql(u8, arg, "--library-directory") or mem.eql(u8, arg, "-L")) { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); - i += 1; - try lib_dirs.append(args[i]); - } else if (mem.eql(u8, arg, "-T")) { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); - i += 1; - linker_script = args[i]; - } else if (mem.eql(u8, arg, "--version-script")) { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); - i += 1; - version_script = args[i]; - } else if (mem.eql(u8, arg, "--library") or mem.eql(u8, arg, "-l")) { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); - // We don't know whether this library is part of libc or libc++ until we resolve the target. - // So we simply append to the list for now. - i += 1; - try system_libs.append(args[i]); - } else if (mem.eql(u8, arg, "-D") or - mem.eql(u8, arg, "-isystem") or - mem.eql(u8, arg, "-I") or - mem.eql(u8, arg, "-dirafter")) - { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); - i += 1; - try clang_argv.append(arg); - try clang_argv.append(args[i]); - } else if (mem.eql(u8, arg, "--version")) { - if (i + 1 >= args.len) { - fatal("expected parameter after --version", .{}); + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); } + try rpath_list.append(linker_args.items[i]); + } else if (mem.eql(u8, arg, "-I") or + mem.eql(u8, arg, "--dynamic-linker") or + mem.eql(u8, arg, "-dynamic-linker")) + { i += 1; - version = std.builtin.Version.parse(args[i]) catch |err| { - fatal("unable to parse --version '{}': {}", .{ args[i], @errorName(err) }); - }; - have_version = true; - } else if (mem.eql(u8, arg, "-target")) { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); - i += 1; - target_arch_os_abi = args[i]; - } else if (mem.eql(u8, arg, "-mcpu")) { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); - i += 1; - target_mcpu = args[i]; - } else if (mem.eql(u8, arg, "-mcmodel")) { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); - i += 1; - machine_code_model = parseCodeModel(args[i]); - } else if (mem.startsWith(u8, arg, "-ofmt=")) { - target_ofmt = arg["-ofmt=".len..]; - } else if (mem.startsWith(u8, arg, "-mcpu=")) { - target_mcpu = arg["-mcpu=".len..]; - } else if (mem.startsWith(u8, arg, "-mcmodel=")) { - machine_code_model = parseCodeModel(arg["-mcmodel=".len..]); - } else if (mem.eql(u8, arg, "--dynamic-linker")) { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); - i += 1; - target_dynamic_linker = args[i]; - } else if (mem.eql(u8, arg, "--libc")) { - if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); - i += 1; - libc_paths_file = args[i]; - } else if (mem.eql(u8, arg, "--watch")) { - watch = true; - } else if (mem.eql(u8, arg, "-ftime-report")) { - time_report = true; - } else if (mem.eql(u8, arg, "-fPIC")) { - want_pic = true; - } else if (mem.eql(u8, arg, "-fno-PIC")) { - want_pic = false; - } else if (mem.eql(u8, arg, "-fstack-check")) { - want_stack_check = true; - } else if (mem.eql(u8, arg, "-fno-stack-check")) { - want_stack_check = false; - } else if (mem.eql(u8, arg, "-fsanitize-c")) { - want_sanitize_c = true; - } else if (mem.eql(u8, arg, "-fno-sanitize-c")) { - want_sanitize_c = false; - } else if (mem.eql(u8, arg, "-fvalgrind")) { - want_valgrind = true; - } else if (mem.eql(u8, arg, "-fno-valgrind")) { - want_valgrind = false; - } else if (mem.eql(u8, arg, "-fLLVM")) { - use_llvm = true; - } else if (mem.eql(u8, arg, "-fno-LLVM")) { - use_llvm = false; - } else if (mem.eql(u8, arg, "-fLLD")) { - use_lld = true; - } else if (mem.eql(u8, arg, "-fno-LLD")) { - use_lld = false; - } else if (mem.eql(u8, arg, "-fClang")) { - use_clang = true; - } else if (mem.eql(u8, arg, "-fno-Clang")) { - use_clang = false; - } else if (mem.eql(u8, arg, "-rdynamic")) { + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + target_dynamic_linker = linker_args.items[i]; + } else if (mem.eql(u8, arg, "-E") or + mem.eql(u8, arg, "--export-dynamic") or + mem.eql(u8, arg, "-export-dynamic")) + { rdynamic = true; - } else if (mem.eql(u8, arg, "-femit-bin")) { - emit_bin = .yes_default_path; - } else if (mem.startsWith(u8, arg, "-femit-bin=")) { - emit_bin = .{ .yes = arg["-femit-bin=".len..] }; - } else if (mem.eql(u8, arg, "-fno-emit-bin")) { - emit_bin = .no; - } else if (mem.eql(u8, arg, "-femit-zir")) { - emit_zir = .yes_default_path; - } else if (mem.startsWith(u8, arg, "-femit-zir=")) { - emit_zir = .{ .yes = arg["-femit-zir=".len..] }; - } else if (mem.eql(u8, arg, "-fno-emit-zir")) { - emit_zir = .no; - } else if (mem.eql(u8, arg, "-femit-h")) { - emit_h = .yes_default_path; - } else if (mem.startsWith(u8, arg, "-femit-h=")) { - emit_h = .{ .yes = arg["-femit-h=".len..] }; - } else if (mem.eql(u8, arg, "-fno-emit-h")) { - emit_h = .no; - } else if (mem.eql(u8, arg, "-dynamic")) { - link_mode = .Dynamic; - } else if (mem.eql(u8, arg, "-static")) { - link_mode = .Static; - } else if (mem.eql(u8, arg, "-fdll-export-fns")) { - dll_export_fns = true; - } else if (mem.eql(u8, arg, "-fno-dll-export-fns")) { - dll_export_fns = false; - } else if (mem.eql(u8, arg, "--show-builtin")) { - show_builtin = true; - } else if (mem.eql(u8, arg, "--strip")) { - strip = true; - } else if (mem.eql(u8, arg, "--single-threaded")) { - single_threaded = true; - } else if (mem.eql(u8, arg, "--eh-frame-hdr")) { - link_eh_frame_hdr = true; + } else if (mem.eql(u8, arg, "--version-script")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + version_script = linker_args.items[i]; + } else if (mem.startsWith(u8, arg, "-O")) { + try lld_argv.append(arg); + } else if (mem.eql(u8, arg, "--gc-sections")) { + linker_gc_sections = true; + } else if (mem.eql(u8, arg, "--no-gc-sections")) { + linker_gc_sections = false; + } else if (mem.eql(u8, arg, "--allow-shlib-undefined") or + mem.eql(u8, arg, "-allow-shlib-undefined")) + { + linker_allow_shlib_undefined = true; + } else if (mem.eql(u8, arg, "--no-allow-shlib-undefined") or + mem.eql(u8, arg, "-no-allow-shlib-undefined")) + { + linker_allow_shlib_undefined = false; } else if (mem.eql(u8, arg, "-Bsymbolic")) { linker_bind_global_refs_locally = true; - } else if (mem.eql(u8, arg, "--verbose-link")) { - verbose_link = true; - } else if (mem.eql(u8, arg, "--verbose-cc")) { - verbose_cc = true; - } else if (mem.eql(u8, arg, "--verbose-tokenize")) { - verbose_tokenize = true; - } else if (mem.eql(u8, arg, "--verbose-ast")) { - verbose_ast = true; - } else if (mem.eql(u8, arg, "--verbose-ir")) { - verbose_ir = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { - verbose_llvm_ir = true; - } else if (mem.eql(u8, arg, "--verbose-cimport")) { - verbose_cimport = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { - verbose_llvm_cpu_features = true; - } else if (mem.startsWith(u8, arg, "-T")) { - linker_script = arg[2..]; - } else if (mem.startsWith(u8, arg, "-L")) { - try lib_dirs.append(arg[2..]); - } else if (mem.startsWith(u8, arg, "-l")) { - // We don't know whether this library is part of libc or libc++ until we resolve the target. - // So we simply append to the list for now. - try system_libs.append(arg[2..]); - } else if (mem.startsWith(u8, arg, "-D") or - mem.startsWith(u8, arg, "-I")) - { - try clang_argv.append(arg); - } else { - fatal("unrecognized parameter: '{}'", .{arg}); - } - } else switch (Compilation.classifyFileExt(arg)) { - .object, .static_library => { - try link_objects.append(arg); - }, - .assembly, .c, .cpp, .h, .ll, .bc => { - // TODO a way to pass extra flags on the CLI - try c_source_files.append(.{ .src_path = arg }); - }, - .shared_library => { - fatal("linking against dynamic libraries not yet supported", .{}); - }, - .zig, .zir => { - if (root_src_file) |other| { - fatal("found another zig file '{}' after root source file '{}'", .{ arg, other }); - } else { - root_src_file = arg; + } else if (mem.eql(u8, arg, "-z")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); } - }, - .unknown => { - fatal("unrecognized file extension of parameter '{}'", .{arg}); - }, + const z_arg = linker_args.items[i]; + if (mem.eql(u8, z_arg, "nodelete")) { + linker_z_nodelete = true; + } else if (mem.eql(u8, z_arg, "defs")) { + linker_z_defs = true; + } else { + warn("unsupported linker arg: -z {}", .{z_arg}); + } + } else if (mem.eql(u8, arg, "--major-image-version")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + version.major = std.fmt.parseInt(u32, linker_args.items[i], 10) catch |err| { + fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); + }; + have_version = true; + } else if (mem.eql(u8, arg, "--minor-image-version")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + version.minor = std.fmt.parseInt(u32, linker_args.items[i], 10) catch |err| { + fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); + }; + have_version = true; + } else if (mem.eql(u8, arg, "--stack")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{}'", .{arg}); + } + stack_size_override = std.fmt.parseInt(u64, linker_args.items[i], 10) catch |err| { + fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); + }; + } else { + warn("unsupported linker arg: {}", .{arg}); + } } - } - } else { - emit_h = .no; - strip = true; - ensure_libc_on_non_freestanding = true; - ensure_libcpp_on_non_freestanding = arg_mode == .cpp; - want_native_include_dirs = true; - var c_arg = false; - var is_shared_lib = false; - var linker_args = std.ArrayList([]const u8).init(arena); - var it = ClangArgIterator.init(arena, all_args); - while (it.has_next) { - it.next() catch |err| { - fatal("unable to parse command line parameters: {}", .{@errorName(err)}); - }; - switch (it.zig_equivalent) { - .target => target_arch_os_abi = it.only_arg, // example: -target riscv64-linux-unknown - .o => { - // -o - emit_bin = .{ .yes = it.only_arg }; - enable_cache = true; - }, - .c => c_arg = true, // -c - .other => { - try clang_argv.appendSlice(it.other_args); - }, - .positional => { - const file_ext = Compilation.classifyFileExt(mem.spanZ(it.only_arg)); - switch (file_ext) { - .assembly, .c, .cpp, .ll, .bc, .h => try c_source_files.append(.{ .src_path = it.only_arg }), - .unknown, .shared_library, .object, .static_library => { - try link_objects.append(it.only_arg); - }, - .zig, .zir => { - if (root_src_file) |other| { - fatal("found another zig file '{}' after root source file '{}'", .{ it.only_arg, other }); - } else { - root_src_file = it.only_arg; - } - }, - } - }, - .l => { - // -l - // We don't know whether this library is part of libc or libc++ until we resolve the target. - // So we simply append to the list for now. - try system_libs.append(it.only_arg); - }, - .ignore => {}, - .driver_punt => { - // Never mind what we're doing, just pass the args directly. For example --help. - return punt_to_clang(arena, all_args); - }, - .pic => want_pic = true, - .no_pic => want_pic = false, - .nostdlib => ensure_libc_on_non_freestanding = false, - .nostdlib_cpp => ensure_libcpp_on_non_freestanding = false, - .shared => { - link_mode = .Dynamic; - is_shared_lib = true; - }, - .rdynamic => rdynamic = true, - .wl => { - var split_it = mem.split(it.only_arg, ","); - while (split_it.next()) |linker_arg| { - try linker_args.append(linker_arg); - } - }, - .pp_or_asm => { - // This handles both -E and -S. - only_pp_or_asm = true; - try clang_argv.appendSlice(it.other_args); - }, - .optimize => { - // Alright, what release mode do they want? - if (mem.eql(u8, it.only_arg, "Os")) { - build_mode = .ReleaseSmall; - } else if (mem.eql(u8, it.only_arg, "O2") or - mem.eql(u8, it.only_arg, "O3") or - mem.eql(u8, it.only_arg, "O4")) - { - build_mode = .ReleaseFast; - } else if (mem.eql(u8, it.only_arg, "Og") or - mem.eql(u8, it.only_arg, "O0")) - { - build_mode = .Debug; - } else { - try clang_argv.appendSlice(it.other_args); - } - }, - .debug => { - strip = false; - if (mem.eql(u8, it.only_arg, "-g")) { - // We handled with strip = false above. - } else { - try clang_argv.appendSlice(it.other_args); - } - }, - .sanitize => { - if (mem.eql(u8, it.only_arg, "undefined")) { - want_sanitize_c = true; - } else { - try clang_argv.appendSlice(it.other_args); - } - }, - .linker_script => linker_script = it.only_arg, - .verbose_cmds => { - verbose_cc = true; - verbose_link = true; - }, - .for_linker => try linker_args.append(it.only_arg), - .linker_input_z => { - try linker_args.append("-z"); - try linker_args.append(it.only_arg); - }, - .lib_dir => try lib_dirs.append(it.only_arg), - .mcpu => target_mcpu = it.only_arg, - .dep_file => { - disable_c_depfile = true; - try clang_argv.appendSlice(it.other_args); - }, - .framework_dir => try framework_dirs.append(it.only_arg), - .framework => try frameworks.append(it.only_arg), - .nostdlibinc => want_native_include_dirs = false, + if (want_sanitize_c) |wsc| { + if (wsc and build_mode == .ReleaseFast) { + build_mode = .ReleaseSafe; + } } - } - // Parse linker args. - var i: usize = 0; - while (i < linker_args.items.len) : (i += 1) { - const arg = linker_args.items[i]; - if (mem.eql(u8, arg, "-soname")) { - i += 1; - if (i >= linker_args.items.len) { - fatal("expected linker arg after '{}'", .{arg}); + + if (only_pp_or_asm) { + output_mode = .Obj; + fatal("TODO implement using zig cc as a preprocessor", .{}); + //// Transfer "link_objects" into c_source_files so that all those + //// args make it onto the command line. + //try c_source_files.appendSlice(link_objects.items); + //for (c_source_files.items) |c_source_file| { + // const src_path = switch (emit_bin) { + // .yes => |p| p, + // else => c_source_file.source_path, + // }; + // const basename = fs.path.basename(src_path); + // c_source_file.preprocessor_only_basename = basename; + //} + //emit_bin = .no; + } else if (!c_arg) { + output_mode = if (is_shared_lib) .Lib else .Exe; + switch (emit_bin) { + .no, .yes_default_path => { + emit_bin = .{ .yes = "a.out" }; + enable_cache = true; + }, + .yes => {}, } - const soname = linker_args.items[i]; - override_soname = soname; - // Use it as --name. - // Example: libsoundio.so.2 - var prefix: usize = 0; - if (mem.startsWith(u8, soname, "lib")) { - prefix = 3; - } - var end: usize = soname.len; - if (mem.endsWith(u8, soname, ".so")) { - end -= 3; - } else { - var found_digit = false; - while (end > 0 and std.ascii.isDigit(soname[end - 1])) { - found_digit = true; - end -= 1; - } - if (found_digit and end > 0 and soname[end - 1] == '.') { - end -= 1; - } else { - end = soname.len; - } - if (mem.endsWith(u8, soname[prefix..end], ".so")) { - end -= 3; - } - } - provided_name = soname[prefix..end]; - } else if (mem.eql(u8, arg, "-rpath")) { - i += 1; - if (i >= linker_args.items.len) { - fatal("expected linker arg after '{}'", .{arg}); - } - try rpath_list.append(linker_args.items[i]); - } else if (mem.eql(u8, arg, "-I") or - mem.eql(u8, arg, "--dynamic-linker") or - mem.eql(u8, arg, "-dynamic-linker")) - { - i += 1; - if (i >= linker_args.items.len) { - fatal("expected linker arg after '{}'", .{arg}); - } - target_dynamic_linker = linker_args.items[i]; - } else if (mem.eql(u8, arg, "-E") or - mem.eql(u8, arg, "--export-dynamic") or - mem.eql(u8, arg, "-export-dynamic")) - { - rdynamic = true; - } else if (mem.eql(u8, arg, "--version-script")) { - i += 1; - if (i >= linker_args.items.len) { - fatal("expected linker arg after '{}'", .{arg}); - } - version_script = linker_args.items[i]; - } else if (mem.startsWith(u8, arg, "-O")) { - try lld_argv.append(arg); - } else if (mem.eql(u8, arg, "--gc-sections")) { - linker_gc_sections = true; - } else if (mem.eql(u8, arg, "--no-gc-sections")) { - linker_gc_sections = false; - } else if (mem.eql(u8, arg, "--allow-shlib-undefined") or - mem.eql(u8, arg, "-allow-shlib-undefined")) - { - linker_allow_shlib_undefined = true; - } else if (mem.eql(u8, arg, "--no-allow-shlib-undefined") or - mem.eql(u8, arg, "-no-allow-shlib-undefined")) - { - linker_allow_shlib_undefined = false; - } else if (mem.eql(u8, arg, "-Bsymbolic")) { - linker_bind_global_refs_locally = true; - } else if (mem.eql(u8, arg, "-z")) { - i += 1; - if (i >= linker_args.items.len) { - fatal("expected linker arg after '{}'", .{arg}); - } - const z_arg = linker_args.items[i]; - if (mem.eql(u8, z_arg, "nodelete")) { - linker_z_nodelete = true; - } else if (mem.eql(u8, z_arg, "defs")) { - linker_z_defs = true; - } else { - warn("unsupported linker arg: -z {}", .{z_arg}); - } - } else if (mem.eql(u8, arg, "--major-image-version")) { - i += 1; - if (i >= linker_args.items.len) { - fatal("expected linker arg after '{}'", .{arg}); - } - version.major = std.fmt.parseInt(u32, linker_args.items[i], 10) catch |err| { - fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); - }; - have_version = true; - } else if (mem.eql(u8, arg, "--minor-image-version")) { - i += 1; - if (i >= linker_args.items.len) { - fatal("expected linker arg after '{}'", .{arg}); - } - version.minor = std.fmt.parseInt(u32, linker_args.items[i], 10) catch |err| { - fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); - }; - have_version = true; - } else if (mem.eql(u8, arg, "--stack")) { - i += 1; - if (i >= linker_args.items.len) { - fatal("expected linker arg after '{}'", .{arg}); - } - stack_size_override = std.fmt.parseInt(u64, linker_args.items[i], 10) catch |err| { - fatal("unable to parse '{}': {}", .{ arg, @errorName(err) }); - }; } else { - warn("unsupported linker arg: {}", .{arg}); + output_mode = .Obj; } - } - - if (want_sanitize_c) |wsc| { - if (wsc and build_mode == .ReleaseFast) { - build_mode = .ReleaseSafe; + if (c_source_files.items.len == 0 and link_objects.items.len == 0) { + // For example `zig cc` and no args should print the "no input files" message. + return punt_to_clang(arena, all_args); } - } - - if (only_pp_or_asm) { - output_mode = .Obj; - fatal("TODO implement using zig cc as a preprocessor", .{}); - //// Transfer "link_objects" into c_source_files so that all those - //// args make it onto the command line. - //try c_source_files.appendSlice(link_objects.items); - //for (c_source_files.items) |c_source_file| { - // const src_path = switch (emit_bin) { - // .yes => |p| p, - // else => c_source_file.source_path, - // }; - // const basename = fs.path.basename(src_path); - // c_source_file.preprocessor_only_basename = basename; - //} - //emit_bin = .no; - } else if (!c_arg) { - output_mode = if (is_shared_lib) .Lib else .Exe; - switch (emit_bin) { - .no, .yes_default_path => { - emit_bin = .{ .yes = "a.out" }; - enable_cache = true; - }, - .yes => {}, - } - } else { - output_mode = .Obj; - } - if (c_source_files.items.len == 0 and link_objects.items.len == 0) { - // For example `zig cc` and no args should print the "no input files" message. - return punt_to_clang(arena, all_args); - } + }, } if (arg_mode == .translate_c and c_source_files.items.len != 1) { @@ -902,7 +953,9 @@ pub fn buildOutputType( } const root_name = if (provided_name) |n| n else blk: { - if (root_src_file) |file| { + if (arg_mode == .zig_test) { + break :blk "test"; + } else if (root_src_file) |file| { const basename = fs.path.basename(file); break :blk mem.split(basename, ".").next().?; } else if (c_source_files.items.len == 1) { @@ -916,6 +969,8 @@ pub fn buildOutputType( break :blk mem.split(basename, ".").next().?; } else if (show_builtin) { break :blk "builtin"; + } else if (arg_mode == .run) { + break :blk "run"; } else { fatal("--name [name] not provided and unable to infer", .{}); } @@ -1220,6 +1275,10 @@ pub fn buildOutputType( .machine_code_model = machine_code_model, .color = color, .time_report = time_report, + .is_test = arg_mode == .zig_test, + .test_evented_io = test_evented_io, + .test_filter = test_filter, + .test_name_prefix = test_name_prefix, }) catch |err| { fatal("unable to create compilation: {}", .{@errorName(err)}); }; @@ -1243,6 +1302,52 @@ pub fn buildOutputType( std.log.warn("--watch is not recommended with the stage1 backend; it leaks memory and is not capable of incremental compilation", .{}); } + switch (arg_mode) { + .run, .zig_test => run: { + const exe_loc = emit_bin_loc orelse break :run; + const exe_directory = exe_loc.directory orelse comp.bin_file.options.directory; + const exe_path = try fs.path.join(arena, &[_][]const u8{ + exe_directory.path orelse ".", exe_loc.basename, + }); + + var argv = std.ArrayList([]const u8).init(gpa); + defer argv.deinit(); + + if (test_exec_args.items.len == 0) { + try argv.append(exe_path); + } else { + for (test_exec_args.items) |arg| { + try argv.append(arg orelse exe_path); + } + } + if (runtime_args_start) |i| { + try argv.appendSlice(all_args[i..]); + } + // TODO On operating systems that support it, do an execve here rather than child process, + // when watch=false. + const child = try std.ChildProcess.init(argv.items, gpa); + defer child.deinit(); + + child.stdin_behavior = .Inherit; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; + + const term = try child.spawnAndWait(); + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO https://github.com/ziglang/zig/issues/6342 + process.exit(1); + } + }, + else => process.exit(1), + } + if (!watch) + process.exit(0); + }, + else => {}, + } + const stdin = std.io.getStdIn().inStream(); const stderr = std.io.getStdErr().outStream(); var repl_buf: [1024]u8 = undefined; From afac5d28951cfd913851094649e8b9f2136694ca Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Sep 2020 21:14:01 -0700 Subject: [PATCH 086/210] fix regressed stage2 test harness --- src/Compilation.zig | 15 +++++++---- src/stage1/stage2.h | 2 +- src/test.zig | 14 ++++++++--- src/windows_sdk.h | 8 +++--- src/zig_clang.h | 2 +- test/stage2/cbe.zig | 2 +- test/stage2/spu-ii.zig | 2 +- test/stage2/test.zig | 56 +++++++++++++++++++++--------------------- test/stage2/zir.zig | 4 +-- 9 files changed, 58 insertions(+), 47 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 725e212b96..948eb1840b 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -772,7 +772,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { if (build_options.is_stage1 and comp.bin_file.options.use_llvm) { try comp.work_queue.writeItem(.{ .stage1_module = {} }); } - if (is_exe_or_dyn_lib) { + if (is_exe_or_dyn_lib and comp.bin_file.options.use_llvm) { try comp.work_queue.writeItem(.{ .libcompiler_rt = {} }); if (!comp.bin_file.options.link_libc) { try comp.work_queue.writeItem(.{ .zig_libc = {} }); @@ -1128,6 +1128,9 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { }; }, .stage1_module => { + if (!build_options.is_stage1) + unreachable; + self.updateStage1Module() catch |err| { fatal("unable to build stage1 zig object: {}", .{@errorName(err)}); }; @@ -1685,11 +1688,13 @@ pub fn classifyFileExt(filename: []const u8) FileExt { test "classifyFileExt" { std.testing.expectEqual(FileExt.cpp, classifyFileExt("foo.cc")); std.testing.expectEqual(FileExt.unknown, classifyFileExt("foo.nim")); - std.testing.expectEqual(FileExt.so, classifyFileExt("foo.so")); - std.testing.expectEqual(FileExt.so, classifyFileExt("foo.so.1")); - std.testing.expectEqual(FileExt.so, classifyFileExt("foo.so.1.2")); - std.testing.expectEqual(FileExt.so, classifyFileExt("foo.so.1.2.3")); + std.testing.expectEqual(FileExt.shared_library, classifyFileExt("foo.so")); + std.testing.expectEqual(FileExt.shared_library, classifyFileExt("foo.so.1")); + std.testing.expectEqual(FileExt.shared_library, classifyFileExt("foo.so.1.2")); + std.testing.expectEqual(FileExt.shared_library, classifyFileExt("foo.so.1.2.3")); std.testing.expectEqual(FileExt.unknown, classifyFileExt("foo.so.1.2.3~")); + std.testing.expectEqual(FileExt.zig, classifyFileExt("foo.zig")); + std.testing.expectEqual(FileExt.zir, classifyFileExt("foo.zir")); } fn haveFramePointer(comp: *Compilation) bool { diff --git a/src/stage1/stage2.h b/src/stage1/stage2.h index 104faa6ca8..613cae2a77 100644 --- a/src/stage1/stage2.h +++ b/src/stage1/stage2.h @@ -29,7 +29,7 @@ #endif // ABI warning: the types and declarations in this file must match both those in -// stage2.cpp and src-self-hosted/stage1.zig. +// stage2.cpp and src/stage1.zig. // ABI warning enum Error { diff --git a/src/test.zig b/src/test.zig index 154a839a3c..0f63c3f6a6 100644 --- a/src/test.zig +++ b/src/test.zig @@ -458,9 +458,15 @@ pub const TestContext = struct { .path = try std.fs.path.join(arena, &[_][]const u8{ bogus_path, "zig-cache" }), }; - const tmp_src_path = if (case.extension == .Zig) "test_case.zig" else if (case.extension == .ZIR) "test_case.zir" else unreachable; - const root_pkg = try Package.create(allocator, tmp.dir, ".", tmp_src_path); - defer root_pkg.destroy(allocator); + const tmp_src_path = switch (case.extension) { + .Zig => "test_case.zig", + .ZIR => "test_case.zir", + }; + + var root_pkg: Package = .{ + .root_src_directory = .{ .path = bogus_path, .handle = tmp.dir }, + .root_src_path = tmp_src_path, + }; const ofmt: ?std.builtin.ObjectFormat = if (case.cbe) .c else null; const bin_name = try std.zig.binNameAlloc(arena, "test_case", target, case.output_mode, null, ofmt); @@ -486,7 +492,7 @@ pub const TestContext = struct { // TODO: support testing optimizations .optimize_mode = .Debug, .emit_bin = emit_bin, - .root_pkg = root_pkg, + .root_pkg = &root_pkg, .keep_source_files_loaded = true, .object_format = ofmt, .is_native_os = case.target.isNativeOs(), diff --git a/src/windows_sdk.h b/src/windows_sdk.h index e887209559..a4707b3de0 100644 --- a/src/windows_sdk.h +++ b/src/windows_sdk.h @@ -16,7 +16,7 @@ #include -// ABI warning - src-self-hosted/windows_sdk.zig +// ABI warning - src/windows_sdk.zig struct ZigWindowsSDK { const char *path10_ptr; size_t path10_len; @@ -34,7 +34,7 @@ struct ZigWindowsSDK { size_t msvc_lib_dir_len; }; -// ABI warning - src-self-hosted/windows_sdk.zig +// ABI warning - src/windows_sdk.zig enum ZigFindWindowsSdkError { ZigFindWindowsSdkErrorNone, ZigFindWindowsSdkErrorOutOfMemory, @@ -42,10 +42,10 @@ enum ZigFindWindowsSdkError { ZigFindWindowsSdkErrorPathTooLong, }; -// ABI warning - src-self-hosted/windows_sdk.zig +// ABI warning - src/windows_sdk.zig ZIG_EXTERN_C enum ZigFindWindowsSdkError zig_find_windows_sdk(struct ZigWindowsSDK **out_sdk); -// ABI warning - src-self-hosted/windows_sdk.zig +// ABI warning - src/windows_sdk.zig ZIG_EXTERN_C void zig_free_windows_sdk(struct ZigWindowsSDK *sdk); #endif diff --git a/src/zig_clang.h b/src/zig_clang.h index 2c358d880e..f9ce9c34ed 100644 --- a/src/zig_clang.h +++ b/src/zig_clang.h @@ -14,7 +14,7 @@ // ATTENTION: If you modify this file, be sure to update the corresponding // extern function declarations in the self-hosted compiler file -// src-self-hosted/clang.zig. +// src/clang.zig. struct ZigClangSourceLocation { unsigned ID; diff --git a/test/stage2/cbe.zig b/test/stage2/cbe.zig index 0608221866..2a176e0368 100644 --- a/test/stage2/cbe.zig +++ b/test/stage2/cbe.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const TestContext = @import("../../src-self-hosted/test.zig").TestContext; +const TestContext = @import("../../src/test.zig").TestContext; // These tests should work with all platforms, but we're using linux_x64 for // now for consistency. Will be expanded eventually. diff --git a/test/stage2/spu-ii.zig b/test/stage2/spu-ii.zig index 1316f19d0d..aa091c4174 100644 --- a/test/stage2/spu-ii.zig +++ b/test/stage2/spu-ii.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const TestContext = @import("../../src-self-hosted/test.zig").TestContext; +const TestContext = @import("../../src/test.zig").TestContext; const spu = std.zig.CrossTarget{ .cpu_arch = .spu_2, diff --git a/test/stage2/test.zig b/test/stage2/test.zig index b37eb63a2e..a22dc23d36 100644 --- a/test/stage2/test.zig +++ b/test/stage2/test.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const TestContext = @import("../../src-self-hosted/test.zig").TestContext; +const TestContext = @import("../../src/test.zig").TestContext; // Self-hosted has differing levels of support for various architectures. For now we pass explicit // target parameters to each test case. At some point we will take this to the next level and have @@ -76,7 +76,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "Hello, World!\n", ); // Now change the message only @@ -108,7 +108,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "What is up? This is a longer message that will force the data to be relocated in virtual address space.\n", ); // Now we print it twice. @@ -184,7 +184,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "Hello, World!\n", ); } @@ -219,7 +219,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "Hello, World!\n", ); } @@ -244,7 +244,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "Hello, World!\n", ); } @@ -271,7 +271,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); } @@ -298,7 +298,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); } @@ -329,7 +329,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -362,7 +362,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -398,7 +398,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -435,7 +435,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -465,7 +465,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -499,7 +499,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -523,7 +523,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -562,7 +562,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "hello\nhello\nhello\nhello\n", ); @@ -599,7 +599,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -641,7 +641,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -693,7 +693,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -755,7 +755,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -788,7 +788,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -820,7 +820,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -845,7 +845,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -871,7 +871,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "", ); @@ -904,7 +904,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); \\ unreachable; \\} - , + , "hello\nhello\nhello\nhello\nhello\n", ); } @@ -923,7 +923,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ bar(); \\} \\fn bar() void {} - , + , "42\n", ); @@ -941,7 +941,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ bar(); \\} \\fn bar() void {} - , + , "42\n", ); @@ -957,7 +957,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ bar(); \\} \\fn bar() void {} - , + , // This is what you get when you take the bits of the IEE-754 // representation of 42.0 and reinterpret them as an unsigned // integer. Guess that's a bug in wasmtime. diff --git a/test/stage2/zir.zig b/test/stage2/zir.zig index 78d971d104..f87fca3748 100644 --- a/test/stage2/zir.zig +++ b/test/stage2/zir.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const TestContext = @import("../../src-self-hosted/test.zig").TestContext; +const TestContext = @import("../../src/test.zig").TestContext; // self-hosted does not yet support PE executable files / COFF object files // or mach-o files. So we do the ZIR transform test cases cross compiling for // x86_64-linux. @@ -156,7 +156,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ %0 = call(@a, []) \\ %1 = returnvoid() \\}) - , + , &[_][]const u8{ ":18:21: error: message", }, From 974333faaf70eb67b78d3fa83bdc92e91379631a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Sep 2020 21:42:27 -0700 Subject: [PATCH 087/210] stage2: fix linking libc trying to depend on itself --- src/Compilation.zig | 6 ++++-- src/glibc.zig | 2 ++ src/link.zig | 1 + src/link/Elf.zig | 3 ++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 948eb1840b..f67e164ecc 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -324,6 +324,8 @@ pub const InitOptions = struct { verbose_cimport: bool = false, verbose_llvm_cpu_features: bool = false, is_test: bool = false, + test_evented_io: bool = false, + is_compiler_rt_or_libc: bool = false, stack_size_override: ?u64 = null, self_exe_path: ?[]const u8 = null, version: ?std.builtin.Version = null, @@ -333,7 +335,6 @@ pub const InitOptions = struct { color: @import("main.zig").Color = .Auto, test_filter: ?[]const u8 = null, test_name_prefix: ?[]const u8 = null, - test_evented_io: bool = false, }; pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { @@ -704,9 +705,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .dll_export_fns = dll_export_fns, .error_return_tracing = error_return_tracing, .llvm_cpu_features = llvm_cpu_features, + .is_compiler_rt_or_libc = options.is_compiler_rt_or_libc, }); errdefer bin_file.destroy(); - comp.* = .{ .gpa = gpa, .arena_state = arena_allocator.state, @@ -2116,6 +2117,7 @@ fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFil .verbose_cimport = comp.verbose_cimport, .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features, .clang_passthrough_mode = comp.clang_passthrough_mode, + .is_compiler_rt_or_libc = true, }); defer sub_compilation.destroy(); diff --git a/src/glibc.zig b/src/glibc.zig index 9ede436f63..22f060f514 100644 --- a/src/glibc.zig +++ b/src/glibc.zig @@ -720,6 +720,7 @@ fn build_crt_file( .verbose_cimport = comp.verbose_cimport, .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features, .clang_passthrough_mode = comp.clang_passthrough_mode, + .is_compiler_rt_or_libc = true, }); defer sub_compilation.destroy(); @@ -1006,6 +1007,7 @@ fn buildSharedLib( .version_script = map_file_path, .override_soname = override_soname, .c_source_files = &c_source_files, + .is_compiler_rt_or_libc = true, }); defer sub_compilation.destroy(); diff --git a/src/link.zig b/src/link.zig index 3ec81715e4..71d49286ba 100644 --- a/src/link.zig +++ b/src/link.zig @@ -64,6 +64,7 @@ pub const Options = struct { verbose_link: bool = false, dll_export_fns: bool, error_return_tracing: bool, + is_compiler_rt_or_libc: bool, gc_sections: ?bool = null, allow_shlib_undefined: ?bool = null, linker_script: ?[]const u8 = null, diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 98deefb3bf..3d8afa8483 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1272,6 +1272,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { ch.hash.add(self.base.options.rdynamic); ch.hash.addListOfBytes(self.base.options.extra_lld_args); ch.hash.addListOfBytes(self.base.options.lib_dirs); + ch.hash.add(self.base.options.is_compiler_rt_or_libc); ch.hash.add(self.base.options.z_nodelete); ch.hash.add(self.base.options.z_defs); if (self.base.options.link_libc) { @@ -1491,7 +1492,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { } // compiler-rt and libc - if (is_exe_or_dyn_lib) { + if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) { if (!self.base.options.link_libc) { try argv.append(comp.libc_static_lib.?.full_object_path); } From 877e4248fc1e8e28b1f43282411aa9868d6d1d9c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Sep 2020 22:38:45 -0700 Subject: [PATCH 088/210] stage2: implement building & linking against libcxx and libcxxabi --- BRANCH_TODO | 1 - src/Compilation.zig | 35 ++++++- src/libcxx.zig | 239 +++++++++++++++++++++++++++++++++++++++++++- src/libunwind.zig | 6 +- src/link/Elf.zig | 4 +- 5 files changed, 273 insertions(+), 12 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index f30f67fc69..026efaddce 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ - * build & link against libcxx and libcxxabi * `zig build` * repair @cImport * make sure zig cc works diff --git a/src/Compilation.zig b/src/Compilation.zig index f67e164ecc..70492b70be 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -16,6 +16,7 @@ const build_options = @import("build_options"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const glibc = @import("glibc.zig"); const libunwind = @import("libunwind.zig"); +const libcxx = @import("libcxx.zig"); const fatal = @import("main.zig").fatal; const Module = @import("Module.zig"); const Cache = @import("Cache.zig"); @@ -69,10 +70,10 @@ rand: *std.rand.Random, /// Populated when we build the libc++ static library. A Job to build this is placed in the queue /// and resolved before calling linker.flush(). -libcxx_static_lib: ?[]const u8 = null, +libcxx_static_lib: ?CRTFile = null, /// Populated when we build the libc++abi static library. A Job to build this is placed in the queue /// and resolved before calling linker.flush(). -libcxxabi_static_lib: ?[]const u8 = null, +libcxxabi_static_lib: ?CRTFile = null, /// Populated when we build the libunwind static library. A Job to build this is placed in the queue /// and resolved before calling linker.flush(). libunwind_static_lib: ?CRTFile = null, @@ -140,6 +141,8 @@ const Job = union(enum) { glibc_shared_objects, /// libunwind.a, usually needed when linking libc libunwind: void, + libcxx: void, + libcxxabi: void, /// needed when producing a dynamic library or executable libcompiler_rt: void, /// needed when not linking libc and using LLVM for code generation because it generates @@ -770,10 +773,18 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { if (comp.wantBuildLibUnwindFromSource()) { try comp.work_queue.writeItem(.{ .libunwind = {} }); } + if (build_options.have_llvm and comp.bin_file.options.output_mode != .Obj and + comp.bin_file.options.link_libcpp) + { + try comp.work_queue.writeItem(.libcxx); + try comp.work_queue.writeItem(.libcxxabi); + } if (build_options.is_stage1 and comp.bin_file.options.use_llvm) { try comp.work_queue.writeItem(.{ .stage1_module = {} }); } - if (is_exe_or_dyn_lib and comp.bin_file.options.use_llvm) { + if (is_exe_or_dyn_lib and !comp.bin_file.options.is_compiler_rt_or_libc and + build_options.is_stage1) + { try comp.work_queue.writeItem(.{ .libcompiler_rt = {} }); if (!comp.bin_file.options.link_libc) { try comp.work_queue.writeItem(.{ .zig_libc = {} }); @@ -811,6 +822,12 @@ pub fn destroy(self: *Compilation) void { if (self.libunwind_static_lib) |*crt_file| { crt_file.deinit(gpa); } + if (self.libcxx_static_lib) |*crt_file| { + crt_file.deinit(gpa); + } + if (self.libcxxabi_static_lib) |*crt_file| { + crt_file.deinit(gpa); + } if (self.compiler_rt_static_lib) |*crt_file| { crt_file.deinit(gpa); } @@ -1109,6 +1126,18 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { fatal("unable to build libunwind: {}", .{@errorName(err)}); }; }, + .libcxx => { + libcxx.buildLibCXX(self) catch |err| { + // TODO Expose this as a normal compile error rather than crashing here. + fatal("unable to build libcxx: {}", .{@errorName(err)}); + }; + }, + .libcxxabi => { + libcxx.buildLibCXXABI(self) catch |err| { + // TODO Expose this as a normal compile error rather than crashing here. + fatal("unable to build libcxxabi: {}", .{@errorName(err)}); + }; + }, .libcompiler_rt => { self.buildStaticLibFromZig("compiler_rt.zig", &self.compiler_rt_static_lib) catch |err| { // TODO Expose this as a normal compile error rather than crashing here. diff --git a/src/libcxx.zig b/src/libcxx.zig index c7dc24ae9f..e6ed35c560 100644 --- a/src/libcxx.zig +++ b/src/libcxx.zig @@ -1,6 +1,13 @@ -//! TODO build libcxx and libcxxabi from source +const std = @import("std"); +const path = std.fs.path; +const assert = std.debug.assert; -pub const libcxxabi_files = [_][]const u8{ +const target_util = @import("target.zig"); +const Compilation = @import("Compilation.zig"); +const build_options = @import("build_options"); +const trace = @import("tracy.zig").trace; + +const libcxxabi_files = [_][]const u8{ "src/abort_message.cpp", "src/cxa_aux_runtime.cpp", "src/cxa_default_handlers.cpp", @@ -22,7 +29,7 @@ pub const libcxxabi_files = [_][]const u8{ "src/stdlib_typeinfo.cpp", }; -pub const libcxx_files = [_][]const u8{ +const libcxx_files = [_][]const u8{ "src/algorithm.cpp", "src/any.cpp", "src/bind.cpp", @@ -64,3 +71,229 @@ pub const libcxx_files = [_][]const u8{ "src/variant.cpp", "src/vector.cpp", }; + +pub fn buildLibCXX(comp: *Compilation) !void { + if (!build_options.have_llvm) { + return error.ZigCompilerNotBuiltWithLLVMExtensions; + } + + const tracy = trace(@src()); + defer tracy.end(); + + var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); + defer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + const root_name = "c++"; + const output_mode = .Lib; + const link_mode = .Static; + const target = comp.getTarget(); + const basename = try std.zig.binNameAlloc(arena, root_name, target, output_mode, link_mode, null); + + const emit_bin = Compilation.EmitLoc{ + .directory = null, // Put it in the cache directory. + .basename = basename, + }; + + const cxxabi_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libcxxabi", "include" }); + const cxx_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libcxx", "include" }); + var c_source_files = std.ArrayList(Compilation.CSourceFile).init(arena); + try c_source_files.ensureCapacity(libcxx_files.len); + + for (libcxx_files) |cxx_src| { + var cflags = std.ArrayList([]const u8).init(arena); + + if (target.os.tag == .windows) { + // Filesystem stuff isn't supported on Windows. + if (std.mem.startsWith(u8, cxx_src, "src/filesystem/")) + continue; + } else { + if (std.mem.startsWith(u8, cxx_src, "src/support/win32/")) + continue; + } + + try cflags.append("-DNDEBUG"); + try cflags.append("-D_LIBCPP_BUILDING_LIBRARY"); + try cflags.append("-D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER"); + try cflags.append("-DLIBCXX_BUILDING_LIBCXXABI"); + try cflags.append("-D_LIBCXXABI_DISABLE_VISIBILITY_ANNOTATIONS"); + try cflags.append("-D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS"); + + if (target.abi.isMusl()) { + try cflags.append("-D_LIBCPP_HAS_MUSL_LIBC"); + } + + try cflags.append("-I"); + try cflags.append(cxx_include_path); + + try cflags.append("-I"); + try cflags.append(cxxabi_include_path); + + try cflags.append("-O3"); + try cflags.append("-DNDEBUG"); + if (target_util.supports_fpic(target)) { + try cflags.append("-fPIC"); + } + try cflags.append("-nostdinc++"); + try cflags.append("-fvisibility-inlines-hidden"); + try cflags.append("-std=c++14"); + try cflags.append("-Wno-user-defined-literals"); + + c_source_files.appendAssumeCapacity(.{ + .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libcxx", cxx_src }), + .extra_flags = cflags.items, + }); + } + + const sub_compilation = try Compilation.create(comp.gpa, .{ + // TODO use the global cache directory here + .zig_cache_directory = comp.zig_cache_directory, + .zig_lib_directory = comp.zig_lib_directory, + .target = target, + .root_name = root_name, + .root_pkg = null, + .output_mode = output_mode, + .rand = comp.rand, + .libc_installation = comp.bin_file.options.libc_installation, + .emit_bin = emit_bin, + .optimize_mode = comp.bin_file.options.optimize_mode, + .link_mode = link_mode, + .want_sanitize_c = false, + .want_stack_check = false, + .want_valgrind = false, + .want_pic = comp.bin_file.options.pic, + .emit_h = null, + .strip = comp.bin_file.options.strip, + .is_native_os = comp.bin_file.options.is_native_os, + .self_exe_path = comp.self_exe_path, + .c_source_files = c_source_files.items, + .verbose_cc = comp.verbose_cc, + .verbose_link = comp.bin_file.options.verbose_link, + .verbose_tokenize = comp.verbose_tokenize, + .verbose_ast = comp.verbose_ast, + .verbose_ir = comp.verbose_ir, + .verbose_llvm_ir = comp.verbose_llvm_ir, + .verbose_cimport = comp.verbose_cimport, + .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features, + .clang_passthrough_mode = comp.clang_passthrough_mode, + .link_libc = true, + }); + defer sub_compilation.destroy(); + + try sub_compilation.updateSubCompilation(); + + assert(comp.libcxx_static_lib == null); + comp.libcxx_static_lib = Compilation.CRTFile{ + .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{basename}), + .lock = sub_compilation.bin_file.toOwnedLock(), + }; +} + +pub fn buildLibCXXABI(comp: *Compilation) !void { + if (!build_options.have_llvm) { + return error.ZigCompilerNotBuiltWithLLVMExtensions; + } + + const tracy = trace(@src()); + defer tracy.end(); + + var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); + defer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + const root_name = "c++abi"; + const output_mode = .Lib; + const link_mode = .Static; + const target = comp.getTarget(); + const basename = try std.zig.binNameAlloc(arena, root_name, target, output_mode, link_mode, null); + + const emit_bin = Compilation.EmitLoc{ + .directory = null, // Put it in the cache directory. + .basename = basename, + }; + + const cxxabi_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libcxxabi", "include" }); + const cxx_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libcxx", "include" }); + + var c_source_files: [libcxxabi_files.len]Compilation.CSourceFile = undefined; + for (libcxxabi_files) |cxxabi_src, i| { + var cflags = std.ArrayList([]const u8).init(arena); + + try cflags.append("-DHAVE___CXA_THREAD_ATEXIT_IMPL"); + try cflags.append("-D_LIBCPP_DISABLE_EXTERN_TEMPLATE"); + try cflags.append("-D_LIBCPP_ENABLE_CXX17_REMOVED_UNEXPECTED_FUNCTIONS"); + try cflags.append("-D_LIBCXXABI_BUILDING_LIBRARY"); + try cflags.append("-D_LIBCXXABI_DISABLE_VISIBILITY_ANNOTATIONS"); + try cflags.append("-D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS"); + + if (target.abi.isMusl()) { + try cflags.append("-D_LIBCPP_HAS_MUSL_LIBC"); + } + + try cflags.append("-I"); + try cflags.append(cxxabi_include_path); + + try cflags.append("-I"); + try cflags.append(cxx_include_path); + + try cflags.append("-O3"); + try cflags.append("-DNDEBUG"); + if (target_util.supports_fpic(target)) { + try cflags.append("-fPIC"); + } + try cflags.append("-nostdinc++"); + try cflags.append("-fstrict-aliasing"); + try cflags.append("-funwind-tables"); + try cflags.append("-D_DEBUG"); + try cflags.append("-UNDEBUG"); + try cflags.append("-std=c++11"); + + c_source_files[i] = .{ + .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libcxxabi", cxxabi_src }), + .extra_flags = cflags.items, + }; + } + + const sub_compilation = try Compilation.create(comp.gpa, .{ + // TODO use the global cache directory here + .zig_cache_directory = comp.zig_cache_directory, + .zig_lib_directory = comp.zig_lib_directory, + .target = target, + .root_name = root_name, + .root_pkg = null, + .output_mode = output_mode, + .rand = comp.rand, + .libc_installation = comp.bin_file.options.libc_installation, + .emit_bin = emit_bin, + .optimize_mode = comp.bin_file.options.optimize_mode, + .link_mode = link_mode, + .want_sanitize_c = false, + .want_stack_check = false, + .want_valgrind = false, + .want_pic = comp.bin_file.options.pic, + .emit_h = null, + .strip = comp.bin_file.options.strip, + .is_native_os = comp.bin_file.options.is_native_os, + .self_exe_path = comp.self_exe_path, + .c_source_files = &c_source_files, + .verbose_cc = comp.verbose_cc, + .verbose_link = comp.bin_file.options.verbose_link, + .verbose_tokenize = comp.verbose_tokenize, + .verbose_ast = comp.verbose_ast, + .verbose_ir = comp.verbose_ir, + .verbose_llvm_ir = comp.verbose_llvm_ir, + .verbose_cimport = comp.verbose_cimport, + .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features, + .clang_passthrough_mode = comp.clang_passthrough_mode, + .link_libc = true, + }); + defer sub_compilation.destroy(); + + try sub_compilation.updateSubCompilation(); + + assert(comp.libcxxabi_static_lib == null); + comp.libcxxabi_static_lib = Compilation.CRTFile{ + .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{basename}), + .lock = sub_compilation.bin_file.toOwnedLock(), + }; +} diff --git a/src/libunwind.zig b/src/libunwind.zig index 0bb44808ff..e7562670e4 100644 --- a/src/libunwind.zig +++ b/src/libunwind.zig @@ -8,13 +8,13 @@ const build_options = @import("build_options"); const trace = @import("tracy.zig").trace; pub fn buildStaticLib(comp: *Compilation) !void { - const tracy = trace(@src()); - defer tracy.end(); - if (!build_options.have_llvm) { return error.ZigCompilerNotBuiltWithLLVMExtensions; } + const tracy = trace(@src()); + defer tracy.end(); + var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); defer arena_allocator.deinit(); const arena = &arena_allocator.allocator; diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 3d8afa8483..b275026e8e 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1513,8 +1513,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { if (!is_obj) { // libc++ dep if (self.base.options.link_libcpp) { - try argv.append(comp.libcxxabi_static_lib.?); - try argv.append(comp.libcxx_static_lib.?); + try argv.append(comp.libcxxabi_static_lib.?.full_object_path); + try argv.append(comp.libcxx_static_lib.?.full_object_path); } // libc dep From bf9a16a037e88d56bec3740698e3fcd34e9ad543 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Sep 2020 23:28:34 -0700 Subject: [PATCH 089/210] stage2: implement `zig init-lib` and `zig init-exe` --- BRANCH_TODO | 11 +++--- src/main.zig | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 7 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 026efaddce..c84dc397bb 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,3 +1,5 @@ + * skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd) + (maybe make it an explicit option and have main.zig disable it) * `zig build` * repair @cImport * make sure zig cc works @@ -12,7 +14,8 @@ * -fno-emit-asm (default) do not output .s (assembly code)\n" * -femit-llvm-ir produce a .ll file with LLVM IR\n" * -fno-emit-llvm-ir (default) do not produce a .ll file with LLVM IR\n" - * --cache-dir [path] override the local cache directory\n" + * --cache-dir [path] override the local cache directory + * --global-cache-dir [path] override the global cache directory * implement proper parsing of LLD stderr/stdout and exposing compile errors * implement proper parsing of clang stderr/stdout and exposing compile errors * support rpaths in ELF linker code @@ -27,8 +30,6 @@ * support cross compiling stage2 with `zig build` * --main-pkg-path * audit the CLI options for stage2 - * `zig init-lib` - * `zig init-exe` * restore error messages for stage2_add_link_lib * audit the base cache hash * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process. @@ -56,10 +57,6 @@ * libc_installation.zig: make it look for msvc only if msvc abi is chosen * switch the default C ABI for windows to be mingw-w64 * change glibc log errors to normal exposed compile errors - * update Package to use Compilation.Directory in create() - - skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd) - (maybe make it an explicit option and have main.zig disable it) - - make it possible for Package to not openDir and reference already existing resources. * improve Directory.join to only use 1 allocation in a clean way. * tracy builds with lc++ * some kind of "zig identifier escape" function rather than unconditionally using @"" syntax diff --git a/src/main.zig b/src/main.zig index c35586e925..3faa8b62a1 100644 --- a/src/main.zig +++ b/src/main.zig @@ -140,6 +140,10 @@ pub fn mainArgs(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v return cmdFmt(gpa, cmd_args); } else if (mem.eql(u8, cmd, "libc")) { return cmdLibC(gpa, cmd_args); + } else if (mem.eql(u8, cmd, "init-exe")) { + return cmdInit(gpa, arena, cmd_args, .Exe); + } else if (mem.eql(u8, cmd, "init-lib")) { + return cmdInit(gpa, arena, cmd_args, .Lib); } else if (mem.eql(u8, cmd, "targets")) { const info = try detectNativeTargetInfo(arena, .{}); const stdout = io.getStdOut().outStream(); @@ -1520,6 +1524,97 @@ pub fn cmdLibC(gpa: *Allocator, args: []const []const u8) !void { } } +pub const usage_init = + \\Usage: zig init-exe + \\ zig init-lib + \\ + \\ Initializes a `zig build` project in the current working + \\ directory. + \\ + \\Options: + \\ --help Print this help and exit + \\ + \\ +; + +pub fn cmdInit( + gpa: *Allocator, + arena: *Allocator, + args: []const []const u8, + output_mode: std.builtin.OutputMode, +) !void { + { + var i: usize = 0; + while (i < args.len) : (i += 1) { + const arg = args[i]; + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "--help")) { + try io.getStdOut().writeAll(usage_init); + process.exit(0); + } else { + fatal("unrecognized parameter: '{}'", .{arg}); + } + } else { + fatal("unexpected extra parameter: '{}'", .{arg}); + } + } + } + const self_exe_path = try fs.selfExePathAlloc(arena); + var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| { + fatal("unable to find zig installation directory: {}\n", .{@errorName(err)}); + }; + defer zig_lib_directory.handle.close(); + + const s = fs.path.sep_str; + const template_sub_path = switch (output_mode) { + .Obj => unreachable, + .Lib => "std" ++ s ++ "special" ++ s ++ "init-lib", + .Exe => "std" ++ s ++ "special" ++ s ++ "init-exe", + }; + var template_dir = try zig_lib_directory.handle.openDir(template_sub_path, .{}); + defer template_dir.close(); + + const cwd_path = try process.getCwdAlloc(arena); + const cwd_basename = fs.path.basename(cwd_path); + + const max_bytes = 10 * 1024 * 1024; + const build_zig_contents = template_dir.readFileAlloc(arena, "build.zig", max_bytes) catch |err| { + fatal("unable to read template file 'build.zig': {}", .{@errorName(err)}); + }; + var modified_build_zig_contents = std.ArrayList(u8).init(arena); + try modified_build_zig_contents.ensureCapacity(build_zig_contents.len); + for (build_zig_contents) |c| { + if (c == '$') { + try modified_build_zig_contents.appendSlice(cwd_basename); + } else { + try modified_build_zig_contents.append(c); + } + } + const main_zig_contents = template_dir.readFileAlloc(arena, "src" ++ s ++ "main.zig", max_bytes) catch |err| { + fatal("unable to read template file 'main.zig': {}", .{@errorName(err)}); + }; + if (fs.cwd().access("build.zig", .{})) |_| { + fatal("existing build.zig file would be overwritten", .{}); + } else |err| switch (err) { + error.FileNotFound => {}, + else => fatal("unable to test existence of build.zig: {}\n", .{@errorName(err)}), + } + var src_dir = try fs.cwd().makeOpenPath("src", .{}); + defer src_dir.close(); + + try src_dir.writeFile("main.zig", main_zig_contents); + try fs.cwd().writeFile("build.zig", modified_build_zig_contents.items); + + std.log.info("Created build.zig", .{}); + std.log.info("Created src" ++ s ++ "main.zig", .{}); + + switch (output_mode) { + .Lib => std.log.info("Next, try `zig build --help` or `zig build test`", .{}), + .Exe => std.log.info("Next, try `zig build --help` or `zig build run`", .{}), + .Obj => unreachable, + } +} + pub const usage_fmt = \\Usage: zig fmt [file]... \\ From dacd36ca9b193444e9e2ca3f132fccf3e08fb4f9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 22 Sep 2020 14:06:57 -0700 Subject: [PATCH 090/210] stage2: implement using the global cache dir --- BRANCH_TODO | 13 ++++------ src/Compilation.zig | 31 ++++++++++++----------- src/glibc.zig | 21 +++++++++------- src/introspect.zig | 7 ------ src/libcxx.zig | 8 +++--- src/libunwind.zig | 4 +-- src/main.zig | 60 ++++++++++++++++++++++++++++++++++++++------- src/test.zig | 3 ++- 8 files changed, 93 insertions(+), 54 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index c84dc397bb..4a55c8192f 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -14,26 +14,23 @@ * -fno-emit-asm (default) do not output .s (assembly code)\n" * -femit-llvm-ir produce a .ll file with LLVM IR\n" * -fno-emit-llvm-ir (default) do not produce a .ll file with LLVM IR\n" - * --cache-dir [path] override the local cache directory - * --global-cache-dir [path] override the global cache directory - * implement proper parsing of LLD stderr/stdout and exposing compile errors - * implement proper parsing of clang stderr/stdout and exposing compile errors * support rpaths in ELF linker code * add CLI support for a way to pass extra flags to c source files * musl * mingw-w64 - * use global zig-cache dir for crt files - * use global zig-cache dir for `zig run` executables but not `zig test` * MachO LLD linking * COFF LLD linking * WASM LLD linking - * support cross compiling stage2 with `zig build` * --main-pkg-path * audit the CLI options for stage2 - * restore error messages for stage2_add_link_lib * audit the base cache hash + * implement proper parsing of LLD stderr/stdout and exposing compile errors + * implement proper parsing of clang stderr/stdout and exposing compile errors * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process. + * restore error messages for stage2_add_link_lib + * update zig build to use new CLI + * support cross compiling stage2 with `zig build` * implement proper compile errors for failing to build glibc crt files and shared libs * implement -fno-emit-bin * improve the stage2 tests to support testing with LLVM extensions enabled diff --git a/src/Compilation.zig b/src/Compilation.zig index 70492b70be..9d2d59b98a 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -64,7 +64,8 @@ cache_parent: *Cache, /// Path to own executable for invoking `zig clang`. self_exe_path: ?[]const u8, zig_lib_directory: Directory, -zig_cache_directory: Directory, +local_cache_directory: Directory, +global_cache_directory: Directory, libc_include_dir_list: []const []const u8, rand: *std.rand.Random, @@ -267,7 +268,8 @@ pub const EmitLoc = struct { pub const InitOptions = struct { zig_lib_directory: Directory, - zig_cache_directory: Directory, + local_cache_directory: Directory, + global_cache_directory: Directory, target: Target, root_name: []const u8, root_pkg: ?*Package, @@ -523,7 +525,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { const cache = try arena.create(Cache); cache.* = .{ .gpa = gpa, - .manifest_dir = try options.zig_cache_directory.handle.makeOpenPath("h", .{}), + .manifest_dir = try options.local_cache_directory.handle.makeOpenPath("h", .{}), }; errdefer cache.manifest_dir.close(); @@ -569,11 +571,11 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { const digest = hash.final(); const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); - var artifact_dir = try options.zig_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); + var artifact_dir = try options.local_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); errdefer artifact_dir.close(); const zig_cache_artifact_directory: Directory = .{ .handle = artifact_dir, - .path = if (options.zig_cache_directory.path) |p| + .path = if (options.local_cache_directory.path) |p| try std.fs.path.join(arena, &[_][]const u8{ p, artifact_sub_dir }) else artifact_sub_dir, @@ -647,11 +649,11 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { const digest = hash.final(); const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); - var artifact_dir = try options.zig_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); + var artifact_dir = try options.local_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); owned_link_dir = artifact_dir; const link_artifact_directory: Directory = .{ .handle = artifact_dir, - .path = try options.zig_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}), + .path = try options.local_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}), }; break :blk link_artifact_directory; }; @@ -715,7 +717,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .gpa = gpa, .arena_state = arena_allocator.state, .zig_lib_directory = options.zig_lib_directory, - .zig_cache_directory = options.zig_cache_directory, + .local_cache_directory = options.local_cache_directory, + .global_cache_directory = options.global_cache_directory, .bin_file = bin_file, .work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa), .keep_source_files_loaded = options.keep_source_files_loaded, @@ -1230,7 +1233,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { // We can't know the digest until we do the C compiler invocation, so we need a temporary filename. const out_obj_path = try comp.tmpFilePath(arena, o_basename); - var zig_cache_tmp_dir = try comp.zig_cache_directory.handle.makeOpenPath("tmp", .{}); + var zig_cache_tmp_dir = try comp.local_cache_directory.handle.makeOpenPath("tmp", .{}); defer zig_cache_tmp_dir.close(); try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang", "-c" }); @@ -1319,7 +1322,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { // Rename into place. const digest = ch.final(); const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); - var o_dir = try comp.zig_cache_directory.handle.makeOpenPath(o_sub_path, .{}); + var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{}); defer o_dir.close(); // TODO https://github.com/ziglang/zig/issues/6344 const tmp_basename = std.fs.path.basename(out_obj_path); @@ -1331,7 +1334,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { break :blk digest; }; - const components = if (comp.zig_cache_directory.path) |p| + const components = if (comp.local_cache_directory.path) |p| &[_][]const u8{ p, "o", &digest, o_basename } else &[_][]const u8{ "o", &digest, o_basename }; @@ -1347,7 +1350,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { fn tmpFilePath(comp: *Compilation, arena: *Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 { const s = std.fs.path.sep_str; const rand_int = comp.rand.int(u64); - if (comp.zig_cache_directory.path) |p| { + if (comp.local_cache_directory.path) |p| { return std.fmt.allocPrint(arena, "{}" ++ s ++ "tmp" ++ s ++ "{x}-{s}", .{ p, rand_int, suffix }); } else { return std.fmt.allocPrint(arena, "tmp" ++ s ++ "{x}-{s}", .{ rand_int, suffix }); @@ -2116,8 +2119,8 @@ fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFil } }; const sub_compilation = try Compilation.create(comp.gpa, .{ - // TODO use the global cache directory here - .zig_cache_directory = comp.zig_cache_directory, + .global_cache_directory = comp.global_cache_directory, + .local_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = comp.getTarget(), .root_name = mem.split(basename, ".").next().?, diff --git a/src/glibc.zig b/src/glibc.zig index 22f060f514..c7c02a4470 100644 --- a/src/glibc.zig +++ b/src/glibc.zig @@ -691,8 +691,8 @@ fn build_crt_file( .basename = basename, }; const sub_compilation = try Compilation.create(comp.gpa, .{ - // TODO use the global cache directory here - .zig_cache_directory = comp.zig_cache_directory, + .local_cache_directory = comp.global_cache_directory, + .global_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = comp.getTarget(), .root_name = mem.split(basename, ".").next().?, @@ -769,11 +769,13 @@ pub fn buildSharedObjects(comp: *Compilation) !void { const target = comp.getTarget(); const target_version = target.os.version_range.linux.glibc; - // TODO use the global cache directory here + // Use the global cache directory. var cache_parent: Cache = .{ .gpa = comp.gpa, - .manifest_dir = comp.cache_parent.manifest_dir, + .manifest_dir = try comp.global_cache_directory.handle.makeOpenPath("h", .{}), }; + defer cache_parent.manifest_dir.close(); + var cache = cache_parent.obtain(); defer cache.deinit(); cache.hash.addBytes(build_options.version); @@ -790,8 +792,8 @@ pub fn buildSharedObjects(comp: *Compilation) !void { // We use the presence of an "ok" file to determine if it is a true hit. var o_directory: Compilation.Directory = .{ - .handle = try comp.zig_cache_directory.handle.makeOpenPath(o_sub_path, .{}), - .path = try path.join(arena, &[_][]const u8{ comp.zig_cache_directory.path.?, o_sub_path }), + .handle = try comp.global_cache_directory.handle.makeOpenPath(o_sub_path, .{}), + .path = try path.join(arena, &[_][]const u8{ comp.global_cache_directory.path.?, o_sub_path }), }; defer o_directory.handle.close(); @@ -931,7 +933,7 @@ pub fn buildSharedObjects(comp: *Compilation) !void { const asm_file_basename = std.fmt.bufPrint(&lib_name_buf, "{s}.s", .{lib.name}) catch unreachable; try o_directory.handle.writeFile(asm_file_basename, zig_body.items); - try buildSharedLib(comp, arena, comp.zig_cache_directory, o_directory, asm_file_basename, lib); + try buildSharedLib(comp, arena, comp.global_cache_directory, o_directory, asm_file_basename, lib); } // No need to write the manifest because there are no file inputs associated with this cache hash. // However we do need to write the ok file now. @@ -945,7 +947,7 @@ pub fn buildSharedObjects(comp: *Compilation) !void { assert(comp.glibc_so_files == null); comp.glibc_so_files = BuiltSharedObjects{ .lock = cache.toOwnedLock(), - .dir_path = try path.join(comp.gpa, &[_][]const u8{ comp.zig_cache_directory.path.?, o_sub_path }), + .dir_path = try path.join(comp.gpa, &[_][]const u8{ comp.global_cache_directory.path.?, o_sub_path }), }; } @@ -976,7 +978,8 @@ fn buildSharedLib( }, }; const sub_compilation = try Compilation.create(comp.gpa, .{ - .zig_cache_directory = zig_cache_directory, + .local_cache_directory = zig_cache_directory, + .global_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = comp.getTarget(), .root_name = lib.name, diff --git a/src/introspect.zig b/src/introspect.zig index 067326ebb6..b75bf8f4b8 100644 --- a/src/introspect.zig +++ b/src/introspect.zig @@ -73,10 +73,3 @@ pub fn resolveGlobalCacheDir(allocator: *mem.Allocator) ![]u8 { return fs.getAppDataDir(allocator, appname); } - -pub fn openGlobalCacheDir() !fs.Dir { - var buf: [fs.MAX_PATH_BYTES]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buf); - const path_name = try resolveGlobalCacheDir(&fba.allocator); - return fs.cwd().makeOpenPath(path_name, .{}); -} diff --git a/src/libcxx.zig b/src/libcxx.zig index e6ed35c560..4fbca42875 100644 --- a/src/libcxx.zig +++ b/src/libcxx.zig @@ -146,8 +146,8 @@ pub fn buildLibCXX(comp: *Compilation) !void { } const sub_compilation = try Compilation.create(comp.gpa, .{ - // TODO use the global cache directory here - .zig_cache_directory = comp.zig_cache_directory, + .local_cache_directory = comp.global_cache_directory, + .global_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = target, .root_name = root_name, @@ -255,8 +255,8 @@ pub fn buildLibCXXABI(comp: *Compilation) !void { } const sub_compilation = try Compilation.create(comp.gpa, .{ - // TODO use the global cache directory here - .zig_cache_directory = comp.zig_cache_directory, + .local_cache_directory = comp.global_cache_directory, + .global_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = target, .root_name = root_name, diff --git a/src/libunwind.zig b/src/libunwind.zig index e7562670e4..28445b7284 100644 --- a/src/libunwind.zig +++ b/src/libunwind.zig @@ -86,8 +86,8 @@ pub fn buildStaticLib(comp: *Compilation) !void { }; } const sub_compilation = try Compilation.create(comp.gpa, .{ - // TODO use the global cache directory here - .zig_cache_directory = comp.zig_cache_directory, + .local_cache_directory = comp.global_cache_directory, + .global_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = target, .root_name = root_name, diff --git a/src/main.zig b/src/main.zig index 3faa8b62a1..e88343e124 100644 --- a/src/main.zig +++ b/src/main.zig @@ -193,6 +193,8 @@ const usage_build_generic = \\ -femit-bin[=path] (default) output machine code \\ -fno-emit-bin Do not output machine code \\ --show-builtin Output the source of @import("builtin") then exit + \\ --cache-dir [path] Override the local cache directory + \\ --global-cache-dir [path] Override the global cache directory \\ \\Compile Options: \\ -target [name] -- see the targets command @@ -353,6 +355,8 @@ pub fn buildOutputType( var runtime_args_start: ?usize = null; var test_filter: ?[]const u8 = null; var test_name_prefix: ?[]const u8 = null; + var override_local_cache_dir: ?[]const u8 = null; + var override_global_cache_dir: ?[]const u8 = null; var system_libs = std.ArrayList([]const u8).init(gpa); defer system_libs.deinit(); @@ -535,6 +539,14 @@ pub fn buildOutputType( if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; try test_exec_args.append(args[i]); + } else if (mem.eql(u8, arg, "--cache-dir")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + override_local_cache_dir = args[i]; + } else if (mem.eql(u8, arg, "--global-cache-dir")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + override_global_cache_dir = args[i]; } else if (mem.eql(u8, arg, "--test-cmd-bin")) { try test_exec_args.append(null); } else if (mem.eql(u8, arg, "--test-evented-io")) { @@ -1093,7 +1105,10 @@ pub fn buildOutputType( const emit_bin_loc: ?Compilation.EmitLoc = switch (emit_bin) { .no => null, .yes_default_path => Compilation.EmitLoc{ - .directory = .{ .path = null, .handle = fs.cwd() }, + .directory = switch (arg_mode) { + .run, .zig_test => null, + else => .{ .path = null, .handle = fs.cwd() }, + }, .basename = try std.zig.binNameAlloc( arena, root_name, @@ -1198,26 +1213,53 @@ pub fn buildOutputType( }; } - const cache_parent_dir = if (root_pkg) |pkg| pkg.root_src_directory.handle else fs.cwd(); - var cache_dir = try cache_parent_dir.makeOpenPath("zig-cache", .{}); - defer cache_dir.close(); - const zig_cache_directory: Compilation.Directory = .{ - .handle = cache_dir, - .path = blk: { + var global_cache_directory: Compilation.Directory = l: { + const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); + break :l .{ + .handle = try fs.cwd().makeOpenPath(p, .{}), + .path = p, + }; + }; + defer global_cache_directory.handle.close(); + + var cleanup_local_cache_dir: ?fs.Dir = null; + defer if (cleanup_local_cache_dir) |*dir| dir.close(); + + var local_cache_directory: Compilation.Directory = l: { + if (override_local_cache_dir) |local_cache_dir_path| { + const dir = try fs.cwd().makeOpenPath(local_cache_dir_path, .{}); + cleanup_local_cache_dir = dir; + break :l .{ + .handle = dir, + .path = local_cache_dir_path, + }; + } + if (arg_mode == .run) { + break :l global_cache_directory; + } + const cache_dir_path = blk: { if (root_pkg) |pkg| { if (pkg.root_src_directory.path) |p| { break :blk try fs.path.join(arena, &[_][]const u8{ p, "zig-cache" }); } } break :blk "zig-cache"; - }, + }; + const cache_parent_dir = if (root_pkg) |pkg| pkg.root_src_directory.handle else fs.cwd(); + const dir = try cache_parent_dir.makeOpenPath("zig-cache", .{}); + cleanup_local_cache_dir = dir; + break :l .{ + .handle = dir, + .path = cache_dir_path, + }; }; gimmeMoreOfThoseSweetSweetFileDescriptors(); const comp = Compilation.create(gpa, .{ .zig_lib_directory = zig_lib_directory, - .zig_cache_directory = zig_cache_directory, + .local_cache_directory = local_cache_directory, + .global_cache_directory = global_cache_directory, .root_name = root_name, .target = target_info.target, .is_native_os = cross_target.isNativeOs(), diff --git a/src/test.zig b/src/test.zig index 0f63c3f6a6..558fe3e95d 100644 --- a/src/test.zig +++ b/src/test.zig @@ -480,7 +480,8 @@ pub const TestContext = struct { .basename = bin_name, }; const comp = try Compilation.create(allocator, .{ - .zig_cache_directory = zig_cache_directory, + .local_cache_directory = zig_cache_directory, + .global_cache_directory = zig_cache_directory, .zig_lib_directory = zig_lib_directory, .rand = rand, .root_name = "test_case", From 250664bea41cc5dde66e3054ca0c1b6e0feab9be Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 22 Sep 2020 14:56:52 -0700 Subject: [PATCH 091/210] build runner: allow for a bit longer -D options so they can display in the --help menu without getting squished. --- lib/std/special/build_runner.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/special/build_runner.zig b/lib/std/special/build_runner.zig index 3ab74a11a2..3c4916a566 100644 --- a/lib/std/special/build_runner.zig +++ b/lib/std/special/build_runner.zig @@ -185,7 +185,7 @@ fn usage(builder: *Builder, already_ran_build: bool, out_stream: anytype) !void Builder.typeIdName(option.type_id), }); defer allocator.free(name); - try out_stream.print("{s:24} {}\n", .{ name, option.description }); + try out_stream.print("{s:32} {}\n", .{ name, option.description }); } } From c2b1cd7c456e274c7ead96e597d147e2df7d317e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 22 Sep 2020 22:18:19 -0700 Subject: [PATCH 092/210] stage2: implement zig build As part of this: * add std.process.cleanExit. closes #6395 - use it in several places * adjust the alignment of text in `zig build --help` menu * Cache: support the concept of "unhit" so that we properly keep track of the cache when we find out using the secondary hash that the cache "hit" was actually a miss. Use this to fix false negatives of caching of stage1 build artifacts. * fix not deleting the symlink hash for stage1 build artifacts causing false positives. * implement support for Package arguments in stage1 build artifacts * update and add missing usage text * add --override-lib-dir and --enable-cache CLI options - `--enable-cache` takes the place of `--cache on` * CLI supports -femit-bin=foo combined with --enable-cache to do an "update file" operation. --enable-cache without that argument will build the output into a cache directory and then print the path to stdout (matching master branch behavior). * errors surfacing from main() now print "error: Foo" instead of "error: error.Foo". --- BRANCH_TODO | 8 +- lib/std/build.zig | 3 +- lib/std/process.zig | 13 ++ lib/std/special/build_runner.zig | 12 +- src/Cache.zig | 37 ++-- src/Compilation.zig | 60 +++++- src/main.zig | 358 ++++++++++++++++++++++++++++--- src/stage1.zig | 4 +- 8 files changed, 431 insertions(+), 64 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 4a55c8192f..fd5cafc737 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,6 +1,3 @@ - * skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd) - (maybe make it an explicit option and have main.zig disable it) - * `zig build` * repair @cImport * make sure zig cc works - using it as a preprocessor (-E) @@ -22,13 +19,16 @@ * COFF LLD linking * WASM LLD linking * --main-pkg-path + * --pkg-begin, --pkg-end + * skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd) + (maybe make it an explicit option and have main.zig disable it) * audit the CLI options for stage2 * audit the base cache hash * implement proper parsing of LLD stderr/stdout and exposing compile errors * implement proper parsing of clang stderr/stdout and exposing compile errors * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process. * restore error messages for stage2_add_link_lib - * update zig build to use new CLI + * update std/build.zig to use new CLI * support cross compiling stage2 with `zig build` * implement proper compile errors for failing to build glibc crt files and shared libs diff --git a/lib/std/build.zig b/lib/std/build.zig index 69f44bad32..6f8c44ccd3 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -2294,8 +2294,7 @@ pub const LibExeObjStep = struct { if (self.kind == Kind.Test) { try builder.spawnChild(zig_args.span()); } else { - try zig_args.append("--cache"); - try zig_args.append("on"); + try zig_args.append("--enable-cache"); const output_dir_nl = try builder.execFromStep(zig_args.span(), &self.step); const build_output_dir = mem.trimRight(u8, output_dir_nl, "\r\n"); diff --git a/lib/std/process.zig b/lib/std/process.zig index 2813d8cbab..5303ada94b 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -19,6 +19,19 @@ pub const exit = os.exit; pub const changeCurDir = os.chdir; pub const changeCurDirC = os.chdirC; +/// Indicate that we are now terminating with a successful exit code. +/// In debug builds, this is a no-op, so that the calling code's +/// cleanup mechanisms are tested and so that external tools that +/// check for resource leaks can be accurate. In release builds, this +/// calls exit(0), and does not return. +pub fn cleanExit() void { + if (builtin.mode == .Debug) { + return; + } else { + exit(0); + } +} + /// The result is a slice of `out_buffer`, from index `0`. pub fn getCwd(out_buffer: []u8) ![]u8 { return os.getcwd(out_buffer); diff --git a/lib/std/special/build_runner.zig b/lib/std/special/build_runner.zig index 3c4916a566..43d6b96536 100644 --- a/lib/std/special/build_runner.zig +++ b/lib/std/special/build_runner.zig @@ -161,16 +161,16 @@ fn usage(builder: *Builder, already_ran_build: bool, out_stream: anytype) !void try fmt.allocPrint(allocator, "{} (default)", .{top_level_step.step.name}) else top_level_step.step.name; - try out_stream.print(" {s:22} {}\n", .{ name, top_level_step.description }); + try out_stream.print(" {s:<27} {}\n", .{ name, top_level_step.description }); } try out_stream.writeAll( \\ \\General Options: - \\ --help Print this help and exit - \\ --verbose Print commands before executing them - \\ --prefix [path] Override default install prefix - \\ --search-prefix [path] Add a path to look for binaries, libraries, headers + \\ --help Print this help and exit + \\ --verbose Print commands before executing them + \\ --prefix [path] Override default install prefix + \\ --search-prefix [path] Add a path to look for binaries, libraries, headers \\ \\Project-Specific Options: \\ @@ -185,7 +185,7 @@ fn usage(builder: *Builder, already_ran_build: bool, out_stream: anytype) !void Builder.typeIdName(option.type_id), }); defer allocator.free(name); - try out_stream.print("{s:32} {}\n", .{ name, option.description }); + try out_stream.print("{s:<29} {}\n", .{ name, option.description }); } } diff --git a/src/Cache.zig b/src/Cache.zig index 24c6ae3ac4..7a0c78a1d9 100644 --- a/src/Cache.zig +++ b/src/Cache.zig @@ -120,6 +120,13 @@ pub const HashHelper = struct { return copy.final(); } + pub fn peekBin(hh: HashHelper) [bin_digest_len]u8 { + var copy = hh; + var bin_digest: [bin_digest_len]u8 = undefined; + copy.hasher.final(&bin_digest); + return bin_digest; + } + /// Returns a hex encoded hash of the inputs, mutating the state of the hasher. pub fn final(hh: *HashHelper) [hex_digest_len]u8 { var bin_digest: [bin_digest_len]u8 = undefined; @@ -338,19 +345,7 @@ pub const CacheHash = struct { if (any_file_changed) { // cache miss // keep the manifest file open - // reset the hash - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); - - // Remove files not in the initial hash - for (self.files.items[input_file_count..]) |*file| { - file.deinit(self.cache.gpa); - } - self.files.shrinkRetainingCapacity(input_file_count); - - for (self.files.items) |file| { - self.hash.hasher.update(&file.bin_digest); - } + self.unhit(bin_digest, input_file_count); return false; } @@ -366,6 +361,22 @@ pub const CacheHash = struct { return true; } + pub fn unhit(self: *CacheHash, bin_digest: [bin_digest_len]u8, input_file_count: usize) void { + // Reset the hash. + self.hash.hasher = hasher_init; + self.hash.hasher.update(&bin_digest); + + // Remove files not in the initial hash. + for (self.files.items[input_file_count..]) |*file| { + file.deinit(self.cache.gpa); + } + self.files.shrinkRetainingCapacity(input_file_count); + + for (self.files.items) |file| { + self.hash.hasher.update(&file.bin_digest); + } + } + fn populateFileHash(self: *CacheHash, ch_file: *File) !void { const file = try fs.cwd().openFile(ch_file.path.?, .{}); defer file.close(); diff --git a/src/Compilation.zig b/src/Compilation.zig index 9d2d59b98a..d0f2e36080 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2200,20 +2200,34 @@ fn updateStage1Module(comp: *Compilation) !void { ch.hash.add(comp.bin_file.options.function_sections); ch.hash.add(comp.is_test); + // Capture the state in case we come back from this branch where the hash doesn't match. + const prev_hash_state = ch.hash.peekBin(); + const input_file_count = ch.files.items.len; + if (try ch.hit()) { const digest = ch.final(); var prev_digest_buf: [digest.len]u8 = undefined; const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { + log.debug("stage1 {} new_digest={} readlink error: {}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; if (mem.eql(u8, prev_digest, &digest)) { + log.debug("stage1 {} digest={} match - skipping invocation", .{ mod.root_pkg.root_src_path, digest }); comp.stage1_lock = ch.toOwnedLock(); return; } + log.debug("stage1 {} prev_digest={} new_digest={}", .{ mod.root_pkg.root_src_path, prev_digest, digest }); + ch.unhit(prev_hash_state, input_file_count); } + // We are about to change the output file to be different, so we invalidate the build hash now. + directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) { + error.FileNotFound => {}, + else => |e| return e, + }; + const stage2_target = try arena.create(stage1.Stage2Target); stage2_target.* = .{ .arch = @enumToInt(target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch @@ -2243,16 +2257,7 @@ fn updateStage1Module(comp: *Compilation) !void { comp.is_test, ) orelse return error.OutOfMemory; - const stage1_pkg = try arena.create(stage1.Pkg); - stage1_pkg.* = .{ - .name_ptr = undefined, - .name_len = 0, - .path_ptr = undefined, - .path_len = 0, - .children_ptr = undefined, - .children_len = 0, - .parent = null, - }; + const stage1_pkg = try createStage1Pkg(arena, "root", mod.root_pkg, null); const output_dir = comp.bin_file.options.directory.path orelse "."; const test_filter = comp.test_filter orelse ""[0..0]; const test_name_prefix = comp.test_name_prefix orelse ""[0..0]; @@ -2303,10 +2308,12 @@ fn updateStage1Module(comp: *Compilation) !void { const digest = ch.final(); + log.debug("stage1 {} final digest={}", .{ mod.root_pkg.root_src_path, digest }); + // Update the dangling symlink with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { - std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); + std.log.warn("failed to save stage1 hash digest symlink: {}", .{@errorName(err)}); }; // Again failure here only means an unnecessary cache miss. ch.writeManifest() catch |err| { @@ -2316,3 +2323,34 @@ fn updateStage1Module(comp: *Compilation) !void { // other processes clobbering it. comp.stage1_lock = ch.toOwnedLock(); } + +fn createStage1Pkg( + arena: *Allocator, + name: []const u8, + pkg: *Package, + parent_pkg: ?*stage1.Pkg, +) error{OutOfMemory}!*stage1.Pkg { + const child_pkg = try arena.create(stage1.Pkg); + + const pkg_children = blk: { + var children = std.ArrayList(*stage1.Pkg).init(arena); + var it = pkg.table.iterator(); + while (it.next()) |entry| { + try children.append(try createStage1Pkg(arena, entry.key, entry.value, child_pkg)); + } + break :blk children.items; + }; + + const src_path = try pkg.root_src_directory.join(arena, &[_][]const u8{pkg.root_src_path}); + + child_pkg.* = .{ + .name_ptr = name.ptr, + .name_len = name.len, + .path_ptr = src_path.ptr, + .path_len = src_path.len, + .children_ptr = pkg_children.ptr, + .children_len = pkg_children.len, + .parent = parent_pkg, + }; + return child_pkg; +} diff --git a/src/main.zig b/src/main.zig index e88343e124..79fd681120 100644 --- a/src/main.zig +++ b/src/main.zig @@ -35,6 +35,7 @@ const usage = \\ \\Commands: \\ + \\ build Build project from build.zig \\ build-exe Create executable from source or object files \\ build-lib Create library from source or object files \\ build-obj Create object from source or assembly @@ -42,6 +43,8 @@ const usage = \\ c++ Use Zig as a drop-in C++ compiler \\ env Print lib path, std path, compiler id and version \\ fmt Parse file and render in canonical zig format + \\ init-exe Initialize a `zig build` application in the cwd + \\ init-lib Initialize a `zig build` library in the cwd \\ libc Display native libc paths file or validate one \\ run Create executable and run immediately \\ translate-c Convert C code to Zig code @@ -136,6 +139,8 @@ pub fn mainArgs(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v mem.eql(u8, cmd, "-cc1") or mem.eql(u8, cmd, "-cc1as")) { return punt_to_clang(arena, args); + } else if (mem.eql(u8, cmd, "build")) { + return cmdBuild(gpa, arena, cmd_args); } else if (mem.eql(u8, cmd, "fmt")) { return cmdFmt(gpa, cmd_args); } else if (mem.eql(u8, cmd, "libc")) { @@ -172,18 +177,18 @@ const usage_build_generic = \\Supported file types: \\ .zig Zig source code \\ .zir Zig Intermediate Representation code - \\ (planned) .o ELF object file - \\ (planned) .o MACH-O (macOS) object file - \\ (planned) .obj COFF (Windows) object file - \\ (planned) .lib COFF (Windows) static library - \\ (planned) .a ELF static library - \\ (planned) .so ELF shared object (dynamic link) - \\ (planned) .dll Windows Dynamic Link Library - \\ (planned) .dylib MACH-O (macOS) dynamic library - \\ (planned) .s Target-specific assembly source code - \\ (planned) .S Assembly with C preprocessor (requires LLVM extensions) - \\ (planned) .c C source code (requires LLVM extensions) - \\ (planned) .cpp C++ source code (requires LLVM extensions) + \\ .o ELF object file + \\ .o MACH-O (macOS) object file + \\ .obj COFF (Windows) object file + \\ .lib COFF (Windows) static library + \\ .a ELF static library + \\ .so ELF shared object (dynamic link) + \\ .dll Windows Dynamic Link Library + \\ .dylib MACH-O (macOS) dynamic library + \\ .s Target-specific assembly source code + \\ .S Assembly with C preprocessor (requires LLVM extensions) + \\ .c C source code (requires LLVM extensions) + \\ .cpp C++ source code (requires LLVM extensions) \\ Other C++ extensions: .C .cc .cxx \\ \\General Options: @@ -195,6 +200,8 @@ const usage_build_generic = \\ --show-builtin Output the source of @import("builtin") then exit \\ --cache-dir [path] Override the local cache directory \\ --global-cache-dir [path] Override the global cache directory + \\ --override-lib-dir [path] Override path to Zig installation lib directory + \\ --enable-cache Output to cache directory; print path to stdout \\ \\Compile Options: \\ -target [name] -- see the targets command @@ -357,6 +364,7 @@ pub fn buildOutputType( var test_name_prefix: ?[]const u8 = null; var override_local_cache_dir: ?[]const u8 = null; var override_global_cache_dir: ?[]const u8 = null; + var override_lib_dir: ?[]const u8 = null; var system_libs = std.ArrayList([]const u8).init(gpa); defer system_libs.deinit(); @@ -412,7 +420,7 @@ pub fn buildOutputType( if (mem.startsWith(u8, arg, "-")) { if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { try io.getStdOut().writeAll(usage_build_generic); - process.exit(0); + return process.cleanExit(); } else if (mem.eql(u8, arg, "--")) { if (arg_mode == .run) { runtime_args_start = i + 1; @@ -547,6 +555,12 @@ pub fn buildOutputType( if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; override_global_cache_dir = args[i]; + } else if (mem.eql(u8, arg, "--override-lib-dir")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + override_lib_dir = args[i]; + } else if (mem.eql(u8, arg, "--enable-cache")) { + enable_cache = true; } else if (mem.eql(u8, arg, "--test-cmd-bin")) { try test_exec_args.append(null); } else if (mem.eql(u8, arg, "--test-evented-io")) { @@ -1102,12 +1116,22 @@ pub fn buildOutputType( var cleanup_emit_bin_dir: ?fs.Dir = null; defer if (cleanup_emit_bin_dir) |*dir| dir.close(); + const have_enable_cache = enable_cache orelse false; + const emit_bin_loc: ?Compilation.EmitLoc = switch (emit_bin) { .no => null, .yes_default_path => Compilation.EmitLoc{ - .directory = switch (arg_mode) { - .run, .zig_test => null, - else => .{ .path = null, .handle = fs.cwd() }, + .directory = blk: { + switch (arg_mode) { + .run, .zig_test => break :blk null, + else => { + if (have_enable_cache) { + break :blk null; + } else { + break :blk .{ .path = null, .handle = fs.cwd() }; + } + }, + } }, .basename = try std.zig.binNameAlloc( arena, @@ -1120,6 +1144,12 @@ pub fn buildOutputType( }, .yes => |full_path| b: { const basename = fs.path.basename(full_path); + if (have_enable_cache) { + break :b Compilation.EmitLoc{ + .basename = basename, + .directory = null, + }; + } if (fs.path.dirname(full_path)) |dirname| { const handle = try fs.cwd().openDir(dirname, .{}); cleanup_emit_bin_dir = handle; @@ -1192,9 +1222,15 @@ pub fn buildOutputType( } else null; const self_exe_path = try fs.selfExePathAlloc(arena); - var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| { - fatal("unable to find zig installation directory: {}\n", .{@errorName(err)}); - }; + var zig_lib_directory: Compilation.Directory = if (override_lib_dir) |lib_dir| + .{ + .path = lib_dir, + .handle = try fs.cwd().openDir(lib_dir, .{}), + } + else + introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| { + fatal("unable to find zig installation directory: {}", .{@errorName(err)}); + }; defer zig_lib_directory.handle.close(); const random_seed = blk: { @@ -1337,7 +1373,20 @@ pub fn buildOutputType( return cmdTranslateC(comp, arena); } - try updateModule(gpa, comp, zir_out_path); + const hook: AfterUpdateHook = blk: { + if (!have_enable_cache) + break :blk .none; + + switch (emit_bin) { + .no => break :blk .none, + .yes_default_path => break :blk .{ + .print = comp.bin_file.options.directory.path orelse ".", + }, + .yes => |full_path| break :blk .{ .update = full_path }, + } + }; + + try updateModule(gpa, comp, zir_out_path, hook); if (build_options.have_llvm and only_pp_or_asm) { // this may include dumping the output to stdout @@ -1389,7 +1438,7 @@ pub fn buildOutputType( else => process.exit(1), } if (!watch) - process.exit(0); + return process.cleanExit(); }, else => {}, } @@ -1413,7 +1462,7 @@ pub fn buildOutputType( if (output_mode == .Exe) { try comp.makeBinFileWritable(); } - try updateModule(gpa, comp, zir_out_path); + try updateModule(gpa, comp, zir_out_path, hook); } else if (mem.eql(u8, actual_line, "exit")) { break; } else if (mem.eql(u8, actual_line, "help")) { @@ -1427,7 +1476,13 @@ pub fn buildOutputType( } } -fn updateModule(gpa: *Allocator, comp: *Compilation, zir_out_path: ?[]const u8) !void { +const AfterUpdateHook = union(enum) { + none, + print: []const u8, + update: []const u8, +}; + +fn updateModule(gpa: *Allocator, comp: *Compilation, zir_out_path: ?[]const u8, hook: AfterUpdateHook) !void { try comp.update(); var errors = try comp.getAllErrorsAlloc(); @@ -1437,6 +1492,15 @@ fn updateModule(gpa: *Allocator, comp: *Compilation, zir_out_path: ?[]const u8) for (errors.list) |full_err_msg| { full_err_msg.renderToStdErr(); } + } else switch (hook) { + .none => {}, + .print => |bin_path| try io.getStdOut().writer().print("{s}\n", .{bin_path}), + .update => |full_path| _ = try comp.bin_file.options.directory.handle.updateFile( + comp.bin_file.options.sub_path, + fs.cwd(), + full_path, + .{}, + ), } if (zir_out_path) |zop| { @@ -1535,7 +1599,7 @@ pub fn cmdLibC(gpa: *Allocator, args: []const []const u8) !void { if (mem.eql(u8, arg, "--help")) { const stdout = io.getStdOut().writer(); try stdout.writeAll(usage_libc); - process.exit(0); + return process.cleanExit(); } else { fatal("unrecognized parameter: '{}'", .{arg}); } @@ -1592,7 +1656,7 @@ pub fn cmdInit( if (mem.startsWith(u8, arg, "-")) { if (mem.eql(u8, arg, "--help")) { try io.getStdOut().writeAll(usage_init); - process.exit(0); + return process.cleanExit(); } else { fatal("unrecognized parameter: '{}'", .{arg}); } @@ -1657,6 +1721,248 @@ pub fn cmdInit( } } +pub const usage_build = + \\Usage: zig build [steps] [options] + \\ + \\ Build a project from build.zig. + \\ + \\Options: + \\ --help Print this help and exit + \\ + \\ +; + +pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !void { + // We want to release all the locks before executing the child process, so we make a nice + // big block here to ensure the cleanup gets run when we extract out our argv. + const lock_and_argv = lock_and_argv: { + const self_exe_path = try fs.selfExePathAlloc(arena); + + var build_file: ?[]const u8 = null; + var override_lib_dir: ?[]const u8 = null; + var override_global_cache_dir: ?[]const u8 = null; + var override_local_cache_dir: ?[]const u8 = null; + var child_argv = std.ArrayList([]const u8).init(arena); + + const argv_index_exe = child_argv.items.len; + _ = try child_argv.addOne(); + + try child_argv.append(self_exe_path); + + const argv_index_build_file = child_argv.items.len; + _ = try child_argv.addOne(); + + const argv_index_cache_dir = child_argv.items.len; + _ = try child_argv.addOne(); + + { + var i: usize = 0; + while (i < args.len) : (i += 1) { + const arg = args[i]; + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "--build-file")) { + if (i + 1 >= args.len) fatal("expected argument after '{}'", .{arg}); + i += 1; + build_file = args[i]; + continue; + } else if (mem.eql(u8, arg, "--override-lib-dir")) { + if (i + 1 >= args.len) fatal("expected argument after '{}'", .{arg}); + i += 1; + override_lib_dir = args[i]; + try child_argv.appendSlice(&[_][]const u8{ arg, args[i] }); + continue; + } else if (mem.eql(u8, arg, "--cache-dir")) { + if (i + 1 >= args.len) fatal("expected argument after '{}'", .{arg}); + i += 1; + override_local_cache_dir = args[i]; + try child_argv.appendSlice(&[_][]const u8{ arg, args[i] }); + continue; + } else if (mem.eql(u8, arg, "--global-cache-dir")) { + if (i + 1 >= args.len) fatal("expected argument after '{}'", .{arg}); + i += 1; + override_global_cache_dir = args[i]; + try child_argv.appendSlice(&[_][]const u8{ arg, args[i] }); + continue; + } + } + try child_argv.append(arg); + } + } + + var zig_lib_directory: Compilation.Directory = if (override_lib_dir) |lib_dir| + .{ + .path = lib_dir, + .handle = try fs.cwd().openDir(lib_dir, .{}), + } + else + introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| { + fatal("unable to find zig installation directory: {}", .{@errorName(err)}); + }; + defer zig_lib_directory.handle.close(); + + const std_special = "std" ++ fs.path.sep_str ++ "special"; + const special_dir_path = try zig_lib_directory.join(arena, &[_][]const u8{std_special}); + + var root_pkg: Package = .{ + .root_src_directory = .{ + .path = special_dir_path, + .handle = try zig_lib_directory.handle.openDir(std_special, .{}), + }, + .root_src_path = "build_runner.zig", + }; + defer root_pkg.root_src_directory.handle.close(); + + var cleanup_build_dir: ?fs.Dir = null; + defer if (cleanup_build_dir) |*dir| dir.close(); + + const cwd_path = try process.getCwdAlloc(arena); + const build_zig_basename = if (build_file) |bf| fs.path.basename(bf) else "build.zig"; + const build_directory: Compilation.Directory = blk: { + if (build_file) |bf| { + if (fs.path.dirname(bf)) |dirname| { + const dir = try fs.cwd().openDir(dirname, .{}); + cleanup_build_dir = dir; + break :blk .{ .path = dirname, .handle = dir }; + } + + break :blk .{ .path = null, .handle = fs.cwd() }; + } + // Search up parent directories until we find build.zig. + var dirname: []const u8 = cwd_path; + while (true) { + const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig_basename }); + if (fs.cwd().access(joined_path, .{})) |_| { + const dir = try fs.cwd().openDir(dirname, .{}); + break :blk .{ .path = dirname, .handle = dir }; + } else |err| switch (err) { + error.FileNotFound => { + dirname = fs.path.dirname(dirname) orelse { + std.log.info("{}", .{ + \\Initialize a 'build.zig' template file with `zig init-lib` or `zig init-exe`, + \\or see `zig --help` for more options. + }); + fatal("No 'build.zig' file found, in the current directory or any parent directories.", .{}); + }; + continue; + }, + else => |e| return e, + } + } + }; + child_argv.items[argv_index_build_file] = build_directory.path orelse cwd_path; + + var build_pkg: Package = .{ + .root_src_directory = build_directory, + .root_src_path = build_zig_basename, + }; + try root_pkg.table.put(arena, "@build", &build_pkg); + + var global_cache_directory: Compilation.Directory = l: { + const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); + break :l .{ + .handle = try fs.cwd().makeOpenPath(p, .{}), + .path = p, + }; + }; + defer global_cache_directory.handle.close(); + + var local_cache_directory: Compilation.Directory = l: { + if (override_local_cache_dir) |local_cache_dir_path| { + break :l .{ + .handle = try fs.cwd().makeOpenPath(local_cache_dir_path, .{}), + .path = local_cache_dir_path, + }; + } + const cache_dir_path = try build_directory.join(arena, &[_][]const u8{"zig-cache"}); + break :l .{ + .handle = try build_directory.handle.makeOpenPath("zig-cache", .{}), + .path = cache_dir_path, + }; + }; + defer local_cache_directory.handle.close(); + + child_argv.items[argv_index_cache_dir] = local_cache_directory.path orelse cwd_path; + + gimmeMoreOfThoseSweetSweetFileDescriptors(); + + const cross_target: std.zig.CrossTarget = .{}; + const target_info = try detectNativeTargetInfo(gpa, cross_target); + + const exe_basename = try std.zig.binNameAlloc(arena, "build", target_info.target, .Exe, null, null); + const emit_bin: Compilation.EmitLoc = .{ + .directory = null, // Use the local zig-cache. + .basename = exe_basename, + }; + const random_seed = blk: { + var random_seed: u64 = undefined; + try std.crypto.randomBytes(mem.asBytes(&random_seed)); + break :blk random_seed; + }; + var default_prng = std.rand.DefaultPrng.init(random_seed); + const comp = Compilation.create(gpa, .{ + .zig_lib_directory = zig_lib_directory, + .local_cache_directory = local_cache_directory, + .global_cache_directory = global_cache_directory, + .root_name = "build", + .target = target_info.target, + .is_native_os = cross_target.isNativeOs(), + .dynamic_linker = target_info.dynamic_linker.get(), + .output_mode = .Exe, + .root_pkg = &root_pkg, + .emit_bin = emit_bin, + .emit_h = null, + .optimize_mode = .Debug, + .self_exe_path = self_exe_path, + .rand = &default_prng.random, + }) catch |err| { + fatal("unable to create compilation: {}", .{@errorName(err)}); + }; + defer comp.destroy(); + + try updateModule(gpa, comp, null, .none); + + child_argv.items[argv_index_exe] = try comp.bin_file.options.directory.join(arena, &[_][]const u8{exe_basename}); + + break :lock_and_argv .{ + .child_argv = child_argv.items, + .lock = comp.bin_file.toOwnedLock(), + }; + }; + const child_argv = lock_and_argv.child_argv; + var lock = lock_and_argv.lock; + defer lock.release(); + + const child = try std.ChildProcess.init(child_argv, gpa); + defer child.deinit(); + + child.stdin_behavior = .Inherit; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; + + var cmd = std.ArrayList(u8).init(arena); + + const term = try child.spawnAndWait(); + switch (term) { + .Exited => |code| { + if (code == 0) return process.cleanExit(); + try cmd.writer().print("failed with exit code {}:\n", .{code}); + }, + else => { + try cmd.appendSlice("crashed:\n"); + }, + } + + try cmd.append('\n'); + for (child_argv[0 .. child_argv.len - 1]) |arg| { + try cmd.appendSlice(arg); + try cmd.append(' '); + } + try cmd.appendSlice(child_argv[child_argv.len - 1]); + + if (true) // Working around erroneous stage1 compile error: unreachable code on child.deinit() + fatal("The following build command {}", .{cmd.items}); +} + pub const usage_fmt = \\Usage: zig fmt [file]... \\ @@ -1699,7 +2005,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { if (mem.eql(u8, arg, "--help")) { const stdout = io.getStdOut().outStream(); try stdout.writeAll(usage_fmt); - process.exit(0); + return process.cleanExit(); } else if (mem.eql(u8, arg, "--color")) { if (i + 1 >= args.len) { fatal("expected [auto|on|off] after --color", .{}); diff --git a/src/stage1.zig b/src/stage1.zig index 1ff7b4cf4c..380d3cbd9f 100644 --- a/src/stage1.zig +++ b/src/stage1.zig @@ -31,11 +31,11 @@ pub export fn main(argc: c_int, argv: [*]const [*:0]const u8) c_int { defer arena_instance.deinit(); const arena = &arena_instance.allocator; - const args = arena.alloc([]const u8, @intCast(usize, argc)) catch fatal("out of memory", .{}); + const args = arena.alloc([]const u8, @intCast(usize, argc)) catch fatal("{}", .{"OutOfMemory"}); for (args) |*arg, i| { arg.* = mem.spanZ(argv[i]); } - stage2.mainArgs(gpa, arena, args) catch |err| fatal("{}", .{err}); + stage2.mainArgs(gpa, arena, args) catch |err| fatal("{}", .{@errorName(err)}); return 0; } From 0638a020cf783486724d1999a4847140c6cc1442 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 22 Sep 2020 23:00:33 -0700 Subject: [PATCH 093/210] stage2: implement --pkg-begin and --pkg-end CLI args --- BRANCH_TODO | 3 ++- lib/std/build.zig | 4 ---- src/Package.zig | 1 + src/main.zig | 43 ++++++++++++++++++++++++++++++++++++++----- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index fd5cafc737..58cf974cb5 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,3 +1,4 @@ + * separate libzigcpp.a and libzigstage1.a so that we can do non-stage1 builds * repair @cImport * make sure zig cc works - using it as a preprocessor (-E) @@ -19,7 +20,6 @@ * COFF LLD linking * WASM LLD linking * --main-pkg-path - * --pkg-begin, --pkg-end * skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd) (maybe make it an explicit option and have main.zig disable it) * audit the CLI options for stage2 @@ -60,3 +60,4 @@ in builtin.zig * rename std.builtin.Mode to std.builtin.OptimizeMode * implement `zig run` and `zig test` when combined with `--watch` + * close the --pkg-begin --pkg-end Package directory handles diff --git a/lib/std/build.zig b/lib/std/build.zig index 6f8c44ccd3..133919afc1 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -1950,7 +1950,6 @@ pub const LibExeObjStep = struct { for (self.link_objects.span()) |link_object| { switch (link_object) { .StaticPath => |static_path| { - try zig_args.append("--object"); try zig_args.append(builder.pathFromRoot(static_path)); }, @@ -1958,12 +1957,10 @@ pub const LibExeObjStep = struct { .Exe => unreachable, .Test => unreachable, .Obj => { - try zig_args.append("--object"); try zig_args.append(other.getOutputPath()); }, .Lib => { if (!other.is_dynamic or self.target.isWindows()) { - try zig_args.append("--object"); try zig_args.append(other.getOutputLibPath()); } else { const full_path_lib = other.getOutputPath(); @@ -1982,7 +1979,6 @@ pub const LibExeObjStep = struct { try zig_args.append(name); }, .AssemblyFile => |asm_file| { - try zig_args.append("--c-source"); try zig_args.append(asm_file.getPath(builder)); }, .CSourceFile => |c_source_file| { diff --git a/src/Package.zig b/src/Package.zig index 14be8b64d6..8a0e89f883 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -4,6 +4,7 @@ root_src_directory: Compilation.Directory, /// Relative to `root_src_directory`. May contain path separators. root_src_path: []const u8, table: Table = .{}, +parent: ?*Package = null, const std = @import("std"); const mem = std.mem; diff --git a/src/main.zig b/src/main.zig index 79fd681120..87ddbed343 100644 --- a/src/main.zig +++ b/src/main.zig @@ -215,6 +215,8 @@ const usage_build_generic = \\ ReleaseFast Optimizations on, safety off \\ ReleaseSafe Optimizations on, safety on \\ ReleaseSmall Optimize for small binary, safety off + \\ --pkg-begin [name] [path] Make pkg available to import and push current pkg + \\ --pkg-end Pop current pkg \\ -fPIC Force-enable Position Independent Code \\ -fno-PIC Force-disable Position Independent Code \\ -fstack-check Enable stack probing in unsafe builds @@ -397,6 +399,13 @@ pub fn buildOutputType( var test_exec_args = std.ArrayList(?[]const u8).init(gpa); defer test_exec_args.deinit(); + var root_pkg_memory: Package = .{ + .root_src_directory = undefined, + .root_src_path = undefined, + }; + defer root_pkg_memory.table.deinit(gpa); + var cur_pkg: *Package = &root_pkg_memory; + switch (arg_mode) { .build, .translate_c, .zig_test, .run => { output_mode = switch (arg_mode) { @@ -427,6 +436,33 @@ pub fn buildOutputType( } else { fatal("unexpected end-of-parameter mark: --", .{}); } + } else if (mem.eql(u8, arg, "--pkg-begin")) { + if (i + 2 >= args.len) fatal("Expected 2 arguments after {}", .{arg}); + i += 1; + const pkg_name = args[i]; + i += 1; + const pkg_path = args[i]; + + const new_cur_pkg = try arena.create(Package); + new_cur_pkg.* = .{ + .root_src_directory = if (fs.path.dirname(pkg_path)) |dirname| + .{ + .path = dirname, + .handle = try fs.cwd().openDir(dirname, .{}), // TODO close this fd + } + else + .{ + .path = null, + .handle = fs.cwd(), + }, + .root_src_path = fs.path.basename(pkg_path), + .parent = cur_pkg, + }; + try cur_pkg.table.put(gpa, pkg_name, new_cur_pkg); + cur_pkg = new_cur_pkg; + } else if (mem.eql(u8, arg, "--pkg-end")) { + cur_pkg = cur_pkg.parent orelse + fatal("encountered --pkg-end with no matching --pkg-begin", .{}); } else if (mem.eql(u8, arg, "--color")) { if (i + 1 >= args.len) { fatal("expected [auto|on|off] after --color", .{}); @@ -1212,12 +1248,9 @@ pub fn buildOutputType( .yes => |p| p, }; - var root_pkg_memory: Package = undefined; const root_pkg: ?*Package = if (root_src_file) |src_path| blk: { - root_pkg_memory = .{ - .root_src_directory = .{ .path = null, .handle = fs.cwd() }, - .root_src_path = src_path, - }; + root_pkg_memory.root_src_directory = .{ .path = null, .handle = fs.cwd() }; + root_pkg_memory.root_src_path = src_path; break :blk &root_pkg_memory; } else null; From fc88d36daea40b69690186730ae7f2a5296585a9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 22 Sep 2020 23:31:32 -0700 Subject: [PATCH 094/210] stage2: link_libc=true on OS's that require it for syscalls --- BRANCH_TODO | 1 + src/Compilation.zig | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 58cf974cb5..1e0efa9a1e 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,3 +1,4 @@ + * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj] * separate libzigcpp.a and libzigstage1.a so that we can do non-stage1 builds * repair @cImport * make sure zig cc works diff --git a/src/Compilation.zig b/src/Compilation.zig index d0f2e36080..f7ffc762aa 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -417,12 +417,14 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { return error.MachineCodeModelNotSupported; } + const link_libc = options.link_libc or target_util.osRequiresLibC(options.target); + const must_dynamic_link = dl: { if (target_util.cannotDynamicLink(options.target)) break :dl false; if (target_util.osRequiresLibC(options.target)) break :dl true; - if (is_exe_or_dyn_lib and options.link_libc and options.target.isGnuLibC()) + if (is_exe_or_dyn_lib and link_libc and options.target.isGnuLibC()) break :dl true; if (options.system_libs.len != 0) break :dl true; @@ -444,12 +446,12 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { options.zig_lib_directory.path.?, options.target, options.is_native_os, - options.link_libc, + link_libc, options.libc_installation, ); const must_pic: bool = b: { - if (target_util.requiresPIC(options.target, options.link_libc)) + if (target_util.requiresPIC(options.target, link_libc)) break :b true; break :b link_mode == .Dynamic; }; @@ -545,7 +547,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { cache.hash.add(link_mode); cache.hash.add(function_sections); cache.hash.add(options.strip); - cache.hash.add(options.link_libc); + cache.hash.add(link_libc); cache.hash.add(options.link_libcpp); cache.hash.add(options.output_mode); cache.hash.add(options.machine_code_model); @@ -676,7 +678,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .optimize_mode = options.optimize_mode, .use_lld = use_lld, .use_llvm = use_llvm, - .link_libc = options.link_libc, + .link_libc = link_libc, .link_libcpp = options.link_libcpp, .objects = options.link_objects, .frameworks = options.frameworks, From 800a4a6cebf363c8ba3d1fcfff55db8bfb71f731 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 23 Sep 2020 00:00:24 -0700 Subject: [PATCH 095/210] eliminate dependency of libzigcpp.a on libzigstage1.a This allows us to create a build of self-hosted with LLVM extensions enabled but without the stage1 backend. --- BRANCH_TODO | 13 ++++++------- CMakeLists.txt | 34 +++++++++++++++++++++------------- build.zig | 4 ++-- lib/std/build/translate_c.zig | 3 +-- src/zig_clang.cpp | 11 ++++++----- 5 files changed, 36 insertions(+), 29 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 1e0efa9a1e..6ced446d48 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,5 +1,10 @@ + * --main-pkg-path + * add CLI support for a way to pass extra flags to c source files + * musl + * support rpaths in ELF linker code + * implement proper parsing of LLD stderr/stdout and exposing compile errors + * tests passing with -Dskip-non-native * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj] - * separate libzigcpp.a and libzigstage1.a so that we can do non-stage1 builds * repair @cImport * make sure zig cc works - using it as a preprocessor (-E) @@ -13,23 +18,17 @@ * -fno-emit-asm (default) do not output .s (assembly code)\n" * -femit-llvm-ir produce a .ll file with LLVM IR\n" * -fno-emit-llvm-ir (default) do not produce a .ll file with LLVM IR\n" - * support rpaths in ELF linker code - * add CLI support for a way to pass extra flags to c source files - * musl * mingw-w64 * MachO LLD linking * COFF LLD linking * WASM LLD linking - * --main-pkg-path * skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd) (maybe make it an explicit option and have main.zig disable it) * audit the CLI options for stage2 * audit the base cache hash - * implement proper parsing of LLD stderr/stdout and exposing compile errors * implement proper parsing of clang stderr/stdout and exposing compile errors * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process. * restore error messages for stage2_add_link_lib - * update std/build.zig to use new CLI * support cross compiling stage2 with `zig build` * implement proper compile errors for failing to build glibc crt files and shared libs diff --git a/CMakeLists.txt b/CMakeLists.txt index e3035213a5..f812da9d76 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,7 +89,7 @@ if(APPLE AND ZIG_WORKAROUND_4799) list(APPEND LLVM_LIBRARIES "-Wl,${CMAKE_PREFIX_PATH}/lib/libPolly.a" "-Wl,${CMAKE_PREFIX_PATH}/lib/libPollyPPCG.a" "-Wl,${CMAKE_PREFIX_PATH}/lib/libPollyISL.a") endif() -set(ZIG_CPP_LIB_DIR "${CMAKE_BINARY_DIR}/zig_cpp") +set(ZIG_CPP_LIB_DIR "${CMAKE_BINARY_DIR}/zigcpp") # Handle multi-config builds and place each into a common lib. The VS generator # for example will append a Debug folder by default if not explicitly specified. @@ -260,7 +260,7 @@ set(ZIG0_SOURCES "${CMAKE_SOURCE_DIR}/src/stage1/zig0.cpp" ) -set(ZIG_SOURCES +set(STAGE1_SOURCES "${CMAKE_SOURCE_DIR}/src/stage1/analyze.cpp" "${CMAKE_SOURCE_DIR}/src/stage1/ast_render.cpp" "${CMAKE_SOURCE_DIR}/src/stage1/bigfloat.cpp" @@ -392,21 +392,19 @@ if(ZIG_TEST_COVERAGE) set(EXE_LDFLAGS "${EXE_LDFLAGS} -fprofile-arcs -ftest-coverage") endif() -add_library(zig_cpp STATIC ${ZIG_SOURCES} ${ZIG_CPP_SOURCES}) -set_target_properties(zig_cpp PROPERTIES +add_library(zigcpp STATIC ${ZIG_CPP_SOURCES}) +set_target_properties(zigcpp PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} ) -target_link_libraries(zig_cpp LINK_PUBLIC - opt_c_util - ${SOFTFLOAT_LIBRARIES} +target_link_libraries(zigcpp LINK_PUBLIC ${CLANG_LIBRARIES} ${LLD_LIBRARIES} ${LLVM_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ) if(ZIG_WORKAROUND_POLLY_SO) - target_link_libraries(zig_cpp LINK_PUBLIC "-Wl,${ZIG_WORKAROUND_POLLY_SO}") + target_link_libraries(zigcpp LINK_PUBLIC "-Wl,${ZIG_WORKAROUND_POLLY_SO}") endif() add_library(opt_c_util STATIC ${OPTIMIZED_C_SOURCES}) @@ -414,16 +412,26 @@ set_target_properties(opt_c_util PROPERTIES COMPILE_FLAGS "${OPTIMIZED_C_FLAGS}" ) +add_library(zigstage1 STATIC ${STAGE1_SOURCES}) +set_target_properties(zigstage1 PROPERTIES + COMPILE_FLAGS ${EXE_CFLAGS} + LINK_FLAGS ${EXE_LDFLAGS} +) +target_link_libraries(zigstage1 LINK_PUBLIC + opt_c_util + ${SOFTFLOAT_LIBRARIES} + zigcpp +) if(NOT MSVC) - target_link_libraries(zig_cpp LINK_PUBLIC ${LIBXML2}) + target_link_libraries(zigstage1 LINK_PUBLIC ${LIBXML2}) endif() if(ZIG_DIA_GUIDS_LIB) - target_link_libraries(zig_cpp LINK_PUBLIC ${ZIG_DIA_GUIDS_LIB}) + target_link_libraries(zigstage1 LINK_PUBLIC ${ZIG_DIA_GUIDS_LIB}) endif() if(MSVC OR MINGW) - target_link_libraries(zig_cpp LINK_PUBLIC version) + target_link_libraries(zigstage1 LINK_PUBLIC version) endif() add_executable(zig0 ${ZIG0_SOURCES}) @@ -431,7 +439,7 @@ set_target_properties(zig0 PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} LINK_FLAGS ${EXE_LDFLAGS} ) -target_link_libraries(zig0 zig_cpp) +target_link_libraries(zig0 zigstage1) if(MSVC) set(ZIG1_OBJECT "${CMAKE_BINARY_DIR}/zig1.obj") @@ -487,7 +495,7 @@ set_target_properties(zig PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} LINK_FLAGS ${EXE_LDFLAGS} ) -target_link_libraries(zig "${ZIG1_OBJECT}" zig_cpp) +target_link_libraries(zig "${ZIG1_OBJECT}" zigstage1) if(MSVC) target_link_libraries(zig ntdll.lib) elseif(MINGW) diff --git a/build.zig b/build.zig index 331af1204c..df575e3356 100644 --- a/build.zig +++ b/build.zig @@ -270,7 +270,7 @@ fn fileExists(filename: []const u8) !bool { fn addCppLib(b: *Builder, lib_exe_obj: anytype, cmake_binary_dir: []const u8, lib_name: []const u8) void { lib_exe_obj.addObjectFile(fs.path.join(b.allocator, &[_][]const u8{ cmake_binary_dir, - "zig_cpp", + "zigcpp", b.fmt("{}{}{}", .{ lib_exe_obj.target.libPrefix(), lib_name, lib_exe_obj.target.staticLibSuffix() }), }) catch unreachable); } @@ -352,7 +352,7 @@ fn findLLVM(b: *Builder, llvm_config_exe: []const u8) !LibraryDep { fn configureStage2(b: *Builder, exe: anytype, ctx: Context, need_cpp_includes: bool) !void { exe.addIncludeDir("src"); exe.addIncludeDir(ctx.cmake_binary_dir); - addCppLib(b, exe, ctx.cmake_binary_dir, "zig_cpp"); + addCppLib(b, exe, ctx.cmake_binary_dir, "zigcpp"); assert(ctx.lld_include_dir.len != 0); exe.addIncludeDir(ctx.lld_include_dir); { diff --git a/lib/std/build/translate_c.zig b/lib/std/build/translate_c.zig index 8ca2b87209..87e153066d 100644 --- a/lib/std/build/translate_c.zig +++ b/lib/std/build/translate_c.zig @@ -72,8 +72,7 @@ pub const TranslateCStep = struct { try argv_list.append("translate-c"); try argv_list.append("-lc"); - try argv_list.append("--cache"); - try argv_list.append("on"); + try argv_list.append("--enable-cache"); if (!self.target.isNative()) { try argv_list.append("-target"); diff --git a/src/zig_clang.cpp b/src/zig_clang.cpp index 21d0c5c0ca..31c4404083 100644 --- a/src/zig_clang.cpp +++ b/src/zig_clang.cpp @@ -13,7 +13,6 @@ * 3. Prevent C++ from infecting the rest of the project. */ #include "zig_clang.h" -#include "list.hpp" #if __GNUC__ >= 8 #pragma GCC diagnostic push @@ -2186,7 +2185,7 @@ ZigClangASTUnit *ZigClangLoadFromCommandLine(const char **args_begin, const char // Take ownership of the err_unit ASTUnit object so that it won't be // free'd when we return, invalidating the error message pointers clang::ASTUnit *unit = ast_unit ? ast_unit : err_unit.release(); - ZigList errors = {}; + Stage2ErrorMsg *errors = nullptr; for (clang::ASTUnit::stored_diag_iterator it = unit->stored_diag_begin(), it_end = unit->stored_diag_end(); it != it_end; ++it) @@ -2204,7 +2203,10 @@ ZigClangASTUnit *ZigClangLoadFromCommandLine(const char **args_begin, const char llvm::StringRef msg_str_ref = it->getMessage(); - Stage2ErrorMsg *msg = errors.add_one(); + *errors_len += 1; + errors = reinterpret_cast(realloc(errors, sizeof(Stage2ErrorMsg) * *errors_len)); + if (errors == nullptr) abort(); + Stage2ErrorMsg *msg = &errors[*errors_len - 1]; memset(msg, 0, sizeof(*msg)); msg->msg_ptr = (const char *)msg_str_ref.bytes_begin(); @@ -2242,8 +2244,7 @@ ZigClangASTUnit *ZigClangLoadFromCommandLine(const char **args_begin, const char } } - *errors_ptr = errors.items; - *errors_len = errors.length; + *errors_ptr = errors; return nullptr; } From 0eed7ec9d51fe8c0dd22b72da0130ad9d31877d0 Mon Sep 17 00:00:00 2001 From: Timon Kruiper Date: Wed, 23 Sep 2020 11:41:31 +0200 Subject: [PATCH 096/210] Eventloop: Fix deadlock in linux event loop implementation A simple empty main with evented-io would not quit, because some threads were still waiting to be resumed (by the os). The os.write to the eventfd only wakes up one thread and thus there are multiple writes needed to wake up all the other threads. --- lib/std/event/loop.zig | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index 2600b337b3..bc2eab3c90 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -687,9 +687,14 @@ pub const Loop = struct { switch (builtin.os.tag) { .linux => { - // writing 8 bytes to an eventfd cannot fail - const amt = os.write(self.os_data.final_eventfd, &wakeup_bytes) catch unreachable; - assert(amt == wakeup_bytes.len); + // writing to the eventfd will only wake up one thread, thus multiple writes + // are needed to wakeup all the threads + var i: usize = 0; + while (i < self.extra_threads.len + 1) : (i += 1) { + // writing 8 bytes to an eventfd cannot fail + const amt = os.write(self.os_data.final_eventfd, &wakeup_bytes) catch unreachable; + assert(amt == wakeup_bytes.len); + } return; }, .macosx, .freebsd, .netbsd, .dragonfly => { From 94024a9fae6e9292770fa377fe2a03b6b85b4249 Mon Sep 17 00:00:00 2001 From: Calle Englund Date: Wed, 23 Sep 2020 16:26:35 +0200 Subject: [PATCH 097/210] Update macOS build instructions with workaround --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 8031aa790e..7b16902d23 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,11 @@ in which case try `-DZIG_WORKAROUND_4799=ON` Hopefully this will be fixed upstream with LLVM 10.0.1. +Building with LLVM 10.0.1 you might run into this problem: +`ld: library not found for -llibxml2.tbd` +[Building with LLVM 10.0.1 installed via Homebrew fails](https://github.com/ziglang/zig/issues/6087), +in which case you can try `-DZIG_WORKAROUND_6087=ON`. + ##### Windows See https://github.com/ziglang/zig/wiki/Building-Zig-on-Windows From bbff6bd6754bbddef03e163406d09c748a479073 Mon Sep 17 00:00:00 2001 From: Timon Kruiper Date: Wed, 23 Sep 2020 18:38:28 +0200 Subject: [PATCH 098/210] Eventloop: Enable basic event loop test, fixed by previous commit Closes #4922 --- lib/std/event/loop.zig | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index bc2eab3c90..c547f50365 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -1257,11 +1257,6 @@ test "std.event.Loop - basic" { // https://github.com/ziglang/zig/issues/1908 if (builtin.single_threaded) return error.SkipZigTest; - if (true) { - // https://github.com/ziglang/zig/issues/4922 - return error.SkipZigTest; - } - var loop: Loop = undefined; try loop.initMultiThreaded(); defer loop.deinit(); From 15b2bae517c54db22604eb303645d2f0023768e0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 23 Sep 2020 09:41:55 -0700 Subject: [PATCH 099/210] fix trying to link libc for static libs and objects --- src/Compilation.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index f7ffc762aa..be26a400c2 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -417,7 +417,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { return error.MachineCodeModelNotSupported; } - const link_libc = options.link_libc or target_util.osRequiresLibC(options.target); + const link_libc = options.link_libc or + (is_exe_or_dyn_lib and target_util.osRequiresLibC(options.target)); const must_dynamic_link = dl: { if (target_util.cannotDynamicLink(options.target)) From d03fcc73fc7083558915f6c170432fce8fb84993 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 23 Sep 2020 09:56:30 -0700 Subject: [PATCH 100/210] stage2: implement --main-pkg-path --- BRANCH_TODO | 1 - src/Compilation.zig | 6 ++++-- src/main.zig | 22 +++++++++++++++++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 6ced446d48..096207a4cf 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ - * --main-pkg-path * add CLI support for a way to pass extra flags to c source files * musl * support rpaths in ELF linker code diff --git a/src/Compilation.zig b/src/Compilation.zig index be26a400c2..656baa898b 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2248,10 +2248,12 @@ fn updateStage1Module(comp: *Compilation) !void { comp.stage1_cache_hash = &ch; + const main_pkg_path = mod.root_pkg.root_src_directory.path orelse ""; + const stage1_module = stage1.create( @enumToInt(comp.bin_file.options.optimize_mode), - undefined, - 0, // TODO --main-pkg-path + main_pkg_path.ptr, + main_pkg_path.len, main_zig_file.ptr, main_zig_file.len, zig_lib_dir.ptr, diff --git a/src/main.zig b/src/main.zig index 87ddbed343..1873656e77 100644 --- a/src/main.zig +++ b/src/main.zig @@ -217,6 +217,7 @@ const usage_build_generic = \\ ReleaseSmall Optimize for small binary, safety off \\ --pkg-begin [name] [path] Make pkg available to import and push current pkg \\ --pkg-end Pop current pkg + \\ --main-pkg-path Set the directory of the root package \\ -fPIC Force-enable Position Independent Code \\ -fno-PIC Force-disable Position Independent Code \\ -fstack-check Enable stack probing in unsafe builds @@ -234,8 +235,8 @@ const usage_build_generic = \\ c Compile to C source code \\ wasm WebAssembly \\ pe Portable Executable (Windows) - \\ coff (planned) Common Object File Format (Windows) - \\ macho (planned) macOS relocatables + \\ coff Common Object File Format (Windows) + \\ macho macOS relocatables \\ hex (planned) Intel IHEX \\ raw (planned) Dump machine code directly \\ -dirafter [dir] Add directory to AFTER include search path @@ -367,6 +368,7 @@ pub fn buildOutputType( var override_local_cache_dir: ?[]const u8 = null; var override_global_cache_dir: ?[]const u8 = null; var override_lib_dir: ?[]const u8 = null; + var main_pkg_path: ?[]const u8 = null; var system_libs = std.ArrayList([]const u8).init(gpa); defer system_libs.deinit(); @@ -463,6 +465,10 @@ pub fn buildOutputType( } else if (mem.eql(u8, arg, "--pkg-end")) { cur_pkg = cur_pkg.parent orelse fatal("encountered --pkg-end with no matching --pkg-begin", .{}); + } else if (mem.eql(u8, arg, "--main-pkg-path")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + main_pkg_path = args[i]; } else if (mem.eql(u8, arg, "--color")) { if (i + 1 >= args.len) { fatal("expected [auto|on|off] after --color", .{}); @@ -1248,8 +1254,18 @@ pub fn buildOutputType( .yes => |p| p, }; + var cleanup_root_dir: ?fs.Dir = null; + defer if (cleanup_root_dir) |*dir| dir.close(); + const root_pkg: ?*Package = if (root_src_file) |src_path| blk: { - root_pkg_memory.root_src_directory = .{ .path = null, .handle = fs.cwd() }; + root_pkg_memory.root_src_directory = m: { + if (main_pkg_path) |p| { + const dir = try fs.cwd().openDir(p, .{}); + cleanup_root_dir = dir; + break :m .{ .path = p, .handle = dir }; + } + break :m .{ .path = null, .handle = fs.cwd() }; + }; root_pkg_memory.root_src_path = src_path; break :blk &root_pkg_memory; } else null; From c0b774fbc65e3e406a38d37b02fffda7c5d3df26 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 23 Sep 2020 10:22:44 -0700 Subject: [PATCH 101/210] stage2: support passing extra C flags to C source files closes #3508 --- BRANCH_TODO | 3 +-- lib/std/build.zig | 21 ++++++++++++++++++--- src/main.zig | 18 ++++++++++++++++-- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 096207a4cf..d3a2ac8f90 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,6 +1,5 @@ - * add CLI support for a way to pass extra flags to c source files - * musl * support rpaths in ELF linker code + * musl * implement proper parsing of LLD stderr/stdout and exposing compile errors * tests passing with -Dskip-non-native * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj] diff --git a/lib/std/build.zig b/lib/std/build.zig index 133919afc1..7499b40533 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -1947,6 +1947,7 @@ pub const LibExeObjStep = struct { if (self.root_src) |root_src| try zig_args.append(root_src.getPath(builder)); + var prev_has_extra_flags = false; for (self.link_objects.span()) |link_object| { switch (link_object) { .StaticPath => |static_path| { @@ -1979,12 +1980,26 @@ pub const LibExeObjStep = struct { try zig_args.append(name); }, .AssemblyFile => |asm_file| { + if (prev_has_extra_flags) { + try zig_args.append("-extra-cflags"); + try zig_args.append("--"); + prev_has_extra_flags = false; + } try zig_args.append(asm_file.getPath(builder)); }, .CSourceFile => |c_source_file| { - try zig_args.append("--c-source"); - for (c_source_file.args) |arg| { - try zig_args.append(arg); + if (c_source_file.args.len == 0) { + if (prev_has_extra_flags) { + try zig_args.append("-cflags"); + try zig_args.append("--"); + prev_has_extra_flags = false; + } + } else { + try zig_args.append("-cflags"); + for (c_source_file.args) |arg| { + try zig_args.append(arg); + } + try zig_args.append("--"); } try zig_args.append(c_source_file.source.getPath(builder)); }, diff --git a/src/main.zig b/src/main.zig index 1873656e77..73676fb75e 100644 --- a/src/main.zig +++ b/src/main.zig @@ -244,6 +244,7 @@ const usage_build_generic = \\ -I[dir] Add directory to include search path \\ -D[macro]=[value] Define C [macro] to [value] (1 if [value] omitted) \\ --libc [file] Provide a file which specifies libc paths + \\ -cflags [flags] -- Set extra flags for the next positional C source files \\ \\Link Options: \\ -l[lib], --library [lib] Link against system library @@ -376,6 +377,9 @@ pub fn buildOutputType( var clang_argv = std.ArrayList([]const u8).init(gpa); defer clang_argv.deinit(); + var extra_cflags = std.ArrayList([]const u8).init(gpa); + defer extra_cflags.deinit(); + var lld_argv = std.ArrayList([]const u8).init(gpa); defer lld_argv.deinit(); @@ -469,6 +473,14 @@ pub fn buildOutputType( if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; main_pkg_path = args[i]; + } else if (mem.eql(u8, arg, "-cflags")) { + extra_cflags.shrinkRetainingCapacity(0); + while (true) { + i += 1; + if (i + 1 >= args.len) fatal("expected -- after -cflags", .{}); + if (mem.eql(u8, args[i], "--")) break; + try extra_cflags.append(args[i]); + } } else if (mem.eql(u8, arg, "--color")) { if (i + 1 >= args.len) { fatal("expected [auto|on|off] after --color", .{}); @@ -713,8 +725,10 @@ pub fn buildOutputType( try link_objects.append(arg); }, .assembly, .c, .cpp, .h, .ll, .bc => { - // TODO a way to pass extra flags on the CLI - try c_source_files.append(.{ .src_path = arg }); + try c_source_files.append(.{ + .src_path = arg, + .extra_flags = try arena.dupe([]const u8, extra_cflags.items), + }); }, .shared_library => { fatal("linking against dynamic libraries not yet supported", .{}); From 02886a8b93ccfe652ac5797784bddc398047f7eb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 23 Sep 2020 11:01:15 -0700 Subject: [PATCH 102/210] stage2: support rpaths --- BRANCH_TODO | 1 - src/Compilation.zig | 2 ++ src/link.zig | 1 + src/link/Elf.zig | 58 ++++++++++++++++++++++++--------------------- src/main.zig | 5 ++++ 5 files changed, 39 insertions(+), 28 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index d3a2ac8f90..72e1c78115 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ - * support rpaths in ELF linker code * musl * implement proper parsing of LLD stderr/stdout and exposing compile errors * tests passing with -Dskip-non-native diff --git a/src/Compilation.zig b/src/Compilation.zig index 656baa898b..1297341926 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -316,6 +316,7 @@ pub const InitOptions = struct { function_sections: ?bool = null, linker_allow_shlib_undefined: ?bool = null, linker_bind_global_refs_locally: ?bool = null, + each_lib_rpath: ?bool = null, disable_c_depfile: bool = false, linker_z_nodelete: bool = false, linker_z_defs: bool = false, @@ -714,6 +715,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .error_return_tracing = error_return_tracing, .llvm_cpu_features = llvm_cpu_features, .is_compiler_rt_or_libc = options.is_compiler_rt_or_libc, + .each_lib_rpath = options.each_lib_rpath orelse false, }); errdefer bin_file.destroy(); comp.* = .{ diff --git a/src/link.zig b/src/link.zig index 71d49286ba..82d67f2a7a 100644 --- a/src/link.zig +++ b/src/link.zig @@ -65,6 +65,7 @@ pub const Options = struct { dll_export_fns: bool, error_return_tracing: bool, is_compiler_rt_or_libc: bool, + each_lib_rpath: bool, gc_sections: ?bool = null, allow_shlib_undefined: ?bool = null, linker_script: ?[]const u8 = null, diff --git a/src/link/Elf.zig b/src/link/Elf.zig index b275026e8e..d4900ccb25 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1272,6 +1272,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { ch.hash.add(self.base.options.rdynamic); ch.hash.addListOfBytes(self.base.options.extra_lld_args); ch.hash.addListOfBytes(self.base.options.lib_dirs); + ch.hash.addListOfBytes(self.base.options.rpath_list); + ch.hash.add(self.base.options.each_lib_rpath); ch.hash.add(self.base.options.is_compiler_rt_or_libc); ch.hash.add(self.base.options.z_nodelete); ch.hash.add(self.base.options.z_defs); @@ -1392,7 +1394,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { } const full_out_path = if (directory.path) |dir_path| - try std.fs.path.join(arena, &[_][]const u8{dir_path, self.base.options.sub_path}) + try fs.path.join(arena, &[_][]const u8{dir_path, self.base.options.sub_path}) else self.base.options.sub_path; try argv.append("-o"); @@ -1420,32 +1422,34 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { } } - // TODO rpaths - // TODO add to cache hash above too - //for (size_t i = 0; i < g->rpath_list.length; i += 1) { - // Buf *rpath = g->rpath_list.at(i); - // add_rpath(lj, rpath); - //} - //if (g->each_lib_rpath) { - // for (size_t i = 0; i < g->lib_dirs.length; i += 1) { - // const char *lib_dir = g->lib_dirs.at(i); - // for (size_t i = 0; i < g->link_libs_list.length; i += 1) { - // LinkLib *link_lib = g->link_libs_list.at(i); - // if (buf_eql_str(link_lib->name, "c")) { - // continue; - // } - // bool does_exist; - // Buf *test_path = buf_sprintf("%s/lib%s.so", lib_dir, buf_ptr(link_lib->name)); - // if (os_file_exists(test_path, &does_exist) != ErrorNone) { - // zig_panic("link: unable to check if file exists: %s", buf_ptr(test_path)); - // } - // if (does_exist) { - // add_rpath(lj, buf_create_from_str(lib_dir)); - // break; - // } - // } - // } - //} + // rpaths + var rpath_table = std.StringHashMap(void).init(self.base.allocator); + defer rpath_table.deinit(); + for (self.base.options.rpath_list) |rpath| { + if ((try rpath_table.fetchPut(rpath, {})) == null) { + try argv.append("-rpath"); + try argv.append(rpath); + } + } + if (self.base.options.each_lib_rpath) { + var test_path = std.ArrayList(u8).init(self.base.allocator); + defer test_path.deinit(); + for (self.base.options.lib_dirs) |lib_dir_path| { + for (self.base.options.system_libs) |link_lib| { + test_path.shrinkRetainingCapacity(0); + const sep = fs.path.sep_str; + try test_path.writer().print("{}" ++ sep ++ "lib{}.so", .{ lib_dir_path, link_lib }); + fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { + error.FileNotFound => continue, + else => |e| return e, + }; + if ((try rpath_table.fetchPut(lib_dir_path, {})) == null) { + try argv.append("-rpath"); + try argv.append(lib_dir_path); + } + } + } + } for (self.base.options.lib_dirs) |lib_dir| { try argv.append("-L"); diff --git a/src/main.zig b/src/main.zig index 73676fb75e..a33e8bfc3b 100644 --- a/src/main.zig +++ b/src/main.zig @@ -251,6 +251,7 @@ const usage_build_generic = \\ -L[d], --library-directory [d] Add a directory to the library search path \\ -T[script] Use a custom linker script \\ --dynamic-linker [path] Set the dynamic interpreter path (usually ld.so) + \\ --each-lib-rpath Add rpath for each used dynamic library \\ --version [ver] Dynamic library semver \\ -rdynamic Add all symbols to the dynamic symbol table \\ -rpath [path] Add directory to the runtime library search path @@ -361,6 +362,7 @@ pub fn buildOutputType( var use_lld: ?bool = null; var use_clang: ?bool = null; var link_eh_frame_hdr = false; + var each_lib_rpath = false; var libc_paths_file: ?[]const u8 = null; var machine_code_model: std.builtin.CodeModel = .default; var runtime_args_start: ?usize = null; @@ -613,6 +615,8 @@ pub fn buildOutputType( if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; override_lib_dir = args[i]; + } else if (mem.eql(u8, arg, "--each-lib-rpath")) { + each_lib_rpath = true; } else if (mem.eql(u8, arg, "--enable-cache")) { enable_cache = true; } else if (mem.eql(u8, arg, "--test-cmd-bin")) { @@ -1421,6 +1425,7 @@ pub fn buildOutputType( .color = color, .time_report = time_report, .is_test = arg_mode == .zig_test, + .each_lib_rpath = each_lib_rpath, .test_evented_io = test_evented_io, .test_filter = test_filter, .test_name_prefix = test_name_prefix, From f4dde4d109ecd722ac82243417cac9f0e736de94 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 23 Sep 2020 11:15:10 -0700 Subject: [PATCH 103/210] test_runner: fix missing newline in log --- lib/std/special/test_runner.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index b9452b79cc..14a35a3aaa 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -103,6 +103,6 @@ pub fn log( log_err_count += 1; } if (@enumToInt(message_level) <= @enumToInt(std.testing.log_level)) { - std.debug.print("[{}] ({}): " ++ format, .{ @tagName(scope), @tagName(message_level) } ++ args); + std.debug.print("[{}] ({}): " ++ format ++ "\n", .{ @tagName(scope), @tagName(message_level) } ++ args); } } From b183b612c938a8c9949fee02fa55038273595496 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 23 Sep 2020 11:15:27 -0700 Subject: [PATCH 104/210] stage2: don't build libunwind on OS's that don't need it --- src/Compilation.zig | 3 ++- src/target.zig | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 1297341926..9ff2f35134 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1883,7 +1883,8 @@ fn wantBuildLibUnwindFromSource(comp: *Compilation) bool { .Exe => true, }; return comp.bin_file.options.link_libc and is_exe_or_dyn_lib and - comp.bin_file.options.libc_installation == null; + comp.bin_file.options.libc_installation == null and + target_util.libcNeedsLibUnwind(comp.getTarget()); } fn updateBuiltinZigFile(comp: *Compilation, mod: *Module) !void { diff --git a/src/target.zig b/src/target.zig index 8ea880e898..0402ccbd51 100644 --- a/src/target.zig +++ b/src/target.zig @@ -128,6 +128,20 @@ pub fn osRequiresLibC(target: std.Target) bool { }; } +pub fn libcNeedsLibUnwind(target: std.Target) bool { + return switch (target.os.tag) { + .windows, + .macosx, + .ios, + .watchos, + .tvos, + .freestanding, + => false, + + else => true, + }; +} + pub fn requiresPIE(target: std.Target) bool { return target.isAndroid(); } From 90b320d0e90c4daa5dcad6248623a69b3f7f2ab1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 23 Sep 2020 14:50:52 -0700 Subject: [PATCH 105/210] use ascii range for the --watch REPL prompt --- src/main.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.zig b/src/main.zig index a33e8bfc3b..bab3932427 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1516,7 +1516,7 @@ pub fn buildOutputType( var repl_buf: [1024]u8 = undefined; while (watch) { - try stderr.print("🦎 ", .{}); + try stderr.print("(zig) ", .{}); if (output_mode == .Exe) { try comp.makeBinFileExecutable(); } From 64deb46859a11588fb97e8464ec9dae53124b96b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 23 Sep 2020 15:21:40 -0700 Subject: [PATCH 106/210] stage2: capture LLD stderr into a buffer --- BRANCH_TODO | 4 ++-- src/link/Elf.zig | 41 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 72e1c78115..e6e88d817a 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,5 +1,4 @@ * musl - * implement proper parsing of LLD stderr/stdout and exposing compile errors * tests passing with -Dskip-non-native * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj] * repair @cImport @@ -23,10 +22,11 @@ (maybe make it an explicit option and have main.zig disable it) * audit the CLI options for stage2 * audit the base cache hash - * implement proper parsing of clang stderr/stdout and exposing compile errors * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process. * restore error messages for stage2_add_link_lib + * implement proper parsing of clang stderr/stdout and exposing compile errors with the Compilation API + * implement proper parsing of LLD stderr/stdout and exposing compile errors with the Compilation API * support cross compiling stage2 with `zig build` * implement proper compile errors for failing to build glibc crt files and shared libs * implement -fno-emit-bin diff --git a/src/link/Elf.zig b/src/link/Elf.zig index d4900ccb25..88255b5280 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1591,9 +1591,34 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { new_argv[i] = try arena.dupeZ(u8, arg); } + var stderr_context: LLDContext = .{ + .elf = self, + .data = std.ArrayList(u8).init(self.base.allocator), + }; + defer stderr_context.data.deinit(); + var stdout_context: LLDContext = .{ + .elf = self, + .data = std.ArrayList(u8).init(self.base.allocator), + }; + defer stdout_context.data.deinit(); const llvm = @import("../llvm.zig"); - const ok = llvm.Link(.ELF, new_argv.ptr, new_argv.len, append_diagnostic, 0, 0); - if (!ok) return error.LLDReportedFailure; + const ok = llvm.Link(.ELF, new_argv.ptr, new_argv.len, append_diagnostic, + @ptrToInt(&stdout_context), + @ptrToInt(&stderr_context), + ); + if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory; + if (stdout_context.data.items.len != 0) { + std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items}); + } + if (!ok) { + // TODO parse this output and surface with the Compilation API rather than + // directly outputting to stderr here. + std.debug.print("{}", .{stderr_context.data.items}); + return error.LLDReportedFailure; + } + if (stderr_context.data.items.len != 0) { + std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items}); + } // Update the dangling symlink with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. @@ -1609,10 +1634,18 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { self.base.lock = ch.toOwnedLock(); } +const LLDContext = struct { + data: std.ArrayList(u8), + elf: *Elf, + oom: bool = false, +}; + fn append_diagnostic(context: usize, ptr: [*]const u8, len: usize) callconv(.C) void { - // TODO collect diagnostics and handle cleanly + const lld_context = @intToPtr(*LLDContext, context); const msg = ptr[0..len]; - std.log.err("LLD: {}", .{msg}); + lld_context.data.appendSlice(msg) catch |err| switch (err) { + error.OutOfMemory => lld_context.oom = true, + }; } fn writeDwarfAddrAssumeCapacity(self: *Elf, buf: *std.ArrayList(u8), addr: u64) void { From 274c55b780c8ef954f335536bdfcf79cd8d9f6ca Mon Sep 17 00:00:00 2001 From: Frank Denis <124872+jedisct1@users.noreply.github.com> Date: Wed, 23 Sep 2020 22:48:17 +0200 Subject: [PATCH 107/210] ZIG_WORKAROUND_4799 is not required with LLVM 10.0.1 That paragraph used to contradict the following one. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b16902d23..2f0d65f584 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ or [error: unable to create target: 'Unable to find target for this triple (no targets are registered)'](https://github.com/ziglang/zig/issues/5055), in which case try `-DZIG_WORKAROUND_4799=ON` -Hopefully this will be fixed upstream with LLVM 10.0.1. +This has been fixed upstream with LLVM 10.0.1. Building with LLVM 10.0.1 you might run into this problem: `ld: library not found for -llibxml2.tbd` From b08fd0e8fca5ff9ac5531437f5749e74dc009a14 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 23 Sep 2020 20:48:47 -0700 Subject: [PATCH 108/210] stage2: building musl libc from source --- BRANCH_TODO | 6 +- lib/std/zig/system.zig | 6 + src/Compilation.zig | 110 ++++++++++++++- src/glibc.zig | 70 +--------- src/link/Elf.zig | 2 +- src/musl.zig | 308 ++++++++++++++++++++++++++++++++++++++++- 6 files changed, 426 insertions(+), 76 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index e6e88d817a..dafcfc742f 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,7 +1,6 @@ - * musl + * repair @cImport * tests passing with -Dskip-non-native * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj] - * repair @cImport * make sure zig cc works - using it as a preprocessor (-E) - try building some software @@ -20,6 +19,7 @@ * WASM LLD linking * skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd) (maybe make it an explicit option and have main.zig disable it) + - make sure that `zig cc -o hello hello.c -target native-native-musl` and `zig build-exe hello.zig -lc -target native-native-musl` will share the same libc build. * audit the CLI options for stage2 * audit the base cache hash * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process. @@ -58,3 +58,5 @@ * rename std.builtin.Mode to std.builtin.OptimizeMode * implement `zig run` and `zig test` when combined with `--watch` * close the --pkg-begin --pkg-end Package directory handles + * make std.Progress support multithreaded + * update musl.zig static data to use native path separator in static data rather than replacing '/' at runtime diff --git a/lib/std/zig/system.zig b/lib/std/zig/system.zig index 57d0449379..d13d0b22ef 100644 --- a/lib/std/zig/system.zig +++ b/lib/std/zig/system.zig @@ -393,6 +393,12 @@ pub const NativeTargetInfo = struct { if (!native_target_has_ld or have_all_info or os_is_non_native) { return defaultAbiAndDynamicLinker(cpu, os, cross_target); } + if (cross_target.abi) |abi| { + if (abi.isMusl()) { + // musl implies static linking. + return defaultAbiAndDynamicLinker(cpu, os, cross_target); + } + } // The current target's ABI cannot be relied on for this. For example, we may build the zig // compiler for target riscv64-linux-musl and provide a tarball for users to download. // A user could then run that zig compiler on riscv64-linux-gnu. This use case is well-defined diff --git a/src/Compilation.zig b/src/Compilation.zig index 9ff2f35134..756425dca5 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -15,6 +15,7 @@ const liveness = @import("liveness.zig"); const build_options = @import("build_options"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const glibc = @import("glibc.zig"); +const musl = @import("musl.zig"); const libunwind = @import("libunwind.zig"); const libcxx = @import("libcxx.zig"); const fatal = @import("main.zig").fatal; @@ -140,6 +141,8 @@ const Job = union(enum) { glibc_crt_file: glibc.CRTFile, /// all of the glibc shared objects glibc_shared_objects, + /// one of the glibc static objects + musl_crt_file: musl.CRTFile, /// libunwind.a, usually needed when linking libc libunwind: void, libcxx: void, @@ -778,6 +781,18 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { if (comp.wantBuildGLibCFromSource()) { try comp.addBuildingGLibCJobs(); } + if (comp.wantBuildMuslFromSource()) { + try comp.work_queue.write(&[_]Job{ + .{ .musl_crt_file = .crti_o }, + .{ .musl_crt_file = .crtn_o }, + .{ .musl_crt_file = .crt1_o }, + .{ .musl_crt_file = .scrt1_o }, + .{ .musl_crt_file = .libc_a }, + }); + } + if (comp.wantBuildMinGWW64FromSource()) { + @panic("TODO"); + } if (comp.wantBuildLibUnwindFromSource()) { try comp.work_queue.writeItem(.{ .libunwind = {} }); } @@ -822,6 +837,7 @@ pub fn destroy(self: *Compilation) void { { var it = self.crt_files.iterator(); while (it.next()) |entry| { + gpa.free(entry.key); entry.value.deinit(gpa); } self.crt_files.deinit(gpa); @@ -1128,6 +1144,12 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { fatal("unable to build glibc shared objects: {}", .{@errorName(err)}); }; }, + .musl_crt_file => |crt_file| { + musl.buildCRTFile(self, crt_file) catch |err| { + // TODO Expose this as a normal compile error rather than crashing here. + fatal("unable to build musl CRT file: {}", .{@errorName(err)}); + }; + }, .libunwind => { libunwind.buildStaticLib(self) catch |err| { // TODO Expose this as a normal compile error rather than crashing here. @@ -1846,7 +1868,10 @@ fn detectLibCFromLibCInstallation(arena: *Allocator, target: Target, lci: *const } pub fn get_libc_crt_file(comp: *Compilation, arena: *Allocator, basename: []const u8) ![]const u8 { - if (comp.wantBuildGLibCFromSource()) { + if (comp.wantBuildGLibCFromSource() or + comp.wantBuildMuslFromSource() or + comp.wantBuildMinGWW64FromSource()) + { return comp.crt_files.get(basename).?.full_object_path; } const lci = comp.bin_file.options.libc_installation orelse return error.LibCInstallationNotAvailable; @@ -1865,15 +1890,26 @@ fn addBuildingGLibCJobs(comp: *Compilation) !void { }); } -fn wantBuildGLibCFromSource(comp: *Compilation) bool { +fn wantBuildLibCFromSource(comp: Compilation) bool { const is_exe_or_dyn_lib = switch (comp.bin_file.options.output_mode) { .Obj => false, .Lib => comp.bin_file.options.link_mode == .Dynamic, .Exe => true, }; return comp.bin_file.options.link_libc and is_exe_or_dyn_lib and - comp.bin_file.options.libc_installation == null and - comp.bin_file.options.target.isGnuLibC(); + comp.bin_file.options.libc_installation == null; +} + +fn wantBuildGLibCFromSource(comp: Compilation) bool { + return comp.wantBuildLibCFromSource() and comp.getTarget().isGnuLibC(); +} + +fn wantBuildMuslFromSource(comp: Compilation) bool { + return comp.wantBuildLibCFromSource() and comp.getTarget().isMusl(); +} + +fn wantBuildMinGWW64FromSource(comp: Compilation) bool { + return comp.wantBuildLibCFromSource() and comp.getTarget().isMinGW(); } fn wantBuildLibUnwindFromSource(comp: *Compilation) bool { @@ -2362,3 +2398,69 @@ fn createStage1Pkg( }; return child_pkg; } + +pub fn build_crt_file( + comp: *Compilation, + root_name: []const u8, + output_mode: std.builtin.OutputMode, + c_source_files: []const Compilation.CSourceFile, +) !void { + const tracy = trace(@src()); + defer tracy.end(); + + const target = comp.getTarget(); + const basename = try std.zig.binNameAlloc(comp.gpa, root_name, target, output_mode, null, null); + errdefer comp.gpa.free(basename); + + // TODO: This is extracted into a local variable to work around a stage1 miscompilation. + const emit_bin = Compilation.EmitLoc{ + .directory = null, // Put it in the cache directory. + .basename = basename, + }; + const sub_compilation = try Compilation.create(comp.gpa, .{ + .local_cache_directory = comp.global_cache_directory, + .global_cache_directory = comp.global_cache_directory, + .zig_lib_directory = comp.zig_lib_directory, + .target = target, + .root_name = root_name, + .root_pkg = null, + .output_mode = output_mode, + .rand = comp.rand, + .libc_installation = comp.bin_file.options.libc_installation, + .emit_bin = emit_bin, + .optimize_mode = comp.bin_file.options.optimize_mode, + .want_sanitize_c = false, + .want_stack_check = false, + .want_valgrind = false, + .want_pic = comp.bin_file.options.pic, + .emit_h = null, + .strip = comp.bin_file.options.strip, + .is_native_os = comp.bin_file.options.is_native_os, + .self_exe_path = comp.self_exe_path, + .c_source_files = c_source_files, + .verbose_cc = comp.verbose_cc, + .verbose_link = comp.bin_file.options.verbose_link, + .verbose_tokenize = comp.verbose_tokenize, + .verbose_ast = comp.verbose_ast, + .verbose_ir = comp.verbose_ir, + .verbose_llvm_ir = comp.verbose_llvm_ir, + .verbose_cimport = comp.verbose_cimport, + .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features, + .clang_passthrough_mode = comp.clang_passthrough_mode, + .is_compiler_rt_or_libc = true, + }); + defer sub_compilation.destroy(); + + try sub_compilation.updateSubCompilation(); + + try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1); + const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p| + try std.fs.path.join(comp.gpa, &[_][]const u8{ p, basename }) + else + try comp.gpa.dupe(u8, basename); + + comp.crt_files.putAssumeCapacityNoClobber(basename, .{ + .full_object_path = artifact_path, + .lock = sub_compilation.bin_file.toOwnedLock(), + }); +} diff --git a/src/glibc.zig b/src/glibc.zig index c7c02a4470..1860726f93 100644 --- a/src/glibc.zig +++ b/src/glibc.zig @@ -274,7 +274,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-g", "-Wa,--noexecstack", }); - return build_crt_file(comp, "crti.o", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crti", .Obj, &[1]Compilation.CSourceFile{ .{ .src_path = try start_asm_path(comp, arena, "crti.S"), .extra_flags = args.items, @@ -292,7 +292,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-g", "-Wa,--noexecstack", }); - return build_crt_file(comp, "crtn.o", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crtn", .Obj, &[1]Compilation.CSourceFile{ .{ .src_path = try start_asm_path(comp, arena, "crtn.S"), .extra_flags = args.items, @@ -343,7 +343,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }; }; - return build_crt_file(comp, "Scrt1.o", .Obj, &[_]Compilation.CSourceFile{ start_os, abi_note_o }); + return comp.build_crt_file("Scrt1", .Obj, &[_]Compilation.CSourceFile{ start_os, abi_note_o }); }, .libc_nonshared_a => { const deps = [_][]const u8{ @@ -433,7 +433,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }; } - return build_crt_file(comp, "libc_nonshared.a", .Lib, &c_source_files); + return comp.build_crt_file("c_nonshared", .Lib, &c_source_files); }, } } @@ -676,68 +676,6 @@ fn lib_path(comp: *Compilation, arena: *Allocator, sub_path: []const u8) ![]cons return path.join(arena, &[_][]const u8{ comp.zig_lib_directory.path.?, sub_path }); } -fn build_crt_file( - comp: *Compilation, - basename: []const u8, - output_mode: std.builtin.OutputMode, - c_source_files: []const Compilation.CSourceFile, -) !void { - const tracy = trace(@src()); - defer tracy.end(); - - // TODO: This is extracted into a local variable to work around a stage1 miscompilation. - const emit_bin = Compilation.EmitLoc{ - .directory = null, // Put it in the cache directory. - .basename = basename, - }; - const sub_compilation = try Compilation.create(comp.gpa, .{ - .local_cache_directory = comp.global_cache_directory, - .global_cache_directory = comp.global_cache_directory, - .zig_lib_directory = comp.zig_lib_directory, - .target = comp.getTarget(), - .root_name = mem.split(basename, ".").next().?, - .root_pkg = null, - .output_mode = output_mode, - .rand = comp.rand, - .libc_installation = comp.bin_file.options.libc_installation, - .emit_bin = emit_bin, - .optimize_mode = comp.bin_file.options.optimize_mode, - .want_sanitize_c = false, - .want_stack_check = false, - .want_valgrind = false, - .want_pic = comp.bin_file.options.pic, - .emit_h = null, - .strip = comp.bin_file.options.strip, - .is_native_os = comp.bin_file.options.is_native_os, - .self_exe_path = comp.self_exe_path, - .c_source_files = c_source_files, - .verbose_cc = comp.verbose_cc, - .verbose_link = comp.bin_file.options.verbose_link, - .verbose_tokenize = comp.verbose_tokenize, - .verbose_ast = comp.verbose_ast, - .verbose_ir = comp.verbose_ir, - .verbose_llvm_ir = comp.verbose_llvm_ir, - .verbose_cimport = comp.verbose_cimport, - .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features, - .clang_passthrough_mode = comp.clang_passthrough_mode, - .is_compiler_rt_or_libc = true, - }); - defer sub_compilation.destroy(); - - try sub_compilation.updateSubCompilation(); - - try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1); - const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p| - try path.join(comp.gpa, &[_][]const u8{ p, basename }) - else - try comp.gpa.dupe(u8, basename); - - comp.crt_files.putAssumeCapacityNoClobber(basename, .{ - .full_object_path = artifact_path, - .lock = sub_compilation.bin_file.toOwnedLock(), - }); -} - pub const BuiltSharedObjects = struct { lock: Cache.Lock, dir_path: []u8, diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 88255b5280..8035adcf0d 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1548,7 +1548,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { try argv.append(try comp.get_libc_crt_file(arena, "libc_nonshared.a")); } else if (target.isMusl()) { try argv.append(comp.libunwind_static_lib.?.full_object_path); - try argv.append(comp.libc_static_lib.?.full_object_path); + try argv.append(try comp.get_libc_crt_file(arena, "libc.a")); } else if (self.base.options.link_libcpp) { try argv.append(comp.libunwind_static_lib.?.full_object_path); } else { diff --git a/src/musl.zig b/src/musl.zig index 88536b90fd..4f03004a70 100644 --- a/src/musl.zig +++ b/src/musl.zig @@ -1,6 +1,308 @@ -//! TODO build musl libc from source +const std = @import("std"); +const Allocator = std.mem.Allocator; +const mem = std.mem; +const path = std.fs.path; +const assert = std.debug.assert; -pub const src_files = [_][]const u8{ +const target_util = @import("target.zig"); +const Compilation = @import("Compilation.zig"); +const build_options = @import("build_options"); +const trace = @import("tracy.zig").trace; +const Cache = @import("Cache.zig"); +const Package = @import("Package.zig"); + +pub const CRTFile = enum { + crti_o, + crtn_o, + crt1_o, + scrt1_o, + libc_a, +}; + +pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { + if (!build_options.have_llvm) { + return error.ZigCompilerNotBuiltWithLLVMExtensions; + } + const gpa = comp.gpa; + var arena_allocator = std.heap.ArenaAllocator.init(gpa); + defer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + switch (crt_file) { + .crti_o => { + var args = std.ArrayList([]const u8).init(arena); + try add_cc_args(comp, arena, &args, false); + try args.appendSlice(&[_][]const u8{ + "-Qunused-arguments", + }); + return comp.build_crt_file("crti", .Obj, &[1]Compilation.CSourceFile{ + .{ + .src_path = try start_asm_path(comp, arena, "crti.s"), + .extra_flags = args.items, + }, + }); + }, + .crtn_o => { + var args = std.ArrayList([]const u8).init(arena); + try add_cc_args(comp, arena, &args, false); + try args.appendSlice(&[_][]const u8{ + "-Qunused-arguments", + }); + return comp.build_crt_file("crtn", .Obj, &[1]Compilation.CSourceFile{ + .{ + .src_path = try start_asm_path(comp, arena, "crtn.s"), + .extra_flags = args.items, + }, + }); + }, + .crt1_o => { + var args = std.ArrayList([]const u8).init(arena); + try add_cc_args(comp, arena, &args, false); + try args.appendSlice(&[_][]const u8{ + "-fno-stack-protector", + "-DCRT", + }); + return comp.build_crt_file("crt1", .Obj, &[1]Compilation.CSourceFile{ + .{ + .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ + "libc", "musl", "crt", "crt1.c", + }), + .extra_flags = args.items, + }, + }); + }, + .scrt1_o => { + var args = std.ArrayList([]const u8).init(arena); + try add_cc_args(comp, arena, &args, false); + try args.appendSlice(&[_][]const u8{ + "-fPIC", + "-fno-stack-protector", + "-DCRT", + }); + return comp.build_crt_file("Scrt1", .Obj, &[1]Compilation.CSourceFile{ + .{ + .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ + "libc", "musl", "crt", "Scrt1.c", + }), + .extra_flags = args.items, + }, + }); + }, + .libc_a => { + // When there is a src//foo.* then it should substitute for src/foo.* + // Even a .s file can substitute for a .c file. + const target = comp.getTarget(); + const arch_name = target_util.archMuslName(target.cpu.arch); + var source_table = std.StringArrayHashMap(Ext).init(comp.gpa); + defer source_table.deinit(); + + try source_table.ensureCapacity(compat_time32_files.len + src_files.len); + + for (src_files) |src_file| { + try addSrcFile(arena, &source_table, src_file); + } + + const time32_compat_arch_list = [_][]const u8{ "arm", "i386", "mips", "powerpc" }; + for (time32_compat_arch_list) |time32_compat_arch| { + if (mem.eql(u8, arch_name, time32_compat_arch)) { + for (compat_time32_files) |compat_time32_file| { + try addSrcFile(arena, &source_table, compat_time32_file); + } + } + } + + var c_source_files = std.ArrayList(Compilation.CSourceFile).init(comp.gpa); + defer c_source_files.deinit(); + + var override_path = std.ArrayList(u8).init(comp.gpa); + defer override_path.deinit(); + + const s = path.sep_str; + + for (source_table.items()) |entry| { + const src_file = entry.key; + const ext = entry.value; + + const dirname = path.dirname(src_file).?; + const basename = path.basename(src_file); + const noextbasename = mem.split(basename, ".").next().?; + const before_arch_dir = path.dirname(dirname).?; + const dirbasename = path.basename(dirname); + + var is_arch_specific = false; + // Architecture-specific implementations are under a / folder. + if (is_musl_arch_name(dirbasename)) { + if (!mem.eql(u8, dirbasename, arch_name)) + continue; // Not the architecture we're compiling for. + is_arch_specific = true; + } + if (!is_arch_specific) { + // Look for an arch specific override. + override_path.shrinkRetainingCapacity(0); + try override_path.writer().print("{}" ++ s ++ "{}" ++ s ++ "{}.s", .{ + dirname, arch_name, noextbasename, + }); + if (source_table.contains(override_path.items)) + continue; + + override_path.shrinkRetainingCapacity(0); + try override_path.writer().print("{}" ++ s ++ "{}" ++ s ++ "{}.S", .{ + dirname, arch_name, noextbasename, + }); + if (source_table.contains(override_path.items)) + continue; + + override_path.shrinkRetainingCapacity(0); + try override_path.writer().print("{}" ++ s ++ "{}" ++ s ++ "{}.c", .{ + dirname, arch_name, noextbasename, + }); + if (source_table.contains(override_path.items)) + continue; + } + + var args = std.ArrayList([]const u8).init(arena); + try add_cc_args(comp, arena, &args, ext == .o3); + try args.appendSlice(&[_][]const u8{ + "-Qunused-arguments", + "-w", // disable all warnings + }); + const c_source_file = try c_source_files.addOne(); + c_source_file.* = .{ + .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", src_file }), + .extra_flags = args.items, + }; + } + return comp.build_crt_file("c", .Lib, c_source_files.items); + }, + } +} + +fn is_musl_arch_name(name: []const u8) bool { + const musl_arch_names = [_][]const u8{ + "aarch64", + "arm", + "generic", + "i386", + "m68k", + "microblaze", + "mips", + "mips64", + "mipsn32", + "or1k", + "powerpc", + "powerpc64", + "riscv64", + "s390x", + "sh", + "x32", + "x86_64", + }; + for (musl_arch_names) |musl_arch_name| { + if (mem.eql(u8, musl_arch_name, name)) { + return true; + } + } + return false; +} + +const Ext = enum { + assembly, + normal, + o3, +}; + +fn addSrcFile(arena: *Allocator, source_table: *std.StringArrayHashMap(Ext), file_path: []const u8) !void { + const ext: Ext = ext: { + if (mem.endsWith(u8, file_path, ".c")) { + if (mem.startsWith(u8, file_path, "musl/src/malloc/") or + mem.startsWith(u8, file_path, "musl/src/string/") or + mem.startsWith(u8, file_path, "musl/src/internal/")) + { + break :ext .o3; + } else { + break :ext .assembly; + } + } else if (mem.endsWith(u8, file_path, ".s") or mem.endsWith(u8, file_path, ".S")) { + break :ext .assembly; + } else { + unreachable; + } + }; + // TODO do this at comptime on the comptime data rather than at runtime + // probably best to wait until self-hosted is done and our comptime execution + // is faster and uses less memory. + const key = if (path.sep != '/') blk: { + const mutable_file_path = try arena.dupe(u8, file_path); + for (mutable_file_path) |*c| { + if (c.* == '/') { + c.* == path.sep; + } + } + break :blk mutable_file_path; + } else file_path; + source_table.putAssumeCapacityNoClobber(key, ext); +} + +fn add_cc_args( + comp: *Compilation, + arena: *Allocator, + args: *std.ArrayList([]const u8), + want_O3: bool, +) error{OutOfMemory}!void { + const target = comp.getTarget(); + const arch_name = target_util.archMuslName(target.cpu.arch); + const os_name = @tagName(target.os.tag); + const triple = try std.fmt.allocPrint(arena, "{}-{}-musl", .{ arch_name, os_name }); + const o_arg = if (want_O3) "-O3" else "-Os"; + + try args.appendSlice(&[_][]const u8{ + "-std=c99", + "-ffreestanding", + // Musl adds these args to builds with gcc but clang does not support them. + //"-fexcess-precision=standard", + //"-frounding-math", + "-Wa,--noexecstack", + "-D_XOPEN_SOURCE=700", + + "-I", + try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "arch", arch_name }), + + "-I", + try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "arch", "generic" }), + + "-I", + try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "src", "include" }), + + "-I", + try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "src", "internal" }), + + "-I", + try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "include" }), + + "-I", + try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "include", triple }), + + "-I", + try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "include", "generic-musl" }), + + o_arg, + + "-fomit-frame-pointer", + "-fno-unwind-tables", + "-fno-asynchronous-unwind-tables", + "-ffunction-sections", + "-fdata-sections", + }); +} + +fn start_asm_path(comp: *Compilation, arena: *Allocator, basename: []const u8) ![]const u8 { + const target = comp.getTarget(); + return comp.zig_lib_directory.join(arena, &[_][]const u8{ + "libc", "musl", "crt", target_util.archMuslName(target.cpu.arch), basename, + }); +} + +const src_files = [_][]const u8{ "musl/src/aio/aio.c", "musl/src/aio/aio_suspend.c", "musl/src/aio/lio_listio.c", @@ -1776,7 +2078,7 @@ pub const src_files = [_][]const u8{ "musl/src/unistd/writev.c", "musl/src/unistd/x32/lseek.c", }; -pub const compat_time32_files = [_][]const u8{ +const compat_time32_files = [_][]const u8{ "musl/compat/time32/__xstat.c", "musl/compat/time32/adjtime32.c", "musl/compat/time32/adjtimex_time32.c", From 7bbd152dcc67e825c1743b5415493fb36bed89c6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 23 Sep 2020 20:52:33 -0700 Subject: [PATCH 109/210] nobody likes my std.process.cleanExit idea it's appropriate for the self-hosted compiler though, so this commit moves it from std lib to stage2 src code. --- lib/std/process.zig | 13 ------------- src/main.zig | 25 +++++++++++++++++++------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/std/process.zig b/lib/std/process.zig index 5303ada94b..2813d8cbab 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -19,19 +19,6 @@ pub const exit = os.exit; pub const changeCurDir = os.chdir; pub const changeCurDirC = os.chdirC; -/// Indicate that we are now terminating with a successful exit code. -/// In debug builds, this is a no-op, so that the calling code's -/// cleanup mechanisms are tested and so that external tools that -/// check for resource leaks can be accurate. In release builds, this -/// calls exit(0), and does not return. -pub fn cleanExit() void { - if (builtin.mode == .Debug) { - return; - } else { - exit(0); - } -} - /// The result is a slice of `out_buffer`, from index `0`. pub fn getCwd(out_buffer: []u8) ![]u8 { return os.getcwd(out_buffer); diff --git a/src/main.zig b/src/main.zig index bab3932427..80bced4ba9 100644 --- a/src/main.zig +++ b/src/main.zig @@ -437,7 +437,7 @@ pub fn buildOutputType( if (mem.startsWith(u8, arg, "-")) { if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { try io.getStdOut().writeAll(usage_build_generic); - return process.cleanExit(); + return cleanExit(); } else if (mem.eql(u8, arg, "--")) { if (arg_mode == .run) { runtime_args_start = i + 1; @@ -1506,7 +1506,7 @@ pub fn buildOutputType( else => process.exit(1), } if (!watch) - return process.cleanExit(); + return cleanExit(); }, else => {}, } @@ -1667,7 +1667,7 @@ pub fn cmdLibC(gpa: *Allocator, args: []const []const u8) !void { if (mem.eql(u8, arg, "--help")) { const stdout = io.getStdOut().writer(); try stdout.writeAll(usage_libc); - return process.cleanExit(); + return cleanExit(); } else { fatal("unrecognized parameter: '{}'", .{arg}); } @@ -1724,7 +1724,7 @@ pub fn cmdInit( if (mem.startsWith(u8, arg, "-")) { if (mem.eql(u8, arg, "--help")) { try io.getStdOut().writeAll(usage_init); - return process.cleanExit(); + return cleanExit(); } else { fatal("unrecognized parameter: '{}'", .{arg}); } @@ -2012,7 +2012,7 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v const term = try child.spawnAndWait(); switch (term) { .Exited => |code| { - if (code == 0) return process.cleanExit(); + if (code == 0) return cleanExit(); try cmd.writer().print("failed with exit code {}:\n", .{code}); }, else => { @@ -2073,7 +2073,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { if (mem.eql(u8, arg, "--help")) { const stdout = io.getStdOut().outStream(); try stdout.writeAll(usage_fmt); - return process.cleanExit(); + return cleanExit(); } else if (mem.eql(u8, arg, "--color")) { if (i + 1 >= args.len) { fatal("expected [auto|on|off] after --color", .{}); @@ -2804,3 +2804,16 @@ fn detectNativeTargetInfo(gpa: *Allocator, cross_target: std.zig.CrossTarget) !s } return info; } + +/// Indicate that we are now terminating with a successful exit code. +/// In debug builds, this is a no-op, so that the calling code's +/// cleanup mechanisms are tested and so that external tools that +/// check for resource leaks can be accurate. In release builds, this +/// calls exit(0), and does not return. +pub fn cleanExit() void { + if (std.builtin.mode == .Debug) { + return; + } else { + process.exit(0); + } +} From 72f4cdb2b4221658b4c85b33394377081ffae6bb Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Wed, 23 Sep 2020 21:31:57 +0200 Subject: [PATCH 110/210] docs: update implementation status for @Type() --- doc/langref.html.in | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index dce23a43e4..504a4e02e2 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -8467,30 +8467,23 @@ test "integer truncation" {
  • {#syntax#}@TypeOf(null){#endsyntax#}
  • {#link|Arrays#}
  • {#link|Optionals#}
    • +
  • {#link|Error Set Type#}
  • {#link|Error Union Type#}
  • {#link|Vectors#}
  • {#link|Opaque Types#}
    • -
  • AnyFrame
    • - -

    - For these types it is a - TODO in the compiler to implement: -

    -
      -
    • ErrorSet
      • -
    • Enum
      • -
    • FnFrame
      • -
    • EnumLiteral
      • -
    -

    - For these types, {#syntax#}@Type{#endsyntax#} is not available. - There is an open proposal to allow unions and structs. -

    -
      +
    • {#link|@Frame#}
      • +
    • {#syntax#}anyframe{#endsyntax#}
      • +
    • {#link|struct#}
      • +
    • {#link|enum#}
      • +
    • {#link|Enum Literals#}
    • {#link|union#}
      • +
    +

    + For these types, {#syntax#}@Type{#endsyntax#} is not available: +

    +
    • {#link|Functions#}
    • BoundFn
      • -
    • {#link|struct#}
    {#header_close#} {#header_open|@typeInfo#} From c8cd6145ac2475e298a3d3b2082e1966fda811d5 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Wed, 23 Sep 2020 21:37:16 +0200 Subject: [PATCH 111/210] Move PBKDF2 to a pwhash category, clarify what that category is Password hashing functions are not general-purpose KDFs, and KDFs don't have to satisfy the same properties as a PHF. This will allow fast KDFs such as the HKDF construction to be in a category of their own, while clarifying what functions are suitable for using passwords as inputs. --- lib/std/crypto.zig | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig index 3a1ae599a0..dd9101aa19 100644 --- a/lib/std/crypto.zig +++ b/lib/std/crypto.zig @@ -35,12 +35,23 @@ pub const onetimeauth = struct { pub const Poly1305 = @import("crypto/poly1305.zig").Poly1305; }; -/// A Key Derivation Function (KDF) is intended to turn a weak, human generated password into a -/// strong key, suitable for cryptographic uses. It does this by salting and stretching the -/// password. Salting injects non-secret random data, so that identical passwords will be converted -/// into unique keys. Stretching applies a deliberately slow hashing function to frustrate -/// brute-force guessing. -pub const kdf = struct { +/// A password hashing function derives a uniform key from low-entropy input material such as passwords. +/// It is intentionally slow or expensive. +/// +/// With the standard definition of a key derivation function, if a key space is small, an exhaustive search may be practical. +/// Password hashing functions make exhaustive searches way slower or way more expensive, even when implemented on GPUs and ASICs, by using different, optionally combined strategies: +/// +/// - Requiring a lot of computation cycles to complete +/// - Requiring a lot of memory to complete +/// - Requiring multiple CPU cores to complete +/// - Requiring cache-local data to complete in reasonable time +/// - Requiring large static tables +/// - Avoiding precomputations and time/memory tradeoffs +/// - Requiring multi-party computations +/// - Combining the input material with random per-entry data (salts), application-specific contexts and keys +/// +/// Password hashing functions must be used whenever sensitive data has to be directly derived from a password. +pub const pwhash = struct { pub const pbkdf2 = @import("crypto/pbkdf2.zig").pbkdf2; }; From 1123c909872569e9a956f9110685c417036118bd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 23 Sep 2020 23:13:18 -0700 Subject: [PATCH 112/210] stage2: print the test command after it fails --- src/main.zig | 60 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/src/main.zig b/src/main.zig index 80bced4ba9..1cc0098e66 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1487,7 +1487,7 @@ pub fn buildOutputType( try argv.appendSlice(all_args[i..]); } // TODO On operating systems that support it, do an execve here rather than child process, - // when watch=false. + // when watch=false and arg_mode == .run const child = try std.ChildProcess.init(argv.items, gpa); defer child.deinit(); @@ -1496,17 +1496,38 @@ pub fn buildOutputType( child.stderr_behavior = .Inherit; const term = try child.spawnAndWait(); - switch (term) { - .Exited => |code| { - if (code != 0) { - // TODO https://github.com/ziglang/zig/issues/6342 - process.exit(1); + switch (arg_mode) { + .run => { + switch (term) { + .Exited => |code| { + if (code == 0) { + if (!watch) return cleanExit(); + } else { + // TODO https://github.com/ziglang/zig/issues/6342 + process.exit(1); + } + }, + else => process.exit(1), } }, - else => process.exit(1), + .zig_test => { + switch (term) { + .Exited => |code| { + if (code == 0) { + if (!watch) return cleanExit(); + } else { + const cmd = try argvCmd(arena, argv.items); + fatal("the following test command failed with exit code {}:\n{}", .{ code, cmd }); + } + }, + else => { + const cmd = try argvCmd(arena, argv.items); + fatal("the following test command crashed:\n{}", .{cmd}); + }, + } + }, + else => unreachable, } - if (!watch) - return cleanExit(); }, else => {}, } @@ -2007,28 +2028,29 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v child.stdout_behavior = .Inherit; child.stderr_behavior = .Inherit; - var cmd = std.ArrayList(u8).init(arena); - const term = try child.spawnAndWait(); switch (term) { .Exited => |code| { if (code == 0) return cleanExit(); - try cmd.writer().print("failed with exit code {}:\n", .{code}); + const cmd = try argvCmd(arena, child_argv); + fatal("the following build command failed with exit code {}:\n{}", .{ code, cmd }); }, else => { - try cmd.appendSlice("crashed:\n"); + const cmd = try argvCmd(arena, child_argv); + fatal("the following build command crashed:\n{}", .{cmd}); }, } +} - try cmd.append('\n'); - for (child_argv[0 .. child_argv.len - 1]) |arg| { +fn argvCmd(allocator: *Allocator, argv: []const []const u8) ![]u8 { + var cmd = std.ArrayList(u8).init(allocator); + defer cmd.deinit(); + for (argv[0 .. argv.len - 1]) |arg| { try cmd.appendSlice(arg); try cmd.append(' '); } - try cmd.appendSlice(child_argv[child_argv.len - 1]); - - if (true) // Working around erroneous stage1 compile error: unreachable code on child.deinit() - fatal("The following build command {}", .{cmd.items}); + try cmd.appendSlice(argv[argv.len - 1]); + return cmd.toOwnedSlice(); } pub const usage_fmt = From bba4576281241e8824de3d48b65c55fa1b1d4c9c Mon Sep 17 00:00:00 2001 From: Ogromny Date: Thu, 24 Sep 2020 15:14:57 +0200 Subject: [PATCH 113/210] Fix typo in documentation --- doc/langref.html.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 504a4e02e2..6b8b07e0b3 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -1840,7 +1840,7 @@ const Point = struct { y: i32, }; -test "compile-time array initalization" { +test "compile-time array initialization" { assert(fancy_array[4].x == 4); assert(fancy_array[4].y == 8); } From bd89bd6fdbcc0ce5ea7763a8043fd46099022b19 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Wed, 23 Sep 2020 10:18:17 +0200 Subject: [PATCH 114/210] Revamp crypto/aes * Reorganize crypto/aes in order to separate parameters, implementations and modes. * Add a zero-cost abstraction over the internal representation of a block, so that blocks can be kept in vector registers in optimized implementations. * Add architecture-independent aesenc/aesdec/aesenclast/aesdeclast operations, so that any AES-based primitive can be implemented, including these that don't use the original key schedule (AES-PRF, AEGIS, MeowHash...) * Add support for parallelization/wide blocks to take advantage of hardware implementations. * Align T-tables to cache lines in the software implementations to slightly reduce side channels. * Add an optimized implementation for modern Intel CPUs with AES-NI. * Add new tests (AES256 key expansion). * Reimplement the counter mode to work with any block cipher, any endianness and to take advantage of wide blocks. * Add benchmarks for AES. --- lib/std/crypto.zig | 8 + lib/std/crypto/aes.zig | 714 ++++------------------------------ lib/std/crypto/aes/aesni.zig | 420 ++++++++++++++++++++ lib/std/crypto/aes/soft.zig | 735 +++++++++++++++++++++++++++++++++++ lib/std/crypto/benchmark.zig | 82 +++- lib/std/crypto/modes.zig | 51 +++ 6 files changed, 1365 insertions(+), 645 deletions(-) create mode 100644 lib/std/crypto/aes/aesni.zig create mode 100644 lib/std/crypto/aes/soft.zig create mode 100644 lib/std/crypto/modes.zig diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig index dd9101aa19..5763348729 100644 --- a/lib/std/crypto.zig +++ b/lib/std/crypto.zig @@ -59,6 +59,13 @@ pub const pwhash = struct { pub const core = struct { pub const aes = @import("crypto/aes.zig"); pub const Gimli = @import("crypto/gimli.zig").State; + + /// Modes are generic compositions to construct encryption/decryption functions from block ciphers and permutations. + /// + /// These modes are designed to be building blocks for higher-level constructions, and should generally not be used directly by applications, as they may not provide the expected properties and security guarantees. + /// + /// Most applications may want to use AEADs instead. + pub const modes = @import("crypto/modes.zig"); }; /// Elliptic-curve arithmetic. @@ -111,6 +118,7 @@ test "crypto" { _ = @import("crypto/gimli.zig"); _ = @import("crypto/hmac.zig"); _ = @import("crypto/md5.zig"); + _ = @import("crypto/modes.zig"); _ = @import("crypto/pbkdf2.zig"); _ = @import("crypto/poly1305.zig"); _ = @import("crypto/sha1.zig"); diff --git a/lib/std/crypto/aes.zig b/lib/std/crypto/aes.zig index 2174fbdd4f..7c509b297f 100644 --- a/lib/std/crypto/aes.zig +++ b/lib/std/crypto/aes.zig @@ -3,249 +3,44 @@ // This file is part of [zig](https://ziglang.org/), which is MIT licensed. // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. -// Based on Go stdlib implementation const std = @import("../std.zig"); -const mem = std.mem; const testing = std.testing; +const builtin = std.builtin; -// Apply sbox0 to each byte in w. -fn subw(w: u32) u32 { - return @as(u32, sbox0[w >> 24]) << 24 | @as(u32, sbox0[w >> 16 & 0xff]) << 16 | @as(u32, sbox0[w >> 8 & 0xff]) << 8 | @as(u32, sbox0[w & 0xff]); -} +const has_aesni = comptime std.Target.x86.featureSetHas(std.Target.current.cpu.features, .aes); +const has_avx = comptime std.Target.x86.featureSetHas(std.Target.current.cpu.features, .avx); +const impl = if (std.Target.current.cpu.arch == .x86_64 and has_aesni and has_avx) @import("aes/aesni.zig") else @import("aes/soft.zig"); -fn rotw(w: u32) u32 { - return w << 8 | w >> 24; -} - -// Encrypt one block from src into dst, using the expanded key xk. -fn encryptBlock(xk: []const u32, dst: []u8, src: []const u8) void { - var s0 = mem.readIntBig(u32, src[0..4]); - var s1 = mem.readIntBig(u32, src[4..8]); - var s2 = mem.readIntBig(u32, src[8..12]); - var s3 = mem.readIntBig(u32, src[12..16]); - - // First round just XORs input with key. - s0 ^= xk[0]; - s1 ^= xk[1]; - s2 ^= xk[2]; - s3 ^= xk[3]; - - // Middle rounds shuffle using tables. - // Number of rounds is set by length of expanded key. - var nr = xk.len / 4 - 2; // - 2: one above, one more below - var k: usize = 4; - var t0: u32 = undefined; - var t1: u32 = undefined; - var t2: u32 = undefined; - var t3: u32 = undefined; - var r: usize = 0; - while (r < nr) : (r += 1) { - t0 = xk[k + 0] ^ te0[@truncate(u8, s0 >> 24)] ^ te1[@truncate(u8, s1 >> 16)] ^ te2[@truncate(u8, s2 >> 8)] ^ te3[@truncate(u8, s3)]; - t1 = xk[k + 1] ^ te0[@truncate(u8, s1 >> 24)] ^ te1[@truncate(u8, s2 >> 16)] ^ te2[@truncate(u8, s3 >> 8)] ^ te3[@truncate(u8, s0)]; - t2 = xk[k + 2] ^ te0[@truncate(u8, s2 >> 24)] ^ te1[@truncate(u8, s3 >> 16)] ^ te2[@truncate(u8, s0 >> 8)] ^ te3[@truncate(u8, s1)]; - t3 = xk[k + 3] ^ te0[@truncate(u8, s3 >> 24)] ^ te1[@truncate(u8, s0 >> 16)] ^ te2[@truncate(u8, s1 >> 8)] ^ te3[@truncate(u8, s2)]; - k += 4; - s0 = t0; - s1 = t1; - s2 = t2; - s3 = t3; - } - - // Last round uses s-box directly and XORs to produce output. - s0 = @as(u32, sbox0[t0 >> 24]) << 24 | @as(u32, sbox0[t1 >> 16 & 0xff]) << 16 | @as(u32, sbox0[t2 >> 8 & 0xff]) << 8 | @as(u32, sbox0[t3 & 0xff]); - s1 = @as(u32, sbox0[t1 >> 24]) << 24 | @as(u32, sbox0[t2 >> 16 & 0xff]) << 16 | @as(u32, sbox0[t3 >> 8 & 0xff]) << 8 | @as(u32, sbox0[t0 & 0xff]); - s2 = @as(u32, sbox0[t2 >> 24]) << 24 | @as(u32, sbox0[t3 >> 16 & 0xff]) << 16 | @as(u32, sbox0[t0 >> 8 & 0xff]) << 8 | @as(u32, sbox0[t1 & 0xff]); - s3 = @as(u32, sbox0[t3 >> 24]) << 24 | @as(u32, sbox0[t0 >> 16 & 0xff]) << 16 | @as(u32, sbox0[t1 >> 8 & 0xff]) << 8 | @as(u32, sbox0[t2 & 0xff]); - - s0 ^= xk[k + 0]; - s1 ^= xk[k + 1]; - s2 ^= xk[k + 2]; - s3 ^= xk[k + 3]; - - mem.writeIntBig(u32, dst[0..4], s0); - mem.writeIntBig(u32, dst[4..8], s1); - mem.writeIntBig(u32, dst[8..12], s2); - mem.writeIntBig(u32, dst[12..16], s3); -} - -// Decrypt one block from src into dst, using the expanded key xk. -pub fn decryptBlock(xk: []const u32, dst: []u8, src: []const u8) void { - var s0 = mem.readIntBig(u32, src[0..4]); - var s1 = mem.readIntBig(u32, src[4..8]); - var s2 = mem.readIntBig(u32, src[8..12]); - var s3 = mem.readIntBig(u32, src[12..16]); - - // First round just XORs input with key. - s0 ^= xk[0]; - s1 ^= xk[1]; - s2 ^= xk[2]; - s3 ^= xk[3]; - - // Middle rounds shuffle using tables. - // Number of rounds is set by length of expanded key. - var nr = xk.len / 4 - 2; // - 2: one above, one more below - var k: usize = 4; - var t0: u32 = undefined; - var t1: u32 = undefined; - var t2: u32 = undefined; - var t3: u32 = undefined; - var r: usize = 0; - while (r < nr) : (r += 1) { - t0 = xk[k + 0] ^ td0[@truncate(u8, s0 >> 24)] ^ td1[@truncate(u8, s3 >> 16)] ^ td2[@truncate(u8, s2 >> 8)] ^ td3[@truncate(u8, s1)]; - t1 = xk[k + 1] ^ td0[@truncate(u8, s1 >> 24)] ^ td1[@truncate(u8, s0 >> 16)] ^ td2[@truncate(u8, s3 >> 8)] ^ td3[@truncate(u8, s2)]; - t2 = xk[k + 2] ^ td0[@truncate(u8, s2 >> 24)] ^ td1[@truncate(u8, s1 >> 16)] ^ td2[@truncate(u8, s0 >> 8)] ^ td3[@truncate(u8, s3)]; - t3 = xk[k + 3] ^ td0[@truncate(u8, s3 >> 24)] ^ td1[@truncate(u8, s2 >> 16)] ^ td2[@truncate(u8, s1 >> 8)] ^ td3[@truncate(u8, s0)]; - k += 4; - s0 = t0; - s1 = t1; - s2 = t2; - s3 = t3; - } - - // Last round uses s-box directly and XORs to produce output. - s0 = @as(u32, sbox1[t0 >> 24]) << 24 | @as(u32, sbox1[t3 >> 16 & 0xff]) << 16 | @as(u32, sbox1[t2 >> 8 & 0xff]) << 8 | @as(u32, sbox1[t1 & 0xff]); - s1 = @as(u32, sbox1[t1 >> 24]) << 24 | @as(u32, sbox1[t0 >> 16 & 0xff]) << 16 | @as(u32, sbox1[t3 >> 8 & 0xff]) << 8 | @as(u32, sbox1[t2 & 0xff]); - s2 = @as(u32, sbox1[t2 >> 24]) << 24 | @as(u32, sbox1[t1 >> 16 & 0xff]) << 16 | @as(u32, sbox1[t0 >> 8 & 0xff]) << 8 | @as(u32, sbox1[t3 & 0xff]); - s3 = @as(u32, sbox1[t3 >> 24]) << 24 | @as(u32, sbox1[t2 >> 16 & 0xff]) << 16 | @as(u32, sbox1[t1 >> 8 & 0xff]) << 8 | @as(u32, sbox1[t0 & 0xff]); - - s0 ^= xk[k + 0]; - s1 ^= xk[k + 1]; - s2 ^= xk[k + 2]; - s3 ^= xk[k + 3]; - - mem.writeIntBig(u32, dst[0..4], s0); - mem.writeIntBig(u32, dst[4..8], s1); - mem.writeIntBig(u32, dst[8..12], s2); - mem.writeIntBig(u32, dst[12..16], s3); -} - -fn xorBytes(dst: []u8, a: []const u8, b: []const u8) usize { - var n = std.math.min(dst.len, std.math.min(a.len, b.len)); - for (dst[0..n]) |_, i| { - dst[i] = a[i] ^ b[i]; - } - return n; -} - -pub const AES128 = AES(128); -pub const AES256 = AES(256); - -fn AES(comptime keysize: usize) type { - return struct { - const Self = @This(); - - pub const Encrypt = AESEncrypt(keysize); - pub const Decrypt = AESDecrypt(keysize); - - const nn = (keysize / 8) + 28; - enc: Encrypt, - dec: Decrypt, - - pub fn init(key: [keysize / 8]u8) Self { - var ctx: Self = undefined; - ctx.enc = Encrypt.init(key); - ctx.dec = ctx.enc.toDecrypt(); - return ctx; - } - - pub fn encrypt(ctx: Self, dst: []u8, src: []const u8) void { - ctx.enc.encrypt(dst, src); - } - pub fn decrypt(ctx: Self, dst: []u8, src: []const u8) void { - ctx.dec.decrypt(dst, src); - } - pub fn ctr(ctx: Self, dst: []u8, src: []const u8, iv: [16]u8) void { - ctx.enc.ctr(dst, src, iv); - } - }; -} - -fn AESEncrypt(comptime keysize: usize) type { - return struct { - const Self = @This(); - - const Decrypt = AESDecrypt(keysize); - - const nn = (keysize / 8) + 28; - enc: [nn]u32, - - pub fn init(key: [keysize / 8]u8) Self { - var ctx: Self = undefined; - expandKeyEncrypt(&key, ctx.enc[0..]); - return ctx; - } - - pub fn toDecrypt(ctx: Self) Decrypt { - var dec: Decrypt = undefined; - expandKeyDecrypt(ctx.enc[0..], dec.dec[0..]); - return dec; - } - - pub fn encrypt(ctx: Self, dst: []u8, src: []const u8) void { - encryptBlock(ctx.enc[0..], dst, src); - } - pub fn ctr(ctx: Self, dst: []u8, src: []const u8, iv: [16]u8) void { - std.debug.assert(dst.len >= src.len); - - var keystream: [16]u8 = undefined; - var ctrbuf = iv; - var n: usize = 0; - while (n < src.len) { - ctx.encrypt(keystream[0..], ctrbuf[0..]); - var ctr_i = std.mem.readIntBig(u128, ctrbuf[0..]); - std.mem.writeIntBig(u128, ctrbuf[0..], ctr_i +% 1); - - n += xorBytes(dst[n..], src[n..], &keystream); - } - } - }; -} - -fn AESDecrypt(comptime keysize: usize) type { - return struct { - const Self = @This(); - - const nn = (keysize / 8) + 28; - dec: [nn]u32, - - pub fn init(key: [keysize / 8]u8) Self { - var ctx: Self = undefined; - var enc: [nn]u32 = undefined; - expandKeyEncrypt(key[0..], enc[0..]); - expandKeyDecrypt(enc[0..], ctx.dec[0..]); - return ctx; - } - - pub fn decrypt(ctx: Self, dst: []u8, src: []const u8) void { - decryptBlock(ctx.dec[0..], dst, src); - } - }; -} +pub const Block = impl.Block; +pub const AESEncryptCtx = impl.AESEncryptCtx; +pub const AESDecryptCtx = impl.AESDecryptCtx; +pub const AES128 = impl.AES128; +pub const AES256 = impl.AES256; test "ctr" { // NIST SP 800-38A pp 55-58 - { - const key = [_]u8{ 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }; - const iv = [_]u8{ 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff }; - const in = [_]u8{ - 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, - 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, - 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, - 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, - }; - const exp_out = [_]u8{ - 0x87, 0x4d, 0x61, 0x91, 0xb6, 0x20, 0xe3, 0x26, 0x1b, 0xef, 0x68, 0x64, 0x99, 0x0d, 0xb6, 0xce, - 0x98, 0x06, 0xf6, 0x6b, 0x79, 0x70, 0xfd, 0xff, 0x86, 0x17, 0x18, 0x7b, 0xb9, 0xff, 0xfd, 0xff, - 0x5a, 0xe4, 0xdf, 0x3e, 0xdb, 0xd5, 0xd3, 0x5e, 0x5b, 0x4f, 0x09, 0x02, 0x0d, 0xb0, 0x3e, 0xab, - 0x1e, 0x03, 0x1d, 0xda, 0x2f, 0xbe, 0x03, 0xd1, 0x79, 0x21, 0x70, 0xa0, 0xf3, 0x00, 0x9c, 0xee, - }; + const ctr = @import("modes.zig").ctr; - var out: [exp_out.len]u8 = undefined; - var aes = AES128.init(key); - aes.ctr(out[0..], in[0..], iv); - testing.expectEqualSlices(u8, exp_out[0..], out[0..]); - } + const key = [_]u8{ 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }; + const iv = [_]u8{ 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff }; + const in = [_]u8{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, + }; + const exp_out = [_]u8{ + 0x87, 0x4d, 0x61, 0x91, 0xb6, 0x20, 0xe3, 0x26, 0x1b, 0xef, 0x68, 0x64, 0x99, 0x0d, 0xb6, 0xce, + 0x98, 0x06, 0xf6, 0x6b, 0x79, 0x70, 0xfd, 0xff, 0x86, 0x17, 0x18, 0x7b, 0xb9, 0xff, 0xfd, 0xff, + 0x5a, 0xe4, 0xdf, 0x3e, 0xdb, 0xd5, 0xd3, 0x5e, 0x5b, 0x4f, 0x09, 0x02, 0x0d, 0xb0, 0x3e, 0xab, + 0x1e, 0x03, 0x1d, 0xda, 0x2f, 0xbe, 0x03, 0xd1, 0x79, 0x21, 0x70, 0xa0, 0xf3, 0x00, 0x9c, 0xee, + }; + + var out: [exp_out.len]u8 = undefined; + var ctx = AES128.initEnc(key); + ctr(AESEncryptCtx(AES128), ctx, out[0..], in[0..], iv, builtin.Endian.Big); + testing.expectEqualSlices(u8, exp_out[0..], out[0..]); } test "encrypt" { @@ -256,8 +51,8 @@ test "encrypt" { const exp_out = [_]u8{ 0x39, 0x25, 0x84, 0x1d, 0x02, 0xdc, 0x09, 0xfb, 0xdc, 0x11, 0x85, 0x97, 0x19, 0x6a, 0x0b, 0x32 }; var out: [exp_out.len]u8 = undefined; - var aes = AES128.init(key); - aes.encrypt(out[0..], in[0..]); + var ctx = AES128.initEnc(key); + ctx.encrypt(out[0..], in[0..]); testing.expectEqualSlices(u8, exp_out[0..], out[0..]); } @@ -271,8 +66,8 @@ test "encrypt" { const exp_out = [_]u8{ 0x8e, 0xa2, 0xb7, 0xca, 0x51, 0x67, 0x45, 0xbf, 0xea, 0xfc, 0x49, 0x90, 0x4b, 0x49, 0x60, 0x89 }; var out: [exp_out.len]u8 = undefined; - var aes = AES256.init(key); - aes.encrypt(out[0..], in[0..]); + var ctx = AES256.initEnc(key); + ctx.encrypt(out[0..], in[0..]); testing.expectEqualSlices(u8, exp_out[0..], out[0..]); } } @@ -285,8 +80,8 @@ test "decrypt" { const exp_out = [_]u8{ 0x32, 0x43, 0xf6, 0xa8, 0x88, 0x5a, 0x30, 0x8d, 0x31, 0x31, 0x98, 0xa2, 0xe0, 0x37, 0x07, 0x34 }; var out: [exp_out.len]u8 = undefined; - var aes = AES128.init(key); - aes.decrypt(out[0..], in[0..]); + var ctx = AES128.initDec(key); + ctx.decrypt(out[0..], in[0..]); testing.expectEqualSlices(u8, exp_out[0..], out[0..]); } @@ -300,413 +95,52 @@ test "decrypt" { const exp_out = [_]u8{ 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff }; var out: [exp_out.len]u8 = undefined; - var aes = AES256.init(key); - aes.decrypt(out[0..], in[0..]); + var ctx = AES256.initDec(key); + ctx.decrypt(out[0..], in[0..]); testing.expectEqualSlices(u8, exp_out[0..], out[0..]); } } -// Key expansion algorithm. See FIPS-197, Figure 11. -fn expandKeyEncrypt(key: []const u8, enc: []u32) void { - var i: usize = 0; - var nk = key.len / 4; - while (i < nk) : (i += 1) { - enc[i] = mem.readIntBig(u32, key[4 * i ..][0..4]); - } - while (i < enc.len) : (i += 1) { - var t = enc[i - 1]; - if (i % nk == 0) { - t = subw(rotw(t)) ^ (@as(u32, powx[i / nk - 1]) << 24); - } else if (nk > 6 and i % nk == 4) { - t = subw(t); - } - enc[i] = enc[i - nk] ^ t; - } -} - -fn expandKeyDecrypt(enc: []const u32, dec: []u32) void { - var i: usize = 0; - var n = enc.len; - while (i < n) : (i += 4) { - var ei = n - i - 4; - var j: usize = 0; - while (j < 4) : (j += 1) { - var x = enc[ei + j]; - if (i > 0 and i + 4 < n) { - x = td0[sbox0[x >> 24]] ^ td1[sbox0[x >> 16 & 0xff]] ^ td2[sbox0[x >> 8 & 0xff]] ^ td3[sbox0[x & 0xff]]; - } - dec[i + j] = x; - } - } -} - -test "expand key" { +test "expand 128-bit key" { const key = [_]u8{ 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }; - const exp_enc = [_]u32{ - 0x2b7e1516, 0x28aed2a6, 0xabf71588, 0x09cf4f3c, - 0xa0fafe17, 0x88542cb1, 0x23a33939, 0x2a6c7605, - 0xf2c295f2, 0x7a96b943, 0x5935807a, 0x7359f67f, - 0x3d80477d, 0x4716fe3e, 0x1e237e44, 0x6d7a883b, - 0xef44a541, 0xa8525b7f, 0xb671253b, 0xdb0bad00, - 0xd4d1c6f8, 0x7c839d87, 0xcaf2b8bc, 0x11f915bc, - 0x6d88a37a, 0x110b3efd, 0xdbf98641, 0xca0093fd, - 0x4e54f70e, 0x5f5fc9f3, 0x84a64fb2, 0x4ea6dc4f, - 0xead27321, 0xb58dbad2, 0x312bf560, 0x7f8d292f, - 0xac7766f3, 0x19fadc21, 0x28d12941, 0x575c006e, - 0xd014f9a8, 0xc9ee2589, 0xe13f0cc8, 0xb6630ca6, + const exp_enc = [_]*const [32:0]u8{ + "2b7e151628aed2a6abf7158809cf4f3c", "a0fafe1788542cb123a339392a6c7605", "f2c295f27a96b9435935807a7359f67f", "3d80477d4716fe3e1e237e446d7a883b", "ef44a541a8525b7fb671253bdb0bad00", "d4d1c6f87c839d87caf2b8bc11f915bc", "6d88a37a110b3efddbf98641ca0093fd", "4e54f70e5f5fc9f384a64fb24ea6dc4f", "ead27321b58dbad2312bf5607f8d292f", "ac7766f319fadc2128d12941575c006e", "d014f9a8c9ee2589e13f0cc8b6630ca6", }; - const exp_dec = [_]u32{ - 0xd014f9a8, 0xc9ee2589, 0xe13f0cc8, 0xb6630ca6, - 0xc7b5a63, 0x1319eafe, 0xb0398890, 0x664cfbb4, - 0xdf7d925a, 0x1f62b09d, 0xa320626e, 0xd6757324, - 0x12c07647, 0xc01f22c7, 0xbc42d2f3, 0x7555114a, - 0x6efcd876, 0xd2df5480, 0x7c5df034, 0xc917c3b9, - 0x6ea30afc, 0xbc238cf6, 0xae82a4b4, 0xb54a338d, - 0x90884413, 0xd280860a, 0x12a12842, 0x1bc89739, - 0x7c1f13f7, 0x4208c219, 0xc021ae48, 0x969bf7b, - 0xcc7505eb, 0x3e17d1ee, 0x82296c51, 0xc9481133, - 0x2b3708a7, 0xf262d405, 0xbc3ebdbf, 0x4b617d62, - 0x2b7e1516, 0x28aed2a6, 0xabf71588, 0x9cf4f3c, + const exp_dec = [_]*const [32:0]u8{ + "2b7e151628aed2a6abf7158809cf4f3c", "a0fafe1788542cb123a339392a6c7605", "f2c295f27a96b9435935807a7359f67f", "3d80477d4716fe3e1e237e446d7a883b", "ef44a541a8525b7fb671253bdb0bad00", "d4d1c6f87c839d87caf2b8bc11f915bc", "6d88a37a110b3efddbf98641ca0093fd", "4e54f70e5f5fc9f384a64fb24ea6dc4f", "ead27321b58dbad2312bf5607f8d292f", "ac7766f319fadc2128d12941575c006e", "d014f9a8c9ee2589e13f0cc8b6630ca6", }; - var enc: [exp_enc.len]u32 = undefined; - var dec: [exp_dec.len]u32 = undefined; - expandKeyEncrypt(key[0..], enc[0..]); - expandKeyDecrypt(enc[0..], dec[0..]); - testing.expectEqualSlices(u32, exp_enc[0..], enc[0..]); - testing.expectEqualSlices(u32, exp_dec[0..], dec[0..]); + const enc = AES128.initEnc(key); + const dec = AES128.initDec(key); + var exp: [16]u8 = undefined; + + for (enc.key_schedule.round_keys) |round_key, i| { + try std.fmt.hexToBytes(&exp, exp_enc[i]); + testing.expectEqualSlices(u8, &exp, &round_key.toBytes()); + } + for (enc.key_schedule.round_keys) |round_key, i| { + try std.fmt.hexToBytes(&exp, exp_dec[i]); + testing.expectEqualSlices(u8, &exp, &round_key.toBytes()); + } } -// constants +test "expand 256-bit key" { + const key = [_]u8{ 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 }; + const exp_enc = [_]*const [32:0]u8{ + "603deb1015ca71be2b73aef0857d7781", "1f352c073b6108d72d9810a30914dff4", "9ba354118e6925afa51a8b5f2067fcde", "a8b09c1a93d194cdbe49846eb75d5b9a", "d59aecb85bf3c917fee94248de8ebe96", "b5a9328a2678a647983122292f6c79b3", "812c81addadf48ba24360af2fab8b464", "98c5bfc9bebd198e268c3ba709e04214", "68007bacb2df331696e939e46c518d80", "c814e20476a9fb8a5025c02d59c58239", "de1369676ccc5a71fa2563959674ee15", "5886ca5d2e2f31d77e0af1fa27cf73c3", "749c47ab18501ddae2757e4f7401905a", "cafaaae3e4d59b349adf6acebd10190d", "fe4890d1e6188d0b046df344706c631e", + }; + const exp_dec = [_]*const [32:0]u8{ + "fe4890d1e6188d0b046df344706c631e", "ada23f4963e23b2455427c8a5c709104", "57c96cf6074f07c0706abb07137f9241", "b668b621ce40046d36a047ae0932ed8e", "34ad1e4450866b367725bcc763152946", "32526c367828b24cf8e043c33f92aa20", "c440b289642b757227a3d7f114309581", "d669a7334a7ade7a80c8f18fc772e9e3", "25ba3c22a06bc7fb4388a28333934270", "54fb808b9c137949cab22ff547ba186c", "6c3d632985d1fbd9e3e36578701be0f3", "4a7459f9c8e8f9c256a156bc8d083799", "42107758e9ec98f066329ea193f8858b", "8ec6bff6829ca03b9e49af7edba96125", "603deb1015ca71be2b73aef0857d7781", + }; + const enc = AES256.initEnc(key); + const dec = AES256.initDec(key); + var exp: [16]u8 = undefined; -const poly = 1 << 8 | 1 << 4 | 1 << 3 | 1 << 1 | 1 << 0; - -const powx = [16]u8{ - 0x01, - 0x02, - 0x04, - 0x08, - 0x10, - 0x20, - 0x40, - 0x80, - 0x1b, - 0x36, - 0x6c, - 0xd8, - 0xab, - 0x4d, - 0x9a, - 0x2f, -}; - -const sbox0 = [256]u8{ - 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, - 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, - 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, - 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, - 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, - 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, - 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, - 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, - 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, - 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, - 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, - 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, - 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, - 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, - 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, - 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, -}; - -const sbox1 = [256]u8{ - 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, - 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, - 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, - 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, - 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, - 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, - 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, - 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, - 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, - 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, - 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, - 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, - 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, - 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, - 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, - 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, -}; - -const te0 = [256]u32{ - 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554, - 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, - 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87, 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, - 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b, - 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, - 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108, 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f, - 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5, - 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f, - 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e, 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, - 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497, - 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, - 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b, 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, - 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594, - 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3, - 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a, 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, - 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d, - 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, - 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47, 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, - 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883, - 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, - 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e, 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, - 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b, - 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, - 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25, 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818, - 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651, - 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, - 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa, 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, - 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9, - 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, - 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920, 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a, - 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8, - 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a, -}; -const te1 = [256]u32{ - 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, - 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, - 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, - 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, - 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, - 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, - 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, - 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, - 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, - 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, - 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, - 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, - 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, - 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, - 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, - 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, - 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, - 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, - 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, - 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, - 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, - 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, - 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, - 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, - 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, - 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, - 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, - 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, - 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, - 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, - 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, - 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, -}; -const te2 = [256]u32{ - 0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b, 0xf20dfff2, 0x6bbdd66b, 0x6fb1de6f, 0xc55491c5, - 0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b, 0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76, - 0xca458fca, 0x829d1f82, 0xc94089c9, 0x7d87fa7d, 0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0, - 0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af, 0x9cbf239c, 0xa4f753a4, 0x7296e472, 0xc05b9bc0, - 0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26, 0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc, - 0x345c6834, 0xa5f451a5, 0xe534d1e5, 0xf108f9f1, 0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15, - 0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3, 0x18283018, 0x96a13796, 0x050f0a05, 0x9ab52f9a, - 0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2, 0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75, - 0x091b1209, 0x839e1d83, 0x2c74582c, 0x1a2e341a, 0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0, - 0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3, 0x297b5229, 0xe33edde3, 0x2f715e2f, 0x84971384, - 0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed, 0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b, - 0x6abed46a, 0xcb468dcb, 0xbed967be, 0x394b7239, 0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf, - 0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb, 0x43c58643, 0x4dd79a4d, 0x33556633, 0x85941185, - 0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f, 0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8, - 0x51f3a251, 0xa3fe5da3, 0x40c08040, 0x8f8a058f, 0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5, - 0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221, 0x10302010, 0xff1ae5ff, 0xf30efdf3, 0xd26dbfd2, - 0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec, 0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17, - 0xc45793c4, 0xa7f255a7, 0x7e82fc7e, 0x3d477a3d, 0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673, - 0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc, 0x22664422, 0x2a7e542a, 0x90ab3b90, 0x88830b88, - 0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814, 0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb, - 0xe03bdbe0, 0x32566432, 0x3a4e743a, 0x0a1e140a, 0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c, - 0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462, 0x91a83991, 0x95a43195, 0xe437d3e4, 0x798bf279, - 0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d, 0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9, - 0x6cb4d86c, 0x56faac56, 0xf407f3f4, 0xea25cfea, 0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008, - 0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e, 0x1c24381c, 0xa6f157a6, 0xb4c773b4, 0xc65197c6, - 0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f, 0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a, - 0x7090e070, 0x3e427c3e, 0xb5c471b5, 0x66aacc66, 0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e, - 0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9, 0x86911786, 0xc15899c1, 0x1d273a1d, 0x9eb9279e, - 0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211, 0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394, - 0x9bb62d9b, 0x1e223c1e, 0x87921587, 0xe920c9e9, 0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df, - 0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d, 0xbfda65bf, 0xe631d7e6, 0x42c68442, 0x68b8d068, - 0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f, 0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16, -}; -const te3 = [256]u32{ - 0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6, 0xf2f20dff, 0x6b6bbdd6, 0x6f6fb1de, 0xc5c55491, - 0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56, 0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec, - 0xcaca458f, 0x82829d1f, 0xc9c94089, 0x7d7d87fa, 0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb, - 0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45, 0x9c9cbf23, 0xa4a4f753, 0x727296e4, 0xc0c05b9b, - 0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c, 0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83, - 0x34345c68, 0xa5a5f451, 0xe5e534d1, 0xf1f108f9, 0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a, - 0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d, 0x18182830, 0x9696a137, 0x05050f0a, 0x9a9ab52f, - 0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf, 0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea, - 0x09091b12, 0x83839e1d, 0x2c2c7458, 0x1a1a2e34, 0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b, - 0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d, 0x29297b52, 0xe3e33edd, 0x2f2f715e, 0x84849713, - 0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1, 0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6, - 0x6a6abed4, 0xcbcb468d, 0xbebed967, 0x39394b72, 0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85, - 0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed, 0x4343c586, 0x4d4dd79a, 0x33335566, 0x85859411, - 0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe, 0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b, - 0x5151f3a2, 0xa3a3fe5d, 0x4040c080, 0x8f8f8a05, 0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1, - 0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342, 0x10103020, 0xffff1ae5, 0xf3f30efd, 0xd2d26dbf, - 0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3, 0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e, - 0xc4c45793, 0xa7a7f255, 0x7e7e82fc, 0x3d3d477a, 0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6, - 0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3, 0x22226644, 0x2a2a7e54, 0x9090ab3b, 0x8888830b, - 0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28, 0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad, - 0xe0e03bdb, 0x32325664, 0x3a3a4e74, 0x0a0a1e14, 0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8, - 0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4, 0x9191a839, 0x9595a431, 0xe4e437d3, 0x79798bf2, - 0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da, 0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049, - 0x6c6cb4d8, 0x5656faac, 0xf4f407f3, 0xeaea25cf, 0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810, - 0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c, 0x1c1c2438, 0xa6a6f157, 0xb4b4c773, 0xc6c65197, - 0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e, 0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f, - 0x707090e0, 0x3e3e427c, 0xb5b5c471, 0x6666aacc, 0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c, - 0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069, 0x86869117, 0xc1c15899, 0x1d1d273a, 0x9e9eb927, - 0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322, 0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733, - 0x9b9bb62d, 0x1e1e223c, 0x87879215, 0xe9e920c9, 0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5, - 0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a, 0xbfbfda65, 0xe6e631d7, 0x4242c684, 0x6868b8d0, - 0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e, 0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c, -}; - -const td0 = [256]u32{ - 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96, 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393, - 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25, 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f, - 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1, 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6, - 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da, 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844, - 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd, 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4, - 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45, 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94, - 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7, 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a, - 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5, 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c, - 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1, 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a, - 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75, 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051, - 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46, 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff, - 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77, 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb, - 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000, 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e, - 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927, 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a, - 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e, 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16, - 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d, 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8, - 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd, 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34, - 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163, 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120, - 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d, 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0, - 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422, 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef, - 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36, 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4, - 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662, 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5, - 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3, 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b, - 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8, 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6, - 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6, 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0, - 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815, 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f, - 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df, 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f, - 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e, 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713, - 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89, 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c, - 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf, 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86, - 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f, 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541, - 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190, 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742, -}; -const td1 = [256]u32{ - 0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e, 0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303, - 0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c, 0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3, - 0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0, 0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9, - 0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259, 0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8, - 0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971, 0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a, - 0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f, 0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b, - 0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8, 0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab, - 0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708, 0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682, - 0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2, 0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe, - 0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb, 0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10, - 0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd, 0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015, - 0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e, 0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee, - 0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000, 0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72, - 0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39, 0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e, - 0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91, 0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a, - 0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17, 0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9, - 0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60, 0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e, - 0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1, 0xcad731dc, 0x10426385, 0x40139722, 0x2084c611, - 0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1, 0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3, - 0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964, 0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390, - 0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b, 0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf, - 0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46, 0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af, - 0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512, 0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb, - 0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a, 0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8, - 0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c, 0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266, - 0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8, 0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6, - 0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604, 0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551, - 0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41, 0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647, - 0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c, 0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1, - 0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737, 0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db, - 0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340, 0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95, - 0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1, 0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857, -}; -const td2 = [256]u32{ - 0xa75051f4, 0x65537e41, 0xa4c31a17, 0x5e963a27, 0x6bcb3bab, 0x45f11f9d, 0x58abacfa, 0x03934be3, - 0xfa552030, 0x6df6ad76, 0x769188cc, 0x4c25f502, 0xd7fc4fe5, 0xcbd7c52a, 0x44802635, 0xa38fb562, - 0x5a49deb1, 0x1b6725ba, 0x0e9845ea, 0xc0e15dfe, 0x7502c32f, 0xf012814c, 0x97a38d46, 0xf9c66bd3, - 0x5fe7038f, 0x9c951592, 0x7aebbf6d, 0x59da9552, 0x832dd4be, 0x21d35874, 0x692949e0, 0xc8448ec9, - 0x896a75c2, 0x7978f48e, 0x3e6b9958, 0x71dd27b9, 0x4fb6bee1, 0xad17f088, 0xac66c920, 0x3ab47dce, - 0x4a1863df, 0x3182e51a, 0x33609751, 0x7f456253, 0x77e0b164, 0xae84bb6b, 0xa01cfe81, 0x2b94f908, - 0x68587048, 0xfd198f45, 0x6c8794de, 0xf8b7527b, 0xd323ab73, 0x02e2724b, 0x8f57e31f, 0xab2a6655, - 0x2807b2eb, 0xc2032fb5, 0x7b9a86c5, 0x08a5d337, 0x87f23028, 0xa5b223bf, 0x6aba0203, 0x825ced16, - 0x1c2b8acf, 0xb492a779, 0xf2f0f307, 0xe2a14e69, 0xf4cd65da, 0xbed50605, 0x621fd134, 0xfe8ac4a6, - 0x539d342e, 0x55a0a2f3, 0xe132058a, 0xeb75a4f6, 0xec390b83, 0xefaa4060, 0x9f065e71, 0x1051bd6e, - 0x8af93e21, 0x063d96dd, 0x05aedd3e, 0xbd464de6, 0x8db59154, 0x5d0571c4, 0xd46f0406, 0x15ff6050, - 0xfb241998, 0xe997d6bd, 0x43cc8940, 0x9e7767d9, 0x42bdb0e8, 0x8b880789, 0x5b38e719, 0xeedb79c8, - 0x0a47a17c, 0x0fe97c42, 0x1ec9f884, 0x00000000, 0x86830980, 0xed48322b, 0x70ac1e11, 0x724e6c5a, - 0xfffbfd0e, 0x38560f85, 0xd51e3dae, 0x3927362d, 0xd9640a0f, 0xa621685c, 0x54d19b5b, 0x2e3a2436, - 0x67b10c0a, 0xe70f9357, 0x96d2b4ee, 0x919e1b9b, 0xc54f80c0, 0x20a261dc, 0x4b695a77, 0x1a161c12, - 0xba0ae293, 0x2ae5c0a0, 0xe0433c22, 0x171d121b, 0x0d0b0e09, 0xc7adf28b, 0xa8b92db6, 0xa9c8141e, - 0x198557f1, 0x074caf75, 0xddbbee99, 0x60fda37f, 0x269ff701, 0xf5bc5c72, 0x3bc54466, 0x7e345bfb, - 0x29768b43, 0xc6dccb23, 0xfc68b6ed, 0xf163b8e4, 0xdccad731, 0x85104263, 0x22401397, 0x112084c6, - 0x247d854a, 0x3df8d2bb, 0x3211aef9, 0xa16dc729, 0x2f4b1d9e, 0x30f3dcb2, 0x52ec0d86, 0xe3d077c1, - 0x166c2bb3, 0xb999a970, 0x48fa1194, 0x642247e9, 0x8cc4a8fc, 0x3f1aa0f0, 0x2cd8567d, 0x90ef2233, - 0x4ec78749, 0xd1c1d938, 0xa2fe8cca, 0x0b3698d4, 0x81cfa6f5, 0xde28a57a, 0x8e26dab7, 0xbfa43fad, - 0x9de42c3a, 0x920d5078, 0xcc9b6a5f, 0x4662547e, 0x13c2f68d, 0xb8e890d8, 0xf75e2e39, 0xaff582c3, - 0x80be9f5d, 0x937c69d0, 0x2da96fd5, 0x12b3cf25, 0x993bc8ac, 0x7da71018, 0x636ee89c, 0xbb7bdb3b, - 0x7809cd26, 0x18f46e59, 0xb701ec9a, 0x9aa8834f, 0x6e65e695, 0xe67eaaff, 0xcf0821bc, 0xe8e6ef15, - 0x9bd9bae7, 0x36ce4a6f, 0x09d4ea9f, 0x7cd629b0, 0xb2af31a4, 0x23312a3f, 0x9430c6a5, 0x66c035a2, - 0xbc37744e, 0xcaa6fc82, 0xd0b0e090, 0xd81533a7, 0x984af104, 0xdaf741ec, 0x500e7fcd, 0xf62f1791, - 0xd68d764d, 0xb04d43ef, 0x4d54ccaa, 0x04dfe496, 0xb5e39ed1, 0x881b4c6a, 0x1fb8c12c, 0x517f4665, - 0xea049d5e, 0x355d018c, 0x7473fa87, 0x412efb0b, 0x1d5ab367, 0xd25292db, 0x5633e910, 0x47136dd6, - 0x618c9ad7, 0x0c7a37a1, 0x148e59f8, 0x3c89eb13, 0x27eecea9, 0xc935b761, 0xe5ede11c, 0xb13c7a47, - 0xdf599cd2, 0x733f55f2, 0xce791814, 0x37bf73c7, 0xcdea53f7, 0xaa5b5ffd, 0x6f14df3d, 0xdb867844, - 0xf381caaf, 0xc43eb968, 0x342c3824, 0x405fc2a3, 0xc372161d, 0x250cbce2, 0x498b283c, 0x9541ff0d, - 0x017139a8, 0xb3de080c, 0xe49cd8b4, 0xc1906456, 0x84617bcb, 0xb670d532, 0x5c74486c, 0x5742d0b8, -}; -const td3 = [256]u32{ - 0xf4a75051, 0x4165537e, 0x17a4c31a, 0x275e963a, 0xab6bcb3b, 0x9d45f11f, 0xfa58abac, 0xe303934b, - 0x30fa5520, 0x766df6ad, 0xcc769188, 0x024c25f5, 0xe5d7fc4f, 0x2acbd7c5, 0x35448026, 0x62a38fb5, - 0xb15a49de, 0xba1b6725, 0xea0e9845, 0xfec0e15d, 0x2f7502c3, 0x4cf01281, 0x4697a38d, 0xd3f9c66b, - 0x8f5fe703, 0x929c9515, 0x6d7aebbf, 0x5259da95, 0xbe832dd4, 0x7421d358, 0xe0692949, 0xc9c8448e, - 0xc2896a75, 0x8e7978f4, 0x583e6b99, 0xb971dd27, 0xe14fb6be, 0x88ad17f0, 0x20ac66c9, 0xce3ab47d, - 0xdf4a1863, 0x1a3182e5, 0x51336097, 0x537f4562, 0x6477e0b1, 0x6bae84bb, 0x81a01cfe, 0x082b94f9, - 0x48685870, 0x45fd198f, 0xde6c8794, 0x7bf8b752, 0x73d323ab, 0x4b02e272, 0x1f8f57e3, 0x55ab2a66, - 0xeb2807b2, 0xb5c2032f, 0xc57b9a86, 0x3708a5d3, 0x2887f230, 0xbfa5b223, 0x036aba02, 0x16825ced, - 0xcf1c2b8a, 0x79b492a7, 0x07f2f0f3, 0x69e2a14e, 0xdaf4cd65, 0x05bed506, 0x34621fd1, 0xa6fe8ac4, - 0x2e539d34, 0xf355a0a2, 0x8ae13205, 0xf6eb75a4, 0x83ec390b, 0x60efaa40, 0x719f065e, 0x6e1051bd, - 0x218af93e, 0xdd063d96, 0x3e05aedd, 0xe6bd464d, 0x548db591, 0xc45d0571, 0x06d46f04, 0x5015ff60, - 0x98fb2419, 0xbde997d6, 0x4043cc89, 0xd99e7767, 0xe842bdb0, 0x898b8807, 0x195b38e7, 0xc8eedb79, - 0x7c0a47a1, 0x420fe97c, 0x841ec9f8, 0x00000000, 0x80868309, 0x2bed4832, 0x1170ac1e, 0x5a724e6c, - 0x0efffbfd, 0x8538560f, 0xaed51e3d, 0x2d392736, 0x0fd9640a, 0x5ca62168, 0x5b54d19b, 0x362e3a24, - 0x0a67b10c, 0x57e70f93, 0xee96d2b4, 0x9b919e1b, 0xc0c54f80, 0xdc20a261, 0x774b695a, 0x121a161c, - 0x93ba0ae2, 0xa02ae5c0, 0x22e0433c, 0x1b171d12, 0x090d0b0e, 0x8bc7adf2, 0xb6a8b92d, 0x1ea9c814, - 0xf1198557, 0x75074caf, 0x99ddbbee, 0x7f60fda3, 0x01269ff7, 0x72f5bc5c, 0x663bc544, 0xfb7e345b, - 0x4329768b, 0x23c6dccb, 0xedfc68b6, 0xe4f163b8, 0x31dccad7, 0x63851042, 0x97224013, 0xc6112084, - 0x4a247d85, 0xbb3df8d2, 0xf93211ae, 0x29a16dc7, 0x9e2f4b1d, 0xb230f3dc, 0x8652ec0d, 0xc1e3d077, - 0xb3166c2b, 0x70b999a9, 0x9448fa11, 0xe9642247, 0xfc8cc4a8, 0xf03f1aa0, 0x7d2cd856, 0x3390ef22, - 0x494ec787, 0x38d1c1d9, 0xcaa2fe8c, 0xd40b3698, 0xf581cfa6, 0x7ade28a5, 0xb78e26da, 0xadbfa43f, - 0x3a9de42c, 0x78920d50, 0x5fcc9b6a, 0x7e466254, 0x8d13c2f6, 0xd8b8e890, 0x39f75e2e, 0xc3aff582, - 0x5d80be9f, 0xd0937c69, 0xd52da96f, 0x2512b3cf, 0xac993bc8, 0x187da710, 0x9c636ee8, 0x3bbb7bdb, - 0x267809cd, 0x5918f46e, 0x9ab701ec, 0x4f9aa883, 0x956e65e6, 0xffe67eaa, 0xbccf0821, 0x15e8e6ef, - 0xe79bd9ba, 0x6f36ce4a, 0x9f09d4ea, 0xb07cd629, 0xa4b2af31, 0x3f23312a, 0xa59430c6, 0xa266c035, - 0x4ebc3774, 0x82caa6fc, 0x90d0b0e0, 0xa7d81533, 0x04984af1, 0xecdaf741, 0xcd500e7f, 0x91f62f17, - 0x4dd68d76, 0xefb04d43, 0xaa4d54cc, 0x9604dfe4, 0xd1b5e39e, 0x6a881b4c, 0x2c1fb8c1, 0x65517f46, - 0x5eea049d, 0x8c355d01, 0x877473fa, 0x0b412efb, 0x671d5ab3, 0xdbd25292, 0x105633e9, 0xd647136d, - 0xd7618c9a, 0xa10c7a37, 0xf8148e59, 0x133c89eb, 0xa927eece, 0x61c935b7, 0x1ce5ede1, 0x47b13c7a, - 0xd2df599c, 0xf2733f55, 0x14ce7918, 0xc737bf73, 0xf7cdea53, 0xfdaa5b5f, 0x3d6f14df, 0x44db8678, - 0xaff381ca, 0x68c43eb9, 0x24342c38, 0xa3405fc2, 0x1dc37216, 0xe2250cbc, 0x3c498b28, 0x0d9541ff, - 0xa8017139, 0x0cb3de08, 0xb4e49cd8, 0x56c19064, 0xcb84617b, 0x32b670d5, 0x6c5c7448, 0xb85742d0, -}; + for (enc.key_schedule.round_keys) |round_key, i| { + try std.fmt.hexToBytes(&exp, exp_enc[i]); + testing.expectEqualSlices(u8, &exp, &round_key.toBytes()); + } + for (dec.key_schedule.round_keys) |round_key, i| { + try std.fmt.hexToBytes(&exp, exp_dec[i]); + testing.expectEqualSlices(u8, &exp, &round_key.toBytes()); + } +} diff --git a/lib/std/crypto/aes/aesni.zig b/lib/std/crypto/aes/aesni.zig new file mode 100644 index 0000000000..f13a4a581c --- /dev/null +++ b/lib/std/crypto/aes/aesni.zig @@ -0,0 +1,420 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. +// Based on Go stdlib implementation + +const std = @import("../../std.zig"); +const mem = std.mem; +const debug = std.debug; +const Vector = std.meta.Vector; + +const BlockVec = Vector(2, u64); + +/// A single AES block. +pub const Block = struct { + pub const block_size: usize = 16; + + /// Internal representation of a block. + repr: BlockVec, + + /// Convert a byte sequence into an internal representation. + pub inline fn fromBytes(bytes: *const [16]u8) Block { + const repr = mem.bytesToValue(BlockVec, bytes); + return Block{ .repr = repr }; + } + + /// Convert the internal representation of a block into a byte sequence. + pub inline fn toBytes(block: Block) [16]u8 { + return mem.toBytes(block.repr); + } + + /// XOR the block with a byte sequence. + pub inline fn xorBytes(block: Block, bytes: *const [16]u8) [16]u8 { + const x = block.repr ^ fromBytes(bytes).repr; + return mem.toBytes(x); + } + + /// Encrypt a block with a round key. + pub inline fn encrypt(block: Block, round_key: Block) Block { + return Block{ + .repr = asm ( + \\ vaesenc %[rk], %[in], %[out] + : [out] "=x" (-> BlockVec) + : [in] "x" (block.repr), + [rk] "x" (round_key.repr) + ), + }; + } + + /// Encrypt a block with the last round key. + pub inline fn encryptLast(block: Block, round_key: Block) Block { + return Block{ + .repr = asm ( + \\ vaesenclast %[rk], %[in], %[out] + : [out] "=x" (-> BlockVec) + : [in] "x" (block.repr), + [rk] "x" (round_key.repr) + ), + }; + } + + /// Decrypt a block with a round key. + pub inline fn decrypt(block: Block, inv_round_key: Block) Block { + return Block{ + .repr = asm ( + \\ vaesdec %[rk], %[in], %[out] + : [out] "=x" (-> BlockVec) + : [in] "x" (block.repr), + [rk] "x" (inv_round_key.repr) + ), + }; + } + + /// Decrypt a block with the last round key. + pub inline fn decryptLast(block: Block, inv_round_key: Block) Block { + return Block{ + .repr = asm ( + \\ vaesdeclast %[rk], %[in], %[out] + : [out] "=x" (-> BlockVec) + : [in] "x" (block.repr), + [rk] "x" (inv_round_key.repr) + ), + }; + } + + /// XOR the content of two blocks. + pub inline fn xor(block1: Block, block2: Block) Block { + return Block{ .repr = block1.repr ^ block2.repr }; + } + + /// Perform operations on multiple blocks in parallel. + pub const parallel = struct { + /// The recommended number of AES encryption/decryption to perform in parallel for the chosen implementation. + pub const optimal_parallel_blocks = 8; + + /// Encrypt multiple blocks in parallel, each their own round key. + pub inline fn encryptParallel(comptime count: usize, blocks: [count]Block, round_keys: [count]Block) [count]Block { + comptime var i = 0; + var out: [count]Block = undefined; + inline while (i < count) : (i += 1) { + out[i] = blocks[i].encrypt(round_keys[i]); + } + return out; + } + + /// Decrypt multiple blocks in parallel, each their own round key. + pub inline fn decryptParallel(comptime count: usize, blocks: [count]Block, round_keys: [count]Block) [count]Block { + comptime var i = 0; + var out: [count]Block = undefined; + inline while (i < count) : (i += 1) { + out[i] = blocks[i].decrypt(round_keys[i]); + } + return out; + } + + /// Encrypt multple blocks in parallel with the same round key. + pub inline fn encryptWide(comptime count: usize, blocks: [count]Block, round_key: Block) [count]Block { + comptime var i = 0; + var out: [count]Block = undefined; + inline while (i < count) : (i += 1) { + out[i] = blocks[i].encrypt(round_key); + } + return out; + } + + /// Decrypt multple blocks in parallel with the same round key. + pub inline fn decryptWide(comptime count: usize, blocks: [count]Block, round_key: Block) [count]Block { + comptime var i = 0; + var out: [count]Block = undefined; + inline while (i < count) : (i += 1) { + out[i] = blocks[i].decrypt(round_key); + } + return out; + } + + /// Encrypt multple blocks in parallel with the same last round key. + pub inline fn encryptLastWide(comptime count: usize, blocks: [count]Block, round_key: Block) [count]Block { + comptime var i = 0; + var out: [count]Block = undefined; + inline while (i < count) : (i += 1) { + out[i] = blocks[i].encryptLast(round_key); + } + return out; + } + + /// Decrypt multple blocks in parallel with the same last round key. + pub inline fn decryptLastWide(comptime count: usize, blocks: [count]Block, round_key: Block) [count]Block { + comptime var i = 0; + var out: [count]Block = undefined; + inline while (i < count) : (i += 1) { + out[i] = blocks[i].decryptLast(round_key); + } + return out; + } + }; +}; + +fn KeySchedule(comptime AES: type) type { + std.debug.assert(AES.rounds == 10 or AES.rounds == 14); + const rounds = AES.rounds; + + return struct { + const Self = @This(); + round_keys: [rounds + 1]Block, + + fn drc(comptime second: bool, comptime rc: u8, t: BlockVec, tx: BlockVec) BlockVec { + var s: BlockVec = undefined; + var ts: BlockVec = undefined; + return asm ( + \\ vaeskeygenassist %[rc], %[t], %[s] + \\ vpslldq $4, %[tx], %[ts] + \\ vpxor %[ts], %[tx], %[r] + \\ vpslldq $8, %[r], %[ts] + \\ vpxor %[ts], %[r], %[r] + \\ vpshufd %[mask], %[s], %[ts] + \\ vpxor %[ts], %[r], %[r] + : [r] "=&x" (-> BlockVec), + [s] "=&x" (s), + [ts] "=&x" (ts) + : [rc] "n" (rc), + [t] "x" (t), + [tx] "x" (tx), + [mask] "n" (@as(u8, if (second) 0xaa else 0xff)) + ); + } + + fn expand128(t1: *Block) Self { + var round_keys: [11]Block = undefined; + const rcs = [_]u8{ 1, 2, 4, 8, 16, 32, 64, 128, 27, 54 }; + inline for (rcs) |rc, round| { + round_keys[round] = t1.*; + t1.repr = drc(false, rc, t1.repr, t1.repr); + } + round_keys[rcs.len] = t1.*; + return Self{ .round_keys = round_keys }; + } + + fn expand256(t1: *Block, t2: *Block) Self { + var round_keys: [15]Block = undefined; + const rcs = [_]u8{ 1, 2, 4, 8, 16, 32 }; + round_keys[0] = t1.*; + inline for (rcs) |rc, round| { + round_keys[round * 2 + 1] = t2.*; + t1.repr = drc(false, rc, t2.repr, t1.repr); + round_keys[round * 2 + 2] = t1.*; + t2.repr = drc(true, rc, t1.repr, t2.repr); + } + round_keys[rcs.len * 2 + 1] = t2.*; + t1.repr = drc(false, 64, t2.repr, t1.repr); + round_keys[rcs.len * 2 + 2] = t1.*; + return Self{ .round_keys = round_keys }; + } + + /// Invert the key schedule. + pub fn invert(key_schedule: Self) Self { + const round_keys = &key_schedule.round_keys; + var inv_round_keys: [rounds + 1]Block = undefined; + inv_round_keys[0] = round_keys[rounds]; + comptime var i = 1; + inline while (i < rounds) : (i += 1) { + inv_round_keys[i] = Block{ + .repr = asm ( + \\ vaesimc %[rk], %[inv_rk] + : [inv_rk] "=x" (-> BlockVec) + : [rk] "x" (round_keys[rounds - i].repr) + ), + }; + } + inv_round_keys[rounds] = round_keys[0]; + return Self{ .round_keys = inv_round_keys }; + } + }; +} + +/// A context to perform encryption using the standard AES key schedule. +pub fn AESEncryptCtx(comptime AES: type) type { + std.debug.assert(AES.key_bits == 128 or AES.key_bits == 256); + const rounds = AES.rounds; + + return struct { + const Self = @This(); + pub const block = AES.block; + pub const block_size = block.block_size; + key_schedule: KeySchedule(AES), + + /// Create a new encryption context with the given key. + pub fn init(key: [AES.key_bits / 8]u8) Self { + var t1 = Block.fromBytes(key[0..16]); + const key_schedule = if (AES.key_bits == 128) ks: { + break :ks KeySchedule(AES).expand128(&t1); + } else ks: { + var t2 = Block.fromBytes(key[16..32]); + break :ks KeySchedule(AES).expand256(&t1, &t2); + }; + return Self{ + .key_schedule = key_schedule, + }; + } + + /// Encrypt a single block. + pub fn encrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void { + const round_keys = ctx.key_schedule.round_keys; + var t = Block.fromBytes(src).xor(round_keys[0]); + comptime var i = 1; + inline while (i < rounds) : (i += 1) { + t = t.encrypt(round_keys[i]); + } + t = t.encryptLast(round_keys[rounds]); + dst.* = t.toBytes(); + } + + /// Encrypt+XOR a single block. + pub fn xor(ctx: Self, dst: *[16]u8, src: *const [16]u8, counter: [16]u8) void { + const round_keys = ctx.key_schedule.round_keys; + var t = Block.fromBytes(&counter).xor(round_keys[0]); + comptime var i = 1; + inline while (i < rounds) : (i += 1) { + t = t.encrypt(round_keys[i]); + } + t = t.encryptLast(round_keys[rounds]); + dst.* = t.xorBytes(src); + } + + /// Encrypt multiple blocks, possibly leveraging parallelization. + pub fn encryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void { + const round_keys = ctx.key_schedule.round_keys; + var ts: [count]Block = undefined; + comptime var j = 0; + inline while (j < count) : (j += 1) { + ts[j] = Block.fromBytes(src[j * 16 .. j * 16 + 16][0..16]).xor(round_keys[0]); + } + comptime var i = 1; + inline while (i < rounds) : (i += 1) { + ts = Block.parallel.encryptWide(count, ts, round_keys[i]); + } + i = 1; + inline while (i < count) : (i += 1) { + ts = Block.parallel.encryptLastWide(count, ts, round_keys[i]); + } + j = 0; + inline while (j < count) : (j += 1) { + dst[16 * j .. 16 * j + 16].* = ts[j].toBytes(); + } + } + + /// Encrypt+XOR multiple blocks, possibly leveraging parallelization. + pub fn xorWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8, counters: [16 * count]u8) void { + const round_keys = ctx.key_schedule.round_keys; + var ts: [count]Block = undefined; + comptime var j = 0; + inline while (j < count) : (j += 1) { + ts[j] = Block.fromBytes(counters[j * 16 .. j * 16 + 16][0..16]).xor(round_keys[0]); + } + comptime var i = 1; + inline while (i < rounds) : (i += 1) { + ts = Block.parallel.encryptWide(count, ts, round_keys[i]); + } + ts = Block.parallel.encryptLastWide(count, ts, round_keys[i]); + j = 0; + inline while (j < count) : (j += 1) { + dst[16 * j .. 16 * j + 16].* = ts[j].xorBytes(src[16 * j .. 16 * j + 16]); + } + } + }; +} + +/// A context to perform decryption using the standard AES key schedule. +pub fn AESDecryptCtx(comptime AES: type) type { + std.debug.assert(AES.key_bits == 128 or AES.key_bits == 256); + const rounds = AES.rounds; + + return struct { + const Self = @This(); + pub const block = AES.block; + pub const block_size = block.block_size; + key_schedule: KeySchedule(AES), + + /// Create a decryption context from an existing encryption context. + pub fn initFromEnc(ctx: AESEncryptCtx(AES)) Self { + return Self{ + .key_schedule = ctx.key_schedule.invert(), + }; + } + + /// Create a new decryption context with the given key. + pub fn init(key: [AES.key_bits / 8]u8) Self { + const enc_ctx = AESEncryptCtx(AES).init(key); + return initFromEnc(enc_ctx); + } + + /// Decrypt a single block. + pub fn decrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void { + const inv_round_keys = ctx.key_schedule.round_keys; + var t = Block.fromBytes(src).xor(inv_round_keys[0]); + comptime var i = 1; + inline while (i < rounds) : (i += 1) { + t = t.decrypt(inv_round_keys[i]); + } + t = t.decryptLast(inv_round_keys[rounds]); + dst.* = t.toBytes(); + } + + /// Decrypt multiple blocks, possibly leveraging parallelization. + pub fn decryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void { + const inv_round_keys = ctx.key_schedule.round_keys; + var ts: [count]Block = undefined; + comptime var j = 0; + inline while (j < count) : (j += 1) { + ts[j] = Block.fromBytes(src[j * 16 .. j * 16 + 16][0..16]).xor(inv_round_keys[0]); + } + comptime var i = 1; + inline while (i < rounds) : (i += 1) { + ts = Block.parallel.decryptWide(count, ts, inv_round_keys[i]); + } + i = 1; + inline while (i < count) : (i += 1) { + ts = Block.parallel.decryptLastWide(count, ts, inv_round_keys[i]); + } + j = 0; + inline while (j < count) : (j += 1) { + dst[16 * j .. 16 * j + 16].* = ts[j].toBytes(); + } + } + }; +} + +/// AES-128 with the standard key schedule. +pub const AES128 = struct { + pub const key_bits: usize = 128; + pub const rounds = ((key_bits - 64) / 32 + 8); + pub const block = Block; + + /// Create a new context for encryption. + pub fn initEnc(key: [key_bits / 8]u8) AESEncryptCtx(AES128) { + return AESEncryptCtx(AES128).init(key); + } + + /// Create a new context for decryption. + pub fn initDec(key: [key_bits / 8]u8) AESDecryptCtx(AES128) { + return AESDecryptCtx(AES128).init(key); + } +}; + +/// AES-256 with the standard key schedule. +pub const AES256 = struct { + pub const key_bits: usize = 256; + pub const rounds = ((key_bits - 64) / 32 + 8); + pub const block = Block; + + /// Create a new context for encryption. + pub fn initEnc(key: [key_bits / 8]u8) AESEncryptCtx(AES256) { + return AESEncryptCtx(AES256).init(key); + } + + /// Create a new context for decryption. + pub fn initDec(key: [key_bits / 8]u8) AESDecryptCtx(AES256) { + return AESDecryptCtx(AES256).init(key); + } +}; diff --git a/lib/std/crypto/aes/soft.zig b/lib/std/crypto/aes/soft.zig new file mode 100644 index 0000000000..c32662fbc5 --- /dev/null +++ b/lib/std/crypto/aes/soft.zig @@ -0,0 +1,735 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. +// Based on Go stdlib implementation + +const std = @import("../../std.zig"); +const mem = std.mem; + +const BlockVec = [4]u32; + +/// A single AES block. +pub const Block = struct { + pub const block_size: usize = 16; + + /// Internal representation of a block. + repr: BlockVec align(16), + + /// Convert a byte sequence into an internal representation. + pub inline fn fromBytes(bytes: *const [16]u8) Block { + const s0 = mem.readIntBig(u32, bytes[0..4]); + const s1 = mem.readIntBig(u32, bytes[4..8]); + const s2 = mem.readIntBig(u32, bytes[8..12]); + const s3 = mem.readIntBig(u32, bytes[12..16]); + return Block{ .repr = BlockVec{ s0, s1, s2, s3 } }; + } + + /// Convert the internal representation of a block into a byte sequence. + pub inline fn toBytes(block: Block) [16]u8 { + var bytes: [16]u8 = undefined; + mem.writeIntBig(u32, bytes[0..4], block.repr[0]); + mem.writeIntBig(u32, bytes[4..8], block.repr[1]); + mem.writeIntBig(u32, bytes[8..12], block.repr[2]); + mem.writeIntBig(u32, bytes[12..16], block.repr[3]); + return bytes; + } + + /// XOR the block with a byte sequence. + pub inline fn xorBytes(block: Block, bytes: *const [16]u8) [16]u8 { + const block_bytes = block.toBytes(); + var x: [16]u8 = undefined; + comptime var i: usize = 0; + inline while (i < 16) : (i += 1) { + x[i] = block_bytes[i] ^ bytes[i]; + } + return x; + } + + /// Encrypt a block with a round key. + pub inline fn encrypt(block: Block, round_key: Block) Block { + const src = &block.repr; + + const s0 = block.repr[0]; + const s1 = block.repr[1]; + const s2 = block.repr[2]; + const s3 = block.repr[3]; + + const t0 = round_key.repr[0] ^ te0[@truncate(u8, s0 >> 24)] ^ te1[@truncate(u8, s1 >> 16)] ^ te2[@truncate(u8, s2 >> 8)] ^ te3[@truncate(u8, s3)]; + const t1 = round_key.repr[1] ^ te0[@truncate(u8, s1 >> 24)] ^ te1[@truncate(u8, s2 >> 16)] ^ te2[@truncate(u8, s3 >> 8)] ^ te3[@truncate(u8, s0)]; + const t2 = round_key.repr[2] ^ te0[@truncate(u8, s2 >> 24)] ^ te1[@truncate(u8, s3 >> 16)] ^ te2[@truncate(u8, s0 >> 8)] ^ te3[@truncate(u8, s1)]; + const t3 = round_key.repr[3] ^ te0[@truncate(u8, s3 >> 24)] ^ te1[@truncate(u8, s0 >> 16)] ^ te2[@truncate(u8, s1 >> 8)] ^ te3[@truncate(u8, s2)]; + + return Block{ .repr = BlockVec{ t0, t1, t2, t3 } }; + } + + /// Encrypt a block with the last round key. + pub inline fn encryptLast(block: Block, round_key: Block) Block { + const src = &block.repr; + + const t0 = block.repr[0]; + const t1 = block.repr[1]; + const t2 = block.repr[2]; + const t3 = block.repr[3]; + + // Last round uses s-box directly and XORs to produce output. + var s0 = @as(u32, sbox0[t0 >> 24]) << 24 | @as(u32, sbox0[t1 >> 16 & 0xff]) << 16 | @as(u32, sbox0[t2 >> 8 & 0xff]) << 8 | @as(u32, sbox0[t3 & 0xff]); + var s1 = @as(u32, sbox0[t1 >> 24]) << 24 | @as(u32, sbox0[t2 >> 16 & 0xff]) << 16 | @as(u32, sbox0[t3 >> 8 & 0xff]) << 8 | @as(u32, sbox0[t0 & 0xff]); + var s2 = @as(u32, sbox0[t2 >> 24]) << 24 | @as(u32, sbox0[t3 >> 16 & 0xff]) << 16 | @as(u32, sbox0[t0 >> 8 & 0xff]) << 8 | @as(u32, sbox0[t1 & 0xff]); + var s3 = @as(u32, sbox0[t3 >> 24]) << 24 | @as(u32, sbox0[t0 >> 16 & 0xff]) << 16 | @as(u32, sbox0[t1 >> 8 & 0xff]) << 8 | @as(u32, sbox0[t2 & 0xff]); + s0 ^= round_key.repr[0]; + s1 ^= round_key.repr[1]; + s2 ^= round_key.repr[2]; + s3 ^= round_key.repr[3]; + + return Block{ .repr = BlockVec{ s0, s1, s2, s3 } }; + } + + /// Decrypt a block with a round key. + pub inline fn decrypt(block: Block, round_key: Block) Block { + const src = &block.repr; + + const s0 = block.repr[0]; + const s1 = block.repr[1]; + const s2 = block.repr[2]; + const s3 = block.repr[3]; + + const t0 = round_key.repr[0] ^ td0[@truncate(u8, s0 >> 24)] ^ td1[@truncate(u8, s3 >> 16)] ^ td2[@truncate(u8, s2 >> 8)] ^ td3[@truncate(u8, s1)]; + const t1 = round_key.repr[1] ^ td0[@truncate(u8, s1 >> 24)] ^ td1[@truncate(u8, s0 >> 16)] ^ td2[@truncate(u8, s3 >> 8)] ^ td3[@truncate(u8, s2)]; + const t2 = round_key.repr[2] ^ td0[@truncate(u8, s2 >> 24)] ^ td1[@truncate(u8, s1 >> 16)] ^ td2[@truncate(u8, s0 >> 8)] ^ td3[@truncate(u8, s3)]; + const t3 = round_key.repr[3] ^ td0[@truncate(u8, s3 >> 24)] ^ td1[@truncate(u8, s2 >> 16)] ^ td2[@truncate(u8, s1 >> 8)] ^ td3[@truncate(u8, s0)]; + + return Block{ .repr = BlockVec{ t0, t1, t2, t3 } }; + } + + /// Decrypt a block with the last round key. + pub inline fn decryptLast(block: Block, round_key: Block) Block { + const src = &block.repr; + + const t0 = block.repr[0]; + const t1 = block.repr[1]; + const t2 = block.repr[2]; + const t3 = block.repr[3]; + + // Last round uses s-box directly and XORs to produce output. + var s0 = @as(u32, sbox1[t0 >> 24]) << 24 | @as(u32, sbox1[t3 >> 16 & 0xff]) << 16 | @as(u32, sbox1[t2 >> 8 & 0xff]) << 8 | @as(u32, sbox1[t1 & 0xff]); + var s1 = @as(u32, sbox1[t1 >> 24]) << 24 | @as(u32, sbox1[t0 >> 16 & 0xff]) << 16 | @as(u32, sbox1[t3 >> 8 & 0xff]) << 8 | @as(u32, sbox1[t2 & 0xff]); + var s2 = @as(u32, sbox1[t2 >> 24]) << 24 | @as(u32, sbox1[t1 >> 16 & 0xff]) << 16 | @as(u32, sbox1[t0 >> 8 & 0xff]) << 8 | @as(u32, sbox1[t3 & 0xff]); + var s3 = @as(u32, sbox1[t3 >> 24]) << 24 | @as(u32, sbox1[t2 >> 16 & 0xff]) << 16 | @as(u32, sbox1[t1 >> 8 & 0xff]) << 8 | @as(u32, sbox1[t0 & 0xff]); + s0 ^= round_key.repr[0]; + s1 ^= round_key.repr[1]; + s2 ^= round_key.repr[2]; + s3 ^= round_key.repr[3]; + + return Block{ .repr = BlockVec{ s0, s1, s2, s3 } }; + } + + /// XOR the content of two blocks. + pub inline fn xor(block1: Block, block2: Block) Block { + var x: BlockVec = undefined; + comptime var i = 0; + inline while (i < 4) : (i += 1) { + x[i] = block1.repr[i] ^ block2.repr[i]; + } + return Block{ .repr = x }; + } + + /// Perform operations on multiple blocks in parallel. + pub const parallel = struct { + /// The recommended number of AES encryption/decryption to perform in parallel for the chosen implementation. + pub const optimal_parallel_blocks = 1; + + /// Encrypt multiple blocks in parallel, each their own round key. + pub fn encryptParallel(comptime count: usize, blocks: [count]Block, round_keys: [count]Block) [count]Block { + var i = 0; + var out: [count]Block = undefined; + while (i < count) : (i += 1) { + out[i] = blocks[i].encrypt(round_keys[i]); + } + return out; + } + + /// Decrypt multiple blocks in parallel, each their own round key. + pub fn decryptParallel(comptime count: usize, blocks: [count]Block, round_keys: [count]Block) [count]Block { + var i = 0; + var out: [count]Block = undefined; + while (i < count) : (i += 1) { + out[i] = blocks[i].decrypt(round_keys[i]); + } + return out; + } + + /// Encrypt multple blocks in parallel with the same round key. + pub fn encryptWide(comptime count: usize, blocks: [count]Block, round_key: Block) [count]Block { + var i = 0; + var out: [count]Block = undefined; + while (i < count) : (i += 1) { + out[i] = blocks[i].encrypt(round_key); + } + return out; + } + + /// Decrypt multple blocks in parallel with the same round key. + pub fn decryptWide(comptime count: usize, blocks: [count]Block, round_key: Block) [count]Block { + var i = 0; + var out: [count]Block = undefined; + while (i < count) : (i += 1) { + out[i] = blocks[i].decrypt(round_key); + } + return out; + } + + /// Encrypt multple blocks in parallel with the same last round key. + pub fn encryptLastWide(comptime count: usize, blocks: [count]Block, round_key: Block) [count]Block { + var i = 0; + var out: [count]Block = undefined; + while (i < count) : (i += 1) { + out[i] = blocks[i].encryptLast(round_key); + } + return out; + } + + /// Decrypt multple blocks in parallel with the same last round key. + pub fn decryptLastWide(comptime count: usize, blocks: [count]Block, round_key: Block) [count]Block { + var i = 0; + var out: [count]Block = undefined; + while (i < count) : (i += 1) { + out[i] = blocks[i].decryptLast(round_key); + } + return out; + } + }; +}; + +fn KeySchedule(comptime AES: type) type { + std.debug.assert(AES.rounds == 10 or AES.rounds == 14); + const key_size = AES.key_bits / 8; + const rounds = AES.rounds; + + return struct { + const Self = @This(); + const words_in_key = key_size / 4; + + round_keys: [rounds + 1]Block, + + // Key expansion algorithm. See FIPS-197, Figure 11. + fn expandKey(key: [key_size]u8) Self { + const subw = struct { + // Apply sbox0 to each byte in w. + fn func(w: u32) u32 { + return @as(u32, sbox0[w >> 24]) << 24 | @as(u32, sbox0[w >> 16 & 0xff]) << 16 | @as(u32, sbox0[w >> 8 & 0xff]) << 8 | @as(u32, sbox0[w & 0xff]); + } + }.func; + + var round_keys: [rounds + 1]Block = undefined; + comptime var i: usize = 0; + inline while (i < words_in_key) : (i += 1) { + round_keys[i / 4].repr[i % 4] = mem.readIntBig(u32, key[4 * i ..][0..4]); + } + inline while (i < round_keys.len * 4) : (i += 1) { + var t = round_keys[(i - 1) / 4].repr[(i - 1) % 4]; + if (i % words_in_key == 0) { + t = subw(std.math.rotl(u32, t, 8)) ^ (@as(u32, powx[i / words_in_key - 1]) << 24); + } else if (words_in_key > 6 and i % words_in_key == 4) { + t = subw(t); + } + round_keys[i / 4].repr[i % 4] = round_keys[(i - words_in_key) / 4].repr[(i - words_in_key) % 4] ^ t; + } + return Self{ .round_keys = round_keys }; + } + + /// Invert the key schedule. + pub fn invert(key_schedule: Self) Self { + const round_keys = &key_schedule.round_keys; + var inv_round_keys: [rounds + 1]Block = undefined; + const total_words = 4 * round_keys.len; + var i: usize = 0; + while (i < total_words) : (i += 4) { + const ei = total_words - i - 4; + comptime var j: usize = 0; + inline while (j < 4) : (j += 1) { + var x = round_keys[(ei + j) / 4].repr[(ei + j) % 4]; + if (i > 0 and i + 4 < total_words) { + x = td0[sbox0[x >> 24]] ^ td1[sbox0[x >> 16 & 0xff]] ^ td2[sbox0[x >> 8 & 0xff]] ^ td3[sbox0[x & 0xff]]; + } + inv_round_keys[(i + j) / 4].repr[(i + j) % 4] = x; + } + } + return Self{ .round_keys = inv_round_keys }; + } + }; +} + +/// A context to perform encryption using the standard AES key schedule. +pub fn AESEncryptCtx(comptime AES: type) type { + std.debug.assert(AES.key_bits == 128 or AES.key_bits == 256); + const rounds = AES.rounds; + + return struct { + const Self = @This(); + pub const block = AES.block; + pub const block_size = block.block_size; + key_schedule: KeySchedule(AES), + + /// Create a new encryption context with the given key. + pub fn init(key: [AES.key_bits / 8]u8) Self { + const key_schedule = KeySchedule(AES).expandKey(key); + return Self{ + .key_schedule = key_schedule, + }; + } + + /// Encrypt a single block. + pub fn encrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void { + const round_keys = ctx.key_schedule.round_keys; + var t = Block.fromBytes(src).xor(round_keys[0]); + comptime var i = 1; + inline while (i < rounds) : (i += 1) { + t = t.encrypt(round_keys[i]); + } + t = t.encryptLast(round_keys[rounds]); + dst.* = t.toBytes(); + } + + /// Encrypt+XOR a single block. + pub fn xor(ctx: Self, dst: *[16]u8, src: *const [16]u8, counter: [16]u8) void { + const round_keys = ctx.key_schedule.round_keys; + var t = Block.fromBytes(&counter).xor(round_keys[0]); + comptime var i = 1; + inline while (i < rounds) : (i += 1) { + t = t.encrypt(round_keys[i]); + } + t = t.encryptLast(round_keys[rounds]); + dst.* = t.xorBytes(src); + } + + /// Encrypt multiple blocks, possibly leveraging parallelization. + pub fn encryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void { + var i: usize = 0; + while (i < count) : (i += 1) { + ctx.encrypt(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]); + } + } + + /// Encrypt+XOR multiple blocks, possibly leveraging parallelization. + pub fn xorWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8, counters: [16 * count]u8) void { + var i: usize = 0; + while (i < count) : (i += 1) { + ctx.xor(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16], counters[16 * i .. 16 * i + 16][0..16].*); + } + } + }; +} + +/// A context to perform decryption using the standard AES key schedule. +pub fn AESDecryptCtx(comptime AES: type) type { + std.debug.assert(AES.key_bits == 128 or AES.key_bits == 256); + const rounds = AES.rounds; + + return struct { + const Self = @This(); + pub const block = AES.block; + pub const block_size = block.block_size; + key_schedule: KeySchedule(AES), + + /// Create a decryption context from an existing encryption context. + pub fn initFromEnc(ctx: AESEncryptCtx(AES)) Self { + return Self{ + .key_schedule = ctx.key_schedule.invert(), + }; + } + + /// Create a new decryption context with the given key. + pub fn init(key: [AES.key_bits / 8]u8) Self { + const enc_ctx = AESEncryptCtx(AES).init(key); + return initFromEnc(enc_ctx); + } + + /// Decrypt a single block. + pub fn decrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void { + const inv_round_keys = ctx.key_schedule.round_keys; + var t = Block.fromBytes(src).xor(inv_round_keys[0]); + comptime var i = 1; + inline while (i < rounds) : (i += 1) { + t = t.decrypt(inv_round_keys[i]); + } + t = t.decryptLast(inv_round_keys[rounds]); + dst.* = t.toBytes(); + } + + /// Decrypt multiple blocks, possibly leveraging parallelization. + pub fn decryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void { + var i: usize = 0; + while (i < count) : (i += 1) { + ctx.decrypt(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]); + } + } + }; +} + +/// AES-128 with the standard key schedule. +pub const AES128 = struct { + pub const key_bits: usize = 128; + pub const rounds = ((key_bits - 64) / 32 + 8); + pub const block = Block; + + /// Create a new context for encryption. + pub fn initEnc(key: [key_bits / 8]u8) AESEncryptCtx(AES128) { + return AESEncryptCtx(AES128).init(key); + } + + /// Create a new context for decryption. + pub fn initDec(key: [key_bits / 8]u8) AESDecryptCtx(AES128) { + return AESDecryptCtx(AES128).init(key); + } +}; + +/// AES-256 with the standard key schedule. +pub const AES256 = struct { + pub const key_bits: usize = 256; + pub const rounds = ((key_bits - 64) / 32 + 8); + pub const block = Block; + + /// Create a new context for encryption. + pub fn initEnc(key: [key_bits / 8]u8) AESEncryptCtx(AES256) { + return AESEncryptCtx(AES256).init(key); + } + + /// Create a new context for decryption. + pub fn initDec(key: [key_bits / 8]u8) AESDecryptCtx(AES256) { + return AESDecryptCtx(AES256).init(key); + } +}; + +// constants +const powx = [16]u8{ + 0x01, + 0x02, + 0x04, + 0x08, + 0x10, + 0x20, + 0x40, + 0x80, + 0x1b, + 0x36, + 0x6c, + 0xd8, + 0xab, + 0x4d, + 0x9a, + 0x2f, +}; + +const sbox0 align(64) = [256]u8{ + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, +}; + +const sbox1 align(64) = [256]u8{ + 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, + 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, + 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, + 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, + 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, + 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, + 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, + 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, + 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, + 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, +}; + +const te0 align(64) = [256]u32{ + 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554, + 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, + 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87, 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, + 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b, + 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, + 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108, 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f, + 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5, + 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f, + 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e, 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, + 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497, + 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, + 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b, 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, + 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594, + 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3, + 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a, 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, + 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d, + 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, + 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47, 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, + 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883, + 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, + 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e, 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, + 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b, + 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, + 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25, 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818, + 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651, + 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, + 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa, 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, + 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9, + 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, + 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920, 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a, + 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8, + 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a, +}; +const te1 align(64) = [256]u32{ + 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, + 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, + 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, + 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, + 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, + 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, + 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, + 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, + 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, + 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, + 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, + 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, + 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, + 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, + 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, + 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, + 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, + 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, + 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, + 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, + 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, + 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, + 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, + 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, + 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, + 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, + 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, + 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, + 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, + 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, + 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, + 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, +}; +const te2 align(64) = [256]u32{ + 0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b, 0xf20dfff2, 0x6bbdd66b, 0x6fb1de6f, 0xc55491c5, + 0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b, 0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76, + 0xca458fca, 0x829d1f82, 0xc94089c9, 0x7d87fa7d, 0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0, + 0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af, 0x9cbf239c, 0xa4f753a4, 0x7296e472, 0xc05b9bc0, + 0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26, 0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc, + 0x345c6834, 0xa5f451a5, 0xe534d1e5, 0xf108f9f1, 0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15, + 0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3, 0x18283018, 0x96a13796, 0x050f0a05, 0x9ab52f9a, + 0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2, 0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75, + 0x091b1209, 0x839e1d83, 0x2c74582c, 0x1a2e341a, 0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0, + 0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3, 0x297b5229, 0xe33edde3, 0x2f715e2f, 0x84971384, + 0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed, 0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b, + 0x6abed46a, 0xcb468dcb, 0xbed967be, 0x394b7239, 0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf, + 0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb, 0x43c58643, 0x4dd79a4d, 0x33556633, 0x85941185, + 0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f, 0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8, + 0x51f3a251, 0xa3fe5da3, 0x40c08040, 0x8f8a058f, 0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5, + 0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221, 0x10302010, 0xff1ae5ff, 0xf30efdf3, 0xd26dbfd2, + 0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec, 0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17, + 0xc45793c4, 0xa7f255a7, 0x7e82fc7e, 0x3d477a3d, 0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673, + 0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc, 0x22664422, 0x2a7e542a, 0x90ab3b90, 0x88830b88, + 0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814, 0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb, + 0xe03bdbe0, 0x32566432, 0x3a4e743a, 0x0a1e140a, 0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c, + 0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462, 0x91a83991, 0x95a43195, 0xe437d3e4, 0x798bf279, + 0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d, 0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9, + 0x6cb4d86c, 0x56faac56, 0xf407f3f4, 0xea25cfea, 0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008, + 0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e, 0x1c24381c, 0xa6f157a6, 0xb4c773b4, 0xc65197c6, + 0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f, 0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a, + 0x7090e070, 0x3e427c3e, 0xb5c471b5, 0x66aacc66, 0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e, + 0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9, 0x86911786, 0xc15899c1, 0x1d273a1d, 0x9eb9279e, + 0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211, 0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394, + 0x9bb62d9b, 0x1e223c1e, 0x87921587, 0xe920c9e9, 0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df, + 0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d, 0xbfda65bf, 0xe631d7e6, 0x42c68442, 0x68b8d068, + 0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f, 0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16, +}; +const te3 align(64) = [256]u32{ + 0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6, 0xf2f20dff, 0x6b6bbdd6, 0x6f6fb1de, 0xc5c55491, + 0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56, 0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec, + 0xcaca458f, 0x82829d1f, 0xc9c94089, 0x7d7d87fa, 0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb, + 0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45, 0x9c9cbf23, 0xa4a4f753, 0x727296e4, 0xc0c05b9b, + 0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c, 0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83, + 0x34345c68, 0xa5a5f451, 0xe5e534d1, 0xf1f108f9, 0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a, + 0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d, 0x18182830, 0x9696a137, 0x05050f0a, 0x9a9ab52f, + 0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf, 0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea, + 0x09091b12, 0x83839e1d, 0x2c2c7458, 0x1a1a2e34, 0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b, + 0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d, 0x29297b52, 0xe3e33edd, 0x2f2f715e, 0x84849713, + 0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1, 0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6, + 0x6a6abed4, 0xcbcb468d, 0xbebed967, 0x39394b72, 0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85, + 0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed, 0x4343c586, 0x4d4dd79a, 0x33335566, 0x85859411, + 0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe, 0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b, + 0x5151f3a2, 0xa3a3fe5d, 0x4040c080, 0x8f8f8a05, 0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1, + 0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342, 0x10103020, 0xffff1ae5, 0xf3f30efd, 0xd2d26dbf, + 0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3, 0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e, + 0xc4c45793, 0xa7a7f255, 0x7e7e82fc, 0x3d3d477a, 0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6, + 0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3, 0x22226644, 0x2a2a7e54, 0x9090ab3b, 0x8888830b, + 0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28, 0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad, + 0xe0e03bdb, 0x32325664, 0x3a3a4e74, 0x0a0a1e14, 0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8, + 0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4, 0x9191a839, 0x9595a431, 0xe4e437d3, 0x79798bf2, + 0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da, 0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049, + 0x6c6cb4d8, 0x5656faac, 0xf4f407f3, 0xeaea25cf, 0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810, + 0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c, 0x1c1c2438, 0xa6a6f157, 0xb4b4c773, 0xc6c65197, + 0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e, 0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f, + 0x707090e0, 0x3e3e427c, 0xb5b5c471, 0x6666aacc, 0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c, + 0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069, 0x86869117, 0xc1c15899, 0x1d1d273a, 0x9e9eb927, + 0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322, 0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733, + 0x9b9bb62d, 0x1e1e223c, 0x87879215, 0xe9e920c9, 0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5, + 0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a, 0xbfbfda65, 0xe6e631d7, 0x4242c684, 0x6868b8d0, + 0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e, 0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c, +}; + +const td0 align(64) = [256]u32{ + 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96, 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393, + 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25, 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f, + 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1, 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6, + 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da, 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844, + 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd, 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4, + 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45, 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94, + 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7, 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a, + 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5, 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c, + 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1, 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a, + 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75, 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051, + 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46, 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff, + 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77, 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb, + 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000, 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e, + 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927, 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a, + 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e, 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16, + 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d, 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8, + 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd, 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34, + 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163, 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120, + 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d, 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0, + 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422, 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef, + 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36, 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4, + 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662, 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5, + 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3, 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b, + 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8, 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6, + 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6, 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0, + 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815, 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f, + 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df, 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f, + 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e, 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713, + 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89, 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c, + 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf, 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86, + 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f, 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541, + 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190, 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742, +}; +const td1 align(64) = [256]u32{ + 0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e, 0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303, + 0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c, 0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3, + 0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0, 0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9, + 0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259, 0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8, + 0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971, 0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a, + 0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f, 0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b, + 0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8, 0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab, + 0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708, 0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682, + 0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2, 0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe, + 0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb, 0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10, + 0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd, 0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015, + 0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e, 0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee, + 0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000, 0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72, + 0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39, 0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e, + 0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91, 0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a, + 0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17, 0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9, + 0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60, 0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e, + 0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1, 0xcad731dc, 0x10426385, 0x40139722, 0x2084c611, + 0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1, 0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3, + 0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964, 0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390, + 0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b, 0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf, + 0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46, 0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af, + 0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512, 0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb, + 0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a, 0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8, + 0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c, 0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266, + 0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8, 0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6, + 0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604, 0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551, + 0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41, 0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647, + 0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c, 0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1, + 0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737, 0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db, + 0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340, 0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95, + 0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1, 0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857, +}; +const td2 align(64) = [256]u32{ + 0xa75051f4, 0x65537e41, 0xa4c31a17, 0x5e963a27, 0x6bcb3bab, 0x45f11f9d, 0x58abacfa, 0x03934be3, + 0xfa552030, 0x6df6ad76, 0x769188cc, 0x4c25f502, 0xd7fc4fe5, 0xcbd7c52a, 0x44802635, 0xa38fb562, + 0x5a49deb1, 0x1b6725ba, 0x0e9845ea, 0xc0e15dfe, 0x7502c32f, 0xf012814c, 0x97a38d46, 0xf9c66bd3, + 0x5fe7038f, 0x9c951592, 0x7aebbf6d, 0x59da9552, 0x832dd4be, 0x21d35874, 0x692949e0, 0xc8448ec9, + 0x896a75c2, 0x7978f48e, 0x3e6b9958, 0x71dd27b9, 0x4fb6bee1, 0xad17f088, 0xac66c920, 0x3ab47dce, + 0x4a1863df, 0x3182e51a, 0x33609751, 0x7f456253, 0x77e0b164, 0xae84bb6b, 0xa01cfe81, 0x2b94f908, + 0x68587048, 0xfd198f45, 0x6c8794de, 0xf8b7527b, 0xd323ab73, 0x02e2724b, 0x8f57e31f, 0xab2a6655, + 0x2807b2eb, 0xc2032fb5, 0x7b9a86c5, 0x08a5d337, 0x87f23028, 0xa5b223bf, 0x6aba0203, 0x825ced16, + 0x1c2b8acf, 0xb492a779, 0xf2f0f307, 0xe2a14e69, 0xf4cd65da, 0xbed50605, 0x621fd134, 0xfe8ac4a6, + 0x539d342e, 0x55a0a2f3, 0xe132058a, 0xeb75a4f6, 0xec390b83, 0xefaa4060, 0x9f065e71, 0x1051bd6e, + 0x8af93e21, 0x063d96dd, 0x05aedd3e, 0xbd464de6, 0x8db59154, 0x5d0571c4, 0xd46f0406, 0x15ff6050, + 0xfb241998, 0xe997d6bd, 0x43cc8940, 0x9e7767d9, 0x42bdb0e8, 0x8b880789, 0x5b38e719, 0xeedb79c8, + 0x0a47a17c, 0x0fe97c42, 0x1ec9f884, 0x00000000, 0x86830980, 0xed48322b, 0x70ac1e11, 0x724e6c5a, + 0xfffbfd0e, 0x38560f85, 0xd51e3dae, 0x3927362d, 0xd9640a0f, 0xa621685c, 0x54d19b5b, 0x2e3a2436, + 0x67b10c0a, 0xe70f9357, 0x96d2b4ee, 0x919e1b9b, 0xc54f80c0, 0x20a261dc, 0x4b695a77, 0x1a161c12, + 0xba0ae293, 0x2ae5c0a0, 0xe0433c22, 0x171d121b, 0x0d0b0e09, 0xc7adf28b, 0xa8b92db6, 0xa9c8141e, + 0x198557f1, 0x074caf75, 0xddbbee99, 0x60fda37f, 0x269ff701, 0xf5bc5c72, 0x3bc54466, 0x7e345bfb, + 0x29768b43, 0xc6dccb23, 0xfc68b6ed, 0xf163b8e4, 0xdccad731, 0x85104263, 0x22401397, 0x112084c6, + 0x247d854a, 0x3df8d2bb, 0x3211aef9, 0xa16dc729, 0x2f4b1d9e, 0x30f3dcb2, 0x52ec0d86, 0xe3d077c1, + 0x166c2bb3, 0xb999a970, 0x48fa1194, 0x642247e9, 0x8cc4a8fc, 0x3f1aa0f0, 0x2cd8567d, 0x90ef2233, + 0x4ec78749, 0xd1c1d938, 0xa2fe8cca, 0x0b3698d4, 0x81cfa6f5, 0xde28a57a, 0x8e26dab7, 0xbfa43fad, + 0x9de42c3a, 0x920d5078, 0xcc9b6a5f, 0x4662547e, 0x13c2f68d, 0xb8e890d8, 0xf75e2e39, 0xaff582c3, + 0x80be9f5d, 0x937c69d0, 0x2da96fd5, 0x12b3cf25, 0x993bc8ac, 0x7da71018, 0x636ee89c, 0xbb7bdb3b, + 0x7809cd26, 0x18f46e59, 0xb701ec9a, 0x9aa8834f, 0x6e65e695, 0xe67eaaff, 0xcf0821bc, 0xe8e6ef15, + 0x9bd9bae7, 0x36ce4a6f, 0x09d4ea9f, 0x7cd629b0, 0xb2af31a4, 0x23312a3f, 0x9430c6a5, 0x66c035a2, + 0xbc37744e, 0xcaa6fc82, 0xd0b0e090, 0xd81533a7, 0x984af104, 0xdaf741ec, 0x500e7fcd, 0xf62f1791, + 0xd68d764d, 0xb04d43ef, 0x4d54ccaa, 0x04dfe496, 0xb5e39ed1, 0x881b4c6a, 0x1fb8c12c, 0x517f4665, + 0xea049d5e, 0x355d018c, 0x7473fa87, 0x412efb0b, 0x1d5ab367, 0xd25292db, 0x5633e910, 0x47136dd6, + 0x618c9ad7, 0x0c7a37a1, 0x148e59f8, 0x3c89eb13, 0x27eecea9, 0xc935b761, 0xe5ede11c, 0xb13c7a47, + 0xdf599cd2, 0x733f55f2, 0xce791814, 0x37bf73c7, 0xcdea53f7, 0xaa5b5ffd, 0x6f14df3d, 0xdb867844, + 0xf381caaf, 0xc43eb968, 0x342c3824, 0x405fc2a3, 0xc372161d, 0x250cbce2, 0x498b283c, 0x9541ff0d, + 0x017139a8, 0xb3de080c, 0xe49cd8b4, 0xc1906456, 0x84617bcb, 0xb670d532, 0x5c74486c, 0x5742d0b8, +}; +const td3 align(64) = [256]u32{ + 0xf4a75051, 0x4165537e, 0x17a4c31a, 0x275e963a, 0xab6bcb3b, 0x9d45f11f, 0xfa58abac, 0xe303934b, + 0x30fa5520, 0x766df6ad, 0xcc769188, 0x024c25f5, 0xe5d7fc4f, 0x2acbd7c5, 0x35448026, 0x62a38fb5, + 0xb15a49de, 0xba1b6725, 0xea0e9845, 0xfec0e15d, 0x2f7502c3, 0x4cf01281, 0x4697a38d, 0xd3f9c66b, + 0x8f5fe703, 0x929c9515, 0x6d7aebbf, 0x5259da95, 0xbe832dd4, 0x7421d358, 0xe0692949, 0xc9c8448e, + 0xc2896a75, 0x8e7978f4, 0x583e6b99, 0xb971dd27, 0xe14fb6be, 0x88ad17f0, 0x20ac66c9, 0xce3ab47d, + 0xdf4a1863, 0x1a3182e5, 0x51336097, 0x537f4562, 0x6477e0b1, 0x6bae84bb, 0x81a01cfe, 0x082b94f9, + 0x48685870, 0x45fd198f, 0xde6c8794, 0x7bf8b752, 0x73d323ab, 0x4b02e272, 0x1f8f57e3, 0x55ab2a66, + 0xeb2807b2, 0xb5c2032f, 0xc57b9a86, 0x3708a5d3, 0x2887f230, 0xbfa5b223, 0x036aba02, 0x16825ced, + 0xcf1c2b8a, 0x79b492a7, 0x07f2f0f3, 0x69e2a14e, 0xdaf4cd65, 0x05bed506, 0x34621fd1, 0xa6fe8ac4, + 0x2e539d34, 0xf355a0a2, 0x8ae13205, 0xf6eb75a4, 0x83ec390b, 0x60efaa40, 0x719f065e, 0x6e1051bd, + 0x218af93e, 0xdd063d96, 0x3e05aedd, 0xe6bd464d, 0x548db591, 0xc45d0571, 0x06d46f04, 0x5015ff60, + 0x98fb2419, 0xbde997d6, 0x4043cc89, 0xd99e7767, 0xe842bdb0, 0x898b8807, 0x195b38e7, 0xc8eedb79, + 0x7c0a47a1, 0x420fe97c, 0x841ec9f8, 0x00000000, 0x80868309, 0x2bed4832, 0x1170ac1e, 0x5a724e6c, + 0x0efffbfd, 0x8538560f, 0xaed51e3d, 0x2d392736, 0x0fd9640a, 0x5ca62168, 0x5b54d19b, 0x362e3a24, + 0x0a67b10c, 0x57e70f93, 0xee96d2b4, 0x9b919e1b, 0xc0c54f80, 0xdc20a261, 0x774b695a, 0x121a161c, + 0x93ba0ae2, 0xa02ae5c0, 0x22e0433c, 0x1b171d12, 0x090d0b0e, 0x8bc7adf2, 0xb6a8b92d, 0x1ea9c814, + 0xf1198557, 0x75074caf, 0x99ddbbee, 0x7f60fda3, 0x01269ff7, 0x72f5bc5c, 0x663bc544, 0xfb7e345b, + 0x4329768b, 0x23c6dccb, 0xedfc68b6, 0xe4f163b8, 0x31dccad7, 0x63851042, 0x97224013, 0xc6112084, + 0x4a247d85, 0xbb3df8d2, 0xf93211ae, 0x29a16dc7, 0x9e2f4b1d, 0xb230f3dc, 0x8652ec0d, 0xc1e3d077, + 0xb3166c2b, 0x70b999a9, 0x9448fa11, 0xe9642247, 0xfc8cc4a8, 0xf03f1aa0, 0x7d2cd856, 0x3390ef22, + 0x494ec787, 0x38d1c1d9, 0xcaa2fe8c, 0xd40b3698, 0xf581cfa6, 0x7ade28a5, 0xb78e26da, 0xadbfa43f, + 0x3a9de42c, 0x78920d50, 0x5fcc9b6a, 0x7e466254, 0x8d13c2f6, 0xd8b8e890, 0x39f75e2e, 0xc3aff582, + 0x5d80be9f, 0xd0937c69, 0xd52da96f, 0x2512b3cf, 0xac993bc8, 0x187da710, 0x9c636ee8, 0x3bbb7bdb, + 0x267809cd, 0x5918f46e, 0x9ab701ec, 0x4f9aa883, 0x956e65e6, 0xffe67eaa, 0xbccf0821, 0x15e8e6ef, + 0xe79bd9ba, 0x6f36ce4a, 0x9f09d4ea, 0xb07cd629, 0xa4b2af31, 0x3f23312a, 0xa59430c6, 0xa266c035, + 0x4ebc3774, 0x82caa6fc, 0x90d0b0e0, 0xa7d81533, 0x04984af1, 0xecdaf741, 0xcd500e7f, 0x91f62f17, + 0x4dd68d76, 0xefb04d43, 0xaa4d54cc, 0x9604dfe4, 0xd1b5e39e, 0x6a881b4c, 0x2c1fb8c1, 0x65517f46, + 0x5eea049d, 0x8c355d01, 0x877473fa, 0x0b412efb, 0x671d5ab3, 0xdbd25292, 0x105633e9, 0xd647136d, + 0xd7618c9a, 0xa10c7a37, 0xf8148e59, 0x133c89eb, 0xa927eece, 0x61c935b7, 0x1ce5ede1, 0x47b13c7a, + 0xd2df599c, 0xf2733f55, 0x14ce7918, 0xc737bf73, 0xf7cdea53, 0xfdaa5b5f, 0x3d6f14df, 0x44db8678, + 0xaff381ca, 0x68c43eb9, 0x24342c38, 0xa3405fc2, 0x1dc37216, 0xe2250cbc, 0x3c498b28, 0x0d9541ff, + 0xa8017139, 0x0cb3de08, 0xb4e49cd8, 0x56c19064, 0xcb84617b, 0x32b670d5, 0x6c5c7448, 0xb85742d0, +}; diff --git a/lib/std/crypto/benchmark.zig b/lib/std/crypto/benchmark.zig index e20b27220e..860f1269f0 100644 --- a/lib/std/crypto/benchmark.zig +++ b/lib/std/crypto/benchmark.zig @@ -179,6 +179,64 @@ pub fn benchmarkAead(comptime Aead: anytype, comptime bytes: comptime_int) !u64 return throughput; } +const aes = [_]Crypto{ + Crypto{ .ty = crypto.core.aes.AES128, .name = "aes128-single" }, + Crypto{ .ty = crypto.core.aes.AES256, .name = "aes256-single" }, +}; + +pub fn benchmarkAES(comptime AES: anytype, comptime count: comptime_int) !u64 { + var key: [AES.key_bits / 8]u8 = undefined; + prng.random.bytes(key[0..]); + const ctx = AES.initEnc(key); + + var in = [_]u8{0} ** 16; + + var timer = try Timer.start(); + const start = timer.lap(); + { + var i: usize = 0; + while (i < count) : (i += 1) { + ctx.encrypt(&in, &in); + } + } + mem.doNotOptimizeAway(&in); + const end = timer.read(); + + const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s; + const throughput = @floatToInt(u64, count / elapsed_s); + + return throughput; +} + +const aes8 = [_]Crypto{ + Crypto{ .ty = crypto.core.aes.AES128, .name = "aes128-8" }, + Crypto{ .ty = crypto.core.aes.AES256, .name = "aes256-8" }, +}; + +pub fn benchmarkAES8(comptime AES: anytype, comptime count: comptime_int) !u64 { + var key: [AES.key_bits / 8]u8 = undefined; + prng.random.bytes(key[0..]); + const ctx = AES.initEnc(key); + + var in = [_]u8{0} ** (8 * 16); + + var timer = try Timer.start(); + const start = timer.lap(); + { + var i: usize = 0; + while (i < count) : (i += 1) { + ctx.encryptWide(8, &in, &in); + } + } + mem.doNotOptimizeAway(&in); + const end = timer.read(); + + const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s; + const throughput = @floatToInt(u64, 8 * count / elapsed_s); + + return throughput; +} + fn usage() void { std.debug.warn( \\throughput_test [options] @@ -238,35 +296,49 @@ pub fn main() !void { inline for (hashes) |H| { if (filter == null or std.mem.indexOf(u8, H.name, filter.?) != null) { const throughput = try benchmarkHash(H.ty, mode(128 * MiB)); - try stdout.print("{:>17}: {:7} MiB/s\n", .{ H.name, throughput / (1 * MiB) }); + try stdout.print("{:>17}: {:10} MiB/s\n", .{ H.name, throughput / (1 * MiB) }); } } inline for (macs) |M| { if (filter == null or std.mem.indexOf(u8, M.name, filter.?) != null) { const throughput = try benchmarkMac(M.ty, mode(128 * MiB)); - try stdout.print("{:>17}: {:7} MiB/s\n", .{ M.name, throughput / (1 * MiB) }); + try stdout.print("{:>17}: {:10} MiB/s\n", .{ M.name, throughput / (1 * MiB) }); } } inline for (exchanges) |E| { if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { const throughput = try benchmarkKeyExchange(E.ty, mode(1000)); - try stdout.print("{:>17}: {:7} exchanges/s\n", .{ E.name, throughput }); + try stdout.print("{:>17}: {:10} exchanges/s\n", .{ E.name, throughput }); } } inline for (signatures) |E| { if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { const throughput = try benchmarkSignature(E.ty, mode(1000)); - try stdout.print("{:>17}: {:7} signatures/s\n", .{ E.name, throughput }); + try stdout.print("{:>17}: {:10} signatures/s\n", .{ E.name, throughput }); } } inline for (aeads) |E| { if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { const throughput = try benchmarkAead(E.ty, mode(128 * MiB)); - try stdout.print("{:>17}: {:7} MiB/s\n", .{ E.name, throughput / (1 * MiB) }); + try stdout.print("{:>17}: {:10} MiB/s\n", .{ E.name, throughput / (1 * MiB) }); + } + } + + inline for (aes) |E| { + if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { + const throughput = try benchmarkAES(E.ty, mode(100000000)); + try stdout.print("{:>17}: {:10} ops/s\n", .{ E.name, throughput }); + } + } + + inline for (aes8) |E| { + if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { + const throughput = try benchmarkAES8(E.ty, mode(10000000)); + try stdout.print("{:>17}: {:10} ops/s\n", .{ E.name, throughput }); } } } diff --git a/lib/std/crypto/modes.zig b/lib/std/crypto/modes.zig new file mode 100644 index 0000000000..5c1fa4b2f3 --- /dev/null +++ b/lib/std/crypto/modes.zig @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. +// Based on Go stdlib implementation + +const std = @import("../std.zig"); +const builtin = std.builtin; +const mem = std.mem; +const debug = std.debug; + +/// Counter mode. +/// +/// This mode creates a key stream by encrypting an incrementing counter using a block cipher, and adding it to the source material. +/// +/// Important: the counter mode doesn't provide authenticated encryption: the ciphertext can be trivially modified without this being detected. +/// As a result, applications should generally never use it directly, but only in a construction that includes a MAC. +pub fn ctr(comptime BlockCipher: anytype, block_cipher: BlockCipher, dst: []u8, src: []const u8, iv: [BlockCipher.block_size]u8, endian: comptime builtin.Endian) void { + debug.assert(dst.len >= src.len); + const block_size = BlockCipher.block_size; + var counter: [BlockCipher.block_size]u8 = undefined; + var counterInt = mem.readInt(u128, &iv, endian); + var i: usize = 0; + + const parallel_count = BlockCipher.block.parallel.optimal_parallel_blocks; + const wide_block_size = parallel_count * 16; + if (src.len >= wide_block_size) { + var counters: [parallel_count * 16]u8 = undefined; + while (i + wide_block_size <= src.len) : (i += wide_block_size) { + comptime var j = 0; + inline while (j < parallel_count) : (j += 1) { + mem.writeInt(u128, counters[j * 16 .. j * 16 + 16], counterInt, endian); + counterInt +%= 1; + } + block_cipher.xorWide(parallel_count, dst[i .. i + wide_block_size][0..wide_block_size], src[i .. i + wide_block_size][0..wide_block_size], counters); + } + } + while (i + block_size <= src.len) : (i += block_size) { + mem.writeInt(u128, &counter, counterInt, endian); + counterInt +%= 1; + block_cipher.xor(dst[i .. i + block_size][0..block_size], src[i .. i + block_size][0..block_size], counter); + } + if (i < src.len) { + mem.writeInt(u128, &counter, counterInt, endian); + var pad = [_]u8{0} ** block_size; + mem.copy(u8, &pad, src[i..]); + block_cipher.xor(&pad, &pad, counter); + mem.copy(u8, dst[i..], pad[0 .. src.len - i]); + } +} From e85c89630e78ccc0e4bab44064779a07a029cecd Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Tue, 16 Jun 2020 22:38:51 +0200 Subject: [PATCH 115/210] accept Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 34 ++++++++++++++++++++++++++++++++++ lib/std/net.zig | 15 ++++++++++----- lib/std/os.zig | 7 +------ 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index c547f50365..f44b2f06e4 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -720,6 +720,40 @@ pub const Loop = struct { } } + /// ------- I/0 APIs ------- + pub fn accept( + self: *Loop, + /// This argument is a socket that has been created with `socket`, bound to a local address + /// with `bind`, and is listening for connections after a `listen`. + sockfd: os.fd_t, + /// This argument is a pointer to a sockaddr structure. This structure is filled in with the + /// address of the peer socket, as known to the communications layer. The exact format of the + /// address returned addr is determined by the socket's address family (see `socket` and the + /// respective protocol man pages). + addr: *os.sockaddr, + /// This argument is a value-result argument: the caller must initialize it to contain the + /// size (in bytes) of the structure pointed to by addr; on return it will contain the actual size + /// of the peer address. + /// + /// The returned address is truncated if the buffer provided is too small; in this case, `addr_size` + /// will return a value greater than was supplied to the call. + addr_size: *os.socklen_t, + /// The following values can be bitwise ORed in flags to obtain different behavior: + /// * `SOCK_CLOEXEC` - Set the close-on-exec (`FD_CLOEXEC`) flag on the new file descriptor. See the + /// description of the `O_CLOEXEC` flag in `open` for reasons why this may be useful. + flags: u32, + ) os.AcceptError!os.fd_t { + while (true) { + return os.accept(sockfd, addr, addr_size, flags | os.SOCK_NONBLOCK) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdReadable(sockfd); + continue; + }, + else => return err, + }; + } + } + /// Performs an async `os.open` using a separate thread. pub fn openZ(self: *Loop, file_path: [*:0]const u8, flags: u32, mode: os.mode_t) os.OpenError!os.fd_t { var req_node = Request.Node{ diff --git a/lib/std/net.zig b/lib/std/net.zig index 45d8f07f04..6b6d234843 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -1661,18 +1661,23 @@ pub const StreamServer = struct { /// If this function succeeds, the returned `Connection` is a caller-managed resource. pub fn accept(self: *StreamServer) AcceptError!Connection { - const nonblock = if (std.io.is_async) os.SOCK_NONBLOCK else 0; - const accept_flags = nonblock | os.SOCK_CLOEXEC; var accepted_addr: Address = undefined; var adr_len: os.socklen_t = @sizeOf(Address); - if (os.accept(self.sockfd.?, &accepted_addr.any, &adr_len, accept_flags)) |fd| { + const accept_result = blk: { + if (std.io.is_async) { + const loop = std.event.Loop.instance orelse return error.UnexpectedError; + break :blk loop.accept(self.sockfd.?, &accepted_addr.any, &adr_len, os.SOCK_CLOEXEC); + } else { + break :blk os.accept(self.sockfd.?, &accepted_addr.any, &adr_len, os.SOCK_CLOEXEC); + } + }; + + if (accept_result) |fd| { return Connection{ .file = fs.File{ .handle = fd }, .address = accepted_addr, }; } else |err| switch (err) { - // We only give SOCK_NONBLOCK when I/O mode is async, in which case this error - // is handled by os.accept4. error.WouldBlock => unreachable, else => |e| return e, } diff --git a/lib/std/os.zig b/lib/std/os.zig index 0b09b1f82a..c5c34d4f40 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -2890,12 +2890,7 @@ pub fn accept( return fd; }, EINTR => continue, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdReadable(sockfd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => unreachable, // always a race condition ECONNABORTED => return error.ConnectionAborted, EFAULT => unreachable, From 730428bfd615cab415b2942fc9b781428a0ff692 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Tue, 16 Jun 2020 22:39:09 +0200 Subject: [PATCH 116/210] connect Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 10 ++++++++++ lib/std/net.zig | 18 ++++++++++++------ lib/std/os.zig | 6 +----- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index f44b2f06e4..2dc1d5659e 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -754,6 +754,16 @@ pub const Loop = struct { } } + pub fn connect(self: *Loop, sockfd: os.socket_t, sock_addr: *const os.sockaddr, len: os.socklen_t) os.ConnectError!void { + os.connect(sockfd, sock_addr, len) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdWritable(sockfd); + return os.getsockoptError(sockfd); + }, + else => return err, + }; + } + /// Performs an async `os.open` using a separate thread. pub fn openZ(self: *Loop, file_path: [*:0]const u8, flags: u32, mode: os.mode_t) os.OpenError!os.fd_t { var req_node = Request.Node{ diff --git a/lib/std/net.zig b/lib/std/net.zig index 6b6d234843..928ebbbce5 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -614,11 +614,11 @@ pub fn connectUnixSocket(path: []const u8) !fs.File { var addr = try std.net.Address.initUnix(path); - try os.connect( - sockfd, - &addr.any, - addr.getOsSockLen(), - ); + if (std.io.is_async) { + try loop.connect(sockfd, &addr.any, addr.getOsSockLen()); + } else { + try os.connect(sockfd, &addr.any, addr.getOsSockLen()); + } return fs.File{ .handle = sockfd, @@ -677,7 +677,13 @@ pub fn tcpConnectToAddress(address: Address) !fs.File { (if (builtin.os.tag == .windows) 0 else os.SOCK_CLOEXEC); const sockfd = try os.socket(address.any.family, sock_flags, os.IPPROTO_TCP); errdefer os.close(sockfd); - try os.connect(sockfd, &address.any, address.getOsSockLen()); + + if (std.io.is_async) { + const loop = std.event.Loop.instance orelse return error.WouldBlock; + try loop.connect(sockfd, &address.any, address.getOsSockLen()); + } else { + try os.connect(sockfd, &address.any, address.getOsSockLen()); + } return fs.File{ .handle = sockfd }; } diff --git a/lib/std/os.zig b/lib/std/os.zig index c5c34d4f40..023c1d5971 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -3108,11 +3108,7 @@ pub fn connect(sockfd: socket_t, sock_addr: *const sockaddr, len: socklen_t) Con EADDRINUSE => return error.AddressInUse, EADDRNOTAVAIL => return error.AddressNotAvailable, EAFNOSUPPORT => return error.AddressFamilyNotSupported, - EAGAIN, EINPROGRESS => { - const loop = std.event.Loop.instance orelse return error.WouldBlock; - loop.waitUntilFdWritable(sockfd); - return getsockoptError(sockfd); - }, + EAGAIN, EINPROGRESS => return error.WouldBlock, EALREADY => unreachable, // The socket is nonblocking and a previous connection attempt has not yet been completed. EBADF => unreachable, // sockfd is not a valid open file descriptor. ECONNREFUSED => return error.ConnectionRefused, From 08364ac773bdc95b9407974b5c761dbdab863f4d Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Tue, 16 Jun 2020 23:14:05 +0200 Subject: [PATCH 117/210] read Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 40 ++++++++++++++++++++++++++-------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index 2dc1d5659e..7adc6e3a8f 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -822,23 +822,35 @@ pub const Loop = struct { /// Performs an async `os.read` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn read(self: *Loop, fd: os.fd_t, buf: []u8) os.ReadError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .read = .{ - .fd = fd, - .buf = buf, - .result = undefined, + pub fn read(self: *Loop, fd: os.fd_t, buf: []u8, simulate_evented: bool) os.ReadError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .read = .{ + .fd = fd, + .buf = buf, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.read.result; + } else { + while (true) { + return os.read(fd, buf) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdReadable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.read.result; } /// Performs an async `os.readv` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 73babf5fa2..bd0b8a2943 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -414,10 +414,12 @@ pub const File = struct { pub fn read(self: File, buffer: []u8) ReadError!usize { if (is_windows) { return windows.ReadFile(self.handle, buffer, null, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.read(self.handle, buffer); - } else { + } + + if (self.intended_io_mode == .blocking or !std.io.is_async) { return os.read(self.handle, buffer); + } else { + return std.event.Loop.instance.?.read(self.handle, buffer, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 023c1d5971..7db545a17c 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -366,12 +366,7 @@ pub fn read(fd: fd_t, buf: []u8) ReadError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdReadable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForReading, // Can be a race condition. EIO => return error.InputOutput, EISDIR => return error.IsDir, From bc35435ca6e51d0e120538398e3c708ada57f6de Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 19 Jun 2020 23:00:17 +0200 Subject: [PATCH 118/210] readv Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 40 ++++++++++++++++++++++++++-------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index 7adc6e3a8f..96774e3f11 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -855,23 +855,35 @@ pub const Loop = struct { /// Performs an async `os.readv` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn readv(self: *Loop, fd: os.fd_t, iov: []const os.iovec) os.ReadError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .readv = .{ - .fd = fd, - .iov = iov, - .result = undefined, + pub fn readv(self: *Loop, fd: os.fd_t, iov: []const os.iovec, simulate_evented: bool) os.ReadError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .readv = .{ + .fd = fd, + .iov = iov, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.readv.result; + } else { + while (true) { + return os.readv(fd, iov) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdReadable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.readv.result; } /// Performs an async `os.pread` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index bd0b8a2943..51ad931504 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -463,10 +463,12 @@ pub const File = struct { if (iovecs.len == 0) return @as(usize, 0); const first = iovecs[0]; return windows.ReadFile(self.handle, first.iov_base[0..first.iov_len], null, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.readv(self.handle, iovecs); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.readv(self.handle, iovecs); + } else { + return std.event.Loop.instance.?.readv(self.handle, iovecs, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 7db545a17c..e5a06965d8 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -423,12 +423,7 @@ pub fn readv(fd: fd_t, iov: []const iovec) ReadError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdReadable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForReading, // can be a race condition EIO => return error.InputOutput, EISDIR => return error.IsDir, From bd9f2369d5c4e5fcd38342c877f9ae6531f78909 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 19 Jun 2020 23:08:34 +0200 Subject: [PATCH 119/210] pread Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 42 +++++++++++++++++++++++++++--------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index 96774e3f11..27c00b9ab3 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -888,24 +888,36 @@ pub const Loop = struct { /// Performs an async `os.pread` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn pread(self: *Loop, fd: os.fd_t, buf: []u8, offset: u64) os.PReadError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .pread = .{ - .fd = fd, - .buf = buf, - .offset = offset, - .result = undefined, + pub fn pread(self: *Loop, fd: os.fd_t, buf: []u8, offset: u64, simulate_evented: bool) os.PReadError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .pread = .{ + .fd = fd, + .buf = buf, + .offset = offset, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.pread.result; + } else { + while (true) { + return os.pread(fd, buf, offset) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdReadable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.pread.result; } /// Performs an async `os.preadv` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 51ad931504..cb3b41ffdf 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -438,10 +438,12 @@ pub const File = struct { pub fn pread(self: File, buffer: []u8, offset: u64) PReadError!usize { if (is_windows) { return windows.ReadFile(self.handle, buffer, offset, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.pread(self.handle, buffer, offset); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.pread(self.handle, buffer, offset); + } else { + return std.event.Loop.instance.?.pread(self.handle, buffer, offset, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index e5a06965d8..6ed555011e 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -482,12 +482,7 @@ pub fn pread(fd: fd_t, buf: []u8, offset: u64) PReadError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdReadable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForReading, // Can be a race condition. EIO => return error.InputOutput, EISDIR => return error.IsDir, From 59ecdaea127cb680d295ad02319dacba75ac1e73 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 19 Jun 2020 23:13:54 +0200 Subject: [PATCH 120/210] preadv Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 42 +++++++++++++++++++++++++++--------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index 27c00b9ab3..a14e798c50 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -922,24 +922,36 @@ pub const Loop = struct { /// Performs an async `os.preadv` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn preadv(self: *Loop, fd: os.fd_t, iov: []const os.iovec, offset: u64) os.ReadError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .preadv = .{ - .fd = fd, - .iov = iov, - .offset = offset, - .result = undefined, + pub fn preadv(self: *Loop, fd: os.fd_t, iov: []const os.iovec, offset: u64, simulate_evented: bool) os.ReadError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .preadv = .{ + .fd = fd, + .iov = iov, + .offset = offset, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.preadv.result; + } else { + while (true) { + return os.preadv(fd, iov, offset) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdReadable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.preadv.result; } /// Performs an async `os.write` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index cb3b41ffdf..4aa33fc5b9 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -506,10 +506,12 @@ pub const File = struct { if (iovecs.len == 0) return @as(usize, 0); const first = iovecs[0]; return windows.ReadFile(self.handle, first.iov_base[0..first.iov_len], offset, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.preadv(self.handle, iovecs, offset); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.preadv(self.handle, iovecs, offset); + } else { + return std.event.Loop.instance.?.preadv(self.handle, iovecs, offset, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 6ed555011e..6fb9d4388b 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -622,12 +622,7 @@ pub fn preadv(fd: fd_t, iov: []const iovec, offset: u64) PReadError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdReadable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForReading, // can be a race condition EIO => return error.InputOutput, EISDIR => return error.IsDir, From 9075f8e5a1a788770cc8193d6f54118434e72a4b Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 19 Jun 2020 23:29:11 +0200 Subject: [PATCH 121/210] write Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 40 ++++++++++++++++++++++++++-------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index a14e798c50..ec45e85630 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -956,23 +956,35 @@ pub const Loop = struct { /// Performs an async `os.write` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn write(self: *Loop, fd: os.fd_t, bytes: []const u8) os.WriteError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .write = .{ - .fd = fd, - .bytes = bytes, - .result = undefined, + pub fn write(self: *Loop, fd: os.fd_t, bytes: []const u8, simulate_evented: bool) os.WriteError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .write = .{ + .fd = fd, + .bytes = bytes, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.write.result; + } else { + while (true) { + return os.write(fd, bytes) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdWritable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.write.result; } /// Performs an async `os.writev` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 4aa33fc5b9..428b758e61 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -547,10 +547,12 @@ pub const File = struct { pub fn write(self: File, bytes: []const u8) WriteError!usize { if (is_windows) { return windows.WriteFile(self.handle, bytes, null, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.write(self.handle, bytes); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.write(self.handle, bytes); + } else { + return std.event.Loop.instance.?.write(self.handle, bytes, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 6fb9d4388b..da5ce13508 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -721,12 +721,7 @@ pub fn write(fd: fd_t, bytes: []const u8) WriteError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdWritable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForWriting, // can be a race condition. EDESTADDRREQ => unreachable, // `connect` was never called. EDQUOT => return error.DiskQuota, From 18f6629bd8ad8bd13108ddce82c32baf6f53628e Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 19 Jun 2020 23:34:02 +0200 Subject: [PATCH 122/210] writev Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 40 ++++++++++++++++++++++++++-------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index ec45e85630..e5fdbd2e3a 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -989,23 +989,35 @@ pub const Loop = struct { /// Performs an async `os.writev` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn writev(self: *Loop, fd: os.fd_t, iov: []const os.iovec_const) os.WriteError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .writev = .{ - .fd = fd, - .iov = iov, - .result = undefined, + pub fn writev(self: *Loop, fd: os.fd_t, iov: []const os.iovec_const, simulate_evented: bool) os.WriteError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .writev = .{ + .fd = fd, + .iov = iov, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.writev.result; + } else { + while (true) { + return os.writev(fd, iov) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdWritable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.writev.result; } /// Performs an async `os.pwritev` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 428b758e61..ab7ea2f579 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -586,10 +586,12 @@ pub const File = struct { if (iovecs.len == 0) return @as(usize, 0); const first = iovecs[0]; return windows.WriteFile(self.handle, first.iov_base[0..first.iov_len], null, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.writev(self.handle, iovecs); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.writev(self.handle, iovecs); + } else { + return std.event.Loop.instance.?.writev(self.handle, iovecs, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index da5ce13508..8f761b48be 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -789,12 +789,7 @@ pub fn writev(fd: fd_t, iov: []const iovec_const) WriteError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdWritable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForWriting, // Can be a race condition. EDESTADDRREQ => unreachable, // `connect` was never called. EDQUOT => return error.DiskQuota, From 7a07c62a075ada32c5eca298e4ed6a4c8a14cf89 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Sat, 20 Jun 2020 00:17:10 +0200 Subject: [PATCH 123/210] pwrite Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 49 +++++++++++++++++++++++++++++++++++++++++- lib/std/fs/file.zig | 8 ++++--- lib/std/os.zig | 7 +----- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index e5fdbd2e3a..b3b62bbe8e 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -1020,9 +1020,43 @@ pub const Loop = struct { } } + /// Performs an async `os.pwrite` using a separate thread. + /// `fd` must block and not return EAGAIN. + pub fn pwrite(self: *Loop, fd: os.fd_t, bytes: []const u8, offset: u64, simulate_evented: bool) os.PWriteError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .pwrite = .{ + .fd = fd, + .bytes = bytes, + .offset = offset, + .result = undefined, + }, + }, + .finish = .{ .TickNode = .{ .data = @frame() } }, + }, + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.pwrite.result; + } else { + while (true) { + return os.pwrite(fd, bytes, offset) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdWritable(fd); + continue; + }, + else => return err, + }; + } + } + } + /// Performs an async `os.pwritev` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn pwritev(self: *Loop, fd: os.fd_t, iov: []const os.iovec_const, offset: u64) os.WriteError!usize { + pub fn pwritev(self: *Loop, fd: os.fd_t, iov: []const os.iovec_const, offset: u64) os.PWriteError!usize { var req_node = Request.Node{ .data = .{ .msg = .{ @@ -1194,6 +1228,9 @@ pub const Loop = struct { .writev => |*msg| { msg.result = os.writev(msg.fd, msg.iov); }, + .pwrite => |*msg| { + msg.result = os.pwrite(msg.fd, msg.bytes, msg.offset); + }, .pwritev => |*msg| { msg.result = os.pwritev(msg.fd, msg.iov, msg.offset); }, @@ -1263,6 +1300,7 @@ pub const Loop = struct { readv: ReadV, write: Write, writev: WriteV, + pwrite: PWrite, pwritev: PWriteV, pread: PRead, preadv: PReadV, @@ -1306,6 +1344,15 @@ pub const Loop = struct { pub const Error = os.WriteError; }; + pub const PWrite = struct { + fd: os.fd_t, + bytes: []const u8, + offset: usize, + result: Error!usize, + + pub const Error = os.PWriteError; + }; + pub const PWriteV = struct { fd: os.fd_t, iov: []const os.iovec_const, diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index ab7ea2f579..60efe28ec2 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -566,10 +566,12 @@ pub const File = struct { pub fn pwrite(self: File, bytes: []const u8, offset: u64) PWriteError!usize { if (is_windows) { return windows.WriteFile(self.handle, bytes, offset, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.pwrite(self.handle, bytes, offset); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.pwrite(self.handle, bytes, offset); + } else { + return std.event.Loop.instance.?.pwrite(self.handle, bytes, offset, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 8f761b48be..595815a1e7 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -875,12 +875,7 @@ pub fn pwrite(fd: fd_t, bytes: []const u8, offset: u64) PWriteError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdWritable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForWriting, // Can be a race condition. EDESTADDRREQ => unreachable, // `connect` was never called. EDQUOT => return error.DiskQuota, From 7fec5b3def36bc73e9e48a777bcca838c4b86770 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Sat, 20 Jun 2020 00:19:38 +0200 Subject: [PATCH 124/210] pwritev Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 44 +++++++++++++++++++++++++++--------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index b3b62bbe8e..ae2d2f1499 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -1022,7 +1022,7 @@ pub const Loop = struct { /// Performs an async `os.pwrite` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn pwrite(self: *Loop, fd: os.fd_t, bytes: []const u8, offset: u64, simulate_evented: bool) os.PWriteError!usize { + pub fn pwrite(self: *Loop, fd: os.fd_t, bytes: []const u8, offset: u64, simulate_evented: bool) os.PerformsWriteError!usize { if (simulate_evented) { var req_node = Request.Node{ .data = .{ @@ -1056,24 +1056,36 @@ pub const Loop = struct { /// Performs an async `os.pwritev` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn pwritev(self: *Loop, fd: os.fd_t, iov: []const os.iovec_const, offset: u64) os.PWriteError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .pwritev = .{ - .fd = fd, - .iov = iov, - .offset = offset, - .result = undefined, + pub fn pwritev(self: *Loop, fd: os.fd_t, iov: []const os.iovec_const, offset: u64, simulate_evented: bool) os.PWriteError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .pwritev = .{ + .fd = fd, + .iov = iov, + .offset = offset, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.pwritev.result; + } else { + while (true) { + return os.pwritev(fd, iov, offset) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdWritable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.pwritev.result; } /// Performs an async `os.faccessatZ` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 60efe28ec2..f3e980b9f0 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -621,10 +621,12 @@ pub const File = struct { if (iovecs.len == 0) return @as(usize, 0); const first = iovecs[0]; return windows.WriteFile(self.handle, first.iov_base[0..first.iov_len], offset, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.pwritev(self.handle, iovecs, offset); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.pwritev(self.handle, iovecs, offset); + } else { + return std.event.Loop.instance.?.pwritev(self.handle, iovecs, offset, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 595815a1e7..1138013e8c 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -958,12 +958,7 @@ pub fn pwritev(fd: fd_t, iov: []const iovec_const, offset: u64) PWriteError!usiz EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdWritable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForWriting, // Can be a race condition. EDESTADDRREQ => unreachable, // `connect` was never called. EDQUOT => return error.DiskQuota, From 419aea54cb30b394191778fcc70effaf5181bf33 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Sat, 20 Jun 2020 00:35:08 +0200 Subject: [PATCH 125/210] sendto Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 21 +++++++++++++++++++++ lib/std/net.zig | 12 ++++++++++-- lib/std/os.zig | 8 +------- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index ae2d2f1499..3a79d36a10 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -1088,6 +1088,27 @@ pub const Loop = struct { } } + pub fn sendto( + self: *Loop, + /// The file descriptor of the sending socket. + sockfd: os.fd_t, + /// Message to send. + buf: []const u8, + flags: u32, + dest_addr: ?*const os.sockaddr, + addrlen: os.socklen_t, + ) os.SendError!usize { + while (true) { + return os.sendto(sockfd, buf, flags, dest_addr, addrlen) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdWritable(sockfd); + continue; + }, + else => return err, + }; + } + } + /// Performs an async `os.faccessatZ` using a separate thread. /// `fd` must block and not return EAGAIN. pub fn faccessatZ( diff --git a/lib/std/net.zig b/lib/std/net.zig index 928ebbbce5..8fe19f955d 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -1435,7 +1435,11 @@ fn resMSendRc( if (answers[i].len == 0) { var j: usize = 0; while (j < ns.len) : (j += 1) { - _ = os.sendto(fd, queries[i], os.MSG_NOSIGNAL, &ns[j].any, sl) catch undefined; + if (std.io.is_async) { + _ = std.event.Loop.instance.?.sendto(fd, queries[i], os.MSG_NOSIGNAL, &ns[j].any, sl) catch undefined; + } else { + _ = os.sendto(fd, queries[i], os.MSG_NOSIGNAL, &ns[j].any, sl) catch undefined; + } } } } @@ -1476,7 +1480,11 @@ fn resMSendRc( 0, 3 => {}, 2 => if (servfail_retry != 0) { servfail_retry -= 1; - _ = os.sendto(fd, queries[i], os.MSG_NOSIGNAL, &ns[j].any, sl) catch undefined; + if (std.io.is_async) { + _ = std.event.Loop.instance.?.sendto(fd, queries[i], os.MSG_NOSIGNAL, &ns[j].any, sl) catch undefined; + } else { + _ = os.sendto(fd, queries[i], os.MSG_NOSIGNAL, &ns[j].any, sl) catch undefined; + } }, else => continue, } diff --git a/lib/std/os.zig b/lib/std/os.zig index 1138013e8c..2f354e33d6 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -4571,14 +4571,8 @@ pub fn sendto( const rc = system.sendto(sockfd, buf.ptr, buf.len, flags, dest_addr, addrlen); switch (errno(rc)) { 0 => return @intCast(usize, rc), - EACCES => return error.AccessDenied, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdWritable(sockfd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EALREADY => return error.FastOpenAlreadyInProgress, EBADF => unreachable, // always a race condition ECONNRESET => return error.ConnectionResetByPeer, From c196c27af86f0f10b2f53240a68477391c4b9820 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Sat, 20 Jun 2020 00:45:51 +0200 Subject: [PATCH 126/210] recvfrom Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 18 ++++++++++++++++++ lib/std/net.zig | 5 ++++- lib/std/os.zig | 7 +------ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index 3a79d36a10..c3bf2495ff 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -1109,6 +1109,24 @@ pub const Loop = struct { } } + pub fn recvfrom( + sockfd: os.fd_t, + buf: []u8, + flags: u32, + src_addr: ?*os.sockaddr, + addrlen: ?*os.socklen_t, + ) os.RecvFromError!usize { + while (true) { + return os.recvfrom(sockfd, buf, flags, src_addr, addrlen) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdReadable(sockfd); + continue; + }, + else => return err, + }; + } + } + /// Performs an async `os.faccessatZ` using a separate thread. /// `fd` must block and not return EAGAIN. pub fn faccessatZ( diff --git a/lib/std/net.zig b/lib/std/net.zig index 8fe19f955d..fe7d0fafe6 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -1454,7 +1454,10 @@ fn resMSendRc( while (true) { var sl_copy = sl; - const rlen = os.recvfrom(fd, answer_bufs[next], 0, &sa.any, &sl_copy) catch break; + const rlen = if (std.io.is_async) + std.event.Loop.instance.?.recvfrom(fd, answer_bufs[next], 0, &sa.any, &sl_copy) catch break + else + os.recvfrom(fd, answer_bufs[next], 0, &sa.any, &sl_copy) catch break; // Ignore non-identifiable packets if (rlen < 4) continue; diff --git a/lib/std/os.zig b/lib/std/os.zig index 2f354e33d6..2c5f3065b2 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -5068,12 +5068,7 @@ pub fn recvfrom( ENOTCONN => unreachable, ENOTSOCK => unreachable, EINTR => continue, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdReadable(sockfd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, ENOMEM => return error.SystemResources, ECONNREFUSED => return error.ConnectionRefused, else => |err| return unexpectedErrno(err), From 054fafd7d9a5226f21e7be1737c6d352fe39f795 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 24 Sep 2020 16:22:45 -0700 Subject: [PATCH 127/210] stage2: implement @cImport Also rename Cache.CacheHash to Cache.Manifest --- BRANCH_TODO | 3 +- src/Cache.zig | 48 ++++---- src/Compilation.zig | 262 +++++++++++++++++++++++++++++++++++--------- src/main.zig | 2 +- src/stage1.zig | 37 ++++++- src/stage1/ir.cpp | 36 +++++- src/stage1/stage2.h | 4 +- src/stage1/zig0.cpp | 5 +- 8 files changed, 311 insertions(+), 86 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index dafcfc742f..8a00c9fded 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,6 +1,4 @@ - * repair @cImport * tests passing with -Dskip-non-native - * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj] * make sure zig cc works - using it as a preprocessor (-E) - try building some software @@ -24,6 +22,7 @@ * audit the base cache hash * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process. * restore error messages for stage2_add_link_lib + * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj] * implement proper parsing of clang stderr/stdout and exposing compile errors with the Compilation API * implement proper parsing of LLD stderr/stdout and exposing compile errors with the Compilation API diff --git a/src/Cache.zig b/src/Cache.zig index 7a0c78a1d9..cfcbc3e76a 100644 --- a/src/Cache.zig +++ b/src/Cache.zig @@ -12,9 +12,9 @@ const mem = std.mem; const fmt = std.fmt; const Allocator = std.mem.Allocator; -/// Be sure to call `CacheHash.deinit` after successful initialization. -pub fn obtain(cache: *const Cache) CacheHash { - return CacheHash{ +/// Be sure to call `Manifest.deinit` after successful initialization. +pub fn obtain(cache: *const Cache) Manifest { + return Manifest{ .cache = cache, .hash = cache.hash, .manifest_file = null, @@ -30,7 +30,7 @@ pub const hex_digest_len = bin_digest_len * 2; const manifest_file_size_max = 50 * 1024 * 1024; /// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it -/// provides enough collision resistance for the CacheHash use cases, while being one of our +/// provides enough collision resistance for the Manifest use cases, while being one of our /// fastest options right now. pub const Hasher = crypto.auth.siphash.SipHash128(1, 3); @@ -147,10 +147,10 @@ pub const Lock = struct { } }; -/// CacheHash manages project-local `zig-cache` directories. +/// Manifest manages project-local `zig-cache` directories. /// This is not a general-purpose cache. /// It is designed to be fast and simple, not to withstand attacks using specially-crafted input. -pub const CacheHash = struct { +pub const Manifest = struct { cache: *const Cache, /// Current state for incremental hashing. hash: HashHelper, @@ -173,7 +173,7 @@ pub const CacheHash = struct { /// ``` /// var file_contents = cache_hash.files.items[file_index].contents.?; /// ``` - pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize { + pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize { assert(self.manifest_file == null); try self.files.ensureCapacity(self.cache.gpa, self.files.items.len + 1); @@ -193,13 +193,13 @@ pub const CacheHash = struct { return idx; } - pub fn addOptionalFile(self: *CacheHash, optional_file_path: ?[]const u8) !void { + pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void { self.hash.add(optional_file_path != null); const file_path = optional_file_path orelse return; _ = try self.addFile(file_path, null); } - pub fn addListOfFiles(self: *CacheHash, list_of_files: []const []const u8) !void { + pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void { self.hash.add(list_of_files.len); for (list_of_files) |file_path| { _ = try self.addFile(file_path, null); @@ -210,13 +210,13 @@ pub const CacheHash = struct { /// A hex encoding of its hash is available by calling `final`. /// /// This function will also acquire an exclusive lock to the manifest file. This means - /// that a process holding a CacheHash will block any other process attempting to + /// that a process holding a Manifest will block any other process attempting to /// acquire the lock. /// /// The lock on the manifest file is released when `deinit` is called. As another /// option, one may call `toOwnedLock` to obtain a smaller object which can represent /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called. - pub fn hit(self: *CacheHash) !bool { + pub fn hit(self: *Manifest) !bool { assert(self.manifest_file == null); const ext = ".txt"; @@ -361,7 +361,7 @@ pub const CacheHash = struct { return true; } - pub fn unhit(self: *CacheHash, bin_digest: [bin_digest_len]u8, input_file_count: usize) void { + pub fn unhit(self: *Manifest, bin_digest: [bin_digest_len]u8, input_file_count: usize) void { // Reset the hash. self.hash.hasher = hasher_init; self.hash.hasher.update(&bin_digest); @@ -377,7 +377,7 @@ pub const CacheHash = struct { } } - fn populateFileHash(self: *CacheHash, ch_file: *File) !void { + fn populateFileHash(self: *Manifest, ch_file: *File) !void { const file = try fs.cwd().openFile(ch_file.path.?, .{}); defer file.close(); @@ -421,7 +421,7 @@ pub const CacheHash = struct { /// calculated. This is useful for processes that don't know the all the files that /// are depended on ahead of time. For example, a source file that can import other files /// will need to be recompiled if the imported file is changed. - pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]const u8 { + pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 { assert(self.manifest_file != null); const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path}); @@ -446,7 +446,7 @@ pub const CacheHash = struct { /// calculated. This is useful for processes that don't know the all the files that /// are depended on ahead of time. For example, a source file that can import other files /// will need to be recompiled if the imported file is changed. - pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void { + pub fn addFilePost(self: *Manifest, file_path: []const u8) !void { assert(self.manifest_file != null); const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path}); @@ -465,7 +465,7 @@ pub const CacheHash = struct { try self.populateFileHash(new_ch_file); } - pub fn addDepFilePost(self: *CacheHash, dir: fs.Dir, dep_file_basename: []const u8) !void { + pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void { assert(self.manifest_file != null); const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max); @@ -501,7 +501,7 @@ pub const CacheHash = struct { } /// Returns a hex encoded hash of the inputs. - pub fn final(self: *CacheHash) [hex_digest_len]u8 { + pub fn final(self: *Manifest) [hex_digest_len]u8 { assert(self.manifest_file != null); // We don't close the manifest file yet, because we want to @@ -519,7 +519,7 @@ pub const CacheHash = struct { return out_digest; } - pub fn writeManifest(self: *CacheHash) !void { + pub fn writeManifest(self: *Manifest) !void { assert(self.manifest_file != null); if (!self.manifest_dirty) return; @@ -544,18 +544,18 @@ pub const CacheHash = struct { } /// Obtain only the data needed to maintain a lock on the manifest file. - /// The `CacheHash` remains safe to deinit. + /// The `Manifest` remains safe to deinit. /// Don't forget to call `writeManifest` before this! - pub fn toOwnedLock(self: *CacheHash) Lock { + pub fn toOwnedLock(self: *Manifest) Lock { const manifest_file = self.manifest_file.?; self.manifest_file = null; return Lock{ .manifest_file = manifest_file }; } - /// Releases the manifest file and frees any memory the CacheHash was using. - /// `CacheHash.hit` must be called first. + /// Releases the manifest file and frees any memory the Manifest was using. + /// `Manifest.hit` must be called first. /// Don't forget to call `writeManifest` before this! - pub fn deinit(self: *CacheHash) void { + pub fn deinit(self: *Manifest) void { if (self.manifest_file) |file| { file.close(); } @@ -808,7 +808,7 @@ test "no file inputs" { testing.expectEqual(digest1, digest2); } -test "CacheHashes with files added after initial hash work" { +test "Manifest with files added after initial hash work" { if (std.Target.current.os.tag == .wasi) { // https://github.com/ziglang/zig/issues/5437 return error.SkipZigTest; diff --git a/src/Compilation.zig b/src/Compilation.zig index 756425dca5..ce55c2aaf4 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -22,6 +22,7 @@ const fatal = @import("main.zig").fatal; const Module = @import("Module.zig"); const Cache = @import("Cache.zig"); const stage1 = @import("stage1.zig"); +const translate_c = @import("translate_c.zig"); /// General-purpose allocator. Used for both temporary and long-term storage. gpa: *Allocator, @@ -30,7 +31,7 @@ arena_state: std.heap.ArenaAllocator.State, bin_file: *link.File, c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{}, stage1_lock: ?Cache.Lock = null, -stage1_cache_hash: *Cache.CacheHash = undefined, +stage1_cache_manifest: *Cache.Manifest = undefined, link_error_flags: link.File.ErrorFlags = .{}, @@ -1198,29 +1199,182 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { }; } -fn updateCObject(comp: *Compilation, c_object: *CObject) !void { +fn obtainCObjectCacheManifest(comp: *Compilation) Cache.Manifest { + var man = comp.cache_parent.obtain(); + + // Only things that need to be added on top of the base hash, and only things + // that apply both to @cImport and compiling C objects. No linking stuff here! + // Also nothing that applies only to compiling .zig code. + + man.hash.add(comp.sanitize_c); + man.hash.addListOfBytes(comp.clang_argv); + man.hash.add(comp.bin_file.options.link_libcpp); + man.hash.addListOfBytes(comp.libc_include_dir_list); + + return man; +} + +test "cImport" { + _ = cImport; +} + +const CImportResult = struct { + out_zig_path: []u8, + errors: []translate_c.ClangErrMsg, +}; + +/// Caller owns returned memory. +/// This API is currently coupled pretty tightly to stage1's needs; it will need to be reworked +/// a bit when we want to start using it from self-hosted. +pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult { + if (!build_options.have_llvm) + return error.ZigCompilerNotBuiltWithLLVMExtensions; + const tracy = trace(@src()); defer tracy.end(); + const cimport_zig_basename = "cimport.zig"; + + var man = comp.obtainCObjectCacheManifest(); + defer man.deinit(); + + man.hash.addBytes(c_src); + + // If the previous invocation resulted in clang errors, we will see a hit + // here with 0 files in the manifest, in which case it is actually a miss. + const actual_hit = (try man.hit()) and man.files.items.len != 0; + const digest = if (!actual_hit) digest: { + var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); + defer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + // We need a place to leave the .h file so we can can log it in case of verbose_cimport. + // This block is so that the defers for closing the tmp directory handle can run before + // we try to delete the directory after the block. + const result: struct { tmp_dir_sub_path: []const u8, digest: [Cache.hex_digest_len]u8 } = blk: { + const tmp_digest = man.hash.peek(); + const tmp_dir_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &tmp_digest }); + var zig_cache_tmp_dir = try comp.local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); + defer zig_cache_tmp_dir.close(); + const cimport_c_basename = "cimport.c"; + const out_h_path = try comp.local_cache_directory.join(arena, &[_][]const u8{ + tmp_dir_sub_path, cimport_c_basename, + }); + const out_dep_path = try std.fmt.allocPrint(arena, "{}.d", .{out_h_path}); + + try zig_cache_tmp_dir.writeFile(cimport_c_basename, c_src); + if (comp.verbose_cimport) { + log.info("C import source: {}", .{out_h_path}); + } + + var argv = std.ArrayList([]const u8).init(comp.gpa); + defer argv.deinit(); + + try comp.addTranslateCCArgs(arena, &argv, .c, out_dep_path); + + try argv.append(out_h_path); + + if (comp.verbose_cc) { + dump_argv(argv.items); + } + + // Convert to null terminated args. + const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1); + new_argv_with_sentinel[argv.items.len] = null; + const new_argv = new_argv_with_sentinel[0..argv.items.len :null]; + for (argv.items) |arg, i| { + new_argv[i] = try arena.dupeZ(u8, arg); + } + + const c_headers_dir_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"include"}); + const c_headers_dir_path_z = try arena.dupeZ(u8, c_headers_dir_path); + var clang_errors: []translate_c.ClangErrMsg = &[0]translate_c.ClangErrMsg{}; + const tree = translate_c.translate( + comp.gpa, + new_argv.ptr, + new_argv.ptr + new_argv.len, + &clang_errors, + c_headers_dir_path_z, + ) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ASTUnitFailure => { + log.warn("clang API returned errors but due to a clang bug, it is not exposing the errors for zig to see. For more details: https://github.com/ziglang/zig/issues/4455", .{}); + return error.ASTUnitFailure; + }, + error.SemanticAnalyzeFail => { + return CImportResult{ + .out_zig_path = "", + .errors = clang_errors, + }; + }, + }; + defer tree.deinit(); + + if (comp.verbose_cimport) { + log.info("C import .d file: {}", .{out_dep_path}); + } + + const dep_basename = std.fs.path.basename(out_dep_path); + try man.addDepFilePost(zig_cache_tmp_dir, dep_basename); + + const digest = man.final(); + const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); + var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{}); + defer o_dir.close(); + + var out_zig_file = try o_dir.createFile(cimport_zig_basename, .{}); + defer out_zig_file.close(); + + var bos = std.io.bufferedOutStream(out_zig_file.writer()); + _ = try std.zig.render(comp.gpa, bos.writer(), tree); + try bos.flush(); + + man.writeManifest() catch |err| { + log.warn("failed to write cache manifest for C import: {}", .{@errorName(err)}); + }; + + break :blk .{ .tmp_dir_sub_path = tmp_dir_sub_path, .digest = digest }; + }; + if (!comp.verbose_cimport) { + // Remove the tmp dir and files to save space because we don't need them again. + comp.local_cache_directory.handle.deleteTree(result.tmp_dir_sub_path) catch |err| { + log.warn("failed to delete tmp files for C import: {}", .{@errorName(err)}); + }; + } + break :digest result.digest; + } else man.final(); + + const out_zig_path = try comp.local_cache_directory.join(comp.gpa, &[_][]const u8{ + "o", &digest, cimport_zig_basename, + }); + if (comp.verbose_cimport) { + log.info("C import output: {}\n", .{out_zig_path}); + } + return CImportResult{ + .out_zig_path = out_zig_path, + .errors = &[0]translate_c.ClangErrMsg{}, + }; +} + +fn updateCObject(comp: *Compilation, c_object: *CObject) !void { if (!build_options.have_llvm) { return comp.failCObj(c_object, "clang not available: compiler built without LLVM extensions", .{}); } const self_exe_path = comp.self_exe_path orelse return comp.failCObj(c_object, "clang compilation disabled", .{}); + const tracy = trace(@src()); + defer tracy.end(); + if (c_object.clearStatus(comp.gpa)) { // There was previous failure. comp.failed_c_objects.removeAssertDiscard(c_object); } - var ch = comp.cache_parent.obtain(); - defer ch.deinit(); + var man = comp.obtainCObjectCacheManifest(); + defer man.deinit(); - ch.hash.add(comp.sanitize_c); - ch.hash.addListOfBytes(comp.clang_argv); - ch.hash.add(comp.bin_file.options.link_libcpp); - ch.hash.addListOfBytes(comp.libc_include_dir_list); - _ = try ch.addFile(c_object.src.src_path, null); + _ = try man.addFile(c_object.src.src_path, null); { // Hash the extra flags, with special care to call addFile for file parameters. // TODO this logic can likely be improved by utilizing clang_options_data.zig. @@ -1228,11 +1382,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { var arg_i: usize = 0; while (arg_i < c_object.src.extra_flags.len) : (arg_i += 1) { const arg = c_object.src.extra_flags[arg_i]; - ch.hash.addBytes(arg); + man.hash.addBytes(arg); for (file_args) |file_arg| { if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_object.src.extra_flags.len) { arg_i += 1; - _ = try ch.addFile(c_object.src.extra_flags[arg_i], null); + _ = try man.addFile(c_object.src.extra_flags[arg_i], null); } } } @@ -1254,7 +1408,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { mem.split(c_source_basename, ".").next().?; const o_basename = try std.fmt.allocPrint(arena, "{}{}", .{ o_basename_noext, comp.getTarget().oFileExt() }); - const digest = if ((try ch.hit()) and !comp.disable_c_depfile) ch.final() else blk: { + const digest = if ((try man.hit()) and !comp.disable_c_depfile) man.final() else blk: { var argv = std.ArrayList([]const u8).init(comp.gpa); defer argv.deinit(); @@ -1270,7 +1424,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { null else try std.fmt.allocPrint(arena, "{}.d", .{out_obj_path}); - try comp.addCCArgs(arena, &argv, ext, false, out_dep_path); + try comp.addCCArgs(arena, &argv, ext, out_dep_path); try argv.append("-o"); try argv.append(out_obj_path); @@ -1325,12 +1479,12 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { if (code != 0) { // TODO parse clang stderr and turn it into an error message // and then call failCObjWithOwnedErrorMsg - std.log.err("clang failed with stderr: {}", .{stderr}); + log.err("clang failed with stderr: {}", .{stderr}); return comp.failCObj(c_object, "clang exited with code {}", .{code}); } }, else => { - std.log.err("clang terminated with stderr: {}", .{stderr}); + log.err("clang terminated with stderr: {}", .{stderr}); return comp.failCObj(c_object, "clang terminated unexpectedly", .{}); }, } @@ -1339,15 +1493,15 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { if (out_dep_path) |dep_file_path| { const dep_basename = std.fs.path.basename(dep_file_path); // Add the files depended on to the cache system. - try ch.addDepFilePost(zig_cache_tmp_dir, dep_basename); + try man.addDepFilePost(zig_cache_tmp_dir, dep_basename); // Just to save disk space, we delete the file because it is never needed again. zig_cache_tmp_dir.deleteFile(dep_basename) catch |err| { - std.log.warn("failed to delete '{}': {}", .{ dep_file_path, @errorName(err) }); + log.warn("failed to delete '{}': {}", .{ dep_file_path, @errorName(err) }); }; } // Rename into place. - const digest = ch.final(); + const digest = man.final(); const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{}); defer o_dir.close(); @@ -1355,8 +1509,8 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { const tmp_basename = std.fs.path.basename(out_obj_path); try std.os.renameat(zig_cache_tmp_dir.fd, tmp_basename, o_dir.fd, o_basename); - ch.writeManifest() catch |err| { - std.log.warn("failed to write cache manifest when compiling '{}': {}", .{ c_object.src.src_path, @errorName(err) }); + man.writeManifest() catch |err| { + log.warn("failed to write cache manifest when compiling '{}': {}", .{ c_object.src.src_path, @errorName(err) }); }; break :blk digest; }; @@ -1369,7 +1523,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { c_object.status = .{ .success = .{ .object_path = try std.fs.path.join(comp.gpa, components), - .lock = ch.toOwnedLock(), + .lock = man.toOwnedLock(), }, }; } @@ -1384,21 +1538,28 @@ fn tmpFilePath(comp: *Compilation, arena: *Allocator, suffix: []const u8) error{ } } +pub fn addTranslateCCArgs( + comp: *Compilation, + arena: *Allocator, + argv: *std.ArrayList([]const u8), + ext: FileExt, + out_dep_path: ?[]const u8, +) !void { + try comp.addCCArgs(arena, argv, ext, out_dep_path); + // This gives us access to preprocessing entities, presumably at the cost of performance. + try argv.appendSlice(&[_][]const u8{ "-Xclang", "-detailed-preprocessing-record" }); +} + /// Add common C compiler args between translate-c and C object compilation. pub fn addCCArgs( comp: *Compilation, arena: *Allocator, argv: *std.ArrayList([]const u8), ext: FileExt, - translate_c: bool, out_dep_path: ?[]const u8, ) !void { const target = comp.getTarget(); - if (translate_c) { - try argv.appendSlice(&[_][]const u8{ "-x", "c" }); - } - if (ext == .cpp) { try argv.append("-nostdinc++"); } @@ -1488,11 +1649,6 @@ pub fn addCCArgs( if (mcmodel != .default) { try argv.append(try std.fmt.allocPrint(arena, "-mcmodel={}", .{@tagName(mcmodel)})); } - if (translate_c) { - // This gives us access to preprocessing entities, presumably at the cost of performance. - try argv.append("-Xclang"); - try argv.append("-detailed-preprocessing-record"); - } // windows.h has files such as pshpack1.h which do #pragma packing, triggering a clang warning. // So for this target, we disable this warning. @@ -2118,7 +2274,7 @@ pub fn updateSubCompilation(sub_compilation: *Compilation) !void { if (errors.list.len != 0) { for (errors.list) |full_err_msg| { - std.log.err("{}:{}:{}: {}\n", .{ + log.err("{}:{}:{}: {}\n", .{ full_err_msg.src_path, full_err_msg.line + 1, full_err_msg.column + 1, @@ -2231,23 +2387,23 @@ fn updateStage1Module(comp: *Compilation) !void { // the artifact directory the same, however, so we take the same strategy as linking // does where we have a file which specifies the hash of the output directory so that we can // skip the expensive compilation step if the hash matches. - var ch = comp.cache_parent.obtain(); - defer ch.deinit(); + var man = comp.cache_parent.obtain(); + defer man.deinit(); - _ = try ch.addFile(main_zig_file, null); - ch.hash.add(comp.bin_file.options.valgrind); - ch.hash.add(comp.bin_file.options.single_threaded); - ch.hash.add(target.os.getVersionRange()); - ch.hash.add(comp.bin_file.options.dll_export_fns); - ch.hash.add(comp.bin_file.options.function_sections); - ch.hash.add(comp.is_test); + _ = try man.addFile(main_zig_file, null); + man.hash.add(comp.bin_file.options.valgrind); + man.hash.add(comp.bin_file.options.single_threaded); + man.hash.add(target.os.getVersionRange()); + man.hash.add(comp.bin_file.options.dll_export_fns); + man.hash.add(comp.bin_file.options.function_sections); + man.hash.add(comp.is_test); // Capture the state in case we come back from this branch where the hash doesn't match. - const prev_hash_state = ch.hash.peekBin(); - const input_file_count = ch.files.items.len; + const prev_hash_state = man.hash.peekBin(); + const input_file_count = man.files.items.len; - if (try ch.hit()) { - const digest = ch.final(); + if (try man.hit()) { + const digest = man.final(); var prev_digest_buf: [digest.len]u8 = undefined; const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { @@ -2257,11 +2413,11 @@ fn updateStage1Module(comp: *Compilation) !void { }; if (mem.eql(u8, prev_digest, &digest)) { log.debug("stage1 {} digest={} match - skipping invocation", .{ mod.root_pkg.root_src_path, digest }); - comp.stage1_lock = ch.toOwnedLock(); + comp.stage1_lock = man.toOwnedLock(); return; } log.debug("stage1 {} prev_digest={} new_digest={}", .{ mod.root_pkg.root_src_path, prev_digest, digest }); - ch.unhit(prev_hash_state, input_file_count); + man.unhit(prev_hash_state, input_file_count); } // We are about to change the output file to be different, so we invalidate the build hash now. @@ -2285,7 +2441,7 @@ fn updateStage1Module(comp: *Compilation) !void { defer main_progress_node.end(); if (comp.color == .Off) progress.terminal = null; - comp.stage1_cache_hash = &ch; + comp.stage1_cache_manifest = &man; const main_pkg_path = mod.root_pkg.root_src_directory.path orelse ""; @@ -2350,22 +2506,22 @@ fn updateStage1Module(comp: *Compilation) !void { stage1_module.build_object(); stage1_module.destroy(); - const digest = ch.final(); + const digest = man.final(); log.debug("stage1 {} final digest={}", .{ mod.root_pkg.root_src_path, digest }); // Update the dangling symlink with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { - std.log.warn("failed to save stage1 hash digest symlink: {}", .{@errorName(err)}); + log.warn("failed to save stage1 hash digest symlink: {}", .{@errorName(err)}); }; // Again failure here only means an unnecessary cache miss. - ch.writeManifest() catch |err| { - std.log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)}); + man.writeManifest() catch |err| { + log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)}); }; // We hang on to this lock so that the output file path can be used without // other processes clobbering it. - comp.stage1_lock = ch.toOwnedLock(); + comp.stage1_lock = man.toOwnedLock(); } fn createStage1Pkg( diff --git a/src/main.zig b/src/main.zig index 1cc0098e66..ca885329d8 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1617,7 +1617,7 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator) !void { const c_source_file = comp.c_source_files[0]; const file_ext = Compilation.classifyFileExt(c_source_file.src_path); - try comp.addCCArgs(arena, &argv, file_ext, true, null); + try comp.addTranslateCCArgs(arena, &argv, file_ext, null); try argv.append(c_source_file.src_path); if (comp.verbose_cc) { diff --git a/src/stage1.zig b/src/stage1.zig index 380d3cbd9f..18d2b5289c 100644 --- a/src/stage1.zig +++ b/src/stage1.zig @@ -11,10 +11,12 @@ const fatal = stage2.fatal; const CrossTarget = std.zig.CrossTarget; const Target = std.Target; const Compilation = @import("Compilation.zig"); +const translate_c = @import("translate_c.zig"); comptime { assert(std.builtin.link_libc); assert(build_options.is_stage1); + assert(build_options.have_llvm); _ = @import("compiler_rt"); } @@ -322,8 +324,37 @@ const Stage2SemVer = extern struct { }; // ABI warning -export fn stage2_cimport(stage1: *Module) [*:0]const u8 { - @panic("TODO implement stage2_cimport"); +export fn stage2_cimport( + stage1: *Module, + c_src_ptr: [*]const u8, + c_src_len: usize, + out_zig_path_ptr: *[*]const u8, + out_zig_path_len: *usize, + out_errors_ptr: *[*]translate_c.ClangErrMsg, + out_errors_len: *usize, +) Error { + const comp = @intToPtr(*Compilation, stage1.userdata); + const c_src = c_src_ptr[0..c_src_len]; + const result = comp.cImport(c_src) catch |err| switch (err) { + error.SystemResources => return .SystemResources, + error.OperationAborted => return .OperationAborted, + error.BrokenPipe => return .BrokenPipe, + error.DiskQuota => return .DiskQuota, + error.FileTooBig => return .FileTooBig, + error.NoSpaceLeft => return .NoSpaceLeft, + error.AccessDenied => return .AccessDenied, + error.OutOfMemory => return .OutOfMemory, + error.Unexpected => return .Unexpected, + error.InputOutput => return .FileSystem, + error.ASTUnitFailure => return .ASTUnitFailure, + else => return .Unexpected, + }; + out_zig_path_ptr.* = result.out_zig_path.ptr; + out_zig_path_len.* = result.out_zig_path.len; + out_errors_ptr.* = result.errors.ptr; + out_errors_len.* = result.errors.len; + if (result.errors.len != 0) return .CCompileErrors; + return Error.None; } export fn stage2_add_link_lib( @@ -345,7 +376,7 @@ export fn stage2_fetch_file( const comp = @intToPtr(*Compilation, stage1.userdata); const file_path = path_ptr[0..path_len]; const max_file_size = std.math.maxInt(u32); - const contents = comp.stage1_cache_hash.addFilePostFetch(file_path, max_file_size) catch return null; + const contents = comp.stage1_cache_manifest.addFilePostFetch(file_path, max_file_size) catch return null; result_len.* = contents.len; return contents.ptr; } diff --git a/src/stage1/ir.cpp b/src/stage1/ir.cpp index f0546f9e01..cb40c3b81e 100644 --- a/src/stage1/ir.cpp +++ b/src/stage1/ir.cpp @@ -26382,7 +26382,41 @@ static IrInstGen *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstSrcCImpo cimport_pkg->package_table.put(buf_create_from_str("std"), ira->codegen->std_package); buf_init_from_buf(&cimport_pkg->pkg_path, namespace_name); - Buf *out_zig_path = buf_create_from_str(stage2_cimport(&ira->codegen->stage1)); + const char *out_zig_path_ptr; + size_t out_zig_path_len; + Stage2ErrorMsg *errors_ptr; + size_t errors_len; + if ((err = stage2_cimport(&ira->codegen->stage1, + buf_ptr(&cimport_scope->buf), buf_len(&cimport_scope->buf), + &out_zig_path_ptr, &out_zig_path_len, + &errors_ptr, &errors_len))) + { + if (err != ErrorCCompileErrors) { + ir_add_error_node(ira, node, buf_sprintf("C import failed: %s", err_str(err))); + return ira->codegen->invalid_inst_gen; + } + + ErrorMsg *parent_err_msg = ir_add_error_node(ira, node, buf_sprintf("C import failed")); + if (!ira->codegen->stage1.link_libc) { + add_error_note(ira->codegen, parent_err_msg, node, + buf_sprintf("libc headers not available; compilation does not link against libc")); + } + for (size_t i = 0; i < errors_len; i += 1) { + Stage2ErrorMsg *clang_err = &errors_ptr[i]; + // Clang can emit "too many errors, stopping now", in which case `source` and `filename_ptr` are null + if (clang_err->source && clang_err->filename_ptr) { + ErrorMsg *err_msg = err_msg_create_with_offset( + clang_err->filename_ptr ? + buf_create_from_mem(clang_err->filename_ptr, clang_err->filename_len) : buf_alloc(), + clang_err->line, clang_err->column, clang_err->offset, clang_err->source, + buf_create_from_mem(clang_err->msg_ptr, clang_err->msg_len)); + err_msg_add_note(parent_err_msg, err_msg); + } + } + + return ira->codegen->invalid_inst_gen; + } + Buf *out_zig_path = buf_create_from_mem(out_zig_path_ptr, out_zig_path_len); Buf *import_code = buf_alloc(); if ((err = file_fetch(ira->codegen, out_zig_path, import_code))) { diff --git a/src/stage1/stage2.h b/src/stage1/stage2.h index 613cae2a77..886a4c2660 100644 --- a/src/stage1/stage2.h +++ b/src/stage1/stage2.h @@ -165,7 +165,9 @@ ZIG_EXTERN_C const char *stage2_fetch_file(struct ZigStage1 *stage1, const char size_t *result_len); // ABI warning -ZIG_EXTERN_C const char *stage2_cimport(struct ZigStage1 *stage1); +ZIG_EXTERN_C Error stage2_cimport(struct ZigStage1 *stage1, const char *c_src_ptr, size_t c_src_len, + const char **out_zig_path_ptr, size_t *out_zig_path_len, + struct Stage2ErrorMsg **out_errors_ptr, size_t *out_errors_len); // ABI warning ZIG_EXTERN_C const char *stage2_add_link_lib(struct ZigStage1 *stage1, diff --git a/src/stage1/zig0.cpp b/src/stage1/zig0.cpp index bd447abded..5c384991f9 100644 --- a/src/stage1/zig0.cpp +++ b/src/stage1/zig0.cpp @@ -511,7 +511,10 @@ const char *stage2_fetch_file(struct ZigStage1 *stage1, const char *path_ptr, si return buf_ptr(&contents_buf); } -const char *stage2_cimport(struct ZigStage1 *stage1) { +Error stage2_cimport(struct ZigStage1 *stage1, const char *c_src_ptr, size_t c_src_len, + const char **out_zig_path_ptr, size_t *out_zig_path_len, + struct Stage2ErrorMsg **out_errors_ptr, size_t *out_errors_len) +{ const char *msg = "stage0 called stage2_cimport"; stage2_panic(msg, strlen(msg)); } From 4f0850fe285937e37a7e9260c399867d3f61a487 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 24 Sep 2020 21:10:11 -0700 Subject: [PATCH 128/210] don't delete C import .c source from zig-cache preventing CacheUnavailable error when doing `@cImport` --- src/Compilation.zig | 172 +++++++++++++++++++++----------------------- src/stage1.zig | 1 + 2 files changed, 81 insertions(+), 92 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index ce55c2aaf4..0b222bd787 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1248,100 +1248,88 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult { defer arena_allocator.deinit(); const arena = &arena_allocator.allocator; - // We need a place to leave the .h file so we can can log it in case of verbose_cimport. - // This block is so that the defers for closing the tmp directory handle can run before - // we try to delete the directory after the block. - const result: struct { tmp_dir_sub_path: []const u8, digest: [Cache.hex_digest_len]u8 } = blk: { - const tmp_digest = man.hash.peek(); - const tmp_dir_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &tmp_digest }); - var zig_cache_tmp_dir = try comp.local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); - defer zig_cache_tmp_dir.close(); - const cimport_c_basename = "cimport.c"; - const out_h_path = try comp.local_cache_directory.join(arena, &[_][]const u8{ - tmp_dir_sub_path, cimport_c_basename, - }); - const out_dep_path = try std.fmt.allocPrint(arena, "{}.d", .{out_h_path}); + const tmp_digest = man.hash.peek(); + const tmp_dir_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &tmp_digest }); + var zig_cache_tmp_dir = try comp.local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); + defer zig_cache_tmp_dir.close(); + const cimport_c_basename = "cimport.c"; + const out_h_path = try comp.local_cache_directory.join(arena, &[_][]const u8{ + tmp_dir_sub_path, cimport_c_basename, + }); + const out_dep_path = try std.fmt.allocPrint(arena, "{}.d", .{out_h_path}); - try zig_cache_tmp_dir.writeFile(cimport_c_basename, c_src); - if (comp.verbose_cimport) { - log.info("C import source: {}", .{out_h_path}); - } - - var argv = std.ArrayList([]const u8).init(comp.gpa); - defer argv.deinit(); - - try comp.addTranslateCCArgs(arena, &argv, .c, out_dep_path); - - try argv.append(out_h_path); - - if (comp.verbose_cc) { - dump_argv(argv.items); - } - - // Convert to null terminated args. - const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1); - new_argv_with_sentinel[argv.items.len] = null; - const new_argv = new_argv_with_sentinel[0..argv.items.len :null]; - for (argv.items) |arg, i| { - new_argv[i] = try arena.dupeZ(u8, arg); - } - - const c_headers_dir_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"include"}); - const c_headers_dir_path_z = try arena.dupeZ(u8, c_headers_dir_path); - var clang_errors: []translate_c.ClangErrMsg = &[0]translate_c.ClangErrMsg{}; - const tree = translate_c.translate( - comp.gpa, - new_argv.ptr, - new_argv.ptr + new_argv.len, - &clang_errors, - c_headers_dir_path_z, - ) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ASTUnitFailure => { - log.warn("clang API returned errors but due to a clang bug, it is not exposing the errors for zig to see. For more details: https://github.com/ziglang/zig/issues/4455", .{}); - return error.ASTUnitFailure; - }, - error.SemanticAnalyzeFail => { - return CImportResult{ - .out_zig_path = "", - .errors = clang_errors, - }; - }, - }; - defer tree.deinit(); - - if (comp.verbose_cimport) { - log.info("C import .d file: {}", .{out_dep_path}); - } - - const dep_basename = std.fs.path.basename(out_dep_path); - try man.addDepFilePost(zig_cache_tmp_dir, dep_basename); - - const digest = man.final(); - const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); - var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{}); - defer o_dir.close(); - - var out_zig_file = try o_dir.createFile(cimport_zig_basename, .{}); - defer out_zig_file.close(); - - var bos = std.io.bufferedOutStream(out_zig_file.writer()); - _ = try std.zig.render(comp.gpa, bos.writer(), tree); - try bos.flush(); - - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest for C import: {}", .{@errorName(err)}); - }; - - break :blk .{ .tmp_dir_sub_path = tmp_dir_sub_path, .digest = digest }; - }; - if (!comp.verbose_cimport) { - // Remove the tmp dir and files to save space because we don't need them again. - comp.local_cache_directory.handle.deleteTree(result.tmp_dir_sub_path) catch |err| { - log.warn("failed to delete tmp files for C import: {}", .{@errorName(err)}); - }; + try zig_cache_tmp_dir.writeFile(cimport_c_basename, c_src); + if (comp.verbose_cimport) { + log.info("C import source: {}", .{out_h_path}); } - break :digest result.digest; + + var argv = std.ArrayList([]const u8).init(comp.gpa); + defer argv.deinit(); + + try comp.addTranslateCCArgs(arena, &argv, .c, out_dep_path); + + try argv.append(out_h_path); + + if (comp.verbose_cc) { + dump_argv(argv.items); + } + + // Convert to null terminated args. + const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1); + new_argv_with_sentinel[argv.items.len] = null; + const new_argv = new_argv_with_sentinel[0..argv.items.len :null]; + for (argv.items) |arg, i| { + new_argv[i] = try arena.dupeZ(u8, arg); + } + + const c_headers_dir_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"include"}); + const c_headers_dir_path_z = try arena.dupeZ(u8, c_headers_dir_path); + var clang_errors: []translate_c.ClangErrMsg = &[0]translate_c.ClangErrMsg{}; + const tree = translate_c.translate( + comp.gpa, + new_argv.ptr, + new_argv.ptr + new_argv.len, + &clang_errors, + c_headers_dir_path_z, + ) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ASTUnitFailure => { + log.warn("clang API returned errors but due to a clang bug, it is not exposing the errors for zig to see. For more details: https://github.com/ziglang/zig/issues/4455", .{}); + return error.ASTUnitFailure; + }, + error.SemanticAnalyzeFail => { + return CImportResult{ + .out_zig_path = "", + .errors = clang_errors, + }; + }, + }; + defer tree.deinit(); + + if (comp.verbose_cimport) { + log.info("C import .d file: {}", .{out_dep_path}); + } + + const dep_basename = std.fs.path.basename(out_dep_path); + try man.addDepFilePost(zig_cache_tmp_dir, dep_basename); + + const digest = man.final(); + const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); + var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{}); + defer o_dir.close(); + + var out_zig_file = try o_dir.createFile(cimport_zig_basename, .{}); + defer out_zig_file.close(); + + var bos = std.io.bufferedOutStream(out_zig_file.writer()); + _ = try std.zig.render(comp.gpa, bos.writer(), tree); + try bos.flush(); + + man.writeManifest() catch |err| { + log.warn("failed to write cache manifest for C import: {}", .{@errorName(err)}); + }; + + break :digest digest; } else man.final(); const out_zig_path = try comp.local_cache_directory.join(comp.gpa, &[_][]const u8{ diff --git a/src/stage1.zig b/src/stage1.zig index 18d2b5289c..7a9609f317 100644 --- a/src/stage1.zig +++ b/src/stage1.zig @@ -347,6 +347,7 @@ export fn stage2_cimport( error.Unexpected => return .Unexpected, error.InputOutput => return .FileSystem, error.ASTUnitFailure => return .ASTUnitFailure, + error.CacheUnavailable => return .CacheUnavailable, else => return .Unexpected, }; out_zig_path_ptr.* = result.out_zig_path.ptr; From 67463c4357d0d0f43b8962231659fb8a1cc4869a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 24 Sep 2020 21:52:37 -0700 Subject: [PATCH 129/210] correct `@cImport` caching to handle "unhit" case if you hit() you have to unhit() to get the same digest if you don't end up treating it as a hit. --- src/Compilation.zig | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 0b222bd787..03da22d80c 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1242,7 +1242,16 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult { // If the previous invocation resulted in clang errors, we will see a hit // here with 0 files in the manifest, in which case it is actually a miss. - const actual_hit = (try man.hit()) and man.files.items.len != 0; + // We need to "unhit" in this case, to keep the digests matching. + const prev_hash_state = man.hash.peekBin(); + const actual_hit = hit: { + const is_hit = try man.hit(); + if (man.files.items.len == 0) { + man.unhit(prev_hash_state, 0); + break :hit false; + } + break :hit true; + }; const digest = if (!actual_hit) digest: { var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); defer arena_allocator.deinit(); @@ -1396,7 +1405,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { mem.split(c_source_basename, ".").next().?; const o_basename = try std.fmt.allocPrint(arena, "{}{}", .{ o_basename_noext, comp.getTarget().oFileExt() }); - const digest = if ((try man.hit()) and !comp.disable_c_depfile) man.final() else blk: { + const digest = if (!comp.disable_c_depfile and try man.hit()) man.final() else blk: { var argv = std.ArrayList([]const u8).init(comp.gpa); defer argv.deinit(); From 5440ca8eb7b7078cbca4b89f6540f08b5bb94960 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 24 Sep 2020 22:15:40 -0700 Subject: [PATCH 130/210] --main-pkg-path properly resolves the relative root src file path --- src/main.zig | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main.zig b/src/main.zig index ca885329d8..be2c55f828 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1276,15 +1276,15 @@ pub fn buildOutputType( defer if (cleanup_root_dir) |*dir| dir.close(); const root_pkg: ?*Package = if (root_src_file) |src_path| blk: { - root_pkg_memory.root_src_directory = m: { - if (main_pkg_path) |p| { - const dir = try fs.cwd().openDir(p, .{}); - cleanup_root_dir = dir; - break :m .{ .path = p, .handle = dir }; - } - break :m .{ .path = null, .handle = fs.cwd() }; - }; - root_pkg_memory.root_src_path = src_path; + if (main_pkg_path) |p| { + const dir = try fs.cwd().openDir(p, .{}); + cleanup_root_dir = dir; + root_pkg_memory.root_src_directory = .{ .path = p, .handle = dir }; + root_pkg_memory.root_src_path = try fs.path.relative(arena, p, src_path); + } else { + root_pkg_memory.root_src_directory = .{ .path = null, .handle = fs.cwd() }; + root_pkg_memory.root_src_path = src_path; + } break :blk &root_pkg_memory; } else null; From 7964341c76fab985b06a5022b88ceb261305ffaf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 24 Sep 2020 22:18:53 -0700 Subject: [PATCH 131/210] zig build: update to use new --version flag instead of legacy flags --- lib/std/build.zig | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/std/build.zig b/lib/std/build.zig index 7499b40533..2c7a0da53a 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -2081,14 +2081,8 @@ pub const LibExeObjStep = struct { if (self.kind == Kind.Lib and self.is_dynamic) { if (self.version) |version| { - zig_args.append("--ver-major") catch unreachable; - zig_args.append(builder.fmt("{}", .{version.major})) catch unreachable; - - zig_args.append("--ver-minor") catch unreachable; - zig_args.append(builder.fmt("{}", .{version.minor})) catch unreachable; - - zig_args.append("--ver-patch") catch unreachable; - zig_args.append(builder.fmt("{}", .{version.patch})) catch unreachable; + zig_args.append("--version") catch unreachable; + zig_args.append(builder.fmt("{}", .{version})) catch unreachable; } } if (self.is_dynamic) { From 1a8669eadaedc50fd7703412389f3c97fdffb913 Mon Sep 17 00:00:00 2001 From: Josh Wolfe Date: Thu, 24 Sep 2020 21:22:39 -0400 Subject: [PATCH 132/210] build.zig: addBuildOptionArtifact --- lib/std/build.zig | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/std/build.zig b/lib/std/build.zig index 69f44bad32..c0d7f0b8ed 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -1165,6 +1165,11 @@ pub const FileSource = union(enum) { } }; +const BuildOptionArtifactArg = struct { + name: []const u8, + artifact: *LibExeObjStep, +}; + pub const LibExeObjStep = struct { step: Step, builder: *Builder, @@ -1210,6 +1215,7 @@ pub const LibExeObjStep = struct { out_pdb_filename: []const u8, packages: ArrayList(Pkg), build_options_contents: std.ArrayList(u8), + build_options_artifact_args: std.ArrayList(BuildOptionArtifactArg), system_linker_hack: bool = false, object_src: []const u8, @@ -1355,6 +1361,7 @@ pub const LibExeObjStep = struct { .framework_dirs = ArrayList([]const u8).init(builder.allocator), .object_src = undefined, .build_options_contents = std.ArrayList(u8).init(builder.allocator), + .build_options_artifact_args = std.ArrayList(BuildOptionArtifactArg).init(builder.allocator), .c_std = Builder.CStd.C99, .override_lib_dir = null, .main_pkg_path = null, @@ -1812,6 +1819,13 @@ pub const LibExeObjStep = struct { out.print("pub const {} = {};\n", .{ name, value }) catch unreachable; } + /// The value is the path in the cache dir. + /// Adds a dependency automatically. + pub fn addBuildOptionArtifact(self: *LibExeObjStep, name: []const u8, artifact: *LibExeObjStep) void { + self.build_options_artifact_args.append(.{ .name = name, .artifact = artifact }) catch unreachable; + self.step.dependOn(&artifact.step); + } + pub fn addSystemIncludeDir(self: *LibExeObjStep, path: []const u8) void { self.include_dirs.append(IncludeDir{ .RawPathSystem = self.builder.dupe(path) }) catch unreachable; } @@ -1995,7 +2009,15 @@ pub const LibExeObjStep = struct { } } - if (self.build_options_contents.items.len > 0) { + if (self.build_options_contents.items.len > 0 or self.build_options_artifact_args.items.len > 0) { + // Render build artifact options at the last minute, now that the path is known. + for (self.build_options_artifact_args.items) |item| { + const out = self.build_options_contents.writer(); + out.print("pub const {}: []const u8 = ", .{item.name}) catch unreachable; + std.zig.renderStringLiteral(item.artifact.getOutputPath(), out) catch unreachable; + out.writeAll(";\n") catch unreachable; + } + const build_options_file = try fs.path.join( builder.allocator, &[_][]const u8{ builder.cache_root, builder.fmt("{}_build_options.zig", .{self.name}) }, From f8b3543cabc28df15e85085a72adb474b345cf93 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 24 Sep 2020 22:57:03 -0700 Subject: [PATCH 133/210] I think this test is still flakey re-opens #4922 --- lib/std/event/loop.zig | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index c547f50365..13e704a8d3 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -112,8 +112,9 @@ pub const Loop = struct { /// have the correct pointer value. /// https://github.com/ziglang/zig/issues/2761 and https://github.com/ziglang/zig/issues/2765 pub fn init(self: *Loop) !void { - if (builtin.single_threaded - or (@hasDecl(root, "event_loop_mode") and root.event_loop_mode == .single_threaded)) { + if (builtin.single_threaded or + (@hasDecl(root, "event_loop_mode") and root.event_loop_mode == .single_threaded)) + { return self.initSingleThreaded(); } else { return self.initMultiThreaded(); @@ -1257,6 +1258,11 @@ test "std.event.Loop - basic" { // https://github.com/ziglang/zig/issues/1908 if (builtin.single_threaded) return error.SkipZigTest; + if (true) { + // https://github.com/ziglang/zig/issues/4922 + return error.SkipZigTest; + } + var loop: Loop = undefined; try loop.initMultiThreaded(); defer loop.deinit(); From 30dfdfdbd09570f97420413015eb8a1517382961 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 24 Sep 2020 23:50:15 -0700 Subject: [PATCH 134/210] fix a round of regressions in this branch * Don't try to generate C header files yet since it will only cause a crash saying the feature is unimplemented. * Rename the CLI options for release modes to use the `-O` prefix to match C compiler precedent. Options are now `-ODebug`, `-OReleaseFast`, `-OReleaseSafe`, `-OReleaseSmall`. The optimization mode matches the enum tags of std.builtin.Mode. It is planned to, at some point, rename std.builtin.Mode to std.builtin.OptimizationMode and modify the tags to be lower case to match the style convention. - Update build.zig code to support this new CLI. * update std.zig.binNameAlloc to support an optional Version and update the implementation to correctly deal with dynamic library version suffixes. --- lib/std/build.zig | 8 +--- lib/std/zig.zig | 47 ++++++++++++++--------- src/Compilation.zig | 6 ++- src/libcxx.zig | 14 ++++++- src/libunwind.zig | 10 +++-- src/main.zig | 92 ++++++++++++++++++++++----------------------- src/test.zig | 7 +++- 7 files changed, 107 insertions(+), 77 deletions(-) diff --git a/lib/std/build.zig b/lib/std/build.zig index 2c7a0da53a..3538629144 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -1692,8 +1692,6 @@ pub const LibExeObjStep = struct { self.main_pkg_path = dir_path; } - pub const setDisableGenH = @compileError("deprecated; set the emit_h field directly"); - pub fn setLibCFile(self: *LibExeObjStep, libc_file: ?[]const u8) void { self.libc_file = libc_file; } @@ -2067,10 +2065,8 @@ pub const LibExeObjStep = struct { } switch (self.build_mode) { - .Debug => {}, - .ReleaseSafe => zig_args.append("--release-safe") catch unreachable, - .ReleaseFast => zig_args.append("--release-fast") catch unreachable, - .ReleaseSmall => zig_args.append("--release-small") catch unreachable, + .Debug => {}, // Skip since it's the default. + else => zig_args.append(builder.fmt("-O{s}", .{@tagName(self.build_mode)})) catch unreachable, } try zig_args.append("--cache-dir"); diff --git a/lib/std/zig.zig b/lib/std/zig.zig index b6dfada6cc..fc173ecda9 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -64,17 +64,21 @@ pub fn lineDelta(source: []const u8, start: usize, end: usize) isize { return line; } -/// Returns the standard file system basename of a binary generated by the Zig compiler. -pub fn binNameAlloc( - allocator: *std.mem.Allocator, +pub const BinNameOptions = struct { root_name: []const u8, target: std.Target, output_mode: std.builtin.OutputMode, - link_mode: ?std.builtin.LinkMode, - object_format: ?std.Target.ObjectFormat, -) error{OutOfMemory}![]u8 { - switch (object_format orelse target.getObjectFormat()) { - .coff, .pe => switch (output_mode) { + link_mode: ?std.builtin.LinkMode = null, + object_format: ?std.Target.ObjectFormat = null, + version: ?std.builtin.Version = null, +}; + +/// Returns the standard file system basename of a binary generated by the Zig compiler. +pub fn binNameAlloc(allocator: *std.mem.Allocator, options: BinNameOptions) error{OutOfMemory}![]u8 { + const root_name = options.root_name; + const target = options.target; + switch (options.object_format orelse target.getObjectFormat()) { + .coff, .pe => switch (options.output_mode) { .Exe => { const suffix = switch (target.os.tag) { .uefi => ".efi", @@ -83,7 +87,7 @@ pub fn binNameAlloc( return std.fmt.allocPrint(allocator, "{}{}", .{ root_name, suffix }); }, .Lib => { - const suffix = switch (link_mode orelse .Static) { + const suffix = switch (options.link_mode orelse .Static) { .Static => ".lib", .Dynamic => ".dll", }; @@ -91,21 +95,30 @@ pub fn binNameAlloc( }, .Obj => return std.fmt.allocPrint(allocator, "{}.obj", .{root_name}), }, - .elf => switch (output_mode) { + .elf => switch (options.output_mode) { .Exe => return allocator.dupe(u8, root_name), .Lib => { - const suffix = switch (link_mode orelse .Static) { - .Static => ".a", - .Dynamic => ".so", - }; - return std.fmt.allocPrint(allocator, "{}{}{}", .{ target.libPrefix(), root_name, suffix }); + switch (options.link_mode orelse .Static) { + .Static => return std.fmt.allocPrint(allocator, "{}{}.a", .{ + target.libPrefix(), root_name, + }), + .Dynamic => { + if (options.version) |ver| { + return std.fmt.allocPrint(allocator, "{}{}.so.{}.{}.{}", .{ + target.libPrefix(), root_name, ver.major, ver.minor, ver.patch, + }); + } else { + return std.fmt.allocPrint(allocator, "{}{}.so", .{ target.libPrefix(), root_name }); + } + }, + } }, .Obj => return std.fmt.allocPrint(allocator, "{}.o", .{root_name}), }, - .macho => switch (output_mode) { + .macho => switch (options.output_mode) { .Exe => return allocator.dupe(u8, root_name), .Lib => { - const suffix = switch (link_mode orelse .Static) { + const suffix = switch (options.link_mode orelse .Static) { .Static => ".a", .Dynamic => ".dylib", }; diff --git a/src/Compilation.zig b/src/Compilation.zig index 03da22d80c..bdd11abc20 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2562,7 +2562,11 @@ pub fn build_crt_file( defer tracy.end(); const target = comp.getTarget(); - const basename = try std.zig.binNameAlloc(comp.gpa, root_name, target, output_mode, null, null); + const basename = try std.zig.binNameAlloc(comp.gpa, .{ + .root_name = root_name, + .target = target, + .output_mode = output_mode, + }); errdefer comp.gpa.free(basename); // TODO: This is extracted into a local variable to work around a stage1 miscompilation. diff --git a/src/libcxx.zig b/src/libcxx.zig index 4fbca42875..16c9581f82 100644 --- a/src/libcxx.zig +++ b/src/libcxx.zig @@ -88,7 +88,12 @@ pub fn buildLibCXX(comp: *Compilation) !void { const output_mode = .Lib; const link_mode = .Static; const target = comp.getTarget(); - const basename = try std.zig.binNameAlloc(arena, root_name, target, output_mode, link_mode, null); + const basename = try std.zig.binNameAlloc(arena, .{ + .root_name = root_name, + .target = target, + .output_mode = output_mode, + .link_mode = link_mode, + }); const emit_bin = Compilation.EmitLoc{ .directory = null, // Put it in the cache directory. @@ -205,7 +210,12 @@ pub fn buildLibCXXABI(comp: *Compilation) !void { const output_mode = .Lib; const link_mode = .Static; const target = comp.getTarget(); - const basename = try std.zig.binNameAlloc(arena, root_name, target, output_mode, link_mode, null); + const basename = try std.zig.binNameAlloc(arena, .{ + .root_name = root_name, + .target = target, + .output_mode = output_mode, + .link_mode = link_mode, + }); const emit_bin = Compilation.EmitLoc{ .directory = null, // Put it in the cache directory. diff --git a/src/libunwind.zig b/src/libunwind.zig index 28445b7284..cbe632ba94 100644 --- a/src/libunwind.zig +++ b/src/libunwind.zig @@ -23,13 +23,16 @@ pub fn buildStaticLib(comp: *Compilation) !void { const output_mode = .Lib; const link_mode = .Static; const target = comp.getTarget(); - const basename = try std.zig.binNameAlloc(arena, root_name, target, output_mode, link_mode, null); - + const basename = try std.zig.binNameAlloc(arena, .{ + .root_name = root_name, + .target = target, + .output_mode = output_mode, + .link_mode = link_mode, + }); const emit_bin = Compilation.EmitLoc{ .directory = null, // Put it in the cache directory. .basename = basename, }; - const unwind_src_list = [_][]const u8{ "libunwind" ++ path.sep_str ++ "src" ++ path.sep_str ++ "libunwind.cpp", "libunwind" ++ path.sep_str ++ "src" ++ path.sep_str ++ "Unwind-EHABI.cpp", @@ -40,7 +43,6 @@ pub fn buildStaticLib(comp: *Compilation) !void { "libunwind" ++ path.sep_str ++ "src" ++ path.sep_str ++ "UnwindRegistersRestore.S", "libunwind" ++ path.sep_str ++ "src" ++ path.sep_str ++ "UnwindRegistersSave.S", }; - var c_source_files: [unwind_src_list.len]Compilation.CSourceFile = undefined; for (unwind_src_list) |unwind_src, i| { var cflags = std.ArrayList([]const u8).init(arena); diff --git a/src/main.zig b/src/main.zig index be2c55f828..fe4c27ced6 100644 --- a/src/main.zig +++ b/src/main.zig @@ -210,11 +210,10 @@ const usage_build_generic = \\ small|kernel| \\ medium|large] \\ --name [name] Override root name (not a file path) - \\ --mode [mode] Set the build mode - \\ Debug (default) optimizations off, safety on - \\ ReleaseFast Optimizations on, safety off - \\ ReleaseSafe Optimizations on, safety on - \\ ReleaseSmall Optimize for small binary, safety off + \\ -ODebug (default) optimizations off, safety on + \\ -OReleaseFast Optimizations on, safety off + \\ -OReleaseSafe Optimizations on, safety on + \\ -OReleaseSmall Optimize for small binary, safety off \\ --pkg-begin [name] [path] Make pkg available to import and push current pkg \\ --pkg-end Pop current pkg \\ --main-pkg-path Set the directory of the root package @@ -307,7 +306,7 @@ pub fn buildOutputType( }, ) !void { var color: Color = .Auto; - var build_mode: std.builtin.Mode = .Debug; + var optimize_mode: std.builtin.Mode = .Debug; var provided_name: ?[]const u8 = null; var link_mode: ?std.builtin.LinkMode = null; var dll_export_fns: ?bool = null; @@ -416,20 +415,23 @@ pub fn buildOutputType( switch (arg_mode) { .build, .translate_c, .zig_test, .run => { + var optimize_mode_string: ?[]const u8 = null; output_mode = switch (arg_mode) { .build => |m| m, .translate_c => .Obj, .zig_test, .run => .Exe, else => unreachable, }; - switch (arg_mode) { - .build => switch (output_mode) { - .Exe => emit_h = .no, - .Obj, .Lib => emit_h = .yes_default_path, - }, - .translate_c, .zig_test, .run => emit_h = .no, - else => unreachable, - } + // TODO finish self-hosted and add support for emitting C header files + emit_h = .no; + //switch (arg_mode) { + // .build => switch (output_mode) { + // .Exe => emit_h = .no, + // .Obj, .Lib => emit_h = .yes_default_path, + // }, + // .translate_c, .zig_test, .run => emit_h = .no, + // else => unreachable, + //} const args = all_args[2..]; var i: usize = 0; while (i < args.len) : (i += 1) { @@ -498,23 +500,10 @@ pub fn buildOutputType( } else { fatal("expected [auto|on|off] after --color, found '{}'", .{next_arg}); } - } else if (mem.eql(u8, arg, "--mode")) { - if (i + 1 >= args.len) { - fatal("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode", .{}); - } + } else if (mem.eql(u8, arg, "-O")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; - const next_arg = args[i]; - if (mem.eql(u8, next_arg, "Debug")) { - build_mode = .Debug; - } else if (mem.eql(u8, next_arg, "ReleaseSafe")) { - build_mode = .ReleaseSafe; - } else if (mem.eql(u8, next_arg, "ReleaseFast")) { - build_mode = .ReleaseFast; - } else if (mem.eql(u8, next_arg, "ReleaseSmall")) { - build_mode = .ReleaseSmall; - } else { - fatal("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode, found '{}'", .{next_arg}); - } + optimize_mode_string = args[i]; } else if (mem.eql(u8, arg, "--stack")) { if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; @@ -583,6 +572,8 @@ pub fn buildOutputType( target_mcpu = arg["-mcpu=".len..]; } else if (mem.startsWith(u8, arg, "-mcmodel=")) { machine_code_model = parseCodeModel(arg["-mcmodel=".len..]); + } else if (mem.startsWith(u8, arg, "-O")) { + optimize_mode_string = arg["-O".len..]; } else if (mem.eql(u8, arg, "--dynamic-linker")) { if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; @@ -749,6 +740,10 @@ pub fn buildOutputType( }, } } + if (optimize_mode_string) |s| { + optimize_mode = std.meta.stringToEnum(std.builtin.Mode, s) orelse + fatal("unrecognized optimization mode: '{}'", .{s}); + } }, .cc, .cpp => { emit_h = .no; @@ -826,16 +821,16 @@ pub fn buildOutputType( .optimize => { // Alright, what release mode do they want? if (mem.eql(u8, it.only_arg, "Os")) { - build_mode = .ReleaseSmall; + optimize_mode = .ReleaseSmall; } else if (mem.eql(u8, it.only_arg, "O2") or mem.eql(u8, it.only_arg, "O3") or mem.eql(u8, it.only_arg, "O4")) { - build_mode = .ReleaseFast; + optimize_mode = .ReleaseFast; } else if (mem.eql(u8, it.only_arg, "Og") or mem.eql(u8, it.only_arg, "O0")) { - build_mode = .Debug; + optimize_mode = .Debug; } else { try clang_argv.appendSlice(it.other_args); } @@ -999,8 +994,8 @@ pub fn buildOutputType( } if (want_sanitize_c) |wsc| { - if (wsc and build_mode == .ReleaseFast) { - build_mode = .ReleaseSafe; + if (wsc and optimize_mode == .ReleaseFast) { + optimize_mode = .ReleaseSafe; } } @@ -1177,6 +1172,7 @@ pub fn buildOutputType( defer if (cleanup_emit_bin_dir) |*dir| dir.close(); const have_enable_cache = enable_cache orelse false; + const optional_version = if (have_version) version else null; const emit_bin_loc: ?Compilation.EmitLoc = switch (emit_bin) { .no => null, @@ -1193,14 +1189,14 @@ pub fn buildOutputType( }, } }, - .basename = try std.zig.binNameAlloc( - arena, - root_name, - target_info.target, - output_mode, - link_mode, - object_format, - ), + .basename = try std.zig.binNameAlloc(arena, .{ + .root_name = root_name, + .target = target_info.target, + .output_mode = output_mode, + .link_mode = link_mode, + .object_format = object_format, + .version = optional_version, + }), }, .yes => |full_path| b: { const basename = fs.path.basename(full_path); @@ -1374,7 +1370,7 @@ pub fn buildOutputType( .link_mode = link_mode, .dll_export_fns = dll_export_fns, .object_format = object_format, - .optimize_mode = build_mode, + .optimize_mode = optimize_mode, .keep_source_files_loaded = zir_out_path != null, .clang_argv = clang_argv.items, .lld_argv = lld_argv.items, @@ -1411,7 +1407,7 @@ pub fn buildOutputType( .self_exe_path = self_exe_path, .rand = &default_prng.random, .clang_passthrough_mode = arg_mode != .build, - .version = if (have_version) version else null, + .version = optional_version, .libc_installation = if (libc_installation) |*lci| lci else null, .verbose_cc = verbose_cc, .verbose_link = verbose_link, @@ -1977,7 +1973,11 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v const cross_target: std.zig.CrossTarget = .{}; const target_info = try detectNativeTargetInfo(gpa, cross_target); - const exe_basename = try std.zig.binNameAlloc(arena, "build", target_info.target, .Exe, null, null); + const exe_basename = try std.zig.binNameAlloc(arena, .{ + .root_name = "build", + .target = target_info.target, + .output_mode = .Exe, + }); const emit_bin: Compilation.EmitLoc = .{ .directory = null, // Use the local zig-cache. .basename = exe_basename, diff --git a/src/test.zig b/src/test.zig index 558fe3e95d..8ad11efa9c 100644 --- a/src/test.zig +++ b/src/test.zig @@ -469,7 +469,12 @@ pub const TestContext = struct { }; const ofmt: ?std.builtin.ObjectFormat = if (case.cbe) .c else null; - const bin_name = try std.zig.binNameAlloc(arena, "test_case", target, case.output_mode, null, ofmt); + const bin_name = try std.zig.binNameAlloc(arena, .{ + .root_name = "test_case", + .target = target, + .output_mode = case.output_mode, + .object_format = ofmt, + }); const emit_directory: Compilation.Directory = .{ .path = bogus_path, From 495d18a2056882d7edc7376751e7f3d4e10ef920 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 25 Sep 2020 00:00:04 -0700 Subject: [PATCH 135/210] std.log: better default for printing logs * prefix with the message level * if the scope is not default, also prefix with the scope This makes the stack trace test pass, with no changes to the test case, because errors returned from main() now print `error: Foo` just like they do in master branch. --- lib/std/log.zig | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/std/log.zig b/lib/std/log.zig index 7b677f698a..bc83e6053d 100644 --- a/lib/std/log.zig +++ b/lib/std/log.zig @@ -132,10 +132,21 @@ fn log( // any I/O configured. return; } else if (builtin.mode != .ReleaseSmall) { + const level_txt = switch (message_level) { + .emerg => "emergency", + .alert => "alert", + .crit => "critical", + .err => "error", + .warn => "warning", + .notice => "notice", + .info => "info", + .debug => "debug", + }; + const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; + const stderr = std.io.getStdErr().writer(); const held = std.debug.getStderrMutex().acquire(); defer held.release(); - const stderr = std.io.getStdErr().writer(); - nosuspend stderr.print(format ++ "\n", args) catch return; + nosuspend stderr.print(level_txt ++ prefix2 ++ format ++ "\n", args) catch return; } } } From 93291cc4722d51afbba7378fab5cfb25da175a79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20=28xq=29=20Quei=C3=9Fner?= Date: Fri, 25 Sep 2020 09:16:43 +0200 Subject: [PATCH 136/210] Implements std.meta.ArgsTuple. --- lib/std/meta.zig | 74 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/lib/std/meta.zig b/lib/std/meta.zig index 1507aa9de8..98f3b284c1 100644 --- a/lib/std/meta.zig +++ b/lib/std/meta.zig @@ -807,7 +807,7 @@ pub fn sizeof(target: anytype) usize { // TODO to get the correct result we have to translate // `1073741824 * 4` as `int(1073741824) *% int(4)` since // sizeof(1073741824 * 4) != sizeof(4294967296). - + // TODO test if target fits in int, long or long long return @sizeOf(c_int); }, @@ -826,3 +826,75 @@ test "sizeof" { testing.expect(sizeof(E.One) == @sizeOf(c_int)); testing.expect(sizeof(S) == 4); } + +/// For a given function type, returns a tuple type which fields will +/// correspond to the argument types. +/// +/// Examples: +/// - `ArgsTuple(fn() void)` ⇒ `tuple { }` +/// - `ArgsTuple(fn(a: u32) u32)` ⇒ `tuple { u32 }` +/// - `ArgsTuple(fn(a: u32, b: f16) noreturn)` ⇒ `tuple { u32, f16 }` +pub fn ArgsTuple(comptime Function: type) type { + const info = @typeInfo(Function); + if (info != .Fn) + @compileError("ArgsTuple expects a function type"); + + const function_info = info.Fn; + if (function_info.is_generic) + @compileError("Cannot create ArgsTuple for generic function"); + if (function_info.is_var_args) + @compileError("Cannot create ArgsTuple for variadic function"); + + var argument_field_list: [function_info.args.len]std.builtin.TypeInfo.StructField = undefined; + inline for (function_info.args) |arg, i| { + @setEvalBranchQuota(10_000); + var num_buf: [128]u8 = undefined; + argument_field_list[i] = std.builtin.TypeInfo.StructField{ + .name = std.fmt.bufPrint(&num_buf, "{d}", .{i}) catch unreachable, + .field_type = arg.arg_type.?, + .default_value = @as(?(arg.arg_type.?), null), + .is_comptime = false, + }; + } + + return @Type(std.builtin.TypeInfo{ + .Struct = std.builtin.TypeInfo.Struct{ + .is_tuple = true, + .layout = .Auto, + .decls = &[_]std.builtin.TypeInfo.Declaration{}, + .fields = &argument_field_list, + }, + }); +} + +comptime { + const T = struct { + fn assertTypeEqual(comptime Expected: type, comptime Actual: type) void { + if (Expected != Actual) + @compileError("Expected type " ++ @typeName(Expected) ++ ", but got type " ++ @typeName(Actual)); + } + + fn assertTuple(comptime expected: anytype, comptime Actual: type) void { + const info = @typeInfo(Actual); + if (info != .Struct) + @compileError("Expected struct type"); + if (!info.Struct.is_tuple) + @compileError("Struct type must be a tuple type"); + + const fields_list = std.meta.fields(Actual); + if (expected.len != fields_list.len) + @compileError("Argument count mismatch"); + + inline for (fields_list) |fld, i| { + if (expected[i] != fld.field_type) { + @compileError("Field " ++ fld.name ++ " expected to be type " ++ @typeName(expected[i]) ++ ", but was type " ++ @typeName(fld.field_type)); + } + } + } + }; + + T.assertTuple(.{}, ArgsTuple(fn () void)); + T.assertTuple(.{u32}, ArgsTuple(fn (a: u32) []const u8)); + T.assertTuple(.{ u32, f16 }, ArgsTuple(fn (a: u32, b: f16) noreturn)); + T.assertTuple(.{ u32, f16, []const u8 }, ArgsTuple(fn (a: u32, b: f16, c: []const u8) noreturn)); +} From 7f68b14377ffe33ded2866f6bf53f213a9c5f620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20=28xq=29=20Quei=C3=9Fner?= Date: Fri, 25 Sep 2020 09:27:00 +0200 Subject: [PATCH 137/210] Implements std.meta.Tuple(), implements #4607 in userland. --- lib/std/meta.zig | 64 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/lib/std/meta.zig b/lib/std/meta.zig index 1507aa9de8..c92637250f 100644 --- a/lib/std/meta.zig +++ b/lib/std/meta.zig @@ -807,7 +807,7 @@ pub fn sizeof(target: anytype) usize { // TODO to get the correct result we have to translate // `1073741824 * 4` as `int(1073741824) *% int(4)` since // sizeof(1073741824 * 4) != sizeof(4294967296). - + // TODO test if target fits in int, long or long long return @sizeOf(c_int); }, @@ -826,3 +826,65 @@ test "sizeof" { testing.expect(sizeof(E.One) == @sizeOf(c_int)); testing.expect(sizeof(S) == 4); } + +/// For a given anonymous list of types, returns a new tuple type +/// with those types as fields. +/// +/// Examples: +/// - `Tuple(.{})` ⇒ `tuple { }` +/// - `Tuple(.{f32})` ⇒ `tuple { f32 }` +/// - `Tuple(.{f32,u32})` ⇒ `tuple { f32, u32 }` +pub fn Tuple(comptime types: anytype) type { + var tuple_fields: [types.len]std.builtin.TypeInfo.StructField = undefined; + inline for (types) |T, i| { + @setEvalBranchQuota(10_000); + var num_buf: [128]u8 = undefined; + tuple_fields[i] = std.builtin.TypeInfo.StructField{ + .name = std.fmt.bufPrint(&num_buf, "{d}", .{i}) catch unreachable, + .field_type = T, + .default_value = @as(?T, null), + .is_comptime = false, + }; + } + + return @Type(std.builtin.TypeInfo{ + .Struct = std.builtin.TypeInfo.Struct{ + .is_tuple = true, + .layout = .Auto, + .decls = &[_]std.builtin.TypeInfo.Declaration{}, + .fields = &tuple_fields, + }, + }); +} + +comptime { + const T = struct { + fn assertTypeEqual(comptime Expected: type, comptime Actual: type) void { + if (Expected != Actual) + @compileError("Expected type " ++ @typeName(Expected) ++ ", but got type " ++ @typeName(Actual)); + } + + fn assertTuple(comptime expected: anytype, comptime Actual: type) void { + const info = @typeInfo(Actual); + if (info != .Struct) + @compileError("Expected struct type"); + if (!info.Struct.is_tuple) + @compileError("Struct type must be a tuple type"); + + const fields_list = std.meta.fields(Actual); + if (expected.len != fields_list.len) + @compileError("Argument count mismatch"); + + inline for (fields_list) |fld, i| { + if (expected[i] != fld.field_type) { + @compileError("Field " ++ fld.name ++ " expected to be type " ++ @typeName(expected[i]) ++ ", but was type " ++ @typeName(fld.field_type)); + } + } + } + }; + + T.assertTuple(.{}, Tuple(.{})); + T.assertTuple(.{u32}, Tuple(.{u32})); + T.assertTuple(.{ u32, f16 }, Tuple(.{ u32, f16 })); + T.assertTuple(.{ u32, f16, []const u8 }, Tuple(.{ u32, f16, []const u8 })); +} From bd9003ed5b16a6e187999fb1190d89eb80bd587b Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 25 Sep 2020 00:21:57 +0200 Subject: [PATCH 138/210] std: ArenaAllocator tries to resize before allocating Closes #5116 --- lib/std/heap.zig | 7 +++++++ lib/std/heap/arena_allocator.zig | 21 +++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 16de215cc2..cf32cff645 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -919,6 +919,13 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void { const zero_bit_ptr = try allocator.create(u0); zero_bit_ptr.* = 0; allocator.destroy(zero_bit_ptr); + + const oversize = try allocator.allocAdvanced(u32, null, 5, .at_least); + testing.expect(oversize.len >= 5); + for (oversize) |*item| { + item.* = 0xDEADBEEF; + } + allocator.free(oversize); } pub fn testAllocatorAligned(base_allocator: *mem.Allocator, comptime alignment: u29) !void { diff --git a/lib/std/heap/arena_allocator.zig b/lib/std/heap/arena_allocator.zig index 0737cb2ef8..b7ee1d54c1 100644 --- a/lib/std/heap/arena_allocator.zig +++ b/lib/std/heap/arena_allocator.zig @@ -75,13 +75,22 @@ pub const ArenaAllocator = struct { const adjusted_addr = mem.alignForward(addr, ptr_align); const adjusted_index = self.state.end_index + (adjusted_addr - addr); const new_end_index = adjusted_index + n; - if (new_end_index > cur_buf.len) { - cur_node = try self.createNode(cur_buf.len, n + ptr_align); - continue; + + if (new_end_index <= cur_buf.len) { + const result = cur_buf[adjusted_index..new_end_index]; + self.state.end_index = new_end_index; + return result; } - const result = cur_buf[adjusted_index..new_end_index]; - self.state.end_index = new_end_index; - return result; + + const bigger_buf_size = @sizeOf(BufNode) + new_end_index; + // Try to grow the buffer in-place + cur_node.data = self.child_allocator.resize(cur_node.data, bigger_buf_size) catch |err| switch (err) { + error.OutOfMemory => { + // Allocate a new node if that's not possible + cur_node = try self.createNode(cur_buf.len, n + ptr_align); + continue; + }, + }; } } From 8d01133bd04423f896dee81b89c56372aa1ee310 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 25 Sep 2020 18:42:24 +0200 Subject: [PATCH 139/210] update doc comments Signed-off-by: Loris Cro --- lib/std/os.zig | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 2c5f3065b2..c06ce4ed00 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -314,8 +314,8 @@ pub const ReadError = error{ /// Returns the number of bytes that were read, which can be less than /// buf.len. If 0 bytes were read, that means EOF. -/// If the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in error.WouldBlock. +/// If `fd` is opened in non blocking mode, the function will return error.WouldBlock +/// when EAGAIN is received. /// /// Linux has a limit on how many bytes may be transferred in one `read` call, which is `0x7ffff000` /// on both 64-bit and 32-bit systems. This is due to using a signed C int as the return value, as @@ -382,8 +382,8 @@ pub fn read(fd: fd_t, buf: []u8) ReadError!usize { /// Number of bytes read is returned. Upon reading end-of-file, zero is returned. /// -/// For POSIX systems, if the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// For POSIX systems, if `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. /// On Windows, if the application has a global event loop enabled, I/O Completion Ports are /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. /// @@ -440,8 +440,8 @@ pub const PReadError = ReadError || error{Unseekable}; /// /// Retries when interrupted by a signal. /// -/// For POSIX systems, if the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// For POSIX systems, if `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. /// On Windows, if the application has a global event loop enabled, I/O Completion Ports are /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. pub fn pread(fd: fd_t, buf: []u8, offset: u64) PReadError!usize { @@ -571,8 +571,8 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void { /// /// Retries when interrupted by a signal. /// -/// For POSIX systems, if the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// For POSIX systems, if `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. /// On Windows, if the application has a global event loop enabled, I/O Completion Ports are /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. /// @@ -667,8 +667,8 @@ pub const WriteError = error{ /// another write() call to transfer the remaining bytes. The subsequent call will either /// transfer further bytes or may result in an error (e.g., if the disk is now full). /// -/// For POSIX systems, if the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// For POSIX systems, if `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. /// On Windows, if the application has a global event loop enabled, I/O Completion Ports are /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. /// @@ -747,8 +747,8 @@ pub fn write(fd: fd_t, bytes: []const u8) WriteError!usize { /// another write() call to transfer the remaining bytes. The subsequent call will either /// transfer further bytes or may result in an error (e.g., if the disk is now full). /// -/// For POSIX systems, if the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// For POSIX systems, if `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received.k`. /// On Windows, if the application has a global event loop enabled, I/O Completion Ports are /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. /// @@ -817,8 +817,8 @@ pub const PWriteError = WriteError || error{Unseekable}; /// another write() call to transfer the remaining bytes. The subsequent call will either /// transfer further bytes or may result in an error (e.g., if the disk is now full). /// -/// For POSIX systems, if the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// For POSIX systems, if `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. /// On Windows, if the application has a global event loop enabled, I/O Completion Ports are /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. /// @@ -904,8 +904,8 @@ pub fn pwrite(fd: fd_t, bytes: []const u8, offset: u64) PWriteError!usize { /// another write() call to transfer the remaining bytes. The subsequent call will either /// transfer further bytes or may result in an error (e.g., if the disk is now full). /// -/// If the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// If `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. /// /// The following systems do not have this syscall, and will return partial writes if more than one /// vector is provided: @@ -2806,8 +2806,8 @@ pub const AcceptError = error{ } || UnexpectedError; /// Accept a connection on a socket. -/// If the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in error.WouldBlock. +/// If `sockfd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. pub fn accept( /// This argument is a socket that has been created with `socket`, bound to a local address /// with `bind`, and is listening for connections after a `listen`. @@ -3036,6 +3036,8 @@ pub const ConnectError = error{ } || UnexpectedError; /// Initiate a connection on a socket. +/// If `sockfd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN or EINPROGRESS is received. pub fn connect(sockfd: socket_t, sock_addr: *const sockaddr, len: socklen_t) ConnectError!void { if (builtin.os.tag == .windows) { const rc = windows.ws2_32.connect(sockfd, sock_addr, len); @@ -5051,6 +5053,8 @@ pub const RecvFromError = error{ SystemResources, } || UnexpectedError; +/// If `sockfd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. pub fn recvfrom( sockfd: fd_t, buf: []u8, From 812e482bef82450eb1010344a59c497feb12ee67 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 25 Sep 2020 13:37:10 -0700 Subject: [PATCH 140/210] ELF linking supports root src directory to be cwd --- src/link/Elf.zig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 8035adcf0d..931c1bc9c0 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -855,7 +855,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation) !void { } // Write the form for the compile unit, which must match the abbrev table above. const name_strp = try self.makeDebugString(module.root_pkg.root_src_path); - const comp_dir_strp = try self.makeDebugString(module.root_pkg.root_src_directory.path.?); + const comp_dir_strp = try self.makeDebugString(module.root_pkg.root_src_directory.path orelse "."); const producer_strp = try self.makeDebugString(link.producer_string); // Currently only one compilation unit is supported, so the address range is simply // identical to the main program header virtual address and memory size. @@ -2858,11 +2858,12 @@ fn dbgLineNeededHeaderBytes(self: Elf) u32 { const file_name_entry_format_count = 1; const directory_count = 1; const file_name_count = 1; + const root_src_dir_path_len = if (self.base.options.module.?.root_pkg.root_src_directory.path) |p| p.len else 1; // "." return @intCast(u32, 53 + directory_entry_format_count * 2 + file_name_entry_format_count * 2 + directory_count * 8 + file_name_count * 8 + // These are encoded as DW.FORM_string rather than DW.FORM_strp as we would like // because of a workaround for readelf and gdb failing to understand DWARFv5 correctly. - self.base.options.module.?.root_pkg.root_src_directory.path.?.len + + root_src_dir_path_len + self.base.options.module.?.root_pkg.root_src_path.len); } From dc01ef738828a4eba08e95eaaf89442ca2f3e2f8 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 25 Sep 2020 23:21:20 +0200 Subject: [PATCH 141/210] Remove noop check Co-authored-by: Andrew Kelley --- lib/std/fs/file.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index f3e980b9f0..8d4f5df2e8 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -416,7 +416,7 @@ pub const File = struct { return windows.ReadFile(self.handle, buffer, null, self.intended_io_mode); } - if (self.intended_io_mode == .blocking or !std.io.is_async) { + if (self.intended_io_mode == .blocking) { return os.read(self.handle, buffer); } else { return std.event.Loop.instance.?.read(self.handle, buffer, self.capable_io_mode != self.intended_io_mode); From 670e7d456c24a8597af6b3809bf1e9ea68746ade Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 25 Sep 2020 14:47:09 -0700 Subject: [PATCH 142/210] add some tracy calls --- src/Compilation.zig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Compilation.zig b/src/Compilation.zig index bdd11abc20..a522a4173c 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2077,6 +2077,9 @@ fn wantBuildLibUnwindFromSource(comp: *Compilation) bool { } fn updateBuiltinZigFile(comp: *Compilation, mod: *Module) !void { + const tracy = trace(@src()); + defer tracy.end(); + const source = try comp.generateBuiltinZigSource(comp.gpa); defer comp.gpa.free(source); try mod.zig_cache_artifact_directory.handle.writeFile("builtin.zig", source); @@ -2090,6 +2093,9 @@ pub fn dump_argv(argv: []const []const u8) void { } pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8 { + const tracy = trace(@src()); + defer tracy.end(); + var buffer = std.ArrayList(u8).init(allocator); defer buffer.deinit(); From f78652484a2bf5ee967d0ed7ca1db495932fad48 Mon Sep 17 00:00:00 2001 From: Suirad Date: Tue, 22 Sep 2020 20:05:12 -0500 Subject: [PATCH 143/210] Stdlib fix for os.windows.deleteFile to fail with a proper error when attempting to delete a directory that isnt empty --- lib/std/fs/test.zig | 23 +++++++++++++++++++++++ lib/std/os/windows.zig | 16 +++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index b3cc1fe569..8d7ef5172e 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -813,3 +813,26 @@ fn run_lock_file_test(contexts: []FileLockTestContext) !void { try threads.append(try std.Thread.spawn(ctx, FileLockTestContext.run)); } } + +test "deleteDir" { + var tmp_dir = tmpDir(.{}); + defer tmp_dir.cleanup(); + + // deleting a non-existent directory + testing.expectError(error.FileNotFound, tmp_dir.dir.deleteDir("test_dir")); + + var dir = try tmp_dir.dir.makeOpenPath("test_dir", .{}); + var file = try dir.createFile("test_file", .{}); + file.close(); + dir.close(); + + // deleting a non-empty directory + testing.expectError(error.DirNotEmpty, tmp_dir.dir.deleteDir("test_dir")); + + dir = try tmp_dir.dir.openDir("test_dir", .{}); + try dir.deleteFile("test_file"); + dir.close(); + + // deleting an empty directory + try tmp_dir.dir.deleteDir("test_dir"); +} diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index de0d0ea45f..c4037ccf0a 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -764,6 +764,7 @@ pub const DeleteFileError = error{ Unexpected, NotDir, IsDir, + DirNotEmpty, }; pub const DeleteFileOptions = struct { @@ -818,7 +819,7 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil 0, ); switch (rc) { - .SUCCESS => return CloseHandle(tmp_handle), + .SUCCESS => CloseHandle(tmp_handle), .OBJECT_NAME_INVALID => unreachable, .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, .INVALID_PARAMETER => unreachable, @@ -826,6 +827,19 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil .NOT_A_DIRECTORY => return error.NotDir, else => return unexpectedStatus(rc), } + + if (options.remove_dir){ + var basic_info: FILE_BASIC_INFORMATION = undefined; + switch (ntdll.NtQueryAttributesFile(&attr, &basic_info)) { + .SUCCESS => return error.DirNotEmpty, + .OBJECT_NAME_NOT_FOUND => return, + .OBJECT_PATH_NOT_FOUND => return, + .INVALID_PARAMETER => unreachable, + .ACCESS_DENIED => return error.AccessDenied, + .OBJECT_PATH_SYNTAX_BAD => unreachable, + else => |urc| return unexpectedStatus(urc), + } + } } pub const MoveFileError = error{ FileNotFound, Unexpected }; From 43cd9eb110f6803f4e19d92347ebf263e6e644af Mon Sep 17 00:00:00 2001 From: Suirad Date: Fri, 25 Sep 2020 18:11:31 -0500 Subject: [PATCH 144/210] Add clarification comment --- lib/std/os/windows.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index c4037ccf0a..2aa222414f 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -828,7 +828,9 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil else => return unexpectedStatus(rc), } - if (options.remove_dir){ + // If a directory fails to be deleted, CloseHandle will still report success + // Check if the directory still exists and return error.DirNotEmpty if true + if (options.remove_dir) { var basic_info: FILE_BASIC_INFORMATION = undefined; switch (ntdll.NtQueryAttributesFile(&attr, &basic_info)) { .SUCCESS => return error.DirNotEmpty, From 70d7d7e919d7f297e63ca421f6be5925259136e2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 25 Sep 2020 18:01:35 -0700 Subject: [PATCH 145/210] stage2: disable lld caching when output dir is owned by user Normally when using LLD to link, Zig uses a file named "lld.id" in the same directory as the output binary which contains the hash of the link operation, allowing Zig to skip linking when the hash would be unchanged. In the case that the output binary is being emitted into a directory which is externally modified - essentially anything other than zig-cache - then this flag would be set to disable this machinery to avoid false positives. * Better defaults when using -fno-LLVM * Fix compiler_rt and libc static libraries were getting a .zig extension instead of .a extension. * when using the stage1 backend, put the object file next to the stage1.id file in the cache directory. this prevents an object file from polluting the cwd when using zig from the CLI. --- BRANCH_TODO | 11 ++- src/Compilation.zig | 84 ++++++++++++++--------- src/link.zig | 1 + src/link/Elf.zig | 164 +++++++++++++++++++++++--------------------- src/main.zig | 1 + 5 files changed, 145 insertions(+), 116 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 8a00c9fded..384f0761c3 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,7 +1,6 @@ + * make sure that `zig cc -o hello hello.c -target native-native-musl` and `zig build-exe hello.zig -lc -target native-native-musl` will share the same libc build. + * zig cc as a preprocessor (-E) * tests passing with -Dskip-non-native - * make sure zig cc works - - using it as a preprocessor (-E) - - try building some software * `-ftime-report` * -fstack-report print stack size diagnostics\n" * -fdump-analysis write analysis.json file with type information\n" @@ -15,14 +14,12 @@ * MachO LLD linking * COFF LLD linking * WASM LLD linking - * skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd) - (maybe make it an explicit option and have main.zig disable it) - - make sure that `zig cc -o hello hello.c -target native-native-musl` and `zig build-exe hello.zig -lc -target native-native-musl` will share the same libc build. * audit the CLI options for stage2 * audit the base cache hash * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process. * restore error messages for stage2_add_link_lib * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj] + * try building some software with zig cc * implement proper parsing of clang stderr/stdout and exposing compile errors with the Compilation API * implement proper parsing of LLD stderr/stdout and exposing compile errors with the Compilation API @@ -59,3 +56,5 @@ * close the --pkg-begin --pkg-end Package directory handles * make std.Progress support multithreaded * update musl.zig static data to use native path separator in static data rather than replacing '/' at runtime + * linking hello world with LLD, lld is silently calling exit(1) instead of reporting ok=false. when run standalone the error message is: ld.lld: error: section [index 3] has a sh_offset (0x57000) + sh_size (0x68) that is greater than the file size (0x57060) + diff --git a/src/Compilation.zig b/src/Compilation.zig index a522a4173c..c28b27b588 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -3,10 +3,11 @@ const Compilation = @This(); const std = @import("std"); const mem = std.mem; const Allocator = std.mem.Allocator; -const Value = @import("value.zig").Value; const assert = std.debug.assert; const log = std.log.scoped(.compilation); const Target = std.Target; + +const Value = @import("value.zig").Value; const target_util = @import("target.zig"); const Package = @import("Package.zig"); const link = @import("link.zig"); @@ -286,6 +287,13 @@ pub const InitOptions = struct { emit_h: ?EmitLoc = null, link_mode: ?std.builtin.LinkMode = null, dll_export_fns: ?bool = false, + /// Normally when using LLD to link, Zig uses a file named "lld.id" in the + /// same directory as the output binary which contains the hash of the link + /// operation, allowing Zig to skip linking when the hash would be unchanged. + /// In the case that the output binary is being emitted into a directory which + /// is externally modified - essentially anything other than zig-cache - then + /// this flag would be set to disable this machinery to avoid false positives. + disable_lld_caching: bool = false, object_format: ?std.builtin.ObjectFormat = null, optimize_mode: std.builtin.Mode = .Debug, keep_source_files_loaded: bool = false, @@ -371,6 +379,26 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { const ofmt = options.object_format orelse options.target.getObjectFormat(); + // Make a decision on whether to use LLVM or our own backend. + const use_llvm = if (options.use_llvm) |explicit| explicit else blk: { + // If we have no zig code to compile, no need for LLVM. + if (options.root_pkg == null) + break :blk false; + + // If we are the stage1 compiler, we depend on the stage1 c++ llvm backend + // to compile zig code. + if (build_options.is_stage1) + break :blk true; + + // We would want to prefer LLVM for release builds when it is available, however + // we don't have an LLVM backend yet :) + // We would also want to prefer LLVM for architectures that we don't have self-hosted support for too. + break :blk false; + }; + if (!use_llvm and options.machine_code_model != .default) { + return error.MachineCodeModelNotSupported; + } + // Make a decision on whether to use LLD or our own linker. const use_lld = if (options.use_lld) |explicit| explicit else blk: { if (!build_options.have_llvm) @@ -393,7 +421,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { break :blk true; } - if (build_options.is_stage1) { + if (use_llvm) { // If stage1 generates an object file, self-hosted linker is not // yet sophisticated enough to handle that. break :blk options.root_pkg != null; @@ -402,25 +430,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { break :blk false; }; - // Make a decision on whether to use LLVM or our own backend. - const use_llvm = if (options.use_llvm) |explicit| explicit else blk: { - // If we have no zig code to compile, no need for LLVM. - if (options.root_pkg == null) - break :blk false; - - // If we are the stage1 compiler, we depend on the stage1 c++ llvm backend - // to compile zig code. - if (build_options.is_stage1) - break :blk true; - - // We would want to prefer LLVM for release builds when it is available, however - // we don't have an LLVM backend yet :) - // We would also want to prefer LLVM for architectures that we don't have self-hosted support for too. - break :blk false; - }; - if (!use_llvm and options.machine_code_model != .default) { - return error.MachineCodeModelNotSupported; - } const link_libc = options.link_libc or (is_exe_or_dyn_lib and target_util.osRequiresLibC(options.target)); @@ -720,6 +729,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .llvm_cpu_features = llvm_cpu_features, .is_compiler_rt_or_libc = options.is_compiler_rt_or_libc, .each_lib_rpath = options.each_lib_rpath orelse false, + .disable_lld_caching = options.disable_lld_caching, }); errdefer bin_file.destroy(); comp.* = .{ @@ -2288,7 +2298,7 @@ pub fn updateSubCompilation(sub_compilation: *Compilation) !void { } } -fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFile) !void { +fn buildStaticLibFromZig(comp: *Compilation, src_basename: []const u8, out: *?CRTFile) !void { const tracy = trace(@src()); defer tracy.end(); @@ -2304,12 +2314,20 @@ fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFil .path = special_path, .handle = special_dir, }, - .root_src_path = basename, + .root_src_path = src_basename, }; + const root_name = mem.split(src_basename, ".").next().?; + const target = comp.getTarget(); + const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{ + .root_name = root_name, + .target = target, + .output_mode = .Lib, + }); + defer comp.gpa.free(bin_basename); const emit_bin = Compilation.EmitLoc{ .directory = null, // Put it in the cache directory. - .basename = basename, + .basename = bin_basename, }; const optimize_mode: std.builtin.Mode = blk: { if (comp.is_test) @@ -2323,8 +2341,8 @@ fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFil .global_cache_directory = comp.global_cache_directory, .local_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, - .target = comp.getTarget(), - .root_name = mem.split(basename, ".").next().?, + .target = target, + .root_name = root_name, .root_pkg = &root_pkg, .output_mode = .Lib, .rand = comp.rand, @@ -2358,7 +2376,9 @@ fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFil assert(out.* == null); out.* = Compilation.CRTFile{ - .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{basename}), + .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{ + sub_compilation.bin_file.options.sub_path, + }), .lock = sub_compilation.bin_file.toOwnedLock(), }; } @@ -2461,7 +2481,7 @@ fn updateStage1Module(comp: *Compilation) !void { ) orelse return error.OutOfMemory; const stage1_pkg = try createStage1Pkg(arena, "root", mod.root_pkg, null); - const output_dir = comp.bin_file.options.directory.path orelse "."; + const output_dir = directory.path orelse "."; const test_filter = comp.test_filter orelse ""[0..0]; const test_name_prefix = comp.test_name_prefix orelse ""[0..0]; stage1_module.* = .{ @@ -2617,13 +2637,11 @@ pub fn build_crt_file( try sub_compilation.updateSubCompilation(); try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1); - const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p| - try std.fs.path.join(comp.gpa, &[_][]const u8{ p, basename }) - else - try comp.gpa.dupe(u8, basename); comp.crt_files.putAssumeCapacityNoClobber(basename, .{ - .full_object_path = artifact_path, + .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{ + sub_compilation.bin_file.options.sub_path, + }), .lock = sub_compilation.bin_file.toOwnedLock(), }); } diff --git a/src/link.zig b/src/link.zig index 82d67f2a7a..4c485063bd 100644 --- a/src/link.zig +++ b/src/link.zig @@ -66,6 +66,7 @@ pub const Options = struct { error_return_tracing: bool, is_compiler_rt_or_libc: bool, each_lib_rpath: bool, + disable_lld_caching: bool, gc_sections: ?bool = null, allow_shlib_undefined: ?bool = null, linker_script: ?[]const u8 = null, diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 931c1bc9c0..82da3f3ae2 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -23,6 +23,7 @@ const File = link.File; const build_options = @import("build_options"); const target_util = @import("../target.zig"); const glibc = @import("../glibc.zig"); +const Cache = @import("../Cache.zig"); const default_entry_addr = 0x8000000; @@ -1225,7 +1226,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { const use_stage1 = build_options.is_stage1 and self.base.options.use_llvm; if (use_stage1) { const obj_basename = try std.fmt.allocPrint(arena, "{}.o", .{self.base.options.root_name}); - const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename}); + const o_directory = self.base.options.module.?.zig_cache_artifact_directory; + const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename}); break :blk full_obj_path; } @@ -1235,6 +1237,12 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { break :blk full_obj_path; } else null; + const is_lib = self.base.options.output_mode == .Lib; + const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib; + const is_exe_or_dyn_lib = is_dyn_lib or self.base.options.output_mode == .Exe; + const have_dynamic_linker = self.base.options.link_libc and + self.base.options.link_mode == .Dynamic and is_exe_or_dyn_lib; + // Here we want to determine whether we can save time by not invoking LLD when the // output is unchanged. None of the linker options or the object files that are being // linked are in the hash that namespaces the directory we are outputting to. Therefore, @@ -1245,78 +1253,78 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { // our digest. If so, we can skip linking. Otherwise, we proceed with invoking LLD. const id_symlink_basename = "lld.id"; - // We are about to obtain this lock, so here we give other processes a chance first. - self.base.releaseLock(); + var man: Cache.Manifest = undefined; + defer if (!self.base.options.disable_lld_caching) man.deinit(); - var ch = comp.cache_parent.obtain(); - defer ch.deinit(); + var digest: [Cache.hex_digest_len]u8 = undefined; - const is_lib = self.base.options.output_mode == .Lib; - const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib; - const is_exe_or_dyn_lib = is_dyn_lib or self.base.options.output_mode == .Exe; - const have_dynamic_linker = self.base.options.link_libc and - self.base.options.link_mode == .Dynamic and is_exe_or_dyn_lib; + if (!self.base.options.disable_lld_caching) { + man = comp.cache_parent.obtain(); - try ch.addOptionalFile(self.base.options.linker_script); - try ch.addOptionalFile(self.base.options.version_script); - try ch.addListOfFiles(self.base.options.objects); - for (comp.c_object_table.items()) |entry| { - _ = try ch.addFile(entry.key.status.success.object_path, null); - } - try ch.addOptionalFile(module_obj_path); - // We can skip hashing libc and libc++ components that we are in charge of building from Zig - // installation sources because they are always a product of the compiler version + target information. - ch.hash.addOptional(self.base.options.stack_size_override); - ch.hash.addOptional(self.base.options.gc_sections); - ch.hash.add(self.base.options.eh_frame_hdr); - ch.hash.add(self.base.options.rdynamic); - ch.hash.addListOfBytes(self.base.options.extra_lld_args); - ch.hash.addListOfBytes(self.base.options.lib_dirs); - ch.hash.addListOfBytes(self.base.options.rpath_list); - ch.hash.add(self.base.options.each_lib_rpath); - ch.hash.add(self.base.options.is_compiler_rt_or_libc); - ch.hash.add(self.base.options.z_nodelete); - ch.hash.add(self.base.options.z_defs); - if (self.base.options.link_libc) { - ch.hash.add(self.base.options.libc_installation != null); - if (self.base.options.libc_installation) |libc_installation| { - ch.hash.addBytes(libc_installation.crt_dir.?); + // We are about to obtain this lock, so here we give other processes a chance first. + self.base.releaseLock(); + + try man.addOptionalFile(self.base.options.linker_script); + try man.addOptionalFile(self.base.options.version_script); + try man.addListOfFiles(self.base.options.objects); + for (comp.c_object_table.items()) |entry| { + _ = try man.addFile(entry.key.status.success.object_path, null); } - if (have_dynamic_linker) { - ch.hash.addOptionalBytes(self.base.options.dynamic_linker); + try man.addOptionalFile(module_obj_path); + // We can skip hashing libc and libc++ components that we are in charge of building from Zig + // installation sources because they are always a product of the compiler version + target information. + man.hash.addOptional(self.base.options.stack_size_override); + man.hash.addOptional(self.base.options.gc_sections); + man.hash.add(self.base.options.eh_frame_hdr); + man.hash.add(self.base.options.rdynamic); + man.hash.addListOfBytes(self.base.options.extra_lld_args); + man.hash.addListOfBytes(self.base.options.lib_dirs); + man.hash.addListOfBytes(self.base.options.rpath_list); + man.hash.add(self.base.options.each_lib_rpath); + man.hash.add(self.base.options.is_compiler_rt_or_libc); + man.hash.add(self.base.options.z_nodelete); + man.hash.add(self.base.options.z_defs); + if (self.base.options.link_libc) { + man.hash.add(self.base.options.libc_installation != null); + if (self.base.options.libc_installation) |libc_installation| { + man.hash.addBytes(libc_installation.crt_dir.?); + } + if (have_dynamic_linker) { + man.hash.addOptionalBytes(self.base.options.dynamic_linker); + } } - } - if (is_dyn_lib) { - ch.hash.addOptionalBytes(self.base.options.override_soname); - ch.hash.addOptional(self.base.options.version); - } - ch.hash.addListOfBytes(self.base.options.system_libs); - ch.hash.addOptional(self.base.options.allow_shlib_undefined); - ch.hash.add(self.base.options.bind_global_refs_locally); + if (is_dyn_lib) { + man.hash.addOptionalBytes(self.base.options.override_soname); + man.hash.addOptional(self.base.options.version); + } + man.hash.addListOfBytes(self.base.options.system_libs); + man.hash.addOptional(self.base.options.allow_shlib_undefined); + man.hash.add(self.base.options.bind_global_refs_locally); - // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock. - _ = try ch.hit(); - const digest = ch.final(); + // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock. + _ = try man.hit(); + digest = man.final(); - var prev_digest_buf: [digest.len]u8 = undefined; - const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { - log.debug("ELF LLD new_digest={} readlink error: {}", .{digest, @errorName(err)}); - // Handle this as a cache miss. - break :blk prev_digest_buf[0..0]; - }; - if (mem.eql(u8, prev_digest, &digest)) { - log.debug("ELF LLD digest={} match - skipping invocation", .{digest}); - // Hot diggity dog! The output binary is already there. - self.base.lock = ch.toOwnedLock(); - return; + var prev_digest_buf: [digest.len]u8 = undefined; + const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { + log.debug("ELF LLD new_digest={} readlink error: {}", .{digest, @errorName(err)}); + // Handle this as a cache miss. + break :blk prev_digest_buf[0..0]; + }; + if (mem.eql(u8, prev_digest, &digest)) { + log.debug("ELF LLD digest={} match - skipping invocation", .{digest}); + // Hot diggity dog! The output binary is already there. + self.base.lock = man.toOwnedLock(); + return; + } + log.debug("ELF LLD prev_digest={} new_digest={}", .{prev_digest, digest}); + + // We are about to change the output file to be different, so we invalidate the build hash now. + directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) { + error.FileNotFound => {}, + else => |e| return e, + }; } - log.debug("ELF LLD prev_digest={} new_digest={}", .{prev_digest, digest}); - - // We are about to change the output file to be different, so we invalidate the build hash now. - directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) { - error.FileNotFound => {}, - else => |e| return e, - }; const target = self.base.options.target; const is_obj = self.base.options.output_mode == .Obj; @@ -1620,18 +1628,20 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items}); } - // Update the dangling symlink with the digest. If it fails we can continue; it only - // means that the next invocation will have an unnecessary cache miss. - directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { - std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); - }; - // Again failure here only means an unnecessary cache miss. - ch.writeManifest() catch |err| { - std.log.warn("failed to write cache manifest when linking: {}", .{ @errorName(err) }); - }; - // We hang on to this lock so that the output file path can be used without - // other processes clobbering it. - self.base.lock = ch.toOwnedLock(); + if (!self.base.options.disable_lld_caching) { + // Update the dangling symlink with the digest. If it fails we can continue; it only + // means that the next invocation will have an unnecessary cache miss. + directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { + std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); + }; + // Again failure here only means an unnecessary cache miss. + man.writeManifest() catch |err| { + std.log.warn("failed to write cache manifest when linking: {}", .{ @errorName(err) }); + }; + // We hang on to this lock so that the output file path can be used without + // other processes clobbering it. + self.base.lock = man.toOwnedLock(); + } } const LLDContext = struct { diff --git a/src/main.zig b/src/main.zig index fe4c27ced6..fc3ebd49c4 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1425,6 +1425,7 @@ pub fn buildOutputType( .test_evented_io = test_evented_io, .test_filter = test_filter, .test_name_prefix = test_name_prefix, + .disable_lld_caching = !have_enable_cache, }) catch |err| { fatal("unable to create compilation: {}", .{@errorName(err)}); }; From 21b407b17f25001b70bbd847f9b2d2782866597c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 25 Sep 2020 18:21:21 -0700 Subject: [PATCH 146/210] update test case for init-exe with respect to new template Now it uses std.log.info instead of std.debug.print --- test/cli.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cli.zig b/test/cli.zig index 77d79ed98e..b9de23e250 100644 --- a/test/cli.zig +++ b/test/cli.zig @@ -98,7 +98,7 @@ fn testZigInitLib(zig_exe: []const u8, dir_path: []const u8) !void { fn testZigInitExe(zig_exe: []const u8, dir_path: []const u8) !void { _ = try exec(dir_path, &[_][]const u8{ zig_exe, "init-exe" }); const run_result = try exec(dir_path, &[_][]const u8{ zig_exe, "build", "run" }); - testing.expect(std.mem.eql(u8, run_result.stderr, "All your codebase are belong to us.\n")); + testing.expect(std.mem.eql(u8, run_result.stderr, "info: All your codebase are belong to us.\n")); } fn testGodboltApi(zig_exe: []const u8, dir_path: []const u8) anyerror!void { From aded86e6909e01dfb45b35204e9dedf6aabb3d58 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 25 Sep 2020 20:50:40 -0700 Subject: [PATCH 147/210] std.ArrayHashMap: count and iterator are not deprecated These APIs allow one to write code that is agnostic of whether it is using an ArrayHashMap or a HashMap, which can be valuable. Specify intent precisely: if you only need the count of the items, it makes sense to have a function for that. --- lib/std/array_hash_map.zig | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/std/array_hash_map.zig b/lib/std/array_hash_map.zig index f8c3623ef2..649c1e1055 100644 --- a/lib/std/array_hash_map.zig +++ b/lib/std/array_hash_map.zig @@ -112,12 +112,10 @@ pub fn ArrayHashMap( return self.unmanaged.clearAndFree(self.allocator); } - /// Deprecated. Use `items().len`. pub fn count(self: Self) usize { - return self.items().len; + return self.unmanaged.count(); } - /// Deprecated. Iterate using `items`. pub fn iterator(self: *const Self) Iterator { return Iterator{ .hm = self, @@ -332,6 +330,10 @@ pub fn ArrayHashMapUnmanaged( } } + pub fn count(self: Self) usize { + return self.entries.items.len; + } + /// If key exists this function cannot fail. /// If there is an existing item with `key`, then the result /// `Entry` pointer points to it, and found_existing is true. From a337046832b936d912b6902e331cb58bdc513a2d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 25 Sep 2020 20:52:02 -0700 Subject: [PATCH 148/210] stage2: properly handle zig cc used as a preprocessor This cleans up how the CLI parses and handles -E, -S, and -c. Compilation explicitly acknowledges when it is being used to do C preprocessing. -S is properly translated to -fno-emit-bin -femit-asm but Compilation does not yet handle -femit-asm. There is not yet a mechanism for skipping the linking step when there is only a single object file, and so to make this work we have to do a file copy in link.flush() to copy the file from zig-cache into the output directory. --- BRANCH_TODO | 4 +- src/Compilation.zig | 38 +++++++++---- src/clang_options_data.zig | 14 +++-- src/link.zig | 21 +++++++- src/link/Elf.zig | 5 +- src/main.zig | 99 ++++++++++++++++++---------------- tools/update_clang_options.zig | 36 ++++++------- 7 files changed, 130 insertions(+), 87 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 384f0761c3..6c04b46935 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,5 +1,3 @@ - * make sure that `zig cc -o hello hello.c -target native-native-musl` and `zig build-exe hello.zig -lc -target native-native-musl` will share the same libc build. - * zig cc as a preprocessor (-E) * tests passing with -Dskip-non-native * `-ftime-report` * -fstack-report print stack size diagnostics\n" @@ -20,6 +18,7 @@ * restore error messages for stage2_add_link_lib * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj] * try building some software with zig cc + * implement support for -femit-asm * implement proper parsing of clang stderr/stdout and exposing compile errors with the Compilation API * implement proper parsing of LLD stderr/stdout and exposing compile errors with the Compilation API @@ -57,4 +56,3 @@ * make std.Progress support multithreaded * update musl.zig static data to use native path separator in static data rather than replacing '/' at runtime * linking hello world with LLD, lld is silently calling exit(1) instead of reporting ok=false. when run standalone the error message is: ld.lld: error: section [index 3] has a sh_offset (0x57000) + sh_size (0x68) that is greater than the file size (0x57060) - diff --git a/src/Compilation.zig b/src/Compilation.zig index c28b27b588..b8856e8a43 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -49,6 +49,7 @@ sanitize_c: bool, /// Otherwise we attempt to parse the error messages and expose them via the Compilation API. /// This is `true` for `zig cc`, `zig c++`, and `zig translate-c`. clang_passthrough_mode: bool, +clang_preprocessor_mode: ClangPreprocessorMode, /// Whether to print clang argvs to stdout. verbose_cc: bool, verbose_tokenize: bool, @@ -271,6 +272,14 @@ pub const EmitLoc = struct { basename: []const u8, }; +pub const ClangPreprocessorMode = enum { + no, + /// This means we are doing `zig cc -E -o `. + yes, + /// This means we are doing `zig cc -E`. + stdout, +}; + pub const InitOptions = struct { zig_lib_directory: Directory, local_cache_directory: Directory, @@ -285,6 +294,8 @@ pub const InitOptions = struct { emit_bin: ?EmitLoc, /// `null` means to not emit a C header file. emit_h: ?EmitLoc = null, + /// `null` means to not emit assembly. + emit_asm: ?EmitLoc = null, link_mode: ?std.builtin.LinkMode = null, dll_export_fns: ?bool = false, /// Normally when using LLD to link, Zig uses a file named "lld.id" in the @@ -349,6 +360,7 @@ pub const InitOptions = struct { version: ?std.builtin.Version = null, libc_installation: ?*const LibCInstallation = null, machine_code_model: std.builtin.CodeModel = .default, + clang_preprocessor_mode: ClangPreprocessorMode = .no, /// This is for stage1 and should be deleted upon completion of self-hosting. color: @import("main.zig").Color = .Auto, test_filter: ?[]const u8 = null, @@ -478,6 +490,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { } else must_pic; if (options.emit_h != null) fatal("-femit-h not supported yet", .{}); // TODO + if (options.emit_asm != null) fatal("-femit-asm not supported yet", .{}); // TODO const emit_bin = options.emit_bin orelse fatal("-fno-emit-bin not supported yet", .{}); // TODO @@ -750,6 +763,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .sanitize_c = sanitize_c, .rand = options.rand, .clang_passthrough_mode = options.clang_passthrough_mode, + .clang_preprocessor_mode = options.clang_preprocessor_mode, .verbose_cc = options.verbose_cc, .verbose_tokenize = options.verbose_tokenize, .verbose_ast = options.verbose_ast, @@ -1215,7 +1229,6 @@ fn obtainCObjectCacheManifest(comp: *Compilation) Cache.Manifest { // Only things that need to be added on top of the base hash, and only things // that apply both to @cImport and compiling C objects. No linking stuff here! // Also nothing that applies only to compiling .zig code. - man.hash.add(comp.sanitize_c); man.hash.addListOfBytes(comp.clang_argv); man.hash.add(comp.bin_file.options.link_libcpp); @@ -1381,6 +1394,8 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { var man = comp.obtainCObjectCacheManifest(); defer man.deinit(); + man.hash.add(comp.clang_preprocessor_mode); + _ = try man.addFile(c_object.src.src_path, null); { // Hash the extra flags, with special care to call addFile for file parameters. @@ -1424,7 +1439,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { var zig_cache_tmp_dir = try comp.local_cache_directory.handle.makeOpenPath("tmp", .{}); defer zig_cache_tmp_dir.close(); - try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang", "-c" }); + try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang" }); const ext = classifyFileExt(c_object.src.src_path); const out_dep_path: ?[]const u8 = if (comp.disable_c_depfile or !ext.clangSupportsDepFile()) @@ -1433,8 +1448,12 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { try std.fmt.allocPrint(arena, "{}.d", .{out_obj_path}); try comp.addCCArgs(arena, &argv, ext, out_dep_path); - try argv.append("-o"); - try argv.append(out_obj_path); + try argv.ensureCapacity(argv.items.len + 3); + switch (comp.clang_preprocessor_mode) { + .no => argv.appendSliceAssumeCapacity(&[_][]const u8{"-c", "-o", out_obj_path}), + .yes => argv.appendSliceAssumeCapacity(&[_][]const u8{"-E", "-o", out_obj_path}), + .stdout => argv.appendAssumeCapacity("-E"), + } try argv.append(c_object.src.src_path); try argv.appendSlice(c_object.src.extra_flags); @@ -1460,6 +1479,8 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { // TODO https://github.com/ziglang/zig/issues/6342 std.process.exit(1); } + if (comp.clang_preprocessor_mode == .stdout) + std.process.exit(0); }, else => std.process.exit(1), } @@ -1522,14 +1543,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { break :blk digest; }; - const components = if (comp.local_cache_directory.path) |p| - &[_][]const u8{ p, "o", &digest, o_basename } - else - &[_][]const u8{ "o", &digest, o_basename }; - c_object.status = .{ .success = .{ - .object_path = try std.fs.path.join(comp.gpa, components), + .object_path = try comp.local_cache_directory.join(comp.gpa, &[_][]const u8{ + "o", &digest, o_basename, + }), .lock = man.toOwnedLock(), }, }; diff --git a/src/clang_options_data.zig b/src/clang_options_data.zig index 889737bdac..bd1237bc00 100644 --- a/src/clang_options_data.zig +++ b/src/clang_options_data.zig @@ -7,7 +7,7 @@ flagpd1("CC"), .{ .name = "E", .syntax = .flag, - .zig_equivalent = .pp_or_asm, + .zig_equivalent = .preprocess_only, .pd1 = true, .pd2 = false, .psl = false, @@ -95,7 +95,7 @@ flagpd1("Qy"), .{ .name = "S", .syntax = .flag, - .zig_equivalent = .pp_or_asm, + .zig_equivalent = .asm_only, .pd1 = true, .pd2 = false, .psl = false, @@ -196,7 +196,7 @@ sepd1("Zlinker-input"), .{ .name = "E", .syntax = .flag, - .zig_equivalent = .pp_or_asm, + .zig_equivalent = .preprocess_only, .pd1 = true, .pd2 = false, .psl = true, @@ -1477,7 +1477,7 @@ flagpsl("MT"), .{ .name = "assemble", .syntax = .flag, - .zig_equivalent = .pp_or_asm, + .zig_equivalent = .asm_only, .pd1 = false, .pd2 = true, .psl = false, @@ -1805,7 +1805,7 @@ flagpsl("MT"), .{ .name = "preprocess", .syntax = .flag, - .zig_equivalent = .pp_or_asm, + .zig_equivalent = .preprocess_only, .pd1 = false, .pd2 = true, .psl = false, @@ -3406,6 +3406,8 @@ flagpd1("mlong-double-128"), flagpd1("mlong-double-64"), flagpd1("mlong-double-80"), flagpd1("mlongcall"), +flagpd1("mlvi-cfi"), +flagpd1("mlvi-hardening"), flagpd1("mlwp"), flagpd1("mlzcnt"), flagpd1("mmadd4"), @@ -3499,6 +3501,8 @@ flagpd1("mno-ldc1-sdc1"), flagpd1("mno-local-sdata"), flagpd1("mno-long-calls"), flagpd1("mno-longcall"), +flagpd1("mno-lvi-cfi"), +flagpd1("mno-lvi-hardening"), flagpd1("mno-lwp"), flagpd1("mno-lzcnt"), flagpd1("mno-madd4"), diff --git a/src/link.zig b/src/link.zig index 4c485063bd..6e8fd3fdc7 100644 --- a/src/link.zig +++ b/src/link.zig @@ -1,16 +1,18 @@ const std = @import("std"); const mem = std.mem; const Allocator = std.mem.Allocator; +const fs = std.fs; +const log = std.log.scoped(.link); +const assert = std.debug.assert; + const Compilation = @import("Compilation.zig"); const Module = @import("Module.zig"); -const fs = std.fs; const trace = @import("tracy.zig").trace; const Package = @import("Package.zig"); const Type = @import("type.zig").Type; const Cache = @import("Cache.zig"); const build_options = @import("build_options"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; -const log = std.log.scoped(.link); pub const producer_string = if (std.builtin.is_test) "zig test" else "zig " ++ build_options.version; @@ -303,6 +305,21 @@ pub const File = struct { /// Commit pending changes and write headers. Takes into account final output mode /// and `use_lld`, not only `effectiveOutputMode`. pub fn flush(base: *File, comp: *Compilation) !void { + if (comp.clang_preprocessor_mode == .yes) { + // TODO: avoid extra link step when it's just 1 object file (the `zig cc -c` case) + // Until then, we do `lld -r -o output.o input.o` even though the output is the same + // as the input. For the preprocessing case (`zig cc -E -o foo`) we copy the file + // to the final location. + const full_out_path = try base.options.directory.join(comp.gpa, &[_][]const u8{ + base.options.sub_path, + }); + defer comp.gpa.free(full_out_path); + assert(comp.c_object_table.count() == 1); + const the_entry = comp.c_object_table.items()[0]; + const cached_pp_file_path = the_entry.key.status.success.object_path; + try fs.cwd().copyFile(cached_pp_file_path, fs.cwd(), full_out_path, .{}); + return; + } const use_lld = build_options.have_llvm and base.options.use_lld; if (use_lld and base.options.output_mode == .Lib and base.options.link_mode == .Static and !base.options.target.isWasm()) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 82da3f3ae2..8cb77013dc 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1401,10 +1401,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { try argv.append("-pie"); } - const full_out_path = if (directory.path) |dir_path| - try fs.path.join(arena, &[_][]const u8{dir_path, self.base.options.sub_path}) - else - self.base.options.sub_path; + const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.sub_path}); try argv.append("-o"); try argv.append(full_out_path); diff --git a/src/main.zig b/src/main.zig index fc3ebd49c4..2204c1f3ed 100644 --- a/src/main.zig +++ b/src/main.zig @@ -327,6 +327,7 @@ pub fn buildOutputType( var time_report = false; var show_builtin = false; var emit_bin: Emit = .yes_default_path; + var emit_asm: Emit = .no; var emit_zir: Emit = .no; var target_arch_os_abi: []const u8 = "native"; var target_mcpu: ?[]const u8 = null; @@ -345,7 +346,6 @@ pub fn buildOutputType( var want_stack_check: ?bool = null; var want_valgrind: ?bool = null; var rdynamic: bool = false; - var only_pp_or_asm = false; var linker_script: ?[]const u8 = null; var version_script: ?[]const u8 = null; var disable_c_depfile = false; @@ -371,6 +371,7 @@ pub fn buildOutputType( var override_global_cache_dir: ?[]const u8 = null; var override_lib_dir: ?[]const u8 = null; var main_pkg_path: ?[]const u8 = null; + var clang_preprocessor_mode: Compilation.ClangPreprocessorMode = .no; var system_libs = std.ArrayList([]const u8).init(gpa); defer system_libs.deinit(); @@ -752,7 +753,14 @@ pub fn buildOutputType( ensure_libcpp_on_non_freestanding = arg_mode == .cpp; want_native_include_dirs = true; - var c_arg = false; + const COutMode = enum { + link, + object, + assembly, + preprocessor, + }; + var c_out_mode: COutMode = .link; + var out_path: ?[]const u8 = null; var is_shared_lib = false; var linker_args = std.ArrayList([]const u8).init(arena); var it = ClangArgIterator.init(arena, all_args); @@ -762,12 +770,10 @@ pub fn buildOutputType( }; switch (it.zig_equivalent) { .target => target_arch_os_abi = it.only_arg, // example: -target riscv64-linux-unknown - .o => { - // -o - emit_bin = .{ .yes = it.only_arg }; - enable_cache = true; - }, - .c => c_arg = true, // -c + .o => out_path = it.only_arg, // -o + .c => c_out_mode = .object, // -c + .asm_only => c_out_mode = .assembly, // -S + .preprocess_only => c_out_mode = .preprocessor, // -E .other => { try clang_argv.appendSlice(it.other_args); }, @@ -813,11 +819,6 @@ pub fn buildOutputType( try linker_args.append(linker_arg); } }, - .pp_or_asm => { - // This handles both -E and -S. - only_pp_or_asm = true; - try clang_argv.appendSlice(it.other_args); - }, .optimize => { // Alright, what release mode do they want? if (mem.eql(u8, it.only_arg, "Os")) { @@ -999,32 +1000,43 @@ pub fn buildOutputType( } } - if (only_pp_or_asm) { - output_mode = .Obj; - fatal("TODO implement using zig cc as a preprocessor", .{}); - //// Transfer "link_objects" into c_source_files so that all those - //// args make it onto the command line. - //try c_source_files.appendSlice(link_objects.items); - //for (c_source_files.items) |c_source_file| { - // const src_path = switch (emit_bin) { - // .yes => |p| p, - // else => c_source_file.source_path, - // }; - // const basename = fs.path.basename(src_path); - // c_source_file.preprocessor_only_basename = basename; - //} - //emit_bin = .no; - } else if (!c_arg) { - output_mode = if (is_shared_lib) .Lib else .Exe; - switch (emit_bin) { - .no, .yes_default_path => { - emit_bin = .{ .yes = "a.out" }; - enable_cache = true; - }, - .yes => {}, - } - } else { - output_mode = .Obj; + switch (c_out_mode) { + .link => { + output_mode = if (is_shared_lib) .Lib else .Exe; + emit_bin = .{ .yes = out_path orelse "a.out" }; + enable_cache = true; + }, + .object => { + output_mode = .Obj; + if (out_path) |p| { + emit_bin = .{ .yes = p }; + } else { + emit_bin = .yes_default_path; + } + }, + .assembly => { + output_mode = .Obj; + emit_bin = .no; + if (out_path) |p| { + emit_asm = .{ .yes = p }; + } else { + emit_asm = .yes_default_path; + } + }, + .preprocessor => { + output_mode = .Obj; + // An error message is generated when there is more than 1 C source file. + if (c_source_files.items.len != 1) { + // For example `zig cc` and no args should print the "no input files" message. + return punt_to_clang(arena, all_args); + } + if (out_path) |p| { + emit_bin = .{ .yes = p }; + clang_preprocessor_mode = .yes; + } else { + clang_preprocessor_mode = .stdout; + } + }, } if (c_source_files.items.len == 0 and link_objects.items.len == 0) { // For example `zig cc` and no args should print the "no input files" message. @@ -1407,6 +1419,7 @@ pub fn buildOutputType( .self_exe_path = self_exe_path, .rand = &default_prng.random, .clang_passthrough_mode = arg_mode != .build, + .clang_preprocessor_mode = clang_preprocessor_mode, .version = optional_version, .libc_installation = if (libc_installation) |*lci| lci else null, .verbose_cc = verbose_cc, @@ -1453,11 +1466,6 @@ pub fn buildOutputType( try updateModule(gpa, comp, zir_out_path, hook); - if (build_options.have_llvm and only_pp_or_asm) { - // this may include dumping the output to stdout - fatal("TODO: implement `zig cc` when using it as a preprocessor", .{}); - } - if (build_options.is_stage1 and comp.stage1_lock != null and watch) { std.log.warn("--watch is not recommended with the stage1 backend; it leaks memory and is not capable of incremental compilation", .{}); } @@ -2436,7 +2444,8 @@ pub const ClangArgIterator = struct { shared, rdynamic, wl, - pp_or_asm, + preprocess_only, + asm_only, optimize, debug, sanitize, diff --git a/tools/update_clang_options.zig b/tools/update_clang_options.zig index ea63e767bb..8b7811aa26 100644 --- a/tools/update_clang_options.zig +++ b/tools/update_clang_options.zig @@ -116,19 +116,19 @@ const known_options = [_]KnownOpt{ }, .{ .name = "E", - .ident = "pp_or_asm", + .ident = "preprocess_only", }, .{ .name = "preprocess", - .ident = "pp_or_asm", + .ident = "preprocess_only", }, .{ .name = "S", - .ident = "pp_or_asm", + .ident = "asm_only", }, .{ .name = "assemble", - .ident = "pp_or_asm", + .ident = "asm_only", }, .{ .name = "O1", @@ -346,7 +346,7 @@ pub fn main() anyerror!void { for (blacklisted_options) |blacklisted_key| { if (std.mem.eql(u8, blacklisted_key, kv.key)) continue :it_map; } - if (kv.value.Object.get("Name").?.value.String.len == 0) continue; + if (kv.value.Object.get("Name").?.String.len == 0) continue; try all_objects.append(&kv.value.Object); } } @@ -365,11 +365,11 @@ pub fn main() anyerror!void { ); for (all_objects.span()) |obj| { - const name = obj.get("Name").?.value.String; + const name = obj.get("Name").?.String; var pd1 = false; var pd2 = false; var pslash = false; - for (obj.get("Prefixes").?.value.Array.span()) |prefix_json| { + for (obj.get("Prefixes").?.Array.span()) |prefix_json| { const prefix = prefix_json.String; if (std.mem.eql(u8, prefix, "-")) { pd1 = true; @@ -465,7 +465,7 @@ const Syntax = union(enum) { self: Syntax, comptime fmt: []const u8, options: std.fmt.FormatOptions, - out_stream: var, + out_stream: anytype, ) !void { switch (self) { .multi_arg => |n| return out_stream.print(".{{.{}={}}}", .{ @tagName(self), n }), @@ -475,8 +475,8 @@ const Syntax = union(enum) { }; fn objSyntax(obj: *json.ObjectMap) Syntax { - const num_args = @intCast(u8, obj.get("NumArgs").?.value.Integer); - for (obj.get("!superclasses").?.value.Array.span()) |superclass_json| { + const num_args = @intCast(u8, obj.get("NumArgs").?.Integer); + for (obj.get("!superclasses").?.Array.span()) |superclass_json| { const superclass = superclass_json.String; if (std.mem.eql(u8, superclass, "Joined")) { return .joined; @@ -510,19 +510,19 @@ fn objSyntax(obj: *json.ObjectMap) Syntax { return .{ .multi_arg = num_args }; } } - const name = obj.get("Name").?.value.String; + const name = obj.get("Name").?.String; if (std.mem.eql(u8, name, "")) { return .flag; } else if (std.mem.eql(u8, name, "")) { return .flag; } - const kind_def = obj.get("Kind").?.value.Object.get("def").?.value.String; + const kind_def = obj.get("Kind").?.Object.get("def").?.String; if (std.mem.eql(u8, kind_def, "KIND_FLAG")) { return .flag; } - const key = obj.get("!name").?.value.String; + const key = obj.get("!name").?.String; std.debug.warn("{} (key {}) has unrecognized superclasses:\n", .{ name, key }); - for (obj.get("!superclasses").?.value.Array.span()) |superclass_json| { + for (obj.get("!superclasses").?.Array.span()) |superclass_json| { std.debug.warn(" {}\n", .{superclass_json.String}); } std.process.exit(1); @@ -560,15 +560,15 @@ fn objectLessThan(context: void, a: *json.ObjectMap, b: *json.ObjectMap) bool { } if (!a_match_with_eql and !b_match_with_eql) { - const a_name = a.get("Name").?.value.String; - const b_name = b.get("Name").?.value.String; + const a_name = a.get("Name").?.String; + const b_name = b.get("Name").?.String; if (a_name.len != b_name.len) { return a_name.len > b_name.len; } } - const a_key = a.get("!name").?.value.String; - const b_key = b.get("!name").?.value.String; + const a_key = a.get("!name").?.String; + const b_key = b.get("!name").?.String; return std.mem.lessThan(u8, a_key, b_key); } From 6af2990549709ec5e2bc1efeb5090a944f1a8bdf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 26 Sep 2020 01:42:54 -0700 Subject: [PATCH 149/210] implement -femit-asm, -femit-docs, -femit-llvm-ir, etc These CLI options are now forwarded to the stage1 backend. We're not going to support the -mllvm CLI option any longer. As a compromise, we unconditionally tell LLVM to output intel x86 syntax when using -femit-asm. Simplify stage1 logic; it no longer has the concept of an output directory. --output-dir is no longer a valid CLI option. cmake uses the `-femit-bin=[path]` option. Note the changes to test/cli.zig. This breaks the CLI API that Godbolt is using so we're going to want to open a PR to help them upgrade to the new CLI for the upcoming Zig 0.7.0 release. --- BRANCH_TODO | 13 +--- CMakeLists.txt | 6 +- src/Compilation.zig | 68 +++++++++++++---- src/llvm.zig | 3 + src/main.zig | 157 ++++++++++++++++++++++++++++++--------- src/stage1.zig | 19 +++-- src/stage1/all_types.hpp | 11 +-- src/stage1/codegen.cpp | 75 ++++++------------- src/stage1/stage1.cpp | 13 ++-- src/stage1/stage1.h | 24 ++++-- src/stage1/zig0.cpp | 13 ++-- test/cli.zig | 19 +++-- 12 files changed, 263 insertions(+), 158 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 6c04b46935..5d4293c5f1 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,13 +1,7 @@ + * support -fno-emit-bin for godbolt * tests passing with -Dskip-non-native * `-ftime-report` * -fstack-report print stack size diagnostics\n" - * -fdump-analysis write analysis.json file with type information\n" - * -femit-docs create a docs/ dir with html documentation\n" - * -fno-emit-docs do not produce docs/ dir with html documentation\n" - * -femit-asm output .s (assembly code)\n" - * -fno-emit-asm (default) do not output .s (assembly code)\n" - * -femit-llvm-ir produce a .ll file with LLVM IR\n" - * -fno-emit-llvm-ir (default) do not produce a .ll file with LLVM IR\n" * mingw-w64 * MachO LLD linking * COFF LLD linking @@ -17,8 +11,8 @@ * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process. * restore error messages for stage2_add_link_lib * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj] - * try building some software with zig cc - * implement support for -femit-asm + * try building some software with zig cc to make sure it didn't regress + * restore the legacy -femit-h feature using the stage1 backend * implement proper parsing of clang stderr/stdout and exposing compile errors with the Compilation API * implement proper parsing of LLD stderr/stdout and exposing compile errors with the Compilation API @@ -56,3 +50,4 @@ * make std.Progress support multithreaded * update musl.zig static data to use native path separator in static data rather than replacing '/' at runtime * linking hello world with LLD, lld is silently calling exit(1) instead of reporting ok=false. when run standalone the error message is: ld.lld: error: section [index 3] has a sh_offset (0x57000) + sh_size (0x68) that is greater than the file size (0x57060) + * submit PR to godbolt and update the CLI options (see changes to test/cli.zig) diff --git a/CMakeLists.txt b/CMakeLists.txt index f812da9d76..c4d7b3976a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -449,7 +449,7 @@ endif() if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") set(ZIG1_RELEASE_ARG "") else() - set(ZIG1_RELEASE_ARG --release-fast --strip) + set(ZIG1_RELEASE_ARG -OReleaseFast --strip) endif() set(BUILD_ZIG1_ARGS @@ -458,8 +458,8 @@ set(BUILD_ZIG1_ARGS "-mcpu=${ZIG_TARGET_MCPU}" --name zig1 --override-lib-dir "${CMAKE_SOURCE_DIR}/lib" - --output-dir "${CMAKE_BINARY_DIR}" - ${ZIG1_RELEASE_ARG} + "-femit-bin=${ZIG1_OBJECT}" + "${ZIG1_RELEASE_ARG}" -lc --pkg-begin build_options "${ZIG_CONFIG_ZIG_OUT}" --pkg-end diff --git a/src/Compilation.zig b/src/Compilation.zig index b8856e8a43..d831403497 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -107,6 +107,12 @@ test_filter: ?[]const u8, test_name_prefix: ?[]const u8, test_evented_io: bool, +emit_h: ?EmitLoc, +emit_asm: ?EmitLoc, +emit_llvm_ir: ?EmitLoc, +emit_analysis: ?EmitLoc, +emit_docs: ?EmitLoc, + pub const InnerError = Module.InnerError; pub const CRTFile = struct { @@ -296,6 +302,12 @@ pub const InitOptions = struct { emit_h: ?EmitLoc = null, /// `null` means to not emit assembly. emit_asm: ?EmitLoc = null, + /// `null` means to not emit LLVM IR. + emit_llvm_ir: ?EmitLoc = null, + /// `null` means to not emit semantic analysis JSON. + emit_analysis: ?EmitLoc = null, + /// `null` means to not emit docs. + emit_docs: ?EmitLoc = null, link_mode: ?std.builtin.LinkMode = null, dll_export_fns: ?bool = false, /// Normally when using LLD to link, Zig uses a file named "lld.id" in the @@ -442,7 +454,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { break :blk false; }; - const link_libc = options.link_libc or (is_exe_or_dyn_lib and target_util.osRequiresLibC(options.target)); @@ -489,9 +500,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { break :pic explicit; } else must_pic; - if (options.emit_h != null) fatal("-femit-h not supported yet", .{}); // TODO - if (options.emit_asm != null) fatal("-femit-asm not supported yet", .{}); // TODO - const emit_bin = options.emit_bin orelse fatal("-fno-emit-bin not supported yet", .{}); // TODO // Make a decision on whether to use Clang for translate-c and compiling C files. @@ -579,6 +587,12 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { cache.hash.add(options.link_libcpp); cache.hash.add(options.output_mode); cache.hash.add(options.machine_code_model); + cache.hash.add(options.emit_bin != null); + cache.hash.add(options.emit_h != null); + cache.hash.add(options.emit_asm != null); + cache.hash.add(options.emit_llvm_ir != null); + cache.hash.add(options.emit_analysis != null); + cache.hash.add(options.emit_docs != null); // TODO audit this and make sure everything is in it const module: ?*Module = if (options.root_pkg) |root_pkg| blk: { @@ -752,6 +766,11 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .local_cache_directory = options.local_cache_directory, .global_cache_directory = options.global_cache_directory, .bin_file = bin_file, + .emit_h = options.emit_h, + .emit_asm = options.emit_asm, + .emit_llvm_ir = options.emit_llvm_ir, + .emit_analysis = options.emit_analysis, + .emit_docs = options.emit_docs, .work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa), .keep_source_files_loaded = options.keep_source_files_loaded, .use_clang = use_clang, @@ -1450,8 +1469,8 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { try argv.ensureCapacity(argv.items.len + 3); switch (comp.clang_preprocessor_mode) { - .no => argv.appendSliceAssumeCapacity(&[_][]const u8{"-c", "-o", out_obj_path}), - .yes => argv.appendSliceAssumeCapacity(&[_][]const u8{"-E", "-o", out_obj_path}), + .no => argv.appendSliceAssumeCapacity(&[_][]const u8{ "-c", "-o", out_obj_path }), + .yes => argv.appendSliceAssumeCapacity(&[_][]const u8{ "-E", "-o", out_obj_path }), .stdout => argv.appendAssumeCapacity("-E"), } @@ -2498,15 +2517,35 @@ fn updateStage1Module(comp: *Compilation) !void { comp.is_test, ) orelse return error.OutOfMemory; + const bin_basename = try std.zig.binNameAlloc(arena, .{ + .root_name = comp.bin_file.options.root_name, + .target = target, + .output_mode = .Obj, + }); + const emit_bin_path = try directory.join(arena, &[_][]const u8{bin_basename}); + const emit_h_path = try stage1LocPath(arena, comp.emit_h, directory); + const emit_asm_path = try stage1LocPath(arena, comp.emit_asm, directory); + const emit_llvm_ir_path = try stage1LocPath(arena, comp.emit_llvm_ir, directory); + const emit_analysis_path = try stage1LocPath(arena, comp.emit_analysis, directory); + const emit_docs_path = try stage1LocPath(arena, comp.emit_docs, directory); const stage1_pkg = try createStage1Pkg(arena, "root", mod.root_pkg, null); - const output_dir = directory.path orelse "."; const test_filter = comp.test_filter orelse ""[0..0]; const test_name_prefix = comp.test_name_prefix orelse ""[0..0]; stage1_module.* = .{ .root_name_ptr = comp.bin_file.options.root_name.ptr, .root_name_len = comp.bin_file.options.root_name.len, - .output_dir_ptr = output_dir.ptr, - .output_dir_len = output_dir.len, + .emit_o_ptr = emit_bin_path.ptr, + .emit_o_len = emit_bin_path.len, + .emit_h_ptr = emit_h_path.ptr, + .emit_h_len = emit_h_path.len, + .emit_asm_ptr = emit_asm_path.ptr, + .emit_asm_len = emit_asm_path.len, + .emit_llvm_ir_ptr = emit_llvm_ir_path.ptr, + .emit_llvm_ir_len = emit_llvm_ir_path.len, + .emit_analysis_json_ptr = emit_analysis_path.ptr, + .emit_analysis_json_len = emit_analysis_path.len, + .emit_docs_ptr = emit_docs_path.ptr, + .emit_docs_len = emit_docs_path.len, .builtin_zig_path_ptr = builtin_zig_path.ptr, .builtin_zig_path_len = builtin_zig_path.len, .test_filter_ptr = test_filter.ptr, @@ -2530,11 +2569,6 @@ fn updateStage1Module(comp: *Compilation) !void { .enable_stack_probing = comp.bin_file.options.stack_check, .enable_time_report = comp.time_report, .enable_stack_report = false, - .dump_analysis = false, - .enable_doc_generation = false, - .emit_bin = true, - .emit_asm = false, - .emit_llvm_ir = false, .test_is_evented = comp.test_evented_io, .verbose_tokenize = comp.verbose_tokenize, .verbose_ast = comp.verbose_ast, @@ -2565,6 +2599,12 @@ fn updateStage1Module(comp: *Compilation) !void { comp.stage1_lock = man.toOwnedLock(); } +fn stage1LocPath(arena: *Allocator, opt_loc: ?EmitLoc, cache_directory: Directory) ![]const u8 { + const loc = opt_loc orelse return ""; + const directory = loc.directory orelse cache_directory; + return directory.join(arena, &[_][]const u8{loc.basename}); +} + fn createStage1Pkg( arena: *Allocator, name: []const u8, diff --git a/src/llvm.zig b/src/llvm.zig index 64a6d4e8b5..ceefb62c5d 100644 --- a/src/llvm.zig +++ b/src/llvm.zig @@ -72,3 +72,6 @@ pub const OSType = extern enum(c_int) { WASI = 34, Emscripten = 35, }; + +pub const ParseCommandLineOptions = ZigLLVMParseCommandLineOptions; +extern fn ZigLLVMParseCommandLineOptions(argc: usize, argv: [*]const [*:0]const u8) void; diff --git a/src/main.zig b/src/main.zig index 2204c1f3ed..30338f11a1 100644 --- a/src/main.zig +++ b/src/main.zig @@ -195,8 +195,20 @@ const usage_build_generic = \\ -h, --help Print this help and exit \\ --watch Enable compiler REPL \\ --color [auto|off|on] Enable or disable colored error messages - \\ -femit-bin[=path] (default) output machine code + \\ -femit-bin[=path] (default) Output machine code \\ -fno-emit-bin Do not output machine code + \\ -femit-asm[=path] Output .s (assembly code) + \\ -fno-emit-asm (default) Do not output .s (assembly code) + \\ -femit-zir[=path] Produce a .zir file with Zig IR + \\ -fno-emit-zir (default) Do not produce a .zir file with Zig IR + \\ -femit-llvm-ir[=path] Produce a .ll file with LLVM IR (requires LLVM extensions) + \\ -fno-emit-llvm-ir (default) Do not produce a .ll file with LLVM IR + \\ -femit-h[=path] Generate a C header file (.h) + \\ -fno-emit-h (default) Do not generate a C header file (.h) + \\ -femit-docs[=path] Create a docs/ dir with html documentation + \\ -fno-emit-docs (default) Do not produce docs/ dir with html documentation + \\ -femit-analysis[=path] Write analysis JSON file with type information + \\ -fno-emit-analysis (default) Do not write analysis JSON file with type information \\ --show-builtin Output the source of @import("builtin") then exit \\ --cache-dir [path] Override the local cache directory \\ --global-cache-dir [path] Override the global cache directory @@ -210,10 +222,11 @@ const usage_build_generic = \\ small|kernel| \\ medium|large] \\ --name [name] Override root name (not a file path) - \\ -ODebug (default) optimizations off, safety on - \\ -OReleaseFast Optimizations on, safety off - \\ -OReleaseSafe Optimizations on, safety on - \\ -OReleaseSmall Optimize for small binary, safety off + \\ -O [mode] Choose what to optimize for + \\ Debug (default) Optimizations off, safety on + \\ ReleaseFast Optimizations on, safety off + \\ ReleaseSafe Optimizations on, safety on + \\ ReleaseSmall Optimize for small binary, safety off \\ --pkg-begin [name] [path] Make pkg available to import and push current pkg \\ --pkg-end Pop current pkg \\ --main-pkg-path Set the directory of the root package @@ -290,9 +303,57 @@ const Emit = union(enum) { no, yes_default_path, yes: []const u8, + + const Resolved = struct { + data: ?Compilation.EmitLoc, + dir: ?fs.Dir, + + fn deinit(self: *Resolved) void { + if (self.dir) |*dir| { + dir.close(); + } + } + }; + + fn resolve(emit: Emit, default_basename: []const u8) !Resolved { + var resolved: Resolved = .{ .data = null, .dir = null }; + errdefer resolved.deinit(); + + switch (emit) { + .no => {}, + .yes_default_path => { + resolved.data = Compilation.EmitLoc{ + .directory = .{ .path = null, .handle = fs.cwd() }, + .basename = default_basename, + }; + }, + .yes => |full_path| { + const basename = fs.path.basename(full_path); + if (fs.path.dirname(full_path)) |dirname| { + const handle = try fs.cwd().openDir(dirname, .{}); + resolved = .{ + .dir = handle, + .data = Compilation.EmitLoc{ + .basename = basename, + .directory = .{ + .path = dirname, + .handle = handle, + }, + }, + }; + } else { + resolved.data = Compilation.EmitLoc{ + .basename = basename, + .directory = .{ .path = null, .handle = fs.cwd() }, + }; + } + }, + } + return resolved; + } }; -pub fn buildOutputType( +fn buildOutputType( gpa: *Allocator, arena: *Allocator, all_args: []const []const u8, @@ -328,7 +389,10 @@ pub fn buildOutputType( var show_builtin = false; var emit_bin: Emit = .yes_default_path; var emit_asm: Emit = .no; + var emit_llvm_ir: Emit = .no; var emit_zir: Emit = .no; + var emit_docs: Emit = .no; + var emit_analysis: Emit = .no; var target_arch_os_abi: []const u8 = "native"; var target_mcpu: ?[]const u8 = null; var target_dynamic_linker: ?[]const u8 = null; @@ -667,6 +731,30 @@ pub fn buildOutputType( emit_h = .{ .yes = arg["-femit-h=".len..] }; } else if (mem.eql(u8, arg, "-fno-emit-h")) { emit_h = .no; + } else if (mem.eql(u8, arg, "-femit-asm")) { + emit_asm = .yes_default_path; + } else if (mem.startsWith(u8, arg, "-femit-asm=")) { + emit_asm = .{ .yes = arg["-femit-asm=".len..] }; + } else if (mem.eql(u8, arg, "-fno-emit-asm")) { + emit_asm = .no; + } else if (mem.eql(u8, arg, "-femit-llvm-ir")) { + emit_llvm_ir = .yes_default_path; + } else if (mem.startsWith(u8, arg, "-femit-llvm-ir=")) { + emit_llvm_ir = .{ .yes = arg["-femit-llvm-ir=".len..] }; + } else if (mem.eql(u8, arg, "-fno-emit-llvm-ir")) { + emit_llvm_ir = .no; + } else if (mem.eql(u8, arg, "-femit-docs")) { + emit_docs = .yes_default_path; + } else if (mem.startsWith(u8, arg, "-femit-docs=")) { + emit_docs = .{ .yes = arg["-femit-docs=".len..] }; + } else if (mem.eql(u8, arg, "-fno-emit-docs")) { + emit_docs = .no; + } else if (mem.eql(u8, arg, "-femit-analysis")) { + emit_analysis = .yes_default_path; + } else if (mem.startsWith(u8, arg, "-femit-analysis=")) { + emit_analysis = .{ .yes = arg["-femit-analysis=".len..] }; + } else if (mem.eql(u8, arg, "-fno-emit-analysis")) { + emit_analysis = .no; } else if (mem.eql(u8, arg, "-dynamic")) { link_mode = .Dynamic; } else if (mem.eql(u8, arg, "-static")) { @@ -1237,35 +1325,24 @@ pub fn buildOutputType( }, }; - var cleanup_emit_h_dir: ?fs.Dir = null; - defer if (cleanup_emit_h_dir) |*dir| dir.close(); + const default_h_basename = try std.fmt.allocPrint(arena, "{}.h", .{root_name}); + var emit_h_resolved = try emit_h.resolve(default_h_basename); + defer emit_h_resolved.deinit(); - const emit_h_loc: ?Compilation.EmitLoc = switch (emit_h) { - .no => null, - .yes_default_path => Compilation.EmitLoc{ - .directory = .{ .path = null, .handle = fs.cwd() }, - .basename = try std.fmt.allocPrint(arena, "{}.h", .{root_name}), - }, - .yes => |full_path| b: { - const basename = fs.path.basename(full_path); - if (fs.path.dirname(full_path)) |dirname| { - const handle = try fs.cwd().openDir(dirname, .{}); - cleanup_emit_h_dir = handle; - break :b Compilation.EmitLoc{ - .basename = basename, - .directory = .{ - .path = dirname, - .handle = handle, - }, - }; - } else { - break :b Compilation.EmitLoc{ - .basename = basename, - .directory = .{ .path = null, .handle = fs.cwd() }, - }; - } - }, - }; + const default_asm_basename = try std.fmt.allocPrint(arena, "{}.s", .{root_name}); + var emit_asm_resolved = try emit_asm.resolve(default_asm_basename); + defer emit_asm_resolved.deinit(); + + const default_llvm_ir_basename = try std.fmt.allocPrint(arena, "{}.ll", .{root_name}); + var emit_llvm_ir_resolved = try emit_llvm_ir.resolve(default_llvm_ir_basename); + defer emit_llvm_ir_resolved.deinit(); + + const default_analysis_basename = try std.fmt.allocPrint(arena, "{}-analysis.json", .{root_name}); + var emit_analysis_resolved = try emit_analysis.resolve(default_analysis_basename); + defer emit_analysis_resolved.deinit(); + + var emit_docs_resolved = try emit_docs.resolve("docs"); + defer emit_docs_resolved.deinit(); const zir_out_path: ?[]const u8 = switch (emit_zir) { .no => null, @@ -1365,6 +1442,12 @@ pub fn buildOutputType( }; }; + if (build_options.have_llvm and emit_asm != .no) { + // LLVM has no way to set this non-globally. + const argv = [_][*:0]const u8{ "zig (LLVM option parsing)", "--x86-asm-syntax=intel" }; + @import("llvm.zig").ParseCommandLineOptions(argv.len, &argv); + } + gimmeMoreOfThoseSweetSweetFileDescriptors(); const comp = Compilation.create(gpa, .{ @@ -1378,7 +1461,11 @@ pub fn buildOutputType( .output_mode = output_mode, .root_pkg = root_pkg, .emit_bin = emit_bin_loc, - .emit_h = emit_h_loc, + .emit_h = emit_h_resolved.data, + .emit_asm = emit_asm_resolved.data, + .emit_llvm_ir = emit_llvm_ir_resolved.data, + .emit_docs = emit_docs_resolved.data, + .emit_analysis = emit_analysis_resolved.data, .link_mode = link_mode, .dll_export_fns = dll_export_fns, .object_format = object_format, diff --git a/src/stage1.zig b/src/stage1.zig index 7a9609f317..d05c0ba7fb 100644 --- a/src/stage1.zig +++ b/src/stage1.zig @@ -75,8 +75,18 @@ pub const Pkg = extern struct { pub const Module = extern struct { root_name_ptr: [*]const u8, root_name_len: usize, - output_dir_ptr: [*]const u8, - output_dir_len: usize, + emit_o_ptr: [*]const u8, + emit_o_len: usize, + emit_h_ptr: [*]const u8, + emit_h_len: usize, + emit_asm_ptr: [*]const u8, + emit_asm_len: usize, + emit_llvm_ir_ptr: [*]const u8, + emit_llvm_ir_len: usize, + emit_analysis_json_ptr: [*]const u8, + emit_analysis_json_len: usize, + emit_docs_ptr: [*]const u8, + emit_docs_len: usize, builtin_zig_path_ptr: [*]const u8, builtin_zig_path_len: usize, test_filter_ptr: [*]const u8, @@ -101,11 +111,6 @@ pub const Module = extern struct { enable_stack_probing: bool, enable_time_report: bool, enable_stack_report: bool, - dump_analysis: bool, - enable_doc_generation: bool, - emit_bin: bool, - emit_asm: bool, - emit_llvm_ir: bool, test_is_evented: bool, verbose_tokenize: bool, verbose_ast: bool, diff --git a/src/stage1/all_types.hpp b/src/stage1/all_types.hpp index 75a89d272f..a1d0cd0aa1 100644 --- a/src/stage1/all_types.hpp +++ b/src/stage1/all_types.hpp @@ -2116,12 +2116,12 @@ struct CodeGen { Buf llvm_triple_str; Buf global_asm; Buf o_file_output_path; + Buf h_file_output_path; Buf asm_file_output_path; Buf llvm_ir_file_output_path; + Buf analysis_json_output_path; + Buf docs_output_path; Buf *cache_dir; - // As an input parameter, mutually exclusive with enable_cache. But it gets - // populated in codegen_build_and_link. - Buf *output_dir; Buf *c_artifact_dir; const char **libc_include_dir_list; size_t libc_include_dir_len; @@ -2186,11 +2186,6 @@ struct CodeGen { bool dll_export_fns; bool have_stack_probing; bool function_sections; - bool enable_dump_analysis; - bool enable_doc_generation; - bool emit_bin; - bool emit_asm; - bool emit_llvm_ir; bool test_is_evented; bool valgrind_enabled; diff --git a/src/stage1/codegen.cpp b/src/stage1/codegen.cpp index 0a8a1f7a95..fff1c325fc 100644 --- a/src/stage1/codegen.cpp +++ b/src/stage1/codegen.cpp @@ -8243,9 +8243,9 @@ static void zig_llvm_emit_output(CodeGen *g) { const char *bin_filename = nullptr; const char *llvm_ir_filename = nullptr; - if (g->emit_bin) bin_filename = buf_ptr(&g->o_file_output_path); - if (g->emit_asm) asm_filename = buf_ptr(&g->asm_file_output_path); - if (g->emit_llvm_ir) llvm_ir_filename = buf_ptr(&g->llvm_ir_file_output_path); + if (buf_len(&g->o_file_output_path) != 0) bin_filename = buf_ptr(&g->o_file_output_path); + if (buf_len(&g->asm_file_output_path) != 0) asm_filename = buf_ptr(&g->asm_file_output_path); + if (buf_len(&g->llvm_ir_file_output_path) != 0) llvm_ir_filename = buf_ptr(&g->llvm_ir_file_output_path); // Unfortunately, LLVM shits the bed when we ask for both binary and assembly. So we call the entire // pipeline multiple times if this is requested. @@ -8858,8 +8858,10 @@ static Error define_builtin_compile_vars(CodeGen *g) { Buf *contents; if (g->builtin_zig_path == nullptr) { // Then this is zig0 building stage2. We can make many assumptions about the compilation. + Buf *out_dir = buf_alloc(); + os_path_split(&g->o_file_output_path, out_dir, nullptr); g->builtin_zig_path = buf_alloc(); - os_path_join(g->output_dir, buf_create_from_str(builtin_zig_basename), g->builtin_zig_path); + os_path_join(out_dir, buf_create_from_str(builtin_zig_basename), g->builtin_zig_path); Buf *resolve_paths[] = { g->builtin_zig_path, }; *g->builtin_zig_path = os_path_resolve(resolve_paths, 1); @@ -8870,7 +8872,7 @@ static Error define_builtin_compile_vars(CodeGen *g) { exit(1); } - g->compile_var_package = new_package(buf_ptr(g->output_dir), builtin_zig_basename, "builtin"); + g->compile_var_package = new_package(buf_ptr(out_dir), builtin_zig_basename, "builtin"); } else { Buf *resolve_paths[] = { g->builtin_zig_path, }; *g->builtin_zig_path = os_path_resolve(resolve_paths, 1); @@ -9232,33 +9234,20 @@ void codegen_add_time_event(CodeGen *g, const char *name) { g->timing_events.append({seconds, name}); } -static void resolve_out_paths(CodeGen *g) { - assert(g->output_dir != nullptr); - assert(g->root_out_name != nullptr); +void codegen_build_object(CodeGen *g) { + g->have_err_ret_tracing = detect_err_ret_tracing(g); - if (g->emit_bin) { - Buf *o_basename = buf_create_from_buf(g->root_out_name); - buf_append_str(o_basename, target_o_file_ext(g->zig_target)); - os_path_join(g->output_dir, o_basename, &g->o_file_output_path); - } - if (g->emit_asm) { - Buf *asm_basename = buf_create_from_buf(g->root_out_name); - const char *asm_ext = target_asm_file_ext(g->zig_target); - buf_append_str(asm_basename, asm_ext); - os_path_join(g->output_dir, asm_basename, &g->asm_file_output_path); - } - if (g->emit_llvm_ir) { - Buf *llvm_ir_basename = buf_create_from_buf(g->root_out_name); - const char *llvm_ir_ext = target_llvm_ir_file_ext(g->zig_target); - buf_append_str(llvm_ir_basename, llvm_ir_ext); - os_path_join(g->output_dir, llvm_ir_basename, &g->llvm_ir_file_output_path); - } -} + init(g); -static void output_type_information(CodeGen *g) { - if (g->enable_dump_analysis) { - const char *analysis_json_filename = buf_ptr(buf_sprintf("%s" OS_SEP "%s-analysis.json", - buf_ptr(g->output_dir), buf_ptr(g->root_out_name))); + codegen_add_time_event(g, "Semantic Analysis"); + const char *progress_name = "Semantic Analysis"; + codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, + progress_name, strlen(progress_name), 0)); + + gen_root_source(g); + + if (buf_len(&g->analysis_json_output_path) != 0) { + const char *analysis_json_filename = buf_ptr(&g->analysis_json_output_path); FILE *f = fopen(analysis_json_filename, "wb"); if (f == nullptr) { fprintf(stderr, "Unable to open '%s': %s\n", analysis_json_filename, strerror(errno)); @@ -9270,9 +9259,9 @@ static void output_type_information(CodeGen *g) { exit(1); } } - if (g->enable_doc_generation) { + if (buf_len(&g->docs_output_path) != 0) { Error err; - Buf *doc_dir_path = buf_sprintf("%s" OS_SEP "docs", buf_ptr(g->output_dir)); + Buf *doc_dir_path = &g->docs_output_path; if ((err = os_make_path(doc_dir_path))) { fprintf(stderr, "Unable to create directory %s: %s\n", buf_ptr(doc_dir_path), err_str(err)); exit(1); @@ -9308,27 +9297,6 @@ static void output_type_information(CodeGen *g) { exit(1); } } -} - -void codegen_build_object(CodeGen *g) { - assert(g->output_dir != nullptr); - - g->have_err_ret_tracing = detect_err_ret_tracing(g); - - init(g); - - codegen_add_time_event(g, "Semantic Analysis"); - const char *progress_name = "Semantic Analysis"; - codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node, - progress_name, strlen(progress_name), 0)); - - gen_root_source(g); - - resolve_out_paths(g); - - if (g->enable_dump_analysis || g->enable_doc_generation) { - output_type_information(g); - } codegen_add_time_event(g, "Code Generation"); { @@ -9379,7 +9347,6 @@ CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget bool is_test_build) { CodeGen *g = heap::c_allocator.create(); - g->emit_bin = true; g->pass1_arena = heap::ArenaAllocator::construct(&heap::c_allocator, &heap::c_allocator, "pass1"); g->subsystem = TargetSubsystemAuto; diff --git a/src/stage1/stage1.cpp b/src/stage1/stage1.cpp index b5b87c05d9..1034f9ff88 100644 --- a/src/stage1/stage1.cpp +++ b/src/stage1/stage1.cpp @@ -69,7 +69,13 @@ void zig_stage1_build_object(struct ZigStage1 *stage1) { CodeGen *g = reinterpret_cast(stage1); g->root_out_name = buf_create_from_mem(stage1->root_name_ptr, stage1->root_name_len); - g->output_dir = buf_create_from_mem(stage1->output_dir_ptr, stage1->output_dir_len); + buf_init_from_mem(&g->o_file_output_path, stage1->emit_o_ptr, stage1->emit_o_len); + buf_init_from_mem(&g->h_file_output_path, stage1->emit_h_ptr, stage1->emit_h_len); + buf_init_from_mem(&g->asm_file_output_path, stage1->emit_asm_ptr, stage1->emit_asm_len); + buf_init_from_mem(&g->llvm_ir_file_output_path, stage1->emit_llvm_ir_ptr, stage1->emit_llvm_ir_len); + buf_init_from_mem(&g->analysis_json_output_path, stage1->emit_analysis_json_ptr, stage1->emit_analysis_json_len); + buf_init_from_mem(&g->docs_output_path, stage1->emit_docs_ptr, stage1->emit_docs_len); + if (stage1->builtin_zig_path_len != 0) { g->builtin_zig_path = buf_create_from_mem(stage1->builtin_zig_path_ptr, stage1->builtin_zig_path_len); } @@ -94,11 +100,6 @@ void zig_stage1_build_object(struct ZigStage1 *stage1) { g->enable_time_report = stage1->enable_time_report; g->enable_stack_report = stage1->enable_stack_report; - g->enable_dump_analysis = stage1->dump_analysis; - g->enable_doc_generation = stage1->enable_doc_generation; - g->emit_bin = stage1->emit_bin; - g->emit_asm = stage1->emit_asm; - g->emit_llvm_ir = stage1->emit_llvm_ir; g->test_is_evented = stage1->test_is_evented; g->verbose_tokenize = stage1->verbose_tokenize; diff --git a/src/stage1/stage1.h b/src/stage1/stage1.h index c0bd0f70fc..e5ffa62e14 100644 --- a/src/stage1/stage1.h +++ b/src/stage1/stage1.h @@ -141,8 +141,23 @@ struct ZigStage1 { const char *root_name_ptr; size_t root_name_len; - const char *output_dir_ptr; - size_t output_dir_len; + const char *emit_o_ptr; + size_t emit_o_len; + + const char *emit_h_ptr; + size_t emit_h_len; + + const char *emit_asm_ptr; + size_t emit_asm_len; + + const char *emit_llvm_ir_ptr; + size_t emit_llvm_ir_len; + + const char *emit_analysis_json_ptr; + size_t emit_analysis_json_len; + + const char *emit_docs_ptr; + size_t emit_docs_len; const char *builtin_zig_path_ptr; size_t builtin_zig_path_len; @@ -173,11 +188,6 @@ struct ZigStage1 { bool enable_stack_probing; bool enable_time_report; bool enable_stack_report; - bool dump_analysis; - bool enable_doc_generation; - bool emit_bin; - bool emit_asm; - bool emit_llvm_ir; bool test_is_evented; bool verbose_tokenize; bool verbose_ast; diff --git a/src/stage1/zig0.cpp b/src/stage1/zig0.cpp index 5c384991f9..7fd20c524e 100644 --- a/src/stage1/zig0.cpp +++ b/src/stage1/zig0.cpp @@ -241,7 +241,7 @@ int main(int argc, char **argv) { Error err; const char *in_file = nullptr; - const char *output_dir = nullptr; + const char *emit_bin_path = nullptr; bool strip = false; const char *out_name = nullptr; bool verbose_tokenize = false; @@ -324,14 +324,14 @@ int main(int argc, char **argv) { cur_pkg = cur_pkg->parent; } else if (str_starts_with(arg, "-mcpu=")) { mcpu = arg + strlen("-mcpu="); + } else if (str_starts_with(arg, "-femit-bin=")) { + emit_bin_path = arg + strlen("-femit-bin="); } else if (i + 1 >= argc) { fprintf(stderr, "Expected another argument after %s\n", arg); return print_error_usage(arg0); } else { i += 1; - if (strcmp(arg, "--output-dir") == 0) { - output_dir = argv[i]; - } else if (strcmp(arg, "--color") == 0) { + if (strcmp(arg, "--color") == 0) { if (strcmp(argv[i], "auto") == 0) { color = ErrColorAuto; } else if (strcmp(argv[i], "on") == 0) { @@ -443,15 +443,14 @@ int main(int argc, char **argv) { stage1->verbose_llvm_ir = verbose_llvm_ir; stage1->verbose_cimport = verbose_cimport; stage1->verbose_llvm_cpu_features = verbose_llvm_cpu_features; - stage1->output_dir_ptr = output_dir; - stage1->output_dir_len = strlen(output_dir); + stage1->emit_o_ptr = emit_bin_path; + stage1->emit_o_len = strlen(emit_bin_path); stage1->root_pkg = cur_pkg; stage1->err_color = color; stage1->link_libc = link_libc; stage1->link_libcpp = link_libcpp; stage1->subsystem = subsystem; stage1->pic = true; - stage1->emit_bin = true; zig_stage1_build_object(stage1); diff --git a/test/cli.zig b/test/cli.zig index b9de23e250..c92636fb3c 100644 --- a/test/cli.zig +++ b/test/cli.zig @@ -118,17 +118,20 @@ fn testGodboltApi(zig_exe: []const u8, dir_path: []const u8) anyerror!void { \\} ); - const args = [_][]const u8{ + var args = std.ArrayList([]const u8).init(a); + try args.appendSlice(&[_][]const u8{ zig_exe, "build-obj", "--cache-dir", dir_path, "--name", "example", - "--output-dir", dir_path, - "--emit", "asm", - "-mllvm", "--x86-asm-syntax=intel", - "--strip", "--release-fast", - example_zig_path, "--disable-gen-h", - }; - _ = try exec(dir_path, &args); + "-fno-emit-bin", "-fno-emit-h", + "--strip", "-OReleaseFast", + example_zig_path, + }); + + const emit_asm_arg = try std.fmt.allocPrint(a, "-femit-asm={s}", .{example_s_path}); + try args.append(emit_asm_arg); + + _ = try exec(dir_path, args.items); const out_asm = try std.fs.cwd().readFileAlloc(a, example_s_path, std.math.maxInt(usize)); testing.expect(std.mem.indexOf(u8, out_asm, "square:") != null); From 819625d2740838ad10440f4af04200a38b5c65e2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 26 Sep 2020 01:58:16 -0700 Subject: [PATCH 150/210] fix logic for choosing when dynamic linking is required When building object files or static libraries, link_mode can still be static even if the OS needs libc for syscalls. --- BRANCH_TODO | 3 ++- src/Compilation.zig | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 5d4293c5f1..05c6f26612 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,6 @@ * support -fno-emit-bin for godbolt + * restore the legacy -femit-h feature using the stage1 backend + * figure out why test-translate-c is failing * tests passing with -Dskip-non-native * `-ftime-report` * -fstack-report print stack size diagnostics\n" @@ -12,7 +14,6 @@ * restore error messages for stage2_add_link_lib * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj] * try building some software with zig cc to make sure it didn't regress - * restore the legacy -femit-h feature using the stage1 backend * implement proper parsing of clang stderr/stdout and exposing compile errors with the Compilation API * implement proper parsing of LLD stderr/stdout and exposing compile errors with the Compilation API diff --git a/src/Compilation.zig b/src/Compilation.zig index d831403497..f51ea74d56 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -454,16 +454,16 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { break :blk false; }; - const link_libc = options.link_libc or - (is_exe_or_dyn_lib and target_util.osRequiresLibC(options.target)); + const link_libc = options.link_libc or target_util.osRequiresLibC(options.target); const must_dynamic_link = dl: { if (target_util.cannotDynamicLink(options.target)) break :dl false; - if (target_util.osRequiresLibC(options.target)) - break :dl true; - if (is_exe_or_dyn_lib and link_libc and options.target.isGnuLibC()) + if (is_exe_or_dyn_lib and link_libc and + (options.target.isGnuLibC() or target_util.osRequiresLibC(options.target))) + { break :dl true; + } if (options.system_libs.len != 0) break :dl true; From 56086d11a463297eea3a52ad8e6b245ed397029f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 26 Sep 2020 02:16:10 -0700 Subject: [PATCH 151/210] stage0: update CLI to match stage2 --- BRANCH_TODO | 1 + src/stage1/zig0.cpp | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 05c6f26612..b5cba8805e 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -4,6 +4,7 @@ * tests passing with -Dskip-non-native * `-ftime-report` * -fstack-report print stack size diagnostics\n" + * subsystem * mingw-w64 * MachO LLD linking * COFF LLD linking diff --git a/src/stage1/zig0.cpp b/src/stage1/zig0.cpp index 7fd20c524e..e63c68bc94 100644 --- a/src/stage1/zig0.cpp +++ b/src/stage1/zig0.cpp @@ -36,9 +36,10 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) { " --output-dir [dir] override output directory (defaults to cwd)\n" " --pkg-begin [name] [path] make pkg available to import and push current pkg\n" " --pkg-end pop current pkg\n" - " --release-fast build with optimizations on and safety off\n" - " --release-safe build with optimizations on and safety on\n" - " --release-small build with size optimizations on and safety off\n" + " -ODebug build with optimizations on and safety off\n" + " -OReleaseFast build with optimizations on and safety off\n" + " -OReleaseSafe build with optimizations on and safety on\n" + " -OReleaseSmall build with size optimizations on and safety off\n" " --single-threaded source may assume it is only used single-threaded\n" " -dynamic create a shared library (.so; .dll; .dylib)\n" " --strip exclude debug symbols\n" @@ -267,11 +268,13 @@ int main(int argc, char **argv) { if (arg[0] == '-') { if (strcmp(arg, "--") == 0) { fprintf(stderr, "Unexpected end-of-parameter mark: %s\n", arg); - } else if (strcmp(arg, "--release-fast") == 0) { + } else if (strcmp(arg, "-ODebug") == 0) { + optimize_mode = BuildModeDebug; + } else if (strcmp(arg, "-OReleaseFast") == 0) { optimize_mode = BuildModeFastRelease; - } else if (strcmp(arg, "--release-safe") == 0) { + } else if (strcmp(arg, "-OReleaseSafe") == 0) { optimize_mode = BuildModeSafeRelease; - } else if (strcmp(arg, "--release-small") == 0) { + } else if (strcmp(arg, "-OReleaseSmall") == 0) { optimize_mode = BuildModeSmallRelease; } else if (strcmp(arg, "--help") == 0) { return print_full_usage(arg0, stdout, EXIT_SUCCESS); From 1337f0bc6adf7f57466228efe09ba2c22e4683ae Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 26 Sep 2020 02:34:18 -0700 Subject: [PATCH 152/210] move to stage1-specific cache manifest emission of extra stuff --- src/Compilation.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index f51ea74d56..1af83078dd 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -588,11 +588,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { cache.hash.add(options.output_mode); cache.hash.add(options.machine_code_model); cache.hash.add(options.emit_bin != null); - cache.hash.add(options.emit_h != null); - cache.hash.add(options.emit_asm != null); - cache.hash.add(options.emit_llvm_ir != null); - cache.hash.add(options.emit_analysis != null); - cache.hash.add(options.emit_docs != null); // TODO audit this and make sure everything is in it const module: ?*Module = if (options.root_pkg) |root_pkg| blk: { @@ -2457,6 +2452,11 @@ fn updateStage1Module(comp: *Compilation) !void { man.hash.add(comp.bin_file.options.dll_export_fns); man.hash.add(comp.bin_file.options.function_sections); man.hash.add(comp.is_test); + man.hash.add(comp.emit_h != null); + man.hash.add(comp.emit_asm != null); + man.hash.add(comp.emit_llvm_ir != null); + man.hash.add(comp.emit_analysis != null); + man.hash.add(comp.emit_docs != null); // Capture the state in case we come back from this branch where the hash doesn't match. const prev_hash_state = man.hash.peekBin(); From a23b1b425480b4e10c539d9032270487d9fe6cd0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 26 Sep 2020 02:34:57 -0700 Subject: [PATCH 153/210] stage2: building musl: fix typo in replacing path separators --- src/musl.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/musl.zig b/src/musl.zig index 4f03004a70..d64d643227 100644 --- a/src/musl.zig +++ b/src/musl.zig @@ -235,7 +235,7 @@ fn addSrcFile(arena: *Allocator, source_table: *std.StringArrayHashMap(Ext), fil const mutable_file_path = try arena.dupe(u8, file_path); for (mutable_file_path) |*c| { if (c.* == '/') { - c.* == path.sep; + c.* = path.sep; } } break :blk mutable_file_path; From 9b83112a1f6af758a1a995ea8838a2319d2fbc1e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 26 Sep 2020 11:26:46 -0700 Subject: [PATCH 154/210] fix _fbsd suffix appended to lld LDM option twice --- src/link/Elf.zig | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 8cb77013dc..903cb57d03 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -3091,12 +3091,9 @@ fn getLDMOption(target: std.Target) ?[]const u8 { .x86_64 => { if (target.abi == .gnux32) { return "elf32_x86_64"; + } else { + return "elf_x86_64"; } - // Any target elf will use the freebsd osabi if suffixed with "_fbsd". - if (target.os.tag == .freebsd) { - return "elf_x86_64_fbsd"; - } - return "elf_x86_64"; }, .riscv32 => return "elf32lriscv", .riscv64 => return "elf64lriscv", From 26f2f9bf1c774caf246317b3fc032449e982a150 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 26 Sep 2020 12:42:07 -0700 Subject: [PATCH 155/210] stage2: implement -fno-emit-bin we are now passing the cli tests --- BRANCH_TODO | 1 - src/Compilation.zig | 129 +++++++++++++++++++++++++------------------- src/libcxx.zig | 10 +++- src/libunwind.zig | 5 +- src/link.zig | 31 ++++++----- src/link/C.zig | 2 +- src/link/Coff.zig | 2 +- src/link/Elf.zig | 6 +-- src/link/MachO.zig | 2 +- src/link/Wasm.zig | 2 +- src/main.zig | 17 +++--- test/cli.zig | 32 +++++------ 12 files changed, 137 insertions(+), 102 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index b5cba8805e..d1d2104c1e 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ - * support -fno-emit-bin for godbolt * restore the legacy -femit-h feature using the stage1 backend * figure out why test-translate-c is failing * tests passing with -Dskip-non-native diff --git a/src/Compilation.zig b/src/Compilation.zig index 1af83078dd..7385825de3 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -500,8 +500,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { break :pic explicit; } else must_pic; - const emit_bin = options.emit_bin orelse fatal("-fno-emit-bin not supported yet", .{}); // TODO - // Make a decision on whether to use Clang for translate-c and compiling C files. const use_clang = if (options.use_clang) |explicit| explicit else blk: { if (build_options.have_llvm) { @@ -667,13 +665,29 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { } else null; errdefer if (module) |zm| zm.deinit(); + const error_return_tracing = !options.strip and switch (options.optimize_mode) { + .Debug, .ReleaseSafe => true, + .ReleaseFast, .ReleaseSmall => false, + }; + // For resource management purposes. var owned_link_dir: ?std.fs.Dir = null; errdefer if (owned_link_dir) |*dir| dir.close(); - const bin_directory = emit_bin.directory orelse blk: { - if (module) |zm| break :blk zm.zig_cache_artifact_directory; - + const bin_file_emit: ?link.Emit = blk: { + const emit_bin = options.emit_bin orelse break :blk null; + if (emit_bin.directory) |directory| { + break :blk link.Emit{ + .directory = directory, + .sub_path = emit_bin.basename, + }; + } + if (module) |zm| { + break :blk link.Emit{ + .directory = zm.zig_cache_artifact_directory, + .sub_path = emit_bin.basename, + }; + } // We could use the cache hash as is no problem, however, we increase // the likelihood of cache hits by adding the first C source file // path name (not contents) to the hash. This way if the user is compiling @@ -694,17 +708,14 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .handle = artifact_dir, .path = try options.local_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}), }; - break :blk link_artifact_directory; - }; - - const error_return_tracing = !options.strip and switch (options.optimize_mode) { - .Debug, .ReleaseSafe => true, - .ReleaseFast, .ReleaseSmall => false, + break :blk link.Emit{ + .directory = link_artifact_directory, + .sub_path = emit_bin.basename, + }; }; const bin_file = try link.File.openPath(gpa, .{ - .directory = bin_directory, - .sub_path = emit_bin.basename, + .emit = bin_file_emit, .root_name = root_name, .module = module, .target = options.target, @@ -815,43 +826,46 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { comp.c_object_table.putAssumeCapacityNoClobber(c_object, {}); } - // If we need to build glibc for the target, add work items for it. - // We go through the work queue so that building can be done in parallel. - if (comp.wantBuildGLibCFromSource()) { - try comp.addBuildingGLibCJobs(); - } - if (comp.wantBuildMuslFromSource()) { - try comp.work_queue.write(&[_]Job{ - .{ .musl_crt_file = .crti_o }, - .{ .musl_crt_file = .crtn_o }, - .{ .musl_crt_file = .crt1_o }, - .{ .musl_crt_file = .scrt1_o }, - .{ .musl_crt_file = .libc_a }, - }); - } - if (comp.wantBuildMinGWW64FromSource()) { - @panic("TODO"); - } - if (comp.wantBuildLibUnwindFromSource()) { - try comp.work_queue.writeItem(.{ .libunwind = {} }); - } - if (build_options.have_llvm and comp.bin_file.options.output_mode != .Obj and - comp.bin_file.options.link_libcpp) - { - try comp.work_queue.writeItem(.libcxx); - try comp.work_queue.writeItem(.libcxxabi); + if (comp.bin_file.options.emit != null) { + // If we need to build glibc for the target, add work items for it. + // We go through the work queue so that building can be done in parallel. + if (comp.wantBuildGLibCFromSource()) { + try comp.addBuildingGLibCJobs(); + } + if (comp.wantBuildMuslFromSource()) { + try comp.work_queue.write(&[_]Job{ + .{ .musl_crt_file = .crti_o }, + .{ .musl_crt_file = .crtn_o }, + .{ .musl_crt_file = .crt1_o }, + .{ .musl_crt_file = .scrt1_o }, + .{ .musl_crt_file = .libc_a }, + }); + } + if (comp.wantBuildMinGWW64FromSource()) { + @panic("TODO"); + } + if (comp.wantBuildLibUnwindFromSource()) { + try comp.work_queue.writeItem(.{ .libunwind = {} }); + } + if (build_options.have_llvm and comp.bin_file.options.output_mode != .Obj and + comp.bin_file.options.link_libcpp) + { + try comp.work_queue.writeItem(.libcxx); + try comp.work_queue.writeItem(.libcxxabi); + } + if (is_exe_or_dyn_lib and !comp.bin_file.options.is_compiler_rt_or_libc and + build_options.is_stage1) + { + try comp.work_queue.writeItem(.{ .libcompiler_rt = {} }); + if (!comp.bin_file.options.link_libc) { + try comp.work_queue.writeItem(.{ .zig_libc = {} }); + } + } } + if (build_options.is_stage1 and comp.bin_file.options.use_llvm) { try comp.work_queue.writeItem(.{ .stage1_module = {} }); } - if (is_exe_or_dyn_lib and !comp.bin_file.options.is_compiler_rt_or_libc and - build_options.is_stage1) - { - try comp.work_queue.writeItem(.{ .libcompiler_rt = {} }); - if (!comp.bin_file.options.link_libc) { - try comp.work_queue.writeItem(.{ .zig_libc = {} }); - } - } return comp; } @@ -2408,8 +2422,8 @@ fn buildStaticLibFromZig(comp: *Compilation, src_basename: []const u8, out: *?CR assert(out.* == null); out.* = Compilation.CRTFile{ - .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{ - sub_compilation.bin_file.options.sub_path, + .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(comp.gpa, &[_][]const u8{ + sub_compilation.bin_file.options.emit.?.sub_path, }), .lock = sub_compilation.bin_file.toOwnedLock(), }; @@ -2452,6 +2466,7 @@ fn updateStage1Module(comp: *Compilation) !void { man.hash.add(comp.bin_file.options.dll_export_fns); man.hash.add(comp.bin_file.options.function_sections); man.hash.add(comp.is_test); + man.hash.add(comp.bin_file.options.emit != null); man.hash.add(comp.emit_h != null); man.hash.add(comp.emit_asm != null); man.hash.add(comp.emit_llvm_ir != null); @@ -2517,12 +2532,14 @@ fn updateStage1Module(comp: *Compilation) !void { comp.is_test, ) orelse return error.OutOfMemory; - const bin_basename = try std.zig.binNameAlloc(arena, .{ - .root_name = comp.bin_file.options.root_name, - .target = target, - .output_mode = .Obj, - }); - const emit_bin_path = try directory.join(arena, &[_][]const u8{bin_basename}); + const emit_bin_path = if (comp.bin_file.options.emit != null) blk: { + const bin_basename = try std.zig.binNameAlloc(arena, .{ + .root_name = comp.bin_file.options.root_name, + .target = target, + .output_mode = .Obj, + }); + break :blk try directory.join(arena, &[_][]const u8{bin_basename}); + } else ""; const emit_h_path = try stage1LocPath(arena, comp.emit_h, directory); const emit_asm_path = try stage1LocPath(arena, comp.emit_asm, directory); const emit_llvm_ir_path = try stage1LocPath(arena, comp.emit_llvm_ir, directory); @@ -2697,8 +2714,8 @@ pub fn build_crt_file( try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1); comp.crt_files.putAssumeCapacityNoClobber(basename, .{ - .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{ - sub_compilation.bin_file.options.sub_path, + .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(comp.gpa, &[_][]const u8{ + sub_compilation.bin_file.options.emit.?.sub_path, }), .lock = sub_compilation.bin_file.toOwnedLock(), }); diff --git a/src/libcxx.zig b/src/libcxx.zig index 16c9581f82..19987082aa 100644 --- a/src/libcxx.zig +++ b/src/libcxx.zig @@ -189,7 +189,10 @@ pub fn buildLibCXX(comp: *Compilation) !void { assert(comp.libcxx_static_lib == null); comp.libcxx_static_lib = Compilation.CRTFile{ - .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{basename}), + .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join( + comp.gpa, + &[_][]const u8{basename}, + ), .lock = sub_compilation.bin_file.toOwnedLock(), }; } @@ -303,7 +306,10 @@ pub fn buildLibCXXABI(comp: *Compilation) !void { assert(comp.libcxxabi_static_lib == null); comp.libcxxabi_static_lib = Compilation.CRTFile{ - .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{basename}), + .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join( + comp.gpa, + &[_][]const u8{basename}, + ), .lock = sub_compilation.bin_file.toOwnedLock(), }; } diff --git a/src/libunwind.zig b/src/libunwind.zig index cbe632ba94..d47eed40dd 100644 --- a/src/libunwind.zig +++ b/src/libunwind.zig @@ -126,7 +126,10 @@ pub fn buildStaticLib(comp: *Compilation) !void { assert(comp.libunwind_static_lib == null); comp.libunwind_static_lib = Compilation.CRTFile{ - .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{basename}), + .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join( + comp.gpa, + &[_][]const u8{basename}, + ), .lock = sub_compilation.bin_file.toOwnedLock(), }; } diff --git a/src/link.zig b/src/link.zig index 6e8fd3fdc7..6cf1d775e4 100644 --- a/src/link.zig +++ b/src/link.zig @@ -16,11 +16,17 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation; pub const producer_string = if (std.builtin.is_test) "zig test" else "zig " ++ build_options.version; -pub const Options = struct { +pub const Emit = struct { /// Where the output will go. directory: Compilation.Directory, /// Path to the output file, relative to `directory`. sub_path: []const u8, +}; + +pub const Options = struct { + /// This is `null` when -fno-emit-bin is used. When `openPath` or `flush` is called, + /// it will have already been null-checked. + emit: ?Emit, target: std.Target, output_mode: std.builtin.OutputMode, link_mode: std.builtin.LinkMode, @@ -141,7 +147,7 @@ pub const File = struct { /// and does not cause Illegal Behavior. This operation is not atomic. pub fn openPath(allocator: *Allocator, options: Options) !*File { const use_stage1 = build_options.is_stage1 and options.use_llvm; - if (use_stage1) { + if (use_stage1 or options.emit == null) { return switch (options.object_format) { .coff, .pe => &(try Coff.createEmpty(allocator, options)).base, .elf => &(try Elf.createEmpty(allocator, options)).base, @@ -152,6 +158,7 @@ pub const File = struct { .raw => return error.RawObjectFormatUnimplemented, }; } + const emit = options.emit.?; const use_lld = build_options.have_llvm and options.use_lld; // comptime known false when !have_llvm const sub_path = if (use_lld) blk: { if (options.module == null) { @@ -167,8 +174,8 @@ pub const File = struct { }; } // Open a temporary object file, not the final output file because we want to link with LLD. - break :blk try std.fmt.allocPrint(allocator, "{}{}", .{ options.sub_path, options.target.oFileExt() }); - } else options.sub_path; + break :blk try std.fmt.allocPrint(allocator, "{}{}", .{ emit.sub_path, options.target.oFileExt() }); + } else emit.sub_path; errdefer if (use_lld) allocator.free(sub_path); const file: *File = switch (options.object_format) { @@ -199,7 +206,8 @@ pub const File = struct { switch (base.tag) { .coff, .elf, .macho => { if (base.file != null) return; - base.file = try base.options.directory.handle.createFile(base.options.sub_path, .{ + const emit = base.options.emit orelse return; + base.file = try emit.directory.handle.createFile(emit.sub_path, .{ .truncate = false, .read = true, .mode = determineMode(base.options), @@ -305,14 +313,14 @@ pub const File = struct { /// Commit pending changes and write headers. Takes into account final output mode /// and `use_lld`, not only `effectiveOutputMode`. pub fn flush(base: *File, comp: *Compilation) !void { + const emit = base.options.emit orelse return; // -fno-emit-bin + if (comp.clang_preprocessor_mode == .yes) { // TODO: avoid extra link step when it's just 1 object file (the `zig cc -c` case) // Until then, we do `lld -r -o output.o input.o` even though the output is the same // as the input. For the preprocessing case (`zig cc -E -o foo`) we copy the file // to the final location. - const full_out_path = try base.options.directory.join(comp.gpa, &[_][]const u8{ - base.options.sub_path, - }); + const full_out_path = try emit.directory.join(comp.gpa, &[_][]const u8{emit.sub_path}); defer comp.gpa.free(full_out_path); assert(comp.c_object_table.count() == 1); const the_entry = comp.c_object_table.items()[0]; @@ -402,7 +410,7 @@ pub const File = struct { defer arena_allocator.deinit(); const arena = &arena_allocator.allocator; - const directory = base.options.directory; // Just an alias to make it shorter to type. + const directory = base.options.emit.?.directory; // Just an alias to make it shorter to type. // If there is no Zig code to compile, then we should skip flushing the output file because it // will not be part of the linker line anyway. @@ -471,10 +479,7 @@ pub const File = struct { object_files.appendAssumeCapacity(try arena.dupeZ(u8, p)); } - const full_out_path = if (directory.path) |dir_path| - try std.fs.path.join(arena, &[_][]const u8{ dir_path, base.options.sub_path }) - else - base.options.sub_path; + const full_out_path = try directory.join(arena, &[_][]const u8{base.options.emit.?.sub_path}); const full_out_path_z = try arena.dupeZ(u8, full_out_path); if (base.options.verbose_link) { diff --git a/src/link/C.zig b/src/link/C.zig index d5d1249244..467e10998e 100644 --- a/src/link/C.zig +++ b/src/link/C.zig @@ -30,7 +30,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio if (options.use_llvm) return error.LLVMHasNoCBackend; if (options.use_lld) return error.LLDHasNoCBackend; - const file = try options.directory.handle.createFile(sub_path, .{ .truncate = true, .read = true, .mode = link.determineMode(options) }); + const file = try options.emit.?.directory.handle.createFile(sub_path, .{ .truncate = true, .read = true, .mode = link.determineMode(options) }); errdefer file.close(); var c_file = try allocator.create(C); diff --git a/src/link/Coff.zig b/src/link/Coff.zig index 31726a5712..c396732bc1 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -120,7 +120,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio if (options.use_llvm) return error.LLVM_BackendIsTODO_ForCoff; // TODO if (options.use_lld) return error.LLD_LinkingIsTODO_ForCoff; // TODO - const file = try options.directory.handle.createFile(sub_path, .{ + const file = try options.emit.?.directory.handle.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options), diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 903cb57d03..c8067058d9 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -229,7 +229,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio if (options.use_llvm) return error.LLVMBackendUnimplementedForELF; // TODO - const file = try options.directory.handle.createFile(sub_path, .{ + const file = try options.emit.?.directory.handle.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options), @@ -1218,7 +1218,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { defer arena_allocator.deinit(); const arena = &arena_allocator.allocator; - const directory = self.base.options.directory; // Just an alias to make it shorter to type. + const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type. // If there is no Zig code to compile, then we should skip flushing the output file because it // will not be part of the linker line anyway. @@ -1401,7 +1401,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { try argv.append("-pie"); } - const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.sub_path}); + const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path}); try argv.append("-o"); try argv.append(full_out_path); diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 3b70a0d710..c38c6b34ff 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -142,7 +142,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio if (options.use_llvm) return error.LLVM_BackendIsTODO_ForMachO; // TODO if (options.use_lld) return error.LLD_LinkingIsTODO_ForMachO; // TODO - const file = try options.directory.handle.createFile(sub_path, .{ + const file = try options.emit.?.directory.handle.createFile(sub_path, .{ .truncate = false, .read = true, .mode = link.determineMode(options), diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index 1160e471fe..4cff09ef69 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -59,7 +59,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio if (options.use_lld) return error.LLD_LinkingIsTODO_ForWasm; // TODO // TODO: read the file and keep vaild parts instead of truncating - const file = try options.directory.handle.createFile(sub_path, .{ .truncate = true, .read = true }); + const file = try options.emit.?.directory.handle.createFile(sub_path, .{ .truncate = true, .read = true }); errdefer file.close(); const wasm = try createEmpty(allocator, options); diff --git a/src/main.zig b/src/main.zig index 30338f11a1..5c1f0b82e7 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1307,7 +1307,9 @@ fn buildOutputType( }; } if (fs.path.dirname(full_path)) |dirname| { - const handle = try fs.cwd().openDir(dirname, .{}); + const handle = fs.cwd().openDir(dirname, .{}) catch |err| { + fatal("unable to open output directory '{}': {}", .{ dirname, @errorName(err) }); + }; cleanup_emit_bin_dir = handle; break :b Compilation.EmitLoc{ .basename = basename, @@ -1545,7 +1547,7 @@ fn buildOutputType( switch (emit_bin) { .no => break :blk .none, .yes_default_path => break :blk .{ - .print = comp.bin_file.options.directory.path orelse ".", + .print = comp.bin_file.options.emit.?.directory.path orelse ".", }, .yes => |full_path| break :blk .{ .update = full_path }, } @@ -1560,7 +1562,7 @@ fn buildOutputType( switch (arg_mode) { .run, .zig_test => run: { const exe_loc = emit_bin_loc orelse break :run; - const exe_directory = exe_loc.directory orelse comp.bin_file.options.directory; + const exe_directory = exe_loc.directory orelse comp.bin_file.options.emit.?.directory; const exe_path = try fs.path.join(arena, &[_][]const u8{ exe_directory.path orelse ".", exe_loc.basename, }); @@ -1676,8 +1678,8 @@ fn updateModule(gpa: *Allocator, comp: *Compilation, zir_out_path: ?[]const u8, } else switch (hook) { .none => {}, .print => |bin_path| try io.getStdOut().writer().print("{s}\n", .{bin_path}), - .update => |full_path| _ = try comp.bin_file.options.directory.handle.updateFile( - comp.bin_file.options.sub_path, + .update => |full_path| _ = try comp.bin_file.options.emit.?.directory.handle.updateFile( + comp.bin_file.options.emit.?.sub_path, fs.cwd(), full_path, .{}, @@ -2106,7 +2108,10 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v try updateModule(gpa, comp, null, .none); - child_argv.items[argv_index_exe] = try comp.bin_file.options.directory.join(arena, &[_][]const u8{exe_basename}); + child_argv.items[argv_index_exe] = try comp.bin_file.options.emit.?.directory.join( + arena, + &[_][]const u8{exe_basename}, + ); break :lock_and_argv .{ .child_argv = child_argv.items, diff --git a/test/cli.zig b/test/cli.zig index c92636fb3c..edb87cc750 100644 --- a/test/cli.zig +++ b/test/cli.zig @@ -58,7 +58,7 @@ fn printCmd(cwd: []const u8, argv: []const []const u8) void { std.debug.warn("\n", .{}); } -fn exec(cwd: []const u8, argv: []const []const u8) !ChildProcess.ExecResult { +fn exec(cwd: []const u8, expect_0: bool, argv: []const []const u8) !ChildProcess.ExecResult { const max_output_size = 100 * 1024; const result = ChildProcess.exec(.{ .allocator = a, @@ -72,7 +72,7 @@ fn exec(cwd: []const u8, argv: []const []const u8) !ChildProcess.ExecResult { }; switch (result.term) { .Exited => |code| { - if (code != 0) { + if ((code != 0) == expect_0) { std.debug.warn("The following command exited with error code {}:\n", .{code}); printCmd(cwd, argv); std.debug.warn("stderr:\n{}\n", .{result.stderr}); @@ -90,14 +90,14 @@ fn exec(cwd: []const u8, argv: []const []const u8) !ChildProcess.ExecResult { } fn testZigInitLib(zig_exe: []const u8, dir_path: []const u8) !void { - _ = try exec(dir_path, &[_][]const u8{ zig_exe, "init-lib" }); - const test_result = try exec(dir_path, &[_][]const u8{ zig_exe, "build", "test" }); + _ = try exec(dir_path, true, &[_][]const u8{ zig_exe, "init-lib" }); + const test_result = try exec(dir_path, true, &[_][]const u8{ zig_exe, "build", "test" }); testing.expect(std.mem.endsWith(u8, test_result.stderr, "All 1 tests passed.\n")); } fn testZigInitExe(zig_exe: []const u8, dir_path: []const u8) !void { - _ = try exec(dir_path, &[_][]const u8{ zig_exe, "init-exe" }); - const run_result = try exec(dir_path, &[_][]const u8{ zig_exe, "build", "run" }); + _ = try exec(dir_path, true, &[_][]const u8{ zig_exe, "init-exe" }); + const run_result = try exec(dir_path, true, &[_][]const u8{ zig_exe, "build", "run" }); testing.expect(std.mem.eql(u8, run_result.stderr, "info: All your codebase are belong to us.\n")); } @@ -131,7 +131,7 @@ fn testGodboltApi(zig_exe: []const u8, dir_path: []const u8) anyerror!void { const emit_asm_arg = try std.fmt.allocPrint(a, "-femit-asm={s}", .{example_s_path}); try args.append(emit_asm_arg); - _ = try exec(dir_path, args.items); + _ = try exec(dir_path, true, args.items); const out_asm = try std.fs.cwd().readFileAlloc(a, example_s_path, std.math.maxInt(usize)); testing.expect(std.mem.indexOf(u8, out_asm, "square:") != null); @@ -140,23 +140,23 @@ fn testGodboltApi(zig_exe: []const u8, dir_path: []const u8) anyerror!void { } fn testMissingOutputPath(zig_exe: []const u8, dir_path: []const u8) !void { - _ = try exec(dir_path, &[_][]const u8{ zig_exe, "init-exe" }); - const output_path = try fs.path.join(a, &[_][]const u8{ "does", "not", "exist" }); + _ = try exec(dir_path, true, &[_][]const u8{ zig_exe, "init-exe" }); + const output_path = try fs.path.join(a, &[_][]const u8{ "does", "not", "exist", "foo.exe" }); + const output_arg = try std.fmt.allocPrint(a, "-femit-bin={s}", .{output_path}); const source_path = try fs.path.join(a, &[_][]const u8{ "src", "main.zig" }); - _ = try exec(dir_path, &[_][]const u8{ - zig_exe, "build-exe", source_path, "--output-dir", output_path, - }); + const result = try exec(dir_path, false, &[_][]const u8{ zig_exe, "build-exe", source_path, output_arg }); + testing.expect(std.mem.eql(u8, result.stderr, "error: unable to open output directory 'does/not/exist': FileNotFound\n")); } fn testZigFmt(zig_exe: []const u8, dir_path: []const u8) !void { - _ = try exec(dir_path, &[_][]const u8{ zig_exe, "init-exe" }); + _ = try exec(dir_path, true, &[_][]const u8{ zig_exe, "init-exe" }); const unformatted_code = " // no reason for indent"; const fmt1_zig_path = try fs.path.join(a, &[_][]const u8{ dir_path, "fmt1.zig" }); try fs.cwd().writeFile(fmt1_zig_path, unformatted_code); - const run_result1 = try exec(dir_path, &[_][]const u8{ zig_exe, "fmt", fmt1_zig_path }); + const run_result1 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", fmt1_zig_path }); // stderr should be file path + \n testing.expect(std.mem.startsWith(u8, run_result1.stderr, fmt1_zig_path)); testing.expect(run_result1.stderr.len == fmt1_zig_path.len + 1 and run_result1.stderr[run_result1.stderr.len - 1] == '\n'); @@ -164,12 +164,12 @@ fn testZigFmt(zig_exe: []const u8, dir_path: []const u8) !void { const fmt2_zig_path = try fs.path.join(a, &[_][]const u8{ dir_path, "fmt2.zig" }); try fs.cwd().writeFile(fmt2_zig_path, unformatted_code); - const run_result2 = try exec(dir_path, &[_][]const u8{ zig_exe, "fmt", dir_path }); + const run_result2 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path }); // running it on the dir, only the new file should be changed testing.expect(std.mem.startsWith(u8, run_result2.stderr, fmt2_zig_path)); testing.expect(run_result2.stderr.len == fmt2_zig_path.len + 1 and run_result2.stderr[run_result2.stderr.len - 1] == '\n'); - const run_result3 = try exec(dir_path, &[_][]const u8{ zig_exe, "fmt", dir_path }); + const run_result3 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path }); // both files have been formatted, nothing should change now testing.expect(run_result3.stderr.len == 0); } From fe4c348f578903d53c490673b1b1dcf5513ae049 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 26 Sep 2020 16:48:02 -0700 Subject: [PATCH 156/210] stage2: `zig translate-c` supports --enable-cache This matches master branch behavior and makes the test-translate-c tests pass. --- BRANCH_TODO | 1 - src/Compilation.zig | 6 +- src/main.zig | 172 ++++++++++++++++++++++++++++++-------------- 3 files changed, 124 insertions(+), 55 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index d1d2104c1e..b224b2a7fe 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,5 +1,4 @@ * restore the legacy -femit-h feature using the stage1 backend - * figure out why test-translate-c is failing * tests passing with -Dskip-non-native * `-ftime-report` * -fstack-report print stack size diagnostics\n" diff --git a/src/Compilation.zig b/src/Compilation.zig index 7385825de3..fc042cdd47 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1251,7 +1251,7 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { }; } -fn obtainCObjectCacheManifest(comp: *Compilation) Cache.Manifest { +pub fn obtainCObjectCacheManifest(comp: *Compilation) Cache.Manifest { var man = comp.cache_parent.obtain(); // Only things that need to be added on top of the base hash, and only things @@ -1289,6 +1289,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult { var man = comp.obtainCObjectCacheManifest(); defer man.deinit(); + man.hash.add(@as(u16, 0xb945)); // Random number to distinguish translate-c from compiling C objects man.hash.addBytes(c_src); // If the previous invocation resulted in clang errors, we will see a hit @@ -1581,7 +1582,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { }; } -fn tmpFilePath(comp: *Compilation, arena: *Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 { +pub fn tmpFilePath(comp: *Compilation, arena: *Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 { const s = std.fs.path.sep_str; const rand_int = comp.rand.int(u64); if (comp.local_cache_directory.path) |p| { @@ -1598,6 +1599,7 @@ pub fn addTranslateCCArgs( ext: FileExt, out_dep_path: ?[]const u8, ) !void { + try argv.appendSlice(&[_][]const u8{ "-x", "c" }); try comp.addCCArgs(arena, argv, ext, out_dep_path); // This gives us access to preprocessing entities, presumably at the cost of performance. try argv.appendSlice(&[_][]const u8{ "-Xclang", "-detailed-preprocessing-record" }); diff --git a/src/main.zig b/src/main.zig index 5c1f0b82e7..572ab4b7b0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -16,6 +16,7 @@ const warn = std.log.warn; const introspect = @import("introspect.zig"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const translate_c = @import("translate_c.zig"); +const Cache = @import("Cache.zig"); pub fn fatal(comptime format: []const u8, args: anytype) noreturn { std.log.emerg(format, args); @@ -481,12 +482,19 @@ fn buildOutputType( switch (arg_mode) { .build, .translate_c, .zig_test, .run => { var optimize_mode_string: ?[]const u8 = null; - output_mode = switch (arg_mode) { - .build => |m| m, - .translate_c => .Obj, - .zig_test, .run => .Exe, + switch (arg_mode) { + .build => |m| { + output_mode = m; + }, + .translate_c => { + emit_bin = .no; + output_mode = .Obj; + }, + .zig_test, .run => { + output_mode = .Exe; + }, else => unreachable, - }; + } // TODO finish self-hosted and add support for emitting C header files emit_h = .no; //switch (arg_mode) { @@ -1537,7 +1545,7 @@ fn buildOutputType( return std.io.getStdOut().writeAll(try comp.generateBuiltinZigSource(arena)); } if (arg_mode == .translate_c) { - return cmdTranslateC(comp, arena); + return cmdTranslateC(comp, arena, have_enable_cache); } const hook: AfterUpdateHook = blk: { @@ -1556,7 +1564,7 @@ fn buildOutputType( try updateModule(gpa, comp, zir_out_path, hook); if (build_options.is_stage1 and comp.stage1_lock != null and watch) { - std.log.warn("--watch is not recommended with the stage1 backend; it leaks memory and is not capable of incremental compilation", .{}); + warn("--watch is not recommended with the stage1 backend; it leaks memory and is not capable of incremental compilation", .{}); } switch (arg_mode) { @@ -1701,61 +1709,121 @@ fn updateModule(gpa: *Allocator, comp: *Compilation, zir_out_path: ?[]const u8, } } -fn cmdTranslateC(comp: *Compilation, arena: *Allocator) !void { +fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !void { if (!build_options.have_llvm) fatal("cannot translate-c: compiler built without LLVM extensions", .{}); assert(comp.c_source_files.len == 1); - - var argv = std.ArrayList([]const u8).init(arena); - const c_source_file = comp.c_source_files[0]; - const file_ext = Compilation.classifyFileExt(c_source_file.src_path); - try comp.addTranslateCCArgs(arena, &argv, file_ext, null); - try argv.append(c_source_file.src_path); - if (comp.verbose_cc) { - std.debug.print("clang ", .{}); - Compilation.dump_argv(argv.items); - } + const translated_zig_basename = try std.fmt.allocPrint(arena, "{}.zig", .{comp.bin_file.options.root_name}); - // Convert to null terminated args. - const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1); - new_argv_with_sentinel[argv.items.len] = null; - const new_argv = new_argv_with_sentinel[0..argv.items.len :null]; - for (argv.items) |arg, i| { - new_argv[i] = try arena.dupeZ(u8, arg); - } + var man: Cache.Manifest = comp.obtainCObjectCacheManifest(); + defer if (enable_cache) man.deinit(); - const c_headers_dir_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"include"}); - const c_headers_dir_path_z = try arena.dupeZ(u8, c_headers_dir_path); - var clang_errors: []translate_c.ClangErrMsg = &[0]translate_c.ClangErrMsg{}; - const tree = translate_c.translate( - comp.gpa, - new_argv.ptr, - new_argv.ptr + new_argv.len, - &clang_errors, - c_headers_dir_path_z, - ) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ASTUnitFailure => fatal("clang API returned errors but due to a clang bug, it is not exposing the errors for zig to see. For more details: https://github.com/ziglang/zig/issues/4455", .{}), - error.SemanticAnalyzeFail => { - for (clang_errors) |clang_err| { - std.debug.print("{}:{}:{}: {}\n", .{ - if (clang_err.filename_ptr) |p| p[0..clang_err.filename_len] else "(no file)", - clang_err.line + 1, - clang_err.column + 1, - clang_err.msg_ptr[0..clang_err.msg_len], - }); - } - process.exit(1); - }, + man.hash.add(@as(u16, 0xb945)); // Random number to distinguish translate-c from compiling C objects + _ = man.addFile(c_source_file.src_path, null) catch |err| { + fatal("unable to process '{}': {}", .{ c_source_file.src_path, @errorName(err) }); }; - defer tree.deinit(); - var bos = io.bufferedOutStream(io.getStdOut().writer()); - _ = try std.zig.render(comp.gpa, bos.writer(), tree); - try bos.flush(); + const digest = if (try man.hit()) man.final() else digest: { + var argv = std.ArrayList([]const u8).init(arena); + + var zig_cache_tmp_dir = try comp.local_cache_directory.handle.makeOpenPath("tmp", .{}); + defer zig_cache_tmp_dir.close(); + + const ext = Compilation.classifyFileExt(c_source_file.src_path); + const out_dep_path: ?[]const u8 = blk: { + if (comp.disable_c_depfile or !ext.clangSupportsDepFile()) + break :blk null; + + const c_src_basename = fs.path.basename(c_source_file.src_path); + const dep_basename = try std.fmt.allocPrint(arena, "{}.d", .{c_src_basename}); + const out_dep_path = try comp.tmpFilePath(arena, dep_basename); + break :blk out_dep_path; + }; + + try comp.addTranslateCCArgs(arena, &argv, ext, out_dep_path); + try argv.append(c_source_file.src_path); + + if (comp.verbose_cc) { + std.debug.print("clang ", .{}); + Compilation.dump_argv(argv.items); + } + + // Convert to null terminated args. + const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1); + new_argv_with_sentinel[argv.items.len] = null; + const new_argv = new_argv_with_sentinel[0..argv.items.len :null]; + for (argv.items) |arg, i| { + new_argv[i] = try arena.dupeZ(u8, arg); + } + + const c_headers_dir_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"include"}); + const c_headers_dir_path_z = try arena.dupeZ(u8, c_headers_dir_path); + var clang_errors: []translate_c.ClangErrMsg = &[0]translate_c.ClangErrMsg{}; + const tree = translate_c.translate( + comp.gpa, + new_argv.ptr, + new_argv.ptr + new_argv.len, + &clang_errors, + c_headers_dir_path_z, + ) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ASTUnitFailure => fatal("clang API returned errors but due to a clang bug, it is not exposing the errors for zig to see. For more details: https://github.com/ziglang/zig/issues/4455", .{}), + error.SemanticAnalyzeFail => { + for (clang_errors) |clang_err| { + std.debug.print("{}:{}:{}: {}\n", .{ + if (clang_err.filename_ptr) |p| p[0..clang_err.filename_len] else "(no file)", + clang_err.line + 1, + clang_err.column + 1, + clang_err.msg_ptr[0..clang_err.msg_len], + }); + } + process.exit(1); + }, + }; + defer tree.deinit(); + + if (out_dep_path) |dep_file_path| { + const dep_basename = std.fs.path.basename(dep_file_path); + // Add the files depended on to the cache system. + try man.addDepFilePost(zig_cache_tmp_dir, dep_basename); + // Just to save disk space, we delete the file because it is never needed again. + zig_cache_tmp_dir.deleteFile(dep_basename) catch |err| { + warn("failed to delete '{}': {}", .{ dep_file_path, @errorName(err) }); + }; + } + + const digest = man.final(); + const o_sub_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest }); + var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{}); + defer o_dir.close(); + var zig_file = try o_dir.createFile(translated_zig_basename, .{}); + defer zig_file.close(); + + var bos = io.bufferedOutStream(zig_file.writer()); + _ = try std.zig.render(comp.gpa, bos.writer(), tree); + try bos.flush(); + + man.writeManifest() catch |err| warn("failed to write cache manifest: {}", .{@errorName(err)}); + + break :digest digest; + }; + + if (enable_cache) { + const full_zig_path = try comp.local_cache_directory.join(arena, &[_][]const u8{ + "o", &digest, translated_zig_basename, + }); + try io.getStdOut().writer().print("{}\n", .{full_zig_path}); + return cleanExit(); + } else { + const out_zig_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest, translated_zig_basename }); + const zig_file = try comp.local_cache_directory.handle.openFile(out_zig_path, .{}); + defer zig_file.close(); + try io.getStdOut().writeFileAll(zig_file, .{}); + return cleanExit(); + } } pub const usage_libc = From b6556c944b88726c2bdb34ce72358ade88c5a984 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 26 Sep 2020 21:03:38 -0700 Subject: [PATCH 157/210] fix another round of regressions in this branch * std.log: still print error messages in ReleaseSmall builds. - when start code gets an error code from main, it uses std.log.err to report the error. this resulted in a test failure because ReleaseSmall wasn't printing `error: TheErrorCode` when an error was returned from main. But that seems like it should keep working. So I changed the std.log defaults. I plan to follow this up with a proposal to change the names of and reduce the quantity of the log levels. * warning emitted when using -femit-h when using stage1 backend; fatal log message when using -femit-h with self-hosted backend (because the feature is not yet available) * fix double `test-cli` build steps in zig's build.zig * update docgen to use new CLI * translate-c uses `-x c` and generates a temporary basename with a `.h` extension. Otherwise clang reports an error. * --show-builtin implies -fno-emit-bin * restore the compile error for using an extern "c" function without putting -lc on the build line. we have to know about the libc dependency up front. * Fix ReleaseFast and ReleaseSmall getting swapped when passing the value to the stage1 backend. * correct the zig0 CLI usage text. * update test harness code to the new CLI. --- BRANCH_TODO | 10 +- build.zig | 5 +- doc/docgen.zig | 247 ++++++++++++++-------------------------- doc/langref.html.in | 1 + lib/std/log.zig | 8 +- src/Compilation.zig | 13 ++- src/main.zig | 65 +---------- src/stage1.zig | 26 ++++- src/stage1/stage1.h | 2 +- src/stage1/zig0.cpp | 2 +- src/target.zig | 56 +++++++++ test/compile_errors.zig | 2 +- test/tests.zig | 49 ++++---- 13 files changed, 213 insertions(+), 273 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index b224b2a7fe..5306f29746 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,18 +1,15 @@ - * restore the legacy -femit-h feature using the stage1 backend - * tests passing with -Dskip-non-native - * `-ftime-report` - * -fstack-report print stack size diagnostics\n" + * MachO LLD linking * subsystem * mingw-w64 - * MachO LLD linking * COFF LLD linking * WASM LLD linking * audit the CLI options for stage2 * audit the base cache hash * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process. - * restore error messages for stage2_add_link_lib * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj] * try building some software with zig cc to make sure it didn't regress + * `-ftime-report` + * -fstack-report print stack size diagnostics\n" * implement proper parsing of clang stderr/stdout and exposing compile errors with the Compilation API * implement proper parsing of LLD stderr/stdout and exposing compile errors with the Compilation API @@ -51,3 +48,4 @@ * update musl.zig static data to use native path separator in static data rather than replacing '/' at runtime * linking hello world with LLD, lld is silently calling exit(1) instead of reporting ok=false. when run standalone the error message is: ld.lld: error: section [index 3] has a sh_offset (0x57000) + sh_size (0x68) that is greater than the file size (0x57060) * submit PR to godbolt and update the CLI options (see changes to test/cli.zig) + * make proposal about log levels diff --git a/build.zig b/build.zig index df575e3356..b4d8b71f79 100644 --- a/build.zig +++ b/build.zig @@ -211,10 +211,7 @@ pub fn build(b: *Builder) !void { test_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes)); test_step.dependOn(tests.addStandaloneTests(b, test_filter, modes)); test_step.dependOn(tests.addStackTraceTests(b, test_filter, modes)); - const test_cli = tests.addCliTests(b, test_filter, modes); - const test_cli_step = b.step("test-cli", "Run zig cli tests"); - test_cli_step.dependOn(test_cli); - test_step.dependOn(test_cli); + test_step.dependOn(tests.addCliTests(b, test_filter, modes)); test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, modes)); test_step.dependOn(tests.addRuntimeSafetyTests(b, test_filter, modes)); test_step.dependOn(tests.addTranslateCTests(b, test_filter)); diff --git a/doc/docgen.zig b/doc/docgen.zig index af4d2530d0..50523d0948 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -4,7 +4,7 @@ const io = std.io; const fs = std.fs; const process = std.process; const ChildProcess = std.ChildProcess; -const warn = std.debug.warn; +const print = std.debug.print; const mem = std.mem; const testing = std.testing; @@ -215,23 +215,23 @@ const Tokenizer = struct { fn parseError(tokenizer: *Tokenizer, token: Token, comptime fmt: []const u8, args: anytype) anyerror { const loc = tokenizer.getTokenLocation(token); const args_prefix = .{ tokenizer.source_file_name, loc.line + 1, loc.column + 1 }; - warn("{}:{}:{}: error: " ++ fmt ++ "\n", args_prefix ++ args); + print("{}:{}:{}: error: " ++ fmt ++ "\n", args_prefix ++ args); if (loc.line_start <= loc.line_end) { - warn("{}\n", .{tokenizer.buffer[loc.line_start..loc.line_end]}); + print("{}\n", .{tokenizer.buffer[loc.line_start..loc.line_end]}); { var i: usize = 0; while (i < loc.column) : (i += 1) { - warn(" ", .{}); + print(" ", .{}); } } { const caret_count = token.end - token.start; var i: usize = 0; while (i < caret_count) : (i += 1) { - warn("~", .{}); + print("~", .{}); } } - warn("\n", .{}); + print("\n", .{}); } return error.ParseError; } @@ -274,6 +274,7 @@ const Code = struct { link_objects: []const []const u8, target_str: ?[]const u8, link_libc: bool, + disable_cache: bool, const Id = union(enum) { Test, @@ -522,6 +523,7 @@ fn genToc(allocator: *mem.Allocator, tokenizer: *Tokenizer) !Toc { defer link_objects.deinit(); var target_str: ?[]const u8 = null; var link_libc = false; + var disable_cache = false; const source_token = while (true) { const content_tok = try eatToken(tokenizer, Token.Id.Content); @@ -532,6 +534,8 @@ fn genToc(allocator: *mem.Allocator, tokenizer: *Tokenizer) !Toc { mode = .ReleaseFast; } else if (mem.eql(u8, end_tag_name, "code_release_safe")) { mode = .ReleaseSafe; + } else if (mem.eql(u8, end_tag_name, "code_disable_cache")) { + disable_cache = true; } else if (mem.eql(u8, end_tag_name, "code_link_object")) { _ = try eatToken(tokenizer, Token.Id.Separator); const obj_tok = try eatToken(tokenizer, Token.Id.TagContent); @@ -572,6 +576,7 @@ fn genToc(allocator: *mem.Allocator, tokenizer: *Tokenizer) !Toc { .link_objects = link_objects.toOwnedSlice(), .target_str = target_str, .link_libc = link_libc, + .disable_cache = disable_cache, }, }); tokenizer.code_node_count += 1; @@ -1032,7 +1037,7 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any }, .Code => |code| { code_progress_index += 1; - warn("docgen example code {}/{}...", .{ code_progress_index, tokenizer.code_node_count }); + print("docgen example code {}/{}...", .{ code_progress_index, tokenizer.code_node_count }); const raw_source = tokenizer.buffer[code.source_token.start..code.source_token.end]; const trimmed_raw_source = mem.trim(u8, raw_source, " \n"); @@ -1055,30 +1060,17 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any var build_args = std.ArrayList([]const u8).init(allocator); defer build_args.deinit(); try build_args.appendSlice(&[_][]const u8{ - zig_exe, - "build-exe", - tmp_source_file_name, - "--name", - code.name, - "--color", - "on", - "--cache", - "on", + zig_exe, "build-exe", + "--name", code.name, + "--color", "on", + "--enable-cache", tmp_source_file_name, }); try out.print("
    $ zig build-exe {}.zig", .{code.name});
                         switch (code.mode) {
                             .Debug => {},
-                            .ReleaseSafe => {
-                                try build_args.append("--release-safe");
-                                try out.print(" --release-safe", .{});
-                            },
-                            .ReleaseFast => {
-                                try build_args.append("--release-fast");
-                                try out.print(" --release-fast", .{});
-                            },
-                            .ReleaseSmall => {
-                                try build_args.append("--release-small");
-                                try out.print(" --release-small", .{});
+                            else => {
+                                try build_args.appendSlice(&[_][]const u8{ "-O", @tagName(code.mode) });
+                                try out.print(" -O {s}", .{@tagName(code.mode)});
                             },
                         }
                         for (code.link_objects) |link_object| {
@@ -1087,9 +1079,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                                 allocator,
                                 &[_][]const u8{ tmp_dir_name, name_with_ext },
                             );
-                            try build_args.append("--object");
                             try build_args.append(full_path_object);
-                            try out.print(" --object {}", .{name_with_ext});
+                            try out.print(" {s}", .{name_with_ext});
                         }
                         if (code.link_libc) {
                             try build_args.append("-lc");
@@ -1114,20 +1105,14 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                             switch (result.term) {
                                 .Exited => |exit_code| {
                                     if (exit_code == 0) {
-                                        warn("{}\nThe following command incorrectly succeeded:\n", .{result.stderr});
-                                        for (build_args.items) |arg|
-                                            warn("{} ", .{arg})
-                                        else
-                                            warn("\n", .{});
+                                        print("{}\nThe following command incorrectly succeeded:\n", .{result.stderr});
+                                        dumpArgs(build_args.items);
                                         return parseError(tokenizer, code.source_token, "example incorrectly compiled", .{});
                                     }
                                 },
                                 else => {
-                                    warn("{}\nThe following command crashed:\n", .{result.stderr});
-                                    for (build_args.items) |arg|
-                                        warn("{} ", .{arg})
-                                    else
-                                        warn("\n", .{});
+                                    print("{}\nThe following command crashed:\n", .{result.stderr});
+                                    dumpArgs(build_args.items);
                                     return parseError(tokenizer, code.source_token, "example compile crashed", .{});
                                 },
                             }
@@ -1174,11 +1159,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                             switch (result.term) {
                                 .Exited => |exit_code| {
                                     if (exit_code == 0) {
-                                        warn("{}\nThe following command incorrectly succeeded:\n", .{result.stderr});
-                                        for (run_args) |arg|
-                                            warn("{} ", .{arg})
-                                        else
-                                            warn("\n", .{});
+                                        print("{}\nThe following command incorrectly succeeded:\n", .{result.stderr});
+                                        dumpArgs(run_args);
                                         return parseError(tokenizer, code.source_token, "example incorrectly compiled", .{});
                                     }
                                 },
@@ -1206,27 +1188,13 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                         var test_args = std.ArrayList([]const u8).init(allocator);
                         defer test_args.deinit();
 
-                        try test_args.appendSlice(&[_][]const u8{
-                            zig_exe,
-                            "test",
-                            tmp_source_file_name,
-                            "--cache",
-                            "on",
-                        });
+                        try test_args.appendSlice(&[_][]const u8{ zig_exe, "test", tmp_source_file_name });
                         try out.print("
    $ zig test {}.zig", .{code.name});
                         switch (code.mode) {
                             .Debug => {},
-                            .ReleaseSafe => {
-                                try test_args.append("--release-safe");
-                                try out.print(" --release-safe", .{});
-                            },
-                            .ReleaseFast => {
-                                try test_args.append("--release-fast");
-                                try out.print(" --release-fast", .{});
-                            },
-                            .ReleaseSmall => {
-                                try test_args.append("--release-small");
-                                try out.print(" --release-small", .{});
+                            else => {
+                                try test_args.appendSlice(&[_][]const u8{ "-O", @tagName(code.mode) });
+                                try out.print(" -O {s}", .{@tagName(code.mode)});
                             },
                         }
                         if (code.link_libc) {
@@ -1252,23 +1220,13 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                             "--color",
                             "on",
                             tmp_source_file_name,
-                            "--output-dir",
-                            tmp_dir_name,
                         });
                         try out.print("
    $ zig test {}.zig", .{code.name});
                         switch (code.mode) {
                             .Debug => {},
-                            .ReleaseSafe => {
-                                try test_args.append("--release-safe");
-                                try out.print(" --release-safe", .{});
-                            },
-                            .ReleaseFast => {
-                                try test_args.append("--release-fast");
-                                try out.print(" --release-fast", .{});
-                            },
-                            .ReleaseSmall => {
-                                try test_args.append("--release-small");
-                                try out.print(" --release-small", .{});
+                            else => {
+                                try test_args.appendSlice(&[_][]const u8{ "-O", @tagName(code.mode) });
+                                try out.print(" -O {s}", .{@tagName(code.mode)});
                             },
                         }
                         const result = try ChildProcess.exec(.{
@@ -1280,25 +1238,19 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                         switch (result.term) {
                             .Exited => |exit_code| {
                                 if (exit_code == 0) {
-                                    warn("{}\nThe following command incorrectly succeeded:\n", .{result.stderr});
-                                    for (test_args.items) |arg|
-                                        warn("{} ", .{arg})
-                                    else
-                                        warn("\n", .{});
+                                    print("{}\nThe following command incorrectly succeeded:\n", .{result.stderr});
+                                    dumpArgs(test_args.items);
                                     return parseError(tokenizer, code.source_token, "example incorrectly compiled", .{});
                                 }
                             },
                             else => {
-                                warn("{}\nThe following command crashed:\n", .{result.stderr});
-                                for (test_args.items) |arg|
-                                    warn("{} ", .{arg})
-                                else
-                                    warn("\n", .{});
+                                print("{}\nThe following command crashed:\n", .{result.stderr});
+                                dumpArgs(test_args.items);
                                 return parseError(tokenizer, code.source_token, "example compile crashed", .{});
                             },
                         }
                         if (mem.indexOf(u8, result.stderr, error_match) == null) {
-                            warn("{}\nExpected to find '{}' in stderr", .{ result.stderr, error_match });
+                            print("{}\nExpected to find '{}' in stderr", .{ result.stderr, error_match });
                             return parseError(tokenizer, code.source_token, "example did not have expected compile error", .{});
                         }
                         const escaped_stderr = try escapeHtml(allocator, result.stderr);
@@ -1314,23 +1266,21 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                             zig_exe,
                             "test",
                             tmp_source_file_name,
-                            "--output-dir",
-                            tmp_dir_name,
                         });
                         var mode_arg: []const u8 = "";
                         switch (code.mode) {
                             .Debug => {},
                             .ReleaseSafe => {
-                                try test_args.append("--release-safe");
-                                mode_arg = " --release-safe";
+                                try test_args.append("-OReleaseSafe");
+                                mode_arg = "-OReleaseSafe";
                             },
                             .ReleaseFast => {
-                                try test_args.append("--release-fast");
-                                mode_arg = " --release-fast";
+                                try test_args.append("-OReleaseFast");
+                                mode_arg = "-OReleaseFast";
                             },
                             .ReleaseSmall => {
-                                try test_args.append("--release-small");
-                                mode_arg = " --release-small";
+                                try test_args.append("-OReleaseSmall");
+                                mode_arg = "-OReleaseSmall";
                             },
                         }
 
@@ -1343,25 +1293,19 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                         switch (result.term) {
                             .Exited => |exit_code| {
                                 if (exit_code == 0) {
-                                    warn("{}\nThe following command incorrectly succeeded:\n", .{result.stderr});
-                                    for (test_args.items) |arg|
-                                        warn("{} ", .{arg})
-                                    else
-                                        warn("\n", .{});
+                                    print("{}\nThe following command incorrectly succeeded:\n", .{result.stderr});
+                                    dumpArgs(test_args.items);
                                     return parseError(tokenizer, code.source_token, "example test incorrectly succeeded", .{});
                                 }
                             },
                             else => {
-                                warn("{}\nThe following command crashed:\n", .{result.stderr});
-                                for (test_args.items) |arg|
-                                    warn("{} ", .{arg})
-                                else
-                                    warn("\n", .{});
+                                print("{}\nThe following command crashed:\n", .{result.stderr});
+                                dumpArgs(test_args.items);
                                 return parseError(tokenizer, code.source_token, "example compile crashed", .{});
                             },
                         }
                         if (mem.indexOf(u8, result.stderr, error_match) == null) {
-                            warn("{}\nExpected to find '{}' in stderr", .{ result.stderr, error_match });
+                            print("{}\nExpected to find '{}' in stderr", .{ result.stderr, error_match });
                             return parseError(tokenizer, code.source_token, "example did not have expected runtime safety error message", .{});
                         }
                         const escaped_stderr = try escapeHtml(allocator, result.stderr);
@@ -1395,32 +1339,20 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                             "on",
                             "--name",
                             code.name,
-                            "--output-dir",
-                            tmp_dir_name,
+                            try std.fmt.allocPrint(allocator, "-femit-bin={s}{c}{s}", .{
+                                tmp_dir_name, fs.path.sep, name_plus_obj_ext,
+                            }),
                         });
-
                         if (!code.is_inline) {
                             try out.print("
    $ zig build-obj {}.zig", .{code.name});
                         }
 
                         switch (code.mode) {
                             .Debug => {},
-                            .ReleaseSafe => {
-                                try build_args.append("--release-safe");
+                            else => {
+                                try build_args.appendSlice(&[_][]const u8{ "-O", @tagName(code.mode) });
                                 if (!code.is_inline) {
-                                    try out.print(" --release-safe", .{});
-                                }
-                            },
-                            .ReleaseFast => {
-                                try build_args.append("--release-fast");
-                                if (!code.is_inline) {
-                                    try out.print(" --release-fast", .{});
-                                }
-                            },
-                            .ReleaseSmall => {
-                                try build_args.append("--release-small");
-                                if (!code.is_inline) {
-                                    try out.print(" --release-small", .{});
+                                    try out.print(" -O {s}", .{@tagName(code.mode)});
                                 }
                             },
                         }
@@ -1440,25 +1372,19 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                             switch (result.term) {
                                 .Exited => |exit_code| {
                                     if (exit_code == 0) {
-                                        warn("{}\nThe following command incorrectly succeeded:\n", .{result.stderr});
-                                        for (build_args.items) |arg|
-                                            warn("{} ", .{arg})
-                                        else
-                                            warn("\n", .{});
+                                        print("{}\nThe following command incorrectly succeeded:\n", .{result.stderr});
+                                        dumpArgs(build_args.items);
                                         return parseError(tokenizer, code.source_token, "example build incorrectly succeeded", .{});
                                     }
                                 },
                                 else => {
-                                    warn("{}\nThe following command crashed:\n", .{result.stderr});
-                                    for (build_args.items) |arg|
-                                        warn("{} ", .{arg})
-                                    else
-                                        warn("\n", .{});
+                                    print("{}\nThe following command crashed:\n", .{result.stderr});
+                                    dumpArgs(build_args.items);
                                     return parseError(tokenizer, code.source_token, "example compile crashed", .{});
                                 },
                             }
                             if (mem.indexOf(u8, result.stderr, error_match) == null) {
-                                warn("{}\nExpected to find '{}' in stderr", .{ result.stderr, error_match });
+                                print("{}\nExpected to find '{}' in stderr", .{ result.stderr, error_match });
                                 return parseError(tokenizer, code.source_token, "example did not have expected compile error message", .{});
                             }
                             const escaped_stderr = try escapeHtml(allocator, result.stderr);
@@ -1472,6 +1398,12 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                         }
                     },
                     Code.Id.Lib => {
+                        const bin_basename = try std.zig.binNameAlloc(allocator, .{
+                            .root_name = code.name,
+                            .target = std.Target.current,
+                            .output_mode = .Lib,
+                        });
+
                         var test_args = std.ArrayList([]const u8).init(allocator);
                         defer test_args.deinit();
 
@@ -1479,23 +1411,16 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                             zig_exe,
                             "build-lib",
                             tmp_source_file_name,
-                            "--output-dir",
-                            tmp_dir_name,
+                            try std.fmt.allocPrint(allocator, "-femit-bin={s}{s}{s}", .{
+                                tmp_dir_name, fs.path.sep_str, bin_basename,
+                            }),
                         });
                         try out.print("
    $ zig build-lib {}.zig", .{code.name});
                         switch (code.mode) {
                             .Debug => {},
-                            .ReleaseSafe => {
-                                try test_args.append("--release-safe");
-                                try out.print(" --release-safe", .{});
-                            },
-                            .ReleaseFast => {
-                                try test_args.append("--release-fast");
-                                try out.print(" --release-fast", .{});
-                            },
-                            .ReleaseSmall => {
-                                try test_args.append("--release-small");
-                                try out.print(" --release-small", .{});
+                            else => {
+                                try test_args.appendSlice(&[_][]const u8{ "-O", @tagName(code.mode) });
+                                try out.print(" -O {s}", .{@tagName(code.mode)});
                             },
                         }
                         if (code.target_str) |triple| {
@@ -1508,7 +1433,7 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: any
                         try out.print("\n{}{}
    \n", .{ escaped_stderr, escaped_stdout });
                     },
                 }
-                warn("OK\n", .{});
+                print("OK\n", .{});
             },
         }
     }
@@ -1524,20 +1449,14 @@ fn exec(allocator: *mem.Allocator, env_map: *std.BufMap, args: []const []const u
     switch (result.term) {
         .Exited => |exit_code| {
             if (exit_code != 0) {
-                warn("{}\nThe following command exited with code {}:\n", .{ result.stderr, exit_code });
-                for (args) |arg|
-                    warn("{} ", .{arg})
-                else
-                    warn("\n", .{});
+                print("{}\nThe following command exited with code {}:\n", .{ result.stderr, exit_code });
+                dumpArgs(args);
                 return error.ChildExitError;
             }
         },
         else => {
-            warn("{}\nThe following command crashed:\n", .{result.stderr});
-            for (args) |arg|
-                warn("{} ", .{arg})
-            else
-                warn("\n", .{});
+            print("{}\nThe following command crashed:\n", .{result.stderr});
+            dumpArgs(args);
             return error.ChildCrashed;
         },
     }
@@ -1545,9 +1464,13 @@ fn exec(allocator: *mem.Allocator, env_map: *std.BufMap, args: []const []const u
 }
 
 fn getBuiltinCode(allocator: *mem.Allocator, env_map: *std.BufMap, zig_exe: []const u8) ![]const u8 {
-    const result = try exec(allocator, env_map, &[_][]const u8{
-        zig_exe,
-        "builtin",
-    });
+    const result = try exec(allocator, env_map, &[_][]const u8{ zig_exe, "build-obj", "--show-builtin" });
     return result.stdout;
 }
+
+fn dumpArgs(args: []const []const u8) void {
+    for (args) |arg|
+        print("{} ", .{arg})
+    else
+        print("\n", .{});
+}
diff --git a/doc/langref.html.in b/doc/langref.html.in
index 740984619b..320b10bbe5 100644
--- a/doc/langref.html.in
+++ b/doc/langref.html.in
@@ -1078,6 +1078,7 @@ const nan = std.math.nan(f128);
           but you can switch to {#syntax#}Optimized{#endsyntax#} mode on a per-block basis:
    
       {#code_begin|obj|foo#}
       {#code_release_fast#}
+      {#code_disable_cache#}
 const std = @import("std");
 const builtin = std.builtin;
 const big = @as(f64, 1 << 40);
diff --git a/lib/std/log.zig b/lib/std/log.zig
index bc83e6053d..0cc2b54452 100644
--- a/lib/std/log.zig
+++ b/lib/std/log.zig
@@ -101,14 +101,12 @@ pub const Level = enum {
     debug,
 };
 
-/// The default log level is based on build mode. Note that in ReleaseSmall
-/// builds the default level is emerg but no messages will be stored/logged
-/// by the default logger to save space.
+/// The default log level is based on build mode.
 pub const default_level: Level = switch (builtin.mode) {
     .Debug => .debug,
     .ReleaseSafe => .notice,
     .ReleaseFast => .err,
-    .ReleaseSmall => .emerg,
+    .ReleaseSmall => .err,
 };
 
 /// The current log level. This is set to root.log_level if present, otherwise
@@ -131,7 +129,7 @@ fn log(
             // On freestanding one must provide a log function; we do not have
             // any I/O configured.
             return;
-        } else if (builtin.mode != .ReleaseSmall) {
+        } else {
             const level_txt = switch (message_level) {
                 .emerg => "emergency",
                 .alert => "alert",
diff --git a/src/Compilation.zig b/src/Compilation.zig
index fc042cdd47..10adfb8c45 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -714,6 +714,10 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             };
         };
 
+        if (!use_llvm and options.emit_h != null) {
+            fatal("TODO implement support for -femit-h in the self-hosted backend", .{});
+        }
+
         const bin_file = try link.File.openPath(gpa, .{
             .emit = bin_file_emit,
             .root_name = root_name,
@@ -1313,13 +1317,13 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
         const tmp_dir_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &tmp_digest });
         var zig_cache_tmp_dir = try comp.local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{});
         defer zig_cache_tmp_dir.close();
-        const cimport_c_basename = "cimport.c";
+        const cimport_basename = "cimport.h";
         const out_h_path = try comp.local_cache_directory.join(arena, &[_][]const u8{
-            tmp_dir_sub_path, cimport_c_basename,
+            tmp_dir_sub_path, cimport_basename,
         });
         const out_dep_path = try std.fmt.allocPrint(arena, "{}.d", .{out_h_path});
 
-        try zig_cache_tmp_dir.writeFile(cimport_c_basename, c_src);
+        try zig_cache_tmp_dir.writeFile(cimport_basename, c_src);
         if (comp.verbose_cimport) {
             log.info("C import source: {}", .{out_h_path});
         }
@@ -2542,6 +2546,9 @@ fn updateStage1Module(comp: *Compilation) !void {
         });
         break :blk try directory.join(arena, &[_][]const u8{bin_basename});
     } else "";
+    if (comp.emit_h != null) {
+        log.warn("-femit-h is not available in the stage1 backend; no .h file will be produced", .{});
+    }
     const emit_h_path = try stage1LocPath(arena, comp.emit_h, directory);
     const emit_asm_path = try stage1LocPath(arena, comp.emit_asm, directory);
     const emit_llvm_ir_path = try stage1LocPath(arena, comp.emit_llvm_ir, directory);
diff --git a/src/main.zig b/src/main.zig
index 572ab4b7b0..31eaf5d8ae 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -7,16 +7,18 @@ const process = std.process;
 const Allocator = mem.Allocator;
 const ArrayList = std.ArrayList;
 const ast = std.zig.ast;
+const warn = std.log.warn;
+
 const Compilation = @import("Compilation.zig");
 const link = @import("link.zig");
 const Package = @import("Package.zig");
 const zir = @import("zir.zig");
 const build_options = @import("build_options");
-const warn = std.log.warn;
 const introspect = @import("introspect.zig");
 const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
 const translate_c = @import("translate_c.zig");
 const Cache = @import("Cache.zig");
+const target_util = @import("target.zig");
 
 pub fn fatal(comptime format: []const u8, args: anytype) noreturn {
     std.log.emerg(format, args);
@@ -773,6 +775,7 @@ fn buildOutputType(
                         dll_export_fns = false;
                     } else if (mem.eql(u8, arg, "--show-builtin")) {
                         show_builtin = true;
+                        emit_bin = .no;
                     } else if (mem.eql(u8, arg, "--strip")) {
                         strip = true;
                     } else if (mem.eql(u8, arg, "--single-threaded")) {
@@ -1219,12 +1222,12 @@ fn buildOutputType(
         var i: usize = 0;
         while (i < system_libs.items.len) {
             const lib_name = system_libs.items[i];
-            if (is_libc_lib_name(target_info.target, lib_name)) {
+            if (target_util.is_libc_lib_name(target_info.target, lib_name)) {
                 link_libc = true;
                 _ = system_libs.orderedRemove(i);
                 continue;
             }
-            if (is_libcpp_lib_name(target_info.target, lib_name)) {
+            if (target_util.is_libcpp_lib_name(target_info.target, lib_name)) {
                 link_libcpp = true;
                 _ = system_libs.orderedRemove(i);
                 continue;
@@ -2809,62 +2812,6 @@ pub const ClangArgIterator = struct {
     }
 };
 
-fn eqlIgnoreCase(ignore_case: bool, a: []const u8, b: []const u8) bool {
-    if (ignore_case) {
-        return std.ascii.eqlIgnoreCase(a, b);
-    } else {
-        return mem.eql(u8, a, b);
-    }
-}
-
-fn is_libc_lib_name(target: std.Target, name: []const u8) bool {
-    const ignore_case = target.os.tag.isDarwin() or target.os.tag == .windows;
-
-    if (eqlIgnoreCase(ignore_case, name, "c"))
-        return true;
-
-    if (target.isMinGW()) {
-        if (eqlIgnoreCase(ignore_case, name, "m"))
-            return true;
-
-        return false;
-    }
-
-    if (target.abi.isGnu() or target.abi.isMusl() or target.os.tag.isDarwin()) {
-        if (eqlIgnoreCase(ignore_case, name, "m"))
-            return true;
-        if (eqlIgnoreCase(ignore_case, name, "rt"))
-            return true;
-        if (eqlIgnoreCase(ignore_case, name, "pthread"))
-            return true;
-        if (eqlIgnoreCase(ignore_case, name, "crypt"))
-            return true;
-        if (eqlIgnoreCase(ignore_case, name, "util"))
-            return true;
-        if (eqlIgnoreCase(ignore_case, name, "xnet"))
-            return true;
-        if (eqlIgnoreCase(ignore_case, name, "resolv"))
-            return true;
-        if (eqlIgnoreCase(ignore_case, name, "dl"))
-            return true;
-        if (eqlIgnoreCase(ignore_case, name, "util"))
-            return true;
-    }
-
-    if (target.os.tag.isDarwin() and eqlIgnoreCase(ignore_case, name, "System"))
-        return true;
-
-    return false;
-}
-
-fn is_libcpp_lib_name(target: std.Target, name: []const u8) bool {
-    const ignore_case = target.os.tag.isDarwin() or target.os.tag == .windows;
-
-    return eqlIgnoreCase(ignore_case, name, "c++") or
-        eqlIgnoreCase(ignore_case, name, "stdc++") or
-        eqlIgnoreCase(ignore_case, name, "c++abi");
-}
-
 fn parseCodeModel(arg: []const u8) std.builtin.CodeModel {
     return std.meta.stringToEnum(std.builtin.CodeModel, arg) orelse
         fatal("unsupported machine code model: '{}'", .{arg});
diff --git a/src/stage1.zig b/src/stage1.zig
index d05c0ba7fb..2124c23f14 100644
--- a/src/stage1.zig
+++ b/src/stage1.zig
@@ -5,13 +5,15 @@
 const std = @import("std");
 const assert = std.debug.assert;
 const mem = std.mem;
+const CrossTarget = std.zig.CrossTarget;
+const Target = std.Target;
+
 const build_options = @import("build_options");
 const stage2 = @import("main.zig");
 const fatal = stage2.fatal;
-const CrossTarget = std.zig.CrossTarget;
-const Target = std.Target;
 const Compilation = @import("Compilation.zig");
 const translate_c = @import("translate_c.zig");
+const target_util = @import("target.zig");
 
 comptime {
     assert(std.builtin.link_libc);
@@ -370,7 +372,25 @@ export fn stage2_add_link_lib(
     symbol_name_ptr: [*c]const u8,
     symbol_name_len: usize,
 ) ?[*:0]const u8 {
-    return null; // no error
+    const comp = @intToPtr(*Compilation, stage1.userdata);
+    const lib_name = lib_name_ptr[0..lib_name_len];
+    const symbol_name = symbol_name_ptr[0..symbol_name_len];
+    const target = comp.getTarget();
+    const is_libc = target_util.is_libc_lib_name(target, lib_name);
+    if (is_libc and !comp.bin_file.options.link_libc) {
+        return "dependency on libc must be explicitly specified in the build command";
+    }
+
+    if (!is_libc and !target.isWasm() and !comp.bin_file.options.pic) {
+        const msg = std.fmt.allocPrint0(
+            comp.gpa,
+            "dependency on dynamic library '{s}' requires enabling Position Independent Code. Fixed by `-l{s}` or `-fPIC`.",
+            .{ lib_name, lib_name },
+        ) catch return "out of memory";
+        return msg.ptr;
+    }
+
+    return null;
 }
 
 export fn stage2_fetch_file(
diff --git a/src/stage1/stage1.h b/src/stage1/stage1.h
index e5ffa62e14..efbd02e393 100644
--- a/src/stage1/stage1.h
+++ b/src/stage1/stage1.h
@@ -117,8 +117,8 @@ struct Stage2ProgressNode;
 
 enum BuildMode {
     BuildModeDebug,
-    BuildModeFastRelease,
     BuildModeSafeRelease,
+    BuildModeFastRelease,
     BuildModeSmallRelease,
 };
 
diff --git a/src/stage1/zig0.cpp b/src/stage1/zig0.cpp
index e63c68bc94..839ff0263f 100644
--- a/src/stage1/zig0.cpp
+++ b/src/stage1/zig0.cpp
@@ -33,7 +33,7 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) {
         "Options:\n"
         "  --color [auto|off|on]        enable or disable colored error messages\n"
         "  --name [name]                override output name\n"
-        "  --output-dir [dir]           override output directory (defaults to cwd)\n"
+        "  -femit-bin=[path]            Output machine code\n"
         "  --pkg-begin [name] [path]    make pkg available to import and push current pkg\n"
         "  --pkg-end                    pop current pkg\n"
         "  -ODebug                      build with optimizations on and safety off\n"
diff --git a/src/target.zig b/src/target.zig
index 0402ccbd51..9b5ea2a366 100644
--- a/src/target.zig
+++ b/src/target.zig
@@ -223,3 +223,59 @@ pub fn osToLLVM(os_tag: std.Target.Os.Tag) llvm.OSType {
         .emscripten => .Emscripten,
     };
 }
+
+fn eqlIgnoreCase(ignore_case: bool, a: []const u8, b: []const u8) bool {
+    if (ignore_case) {
+        return std.ascii.eqlIgnoreCase(a, b);
+    } else {
+        return std.mem.eql(u8, a, b);
+    }
+}
+
+pub fn is_libc_lib_name(target: std.Target, name: []const u8) bool {
+    const ignore_case = target.os.tag.isDarwin() or target.os.tag == .windows;
+
+    if (eqlIgnoreCase(ignore_case, name, "c"))
+        return true;
+
+    if (target.isMinGW()) {
+        if (eqlIgnoreCase(ignore_case, name, "m"))
+            return true;
+
+        return false;
+    }
+
+    if (target.abi.isGnu() or target.abi.isMusl() or target.os.tag.isDarwin()) {
+        if (eqlIgnoreCase(ignore_case, name, "m"))
+            return true;
+        if (eqlIgnoreCase(ignore_case, name, "rt"))
+            return true;
+        if (eqlIgnoreCase(ignore_case, name, "pthread"))
+            return true;
+        if (eqlIgnoreCase(ignore_case, name, "crypt"))
+            return true;
+        if (eqlIgnoreCase(ignore_case, name, "util"))
+            return true;
+        if (eqlIgnoreCase(ignore_case, name, "xnet"))
+            return true;
+        if (eqlIgnoreCase(ignore_case, name, "resolv"))
+            return true;
+        if (eqlIgnoreCase(ignore_case, name, "dl"))
+            return true;
+        if (eqlIgnoreCase(ignore_case, name, "util"))
+            return true;
+    }
+
+    if (target.os.tag.isDarwin() and eqlIgnoreCase(ignore_case, name, "System"))
+        return true;
+
+    return false;
+}
+
+pub fn is_libcpp_lib_name(target: std.Target, name: []const u8) bool {
+    const ignore_case = target.os.tag.isDarwin() or target.os.tag == .windows;
+
+    return eqlIgnoreCase(ignore_case, name, "c++") or
+        eqlIgnoreCase(ignore_case, name, "stdc++") or
+        eqlIgnoreCase(ignore_case, name, "c++abi");
+}
diff --git a/test/compile_errors.zig b/test/compile_errors.zig
index f457c74609..d27517bc38 100644
--- a/test/compile_errors.zig
+++ b/test/compile_errors.zig
@@ -2347,7 +2347,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
             \\    exit(0);
             \\}
         , &[_][]const u8{
-            "tmp.zig:3:5: error: dependency on library c must be explicitly specified in the build command",
+            "tmp.zig:3:5: error: dependency on libc must be explicitly specified in the build command",
         });
 
         cases.addTest("libc headers note",
diff --git a/test/tests.zig b/test/tests.zig
index 6598a05ea7..58b2b50094 100644
--- a/test/tests.zig
+++ b/test/tests.zig
@@ -634,7 +634,7 @@ pub const StackTracesContext = struct {
 
             warn("Test {}/{} {}...", .{ self.test_index + 1, self.context.test_index, self.name });
 
-            const child = std.ChildProcess.init(args.span(), b.allocator) catch unreachable;
+            const child = std.ChildProcess.init(args.items, b.allocator) catch unreachable;
             defer child.deinit();
 
             child.stdin_behavior = .Ignore;
@@ -643,7 +643,7 @@ pub const StackTracesContext = struct {
             child.env_map = b.env_map;
 
             if (b.verbose) {
-                printInvocation(args.span());
+                printInvocation(args.items);
             }
             child.spawn() catch |err| debug.panic("Unable to spawn {}: {}\n", .{ full_exe_path, @errorName(err) });
 
@@ -666,23 +666,23 @@ pub const StackTracesContext = struct {
                             code,
                             expect_code,
                         });
-                        printInvocation(args.span());
+                        printInvocation(args.items);
                         return error.TestFailed;
                     }
                 },
                 .Signal => |signum| {
                     warn("Process {} terminated on signal {}\n", .{ full_exe_path, signum });
-                    printInvocation(args.span());
+                    printInvocation(args.items);
                     return error.TestFailed;
                 },
                 .Stopped => |signum| {
                     warn("Process {} stopped on signal {}\n", .{ full_exe_path, signum });
-                    printInvocation(args.span());
+                    printInvocation(args.items);
                     return error.TestFailed;
                 },
                 .Unknown => |code| {
                     warn("Process {} terminated unexpectedly with error code {}\n", .{ full_exe_path, code });
-                    printInvocation(args.span());
+                    printInvocation(args.items);
                     return error.TestFailed;
                 },
             }
@@ -837,34 +837,27 @@ pub const CompileErrorContext = struct {
             } else {
                 try zig_args.append("build-obj");
             }
-            const root_src_basename = self.case.sources.span()[0].filename;
+            const root_src_basename = self.case.sources.items[0].filename;
             try zig_args.append(self.write_src.getOutputPath(root_src_basename));
 
             zig_args.append("--name") catch unreachable;
             zig_args.append("test") catch unreachable;
 
-            zig_args.append("--output-dir") catch unreachable;
-            zig_args.append(b.pathFromRoot(b.cache_root)) catch unreachable;
-
             if (!self.case.target.isNative()) {
                 try zig_args.append("-target");
                 try zig_args.append(try self.case.target.zigTriple(b.allocator));
             }
 
-            switch (self.build_mode) {
-                Mode.Debug => {},
-                Mode.ReleaseSafe => zig_args.append("--release-safe") catch unreachable,
-                Mode.ReleaseFast => zig_args.append("--release-fast") catch unreachable,
-                Mode.ReleaseSmall => zig_args.append("--release-small") catch unreachable,
-            }
+            zig_args.append("-O") catch unreachable;
+            zig_args.append(@tagName(self.build_mode)) catch unreachable;
 
             warn("Test {}/{} {}...", .{ self.test_index + 1, self.context.test_index, self.name });
 
             if (b.verbose) {
-                printInvocation(zig_args.span());
+                printInvocation(zig_args.items);
             }
 
-            const child = std.ChildProcess.init(zig_args.span(), b.allocator) catch unreachable;
+            const child = std.ChildProcess.init(zig_args.items, b.allocator) catch unreachable;
             defer child.deinit();
 
             child.env_map = b.env_map;
@@ -886,19 +879,19 @@ pub const CompileErrorContext = struct {
             switch (term) {
                 .Exited => |code| {
                     if (code == 0) {
-                        printInvocation(zig_args.span());
+                        printInvocation(zig_args.items);
                         return error.CompilationIncorrectlySucceeded;
                     }
                 },
                 else => {
                     warn("Process {} terminated unexpectedly\n", .{b.zig_exe});
-                    printInvocation(zig_args.span());
+                    printInvocation(zig_args.items);
                     return error.TestFailed;
                 },
             }
 
-            const stdout = stdout_buf.span();
-            const stderr = stderr_buf.span();
+            const stdout = stdout_buf.items;
+            const stderr = stderr_buf.items;
 
             if (stdout.len != 0) {
                 warn(
@@ -927,12 +920,12 @@ pub const CompileErrorContext = struct {
 
                 if (!ok) {
                     warn("\n======== Expected these compile errors: ========\n", .{});
-                    for (self.case.expected_errors.span()) |expected| {
+                    for (self.case.expected_errors.items) |expected| {
                         warn("{}\n", .{expected});
                     }
                 }
             } else {
-                for (self.case.expected_errors.span()) |expected| {
+                for (self.case.expected_errors.items) |expected| {
                     if (mem.indexOf(u8, stderr, expected) == null) {
                         warn(
                             \\
@@ -1032,7 +1025,7 @@ pub const CompileErrorContext = struct {
             if (mem.indexOf(u8, annotated_case_name, filter) == null) return;
         }
         const write_src = b.addWriteFiles();
-        for (case.sources.span()) |src_file| {
+        for (case.sources.items) |src_file| {
             write_src.add(src_file.filename, src_file.source);
         }
 
@@ -1079,7 +1072,7 @@ pub const StandaloneContext = struct {
             zig_args.append("--verbose") catch unreachable;
         }
 
-        const run_cmd = b.addSystemCommand(zig_args.span());
+        const run_cmd = b.addSystemCommand(zig_args.items);
 
         const log_step = b.addLog("PASS {}\n", .{annotated_case_name});
         log_step.step.dependOn(&run_cmd.step);
@@ -1179,7 +1172,7 @@ pub const GenHContext = struct {
             const full_h_path = self.obj.getOutputHPath();
             const actual_h = try io.readFileAlloc(b.allocator, full_h_path);
 
-            for (self.case.expected_lines.span()) |expected_line| {
+            for (self.case.expected_lines.items) |expected_line| {
                 if (mem.indexOf(u8, actual_h, expected_line) == null) {
                     warn(
                         \\
@@ -1240,7 +1233,7 @@ pub const GenHContext = struct {
         }
 
         const write_src = b.addWriteFiles();
-        for (case.sources.span()) |src_file| {
+        for (case.sources.items) |src_file| {
             write_src.add(src_file.filename, src_file.source);
         }
 

From 4b403c7eaca50815cae8f2ddde19b4fb476ae8ca Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Sat, 26 Sep 2020 21:23:12 -0700
Subject: [PATCH 158/210] fix non-ELF linkAsArchive

---
 BRANCH_TODO  | 4 ++--
 src/link.zig | 6 +++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/BRANCH_TODO b/BRANCH_TODO
index 5306f29746..6eb9cbc34b 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,7 +1,7 @@
- * MachO LLD linking
  * subsystem
- * mingw-w64
  * COFF LLD linking
+ * mingw-w64
+ * MachO LLD linking
  * WASM LLD linking
  * audit the CLI options for stage2
  * audit the base cache hash
diff --git a/src/link.zig b/src/link.zig
index 6cf1d775e4..b90dce9766 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -417,7 +417,11 @@ pub const File = struct {
         const module_obj_path: ?[]const u8 = if (base.options.module) |module| blk: {
             const use_stage1 = build_options.is_stage1 and base.options.use_llvm;
             if (use_stage1) {
-                const obj_basename = try std.fmt.allocPrint(arena, "{}.o", .{base.options.root_name});
+                const obj_basename = try std.zig.binNameAlloc(arena, .{
+                    .root_name = base.options.root_name,
+                    .target = base.options.target,
+                    .output_mode = .Obj,
+                });
                 const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
                 break :blk full_obj_path;
             }

From a9082b4ec51debdbffc20b56e9b37cb82dd04750 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Sat, 26 Sep 2020 22:37:19 -0700
Subject: [PATCH 159/210] stage2: add CLI support for --subsystem

---
 BRANCH_TODO         |  1 -
 src/Compilation.zig |  8 +++++++-
 src/link.zig        |  1 +
 src/main.zig        | 36 ++++++++++++++++++++++++++++++++++++
 4 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/BRANCH_TODO b/BRANCH_TODO
index 6eb9cbc34b..0bd4766207 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,4 +1,3 @@
- * subsystem
  * COFF LLD linking
  * mingw-w64
  * MachO LLD linking
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 10adfb8c45..f641f96832 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -377,6 +377,7 @@ pub const InitOptions = struct {
     color: @import("main.zig").Color = .Auto,
     test_filter: ?[]const u8 = null,
     test_name_prefix: ?[]const u8 = null,
+    subsystem: ?std.Target.SubSystem = null,
 };
 
 pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
@@ -767,6 +768,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .is_compiler_rt_or_libc = options.is_compiler_rt_or_libc,
             .each_lib_rpath = options.each_lib_rpath orelse false,
             .disable_lld_caching = options.disable_lld_caching,
+            .subsystem = options.subsystem,
         });
         errdefer bin_file.destroy();
         comp.* = .{
@@ -2557,6 +2559,10 @@ fn updateStage1Module(comp: *Compilation) !void {
     const stage1_pkg = try createStage1Pkg(arena, "root", mod.root_pkg, null);
     const test_filter = comp.test_filter orelse ""[0..0];
     const test_name_prefix = comp.test_name_prefix orelse ""[0..0];
+    const subsystem = if (comp.bin_file.options.subsystem) |s|
+        @intToEnum(stage1.TargetSubsystem, @enumToInt(s))
+    else
+        stage1.TargetSubsystem.Auto;
     stage1_module.* = .{
         .root_name_ptr = comp.bin_file.options.root_name.ptr,
         .root_name_len = comp.bin_file.options.root_name.len,
@@ -2581,7 +2587,7 @@ fn updateStage1Module(comp: *Compilation) !void {
         .userdata = @ptrToInt(comp),
         .root_pkg = stage1_pkg,
         .code_model = @enumToInt(comp.bin_file.options.machine_code_model),
-        .subsystem = stage1.TargetSubsystem.Auto,
+        .subsystem = subsystem,
         .err_color = @enumToInt(comp.color),
         .pic = comp.bin_file.options.pic,
         .link_libc = comp.bin_file.options.link_libc,
diff --git a/src/link.zig b/src/link.zig
index b90dce9766..a0372da4a1 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -77,6 +77,7 @@ pub const Options = struct {
     disable_lld_caching: bool,
     gc_sections: ?bool = null,
     allow_shlib_undefined: ?bool = null,
+    subsystem: ?std.Target.SubSystem = null,
     linker_script: ?[]const u8 = null,
     version_script: ?[]const u8 = null,
     override_soname: ?[]const u8 = null,
diff --git a/src/main.zig b/src/main.zig
index 31eaf5d8ae..d96cfaf526 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -273,6 +273,7 @@ const usage_build_generic =
     \\  --eh-frame-hdr                 Enable C++ exception handling by passing --eh-frame-hdr to linker
     \\  -dynamic                       Force output to be dynamically linked
     \\  -static                        Force output to be statically linked
+    \\  --subsystem [subsystem]        (windows) /SUBSYSTEM: to the linker\n"
     \\
     \\Test Options:
     \\  --test-filter [text]           Skip tests that do not match filter
@@ -439,6 +440,7 @@ fn buildOutputType(
     var override_lib_dir: ?[]const u8 = null;
     var main_pkg_path: ?[]const u8 = null;
     var clang_preprocessor_mode: Compilation.ClangPreprocessorMode = .no;
+    var subsystem: ?std.Target.SubSystem = null;
 
     var system_libs = std.ArrayList([]const u8).init(gpa);
     defer system_libs.deinit();
@@ -575,6 +577,39 @@ fn buildOutputType(
                         } else {
                             fatal("expected [auto|on|off] after --color, found '{}'", .{next_arg});
                         }
+                    } else if (mem.eql(u8, arg, "--subsystem")) {
+                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        i += 1;
+                        if (mem.eql(u8, args[i], "console")) {
+                            subsystem = .Console;
+                        } else if (mem.eql(u8, args[i], "windows")) {
+                            subsystem = .Windows;
+                        } else if (mem.eql(u8, args[i], "posix")) {
+                            subsystem = .Posix;
+                        } else if (mem.eql(u8, args[i], "native")) {
+                            subsystem = .Native;
+                        } else if (mem.eql(u8, args[i], "efi_application")) {
+                            subsystem = .EfiApplication;
+                        } else if (mem.eql(u8, args[i], "efi_boot_service_driver")) {
+                            subsystem = .EfiBootServiceDriver;
+                        } else if (mem.eql(u8, args[i], "efi_rom")) {
+                            subsystem = .EfiRom;
+                        } else if (mem.eql(u8, args[i], "efi_runtime_driver")) {
+                            subsystem = .EfiRuntimeDriver;
+                        } else {
+                            fatal("invalid: --subsystem: '{s}'. Options are:\n{s}", .{
+                                args[i],
+                                \\  console
+                                \\  windows
+                                \\  posix
+                                \\  native
+                                \\  efi_application
+                                \\  efi_boot_service_driver
+                                \\  efi_rom
+                                \\  efi_runtime_driver
+                                \\
+                            });
+                        }
                     } else if (mem.eql(u8, arg, "-O")) {
                         if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
                         i += 1;
@@ -1539,6 +1574,7 @@ fn buildOutputType(
         .test_filter = test_filter,
         .test_name_prefix = test_name_prefix,
         .disable_lld_caching = !have_enable_cache,
+        .subsystem = subsystem,
     }) catch |err| {
         fatal("unable to create compilation: {}", .{@errorName(err)});
     };

From ed357f9897a6c96a1744307b1bc75a370dd85461 Mon Sep 17 00:00:00 2001
From: Woze Parrrot 
Date: Sat, 26 Sep 2020 20:44:49 -0400
Subject: [PATCH 160/210] uefi system_table

---
 lib/std/os/uefi/tables/system_table.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/std/os/uefi/tables/system_table.zig b/lib/std/os/uefi/tables/system_table.zig
index cbe66fbb68..3f0624d2ce 100644
--- a/lib/std/os/uefi/tables/system_table.zig
+++ b/lib/std/os/uefi/tables/system_table.zig
@@ -35,7 +35,7 @@ pub const SystemTable = extern struct {
     runtime_services: *RuntimeServices,
     boot_services: ?*BootServices,
     number_of_table_entries: usize,
-    configuration_table: *ConfigurationTable,
+    configuration_table: [*]ConfigurationTable,
 
     pub const signature: u64 = 0x5453595320494249;
     pub const revision_1_02: u32 = (1 << 16) | 2;

From eab51b7785ce0989f90a5cdde4b1110f40875ddb Mon Sep 17 00:00:00 2001
From: Ian Simonson 
Date: Sun, 27 Sep 2020 15:07:50 +1000
Subject: [PATCH 161/210] Make LinearFifo not crash when discarding from empty
 buffer

Previously if a LinearFifo was empty and discard was called
an unsigned overflow would occur. However it is safe to perform
this overflow as a bitwise & operation with 0xFFFFFFFFFFFFFF is a noop
---
 lib/std/fifo.zig | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/lib/std/fifo.zig b/lib/std/fifo.zig
index c92da615f9..91d4c0330b 100644
--- a/lib/std/fifo.zig
+++ b/lib/std/fifo.zig
@@ -186,7 +186,9 @@ pub fn LinearFifo(
             } else {
                 var head = self.head + count;
                 if (powers_of_two) {
-                    head &= self.buf.len - 1;
+                    // Note it is safe to do a wrapping subtract as
+                    // bitwise & with all 1s is a noop
+                    head &= self.buf.len -% 1;
                 } else {
                     head %= self.buf.len;
                 }
@@ -376,6 +378,14 @@ pub fn LinearFifo(
     };
 }
 
+test "LinearFifo(u8, .Dynamic) discard(0) from empty buffer should not error on overflow" {
+    var fifo = LinearFifo(u8, .Dynamic).init(testing.allocator);
+    defer fifo.deinit();
+
+    // If overflow is not explicitly allowed this will crash in debug / safe mode
+    fifo.discard(0);
+}
+
 test "LinearFifo(u8, .Dynamic)" {
     var fifo = LinearFifo(u8, .Dynamic).init(testing.allocator);
     defer fifo.deinit();

From a31d9f92f282a878836a3ecac5a48d5f4037868c Mon Sep 17 00:00:00 2001
From: kprotty <45520026+kprotty@users.noreply.github.com>
Date: Sun, 27 Sep 2020 14:05:38 -0500
Subject: [PATCH 162/210] new std.event.Lock implementation

---
 lib/std/event/lock.zig | 194 ++++++++++++++++-------------------------
 1 file changed, 76 insertions(+), 118 deletions(-)

diff --git a/lib/std/event/lock.zig b/lib/std/event/lock.zig
index d27a12aef8..452420b9cd 100644
--- a/lib/std/event/lock.zig
+++ b/lib/std/event/lock.zig
@@ -16,107 +16,90 @@ const Loop = std.event.Loop;
 /// Allows only one actor to hold the lock.
 /// TODO: make this API also work in blocking I/O mode.
 pub const Lock = struct {
-    shared: bool,
-    queue: Queue,
-    queue_empty: bool,
+    mutex: std.Mutex = std.Mutex{},
+    head: usize = UNLOCKED,
 
-    const Queue = std.atomic.Queue(anyframe);
+    const UNLOCKED = 0;
+    const LOCKED = 69;
 
     const global_event_loop = Loop.instance orelse
         @compileError("std.event.Lock currently only works with event-based I/O");
 
-    pub const Held = struct {
-        lock: *Lock,
-
-        pub fn release(self: Held) void {
-            // Resume the next item from the queue.
-            if (self.lock.queue.get()) |node| {
-                global_event_loop.onNextTick(node);
-                return;
-            }
-
-            // We need to release the lock.
-            @atomicStore(bool, &self.lock.queue_empty, true, .SeqCst);
-            @atomicStore(bool, &self.lock.shared, false, .SeqCst);
-
-            // There might be a queue item. If we know the queue is empty, we can be done,
-            // because the other actor will try to obtain the lock.
-            // But if there's a queue item, we are the actor which must loop and attempt
-            // to grab the lock again.
-            if (@atomicLoad(bool, &self.lock.queue_empty, .SeqCst)) {
-                return;
-            }
-
-            while (true) {
-                if (@atomicRmw(bool, &self.lock.shared, .Xchg, true, .SeqCst)) {
-                    // We did not obtain the lock. Great, the queue is someone else's problem.
-                    return;
-                }
-
-                // Resume the next item from the queue.
-                if (self.lock.queue.get()) |node| {
-                    global_event_loop.onNextTick(node);
-                    return;
-                }
-
-                // Release the lock again.
-                @atomicStore(bool, &self.lock.queue_empty, true, .SeqCst);
-                @atomicStore(bool, &self.lock.shared, false, .SeqCst);
-
-                // Find out if we can be done.
-                if (@atomicLoad(bool, &self.lock.queue_empty, .SeqCst)) {
-                    return;
-                }
-            }
-        }
+    const Waiter = struct {
+        next: ?*Waiter,
+        tail: *Waiter,
+        node: Loop.NextTickNode,
     };
 
-    pub fn init() Lock {
-        return Lock{
-            .shared = false,
-            .queue = Queue.init(),
-            .queue_empty = true,
+    pub fn acquire(self: *Lock) Held {
+        const held = self.mutex.acquire();
+
+        if (self.head == UNLOCKED) {
+            self.head = LOCKED;
+            held.release();
+            return Held{ .lock = self };
+        }
+
+        var waiter: Waiter = undefined;
+        waiter.next = null;
+        waiter.tail = &waiter; 
+
+        const head = switch (self.head) {
+            UNLOCKED => unreachable,
+            LOCKED => null,
+            else => @intToPtr(?*Waiter, self.head),
         };
-    }
 
-    pub fn initLocked() Lock {
-        return Lock{
-            .shared = true,
-            .queue = Queue.init(),
-            .queue_empty = true,
-        };
-    }
+        if (head) |h| {
+            h.tail.next = &waiter;
+            h.tail = &waiter;
+        } else {
+            self.head = @ptrToInt(&waiter);
+        }
 
-    /// Must be called when not locked. Not thread safe.
-    /// All calls to acquire() and release() must complete before calling deinit().
-    pub fn deinit(self: *Lock) void {
-        assert(!self.shared);
-        while (self.queue.get()) |node| resume node.data;
-    }
-
-    pub fn acquire(self: *Lock) callconv(.Async) Held {
-        var my_tick_node = Loop.NextTickNode.init(@frame());
-
-        errdefer _ = self.queue.remove(&my_tick_node); // TODO test canceling an acquire
         suspend {
-            self.queue.put(&my_tick_node);
-
-            // At this point, we are in the queue, so we might have already been resumed.
-
-            // We set this bit so that later we can rely on the fact, that if queue_empty == true, some actor
-            // will attempt to grab the lock.
-            @atomicStore(bool, &self.queue_empty, false, .SeqCst);
-
-            if (!@atomicRmw(bool, &self.shared, .Xchg, true, .SeqCst)) {
-                if (self.queue.get()) |node| {
-                    // Whether this node is us or someone else, we tail resume it.
-                    resume node.data;
-                }
-            }
+            waiter.node = Loop.NextTickNode{
+                .prev = undefined,
+                .next = undefined,
+                .data = @frame(),
+            };
+            held.release();
         }
 
         return Held{ .lock = self };
     }
+
+    pub const Held = struct {
+        lock: *Lock, 
+    
+        pub fn release(self: Held) void {
+            const waiter = blk: {
+                const held = self.lock.mutex.acquire();
+                defer held.release();
+
+                switch (self.lock.head) {
+                    UNLOCKED => {
+                        std.debug.panic("Lock unlocked when already unlocked", .{});
+                    },
+                    LOCKED => {
+                        self.lock.head = UNLOCKED;
+                        break :blk null;
+                    },
+                    else => {
+                        const waiter = @intToPtr(*Waiter, self.lock.head);
+                        self.lock.head = if (waiter.next == null) LOCKED else @ptrToInt(waiter.next);
+                        if (waiter.next) |next|
+                            next.tail = waiter.tail;
+                        break :blk waiter;
+                    },
+                }
+            };
+
+            if (waiter) |w| {
+                global_event_loop.onNextTick(&w.node);
+            }
+        }
+    };
 };
 
 test "std.event.Lock" {
@@ -128,41 +111,16 @@ test "std.event.Lock" {
     // TODO https://github.com/ziglang/zig/issues/3251
     if (builtin.os.tag == .freebsd) return error.SkipZigTest;
 
-    // TODO this file has bit-rotted. repair it
-    if (true) return error.SkipZigTest;
-
-    var lock = Lock.init();
-    defer lock.deinit();
-
-    _ = async testLock(&lock);
+    var lock = Lock{};
+    testLock(&lock);
 
     const expected_result = [1]i32{3 * @intCast(i32, shared_test_data.len)} ** shared_test_data.len;
     testing.expectEqualSlices(i32, &expected_result, &shared_test_data);
 }
-fn testLock(lock: *Lock) callconv(.Async) void {
+fn testLock(lock: *Lock) void {
     var handle1 = async lockRunner(lock);
-    var tick_node1 = Loop.NextTickNode{
-        .prev = undefined,
-        .next = undefined,
-        .data = &handle1,
-    };
-    Loop.instance.?.onNextTick(&tick_node1);
-
     var handle2 = async lockRunner(lock);
-    var tick_node2 = Loop.NextTickNode{
-        .prev = undefined,
-        .next = undefined,
-        .data = &handle2,
-    };
-    Loop.instance.?.onNextTick(&tick_node2);
-
     var handle3 = async lockRunner(lock);
-    var tick_node3 = Loop.NextTickNode{
-        .prev = undefined,
-        .next = undefined,
-        .data = &handle3,
-    };
-    Loop.instance.?.onNextTick(&tick_node3);
 
     await handle1;
     await handle2;
@@ -171,13 +129,13 @@ fn testLock(lock: *Lock) callconv(.Async) void {
 
 var shared_test_data = [1]i32{0} ** 10;
 var shared_test_index: usize = 0;
-fn lockRunner(lock: *Lock) callconv(.Async) void {
-    suspend; // resumed by onNextTick
+
+fn lockRunner(lock: *Lock) void {
+    Lock.global_event_loop.yield();
 
     var i: usize = 0;
     while (i < shared_test_data.len) : (i += 1) {
-        var lock_frame = async lock.acquire();
-        const handle = await lock_frame;
+        const handle = lock.acquire();
         defer handle.release();
 
         shared_test_index = 0;

From 91a73a177bc20fa0219dbb6c3cf3dda1c2a465db Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Mon, 28 Sep 2020 00:06:06 -0700
Subject: [PATCH 163/210] stage2: building mingw-w64 and COFF LDD linking

still TODO is the task of creating import .lib files for DLLs on the fly
both for -lfoo and for e.g. `extern "kernel32"`
---
 BRANCH_TODO              |  10 +-
 lib/std/zig.zig          |   2 +-
 src/Cache.zig            |   8 +
 src/Compilation.zig      |  68 ++-
 src/Module.zig           |   9 +-
 src/link.zig             |  53 +--
 src/link/Coff.zig        | 576 ++++++++++++++++++++++----
 src/link/Elf.zig         |  25 +-
 src/mingw.zig            | 866 +++++++++++++++++++++++++++++++++++++++
 src/musl.zig             |   3 -
 src/stage1.zig           |   8 +
 src/stage1/all_types.hpp |   6 -
 src/stage1/analyze.cpp   |  12 +-
 src/stage1/codegen.cpp   |   6 +-
 src/stage1/stage1.h      |   8 +
 15 files changed, 1505 insertions(+), 155 deletions(-)
 create mode 100644 src/mingw.zig

diff --git a/BRANCH_TODO b/BRANCH_TODO
index 0bd4766207..6853b031d7 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,14 +1,13 @@
- * COFF LLD linking
- * mingw-w64
+ * add jobs to build import libs for windows DLLs for explicitly linked libs
+ * add jobs to build import libs for windows DLLs for extern "foo" functions used
  * MachO LLD linking
  * WASM LLD linking
  * audit the CLI options for stage2
  * audit the base cache hash
  * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process.
- * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj]
- * try building some software with zig cc to make sure it didn't regress
  * `-ftime-report`
  *  -fstack-report               print stack size diagnostics\n"
+ * try building some software with zig cc to make sure it didn't regress
 
  * implement proper parsing of clang stderr/stdout and exposing compile errors with the Compilation API
  * implement proper parsing of LLD stderr/stdout and exposing compile errors with the Compilation API
@@ -35,6 +34,7 @@
  * integrate target features into building assembly code
  * libc_installation.zig: make it look for msvc only if msvc abi is chosen
  * switch the default C ABI for windows to be mingw-w64
+   - make it .obj instead of .o always for coff
  * change glibc log errors to normal exposed compile errors
  * improve Directory.join to only use 1 allocation in a clean way.
  * tracy builds with lc++
@@ -48,3 +48,5 @@
  * linking hello world with LLD, lld is silently calling exit(1) instead of reporting ok=false. when run standalone the error message is: ld.lld: error: section [index 3] has a sh_offset (0x57000) + sh_size (0x68) that is greater than the file size (0x57060)
  * submit PR to godbolt and update the CLI options (see changes to test/cli.zig)
  * make proposal about log levels
+ * proposal for changing fs Z/W functions to be native paths and have a way to do native path string literals
+ * proposal for block { break x; }
diff --git a/lib/std/zig.zig b/lib/std/zig.zig
index fc173ecda9..59855f62d4 100644
--- a/lib/std/zig.zig
+++ b/lib/std/zig.zig
@@ -93,7 +93,7 @@ pub fn binNameAlloc(allocator: *std.mem.Allocator, options: BinNameOptions) erro
                 };
                 return std.fmt.allocPrint(allocator, "{}{}{}", .{ target.libPrefix(), root_name, suffix });
             },
-            .Obj => return std.fmt.allocPrint(allocator, "{}.obj", .{root_name}),
+            .Obj => return std.fmt.allocPrint(allocator, "{}{}", .{ root_name, target.abi.oFileExt() }),
         },
         .elf => switch (options.output_mode) {
             .Exe => return allocator.dupe(u8, root_name),
diff --git a/src/Cache.zig b/src/Cache.zig
index cfcbc3e76a..425c6407a3 100644
--- a/src/Cache.zig
+++ b/src/Cache.zig
@@ -76,6 +76,14 @@ pub const HashHelper = struct {
         for (list_of_bytes) |bytes| hh.addBytes(bytes);
     }
 
+    pub fn addStringSet(hh: *HashHelper, hm: std.StringArrayHashMapUnmanaged(void)) void {
+        const entries = hm.items();
+        hh.add(entries.len);
+        for (entries) |entry| {
+            hh.addBytes(entry.key);
+        }
+    }
+
     /// Convert the input value into bytes and record it as a dependency of the process being cached.
     pub fn add(hh: *HashHelper, x: anytype) void {
         switch (@TypeOf(x)) {
diff --git a/src/Compilation.zig b/src/Compilation.zig
index f641f96832..e539950aba 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -17,6 +17,7 @@ const build_options = @import("build_options");
 const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
 const glibc = @import("glibc.zig");
 const musl = @import("musl.zig");
+const mingw = @import("mingw.zig");
 const libunwind = @import("libunwind.zig");
 const libcxx = @import("libcxx.zig");
 const fatal = @import("main.zig").fatal;
@@ -59,7 +60,6 @@ verbose_llvm_ir: bool,
 verbose_cimport: bool,
 verbose_llvm_cpu_features: bool,
 disable_c_depfile: bool,
-is_test: bool,
 time_report: bool,
 
 c_source_files: []const CSourceFile,
@@ -150,8 +150,10 @@ const Job = union(enum) {
     glibc_crt_file: glibc.CRTFile,
     /// all of the glibc shared objects
     glibc_shared_objects,
-    /// one of the glibc static objects
+    /// one of the musl static objects
     musl_crt_file: musl.CRTFile,
+    /// one of the mingw-w64 static objects
+    mingw_crt_file: mingw.CRTFile,
     /// libunwind.a, usually needed when linking libc
     libunwind: void,
     libcxx: void,
@@ -719,6 +721,13 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             fatal("TODO implement support for -femit-h in the self-hosted backend", .{});
         }
 
+        var system_libs: std.StringArrayHashMapUnmanaged(void) = .{};
+        errdefer system_libs.deinit(gpa);
+        try system_libs.ensureCapacity(gpa, options.system_libs.len);
+        for (options.system_libs) |lib_name| {
+            system_libs.putAssumeCapacity(lib_name, {});
+        }
+
         const bin_file = try link.File.openPath(gpa, .{
             .emit = bin_file_emit,
             .root_name = root_name,
@@ -736,7 +745,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .objects = options.link_objects,
             .frameworks = options.frameworks,
             .framework_dirs = options.framework_dirs,
-            .system_libs = options.system_libs,
+            .system_libs = system_libs,
             .lib_dirs = options.lib_dirs,
             .rpath_list = options.rpath_list,
             .strip = options.strip,
@@ -769,6 +778,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .each_lib_rpath = options.each_lib_rpath orelse false,
             .disable_lld_caching = options.disable_lld_caching,
             .subsystem = options.subsystem,
+            .is_test = options.is_test,
         });
         errdefer bin_file.destroy();
         comp.* = .{
@@ -804,7 +814,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .verbose_llvm_cpu_features = options.verbose_llvm_cpu_features,
             .disable_c_depfile = options.disable_c_depfile,
             .owned_link_dir = owned_link_dir,
-            .is_test = options.is_test,
             .color = options.color,
             .time_report = options.time_report,
             .test_filter = options.test_filter,
@@ -847,8 +856,17 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
                 .{ .musl_crt_file = .libc_a },
             });
         }
-        if (comp.wantBuildMinGWW64FromSource()) {
-            @panic("TODO");
+        if (comp.wantBuildMinGWFromSource()) {
+            const static_lib_jobs = [_]Job{
+                .{ .mingw_crt_file = .mingw32_lib },
+                .{ .mingw_crt_file = .msvcrt_os_lib },
+                .{ .mingw_crt_file = .mingwex_lib },
+                .{ .mingw_crt_file = .uuid_lib },
+            };
+            const crt_job: Job = .{ .mingw_crt_file = if (is_dyn_lib) .dllcrt2_o else .crt2_o };
+            try comp.work_queue.ensureUnusedCapacity(static_lib_jobs.len + 1);
+            comp.work_queue.writeAssumeCapacity(&static_lib_jobs);
+            comp.work_queue.writeItemAssumeCapacity(crt_job);
         }
         if (comp.wantBuildLibUnwindFromSource()) {
             try comp.work_queue.writeItem(.{ .libunwind = {} });
@@ -1209,6 +1227,12 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void {
                 fatal("unable to build musl CRT file: {}", .{@errorName(err)});
             };
         },
+        .mingw_crt_file => |crt_file| {
+            mingw.buildCRTFile(self, crt_file) catch |err| {
+                // TODO Expose this as a normal compile error rather than crashing here.
+                fatal("unable to build mingw-w64 CRT file: {}", .{@errorName(err)});
+            };
+        },
         .libunwind => {
             libunwind.buildStaticLib(self) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
@@ -2087,7 +2111,7 @@ fn detectLibCFromLibCInstallation(arena: *Allocator, target: Target, lci: *const
 pub fn get_libc_crt_file(comp: *Compilation, arena: *Allocator, basename: []const u8) ![]const u8 {
     if (comp.wantBuildGLibCFromSource() or
         comp.wantBuildMuslFromSource() or
-        comp.wantBuildMinGWW64FromSource())
+        comp.wantBuildMinGWFromSource())
     {
         return comp.crt_files.get(basename).?.full_object_path;
     }
@@ -2125,7 +2149,7 @@ fn wantBuildMuslFromSource(comp: Compilation) bool {
     return comp.wantBuildLibCFromSource() and comp.getTarget().isMusl();
 }
 
-fn wantBuildMinGWW64FromSource(comp: Compilation) bool {
+fn wantBuildMinGWFromSource(comp: Compilation) bool {
     return comp.wantBuildLibCFromSource() and comp.getTarget().isMinGW();
 }
 
@@ -2186,7 +2210,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
     , .{
         @tagName(comp.bin_file.options.output_mode),
         @tagName(comp.bin_file.options.link_mode),
-        comp.is_test,
+        comp.bin_file.options.is_test,
         comp.bin_file.options.single_threaded,
         @tagName(target.abi),
         @tagName(target.cpu.arch),
@@ -2214,7 +2238,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         \\pub const os = Os{{
         \\    .tag = .{},
         \\    .version_range = .{{
-        ,
+    ,
         .{@tagName(target.os.tag)},
     );
 
@@ -2283,7 +2307,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
             \\        .max = {s},
             \\    }}}},
             \\
-            ,
+        ,
             .{ windows.min, windows.max },
         ),
     }
@@ -2311,7 +2335,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         @tagName(comp.bin_file.options.machine_code_model),
     });
 
-    if (comp.is_test) {
+    if (comp.bin_file.options.is_test) {
         try buffer.appendSlice(
             \\pub var test_functions: []TestFn = undefined; // overwritten later
             \\
@@ -2384,7 +2408,7 @@ fn buildStaticLibFromZig(comp: *Compilation, src_basename: []const u8, out: *?CR
         .basename = bin_basename,
     };
     const optimize_mode: std.builtin.Mode = blk: {
-        if (comp.is_test)
+        if (comp.bin_file.options.is_test)
             break :blk comp.bin_file.options.optimize_mode;
         switch (comp.bin_file.options.optimize_mode) {
             .Debug, .ReleaseFast, .ReleaseSafe => break :blk .ReleaseFast,
@@ -2473,7 +2497,7 @@ fn updateStage1Module(comp: *Compilation) !void {
     man.hash.add(target.os.getVersionRange());
     man.hash.add(comp.bin_file.options.dll_export_fns);
     man.hash.add(comp.bin_file.options.function_sections);
-    man.hash.add(comp.is_test);
+    man.hash.add(comp.bin_file.options.is_test);
     man.hash.add(comp.bin_file.options.emit != null);
     man.hash.add(comp.emit_h != null);
     man.hash.add(comp.emit_asm != null);
@@ -2537,7 +2561,7 @@ fn updateStage1Module(comp: *Compilation) !void {
         zig_lib_dir.ptr,
         zig_lib_dir.len,
         stage2_target,
-        comp.is_test,
+        comp.bin_file.options.is_test,
     ) orelse return error.OutOfMemory;
 
     const emit_bin_path = if (comp.bin_file.options.emit != null) blk: {
@@ -2609,8 +2633,22 @@ fn updateStage1Module(comp: *Compilation) !void {
         .verbose_cimport = comp.verbose_cimport,
         .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
         .main_progress_node = main_progress_node,
+        .have_c_main = false,
+        .have_winmain = false,
+        .have_wwinmain = false,
+        .have_winmain_crt_startup = false,
+        .have_wwinmain_crt_startup = false,
+        .have_dllmain_crt_startup = false,
     };
     stage1_module.build_object();
+
+    mod.have_c_main = stage1_module.have_c_main;
+    mod.have_winmain = stage1_module.have_winmain;
+    mod.have_wwinmain = stage1_module.have_wwinmain;
+    mod.have_winmain_crt_startup = stage1_module.have_winmain_crt_startup;
+    mod.have_wwinmain_crt_startup = stage1_module.have_wwinmain_crt_startup;
+    mod.have_dllmain_crt_startup = stage1_module.have_dllmain_crt_startup;
+
     stage1_module.destroy();
 
     const digest = man.final();
diff --git a/src/Module.zig b/src/Module.zig
index ebe7cdfb1e..8f87b7a521 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -75,6 +75,13 @@ global_error_set: std.StringHashMapUnmanaged(u16) = .{},
 /// previous analysis.
 generation: u32 = 0,
 
+have_winmain: bool = false,
+have_wwinmain: bool = false,
+have_winmain_crt_startup: bool = false,
+have_wwinmain_crt_startup: bool = false,
+have_dllmain_crt_startup: bool = false,
+have_c_main: bool = false,
+
 pub const Export = struct {
     options: std.builtin.ExportOptions,
     /// Byte offset into the file that contains the export directive.
@@ -2668,7 +2675,7 @@ pub fn coerce(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst
         const src_info = inst.ty.intInfo(self.getTarget());
         const dst_info = dest_type.intInfo(self.getTarget());
         if ((src_info.signed == dst_info.signed and dst_info.bits >= src_info.bits) or
-        // small enough unsigned ints can get casted to large enough signed ints
+            // small enough unsigned ints can get casted to large enough signed ints
             (src_info.signed and !dst_info.signed and dst_info.bits > src_info.bits))
         {
             const b = try self.requireRuntimeBlock(scope, inst.src);
diff --git a/src/link.zig b/src/link.zig
index a0372da4a1..9112bd8d1d 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -36,7 +36,7 @@ pub const Options = struct {
     root_name: []const u8,
     /// Not every Compilation compiles .zig code! For example you could do `zig build-exe foo.o`.
     module: ?*Module,
-    dynamic_linker: ?[]const u8 = null,
+    dynamic_linker: ?[]const u8,
     /// Used for calculating how much space to reserve for symbols in case the binary file
     /// does not already have a symbol table.
     symbol_count_hint: u64 = 32,
@@ -44,53 +44,54 @@ pub const Options = struct {
     /// the binary file does not already have such a section.
     program_code_size_hint: u64 = 256 * 1024,
     entry_addr: ?u64 = null,
-    stack_size_override: ?u64 = null,
+    stack_size_override: ?u64,
     /// Set to `true` to omit debug info.
-    strip: bool = false,
+    strip: bool,
     /// If this is true then this link code is responsible for outputting an object
     /// file and then using LLD to link it together with the link options and other objects.
     /// Otherwise (depending on `use_llvm`) this link code directly outputs and updates the final binary.
-    use_lld: bool = false,
+    use_lld: bool,
     /// If this is true then this link code is responsible for making an LLVM IR Module,
     /// outputting it to an object file, and then linking that together with link options and
     /// other objects.
     /// Otherwise (depending on `use_lld`) this link code directly outputs and updates the final binary.
-    use_llvm: bool = false,
-    link_libc: bool = false,
-    link_libcpp: bool = false,
-    function_sections: bool = false,
-    eh_frame_hdr: bool = false,
-    rdynamic: bool = false,
-    z_nodelete: bool = false,
-    z_defs: bool = false,
+    use_llvm: bool,
+    link_libc: bool,
+    link_libcpp: bool,
+    function_sections: bool,
+    eh_frame_hdr: bool,
+    rdynamic: bool,
+    z_nodelete: bool,
+    z_defs: bool,
     bind_global_refs_locally: bool,
     is_native_os: bool,
     pic: bool,
     valgrind: bool,
     stack_check: bool,
     single_threaded: bool,
-    verbose_link: bool = false,
+    verbose_link: bool,
     dll_export_fns: bool,
     error_return_tracing: bool,
     is_compiler_rt_or_libc: bool,
     each_lib_rpath: bool,
     disable_lld_caching: bool,
+    is_test: bool,
     gc_sections: ?bool = null,
-    allow_shlib_undefined: ?bool = null,
-    subsystem: ?std.Target.SubSystem = null,
-    linker_script: ?[]const u8 = null,
-    version_script: ?[]const u8 = null,
-    override_soname: ?[]const u8 = null,
-    llvm_cpu_features: ?[*:0]const u8 = null,
+    allow_shlib_undefined: ?bool,
+    subsystem: ?std.Target.SubSystem,
+    linker_script: ?[]const u8,
+    version_script: ?[]const u8,
+    override_soname: ?[]const u8,
+    llvm_cpu_features: ?[*:0]const u8,
     /// Extra args passed directly to LLD. Ignored when not linking with LLD.
-    extra_lld_args: []const []const u8 = &[0][]const u8,
+    extra_lld_args: []const []const u8,
 
-    objects: []const []const u8 = &[0][]const u8{},
-    framework_dirs: []const []const u8 = &[0][]const u8{},
-    frameworks: []const []const u8 = &[0][]const u8{},
-    system_libs: []const []const u8 = &[0][]const u8{},
-    lib_dirs: []const []const u8 = &[0][]const u8{},
-    rpath_list: []const []const u8 = &[0][]const u8{},
+    objects: []const []const u8,
+    framework_dirs: []const []const u8,
+    frameworks: []const []const u8,
+    system_libs: std.StringArrayHashMapUnmanaged(void),
+    lib_dirs: []const []const u8,
+    rpath_list: []const []const u8,
 
     version: ?std.builtin.Version,
     libc_installation: ?*const LibCInstallation,
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index c396732bc1..e1d7a07fbc 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -5,6 +5,8 @@ const log = std.log.scoped(.link);
 const Allocator = std.mem.Allocator;
 const assert = std.debug.assert;
 const fs = std.fs;
+const allocPrint = std.fmt.allocPrint;
+const mem = std.mem;
 
 const trace = @import("../tracy.zig").trace;
 const Module = @import("../Module.zig");
@@ -12,6 +14,8 @@ const Compilation = @import("../Compilation.zig");
 const codegen = @import("../codegen.zig");
 const link = @import("../link.zig");
 const build_options = @import("build_options");
+const Cache = @import("../Cache.zig");
+const mingw = @import("../mingw.zig");
 
 const allocation_padding = 4 / 3;
 const minimum_text_block_size = 64 * allocation_padding;
@@ -21,7 +25,7 @@ const file_alignment = 512;
 const image_base = 0x400_000;
 const section_table_size = 2 * 40;
 comptime {
-    assert(std.mem.isAligned(image_base, section_alignment));
+    assert(mem.isAligned(image_base, section_alignment));
 }
 
 pub const base_tag: link.File.Tag = .coff;
@@ -155,14 +159,14 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio
     if (machine == .Unknown) {
         return error.UnsupportedCOFFArchitecture;
     }
-    std.mem.writeIntLittle(u16, hdr_data[0..2], @enumToInt(machine));
+    mem.writeIntLittle(u16, hdr_data[0..2], @enumToInt(machine));
     index += 2;
 
     // Number of sections (we only use .got, .text)
-    std.mem.writeIntLittle(u16, hdr_data[index..][0..2], 2);
+    mem.writeIntLittle(u16, hdr_data[index..][0..2], 2);
     index += 2;
     // TimeDateStamp (u32), PointerToSymbolTable (u32), NumberOfSymbols (u32)
-    std.mem.set(u8, hdr_data[index..][0..12], 0);
+    mem.set(u8, hdr_data[index..][0..12], 0);
     index += 12;
 
     const optional_header_size = switch (options.output_mode) {
@@ -177,8 +181,8 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio
     const default_offset_table_size = file_alignment;
     const default_size_of_code = 0;
 
-    self.section_data_offset = std.mem.alignForwardGeneric(u32, self.section_table_offset + section_table_size, file_alignment);
-    const section_data_relative_virtual_address = std.mem.alignForwardGeneric(u32, self.section_table_offset + section_table_size, section_alignment);
+    self.section_data_offset = mem.alignForwardGeneric(u32, self.section_table_offset + section_table_size, file_alignment);
+    const section_data_relative_virtual_address = mem.alignForwardGeneric(u32, self.section_table_offset + section_table_size, section_alignment);
     self.offset_table_virtual_address = image_base + section_data_relative_virtual_address;
     self.offset_table_size = default_offset_table_size;
     self.section_table_offset = section_table_offset;
@@ -186,9 +190,9 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio
     self.text_section_size = default_size_of_code;
 
     // Size of file when loaded in memory
-    const size_of_image = std.mem.alignForwardGeneric(u32, self.text_section_virtual_address - image_base + default_size_of_code, section_alignment);
+    const size_of_image = mem.alignForwardGeneric(u32, self.text_section_virtual_address - image_base + default_size_of_code, section_alignment);
 
-    std.mem.writeIntLittle(u16, hdr_data[index..][0..2], optional_header_size);
+    mem.writeIntLittle(u16, hdr_data[index..][0..2], optional_header_size);
     index += 2;
 
     // Characteristics
@@ -200,7 +204,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio
         .p32 => characteristics |= std.coff.IMAGE_FILE_32BIT_MACHINE,
         .p64 => characteristics |= std.coff.IMAGE_FILE_LARGE_ADDRESS_AWARE,
     }
-    std.mem.writeIntLittle(u16, hdr_data[index..][0..2], characteristics);
+    mem.writeIntLittle(u16, hdr_data[index..][0..2], characteristics);
     index += 2;
 
     assert(index == 20);
@@ -210,106 +214,106 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio
         self.optional_header_offset = coff_file_header_offset + 20;
         // Optional header
         index = 0;
-        std.mem.writeIntLittle(u16, hdr_data[0..2], switch (self.ptr_width) {
+        mem.writeIntLittle(u16, hdr_data[0..2], switch (self.ptr_width) {
             .p32 => @as(u16, 0x10b),
             .p64 => 0x20b,
         });
         index += 2;
 
         // Linker version (u8 + u8)
-        std.mem.set(u8, hdr_data[index..][0..2], 0);
+        mem.set(u8, hdr_data[index..][0..2], 0);
         index += 2;
 
         // SizeOfCode (UNUSED, u32), SizeOfInitializedData (u32), SizeOfUninitializedData (u32), AddressOfEntryPoint (u32), BaseOfCode (UNUSED, u32)
-        std.mem.set(u8, hdr_data[index..][0..20], 0);
+        mem.set(u8, hdr_data[index..][0..20], 0);
         index += 20;
 
         if (self.ptr_width == .p32) {
             // Base of data relative to the image base (UNUSED)
-            std.mem.set(u8, hdr_data[index..][0..4], 0);
+            mem.set(u8, hdr_data[index..][0..4], 0);
             index += 4;
 
             // Image base address
-            std.mem.writeIntLittle(u32, hdr_data[index..][0..4], image_base);
+            mem.writeIntLittle(u32, hdr_data[index..][0..4], image_base);
             index += 4;
         } else {
             // Image base address
-            std.mem.writeIntLittle(u64, hdr_data[index..][0..8], image_base);
+            mem.writeIntLittle(u64, hdr_data[index..][0..8], image_base);
             index += 8;
         }
 
         // Section alignment
-        std.mem.writeIntLittle(u32, hdr_data[index..][0..4], section_alignment);
+        mem.writeIntLittle(u32, hdr_data[index..][0..4], section_alignment);
         index += 4;
         // File alignment
-        std.mem.writeIntLittle(u32, hdr_data[index..][0..4], file_alignment);
+        mem.writeIntLittle(u32, hdr_data[index..][0..4], file_alignment);
         index += 4;
         // Required OS version, 6.0 is vista
-        std.mem.writeIntLittle(u16, hdr_data[index..][0..2], 6);
+        mem.writeIntLittle(u16, hdr_data[index..][0..2], 6);
         index += 2;
-        std.mem.writeIntLittle(u16, hdr_data[index..][0..2], 0);
+        mem.writeIntLittle(u16, hdr_data[index..][0..2], 0);
         index += 2;
         // Image version
-        std.mem.set(u8, hdr_data[index..][0..4], 0);
+        mem.set(u8, hdr_data[index..][0..4], 0);
         index += 4;
         // Required subsystem version, same as OS version
-        std.mem.writeIntLittle(u16, hdr_data[index..][0..2], 6);
+        mem.writeIntLittle(u16, hdr_data[index..][0..2], 6);
         index += 2;
-        std.mem.writeIntLittle(u16, hdr_data[index..][0..2], 0);
+        mem.writeIntLittle(u16, hdr_data[index..][0..2], 0);
         index += 2;
         // Reserved zeroes (u32)
-        std.mem.set(u8, hdr_data[index..][0..4], 0);
+        mem.set(u8, hdr_data[index..][0..4], 0);
         index += 4;
-        std.mem.writeIntLittle(u32, hdr_data[index..][0..4], size_of_image);
+        mem.writeIntLittle(u32, hdr_data[index..][0..4], size_of_image);
         index += 4;
-        std.mem.writeIntLittle(u32, hdr_data[index..][0..4], self.section_data_offset);
+        mem.writeIntLittle(u32, hdr_data[index..][0..4], self.section_data_offset);
         index += 4;
         // CheckSum (u32)
-        std.mem.set(u8, hdr_data[index..][0..4], 0);
+        mem.set(u8, hdr_data[index..][0..4], 0);
         index += 4;
         // Subsystem, TODO: Let users specify the subsystem, always CUI for now
-        std.mem.writeIntLittle(u16, hdr_data[index..][0..2], 3);
+        mem.writeIntLittle(u16, hdr_data[index..][0..2], 3);
         index += 2;
         // DLL characteristics
-        std.mem.writeIntLittle(u16, hdr_data[index..][0..2], 0x0);
+        mem.writeIntLittle(u16, hdr_data[index..][0..2], 0x0);
         index += 2;
 
         switch (self.ptr_width) {
             .p32 => {
                 // Size of stack reserve + commit
-                std.mem.writeIntLittle(u32, hdr_data[index..][0..4], 0x1_000_000);
+                mem.writeIntLittle(u32, hdr_data[index..][0..4], 0x1_000_000);
                 index += 4;
-                std.mem.writeIntLittle(u32, hdr_data[index..][0..4], 0x1_000);
+                mem.writeIntLittle(u32, hdr_data[index..][0..4], 0x1_000);
                 index += 4;
                 // Size of heap reserve + commit
-                std.mem.writeIntLittle(u32, hdr_data[index..][0..4], 0x100_000);
+                mem.writeIntLittle(u32, hdr_data[index..][0..4], 0x100_000);
                 index += 4;
-                std.mem.writeIntLittle(u32, hdr_data[index..][0..4], 0x1_000);
+                mem.writeIntLittle(u32, hdr_data[index..][0..4], 0x1_000);
                 index += 4;
             },
             .p64 => {
                 // Size of stack reserve + commit
-                std.mem.writeIntLittle(u64, hdr_data[index..][0..8], 0x1_000_000);
+                mem.writeIntLittle(u64, hdr_data[index..][0..8], 0x1_000_000);
                 index += 8;
-                std.mem.writeIntLittle(u64, hdr_data[index..][0..8], 0x1_000);
+                mem.writeIntLittle(u64, hdr_data[index..][0..8], 0x1_000);
                 index += 8;
                 // Size of heap reserve + commit
-                std.mem.writeIntLittle(u64, hdr_data[index..][0..8], 0x100_000);
+                mem.writeIntLittle(u64, hdr_data[index..][0..8], 0x100_000);
                 index += 8;
-                std.mem.writeIntLittle(u64, hdr_data[index..][0..8], 0x1_000);
+                mem.writeIntLittle(u64, hdr_data[index..][0..8], 0x1_000);
                 index += 8;
             },
         }
 
         // Reserved zeroes
-        std.mem.set(u8, hdr_data[index..][0..4], 0);
+        mem.set(u8, hdr_data[index..][0..4], 0);
         index += 4;
 
         // Number of data directories
-        std.mem.writeIntLittle(u32, hdr_data[index..][0..4], data_directory_count);
+        mem.writeIntLittle(u32, hdr_data[index..][0..4], data_directory_count);
         index += 4;
         // Initialize data directories to zero
-        std.mem.set(u8, hdr_data[index..][0 .. data_directory_count * 8], 0);
+        mem.set(u8, hdr_data[index..][0 .. data_directory_count * 8], 0);
         index += data_directory_count * 8;
 
         assert(index == optional_header_size);
@@ -321,52 +325,52 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio
     index += 8;
     if (options.output_mode == .Exe) {
         // Virtual size (u32)
-        std.mem.writeIntLittle(u32, hdr_data[index..][0..4], default_offset_table_size);
+        mem.writeIntLittle(u32, hdr_data[index..][0..4], default_offset_table_size);
         index += 4;
         // Virtual address (u32)
-        std.mem.writeIntLittle(u32, hdr_data[index..][0..4], self.offset_table_virtual_address - image_base);
+        mem.writeIntLittle(u32, hdr_data[index..][0..4], self.offset_table_virtual_address - image_base);
         index += 4;
     } else {
-        std.mem.set(u8, hdr_data[index..][0..8], 0);
+        mem.set(u8, hdr_data[index..][0..8], 0);
         index += 8;
     }
     // Size of raw data (u32)
-    std.mem.writeIntLittle(u32, hdr_data[index..][0..4], default_offset_table_size);
+    mem.writeIntLittle(u32, hdr_data[index..][0..4], default_offset_table_size);
     index += 4;
     // File pointer to the start of the section
-    std.mem.writeIntLittle(u32, hdr_data[index..][0..4], self.section_data_offset);
+    mem.writeIntLittle(u32, hdr_data[index..][0..4], self.section_data_offset);
     index += 4;
     // Pointer to relocations (u32), PointerToLinenumbers (u32), NumberOfRelocations (u16), NumberOfLinenumbers (u16)
-    std.mem.set(u8, hdr_data[index..][0..12], 0);
+    mem.set(u8, hdr_data[index..][0..12], 0);
     index += 12;
     // Section flags
-    std.mem.writeIntLittle(u32, hdr_data[index..][0..4], std.coff.IMAGE_SCN_CNT_INITIALIZED_DATA | std.coff.IMAGE_SCN_MEM_READ);
+    mem.writeIntLittle(u32, hdr_data[index..][0..4], std.coff.IMAGE_SCN_CNT_INITIALIZED_DATA | std.coff.IMAGE_SCN_MEM_READ);
     index += 4;
     // Then, the .text section
     hdr_data[index..][0..8].* = ".text\x00\x00\x00".*;
     index += 8;
     if (options.output_mode == .Exe) {
         // Virtual size (u32)
-        std.mem.writeIntLittle(u32, hdr_data[index..][0..4], default_size_of_code);
+        mem.writeIntLittle(u32, hdr_data[index..][0..4], default_size_of_code);
         index += 4;
         // Virtual address (u32)
-        std.mem.writeIntLittle(u32, hdr_data[index..][0..4], self.text_section_virtual_address - image_base);
+        mem.writeIntLittle(u32, hdr_data[index..][0..4], self.text_section_virtual_address - image_base);
         index += 4;
     } else {
-        std.mem.set(u8, hdr_data[index..][0..8], 0);
+        mem.set(u8, hdr_data[index..][0..8], 0);
         index += 8;
     }
     // Size of raw data (u32)
-    std.mem.writeIntLittle(u32, hdr_data[index..][0..4], default_size_of_code);
+    mem.writeIntLittle(u32, hdr_data[index..][0..4], default_size_of_code);
     index += 4;
     // File pointer to the start of the section
-    std.mem.writeIntLittle(u32, hdr_data[index..][0..4], self.section_data_offset + default_offset_table_size);
+    mem.writeIntLittle(u32, hdr_data[index..][0..4], self.section_data_offset + default_offset_table_size);
     index += 4;
     // Pointer to relocations (u32), PointerToLinenumbers (u32), NumberOfRelocations (u16), NumberOfLinenumbers (u16)
-    std.mem.set(u8, hdr_data[index..][0..12], 0);
+    mem.set(u8, hdr_data[index..][0..12], 0);
     index += 12;
     // Section flags
-    std.mem.writeIntLittle(
+    mem.writeIntLittle(
         u32,
         hdr_data[index..][0..4],
         std.coff.IMAGE_SCN_CNT_CODE | std.coff.IMAGE_SCN_MEM_EXECUTE | std.coff.IMAGE_SCN_MEM_READ | std.coff.IMAGE_SCN_MEM_WRITE,
@@ -434,7 +438,7 @@ fn allocateTextBlock(self: *Coff, text_block: *TextBlock, new_block_size: u64, a
             const free_block = self.text_block_free_list.items[i];
 
             const next_block_text_offset = free_block.text_offset + free_block.capacity();
-            const new_block_text_offset = std.mem.alignForwardGeneric(u64, free_block.getVAddr(self.*) + free_block.size, alignment) - self.text_section_virtual_address;
+            const new_block_text_offset = mem.alignForwardGeneric(u64, free_block.getVAddr(self.*) + free_block.size, alignment) - self.text_section_virtual_address;
             if (new_block_text_offset < next_block_text_offset and next_block_text_offset - new_block_text_offset >= new_block_min_capacity) {
                 block_placement = free_block;
 
@@ -453,7 +457,7 @@ fn allocateTextBlock(self: *Coff, text_block: *TextBlock, new_block_size: u64, a
                 continue;
             }
         } else if (self.last_text_block) |last| {
-            const new_block_vaddr = std.mem.alignForwardGeneric(u64, last.getVAddr(self.*) + last.size, alignment);
+            const new_block_vaddr = mem.alignForwardGeneric(u64, last.getVAddr(self.*) + last.size, alignment);
             block_placement = last;
             break :blk new_block_vaddr;
         } else {
@@ -463,15 +467,15 @@ fn allocateTextBlock(self: *Coff, text_block: *TextBlock, new_block_size: u64, a
 
     const expand_text_section = block_placement == null or block_placement.?.next == null;
     if (expand_text_section) {
-        const needed_size = @intCast(u32, std.mem.alignForwardGeneric(u64, vaddr + new_block_size - self.text_section_virtual_address, file_alignment));
+        const needed_size = @intCast(u32, mem.alignForwardGeneric(u64, vaddr + new_block_size - self.text_section_virtual_address, file_alignment));
         if (needed_size > self.text_section_size) {
-            const current_text_section_virtual_size = std.mem.alignForwardGeneric(u32, self.text_section_size, section_alignment);
-            const new_text_section_virtual_size = std.mem.alignForwardGeneric(u32, needed_size, section_alignment);
+            const current_text_section_virtual_size = mem.alignForwardGeneric(u32, self.text_section_size, section_alignment);
+            const new_text_section_virtual_size = mem.alignForwardGeneric(u32, needed_size, section_alignment);
             if (current_text_section_virtual_size != new_text_section_virtual_size) {
                 self.size_of_image_dirty = true;
                 // Write new virtual size
                 var buf: [4]u8 = undefined;
-                std.mem.writeIntLittle(u32, &buf, new_text_section_virtual_size);
+                mem.writeIntLittle(u32, &buf, new_text_section_virtual_size);
                 try self.base.file.?.pwriteAll(&buf, self.section_table_offset + 40 + 8);
             }
 
@@ -509,7 +513,7 @@ fn allocateTextBlock(self: *Coff, text_block: *TextBlock, new_block_size: u64, a
 
 fn growTextBlock(self: *Coff, text_block: *TextBlock, new_block_size: u64, alignment: u64) !u64 {
     const block_vaddr = text_block.getVAddr(self.*);
-    const align_ok = std.mem.alignBackwardGeneric(u64, block_vaddr, alignment) == block_vaddr;
+    const align_ok = mem.alignBackwardGeneric(u64, block_vaddr, alignment) == block_vaddr;
     const need_realloc = !align_ok or new_block_size > text_block.capacity();
     if (!need_realloc) return @as(u64, block_vaddr);
     return self.allocateTextBlock(text_block, new_block_size, alignment);
@@ -575,14 +579,14 @@ fn writeOffsetTableEntry(self: *Coff, index: usize) !void {
 
         // Write the new raw size in the .got header
         var buf: [8]u8 = undefined;
-        std.mem.writeIntLittle(u32, buf[0..4], new_raw_size);
+        mem.writeIntLittle(u32, buf[0..4], new_raw_size);
         try self.base.file.?.pwriteAll(buf[0..4], self.section_table_offset + 16);
         // Write the new .text section file offset in the .text section header
-        std.mem.writeIntLittle(u32, buf[0..4], new_text_section_start);
+        mem.writeIntLittle(u32, buf[0..4], new_text_section_start);
         try self.base.file.?.pwriteAll(buf[0..4], self.section_table_offset + 40 + 20);
 
-        const current_virtual_size = std.mem.alignForwardGeneric(u32, self.offset_table_size, section_alignment);
-        const new_virtual_size = std.mem.alignForwardGeneric(u32, new_raw_size, section_alignment);
+        const current_virtual_size = mem.alignForwardGeneric(u32, self.offset_table_size, section_alignment);
+        const new_virtual_size = mem.alignForwardGeneric(u32, new_raw_size, section_alignment);
         // If we had to move in the virtual address space, we need to fix the VAs in the offset table, as well as the virtual address of the `.text` section
         // and the virutal size of the `.got` section
 
@@ -592,12 +596,12 @@ fn writeOffsetTableEntry(self: *Coff, index: usize) !void {
             const va_offset = new_virtual_size - current_virtual_size;
 
             // Write .got virtual size
-            std.mem.writeIntLittle(u32, buf[0..4], new_virtual_size);
+            mem.writeIntLittle(u32, buf[0..4], new_virtual_size);
             try self.base.file.?.pwriteAll(buf[0..4], self.section_table_offset + 8);
 
             // Write .text new virtual address
             self.text_section_virtual_address = self.text_section_virtual_address + va_offset;
-            std.mem.writeIntLittle(u32, buf[0..4], self.text_section_virtual_address - image_base);
+            mem.writeIntLittle(u32, buf[0..4], self.text_section_virtual_address - image_base);
             try self.base.file.?.pwriteAll(buf[0..4], self.section_table_offset + 40 + 12);
 
             // Fix the VAs in the offset table
@@ -607,11 +611,11 @@ fn writeOffsetTableEntry(self: *Coff, index: usize) !void {
 
                     switch (entry_size) {
                         4 => {
-                            std.mem.writeInt(u32, buf[0..4], @intCast(u32, va.*), endian);
+                            mem.writeInt(u32, buf[0..4], @intCast(u32, va.*), endian);
                             try self.base.file.?.pwriteAll(buf[0..4], offset_table_start + idx * entry_size);
                         },
                         8 => {
-                            std.mem.writeInt(u64, &buf, va.*, endian);
+                            mem.writeInt(u64, &buf, va.*, endian);
                             try self.base.file.?.pwriteAll(&buf, offset_table_start + idx * entry_size);
                         },
                         else => unreachable,
@@ -626,12 +630,12 @@ fn writeOffsetTableEntry(self: *Coff, index: usize) !void {
     switch (entry_size) {
         4 => {
             var buf: [4]u8 = undefined;
-            std.mem.writeInt(u32, &buf, @intCast(u32, self.offset_table.items[index]), endian);
+            mem.writeInt(u32, &buf, @intCast(u32, self.offset_table.items[index]), endian);
             try self.base.file.?.pwriteAll(&buf, offset_table_start + index * entry_size);
         },
         8 => {
             var buf: [8]u8 = undefined;
-            std.mem.writeInt(u64, &buf, self.offset_table.items[index], endian);
+            mem.writeInt(u64, &buf, self.offset_table.items[index], endian);
             try self.base.file.?.pwriteAll(&buf, offset_table_start + index * entry_size);
         },
         else => unreachable,
@@ -664,7 +668,7 @@ pub fn updateDecl(self: *Coff, module: *Module, decl: *Module.Decl) !void {
     if (curr_size != 0) {
         const capacity = decl.link.coff.capacity();
         const need_realloc = code.len > capacity or
-            !std.mem.isAlignedGeneric(u32, decl.link.coff.text_offset, required_alignment);
+            !mem.isAlignedGeneric(u32, decl.link.coff.text_offset, required_alignment);
         if (need_realloc) {
             const curr_vaddr = self.getDeclVAddr(decl);
             const vaddr = try self.growTextBlock(&decl.link.coff, code.len, required_alignment);
@@ -679,7 +683,7 @@ pub fn updateDecl(self: *Coff, module: *Module, decl: *Module.Decl) !void {
         }
     } else {
         const vaddr = try self.allocateTextBlock(&decl.link.coff, code.len, required_alignment);
-        log.debug("allocated text block for {} at 0x{x} (size: {Bi})\n", .{ std.mem.spanZ(decl.name), vaddr, code.len });
+        log.debug("allocated text block for {} at 0x{x} (size: {Bi})\n", .{ mem.spanZ(decl.name), vaddr, code.len });
         errdefer self.freeTextBlock(&decl.link.coff);
         self.offset_table.items[decl.link.coff.offset_table_index] = vaddr;
         try self.writeOffsetTableEntry(decl.link.coff.offset_table_index);
@@ -702,7 +706,7 @@ pub fn freeDecl(self: *Coff, decl: *Module.Decl) void {
 pub fn updateDeclExports(self: *Coff, module: *Module, decl: *const Module.Decl, exports: []const *Module.Export) !void {
     for (exports) |exp| {
         if (exp.options.section) |section_name| {
-            if (!std.mem.eql(u8, section_name, ".text")) {
+            if (!mem.eql(u8, section_name, ".text")) {
                 try module.failed_exports.ensureCapacity(module.gpa, module.failed_exports.items().len + 1);
                 module.failed_exports.putAssumeCapacityNoClobber(
                     exp,
@@ -711,7 +715,7 @@ pub fn updateDeclExports(self: *Coff, module: *Module, decl: *const Module.Decl,
                 continue;
             }
         }
-        if (std.mem.eql(u8, exp.options.name, "_start")) {
+        if (mem.eql(u8, exp.options.name, "_start")) {
             self.entry_addr = decl.link.coff.getVAddr(self.*) - image_base;
         } else {
             try module.failed_exports.ensureCapacity(module.gpa, module.failed_exports.items().len + 1);
@@ -726,8 +730,12 @@ pub fn updateDeclExports(self: *Coff, module: *Module, decl: *const Module.Decl,
 
 pub fn flush(self: *Coff, comp: *Compilation) !void {
     if (build_options.have_llvm and self.base.options.use_lld) {
-        return error.CoffLinkingWithLLDUnimplemented;
+        return self.linkWithLLD(comp);
     } else {
+        switch (self.base.options.effectiveOutputMode()) {
+            .Exe, .Obj => {},
+            .Lib => return error.TODOImplementWritingLibFiles,
+        }
         return self.flushModule(comp);
     }
 }
@@ -739,16 +747,16 @@ pub fn flushModule(self: *Coff, comp: *Compilation) !void {
     if (self.text_section_size_dirty) {
         // Write the new raw size in the .text header
         var buf: [4]u8 = undefined;
-        std.mem.writeIntLittle(u32, &buf, self.text_section_size);
+        mem.writeIntLittle(u32, &buf, self.text_section_size);
         try self.base.file.?.pwriteAll(&buf, self.section_table_offset + 40 + 16);
         try self.base.file.?.setEndPos(self.section_data_offset + self.offset_table_size + self.text_section_size);
         self.text_section_size_dirty = false;
     }
 
     if (self.base.options.output_mode == .Exe and self.size_of_image_dirty) {
-        const new_size_of_image = std.mem.alignForwardGeneric(u32, self.text_section_virtual_address - image_base + self.text_section_size, section_alignment);
+        const new_size_of_image = mem.alignForwardGeneric(u32, self.text_section_virtual_address - image_base + self.text_section_size, section_alignment);
         var buf: [4]u8 = undefined;
-        std.mem.writeIntLittle(u32, &buf, new_size_of_image);
+        mem.writeIntLittle(u32, &buf, new_size_of_image);
         try self.base.file.?.pwriteAll(&buf, self.optional_header_offset + 56);
         self.size_of_image_dirty = false;
     }
@@ -763,12 +771,422 @@ pub fn flushModule(self: *Coff, comp: *Compilation) !void {
         if (self.base.options.output_mode == .Exe) {
             // Write AddressOfEntryPoint
             var buf: [4]u8 = undefined;
-            std.mem.writeIntLittle(u32, &buf, self.entry_addr.?);
+            mem.writeIntLittle(u32, &buf, self.entry_addr.?);
             try self.base.file.?.pwriteAll(&buf, self.optional_header_offset + 16);
         }
     }
 }
 
+fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
+    const tracy = trace(@src());
+    defer tracy.end();
+
+    var arena_allocator = std.heap.ArenaAllocator.init(self.base.allocator);
+    defer arena_allocator.deinit();
+    const arena = &arena_allocator.allocator;
+
+    const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
+
+    // If there is no Zig code to compile, then we should skip flushing the output file because it
+    // will not be part of the linker line anyway.
+    const module_obj_path: ?[]const u8 = if (self.base.options.module) |module| blk: {
+        const use_stage1 = build_options.is_stage1 and self.base.options.use_llvm;
+        if (use_stage1) {
+            const obj_basename = try std.zig.binNameAlloc(arena, .{
+                .root_name = self.base.options.root_name,
+                .target = self.base.options.target,
+                .output_mode = .Obj,
+            });
+            const o_directory = self.base.options.module.?.zig_cache_artifact_directory;
+            const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
+            break :blk full_obj_path;
+        }
+
+        try self.flushModule(comp);
+        const obj_basename = self.base.intermediary_basename.?;
+        const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
+        break :blk full_obj_path;
+    } else null;
+
+    const is_lib = self.base.options.output_mode == .Lib;
+    const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib;
+    const is_exe_or_dyn_lib = is_dyn_lib or self.base.options.output_mode == .Exe;
+    const link_in_crt = self.base.options.link_libc and self.base.options.output_mode == .Exe;
+    const target = self.base.options.target;
+
+    // See link/Elf.zig for comments on how this mechanism works.
+    const id_symlink_basename = "lld.id";
+
+    var man: Cache.Manifest = undefined;
+    defer if (!self.base.options.disable_lld_caching) man.deinit();
+
+    var digest: [Cache.hex_digest_len]u8 = undefined;
+
+    if (!self.base.options.disable_lld_caching) {
+        man = comp.cache_parent.obtain();
+        self.base.releaseLock();
+
+        try man.addListOfFiles(self.base.options.objects);
+        for (comp.c_object_table.items()) |entry| {
+            _ = try man.addFile(entry.key.status.success.object_path, null);
+        }
+        try man.addOptionalFile(module_obj_path);
+        man.hash.addOptional(self.base.options.stack_size_override);
+        man.hash.addListOfBytes(self.base.options.extra_lld_args);
+        man.hash.addListOfBytes(self.base.options.lib_dirs);
+        man.hash.add(self.base.options.is_compiler_rt_or_libc);
+        if (self.base.options.link_libc) {
+            man.hash.add(self.base.options.libc_installation != null);
+            if (self.base.options.libc_installation) |libc_installation| {
+                man.hash.addBytes(libc_installation.crt_dir.?);
+                if (target.abi == .msvc) {
+                    man.hash.addBytes(libc_installation.msvc_lib_dir.?);
+                    man.hash.addBytes(libc_installation.kernel32_lib_dir.?);
+                }
+            }
+        }
+        man.hash.addStringSet(self.base.options.system_libs);
+        man.hash.addOptional(self.base.options.subsystem);
+        man.hash.add(self.base.options.is_test);
+
+        // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
+        _ = try man.hit();
+        digest = man.final();
+        var prev_digest_buf: [digest.len]u8 = undefined;
+        const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {
+            log.debug("COFF LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) });
+            // Handle this as a cache miss.
+            break :blk prev_digest_buf[0..0];
+        };
+        if (mem.eql(u8, prev_digest, &digest)) {
+            log.debug("COFF LLD digest={} match - skipping invocation", .{digest});
+            // Hot diggity dog! The output binary is already there.
+            self.base.lock = man.toOwnedLock();
+            return;
+        }
+        log.debug("COFF LLD prev_digest={} new_digest={}", .{ prev_digest, digest });
+
+        // We are about to change the output file to be different, so we invalidate the build hash now.
+        directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
+            error.FileNotFound => {},
+            else => |e| return e,
+        };
+    }
+
+    const is_obj = self.base.options.output_mode == .Obj;
+
+    // Create an LLD command line and invoke it.
+    var argv = std.ArrayList([]const u8).init(self.base.allocator);
+    defer argv.deinit();
+    // Even though we're calling LLD as a library it thinks the first argument is its own exe name.
+    try argv.append("lld");
+    if (is_obj) {
+        try argv.append("-r");
+    }
+
+    try argv.append("-ERRORLIMIT:0");
+    try argv.append("-NOLOGO");
+    if (!self.base.options.strip) {
+        try argv.append("-DEBUG");
+    }
+    if (self.base.options.output_mode == .Exe) {
+        const stack_size = self.base.options.stack_size_override orelse 16777216;
+        try argv.append(try allocPrint(arena, "-STACK:{d}", .{stack_size}));
+    }
+
+    if (target.cpu.arch == .i386) {
+        try argv.append("-MACHINE:X86");
+    } else if (target.cpu.arch == .x86_64) {
+        try argv.append("-MACHINE:X64");
+    } else if (target.cpu.arch.isARM()) {
+        if (target.cpu.arch.ptrBitWidth() == 32) {
+            try argv.append("-MACHINE:ARM");
+        } else {
+            try argv.append("-MACHINE:ARM64");
+        }
+    }
+
+    if (is_dyn_lib) {
+        try argv.append("-DLL");
+    }
+
+    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
+    try argv.append(try allocPrint(arena, "-OUT:{s}", .{full_out_path}));
+
+    if (self.base.options.link_libc) {
+        if (self.base.options.libc_installation) |libc_installation| {
+            try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.crt_dir.?}));
+
+            if (target.abi == .msvc) {
+                try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.msvc_lib_dir.?}));
+                try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.kernel32_lib_dir.?}));
+            }
+        }
+    }
+
+    for (self.base.options.lib_dirs) |lib_dir| {
+        try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{lib_dir}));
+    }
+
+    try argv.appendSlice(self.base.options.objects);
+
+    for (comp.c_object_table.items()) |entry| {
+        try argv.append(entry.key.status.success.object_path);
+    }
+
+    if (module_obj_path) |p| {
+        try argv.append(p);
+    }
+
+    const resolved_subsystem: ?std.Target.SubSystem = blk: {
+        if (self.base.options.subsystem) |explicit| break :blk explicit;
+        switch (target.os.tag) {
+            .windows => {
+                if (self.base.options.module) |module| {
+                    if (module.have_dllmain_crt_startup or is_dyn_lib)
+                        break :blk null;
+                    if (module.have_c_main or self.base.options.is_test or
+                        module.have_winmain_crt_startup or module.have_wwinmain_crt_startup)
+                    {
+                        break :blk .Console;
+                    }
+                    if (module.have_winmain or module.have_wwinmain)
+                        break :blk .Windows;
+                }
+            },
+            .uefi => break :blk .EfiApplication,
+            else => {},
+        }
+        break :blk null;
+    };
+    const Mode = enum { uefi, win32 };
+    const mode: Mode = mode: {
+        if (resolved_subsystem) |subsystem| switch (subsystem) {
+            .Console => {
+                try argv.append("-SUBSYSTEM:console");
+                break :mode .win32;
+            },
+            .EfiApplication => {
+                try argv.append("-SUBSYSTEM:efi_application");
+                break :mode .uefi;
+            },
+            .EfiBootServiceDriver => {
+                try argv.append("-SUBSYSTEM:efi_boot_service_driver");
+                break :mode .uefi;
+            },
+            .EfiRom => {
+                try argv.append("-SUBSYSTEM:efi_rom");
+                break :mode .uefi;
+            },
+            .EfiRuntimeDriver => {
+                try argv.append("-SUBSYSTEM:efi_runtime_driver");
+                break :mode .uefi;
+            },
+            .Native => {
+                try argv.append("-SUBSYSTEM:native");
+                break :mode .win32;
+            },
+            .Posix => {
+                try argv.append("-SUBSYSTEM:posix");
+                break :mode .win32;
+            },
+            .Windows => {
+                try argv.append("-SUBSYSTEM:windows");
+                break :mode .win32;
+            },
+        } else if (target.os.tag == .uefi) {
+            break :mode .uefi;
+        } else {
+            break :mode .win32;
+        }
+    };
+
+    switch (mode) {
+        .uefi => try argv.appendSlice(&[_][]const u8{
+            "-BASE:0",
+            "-ENTRY:EfiMain",
+            "-OPT:REF",
+            "-SAFESEH:NO",
+            "-MERGE:.rdata=.data",
+            "-ALIGN:32",
+            "-NODEFAULTLIB",
+            "-SECTION:.xdata,D",
+        }),
+        .win32 => {
+            if (link_in_crt) {
+                if (target.abi.isGnu()) {
+                    try argv.append("-lldmingw");
+
+                    if (target.cpu.arch == .i386) {
+                        try argv.append("-ALTERNATENAME:__image_base__=___ImageBase");
+                    } else {
+                        try argv.append("-ALTERNATENAME:__image_base__=__ImageBase");
+                    }
+
+                    if (is_dyn_lib) {
+                        try argv.append(try comp.get_libc_crt_file(arena, "dllcrt2.o"));
+                    } else {
+                        try argv.append(try comp.get_libc_crt_file(arena, "crt2.o"));
+                    }
+
+                    try argv.append(try comp.get_libc_crt_file(arena, "mingw32.lib"));
+                    try argv.append(try comp.get_libc_crt_file(arena, "mingwex.lib"));
+                    try argv.append(try comp.get_libc_crt_file(arena, "msvcrt-os.lib"));
+
+                    for (mingw.always_link_libs) |name| {
+                        if (!self.base.options.system_libs.contains(name)) {
+                            const lib_basename = try allocPrint(arena, "{s}.lib", .{name});
+                            try argv.append(try comp.get_libc_crt_file(arena, lib_basename));
+                        }
+                    }
+                } else {
+                    const lib_str = switch (self.base.options.link_mode) {
+                        .Dynamic => "",
+                        .Static => "lib",
+                    };
+                    const d_str = switch (self.base.options.optimize_mode) {
+                        .Debug => "d",
+                        else => "",
+                    };
+                    switch (self.base.options.link_mode) {
+                        .Static => try argv.append(try allocPrint(arena, "libcmt{s}.lib", .{d_str})),
+                        .Dynamic => try argv.append(try allocPrint(arena, "msvcrt{s}.lib", .{d_str})),
+                    }
+
+                    try argv.append(try allocPrint(arena, "{s}vcruntime{s}.lib", .{ lib_str, d_str }));
+                    try argv.append(try allocPrint(arena, "{s}ucrt{s}.lib", .{ lib_str, d_str }));
+
+                    //Visual C++ 2015 Conformance Changes
+                    //https://msdn.microsoft.com/en-us/library/bb531344.aspx
+                    try argv.append("legacy_stdio_definitions.lib");
+
+                    // msvcrt depends on kernel32 and ntdll
+                    try argv.append("kernel32.lib");
+                    try argv.append("ntdll.lib");
+                }
+            } else {
+                try argv.append("-NODEFAULTLIB");
+                if (!is_lib) {
+                    if (self.base.options.module) |module| {
+                        if (module.have_winmain) {
+                            try argv.append("-ENTRY:WinMain");
+                        } else if (module.have_wwinmain) {
+                            try argv.append("-ENTRY:wWinMain");
+                        } else if (module.have_wwinmain_crt_startup) {
+                            try argv.append("-ENTRY:wWinMainCRTStartup");
+                        } else {
+                            try argv.append("-ENTRY:WinMainCRTStartup");
+                        }
+                    } else {
+                        try argv.append("-ENTRY:WinMainCRTStartup");
+                    }
+                }
+            }
+        },
+    }
+
+    if (!is_obj) {
+        // libc++ dep
+        if (self.base.options.link_libcpp) {
+            try argv.append(comp.libcxxabi_static_lib.?.full_object_path);
+            try argv.append(comp.libcxx_static_lib.?.full_object_path);
+            try argv.append(comp.libunwind_static_lib.?.full_object_path);
+        }
+    }
+
+    // compiler-rt and libc
+    if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) {
+        if (!self.base.options.link_libc) {
+            try argv.append(comp.libc_static_lib.?.full_object_path);
+        }
+        // MSVC compiler_rt is missing some stuff, so we build it unconditionally but
+        // and rely on weak linkage to allow MSVC compiler_rt functions to override ours.
+        try argv.append(comp.compiler_rt_static_lib.?.full_object_path);
+    }
+
+    for (self.base.options.system_libs.items()) |entry| {
+        const lib_basename = try allocPrint(arena, "{s}.lib", .{entry.key});
+        if (comp.crt_files.get(lib_basename)) |crt_file| {
+            try argv.append(crt_file.full_object_path);
+        } else {
+            try argv.append(lib_basename);
+        }
+    }
+
+    if (self.base.options.verbose_link) {
+        Compilation.dump_argv(argv.items);
+    }
+
+    const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1);
+    new_argv_with_sentinel[argv.items.len] = null;
+    const new_argv = new_argv_with_sentinel[0..argv.items.len :null];
+    for (argv.items) |arg, i| {
+        new_argv[i] = try arena.dupeZ(u8, arg);
+    }
+
+    var stderr_context: LLDContext = .{
+        .coff = self,
+        .data = std.ArrayList(u8).init(self.base.allocator),
+    };
+    defer stderr_context.data.deinit();
+    var stdout_context: LLDContext = .{
+        .coff = self,
+        .data = std.ArrayList(u8).init(self.base.allocator),
+    };
+    defer stdout_context.data.deinit();
+    const llvm = @import("../llvm.zig");
+    const ok = llvm.Link(
+        .COFF,
+        new_argv.ptr,
+        new_argv.len,
+        append_diagnostic,
+        @ptrToInt(&stdout_context),
+        @ptrToInt(&stderr_context),
+    );
+    if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory;
+    if (stdout_context.data.items.len != 0) {
+        std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items});
+    }
+    if (!ok) {
+        // TODO parse this output and surface with the Compilation API rather than
+        // directly outputting to stderr here.
+        std.debug.print("{}", .{stderr_context.data.items});
+        return error.LLDReportedFailure;
+    }
+    if (stderr_context.data.items.len != 0) {
+        std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items});
+    }
+
+    if (!self.base.options.disable_lld_caching) {
+        // Update the dangling symlink with the digest. If it fails we can continue; it only
+        // means that the next invocation will have an unnecessary cache miss.
+        directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {
+            std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});
+        };
+        // Again failure here only means an unnecessary cache miss.
+        man.writeManifest() catch |err| {
+            std.log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
+        };
+        // We hang on to this lock so that the output file path can be used without
+        // other processes clobbering it.
+        self.base.lock = man.toOwnedLock();
+    }
+}
+
+const LLDContext = struct {
+    data: std.ArrayList(u8),
+    coff: *Coff,
+    oom: bool = false,
+};
+
+fn append_diagnostic(context: usize, ptr: [*]const u8, len: usize) callconv(.C) void {
+    const lld_context = @intToPtr(*LLDContext, context);
+    const msg = ptr[0..len];
+    lld_context.data.appendSlice(msg) catch |err| switch (err) {
+        error.OutOfMemory => lld_context.oom = true,
+    };
+}
+
 pub fn getDeclVAddr(self: *Coff, decl: *const Module.Decl) u64 {
     return self.text_section_virtual_address + decl.link.coff.text_offset;
 }
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index c8067058d9..88f5040761 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -1225,7 +1225,11 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
     const module_obj_path: ?[]const u8 = if (self.base.options.module) |module| blk: {
         const use_stage1 = build_options.is_stage1 and self.base.options.use_llvm;
         if (use_stage1) {
-            const obj_basename = try std.fmt.allocPrint(arena, "{}.o", .{self.base.options.root_name});
+            const obj_basename = try std.zig.binNameAlloc(arena, .{
+                .root_name = self.base.options.root_name,
+                .target = self.base.options.target,
+                .output_mode = .Obj,
+            });
             const o_directory = self.base.options.module.?.zig_cache_artifact_directory;
             const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
             break :blk full_obj_path;
@@ -1242,6 +1246,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
     const is_exe_or_dyn_lib = is_dyn_lib or self.base.options.output_mode == .Exe;
     const have_dynamic_linker = self.base.options.link_libc and
         self.base.options.link_mode == .Dynamic and is_exe_or_dyn_lib;
+    const link_in_crt = self.base.options.link_libc and self.base.options.output_mode == .Exe;
+    const target = self.base.options.target;
 
     // Here we want to determine whether we can save time by not invoking LLD when the
     // output is unchanged. None of the linker options or the object files that are being
@@ -1297,7 +1303,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
             man.hash.addOptionalBytes(self.base.options.override_soname);
             man.hash.addOptional(self.base.options.version);
         }
-        man.hash.addListOfBytes(self.base.options.system_libs);
+        man.hash.addStringSet(self.base.options.system_libs);
         man.hash.addOptional(self.base.options.allow_shlib_undefined);
         man.hash.add(self.base.options.bind_global_refs_locally);
 
@@ -1326,7 +1332,6 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         };
     }
 
-    const target = self.base.options.target;
     const is_obj = self.base.options.output_mode == .Obj;
 
     // Create an LLD command line and invoke it.
@@ -1337,7 +1342,6 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
     if (is_obj) {
         try argv.append("-r");
     }
-    const link_in_crt = self.base.options.link_libc and self.base.options.output_mode == .Exe;
 
     try argv.append("-error-limit=0");
 
@@ -1440,7 +1444,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         var test_path = std.ArrayList(u8).init(self.base.allocator);
         defer test_path.deinit();
         for (self.base.options.lib_dirs) |lib_dir_path| {
-            for (self.base.options.system_libs) |link_lib| {
+            for (self.base.options.system_libs.items()) |link_lib| {
                 test_path.shrinkRetainingCapacity(0);
                 const sep = fs.path.sep_str;
                 try test_path.writer().print("{}" ++ sep ++ "lib{}.so", .{ lib_dir_path, link_lib });
@@ -1509,8 +1513,10 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
     }
 
     // Shared libraries.
-    try argv.ensureCapacity(argv.items.len + self.base.options.system_libs.len);
-    for (self.base.options.system_libs) |link_lib| {
+    const system_libs = self.base.options.system_libs.items();
+    try argv.ensureCapacity(argv.items.len + system_libs.len);
+    for (system_libs) |entry| {
+        const link_lib = entry.key;
         // By this time, we depend on these libs being dynamically linked libraries and not static libraries
         // (the check for that needs to be earlier), but they could be full paths to .so files, in which
         // case we want to avoid prepending "-l".
@@ -1581,10 +1587,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
     }
 
     if (self.base.options.verbose_link) {
-        for (argv.items[0 .. argv.items.len - 1]) |arg| {
-            std.debug.print("{} ", .{arg});
-        }
-        std.debug.print("{}\n", .{argv.items[argv.items.len - 1]});
+        Compilation.dump_argv(argv.items);
     }
 
     // Oh, snapplesauce! We need null terminated argv.
diff --git a/src/mingw.zig b/src/mingw.zig
new file mode 100644
index 0000000000..2a2879de05
--- /dev/null
+++ b/src/mingw.zig
@@ -0,0 +1,866 @@
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const mem = std.mem;
+const path = std.fs.path;
+const assert = std.debug.assert;
+
+const target_util = @import("target.zig");
+const Compilation = @import("Compilation.zig");
+const build_options = @import("build_options");
+
+pub const CRTFile = enum {
+    crt2_o,
+    dllcrt2_o,
+    mingw32_lib,
+    msvcrt_os_lib,
+    mingwex_lib,
+    uuid_lib,
+};
+
+pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
+    if (!build_options.have_llvm) {
+        return error.ZigCompilerNotBuiltWithLLVMExtensions;
+    }
+    var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
+    defer arena_allocator.deinit();
+    const arena = &arena_allocator.allocator;
+
+    switch (crt_file) {
+        .crt2_o => {
+            var args = std.ArrayList([]const u8).init(arena);
+            try add_cc_args(comp, arena, &args);
+            try args.appendSlice(&[_][]const u8{
+                "-U__CRTDLL__",
+                "-D__MSVCRT__",
+                // Uncomment these 3 things for crtu
+                //"-DUNICODE",
+                //"-D_UNICODE",
+                //"-DWPRFLAG=1",
+            });
+            return comp.build_crt_file("crt2", .Obj, &[1]Compilation.CSourceFile{
+                .{
+                    .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                        "libc", "mingw", "crt", "crtexe.c",
+                    }),
+                    .extra_flags = args.items,
+                },
+            });
+        },
+
+        .dllcrt2_o => {
+            var args = std.ArrayList([]const u8).init(arena);
+            try add_cc_args(comp, arena, &args);
+            try args.appendSlice(&[_][]const u8{
+                "-U__CRTDLL__",
+                "-D__MSVCRT__",
+            });
+            return comp.build_crt_file("dllcrt2", .Obj, &[1]Compilation.CSourceFile{
+                .{
+                    .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                        "libc", "mingw", "crt", "crtdll.c",
+                    }),
+                    .extra_flags = args.items,
+                },
+            });
+        },
+
+        .mingw32_lib => {
+            var c_source_files: [mingw32_lib_deps.len]Compilation.CSourceFile = undefined;
+            for (mingw32_lib_deps) |dep, i| {
+                var args = std.ArrayList([]const u8).init(arena);
+                try args.appendSlice(&[_][]const u8{
+                    "-DHAVE_CONFIG_H",
+                    "-D_SYSCRT=1",
+                    "-DCRTDLL=1",
+
+                    "-isystem",
+                    try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                        "libc", "include", "any-windows-any",
+                    }),
+
+                    "-isystem",
+                    try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw", "include" }),
+
+                    "-std=gnu99",
+                    "-D_CRTBLD",
+                    "-D_WIN32_WINNT=0x0f00",
+                    "-D__MSVCRT_VERSION__=0x700",
+                    "-g",
+                    "-O2",
+                });
+                c_source_files[i] = .{
+                    .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                        "libc", "mingw", "crt", dep,
+                    }),
+                    .extra_flags = args.items,
+                };
+            }
+            return comp.build_crt_file("mingw32", .Lib, &c_source_files);
+        },
+
+        .msvcrt_os_lib => {
+            const extra_flags = try arena.dupe([]const u8, &[_][]const u8{
+                "-DHAVE_CONFIG_H",
+                "-D__LIBMSVCRT__",
+
+                "-I",
+                try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw", "include" }),
+
+                "-std=gnu99",
+                "-D_CRTBLD",
+                "-D_WIN32_WINNT=0x0f00",
+                "-D__MSVCRT_VERSION__=0x700",
+
+                "-isystem",
+                try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "include", "any-windows-any" }),
+
+                "-g",
+                "-O2",
+            });
+            var c_source_files = std.ArrayList(Compilation.CSourceFile).init(arena);
+
+            for (msvcrt_common_src) |dep| {
+                (try c_source_files.addOne()).* = .{
+                    .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw", dep }),
+                    .extra_flags = extra_flags,
+                };
+            }
+            if (comp.getTarget().cpu.arch == .i386) {
+                for (msvcrt_i386_src) |dep| {
+                    (try c_source_files.addOne()).* = .{
+                        .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                            "libc", "mingw", dep,
+                        }),
+                        .extra_flags = extra_flags,
+                    };
+                }
+            } else {
+                for (msvcrt_other_src) |dep| {
+                    (try c_source_files.addOne()).* = .{
+                        .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                            "libc", "mingw", dep,
+                        }),
+                        .extra_flags = extra_flags,
+                    };
+                }
+            }
+            return comp.build_crt_file("msvcrt-os", .Lib, c_source_files.items);
+        },
+
+        .mingwex_lib => {
+            const extra_flags = try arena.dupe([]const u8, &[_][]const u8{
+                "-DHAVE_CONFIG_H",
+
+                "-I",
+                try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw" }),
+
+                "-I",
+                try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw", "include" }),
+
+                "-std=gnu99",
+                "-D_CRTBLD",
+                "-D_WIN32_WINNT=0x0f00",
+                "-D__MSVCRT_VERSION__=0x700",
+                "-g",
+                "-O2",
+
+                "-isystem",
+                try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "include", "any-windows-any" }),
+            });
+            var c_source_files = std.ArrayList(Compilation.CSourceFile).init(arena);
+
+            for (mingwex_generic_src) |dep| {
+                (try c_source_files.addOne()).* = .{
+                    .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                        "libc", "mingw", dep,
+                    }),
+                    .extra_flags = extra_flags,
+                };
+            }
+            const target = comp.getTarget();
+            if (target.cpu.arch == .i386 or target.cpu.arch == .x86_64) {
+                for (mingwex_x86_src) |dep| {
+                    (try c_source_files.addOne()).* = .{
+                        .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                            "libc", "mingw", dep,
+                        }),
+                        .extra_flags = extra_flags,
+                    };
+                }
+            } else if (target.cpu.arch.isARM()) {
+                if (target.cpu.arch.ptrBitWidth() == 32) {
+                    for (mingwex_arm32_src) |dep| {
+                        (try c_source_files.addOne()).* = .{
+                            .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                                "libc", "mingw", dep,
+                            }),
+                            .extra_flags = extra_flags,
+                        };
+                    }
+                } else {
+                    for (mingwex_arm64_src) |dep| {
+                        (try c_source_files.addOne()).* = .{
+                            .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                                "libc", "mingw", dep,
+                            }),
+                            .extra_flags = extra_flags,
+                        };
+                    }
+                }
+            } else {
+                unreachable;
+            }
+            return comp.build_crt_file("mingwex", .Lib, c_source_files.items);
+        },
+
+        .uuid_lib => {
+            const extra_flags = try arena.dupe([]const u8, &[_][]const u8{
+                "-DHAVE_CONFIG_H",
+
+                "-I",
+                try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw" }),
+
+                "-I",
+                try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw", "include" }),
+
+                "-std=gnu99",
+                "-D_CRTBLD",
+                "-D_WIN32_WINNT=0x0f00",
+                "-D__MSVCRT_VERSION__=0x700",
+                "-g",
+                "-O2",
+
+                "-isystem",
+                try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                    "libc", "include", "any-windows-any",
+                }),
+            });
+            var c_source_files: [uuid_src.len]Compilation.CSourceFile = undefined;
+            for (uuid_src) |dep, i| {
+                c_source_files[i] = .{
+                    .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                        "libc", "mingw", "libsrc", dep,
+                    }),
+                    .extra_flags = extra_flags,
+                };
+            }
+            return comp.build_crt_file("uuid", .Lib, &c_source_files);
+        },
+    }
+}
+
+fn add_cc_args(
+    comp: *Compilation,
+    arena: *Allocator,
+    args: *std.ArrayList([]const u8),
+) error{OutOfMemory}!void {
+    try args.appendSlice(&[_][]const u8{
+        "-DHAVE_CONFIG_H",
+
+        "-I",
+        try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw", "include" }),
+
+        "-isystem",
+        try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "include", "any-windows-any" }),
+    });
+
+    const target = comp.getTarget();
+    if (target.cpu.arch.isARM() and target.cpu.arch.ptrBitWidth() == 32) {
+        try args.append("-mfpu=vfp");
+    }
+
+    try args.appendSlice(&[_][]const u8{
+        "-std=gnu11",
+        "-D_CRTBLD",
+        "-D_WIN32_WINNT=0x0f00",
+        "-D__MSVCRT_VERSION__=0x700",
+    });
+}
+
+const mingw32_lib_deps = [_][]const u8{
+    "crt0_c.c",
+    "dll_argv.c",
+    "gccmain.c",
+    "natstart.c",
+    "pseudo-reloc-list.c",
+    "wildcard.c",
+    "charmax.c",
+    "crt0_w.c",
+    "dllargv.c",
+    "gs_support.c",
+    "_newmode.c",
+    "tlssup.c",
+    "xncommod.c",
+    "cinitexe.c",
+    "merr.c",
+    "usermatherr.c",
+    "pesect.c",
+    "udllargc.c",
+    "xthdloc.c",
+    "CRT_fp10.c",
+    "mingw_helpers.c",
+    "pseudo-reloc.c",
+    "udll_argv.c",
+    "xtxtmode.c",
+    "crt_handler.c",
+    "tlsthrd.c",
+    "tlsmthread.c",
+    "tlsmcrt.c",
+    "cxa_atexit.c",
+};
+const msvcrt_common_src = [_][]const u8{
+    "misc" ++ path.sep_str ++ "_create_locale.c",
+    "misc" ++ path.sep_str ++ "_free_locale.c",
+    "misc" ++ path.sep_str ++ "onexit_table.c",
+    "misc" ++ path.sep_str ++ "register_tls_atexit.c",
+    "stdio" ++ path.sep_str ++ "acrt_iob_func.c",
+    "misc" ++ path.sep_str ++ "_configthreadlocale.c",
+    "misc" ++ path.sep_str ++ "_get_current_locale.c",
+    "misc" ++ path.sep_str ++ "invalid_parameter_handler.c",
+    "misc" ++ path.sep_str ++ "output_format.c",
+    "misc" ++ path.sep_str ++ "purecall.c",
+    "secapi" ++ path.sep_str ++ "_access_s.c",
+    "secapi" ++ path.sep_str ++ "_cgets_s.c",
+    "secapi" ++ path.sep_str ++ "_cgetws_s.c",
+    "secapi" ++ path.sep_str ++ "_chsize_s.c",
+    "secapi" ++ path.sep_str ++ "_controlfp_s.c",
+    "secapi" ++ path.sep_str ++ "_cprintf_s.c",
+    "secapi" ++ path.sep_str ++ "_cprintf_s_l.c",
+    "secapi" ++ path.sep_str ++ "_ctime32_s.c",
+    "secapi" ++ path.sep_str ++ "_ctime64_s.c",
+    "secapi" ++ path.sep_str ++ "_cwprintf_s.c",
+    "secapi" ++ path.sep_str ++ "_cwprintf_s_l.c",
+    "secapi" ++ path.sep_str ++ "_gmtime32_s.c",
+    "secapi" ++ path.sep_str ++ "_gmtime64_s.c",
+    "secapi" ++ path.sep_str ++ "_localtime32_s.c",
+    "secapi" ++ path.sep_str ++ "_localtime64_s.c",
+    "secapi" ++ path.sep_str ++ "_mktemp_s.c",
+    "secapi" ++ path.sep_str ++ "_sopen_s.c",
+    "secapi" ++ path.sep_str ++ "_strdate_s.c",
+    "secapi" ++ path.sep_str ++ "_strtime_s.c",
+    "secapi" ++ path.sep_str ++ "_umask_s.c",
+    "secapi" ++ path.sep_str ++ "_vcprintf_s.c",
+    "secapi" ++ path.sep_str ++ "_vcprintf_s_l.c",
+    "secapi" ++ path.sep_str ++ "_vcwprintf_s.c",
+    "secapi" ++ path.sep_str ++ "_vcwprintf_s_l.c",
+    "secapi" ++ path.sep_str ++ "_vscprintf_p.c",
+    "secapi" ++ path.sep_str ++ "_vscwprintf_p.c",
+    "secapi" ++ path.sep_str ++ "_vswprintf_p.c",
+    "secapi" ++ path.sep_str ++ "_waccess_s.c",
+    "secapi" ++ path.sep_str ++ "_wasctime_s.c",
+    "secapi" ++ path.sep_str ++ "_wctime32_s.c",
+    "secapi" ++ path.sep_str ++ "_wctime64_s.c",
+    "secapi" ++ path.sep_str ++ "_wstrtime_s.c",
+    "secapi" ++ path.sep_str ++ "_wmktemp_s.c",
+    "secapi" ++ path.sep_str ++ "_wstrdate_s.c",
+    "secapi" ++ path.sep_str ++ "asctime_s.c",
+    "secapi" ++ path.sep_str ++ "memcpy_s.c",
+    "secapi" ++ path.sep_str ++ "memmove_s.c",
+    "secapi" ++ path.sep_str ++ "rand_s.c",
+    "secapi" ++ path.sep_str ++ "sprintf_s.c",
+    "secapi" ++ path.sep_str ++ "strerror_s.c",
+    "secapi" ++ path.sep_str ++ "vsprintf_s.c",
+    "secapi" ++ path.sep_str ++ "wmemcpy_s.c",
+    "secapi" ++ path.sep_str ++ "wmemmove_s.c",
+    "stdio" ++ path.sep_str ++ "mingw_lock.c",
+};
+const msvcrt_i386_src = [_][]const u8{
+    "misc" ++ path.sep_str ++ "lc_locale_func.c",
+    "misc" ++ path.sep_str ++ "___mb_cur_max_func.c",
+};
+
+const msvcrt_other_src = [_][]const u8{
+    "misc" ++ path.sep_str ++ "__p___argv.c",
+    "misc" ++ path.sep_str ++ "__p__acmdln.c",
+    "misc" ++ path.sep_str ++ "__p__fmode.c",
+    "misc" ++ path.sep_str ++ "__p__wcmdln.c",
+};
+const mingwex_generic_src = [_][]const u8{
+    "complex" ++ path.sep_str ++ "_cabs.c",
+    "complex" ++ path.sep_str ++ "cabs.c",
+    "complex" ++ path.sep_str ++ "cabsf.c",
+    "complex" ++ path.sep_str ++ "cabsl.c",
+    "complex" ++ path.sep_str ++ "cacos.c",
+    "complex" ++ path.sep_str ++ "cacosf.c",
+    "complex" ++ path.sep_str ++ "cacosl.c",
+    "complex" ++ path.sep_str ++ "carg.c",
+    "complex" ++ path.sep_str ++ "cargf.c",
+    "complex" ++ path.sep_str ++ "cargl.c",
+    "complex" ++ path.sep_str ++ "casin.c",
+    "complex" ++ path.sep_str ++ "casinf.c",
+    "complex" ++ path.sep_str ++ "casinl.c",
+    "complex" ++ path.sep_str ++ "catan.c",
+    "complex" ++ path.sep_str ++ "catanf.c",
+    "complex" ++ path.sep_str ++ "catanl.c",
+    "complex" ++ path.sep_str ++ "ccos.c",
+    "complex" ++ path.sep_str ++ "ccosf.c",
+    "complex" ++ path.sep_str ++ "ccosl.c",
+    "complex" ++ path.sep_str ++ "cexp.c",
+    "complex" ++ path.sep_str ++ "cexpf.c",
+    "complex" ++ path.sep_str ++ "cexpl.c",
+    "complex" ++ path.sep_str ++ "cimag.c",
+    "complex" ++ path.sep_str ++ "cimagf.c",
+    "complex" ++ path.sep_str ++ "cimagl.c",
+    "complex" ++ path.sep_str ++ "clog.c",
+    "complex" ++ path.sep_str ++ "clog10.c",
+    "complex" ++ path.sep_str ++ "clog10f.c",
+    "complex" ++ path.sep_str ++ "clog10l.c",
+    "complex" ++ path.sep_str ++ "clogf.c",
+    "complex" ++ path.sep_str ++ "clogl.c",
+    "complex" ++ path.sep_str ++ "conj.c",
+    "complex" ++ path.sep_str ++ "conjf.c",
+    "complex" ++ path.sep_str ++ "conjl.c",
+    "complex" ++ path.sep_str ++ "cpow.c",
+    "complex" ++ path.sep_str ++ "cpowf.c",
+    "complex" ++ path.sep_str ++ "cpowl.c",
+    "complex" ++ path.sep_str ++ "cproj.c",
+    "complex" ++ path.sep_str ++ "cprojf.c",
+    "complex" ++ path.sep_str ++ "cprojl.c",
+    "complex" ++ path.sep_str ++ "creal.c",
+    "complex" ++ path.sep_str ++ "crealf.c",
+    "complex" ++ path.sep_str ++ "creall.c",
+    "complex" ++ path.sep_str ++ "csin.c",
+    "complex" ++ path.sep_str ++ "csinf.c",
+    "complex" ++ path.sep_str ++ "csinl.c",
+    "complex" ++ path.sep_str ++ "csqrt.c",
+    "complex" ++ path.sep_str ++ "csqrtf.c",
+    "complex" ++ path.sep_str ++ "csqrtl.c",
+    "complex" ++ path.sep_str ++ "ctan.c",
+    "complex" ++ path.sep_str ++ "ctanf.c",
+    "complex" ++ path.sep_str ++ "ctanl.c",
+    "crt" ++ path.sep_str ++ "dllentry.c",
+    "crt" ++ path.sep_str ++ "dllmain.c",
+    "gdtoa" ++ path.sep_str ++ "arithchk.c",
+    "gdtoa" ++ path.sep_str ++ "dmisc.c",
+    "gdtoa" ++ path.sep_str ++ "dtoa.c",
+    "gdtoa" ++ path.sep_str ++ "g__fmt.c",
+    "gdtoa" ++ path.sep_str ++ "g_dfmt.c",
+    "gdtoa" ++ path.sep_str ++ "g_ffmt.c",
+    "gdtoa" ++ path.sep_str ++ "g_xfmt.c",
+    "gdtoa" ++ path.sep_str ++ "gdtoa.c",
+    "gdtoa" ++ path.sep_str ++ "gethex.c",
+    "gdtoa" ++ path.sep_str ++ "gmisc.c",
+    "gdtoa" ++ path.sep_str ++ "hd_init.c",
+    "gdtoa" ++ path.sep_str ++ "hexnan.c",
+    "gdtoa" ++ path.sep_str ++ "misc.c",
+    "gdtoa" ++ path.sep_str ++ "qnan.c",
+    "gdtoa" ++ path.sep_str ++ "smisc.c",
+    "gdtoa" ++ path.sep_str ++ "strtodg.c",
+    "gdtoa" ++ path.sep_str ++ "strtodnrp.c",
+    "gdtoa" ++ path.sep_str ++ "strtof.c",
+    "gdtoa" ++ path.sep_str ++ "strtopx.c",
+    "gdtoa" ++ path.sep_str ++ "sum.c",
+    "gdtoa" ++ path.sep_str ++ "ulp.c",
+    "math" ++ path.sep_str ++ "abs64.c",
+    "math" ++ path.sep_str ++ "cbrt.c",
+    "math" ++ path.sep_str ++ "cbrtf.c",
+    "math" ++ path.sep_str ++ "cbrtl.c",
+    "math" ++ path.sep_str ++ "cephes_emath.c",
+    "math" ++ path.sep_str ++ "copysign.c",
+    "math" ++ path.sep_str ++ "copysignf.c",
+    "math" ++ path.sep_str ++ "coshf.c",
+    "math" ++ path.sep_str ++ "coshl.c",
+    "math" ++ path.sep_str ++ "erfl.c",
+    "math" ++ path.sep_str ++ "expf.c",
+    "math" ++ path.sep_str ++ "fabs.c",
+    "math" ++ path.sep_str ++ "fabsf.c",
+    "math" ++ path.sep_str ++ "fabsl.c",
+    "math" ++ path.sep_str ++ "fdim.c",
+    "math" ++ path.sep_str ++ "fdimf.c",
+    "math" ++ path.sep_str ++ "fdiml.c",
+    "math" ++ path.sep_str ++ "fma.c",
+    "math" ++ path.sep_str ++ "fmaf.c",
+    "math" ++ path.sep_str ++ "fmal.c",
+    "math" ++ path.sep_str ++ "fmax.c",
+    "math" ++ path.sep_str ++ "fmaxf.c",
+    "math" ++ path.sep_str ++ "fmaxl.c",
+    "math" ++ path.sep_str ++ "fmin.c",
+    "math" ++ path.sep_str ++ "fminf.c",
+    "math" ++ path.sep_str ++ "fminl.c",
+    "math" ++ path.sep_str ++ "fp_consts.c",
+    "math" ++ path.sep_str ++ "fp_constsf.c",
+    "math" ++ path.sep_str ++ "fp_constsl.c",
+    "math" ++ path.sep_str ++ "fpclassify.c",
+    "math" ++ path.sep_str ++ "fpclassifyf.c",
+    "math" ++ path.sep_str ++ "fpclassifyl.c",
+    "math" ++ path.sep_str ++ "frexpf.c",
+    "math" ++ path.sep_str ++ "hypot.c",
+    "math" ++ path.sep_str ++ "hypotf.c",
+    "math" ++ path.sep_str ++ "hypotl.c",
+    "math" ++ path.sep_str ++ "isnan.c",
+    "math" ++ path.sep_str ++ "isnanf.c",
+    "math" ++ path.sep_str ++ "isnanl.c",
+    "math" ++ path.sep_str ++ "ldexpf.c",
+    "math" ++ path.sep_str ++ "lgamma.c",
+    "math" ++ path.sep_str ++ "lgammaf.c",
+    "math" ++ path.sep_str ++ "lgammal.c",
+    "math" ++ path.sep_str ++ "llrint.c",
+    "math" ++ path.sep_str ++ "llrintf.c",
+    "math" ++ path.sep_str ++ "llrintl.c",
+    "math" ++ path.sep_str ++ "llround.c",
+    "math" ++ path.sep_str ++ "llroundf.c",
+    "math" ++ path.sep_str ++ "llroundl.c",
+    "math" ++ path.sep_str ++ "log10f.c",
+    "math" ++ path.sep_str ++ "logf.c",
+    "math" ++ path.sep_str ++ "lrint.c",
+    "math" ++ path.sep_str ++ "lrintf.c",
+    "math" ++ path.sep_str ++ "lrintl.c",
+    "math" ++ path.sep_str ++ "lround.c",
+    "math" ++ path.sep_str ++ "lroundf.c",
+    "math" ++ path.sep_str ++ "lroundl.c",
+    "math" ++ path.sep_str ++ "modf.c",
+    "math" ++ path.sep_str ++ "modff.c",
+    "math" ++ path.sep_str ++ "modfl.c",
+    "math" ++ path.sep_str ++ "nextafterf.c",
+    "math" ++ path.sep_str ++ "nextafterl.c",
+    "math" ++ path.sep_str ++ "nexttoward.c",
+    "math" ++ path.sep_str ++ "nexttowardf.c",
+    "math" ++ path.sep_str ++ "powf.c",
+    "math" ++ path.sep_str ++ "powi.c",
+    "math" ++ path.sep_str ++ "powif.c",
+    "math" ++ path.sep_str ++ "powil.c",
+    "math" ++ path.sep_str ++ "rint.c",
+    "math" ++ path.sep_str ++ "rintf.c",
+    "math" ++ path.sep_str ++ "rintl.c",
+    "math" ++ path.sep_str ++ "round.c",
+    "math" ++ path.sep_str ++ "roundf.c",
+    "math" ++ path.sep_str ++ "roundl.c",
+    "math" ++ path.sep_str ++ "s_erf.c",
+    "math" ++ path.sep_str ++ "sf_erf.c",
+    "math" ++ path.sep_str ++ "signbit.c",
+    "math" ++ path.sep_str ++ "signbitf.c",
+    "math" ++ path.sep_str ++ "signbitl.c",
+    "math" ++ path.sep_str ++ "signgam.c",
+    "math" ++ path.sep_str ++ "sinhf.c",
+    "math" ++ path.sep_str ++ "sinhl.c",
+    "math" ++ path.sep_str ++ "sqrt.c",
+    "math" ++ path.sep_str ++ "sqrtf.c",
+    "math" ++ path.sep_str ++ "sqrtl.c",
+    "math" ++ path.sep_str ++ "tanhf.c",
+    "math" ++ path.sep_str ++ "tanhl.c",
+    "math" ++ path.sep_str ++ "tgamma.c",
+    "math" ++ path.sep_str ++ "tgammaf.c",
+    "math" ++ path.sep_str ++ "tgammal.c",
+    "math" ++ path.sep_str ++ "truncl.c",
+    "misc" ++ path.sep_str ++ "alarm.c",
+    "misc" ++ path.sep_str ++ "basename.c",
+    "misc" ++ path.sep_str ++ "btowc.c",
+    "misc" ++ path.sep_str ++ "delay-f.c",
+    "misc" ++ path.sep_str ++ "delay-n.c",
+    "misc" ++ path.sep_str ++ "delayimp.c",
+    "misc" ++ path.sep_str ++ "dirent.c",
+    "misc" ++ path.sep_str ++ "dirname.c",
+    "misc" ++ path.sep_str ++ "feclearexcept.c",
+    "misc" ++ path.sep_str ++ "fegetenv.c",
+    "misc" ++ path.sep_str ++ "fegetexceptflag.c",
+    "misc" ++ path.sep_str ++ "fegetround.c",
+    "misc" ++ path.sep_str ++ "feholdexcept.c",
+    "misc" ++ path.sep_str ++ "feraiseexcept.c",
+    "misc" ++ path.sep_str ++ "fesetenv.c",
+    "misc" ++ path.sep_str ++ "fesetexceptflag.c",
+    "misc" ++ path.sep_str ++ "fesetround.c",
+    "misc" ++ path.sep_str ++ "fetestexcept.c",
+    "misc" ++ path.sep_str ++ "feupdateenv.c",
+    "misc" ++ path.sep_str ++ "ftruncate.c",
+    "misc" ++ path.sep_str ++ "ftw.c",
+    "misc" ++ path.sep_str ++ "ftw64.c",
+    "misc" ++ path.sep_str ++ "fwide.c",
+    "misc" ++ path.sep_str ++ "getlogin.c",
+    "misc" ++ path.sep_str ++ "getopt.c",
+    "misc" ++ path.sep_str ++ "gettimeofday.c",
+    "misc" ++ path.sep_str ++ "imaxabs.c",
+    "misc" ++ path.sep_str ++ "imaxdiv.c",
+    "misc" ++ path.sep_str ++ "isblank.c",
+    "misc" ++ path.sep_str ++ "iswblank.c",
+    "misc" ++ path.sep_str ++ "mbrtowc.c",
+    "misc" ++ path.sep_str ++ "mbsinit.c",
+    "misc" ++ path.sep_str ++ "mempcpy.c",
+    "misc" ++ path.sep_str ++ "mingw-aligned-malloc.c",
+    "misc" ++ path.sep_str ++ "mingw-fseek.c",
+    "misc" ++ path.sep_str ++ "mingw_getsp.S",
+    "misc" ++ path.sep_str ++ "mingw_matherr.c",
+    "misc" ++ path.sep_str ++ "mingw_mbwc_convert.c",
+    "misc" ++ path.sep_str ++ "mingw_usleep.c",
+    "misc" ++ path.sep_str ++ "mingw_wcstod.c",
+    "misc" ++ path.sep_str ++ "mingw_wcstof.c",
+    "misc" ++ path.sep_str ++ "mingw_wcstold.c",
+    "misc" ++ path.sep_str ++ "mkstemp.c",
+    "misc" ++ path.sep_str ++ "seterrno.c",
+    "misc" ++ path.sep_str ++ "sleep.c",
+    "misc" ++ path.sep_str ++ "strnlen.c",
+    "misc" ++ path.sep_str ++ "strsafe.c",
+    "misc" ++ path.sep_str ++ "strtoimax.c",
+    "misc" ++ path.sep_str ++ "strtold.c",
+    "misc" ++ path.sep_str ++ "strtoumax.c",
+    "misc" ++ path.sep_str ++ "tdelete.c",
+    "misc" ++ path.sep_str ++ "tfind.c",
+    "misc" ++ path.sep_str ++ "tsearch.c",
+    "misc" ++ path.sep_str ++ "twalk.c",
+    "misc" ++ path.sep_str ++ "uchar_c16rtomb.c",
+    "misc" ++ path.sep_str ++ "uchar_c32rtomb.c",
+    "misc" ++ path.sep_str ++ "uchar_mbrtoc16.c",
+    "misc" ++ path.sep_str ++ "uchar_mbrtoc32.c",
+    "misc" ++ path.sep_str ++ "wassert.c",
+    "misc" ++ path.sep_str ++ "wcrtomb.c",
+    "misc" ++ path.sep_str ++ "wcsnlen.c",
+    "misc" ++ path.sep_str ++ "wcstof.c",
+    "misc" ++ path.sep_str ++ "wcstoimax.c",
+    "misc" ++ path.sep_str ++ "wcstold.c",
+    "misc" ++ path.sep_str ++ "wcstoumax.c",
+    "misc" ++ path.sep_str ++ "wctob.c",
+    "misc" ++ path.sep_str ++ "wctrans.c",
+    "misc" ++ path.sep_str ++ "wctype.c",
+    "misc" ++ path.sep_str ++ "wdirent.c",
+    "misc" ++ path.sep_str ++ "winbs_uint64.c",
+    "misc" ++ path.sep_str ++ "winbs_ulong.c",
+    "misc" ++ path.sep_str ++ "winbs_ushort.c",
+    "misc" ++ path.sep_str ++ "wmemchr.c",
+    "misc" ++ path.sep_str ++ "wmemcmp.c",
+    "misc" ++ path.sep_str ++ "wmemcpy.c",
+    "misc" ++ path.sep_str ++ "wmemmove.c",
+    "misc" ++ path.sep_str ++ "wmempcpy.c",
+    "misc" ++ path.sep_str ++ "wmemset.c",
+    "stdio" ++ path.sep_str ++ "_Exit.c",
+    "stdio" ++ path.sep_str ++ "_findfirst64i32.c",
+    "stdio" ++ path.sep_str ++ "_findnext64i32.c",
+    "stdio" ++ path.sep_str ++ "_fstat.c",
+    "stdio" ++ path.sep_str ++ "_fstat64i32.c",
+    "stdio" ++ path.sep_str ++ "_ftime.c",
+    "stdio" ++ path.sep_str ++ "_getc_nolock.c",
+    "stdio" ++ path.sep_str ++ "_getwc_nolock.c",
+    "stdio" ++ path.sep_str ++ "_putc_nolock.c",
+    "stdio" ++ path.sep_str ++ "_putwc_nolock.c",
+    "stdio" ++ path.sep_str ++ "_stat.c",
+    "stdio" ++ path.sep_str ++ "_stat64i32.c",
+    "stdio" ++ path.sep_str ++ "_wfindfirst64i32.c",
+    "stdio" ++ path.sep_str ++ "_wfindnext64i32.c",
+    "stdio" ++ path.sep_str ++ "_wstat.c",
+    "stdio" ++ path.sep_str ++ "_wstat64i32.c",
+    "stdio" ++ path.sep_str ++ "asprintf.c",
+    "stdio" ++ path.sep_str ++ "atoll.c",
+    "stdio" ++ path.sep_str ++ "fgetpos64.c",
+    "stdio" ++ path.sep_str ++ "fopen64.c",
+    "stdio" ++ path.sep_str ++ "fseeko32.c",
+    "stdio" ++ path.sep_str ++ "fseeko64.c",
+    "stdio" ++ path.sep_str ++ "fsetpos64.c",
+    "stdio" ++ path.sep_str ++ "ftello.c",
+    "stdio" ++ path.sep_str ++ "ftello64.c",
+    "stdio" ++ path.sep_str ++ "ftruncate64.c",
+    "stdio" ++ path.sep_str ++ "lltoa.c",
+    "stdio" ++ path.sep_str ++ "lltow.c",
+    "stdio" ++ path.sep_str ++ "lseek64.c",
+    "stdio" ++ path.sep_str ++ "mingw_asprintf.c",
+    "stdio" ++ path.sep_str ++ "mingw_fprintf.c",
+    "stdio" ++ path.sep_str ++ "mingw_fprintfw.c",
+    "stdio" ++ path.sep_str ++ "mingw_fscanf.c",
+    "stdio" ++ path.sep_str ++ "mingw_fwscanf.c",
+    "stdio" ++ path.sep_str ++ "mingw_pformat.c",
+    "stdio" ++ path.sep_str ++ "mingw_pformatw.c",
+    "stdio" ++ path.sep_str ++ "mingw_printf.c",
+    "stdio" ++ path.sep_str ++ "mingw_printfw.c",
+    "stdio" ++ path.sep_str ++ "mingw_scanf.c",
+    "stdio" ++ path.sep_str ++ "mingw_snprintf.c",
+    "stdio" ++ path.sep_str ++ "mingw_snprintfw.c",
+    "stdio" ++ path.sep_str ++ "mingw_sprintf.c",
+    "stdio" ++ path.sep_str ++ "mingw_sprintfw.c",
+    "stdio" ++ path.sep_str ++ "mingw_sscanf.c",
+    "stdio" ++ path.sep_str ++ "mingw_swscanf.c",
+    "stdio" ++ path.sep_str ++ "mingw_vasprintf.c",
+    "stdio" ++ path.sep_str ++ "mingw_vfprintf.c",
+    "stdio" ++ path.sep_str ++ "mingw_vfprintfw.c",
+    "stdio" ++ path.sep_str ++ "mingw_vfscanf.c",
+    "stdio" ++ path.sep_str ++ "mingw_vprintf.c",
+    "stdio" ++ path.sep_str ++ "mingw_vprintfw.c",
+    "stdio" ++ path.sep_str ++ "mingw_vsnprintf.c",
+    "stdio" ++ path.sep_str ++ "mingw_vsnprintfw.c",
+    "stdio" ++ path.sep_str ++ "mingw_vsprintf.c",
+    "stdio" ++ path.sep_str ++ "mingw_vsprintfw.c",
+    "stdio" ++ path.sep_str ++ "mingw_wscanf.c",
+    "stdio" ++ path.sep_str ++ "mingw_wvfscanf.c",
+    "stdio" ++ path.sep_str ++ "scanf.S",
+    "stdio" ++ path.sep_str ++ "snprintf.c",
+    "stdio" ++ path.sep_str ++ "snwprintf.c",
+    "stdio" ++ path.sep_str ++ "strtof.c",
+    "stdio" ++ path.sep_str ++ "strtok_r.c",
+    "stdio" ++ path.sep_str ++ "truncate.c",
+    "stdio" ++ path.sep_str ++ "ulltoa.c",
+    "stdio" ++ path.sep_str ++ "ulltow.c",
+    "stdio" ++ path.sep_str ++ "vasprintf.c",
+    "stdio" ++ path.sep_str ++ "vfscanf.c",
+    "stdio" ++ path.sep_str ++ "vfscanf2.S",
+    "stdio" ++ path.sep_str ++ "vfwscanf.c",
+    "stdio" ++ path.sep_str ++ "vfwscanf2.S",
+    "stdio" ++ path.sep_str ++ "vscanf.c",
+    "stdio" ++ path.sep_str ++ "vscanf2.S",
+    "stdio" ++ path.sep_str ++ "vsnprintf.c",
+    "stdio" ++ path.sep_str ++ "vsnwprintf.c",
+    "stdio" ++ path.sep_str ++ "vsscanf.c",
+    "stdio" ++ path.sep_str ++ "vsscanf2.S",
+    "stdio" ++ path.sep_str ++ "vswscanf.c",
+    "stdio" ++ path.sep_str ++ "vswscanf2.S",
+    "stdio" ++ path.sep_str ++ "vwscanf.c",
+    "stdio" ++ path.sep_str ++ "vwscanf2.S",
+    "stdio" ++ path.sep_str ++ "wtoll.c",
+};
+
+const mingwex_x86_src = [_][]const u8{
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "acosf.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "acosh.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "acoshf.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "acoshl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "acosl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "asinf.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "asinh.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "asinhf.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "asinhl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "asinl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "atan2.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "atan2f.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "atan2l.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "atanf.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "atanh.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "atanhf.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "atanhl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "atanl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "ceilf.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "ceill.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "ceil.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "_chgsignl.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "copysignl.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "cos.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "cosf.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "cosl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "cosl_internal.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "cossin.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "exp2f.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "exp2l.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "exp2.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "exp.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "expl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "expm1.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "expm1f.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "expm1l.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "floorf.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "floorl.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "floor.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "fmod.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "fmodf.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "fmodl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "fucom.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "ilogbf.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "ilogbl.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "ilogb.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "internal_logl.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "ldexp.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "ldexpl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "log10l.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "log1pf.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "log1pl.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "log1p.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "log2f.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "log2l.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "log2.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "logb.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "logbf.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "logbl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "log.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "logl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "nearbyintf.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "nearbyintl.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "nearbyint.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "pow.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "powl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "remainderf.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "remainderl.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "remainder.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "remquof.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "remquol.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "remquo.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "scalbnf.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "scalbnl.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "scalbn.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "sin.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "sinf.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "sinl.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "sinl_internal.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "tanf.c",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "tanl.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "truncf.S",
+    "math" ++ path.sep_str ++ "x86" ++ path.sep_str ++ "trunc.S",
+};
+
+const mingwex_arm32_src = [_][]const u8{
+    "math" ++ path.sep_str ++ "arm" ++ path.sep_str ++ "_chgsignl.S",
+    "math" ++ path.sep_str ++ "arm" ++ path.sep_str ++ "exp2.c",
+    "math" ++ path.sep_str ++ "arm" ++ path.sep_str ++ "nearbyint.S",
+    "math" ++ path.sep_str ++ "arm" ++ path.sep_str ++ "nearbyintf.S",
+    "math" ++ path.sep_str ++ "arm" ++ path.sep_str ++ "nearbyintl.S",
+    "math" ++ path.sep_str ++ "arm" ++ path.sep_str ++ "trunc.S",
+    "math" ++ path.sep_str ++ "arm" ++ path.sep_str ++ "truncf.S",
+};
+
+const mingwex_arm64_src = [_][]const u8{
+    "math" ++ path.sep_str ++ "arm64" ++ path.sep_str ++ "_chgsignl.S",
+    "math" ++ path.sep_str ++ "arm64" ++ path.sep_str ++ "exp2f.S",
+    "math" ++ path.sep_str ++ "arm64" ++ path.sep_str ++ "exp2.S",
+    "math" ++ path.sep_str ++ "arm64" ++ path.sep_str ++ "nearbyintf.S",
+    "math" ++ path.sep_str ++ "arm64" ++ path.sep_str ++ "nearbyintl.S",
+    "math" ++ path.sep_str ++ "arm64" ++ path.sep_str ++ "nearbyint.S",
+    "math" ++ path.sep_str ++ "arm64" ++ path.sep_str ++ "truncf.S",
+    "math" ++ path.sep_str ++ "arm64" ++ path.sep_str ++ "trunc.S",
+};
+
+const uuid_src = [_][]const u8{
+    "ativscp-uuid.c",
+    "atsmedia-uuid.c",
+    "bth-uuid.c",
+    "cguid-uuid.c",
+    "comcat-uuid.c",
+    "devguid.c",
+    "docobj-uuid.c",
+    "dxva-uuid.c",
+    "exdisp-uuid.c",
+    "extras-uuid.c",
+    "fwp-uuid.c",
+    "guid_nul.c",
+    "hlguids-uuid.c",
+    "hlink-uuid.c",
+    "mlang-uuid.c",
+    "msctf-uuid.c",
+    "mshtmhst-uuid.c",
+    "mshtml-uuid.c",
+    "msxml-uuid.c",
+    "netcon-uuid.c",
+    "ntddkbd-uuid.c",
+    "ntddmou-uuid.c",
+    "ntddpar-uuid.c",
+    "ntddscsi-uuid.c",
+    "ntddser-uuid.c",
+    "ntddstor-uuid.c",
+    "ntddvdeo-uuid.c",
+    "oaidl-uuid.c",
+    "objidl-uuid.c",
+    "objsafe-uuid.c",
+    "ocidl-uuid.c",
+    "oleacc-uuid.c",
+    "olectlid-uuid.c",
+    "oleidl-uuid.c",
+    "power-uuid.c",
+    "powrprof-uuid.c",
+    "uianimation-uuid.c",
+    "usbcamdi-uuid.c",
+    "usbiodef-uuid.c",
+    "uuid.c",
+    "vds-uuid.c",
+    "virtdisk-uuid.c",
+    "wia-uuid.c",
+};
+
+pub const always_link_libs = [_][]const u8{
+    "advapi32",
+    "kernel32",
+    "msvcrt",
+    "ntdll",
+    "shell32",
+    "user32",
+};
diff --git a/src/musl.zig b/src/musl.zig
index d64d643227..ef4ea7236b 100644
--- a/src/musl.zig
+++ b/src/musl.zig
@@ -7,9 +7,6 @@ const assert = std.debug.assert;
 const target_util = @import("target.zig");
 const Compilation = @import("Compilation.zig");
 const build_options = @import("build_options");
-const trace = @import("tracy.zig").trace;
-const Cache = @import("Cache.zig");
-const Package = @import("Package.zig");
 
 pub const CRTFile = enum {
     crti_o,
diff --git a/src/stage1.zig b/src/stage1.zig
index 2124c23f14..8142ce901c 100644
--- a/src/stage1.zig
+++ b/src/stage1.zig
@@ -121,6 +121,14 @@ pub const Module = extern struct {
     verbose_cimport: bool,
     verbose_llvm_cpu_features: bool,
 
+    // Set by stage1
+    have_c_main: bool,
+    have_winmain: bool,
+    have_wwinmain: bool,
+    have_winmain_crt_startup: bool,
+    have_wwinmain_crt_startup: bool,
+    have_dllmain_crt_startup: bool,
+
     pub fn build_object(mod: *Module) void {
         zig_stage1_build_object(mod);
     }
diff --git a/src/stage1/all_types.hpp b/src/stage1/all_types.hpp
index a1d0cd0aa1..7a5016d004 100644
--- a/src/stage1/all_types.hpp
+++ b/src/stage1/all_types.hpp
@@ -2152,12 +2152,6 @@ struct CodeGen {
     uint32_t next_unresolved_index;
     unsigned pointer_size_bytes;
     bool is_big_endian;
-    bool have_c_main;
-    bool have_winmain;
-    bool have_wwinmain;
-    bool have_winmain_crt_startup;
-    bool have_wwinmain_crt_startup;
-    bool have_dllmain_crt_startup;
     bool have_err_ret_tracing;
     bool verbose_tokenize;
     bool verbose_ast;
diff --git a/src/stage1/analyze.cpp b/src/stage1/analyze.cpp
index 15c30350d7..369c284684 100644
--- a/src/stage1/analyze.cpp
+++ b/src/stage1/analyze.cpp
@@ -3496,18 +3496,18 @@ void add_var_export(CodeGen *g, ZigVar *var, const char *symbol_name, GlobalLink
 
 void add_fn_export(CodeGen *g, ZigFn *fn_table_entry, const char *symbol_name, GlobalLinkageId linkage, CallingConvention cc) {
     if (cc == CallingConventionC && strcmp(symbol_name, "main") == 0 && g->link_libc) {
-        g->have_c_main = true;
+        g->stage1.have_c_main = true;
     } else if (cc == CallingConventionStdcall && g->zig_target->os == OsWindows) {
         if (strcmp(symbol_name, "WinMain") == 0) {
-            g->have_winmain = true;
+            g->stage1.have_winmain = true;
         } else if (strcmp(symbol_name, "wWinMain") == 0) {
-            g->have_wwinmain = true;
+            g->stage1.have_wwinmain = true;
         } else if (strcmp(symbol_name, "WinMainCRTStartup") == 0) {
-            g->have_winmain_crt_startup = true;
+            g->stage1.have_winmain_crt_startup = true;
         } else if (strcmp(symbol_name, "wWinMainCRTStartup") == 0) {
-            g->have_wwinmain_crt_startup = true;
+            g->stage1.have_wwinmain_crt_startup = true;
         } else if (strcmp(symbol_name, "DllMainCRTStartup") == 0) {
-            g->have_dllmain_crt_startup = true;
+            g->stage1.have_dllmain_crt_startup = true;
         }
     }
 
diff --git a/src/stage1/codegen.cpp b/src/stage1/codegen.cpp
index fff1c325fc..2fc85b42fe 100644
--- a/src/stage1/codegen.cpp
+++ b/src/stage1/codegen.cpp
@@ -8666,11 +8666,11 @@ TargetSubsystem detect_subsystem(CodeGen *g) {
     if (g->subsystem != TargetSubsystemAuto)
         return g->subsystem;
     if (g->zig_target->os == OsWindows) {
-        if (g->have_dllmain_crt_startup)
+        if (g->stage1.have_dllmain_crt_startup)
             return TargetSubsystemAuto;
-        if (g->have_c_main || g->is_test_build || g->have_winmain_crt_startup || g->have_wwinmain_crt_startup)
+        if (g->stage1.have_c_main || g->is_test_build || g->stage1.have_winmain_crt_startup || g->stage1.have_wwinmain_crt_startup)
             return TargetSubsystemConsole;
-        if (g->have_winmain || g->have_wwinmain)
+        if (g->stage1.have_winmain || g->stage1.have_wwinmain)
             return TargetSubsystemWindows;
     } else if (g->zig_target->os == OsUefi) {
         return TargetSubsystemEfiApplication;
diff --git a/src/stage1/stage1.h b/src/stage1/stage1.h
index efbd02e393..412118a6fa 100644
--- a/src/stage1/stage1.h
+++ b/src/stage1/stage1.h
@@ -195,6 +195,14 @@ struct ZigStage1 {
     bool verbose_llvm_ir;
     bool verbose_cimport;
     bool verbose_llvm_cpu_features;
+
+    // Set by stage1
+    bool have_c_main;
+    bool have_winmain;
+    bool have_wwinmain;
+    bool have_winmain_crt_startup;
+    bool have_wwinmain_crt_startup;
+    bool have_dllmain_crt_startup;
 };
 
 ZIG_EXTERN_C void zig_stage1_os_init(void);

From 933146699806fc25f8182f53cd8c217654d51664 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20=28xq=29=20Quei=C3=9Fner?= 
Date: Mon, 28 Sep 2020 11:42:39 +0200
Subject: [PATCH 164/210] Changes comptime block to test.

---
 lib/std/meta.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/std/meta.zig b/lib/std/meta.zig
index c92637250f..5b4920157a 100644
--- a/lib/std/meta.zig
+++ b/lib/std/meta.zig
@@ -857,7 +857,7 @@ pub fn Tuple(comptime types: anytype) type {
     });
 }
 
-comptime {
+test "Tuple" {
     const T = struct {
         fn assertTypeEqual(comptime Expected: type, comptime Actual: type) void {
             if (Expected != Actual)

From 55dfe729b480c2ba121c6f650d160921560d9535 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20=28xq=29=20Quei=C3=9Fner?= 
Date: Mon, 28 Sep 2020 11:44:55 +0200
Subject: [PATCH 165/210] Changes comptime block to test.

---
 lib/std/meta.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/std/meta.zig b/lib/std/meta.zig
index 98f3b284c1..2744690555 100644
--- a/lib/std/meta.zig
+++ b/lib/std/meta.zig
@@ -867,7 +867,7 @@ pub fn ArgsTuple(comptime Function: type) type {
     });
 }
 
-comptime {
+test "ArgsTuple" {
     const T = struct {
         fn assertTypeEqual(comptime Expected: type, comptime Actual: type) void {
             if (Expected != Actual)

From c2d60bc5b5dd9ac39760588c777cc6809732af19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20=28xq=29=20Quei=C3=9Fner?= 
Date: Mon, 28 Sep 2020 12:24:22 +0200
Subject: [PATCH 166/210] Follows @tadeokondrak remark about taking `[]const
 type`.

---
 lib/std/meta.zig | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/std/meta.zig b/lib/std/meta.zig
index 5b4920157a..13d3c574e9 100644
--- a/lib/std/meta.zig
+++ b/lib/std/meta.zig
@@ -831,10 +831,10 @@ test "sizeof" {
 /// with those types as fields.
 ///
 /// Examples:
-/// - `Tuple(.{})` ⇒ `tuple { }`
-/// - `Tuple(.{f32})` ⇒ `tuple { f32 }`
-/// - `Tuple(.{f32,u32})` ⇒ `tuple { f32, u32 }`
-pub fn Tuple(comptime types: anytype) type {
+/// - `Tuple(&[_]type {})` ⇒ `tuple { }`
+/// - `Tuple(&[_]type {f32})` ⇒ `tuple { f32 }`
+/// - `Tuple(&[_]type {f32,u32})` ⇒ `tuple { f32, u32 }`
+pub fn Tuple(comptime types: []const type) type {
     var tuple_fields: [types.len]std.builtin.TypeInfo.StructField = undefined;
     inline for (types) |T, i| {
         @setEvalBranchQuota(10_000);
@@ -883,8 +883,8 @@ test "Tuple" {
         }
     };
 
-    T.assertTuple(.{}, Tuple(.{}));
-    T.assertTuple(.{u32}, Tuple(.{u32}));
-    T.assertTuple(.{ u32, f16 }, Tuple(.{ u32, f16 }));
-    T.assertTuple(.{ u32, f16, []const u8 }, Tuple(.{ u32, f16, []const u8 }));
+    T.assertTuple(.{}, Tuple(&[_]type{}));
+    T.assertTuple(.{u32}, Tuple(&[_]type{u32}));
+    T.assertTuple(.{ u32, f16 }, Tuple(&[_]type{ u32, f16 }));
+    T.assertTuple(.{ u32, f16, []const u8 }, Tuple(&[_]type{ u32, f16, []const u8 }));
 }

From 468a4bf0b443067b9d4a0bf68ea7e675a9bca727 Mon Sep 17 00:00:00 2001
From: kprotty <45520026+kprotty@users.noreply.github.com>
Date: Mon, 28 Sep 2020 07:25:51 -0500
Subject: [PATCH 167/210] address some review changes

---
 lib/std/event/lock.zig | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/lib/std/event/lock.zig b/lib/std/event/lock.zig
index 452420b9cd..a83395d7d0 100644
--- a/lib/std/event/lock.zig
+++ b/lib/std/event/lock.zig
@@ -20,13 +20,14 @@ pub const Lock = struct {
     head: usize = UNLOCKED,
 
     const UNLOCKED = 0;
-    const LOCKED = 69;
+    const LOCKED = 1;
 
     const global_event_loop = Loop.instance orelse
         @compileError("std.event.Lock currently only works with event-based I/O");
 
     const Waiter = struct {
-        next: ?*Waiter,
+        // forced Waiter alignment to ensure it doesn't clash with LOCKED
+        next: ?*Waiter align(2), 
         tail: *Waiter,
         node: Loop.NextTickNode,
     };
@@ -34,6 +35,14 @@ pub const Lock = struct {
     pub fn acquire(self: *Lock) Held {
         const held = self.mutex.acquire();
 
+        // self.head transitions from multiple stages depending on the value:
+        // UNLOCKED -> LOCKED: 
+        //   acquire Lock ownership when theres no waiters
+        // LOCKED -> :
+        //   Lock is already owned, enqueue first Waiter
+        //  -> : 
+        //   Lock is owned with pending waiters. Push our waiter to the queue.
+
         if (self.head == UNLOCKED) {
             self.head = LOCKED;
             held.release();
@@ -47,7 +56,7 @@ pub const Lock = struct {
         const head = switch (self.head) {
             UNLOCKED => unreachable,
             LOCKED => null,
-            else => @intToPtr(?*Waiter, self.head),
+            else => @intToPtr(*Waiter, self.head),
         };
 
         if (head) |h| {
@@ -77,9 +86,17 @@ pub const Lock = struct {
                 const held = self.lock.mutex.acquire();
                 defer held.release();
 
+                // self.head goes through the reverse transition from acquire():
+                //  -> : 
+                //   pop a waiter from the queue to give Lock ownership when theres still others pending
+                //  -> LOCKED:
+                //   pop the laster waiter from the queue, while also giving it lock ownership when awaken
+                // LOCKED -> UNLOCKED:
+                //   last lock owner releases lock while no one else is waiting for it
+
                 switch (self.lock.head) {
                     UNLOCKED => {
-                        std.debug.panic("Lock unlocked when already unlocked", .{});
+                        unreachable; // Lock unlocked while unlocking
                     },
                     LOCKED => {
                         self.lock.head = UNLOCKED;

From 56b52dd0a357f87627fe96dc99377a397f3bb9a1 Mon Sep 17 00:00:00 2001
From: LemonBoy 
Date: Mon, 28 Sep 2020 17:16:12 +0200
Subject: [PATCH 168/210] stage1: Detect OOB access of vector value

Fixes #5710
---
 src/ir.cpp              | 10 ++++++++++
 test/compile_errors.zig | 10 +++++++++-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/ir.cpp b/src/ir.cpp
index 7d446c82a0..871c6739e6 100644
--- a/src/ir.cpp
+++ b/src/ir.cpp
@@ -21934,7 +21934,17 @@ static IrInstGen *ir_analyze_instruction_elem_ptr(IrAnalyze *ira, IrInstSrcElemP
                 return ira->codegen->invalid_inst_gen;
             }
             safety_check_on = false;
+        } else if (array_type->id == ZigTypeIdVector) {
+            uint64_t vector_len = array_type->data.vector.len;
+            if (index >= vector_len) {
+                ir_add_error_node(ira, elem_ptr_instruction->base.base.source_node,
+                    buf_sprintf("index %" ZIG_PRI_u64 " outside vector of size %" ZIG_PRI_u64,
+                            index, vector_len));
+                return ira->codegen->invalid_inst_gen;
+            }
+            safety_check_on = false;
         }
+
         if (array_type->id == ZigTypeIdVector) {
             ZigType *elem_type = array_type->data.vector.elem_type;
             uint32_t host_vec_len = array_type->data.vector.len;
diff --git a/test/compile_errors.zig b/test/compile_errors.zig
index f457c74609..6ae857d1a8 100644
--- a/test/compile_errors.zig
+++ b/test/compile_errors.zig
@@ -2,6 +2,14 @@ const tests = @import("tests.zig");
 const std = @import("std");
 
 pub fn addCases(cases: *tests.CompileErrorContext) void {
+    cases.add("slice sentinel mismatch",
+        \\export fn entry() void {
+        \\    const x = @import("std").meta.Vector(3, f32){ 25, 75, 5, 0 };
+        \\}
+    , &[_][]const u8{
+        "tmp.zig:2:62: error: index 3 outside vector of size 3",
+    });
+
     cases.add("slice sentinel mismatch",
         \\export fn entry() void {
         \\    const y: [:1]const u8 = &[_:2]u8{ 1, 2 };
@@ -7548,7 +7556,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
     });
 
     cases.add( // fixed bug #2032
-    "compile diagnostic string for top level decl type",
+        "compile diagnostic string for top level decl type",
         \\export fn entry() void {
         \\    var foo: u32 = @This(){};
         \\}

From cbbcf609688d70ce9e45bde444b2904b39a0ae2c Mon Sep 17 00:00:00 2001
From: LemonBoy 
Date: Mon, 28 Sep 2020 17:16:57 +0200
Subject: [PATCH 169/210] stage1: Allow comparison with comptime-known vectors

Since comptime_{int,float} vectors are not allowed (thanks $DEITY) we
can use the element type infos to determine the minimum operand size.
---
 src/ir.cpp                      | 14 ++++----------
 test/stage1/behavior/vector.zig |  8 ++++++++
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/ir.cpp b/src/ir.cpp
index 871c6739e6..d8d7289dae 100644
--- a/src/ir.cpp
+++ b/src/ir.cpp
@@ -16715,16 +16715,12 @@ static IrInstGen *ir_analyze_bin_op_cmp_numeric(IrAnalyze *ira, IrInst *source_i
     }
     ZigType *dest_float_type = nullptr;
     uint32_t op1_bits;
-    if (instr_is_comptime(op1)) {
+    if (instr_is_comptime(op1) && result_type->id != ZigTypeIdVector) {
         ZigValue *op1_val = ir_resolve_const(ira, op1, UndefOk);
         if (op1_val == nullptr)
             return ira->codegen->invalid_inst_gen;
         if (op1_val->special == ConstValSpecialUndef)
             return ir_const_undef(ira, source_instr, ira->codegen->builtin_types.entry_bool);
-        if (result_type->id == ZigTypeIdVector) {
-            ir_add_error(ira, &op1->base, buf_sprintf("compiler bug: TODO: support comptime vector here"));
-            return ira->codegen->invalid_inst_gen;
-        }
         bool is_unsigned;
         if (op1_is_float) {
             BigInt bigint = {};
@@ -16750,6 +16746,7 @@ static IrInstGen *ir_analyze_bin_op_cmp_numeric(IrAnalyze *ira, IrInst *source_i
             op1_bits += 1;
         }
     } else if (op1_is_float) {
+        ir_assert(op1_scalar_type->id == ZigTypeIdFloat, source_instr);
         dest_float_type = op1_scalar_type;
     } else {
         ir_assert(op1_scalar_type->id == ZigTypeIdInt, source_instr);
@@ -16759,16 +16756,12 @@ static IrInstGen *ir_analyze_bin_op_cmp_numeric(IrAnalyze *ira, IrInst *source_i
         }
     }
     uint32_t op2_bits;
-    if (instr_is_comptime(op2)) {
+    if (instr_is_comptime(op2) && result_type->id != ZigTypeIdVector) {
         ZigValue *op2_val = ir_resolve_const(ira, op2, UndefOk);
         if (op2_val == nullptr)
             return ira->codegen->invalid_inst_gen;
         if (op2_val->special == ConstValSpecialUndef)
             return ir_const_undef(ira, source_instr, ira->codegen->builtin_types.entry_bool);
-        if (result_type->id == ZigTypeIdVector) {
-            ir_add_error(ira, &op2->base, buf_sprintf("compiler bug: TODO: support comptime vector here"));
-            return ira->codegen->invalid_inst_gen;
-        }
         bool is_unsigned;
         if (op2_is_float) {
             BigInt bigint = {};
@@ -16794,6 +16787,7 @@ static IrInstGen *ir_analyze_bin_op_cmp_numeric(IrAnalyze *ira, IrInst *source_i
             op2_bits += 1;
         }
     } else if (op2_is_float) {
+        ir_assert(op2_scalar_type->id == ZigTypeIdFloat, source_instr);
         dest_float_type = op2_scalar_type;
     } else {
         ir_assert(op2_scalar_type->id == ZigTypeIdInt, source_instr);
diff --git a/test/stage1/behavior/vector.zig b/test/stage1/behavior/vector.zig
index 8bdffcd500..dc9e49da43 100644
--- a/test/stage1/behavior/vector.zig
+++ b/test/stage1/behavior/vector.zig
@@ -274,6 +274,14 @@ test "vector comparison operators" {
                 expectEqual(@splat(4, true), v1 != v3);
                 expectEqual(@splat(4, false), v1 != v2);
             }
+            {
+                // Comptime-known LHS/RHS
+                var v1: @Vector(4, u32) = [_]u32{ 2, 1, 2, 1 };
+                const v2 = @splat(4, @as(u32, 2));
+                const v3: @Vector(4, bool) = [_]bool{ true, false, true, false };
+                expectEqual(v3, v1 == v2);
+                expectEqual(v3, v2 == v1);
+            }
         }
     };
     S.doTheTest();

From 5c6cd5e2c9e8b2d0feb0026bad7c201035a175b4 Mon Sep 17 00:00:00 2001
From: LemonBoy 
Date: Sun, 27 Sep 2020 17:17:27 +0200
Subject: [PATCH 170/210] stage{1,2}: Fix parsing of range literals

stage1 was unable to parse ranges whose starting point was written in
binary/octal as the first dot in '...' was incorrectly interpreted as
decimal point.

stage2 forgot to reset the literal type to IntegerLiteral when it
discovered the dot was not a decimal point.

I've only stumbled across this bug because zig fmt keeps formatting the
ranges without any space around the ...
---
 lib/std/zig/tokenizer.zig | 9 +++++++++
 src/tokenizer.cpp         | 6 +++---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig
index 86968c73b2..e40483c022 100644
--- a/lib/std/zig/tokenizer.zig
+++ b/lib/std/zig/tokenizer.zig
@@ -1195,6 +1195,7 @@ pub const Tokenizer = struct {
                 },
                 .num_dot_hex => switch (c) {
                     '.' => {
+                        result.id = .IntegerLiteral;
                         self.index -= 1;
                         state = .start;
                         break;
@@ -1758,6 +1759,14 @@ test "correctly parse pointer assignment" {
     });
 }
 
+test "tokenizer - range literals" {
+    testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
+    testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral });
+    testTokenize("0x00...0x09", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
+    testTokenize("0b00...0b11", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
+    testTokenize("0o00...0o11", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
+}
+
 test "tokenizer - number literals decimal" {
     testTokenize("0", &[_]Token.Id{.IntegerLiteral});
     testTokenize("1", &[_]Token.Id{.IntegerLiteral});
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 4415bdf431..fa14dd40fa 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -1225,9 +1225,6 @@ void tokenize(Buf *buf, Tokenization *out) {
                             invalid_char_error(&t, c);
                             break;
                         }
-                        if (t.radix != 16 && t.radix != 10) {
-                            invalid_char_error(&t, c);
-                        }
                         t.state = TokenizeStateNumberDot;
                         break;
                     }
@@ -1281,6 +1278,9 @@ void tokenize(Buf *buf, Tokenization *out) {
                         t.state = TokenizeStateStart;
                         continue;
                     }
+                    if (t.radix != 16 && t.radix != 10) {
+                        invalid_char_error(&t, c);
+                    }
                     t.pos -= 1;
                     t.state = TokenizeStateFloatFractionNoUnderscore;
                     assert(t.cur_tok->id == TokenIdIntLiteral);

From 868a46eb43e68971634c046c8317c1b83cae21ae Mon Sep 17 00:00:00 2001
From: Frank Denis 
Date: Mon, 28 Sep 2020 23:23:32 +0200
Subject: [PATCH 171/210] std/crypto: make gimli slightly faster

Before:
       gimli-hash:        120 MiB/s
       gimli-aead:        130 MiB/s

After:
       gimli-hash:        195 MiB/s
       gimli-aead:        208 MiB/s

Also fixes in-place decryption by the way.

If the input & output buffers were the same, decryption used to fail.

Return on decryption error in the benchmark to detect similar issues
in future AEADs even in non release-fast mode.
---
 lib/std/crypto/benchmark.zig |  2 +-
 lib/std/crypto/gimli.zig     | 34 +++++++++++++++++++++-------------
 2 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/lib/std/crypto/benchmark.zig b/lib/std/crypto/benchmark.zig
index 860f1269f0..4397f7312a 100644
--- a/lib/std/crypto/benchmark.zig
+++ b/lib/std/crypto/benchmark.zig
@@ -168,7 +168,7 @@ pub fn benchmarkAead(comptime Aead: anytype, comptime bytes: comptime_int) !u64
     const start = timer.lap();
     while (offset < bytes) : (offset += in.len) {
         Aead.encrypt(in[0..], tag[0..], in[0..], &[_]u8{}, nonce, key);
-        Aead.decrypt(in[0..], in[0..], tag, &[_]u8{}, nonce, key) catch unreachable;
+        try Aead.decrypt(in[0..], in[0..], tag, &[_]u8{}, nonce, key);
     }
     mem.doNotOptimizeAway(&in);
     const end = timer.read();
diff --git a/lib/std/crypto/gimli.zig b/lib/std/crypto/gimli.zig
index 5b572aad7d..e5f93f5833 100644
--- a/lib/std/crypto/gimli.zig
+++ b/lib/std/crypto/gimli.zig
@@ -40,8 +40,8 @@ pub const State = struct {
 
     pub fn permute(self: *Self) void {
         const state = &self.data;
-        var round = @as(u32, 24);
-        while (round > 0) : (round -= 1) {
+        comptime var round = @as(u32, 24);
+        inline while (round > 0) : (round -= 1) {
             var column = @as(usize, 0);
             while (column < 4) : (column += 1) {
                 const x = math.rotl(u32, state[column], 24);
@@ -249,15 +249,19 @@ pub const Aead = struct {
             in = in[State.RATE..];
             out = out[State.RATE..];
         }) {
-            for (buf[0..State.RATE]) |*p, i| {
-                p.* ^= in[i];
-                out[i] = p.*;
+            const d = in[0..State.RATE];
+            for (d) |v, i| {
+                buf[i] ^= v;
+            }
+            for (d) |_, i| {
+                out[i] = buf[i];
             }
             state.permute();
         }
-        for (buf[0..in.len]) |*p, i| {
-            p.* ^= in[i];
-            out[i] = p.*;
+        const d = in[0..];
+        for (d) |v, i| {
+            buf[i] ^= v;
+            out[i] = buf[i];
         }
 
         // XOR 1 into the next byte of the state
@@ -291,15 +295,19 @@ pub const Aead = struct {
             in = in[State.RATE..];
             out = out[State.RATE..];
         }) {
-            for (buf[0..State.RATE]) |*p, i| {
-                out[i] = p.* ^ in[i];
-                p.* = in[i];
+            const d = in[0..State.RATE].*;
+            for (d) |v, i| {
+                out[i] = buf[i] ^ v;
+            }
+            for (d) |v, i| {
+                buf[i] = v;
             }
             state.permute();
         }
         for (buf[0..in.len]) |*p, i| {
-            out[i] = p.* ^ in[i];
-            p.* = in[i];
+            const d = in[i];
+            out[i] = p.* ^ d;
+            p.* = d;
         }
 
         // XOR 1 into the next byte of the state

From 412a2f966e18aa792089ac1f41482222d7f2434f Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Mon, 28 Sep 2020 15:42:09 -0700
Subject: [PATCH 172/210] store stage1 flags in a trailing byte in the hash id
 symlink

When we get a cache hit for a stage1 compilation, we need to know about
some of the flags such as have_winmain or have_dllmain to know which
subsystem to infer during linking.

To do this, we append a hex-encoded byte into the intentionally-dangling
symlink which contains the cache hash digest rather than a filename. The
hex-encoded byte contains the flags we need to infer the subsystem
during linking.
---
 BRANCH_TODO         |  3 +++
 src/Compilation.zig | 50 ++++++++++++++++++++++++++++++++-------------
 src/Module.zig      | 15 ++++++++------
 src/link/Coff.zig   | 15 +++++++-------
 4 files changed, 56 insertions(+), 27 deletions(-)

diff --git a/BRANCH_TODO b/BRANCH_TODO
index 6853b031d7..d134b5c733 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,3 +1,6 @@
+ * the have_foo flags that we get from stage1 have to be stored in the cache otherwise we get
+   a different result for subsystem when we have a cached stage1 execution result.
+   same deal with extern "foo" libraries used
  * add jobs to build import libs for windows DLLs for explicitly linked libs
  * add jobs to build import libs for windows DLLs for extern "foo" functions used
  * MachO LLD linking
diff --git a/src/Compilation.zig b/src/Compilation.zig
index e539950aba..00eb8c374d 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2512,16 +2512,25 @@ fn updateStage1Module(comp: *Compilation) !void {
     if (try man.hit()) {
         const digest = man.final();
 
-        var prev_digest_buf: [digest.len]u8 = undefined;
+        // We use an extra hex-encoded byte here to store some flags.
+        var prev_digest_buf: [digest.len + 2]u8 = undefined;
         const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {
             log.debug("stage1 {} new_digest={} readlink error: {}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) });
             // Handle this as a cache miss.
             break :blk prev_digest_buf[0..0];
         };
-        if (mem.eql(u8, prev_digest, &digest)) {
-            log.debug("stage1 {} digest={} match - skipping invocation", .{ mod.root_pkg.root_src_path, digest });
-            comp.stage1_lock = man.toOwnedLock();
-            return;
+        if (prev_digest.len >= digest.len + 2) {
+            if (mem.eql(u8, prev_digest[0..digest.len], &digest)) {
+                log.debug("stage1 {} digest={} match - skipping invocation", .{ mod.root_pkg.root_src_path, digest });
+                var flags_bytes: [1]u8 = undefined;
+                if (std.fmt.hexToBytes(&flags_bytes, prev_digest[digest.len..])) |_| {
+                    comp.stage1_lock = man.toOwnedLock();
+                    mod.stage1_flags = @bitCast(@TypeOf(mod.stage1_flags), flags_bytes[0]);
+                    return;
+                } else |err| {
+                    log.warn("bad cache stage1 digest: '{s}'", .{prev_digest});
+                }
+            }
         }
         log.debug("stage1 {} prev_digest={} new_digest={}", .{ mod.root_pkg.root_src_path, prev_digest, digest });
         man.unhit(prev_hash_state, input_file_count);
@@ -2642,22 +2651,35 @@ fn updateStage1Module(comp: *Compilation) !void {
     };
     stage1_module.build_object();
 
-    mod.have_c_main = stage1_module.have_c_main;
-    mod.have_winmain = stage1_module.have_winmain;
-    mod.have_wwinmain = stage1_module.have_wwinmain;
-    mod.have_winmain_crt_startup = stage1_module.have_winmain_crt_startup;
-    mod.have_wwinmain_crt_startup = stage1_module.have_wwinmain_crt_startup;
-    mod.have_dllmain_crt_startup = stage1_module.have_dllmain_crt_startup;
+    mod.stage1_flags = .{
+        .have_c_main = stage1_module.have_c_main,
+        .have_winmain = stage1_module.have_winmain,
+        .have_wwinmain = stage1_module.have_wwinmain,
+        .have_winmain_crt_startup = stage1_module.have_winmain_crt_startup,
+        .have_wwinmain_crt_startup = stage1_module.have_wwinmain_crt_startup,
+        .have_dllmain_crt_startup = stage1_module.have_dllmain_crt_startup,
+    };
 
     stage1_module.destroy();
 
     const digest = man.final();
 
-    log.debug("stage1 {} final digest={}", .{ mod.root_pkg.root_src_path, digest });
-
     // Update the dangling symlink with the digest. If it fails we can continue; it only
     // means that the next invocation will have an unnecessary cache miss.
-    directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {
+    const stage1_flags_byte = @bitCast(u8, mod.stage1_flags);
+    log.debug("stage1 {} final digest={} flags={x}", .{
+        mod.root_pkg.root_src_path, digest, stage1_flags_byte,
+    });
+    var digest_plus_flags: [digest.len + 2]u8 = undefined;
+    digest_plus_flags[0..digest.len].* = digest;
+    assert(std.fmt.formatIntBuf(digest_plus_flags[digest.len..], stage1_flags_byte, 16, false, .{
+        .width = 2,
+        .fill = '0',
+    }) == 2);
+    log.debug("saved digest + flags: '{s}' (byte = {}) have_winmain_crt_startup={}", .{
+        digest_plus_flags, stage1_flags_byte, mod.stage1_flags.have_winmain_crt_startup,
+    });
+    directory.handle.symLink(&digest_plus_flags, id_symlink_basename, .{}) catch |err| {
         log.warn("failed to save stage1 hash digest symlink: {}", .{@errorName(err)});
     };
     // Again failure here only means an unnecessary cache miss.
diff --git a/src/Module.zig b/src/Module.zig
index 8f87b7a521..4fcf72f4ff 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -75,12 +75,15 @@ global_error_set: std.StringHashMapUnmanaged(u16) = .{},
 /// previous analysis.
 generation: u32 = 0,
 
-have_winmain: bool = false,
-have_wwinmain: bool = false,
-have_winmain_crt_startup: bool = false,
-have_wwinmain_crt_startup: bool = false,
-have_dllmain_crt_startup: bool = false,
-have_c_main: bool = false,
+stage1_flags: packed struct {
+    have_winmain: bool = false,
+    have_wwinmain: bool = false,
+    have_winmain_crt_startup: bool = false,
+    have_wwinmain_crt_startup: bool = false,
+    have_dllmain_crt_startup: bool = false,
+    have_c_main: bool = false,
+    reserved: u2 = 0,
+} = .{},
 
 pub const Export = struct {
     options: std.builtin.ExportOptions,
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index e1d7a07fbc..0356a327ba 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -943,14 +943,15 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
         switch (target.os.tag) {
             .windows => {
                 if (self.base.options.module) |module| {
-                    if (module.have_dllmain_crt_startup or is_dyn_lib)
+                    if (module.stage1_flags.have_dllmain_crt_startup or is_dyn_lib)
                         break :blk null;
-                    if (module.have_c_main or self.base.options.is_test or
-                        module.have_winmain_crt_startup or module.have_wwinmain_crt_startup)
+                    if (module.stage1_flags.have_c_main or self.base.options.is_test or
+                        module.stage1_flags.have_winmain_crt_startup or
+                        module.stage1_flags.have_wwinmain_crt_startup)
                     {
                         break :blk .Console;
                     }
-                    if (module.have_winmain or module.have_wwinmain)
+                    if (module.stage1_flags.have_winmain or module.stage1_flags.have_wwinmain)
                         break :blk .Windows;
                 }
             },
@@ -1068,11 +1069,11 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
                 try argv.append("-NODEFAULTLIB");
                 if (!is_lib) {
                     if (self.base.options.module) |module| {
-                        if (module.have_winmain) {
+                        if (module.stage1_flags.have_winmain) {
                             try argv.append("-ENTRY:WinMain");
-                        } else if (module.have_wwinmain) {
+                        } else if (module.stage1_flags.have_wwinmain) {
                             try argv.append("-ENTRY:wWinMain");
-                        } else if (module.have_wwinmain_crt_startup) {
+                        } else if (module.stage1_flags.have_wwinmain_crt_startup) {
                             try argv.append("-ENTRY:wWinMainCRTStartup");
                         } else {
                             try argv.append("-ENTRY:WinMainCRTStartup");

From ada19c498d6cb52dd8f0de71d42baf845cfadc21 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Mon, 28 Sep 2020 19:20:58 -0700
Subject: [PATCH 173/210] stage2: building DLL import lib files

---
 BRANCH_TODO         |   2 +-
 src/Compilation.zig |  19 ++++
 src/llvm.zig        |  63 ++++++++++++
 src/mingw.zig       | 229 ++++++++++++++++++++++++++++++++++++++++++++
 src/target.zig      |  57 +++++++++++
 src/zig_llvm.cpp    |   2 +-
 src/zig_llvm.h      |   4 +-
 7 files changed, 372 insertions(+), 4 deletions(-)

diff --git a/BRANCH_TODO b/BRANCH_TODO
index d134b5c733..3f725e208c 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,7 +1,6 @@
  * the have_foo flags that we get from stage1 have to be stored in the cache otherwise we get
    a different result for subsystem when we have a cached stage1 execution result.
    same deal with extern "foo" libraries used
- * add jobs to build import libs for windows DLLs for explicitly linked libs
  * add jobs to build import libs for windows DLLs for extern "foo" functions used
  * MachO LLD linking
  * WASM LLD linking
@@ -53,3 +52,4 @@
  * make proposal about log levels
  * proposal for changing fs Z/W functions to be native paths and have a way to do native path string literals
  * proposal for block { break x; }
+ * generally look for the "TODO surface this as a real compile error message" and fix all that stuff
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 00eb8c374d..3095ffab81 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -168,6 +168,9 @@ const Job = union(enum) {
     generate_builtin_zig: void,
     /// Use stage1 C++ code to compile zig code into an object file.
     stage1_module: void,
+
+    /// The value is the index into `link.File.Options.system_libs`.
+    windows_import_lib: usize,
 };
 
 pub const CObject = struct {
@@ -868,6 +871,15 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             comp.work_queue.writeAssumeCapacity(&static_lib_jobs);
             comp.work_queue.writeItemAssumeCapacity(crt_job);
         }
+        // Generate Windows import libs.
+        if (comp.getTarget().os.tag == .windows) {
+            const count = comp.bin_file.options.system_libs.count();
+            try comp.work_queue.ensureUnusedCapacity(count);
+            var i: usize = 0;
+            while (i < count) : (i += 1) {
+                comp.work_queue.writeItemAssumeCapacity(.{ .windows_import_lib = i });
+            }
+        }
         if (comp.wantBuildLibUnwindFromSource()) {
             try comp.work_queue.writeItem(.{ .libunwind = {} });
         }
@@ -1233,6 +1245,13 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void {
                 fatal("unable to build mingw-w64 CRT file: {}", .{@errorName(err)});
             };
         },
+        .windows_import_lib => |index| {
+            const link_lib = self.bin_file.options.system_libs.items()[index].key;
+            mingw.buildImportLib(self, link_lib) catch |err| {
+                // TODO Expose this as a normal compile error rather than crashing here.
+                fatal("unable to generate DLL import .lib file: {}", .{@errorName(err)});
+            };
+        },
         .libunwind => {
             libunwind.buildStaticLib(self) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
diff --git a/src/llvm.zig b/src/llvm.zig
index ceefb62c5d..3aebf46b81 100644
--- a/src/llvm.zig
+++ b/src/llvm.zig
@@ -73,5 +73,68 @@ pub const OSType = extern enum(c_int) {
     Emscripten = 35,
 };
 
+pub const ArchType = extern enum(c_int) {
+    UnknownArch = 0,
+    arm = 1,
+    armeb = 2,
+    aarch64 = 3,
+    aarch64_be = 4,
+    aarch64_32 = 5,
+    arc = 6,
+    avr = 7,
+    bpfel = 8,
+    bpfeb = 9,
+    hexagon = 10,
+    mips = 11,
+    mipsel = 12,
+    mips64 = 13,
+    mips64el = 14,
+    msp430 = 15,
+    ppc = 16,
+    ppc64 = 17,
+    ppc64le = 18,
+    r600 = 19,
+    amdgcn = 20,
+    riscv32 = 21,
+    riscv64 = 22,
+    sparc = 23,
+    sparcv9 = 24,
+    sparcel = 25,
+    systemz = 26,
+    tce = 27,
+    tcele = 28,
+    thumb = 29,
+    thumbeb = 30,
+    x86 = 31,
+    x86_64 = 32,
+    xcore = 33,
+    nvptx = 34,
+    nvptx64 = 35,
+    le32 = 36,
+    le64 = 37,
+    amdil = 38,
+    amdil64 = 39,
+    hsail = 40,
+    hsail64 = 41,
+    spir = 42,
+    spir64 = 43,
+    kalimba = 44,
+    shave = 45,
+    lanai = 46,
+    wasm32 = 47,
+    wasm64 = 48,
+    renderscript32 = 49,
+    renderscript64 = 50,
+    ve = 51,
+};
+
 pub const ParseCommandLineOptions = ZigLLVMParseCommandLineOptions;
 extern fn ZigLLVMParseCommandLineOptions(argc: usize, argv: [*]const [*:0]const u8) void;
+
+pub const WriteImportLibrary = ZigLLVMWriteImportLibrary;
+extern fn ZigLLVMWriteImportLibrary(
+    def_path: [*:0]const u8,
+    arch: ArchType,
+    output_lib_path: [*c]const u8,
+    kill_at: bool,
+) bool;
diff --git a/src/mingw.zig b/src/mingw.zig
index 2a2879de05..41a328c6f4 100644
--- a/src/mingw.zig
+++ b/src/mingw.zig
@@ -3,10 +3,12 @@ const Allocator = std.mem.Allocator;
 const mem = std.mem;
 const path = std.fs.path;
 const assert = std.debug.assert;
+const log = std.log.scoped(.mingw);
 
 const target_util = @import("target.zig");
 const Compilation = @import("Compilation.zig");
 const build_options = @import("build_options");
+const Cache = @import("Cache.zig");
 
 pub const CRTFile = enum {
     crt2_o,
@@ -277,6 +279,233 @@ fn add_cc_args(
     });
 }
 
+pub fn buildImportLib(comp: *Compilation, lib_name: []const u8) !void {
+    var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
+    defer arena_allocator.deinit();
+    const arena = &arena_allocator.allocator;
+
+    const def_file_path = findDef(comp, arena, lib_name) catch |err| switch (err) {
+        error.FileNotFound => {
+            log.debug("no {s}.def file available to make a DLL import {s}.lib", .{ lib_name, lib_name });
+            // In this case we will end up putting foo.lib onto the linker line and letting the linker
+            // use its library paths to look for libraries and report any problems.
+            return;
+        },
+        else => |e| return e,
+    };
+
+    // We need to invoke `zig clang` to use the preprocessor.
+    if (!build_options.have_llvm) return error.ZigCompilerNotBuiltWithLLVMExtensions;
+    const self_exe_path = comp.self_exe_path orelse return error.PreprocessorDisabled;
+
+    const target = comp.getTarget();
+
+    var cache: Cache = .{
+        .gpa = comp.gpa,
+        .manifest_dir = comp.cache_parent.manifest_dir,
+    };
+    cache.hash.addBytes(build_options.version);
+    cache.hash.addOptionalBytes(comp.zig_lib_directory.path);
+    cache.hash.add(target.cpu.arch);
+
+    var man = cache.obtain();
+    defer man.deinit();
+
+    _ = try man.addFile(def_file_path, null);
+
+    const final_lib_basename = try std.fmt.allocPrint(comp.gpa, "{s}.lib", .{lib_name});
+    errdefer comp.gpa.free(final_lib_basename);
+
+    if (try man.hit()) {
+        const digest = man.final();
+
+        try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1);
+        comp.crt_files.putAssumeCapacityNoClobber(final_lib_basename, .{
+            .full_object_path = try comp.global_cache_directory.join(comp.gpa, &[_][]const u8{
+                "o", &digest, final_lib_basename,
+            }),
+            .lock = man.toOwnedLock(),
+        });
+        return;
+    }
+
+    const digest = man.final();
+    const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });
+    var o_dir = try comp.global_cache_directory.handle.makeOpenPath(o_sub_path, .{});
+    defer o_dir.close();
+
+    const final_def_basename = try std.fmt.allocPrint(arena, "{s}.def", .{lib_name});
+    const def_final_path = try comp.global_cache_directory.join(arena, &[_][]const u8{
+        "o", &digest, final_def_basename,
+    });
+
+    const target_def_arg = switch (target.cpu.arch) {
+        .i386 => "-DDEF_I386",
+        .x86_64 => "-DDEF_X64",
+        .arm, .armeb => switch (target.cpu.arch.ptrBitWidth()) {
+            32 => "-DDEF_ARM32",
+            64 => "-DDEF_ARM64",
+            else => unreachable,
+        },
+        else => unreachable,
+    };
+
+    const args = [_][]const u8{
+        self_exe_path,
+        "clang",
+        "-x",
+        "c",
+        def_file_path,
+        "-Wp,-w",
+        "-undef",
+        "-P",
+        "-I",
+        try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw", "def-include" }),
+        target_def_arg,
+        "-E",
+        "-o",
+        def_final_path,
+    };
+
+    if (comp.verbose_cc) {
+        Compilation.dump_argv(&args);
+    }
+
+    const child = try std.ChildProcess.init(&args, arena);
+    defer child.deinit();
+
+    child.stdin_behavior = .Ignore;
+    child.stdout_behavior = .Pipe;
+    child.stderr_behavior = .Pipe;
+
+    try child.spawn();
+
+    const stdout_reader = child.stdout.?.reader();
+    const stderr_reader = child.stderr.?.reader();
+
+    // TODO https://github.com/ziglang/zig/issues/6343
+    const stdout = try stdout_reader.readAllAlloc(arena, std.math.maxInt(u32));
+    const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024);
+
+    const term = child.wait() catch |err| {
+        // TODO surface a proper error here
+        log.err("unable to spawn {}: {}", .{ args[0], @errorName(err) });
+        return error.ClangPreprocessorFailed;
+    };
+
+    switch (term) {
+        .Exited => |code| {
+            if (code != 0) {
+                // TODO surface a proper error here
+                log.err("clang exited with code {d} and stderr: {s}", .{ code, stderr });
+                return error.ClangPreprocessorFailed;
+            }
+        },
+        else => {
+            // TODO surface a proper error here
+            log.err("clang terminated unexpectedly with stderr: {}", .{stderr});
+            return error.ClangPreprocessorFailed;
+        },
+    }
+
+    const lib_final_path = try comp.global_cache_directory.join(comp.gpa, &[_][]const u8{
+        "o", &digest, final_lib_basename,
+    });
+    errdefer comp.gpa.free(lib_final_path);
+
+    const llvm = @import("llvm.zig");
+    const arch_type = @import("target.zig").archToLLVM(target.cpu.arch);
+    const def_final_path_z = try arena.dupeZ(u8, def_final_path);
+    const lib_final_path_z = try arena.dupeZ(u8, lib_final_path);
+    if (llvm.WriteImportLibrary(def_final_path_z.ptr, arch_type, lib_final_path_z.ptr, true)) {
+        // TODO surface a proper error here
+        log.err("unable to turn {s}.def into {s}.lib", .{ lib_name, lib_name });
+        return error.WritingImportLibFailed;
+    }
+
+    man.writeManifest() catch |err| {
+        log.warn("failed to write cache manifest for DLL import {s}.lib: {s}", .{ lib_name, @errorName(err) });
+    };
+
+    try comp.crt_files.putNoClobber(comp.gpa, final_lib_basename, .{
+        .full_object_path = lib_final_path,
+        .lock = man.toOwnedLock(),
+    });
+}
+
+/// This function body is verbose but all it does is test 3 different paths and see if a .def file exists.
+fn findDef(comp: *Compilation, allocator: *Allocator, lib_name: []const u8) ![]u8 {
+    const target = comp.getTarget();
+
+    const lib_path = switch (target.cpu.arch) {
+        .i386 => "lib32",
+        .x86_64 => "lib64",
+        .arm, .armeb => switch (target.cpu.arch.ptrBitWidth()) {
+            32 => "libarm32",
+            64 => "libarm64",
+            else => unreachable,
+        },
+        else => unreachable,
+    };
+
+    var override_path = std.ArrayList(u8).init(allocator);
+    defer override_path.deinit();
+
+    const s = path.sep_str;
+
+    {
+        // Try the archtecture-specific path first.
+        const fmt_path = "libc" ++ s ++ "mingw" ++ s ++ "{s}" ++ s ++ "{s}.def";
+        if (comp.zig_lib_directory.path) |p| {
+            try override_path.writer().print("{s}" ++ s ++ fmt_path, .{ p, lib_path, lib_name });
+        } else {
+            try override_path.writer().print(fmt_path, .{ lib_path, lib_name });
+        }
+        if (std.fs.cwd().access(override_path.items, .{})) |_| {
+            return override_path.toOwnedSlice();
+        } else |err| switch (err) {
+            error.FileNotFound => {},
+            else => |e| return e,
+        }
+    }
+
+    {
+        // Try the generic version.
+        override_path.shrinkRetainingCapacity(0);
+        const fmt_path = "libc" ++ s ++ "mingw" ++ s ++ "lib-common" ++ s ++ "{s}.def";
+        if (comp.zig_lib_directory.path) |p| {
+            try override_path.writer().print("{s}" ++ s ++ fmt_path, .{ p, lib_name });
+        } else {
+            try override_path.writer().print(fmt_path, .{lib_name});
+        }
+        if (std.fs.cwd().access(override_path.items, .{})) |_| {
+            return override_path.toOwnedSlice();
+        } else |err| switch (err) {
+            error.FileNotFound => {},
+            else => |e| return e,
+        }
+    }
+
+    {
+        // Try the generic version and preprocess it.
+        override_path.shrinkRetainingCapacity(0);
+        const fmt_path = "libc" ++ s ++ "mingw" ++ s ++ "lib-common" ++ s ++ "{s}.def.in";
+        if (comp.zig_lib_directory.path) |p| {
+            try override_path.writer().print("{s}" ++ s ++ fmt_path, .{ p, lib_name });
+        } else {
+            try override_path.writer().print(fmt_path, .{lib_name});
+        }
+        if (std.fs.cwd().access(override_path.items, .{})) |_| {
+            return override_path.toOwnedSlice();
+        } else |err| switch (err) {
+            error.FileNotFound => {},
+            else => |e| return e,
+        }
+    }
+
+    return error.FileNotFound;
+}
+
 const mingw32_lib_deps = [_][]const u8{
     "crt0_c.c",
     "dll_argv.c",
diff --git a/src/target.zig b/src/target.zig
index 9b5ea2a366..f6d0f41112 100644
--- a/src/target.zig
+++ b/src/target.zig
@@ -224,6 +224,63 @@ pub fn osToLLVM(os_tag: std.Target.Os.Tag) llvm.OSType {
     };
 }
 
+pub fn archToLLVM(arch_tag: std.Target.Cpu.Arch) llvm.ArchType {
+    return switch (arch_tag) {
+        .arm => .arm,
+        .armeb => .armeb,
+        .aarch64 => .aarch64,
+        .aarch64_be => .aarch64_be,
+        .aarch64_32 => .aarch64_32,
+        .arc => .arc,
+        .avr => .avr,
+        .bpfel => .bpfel,
+        .bpfeb => .bpfeb,
+        .hexagon => .hexagon,
+        .mips => .mips,
+        .mipsel => .mipsel,
+        .mips64 => .mips64,
+        .mips64el => .mips64el,
+        .msp430 => .msp430,
+        .powerpc => .ppc,
+        .powerpc64 => .ppc64,
+        .powerpc64le => .ppc64le,
+        .r600 => .r600,
+        .amdgcn => .amdgcn,
+        .riscv32 => .riscv32,
+        .riscv64 => .riscv64,
+        .sparc => .sparc,
+        .sparcv9 => .sparcv9,
+        .sparcel => .sparcel,
+        .s390x => .systemz,
+        .tce => .tce,
+        .tcele => .tcele,
+        .thumb => .thumb,
+        .thumbeb => .thumbeb,
+        .i386 => .x86,
+        .x86_64 => .x86_64,
+        .xcore => .xcore,
+        .nvptx => .nvptx,
+        .nvptx64 => .nvptx64,
+        .le32 => .le32,
+        .le64 => .le64,
+        .amdil => .amdil,
+        .amdil64 => .amdil64,
+        .hsail => .hsail,
+        .hsail64 => .hsail64,
+        .spir => .spir,
+        .spir64 => .spir64,
+        .kalimba => .kalimba,
+        .shave => .shave,
+        .lanai => .lanai,
+        .wasm32 => .wasm32,
+        .wasm64 => .wasm64,
+        .renderscript32 => .renderscript32,
+        .renderscript64 => .renderscript64,
+        .ve => .ve,
+        .spu_2 => .UnknownArch,
+    };
+}
+
 fn eqlIgnoreCase(ignore_case: bool, a: []const u8, b: []const u8) bool {
     if (ignore_case) {
         return std.ascii.eqlIgnoreCase(a, b);
diff --git a/src/zig_llvm.cpp b/src/zig_llvm.cpp
index e5b9df625c..08823050ad 100644
--- a/src/zig_llvm.cpp
+++ b/src/zig_llvm.cpp
@@ -927,7 +927,7 @@ class MyOStream: public raw_ostream {
 };
 
 bool ZigLLVMWriteImportLibrary(const char *def_path, const ZigLLVM_ArchType arch,
-                               const char *output_lib_path, const bool kill_at)
+                               const char *output_lib_path, bool kill_at)
 {
     COFF::MachineTypes machine = COFF::IMAGE_FILE_MACHINE_UNKNOWN;
 
diff --git a/src/zig_llvm.h b/src/zig_llvm.h
index 4de048a3d8..007d8afc1f 100644
--- a/src/zig_llvm.h
+++ b/src/zig_llvm.h
@@ -500,8 +500,8 @@ ZIG_EXTERN_C bool ZigLLDLink(enum ZigLLVM_ObjectFormatType oformat, const char *
 ZIG_EXTERN_C bool ZigLLVMWriteArchive(const char *archive_name, const char **file_names, size_t file_name_count,
         enum ZigLLVM_OSType os_type);
 
-bool ZigLLVMWriteImportLibrary(const char *def_path, const enum ZigLLVM_ArchType arch,
-                               const char *output_lib_path, const bool kill_at);
+ZIG_EXTERN_C bool ZigLLVMWriteImportLibrary(const char *def_path, const enum ZigLLVM_ArchType arch,
+                               const char *output_lib_path, bool kill_at);
 
 ZIG_EXTERN_C void ZigLLVMGetNativeTarget(enum ZigLLVM_ArchType *arch_type,
         enum ZigLLVM_VendorType *vendor_type, enum ZigLLVM_OSType *os_type, enum ZigLLVM_EnvironmentType *environ_type,

From 5fed42d70a9ce23581bc8e1651a417161f269663 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Mon, 28 Sep 2020 20:55:52 -0700
Subject: [PATCH 174/210] the stage1 backend cache stores inferred link libs

So that we properly learn about extern "foo" functions called even when
we get a stage1 cache hit.
---
 BRANCH_TODO         |  4 ---
 src/Compilation.zig | 59 +++++++++++++++++++++++++++++++++++++--------
 src/stage1.zig      | 30 +++++++++++++++--------
 3 files changed, 69 insertions(+), 24 deletions(-)

diff --git a/BRANCH_TODO b/BRANCH_TODO
index 3f725e208c..a579d4a5eb 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,7 +1,3 @@
- * the have_foo flags that we get from stage1 have to be stored in the cache otherwise we get
-   a different result for subsystem when we have a cached stage1 execution result.
-   same deal with extern "foo" libraries used
- * add jobs to build import libs for windows DLLs for extern "foo" functions used
  * MachO LLD linking
  * WASM LLD linking
  * audit the CLI options for stage2
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 3095ffab81..314b7cdbdd 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2498,6 +2498,7 @@ fn updateStage1Module(comp: *Compilation) !void {
     const builtin_zig_path = try directory.join(arena, &[_][]const u8{"builtin.zig"});
     const target = comp.getTarget();
     const id_symlink_basename = "stage1.id";
+    const libs_txt_basename = "libs.txt";
 
     // We are about to obtain this lock, so here we give other processes a chance first.
     comp.releaseStage1Lock();
@@ -2538,18 +2539,32 @@ fn updateStage1Module(comp: *Compilation) !void {
             // Handle this as a cache miss.
             break :blk prev_digest_buf[0..0];
         };
-        if (prev_digest.len >= digest.len + 2) {
-            if (mem.eql(u8, prev_digest[0..digest.len], &digest)) {
-                log.debug("stage1 {} digest={} match - skipping invocation", .{ mod.root_pkg.root_src_path, digest });
-                var flags_bytes: [1]u8 = undefined;
-                if (std.fmt.hexToBytes(&flags_bytes, prev_digest[digest.len..])) |_| {
-                    comp.stage1_lock = man.toOwnedLock();
-                    mod.stage1_flags = @bitCast(@TypeOf(mod.stage1_flags), flags_bytes[0]);
-                    return;
-                } else |err| {
-                    log.warn("bad cache stage1 digest: '{s}'", .{prev_digest});
+        if (prev_digest.len >= digest.len + 2) hit: {
+            if (!mem.eql(u8, prev_digest[0..digest.len], &digest))
+                break :hit;
+
+            log.debug("stage1 {} digest={} match - skipping invocation", .{ mod.root_pkg.root_src_path, digest });
+            var flags_bytes: [1]u8 = undefined;
+            _ = std.fmt.hexToBytes(&flags_bytes, prev_digest[digest.len..]) catch {
+                log.warn("bad cache stage1 digest: '{s}'", .{prev_digest});
+                break :hit;
+            };
+
+            if (directory.handle.readFileAlloc(comp.gpa, libs_txt_basename, 10 * 1024 * 1024)) |libs_txt| {
+                var it = mem.tokenize(libs_txt, "\n");
+                while (it.next()) |lib_name| {
+                    try comp.stage1AddLinkLib(lib_name);
                 }
+            } else |err| switch (err) {
+                error.FileNotFound => {}, // That's OK, it just means 0 libs.
+                else => {
+                    log.warn("unable to read cached list of link libs: {s}", .{@errorName(err)});
+                    break :hit;
+                },
             }
+            comp.stage1_lock = man.toOwnedLock();
+            mod.stage1_flags = @bitCast(@TypeOf(mod.stage1_flags), flags_bytes[0]);
+            return;
         }
         log.debug("stage1 {} prev_digest={} new_digest={}", .{ mod.root_pkg.root_src_path, prev_digest, digest });
         man.unhit(prev_hash_state, input_file_count);
@@ -2668,8 +2683,20 @@ fn updateStage1Module(comp: *Compilation) !void {
         .have_wwinmain_crt_startup = false,
         .have_dllmain_crt_startup = false,
     };
+
+    const inferred_lib_start_index = comp.bin_file.options.system_libs.count();
     stage1_module.build_object();
 
+    if (comp.bin_file.options.system_libs.count() > inferred_lib_start_index) {
+        // We need to save the inferred link libs to the cache, otherwise if we get a cache hit
+        // next time we will be missing these libs.
+        var libs_txt = std.ArrayList(u8).init(arena);
+        for (comp.bin_file.options.system_libs.items()[inferred_lib_start_index..]) |entry| {
+            try libs_txt.writer().print("{s}\n", .{entry.key});
+        }
+        try directory.handle.writeFile(libs_txt_basename, libs_txt.items);
+    }
+
     mod.stage1_flags = .{
         .have_c_main = stage1_module.have_c_main,
         .have_winmain = stage1_module.have_winmain,
@@ -2814,3 +2841,15 @@ pub fn build_crt_file(
         .lock = sub_compilation.bin_file.toOwnedLock(),
     });
 }
+
+pub fn stage1AddLinkLib(comp: *Compilation, lib_name: []const u8) !void {
+    // This happens when an `extern "foo"` function is referenced by the stage1 backend.
+    // If we haven't seen this library yet and we're targeting Windows, we need to queue up
+    // a work item to produce the DLL import library for this.
+    const gop = try comp.bin_file.options.system_libs.getOrPut(comp.gpa, lib_name);
+    if (!gop.found_existing and comp.getTarget().os.tag == .windows) {
+        try comp.work_queue.writeItem(.{
+            .windows_import_lib = comp.bin_file.options.system_libs.count() - 1,
+        });
+    }
+}
diff --git a/src/stage1.zig b/src/stage1.zig
index 8142ce901c..a989ad4be3 100644
--- a/src/stage1.zig
+++ b/src/stage1.zig
@@ -381,23 +381,33 @@ export fn stage2_add_link_lib(
     symbol_name_len: usize,
 ) ?[*:0]const u8 {
     const comp = @intToPtr(*Compilation, stage1.userdata);
-    const lib_name = lib_name_ptr[0..lib_name_len];
-    const symbol_name = symbol_name_ptr[0..symbol_name_len];
+    const lib_name = std.ascii.allocLowerString(comp.gpa, lib_name_ptr[0..lib_name_len]) catch return "out of memory";
     const target = comp.getTarget();
     const is_libc = target_util.is_libc_lib_name(target, lib_name);
-    if (is_libc and !comp.bin_file.options.link_libc) {
-        return "dependency on libc must be explicitly specified in the build command";
+    if (is_libc) {
+        if (!comp.bin_file.options.link_libc) {
+            return "dependency on libc must be explicitly specified in the build command";
+        }
+        return null;
     }
-
-    if (!is_libc and !target.isWasm() and !comp.bin_file.options.pic) {
-        const msg = std.fmt.allocPrint0(
+    if (target_util.is_libcpp_lib_name(target, lib_name)) {
+        if (!comp.bin_file.options.link_libcpp) {
+            return "dependency on libc++ must be explicitly specified in the build command";
+        }
+        return null;
+    }
+    if (!target.isWasm() and !comp.bin_file.options.pic) {
+        return std.fmt.allocPrint0(
             comp.gpa,
             "dependency on dynamic library '{s}' requires enabling Position Independent Code. Fixed by `-l{s}` or `-fPIC`.",
             .{ lib_name, lib_name },
-        ) catch return "out of memory";
-        return msg.ptr;
+        ) catch "out of memory";
     }
-
+    comp.stage1AddLinkLib(lib_name) catch |err| {
+        return std.fmt.allocPrint0(comp.gpa, "unable to add link lib '{s}': {s}", .{
+            lib_name, @errorName(err),
+        }) catch "out of memory";
+    };
     return null;
 }
 

From 29fd13009391060a2d6783fb0b91cb075c2e6cce Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Mon, 28 Sep 2020 22:17:50 -0700
Subject: [PATCH 175/210] std.ChildProcess: bypass libc exit() in fork child
 error case

Comment reproduced here:

If we're linking libc, some naughty applications may have
registered atexit handlers which we really do not want to
run in the fork child. I caught LLVM doing this and it
caused a deadlock instead of doing an exit syscall. In
the words of Avril Lavigne, "Why'd you have to go and
make things so complicated?"
---
 lib/std/child_process.zig | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig
index 82ed938f33..e706302ebd 100644
--- a/lib/std/child_process.zig
+++ b/lib/std/child_process.zig
@@ -485,7 +485,7 @@ pub const ChildProcess = struct {
         const any_ignore = (self.stdin_behavior == StdIo.Ignore or self.stdout_behavior == StdIo.Ignore or self.stderr_behavior == StdIo.Ignore);
 
         const nul_handle = if (any_ignore)
-        // "\Device\Null" or "\??\NUL"
+            // "\Device\Null" or "\??\NUL"
             windows.OpenFile(&[_]u16{ '\\', 'D', 'e', 'v', 'i', 'c', 'e', '\\', 'N', 'u', 'l', 'l' }, .{
                 .access_mask = windows.GENERIC_READ | windows.SYNCHRONIZE,
                 .share_access = windows.FILE_SHARE_READ,
@@ -816,6 +816,13 @@ fn destroyPipe(pipe: [2]os.fd_t) void {
 // Then the child exits.
 fn forkChildErrReport(fd: i32, err: ChildProcess.SpawnError) noreturn {
     writeIntFd(fd, @as(ErrInt, @errorToInt(err))) catch {};
+    // If we're linking libc, some naughty applications may have registered atexit handlers
+    // which we really do not want to run in the fork child. I caught LLVM doing this and
+    // it caused a deadlock instead of doing an exit syscall. In the words of Avril Lavigne,
+    // "Why'd you have to go and make things so complicated?"
+    if (std.Target.current.os.tag == .linux) {
+        std.os.linux.exit(1); // By-pass libc regardless of whether it is linked.
+    }
     os.exit(1);
 }
 

From ef9582a1ec452aab816b67cd2d0d35ef7356ddae Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Mon, 28 Sep 2020 22:19:00 -0700
Subject: [PATCH 176/210] `zig test` and `zig run` do not try to run foreign
 binaries

---
 BRANCH_TODO        |  1 +
 lib/std/target.zig | 21 +++++++++++++++++++++
 src/main.zig       | 15 +++++++++++++--
 3 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/BRANCH_TODO b/BRANCH_TODO
index a579d4a5eb..9665ba3d0e 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,3 +1,4 @@
+ * docs are failing to build
  * MachO LLD linking
  * WASM LLD linking
  * audit the CLI options for stage2
diff --git a/lib/std/target.zig b/lib/std/target.zig
index 1b8e7c1519..65e9f75457 100644
--- a/lib/std/target.zig
+++ b/lib/std/target.zig
@@ -1483,6 +1483,27 @@ pub const Target = struct {
             => return result,
         }
     }
+
+    /// Return whether or not the given host target is capable of executing natively executables
+    /// of the other target.
+    pub fn canExecBinariesOf(host_target: std.Target, binary_target: std.Target) bool {
+        if (host_target.os.tag != binary_target.os.tag)
+            return false;
+
+        if (host_target.cpu.arch == binary_target.cpu.arch)
+            return true;
+
+        if (host_target.cpu.arch == .x86_64 and binary_target.cpu.arch == .i386)
+            return true;
+
+        if (host_target.cpu.arch == .aarch64 and binary_target.cpu.arch == .arm)
+            return true;
+
+        if (host_target.cpu.arch == .aarch64_be and binary_target.cpu.arch == .armeb)
+            return true;
+
+        return false;
+    }
 };
 
 test "" {
diff --git a/src/main.zig b/src/main.zig
index d96cfaf526..61ef77cce7 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -1618,6 +1618,17 @@ fn buildOutputType(
             defer argv.deinit();
 
             if (test_exec_args.items.len == 0) {
+                if (!std.Target.current.canExecBinariesOf(target_info.target)) {
+                    switch (arg_mode) {
+                        .zig_test => {
+                            warn("created {s} but skipping execution because it is non-native", .{exe_path});
+                            if (!watch) return cleanExit();
+                            break :run;
+                        },
+                        .run => fatal("unable to execute {s}: non-native", .{exe_path}),
+                        else => unreachable,
+                    }
+                }
                 try argv.append(exe_path);
             } else {
                 for (test_exec_args.items) |arg| {
@@ -2128,8 +2139,8 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
                     error.FileNotFound => {
                         dirname = fs.path.dirname(dirname) orelse {
                             std.log.info("{}", .{
-                            \\Initialize a 'build.zig' template file with `zig init-lib` or `zig init-exe`,
-                            \\or see `zig --help` for more options.
+                                \\Initialize a 'build.zig' template file with `zig init-lib` or `zig init-exe`,
+                                \\or see `zig --help` for more options.
                             });
                             fatal("No 'build.zig' file found, in the current directory or any parent directories.", .{});
                         };

From 73167e80f8a1d440ebc5725a165b7cf484b96170 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Mon, 28 Sep 2020 22:40:50 -0700
Subject: [PATCH 177/210] stage2: fix Cache not calling ftruncate in
 writeManifest

this led to a corrupt cache when the number of files got smaller. it is
now fixed.
---
 BRANCH_TODO   |  1 -
 src/Cache.zig | 10 ++++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/BRANCH_TODO b/BRANCH_TODO
index 9665ba3d0e..a579d4a5eb 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,4 +1,3 @@
- * docs are failing to build
  * MachO LLD linking
  * WASM LLD linking
  * audit the CLI options for stage2
diff --git a/src/Cache.zig b/src/Cache.zig
index 425c6407a3..dff6f7e38e 100644
--- a/src/Cache.zig
+++ b/src/Cache.zig
@@ -528,14 +528,15 @@ pub const Manifest = struct {
     }
 
     pub fn writeManifest(self: *Manifest) !void {
-        assert(self.manifest_file != null);
+        const manifest_file = self.manifest_file.?;
         if (!self.manifest_dirty) return;
 
-        var encoded_digest: [hex_digest_len]u8 = undefined;
         var contents = std.ArrayList(u8).init(self.cache.gpa);
-        var writer = contents.writer();
         defer contents.deinit();
 
+        const writer = contents.writer();
+        var encoded_digest: [hex_digest_len]u8 = undefined;
+
         for (self.files.items) |file| {
             _ = std.fmt.bufPrint(&encoded_digest, "{x}", .{file.bin_digest}) catch unreachable;
             try writer.print("{d} {d} {d} {s} {s}\n", .{
@@ -547,7 +548,8 @@ pub const Manifest = struct {
             });
         }
 
-        try self.manifest_file.?.pwriteAll(contents.items, 0);
+        try manifest_file.setEndPos(contents.items.len);
+        try manifest_file.pwriteAll(contents.items, 0);
         self.manifest_dirty = false;
     }
 

From ed06a78f35e7281289249b0d0c119bd64845dd51 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Mon, 28 Sep 2020 23:20:14 -0700
Subject: [PATCH 178/210] stage2: WASM LLD linking

---
 BRANCH_TODO         |   1 -
 lib/std/target.zig  |   2 +-
 src/Compilation.zig |   4 +-
 src/link/Wasm.zig   | 210 +++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 212 insertions(+), 5 deletions(-)

diff --git a/BRANCH_TODO b/BRANCH_TODO
index a579d4a5eb..5c0b4ce27a 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,5 +1,4 @@
  * MachO LLD linking
- * WASM LLD linking
  * audit the CLI options for stage2
  * audit the base cache hash
  * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process.
diff --git a/lib/std/target.zig b/lib/std/target.zig
index 65e9f75457..e1a7e1a2bf 100644
--- a/lib/std/target.zig
+++ b/lib/std/target.zig
@@ -1486,7 +1486,7 @@ pub const Target = struct {
 
     /// Return whether or not the given host target is capable of executing natively executables
     /// of the other target.
-    pub fn canExecBinariesOf(host_target: std.Target, binary_target: std.Target) bool {
+    pub fn canExecBinariesOf(host_target: Target, binary_target: Target) bool {
         if (host_target.os.tag != binary_target.os.tag)
             return false;
 
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 314b7cdbdd..4bc9679ce2 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2418,7 +2418,7 @@ fn buildStaticLibFromZig(comp: *Compilation, src_basename: []const u8, out: *?CR
     const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{
         .root_name = root_name,
         .target = target,
-        .output_mode = .Lib,
+        .output_mode = .Obj,
     });
     defer comp.gpa.free(bin_basename);
 
@@ -2441,7 +2441,7 @@ fn buildStaticLibFromZig(comp: *Compilation, src_basename: []const u8, out: *?CR
         .target = target,
         .root_name = root_name,
         .root_pkg = &root_pkg,
-        .output_mode = .Lib,
+        .output_mode = .Obj,
         .rand = comp.rand,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig
index 4cff09ef69..509544c94f 100644
--- a/src/link/Wasm.zig
+++ b/src/link/Wasm.zig
@@ -1,10 +1,12 @@
 const Wasm = @This();
 
 const std = @import("std");
+const mem = std.mem;
 const Allocator = std.mem.Allocator;
 const assert = std.debug.assert;
 const fs = std.fs;
 const leb = std.debug.leb;
+const log = std.log.scoped(.link);
 
 const Module = @import("../Module.zig");
 const Compilation = @import("../Compilation.zig");
@@ -12,6 +14,7 @@ const codegen = @import("../codegen/wasm.zig");
 const link = @import("../link.zig");
 const trace = @import("../tracy.zig").trace;
 const build_options = @import("build_options");
+const Cache = @import("../Cache.zig");
 
 /// Various magic numbers defined by the wasm spec
 const spec = struct {
@@ -137,7 +140,7 @@ pub fn freeDecl(self: *Wasm, decl: *Module.Decl) void {
 
 pub fn flush(self: *Wasm, comp: *Compilation) !void {
     if (build_options.have_llvm and self.base.options.use_lld) {
-        return error.WasmLinkingWithLLDUnimplemented;
+        return self.linkWithLLD(comp);
     } else {
         return self.flushModule(comp);
     }
@@ -248,6 +251,211 @@ pub fn flushModule(self: *Wasm, comp: *Compilation) !void {
     }
 }
 
+fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
+    const tracy = trace(@src());
+    defer tracy.end();
+
+    var arena_allocator = std.heap.ArenaAllocator.init(self.base.allocator);
+    defer arena_allocator.deinit();
+    const arena = &arena_allocator.allocator;
+
+    const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
+
+    // If there is no Zig code to compile, then we should skip flushing the output file because it
+    // will not be part of the linker line anyway.
+    const module_obj_path: ?[]const u8 = if (self.base.options.module) |module| blk: {
+        const use_stage1 = build_options.is_stage1 and self.base.options.use_llvm;
+        if (use_stage1) {
+            const obj_basename = try std.zig.binNameAlloc(arena, .{
+                .root_name = self.base.options.root_name,
+                .target = self.base.options.target,
+                .output_mode = .Obj,
+            });
+            const o_directory = self.base.options.module.?.zig_cache_artifact_directory;
+            const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
+            break :blk full_obj_path;
+        }
+
+        try self.flushModule(comp);
+        const obj_basename = self.base.intermediary_basename.?;
+        const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
+        break :blk full_obj_path;
+    } else null;
+
+    const target = self.base.options.target;
+
+    const id_symlink_basename = "lld.id";
+
+    var man: Cache.Manifest = undefined;
+    defer if (!self.base.options.disable_lld_caching) man.deinit();
+
+    var digest: [Cache.hex_digest_len]u8 = undefined;
+
+    if (!self.base.options.disable_lld_caching) {
+        man = comp.cache_parent.obtain();
+
+        // We are about to obtain this lock, so here we give other processes a chance first.
+        self.base.releaseLock();
+
+        try man.addListOfFiles(self.base.options.objects);
+        for (comp.c_object_table.items()) |entry| {
+            _ = try man.addFile(entry.key.status.success.object_path, null);
+        }
+        try man.addOptionalFile(module_obj_path);
+        man.hash.addOptional(self.base.options.stack_size_override);
+        man.hash.addListOfBytes(self.base.options.extra_lld_args);
+
+        // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
+        _ = try man.hit();
+        digest = man.final();
+
+        var prev_digest_buf: [digest.len]u8 = undefined;
+        const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {
+            log.debug("WASM LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) });
+            // Handle this as a cache miss.
+            break :blk prev_digest_buf[0..0];
+        };
+        if (mem.eql(u8, prev_digest, &digest)) {
+            log.debug("WASM LLD digest={} match - skipping invocation", .{digest});
+            // Hot diggity dog! The output binary is already there.
+            self.base.lock = man.toOwnedLock();
+            return;
+        }
+        log.debug("WASM LLD prev_digest={} new_digest={}", .{ prev_digest, digest });
+
+        // We are about to change the output file to be different, so we invalidate the build hash now.
+        directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
+            error.FileNotFound => {},
+            else => |e| return e,
+        };
+    }
+
+    const is_obj = self.base.options.output_mode == .Obj;
+
+    // Create an LLD command line and invoke it.
+    var argv = std.ArrayList([]const u8).init(self.base.allocator);
+    defer argv.deinit();
+    // Even though we're calling LLD as a library it thinks the first argument is its own exe name.
+    try argv.append("lld");
+    if (is_obj) {
+        try argv.append("-r");
+    }
+
+    try argv.append("-error-limit=0");
+
+    if (self.base.options.output_mode == .Exe) {
+        // Increase the default stack size to a more reasonable value of 1MB instead of
+        // the default of 1 Wasm page being 64KB, unless overriden by the user.
+        try argv.append("-z");
+        const stack_size = self.base.options.stack_size_override orelse 1048576;
+        const arg = try std.fmt.allocPrint(arena, "stack-size={d}", .{stack_size});
+        try argv.append(arg);
+
+        // Put stack before globals so that stack overflow results in segfault immediately
+        // before corrupting globals. See https://github.com/ziglang/zig/issues/4496
+        try argv.append("--stack-first");
+    } else {
+        try argv.append("--no-entry"); // So lld doesn't look for _start.
+        try argv.append("--export-all");
+    }
+    try argv.appendSlice(&[_][]const u8{
+        "--allow-undefined",
+        "-o",
+        try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path}),
+    });
+
+    // Positional arguments to the linker such as object files.
+    try argv.appendSlice(self.base.options.objects);
+
+    for (comp.c_object_table.items()) |entry| {
+        try argv.append(entry.key.status.success.object_path);
+    }
+    if (module_obj_path) |p| {
+        try argv.append(p);
+    }
+
+    if (self.base.options.output_mode == .Exe and !self.base.options.is_compiler_rt_or_libc) {
+        if (!self.base.options.link_libc) {
+            try argv.append(comp.libc_static_lib.?.full_object_path);
+        }
+        try argv.append(comp.compiler_rt_static_lib.?.full_object_path);
+    }
+
+    if (self.base.options.verbose_link) {
+        Compilation.dump_argv(argv.items);
+    }
+
+    // TODO allocSentinel crashed stage1 so this is working around it.
+    const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1);
+    new_argv_with_sentinel[argv.items.len] = null;
+    const new_argv = new_argv_with_sentinel[0..argv.items.len :null];
+    for (argv.items) |arg, i| {
+        new_argv[i] = try arena.dupeZ(u8, arg);
+    }
+
+    var stderr_context: LLDContext = .{
+        .wasm = self,
+        .data = std.ArrayList(u8).init(self.base.allocator),
+    };
+    defer stderr_context.data.deinit();
+    var stdout_context: LLDContext = .{
+        .wasm = self,
+        .data = std.ArrayList(u8).init(self.base.allocator),
+    };
+    defer stdout_context.data.deinit();
+    const llvm = @import("../llvm.zig");
+    const ok = llvm.Link(
+        .Wasm,
+        new_argv.ptr,
+        new_argv.len,
+        append_diagnostic,
+        @ptrToInt(&stdout_context),
+        @ptrToInt(&stderr_context),
+    );
+    if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory;
+    if (stdout_context.data.items.len != 0) {
+        std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items});
+    }
+    if (!ok) {
+        // TODO parse this output and surface with the Compilation API rather than
+        // directly outputting to stderr here.
+        std.debug.print("{}", .{stderr_context.data.items});
+        return error.LLDReportedFailure;
+    }
+    if (stderr_context.data.items.len != 0) {
+        std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items});
+    }
+
+    if (!self.base.options.disable_lld_caching) {
+        // Update the dangling symlink with the digest. If it fails we can continue; it only
+        // means that the next invocation will have an unnecessary cache miss.
+        directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {
+            std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});
+        };
+        // Again failure here only means an unnecessary cache miss.
+        man.writeManifest() catch |err| {
+            std.log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
+        };
+        // We hang on to this lock so that the output file path can be used without
+        // other processes clobbering it.
+        self.base.lock = man.toOwnedLock();
+    }
+}
+
+const LLDContext = struct {
+    data: std.ArrayList(u8),
+    wasm: *Wasm,
+    oom: bool = false,
+};
+
+fn append_diagnostic(context: usize, ptr: [*]const u8, len: usize) callconv(.C) void {
+    const lld_context = @intToPtr(*LLDContext, context);
+    const msg = ptr[0..len];
+    lld_context.data.appendSlice(msg) catch |err| switch (err) {
+        error.OutOfMemory => lld_context.oom = true,
+    };
+}
+
 /// Get the current index of a given Decl in the function list
 /// TODO: we could maintain a hash map to potentially make this
 fn getFuncidx(self: Wasm, decl: *Module.Decl) ?u32 {

From 9d0da1612e6abf4a05fd6461231f727238d71759 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 00:24:17 -0700
Subject: [PATCH 179/210] langref: use general purpose allocator in the wasi
 example

---
 doc/langref.html.in | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/doc/langref.html.in b/doc/langref.html.in
index 320b10bbe5..870764f054 100644
--- a/doc/langref.html.in
+++ b/doc/langref.html.in
@@ -9889,12 +9889,13 @@ The result is 3
    
 const std = @import("std");
 
 pub fn main() !void {
-    // TODO a better default allocator that isn't as wasteful!
-    const args = try std.process.argsAlloc(std.heap.page_allocator);
-    defer std.process.argsFree(std.heap.page_allocator, args);
+    var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){};
+    const gpa = &general_purpose_allocator.allocator;
+    const args = try std.process.argsAlloc(gpa);
+    defer std.process.argsFree(gpa, args);
 
     for (args) |arg, i| {
-        std.debug.print("{}: {}\n", .{i, arg});
+        std.debug.print("{}: {}\n", .{ i, arg });
     }
 }
       {#code_end#}

From af6c3a39340ea5be3435dff95ab7d309945b048e Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 00:24:54 -0700
Subject: [PATCH 180/210] mingw: better -D arg for processing def.in files

Thanks @LemonBoy
---
 src/mingw.zig | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/mingw.zig b/src/mingw.zig
index 41a328c6f4..b6c8591ea4 100644
--- a/src/mingw.zig
+++ b/src/mingw.zig
@@ -342,11 +342,8 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8) !void {
     const target_def_arg = switch (target.cpu.arch) {
         .i386 => "-DDEF_I386",
         .x86_64 => "-DDEF_X64",
-        .arm, .armeb => switch (target.cpu.arch.ptrBitWidth()) {
-            32 => "-DDEF_ARM32",
-            64 => "-DDEF_ARM64",
-            else => unreachable,
-        },
+        .arm, .armeb, .thumb, .thumbeb, .aarch64_32 => "-DDEF_ARM32",
+        .aarch64, .aarch64_be => "-DDEF_ARM64",
         else => unreachable,
     };
 

From 41f6627521f7ff03962ff73944f5be3722346385 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 00:25:48 -0700
Subject: [PATCH 181/210] stage2: infer --strip on wasm builds

---
 src/target.zig | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/target.zig b/src/target.zig
index f6d0f41112..fc0c7a0745 100644
--- a/src/target.zig
+++ b/src/target.zig
@@ -143,7 +143,7 @@ pub fn libcNeedsLibUnwind(target: std.Target) bool {
 }
 
 pub fn requiresPIE(target: std.Target) bool {
-    return target.isAndroid();
+    return target.isAndroid() or target.isDarwin();
 }
 
 /// This function returns whether non-pic code is completely invalid on the given target.
@@ -336,3 +336,7 @@ pub fn is_libcpp_lib_name(target: std.Target, name: []const u8) bool {
         eqlIgnoreCase(ignore_case, name, "stdc++") or
         eqlIgnoreCase(ignore_case, name, "c++abi");
 }
+
+pub fn hasDebugInfo(target: std.Target) bool {
+    return !target.cpu.arch.isWasm();
+}

From fa6d150441d1d8679a77c5e9a6071fa952851376 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 00:26:18 -0700
Subject: [PATCH 182/210] stage2: MachO LLD Linking

---
 BRANCH_TODO         |   1 -
 src/Compilation.zig |  13 +-
 src/link/Elf.zig    |  18 +--
 src/link/MachO.zig  | 370 +++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 385 insertions(+), 17 deletions(-)

diff --git a/BRANCH_TODO b/BRANCH_TODO
index 5c0b4ce27a..3135689e12 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,4 +1,3 @@
- * MachO LLD linking
  * audit the CLI options for stage2
  * audit the base cache hash
  * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process.
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 4bc9679ce2..c636ccb0b2 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -558,6 +558,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             break :blk buf.items[0 .. buf.items.len - 1 :0].ptr;
         } else null;
 
+        const strip = options.strip or !target_util.hasDebugInfo(options.target);
+
         // We put everything into the cache hash that *cannot be modified during an incremental update*.
         // For example, one cannot change the target between updates, but one can change source files,
         // so the target goes into the cache hash, but source files do not. This is so that we can
@@ -586,7 +588,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         cache.hash.add(stack_check);
         cache.hash.add(link_mode);
         cache.hash.add(function_sections);
-        cache.hash.add(options.strip);
+        cache.hash.add(strip);
         cache.hash.add(link_libc);
         cache.hash.add(options.link_libcpp);
         cache.hash.add(options.output_mode);
@@ -671,7 +673,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         } else null;
         errdefer if (module) |zm| zm.deinit();
 
-        const error_return_tracing = !options.strip and switch (options.optimize_mode) {
+        const error_return_tracing = !strip and switch (options.optimize_mode) {
             .Debug, .ReleaseSafe => true,
             .ReleaseFast, .ReleaseSmall => false,
         };
@@ -751,7 +753,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .system_libs = system_libs,
             .lib_dirs = options.lib_dirs,
             .rpath_list = options.rpath_list,
-            .strip = options.strip,
+            .strip = strip,
             .is_native_os = options.is_native_os,
             .function_sections = options.function_sections orelse false,
             .allow_shlib_undefined = options.linker_allow_shlib_undefined,
@@ -2415,10 +2417,11 @@ fn buildStaticLibFromZig(comp: *Compilation, src_basename: []const u8, out: *?CR
     };
     const root_name = mem.split(src_basename, ".").next().?;
     const target = comp.getTarget();
+    const output_mode: std.builtin.OutputMode = if (target.cpu.arch.isWasm()) .Obj else .Lib;
     const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{
         .root_name = root_name,
         .target = target,
-        .output_mode = .Obj,
+        .output_mode = output_mode,
     });
     defer comp.gpa.free(bin_basename);
 
@@ -2441,7 +2444,7 @@ fn buildStaticLibFromZig(comp: *Compilation, src_basename: []const u8, out: *?CR
         .target = target,
         .root_name = root_name,
         .root_pkg = &root_pkg,
-        .output_mode = .Obj,
+        .output_mode = output_mode,
         .rand = comp.rand,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 88f5040761..dc664b14be 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -1241,6 +1241,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         break :blk full_obj_path;
     } else null;
 
+    const is_obj = self.base.options.output_mode == .Obj;
     const is_lib = self.base.options.output_mode == .Lib;
     const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib;
     const is_exe_or_dyn_lib = is_dyn_lib or self.base.options.output_mode == .Exe;
@@ -1248,6 +1249,9 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         self.base.options.link_mode == .Dynamic and is_exe_or_dyn_lib;
     const link_in_crt = self.base.options.link_libc and self.base.options.output_mode == .Exe;
     const target = self.base.options.target;
+    const gc_sections = self.base.options.gc_sections orelse !is_obj;
+    const stack_size = self.base.options.stack_size_override orelse 16777216;
+    const allow_shlib_undefined = self.base.options.allow_shlib_undefined orelse !self.base.options.is_native_os;
 
     // Here we want to determine whether we can save time by not invoking LLD when the
     // output is unchanged. None of the linker options or the object files that are being
@@ -1279,8 +1283,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         try man.addOptionalFile(module_obj_path);
         // We can skip hashing libc and libc++ components that we are in charge of building from Zig
         // installation sources because they are always a product of the compiler version + target information.
-        man.hash.addOptional(self.base.options.stack_size_override);
-        man.hash.addOptional(self.base.options.gc_sections);
+        man.hash.add(stack_size);
+        man.hash.add(gc_sections);
         man.hash.add(self.base.options.eh_frame_hdr);
         man.hash.add(self.base.options.rdynamic);
         man.hash.addListOfBytes(self.base.options.extra_lld_args);
@@ -1304,7 +1308,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
             man.hash.addOptional(self.base.options.version);
         }
         man.hash.addStringSet(self.base.options.system_libs);
-        man.hash.addOptional(self.base.options.allow_shlib_undefined);
+        man.hash.add(allow_shlib_undefined);
         man.hash.add(self.base.options.bind_global_refs_locally);
 
         // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
@@ -1332,8 +1336,6 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         };
     }
 
-    const is_obj = self.base.options.output_mode == .Obj;
-
     // Create an LLD command line and invoke it.
     var argv = std.ArrayList([]const u8).init(self.base.allocator);
     defer argv.deinit();
@@ -1347,9 +1349,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
 
     if (self.base.options.output_mode == .Exe) {
         try argv.append("-z");
-        const stack_size = self.base.options.stack_size_override orelse 16777216;
-        const arg = try std.fmt.allocPrint(arena, "stack-size={}", .{stack_size});
-        try argv.append(arg);
+        try argv.append(try std.fmt.allocPrint(arena, "stack-size={}", .{stack_size}));
     }
 
     if (self.base.options.linker_script) |linker_script| {
@@ -1357,7 +1357,6 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         try argv.append(linker_script);
     }
 
-    const gc_sections = self.base.options.gc_sections orelse !is_obj;
     if (gc_sections) {
         try argv.append("--gc-sections");
     }
@@ -1577,7 +1576,6 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         }
     }
 
-    const allow_shlib_undefined = self.base.options.allow_shlib_undefined orelse !self.base.options.is_native_os;
     if (allow_shlib_undefined) {
         try argv.append("--allow-shlib-undefined");
     }
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index c38c6b34ff..146f2616c2 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -17,6 +17,8 @@ const Module = @import("../Module.zig");
 const Compilation = @import("../Compilation.zig");
 const link = @import("../link.zig");
 const File = link.File;
+const Cache = @import("../Cache.zig");
+const target_util = @import("../target.zig");
 
 pub const base_tag: File.Tag = File.Tag.macho;
 
@@ -180,8 +182,12 @@ pub fn createEmpty(gpa: *Allocator, options: link.Options) !*MachO {
 
 pub fn flush(self: *MachO, comp: *Compilation) !void {
     if (build_options.have_llvm and self.base.options.use_lld) {
-        return error.MachOLLDLinkingUnimplemented;
+        return self.linkWithLLD(comp);
     } else {
+        switch (self.base.options.effectiveOutputMode()) {
+            .Exe, .Obj => {},
+            .Lib => return error.TODOImplementWritingLibFiles,
+        }
         return self.flushModule(comp);
     }
 }
@@ -282,6 +288,368 @@ pub fn flushModule(self: *MachO, comp: *Compilation) !void {
     }
 }
 
+fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
+    const tracy = trace(@src());
+    defer tracy.end();
+
+    var arena_allocator = std.heap.ArenaAllocator.init(self.base.allocator);
+    defer arena_allocator.deinit();
+    const arena = &arena_allocator.allocator;
+
+    const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
+
+    // If there is no Zig code to compile, then we should skip flushing the output file because it
+    // will not be part of the linker line anyway.
+    const module_obj_path: ?[]const u8 = if (self.base.options.module) |module| blk: {
+        const use_stage1 = build_options.is_stage1 and self.base.options.use_llvm;
+        if (use_stage1) {
+            const obj_basename = try std.zig.binNameAlloc(arena, .{
+                .root_name = self.base.options.root_name,
+                .target = self.base.options.target,
+                .output_mode = .Obj,
+            });
+            const o_directory = self.base.options.module.?.zig_cache_artifact_directory;
+            const full_obj_path = try o_directory.join(arena, &[_][]const u8{obj_basename});
+            break :blk full_obj_path;
+        }
+
+        try self.flushModule(comp);
+        const obj_basename = self.base.intermediary_basename.?;
+        const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
+        break :blk full_obj_path;
+    } else null;
+
+    const is_obj = self.base.options.output_mode == .Obj;
+    const is_lib = self.base.options.output_mode == .Lib;
+    const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib;
+    const is_exe_or_dyn_lib = is_dyn_lib or self.base.options.output_mode == .Exe;
+    const have_dynamic_linker = self.base.options.link_libc and
+        self.base.options.link_mode == .Dynamic and is_exe_or_dyn_lib;
+    const link_in_crt = self.base.options.link_libc and self.base.options.output_mode == .Exe;
+    const target = self.base.options.target;
+    const stack_size = self.base.options.stack_size_override orelse 16777216;
+    const allow_shlib_undefined = self.base.options.allow_shlib_undefined orelse !self.base.options.is_native_os;
+
+    const id_symlink_basename = "lld.id";
+
+    var man: Cache.Manifest = undefined;
+    defer if (!self.base.options.disable_lld_caching) man.deinit();
+
+    var digest: [Cache.hex_digest_len]u8 = undefined;
+
+    if (!self.base.options.disable_lld_caching) {
+        man = comp.cache_parent.obtain();
+
+        // We are about to obtain this lock, so here we give other processes a chance first.
+        self.base.releaseLock();
+
+        try man.addOptionalFile(self.base.options.linker_script);
+        try man.addOptionalFile(self.base.options.version_script);
+        try man.addListOfFiles(self.base.options.objects);
+        for (comp.c_object_table.items()) |entry| {
+            _ = try man.addFile(entry.key.status.success.object_path, null);
+        }
+        try man.addOptionalFile(module_obj_path);
+        // We can skip hashing libc and libc++ components that we are in charge of building from Zig
+        // installation sources because they are always a product of the compiler version + target information.
+        man.hash.add(stack_size);
+        man.hash.add(self.base.options.rdynamic);
+        man.hash.addListOfBytes(self.base.options.extra_lld_args);
+        man.hash.addListOfBytes(self.base.options.lib_dirs);
+        man.hash.addListOfBytes(self.base.options.framework_dirs);
+        man.hash.addListOfBytes(self.base.options.frameworks);
+        man.hash.addListOfBytes(self.base.options.rpath_list);
+        man.hash.add(self.base.options.is_compiler_rt_or_libc);
+        man.hash.add(self.base.options.z_nodelete);
+        man.hash.add(self.base.options.z_defs);
+        if (is_dyn_lib) {
+            man.hash.addOptional(self.base.options.version);
+        }
+        man.hash.addStringSet(self.base.options.system_libs);
+        man.hash.add(allow_shlib_undefined);
+        man.hash.add(self.base.options.bind_global_refs_locally);
+
+        // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
+        _ = try man.hit();
+        digest = man.final();
+
+        var prev_digest_buf: [digest.len]u8 = undefined;
+        const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {
+            log.debug("MachO LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) });
+            // Handle this as a cache miss.
+            break :blk prev_digest_buf[0..0];
+        };
+        if (mem.eql(u8, prev_digest, &digest)) {
+            log.debug("MachO LLD digest={} match - skipping invocation", .{digest});
+            // Hot diggity dog! The output binary is already there.
+            self.base.lock = man.toOwnedLock();
+            return;
+        }
+        log.debug("MachO LLD prev_digest={} new_digest={}", .{ prev_digest, digest });
+
+        // We are about to change the output file to be different, so we invalidate the build hash now.
+        directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
+            error.FileNotFound => {},
+            else => |e| return e,
+        };
+    }
+
+    // Create an LLD command line and invoke it.
+    var argv = std.ArrayList([]const u8).init(self.base.allocator);
+    defer argv.deinit();
+    // Even though we're calling LLD as a library it thinks the first argument is its own exe name.
+    try argv.append("lld");
+    if (is_obj) {
+        try argv.append("-r");
+    }
+
+    try argv.append("-error-limit");
+    try argv.append("0");
+
+    try argv.append("-demangle");
+
+    if (self.base.options.rdynamic) {
+        try argv.append("--export-dynamic");
+    }
+
+    try argv.appendSlice(self.base.options.extra_lld_args);
+
+    if (self.base.options.z_nodelete) {
+        try argv.append("-z");
+        try argv.append("nodelete");
+    }
+    if (self.base.options.z_defs) {
+        try argv.append("-z");
+        try argv.append("defs");
+    }
+
+    if (is_dyn_lib) {
+        try argv.append("-static");
+    } else {
+        try argv.append("-dynamic");
+    }
+
+    if (is_dyn_lib) {
+        try argv.append("-dylib");
+
+        if (self.base.options.version) |ver| {
+            const compat_vers = try std.fmt.allocPrint(arena, "{d}.0.0", .{ver.major});
+            try argv.append("-compatibility_version");
+            try argv.append(compat_vers);
+
+            const cur_vers = try std.fmt.allocPrint(arena, "{d}.{d}.{d}", .{ ver.major, ver.minor, ver.patch });
+            try argv.append("-current_version");
+            try argv.append(cur_vers);
+        }
+
+        // TODO getting an error when running an executable when doing this rpath thing
+        //Buf *dylib_install_name = buf_sprintf("@rpath/lib%s.%" ZIG_PRI_usize ".dylib",
+        //    buf_ptr(g->root_out_name), g->version_major);
+        //try argv.append("-install_name");
+        //try argv.append(buf_ptr(dylib_install_name));
+    }
+
+    try argv.append("-arch");
+    try argv.append(darwinArchString(target.cpu.arch));
+
+    switch (target.os.tag) {
+        .macosx => {
+            try argv.append("-macosx_version_min");
+        },
+        .ios, .tvos, .watchos => switch (target.cpu.arch) {
+            .i386, .x86_64 => {
+                try argv.append("-ios_simulator_version_min");
+            },
+            else => {
+                try argv.append("-iphoneos_version_min");
+            },
+        },
+        else => unreachable,
+    }
+    const ver = target.os.version_range.semver.min;
+    const version_string = try std.fmt.allocPrint(arena, "{d}.{d}.{d}", .{ ver.major, ver.minor, ver.patch });
+    try argv.append(version_string);
+
+    try argv.append("-sdk_version");
+    try argv.append(version_string);
+
+    if (target_util.requiresPIE(target) and self.base.options.output_mode == .Exe) {
+        try argv.append("-pie");
+    }
+
+    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
+    try argv.append("-o");
+    try argv.append(full_out_path);
+
+    // rpaths
+    var rpath_table = std.StringHashMap(void).init(self.base.allocator);
+    defer rpath_table.deinit();
+    for (self.base.options.rpath_list) |rpath| {
+        if ((try rpath_table.fetchPut(rpath, {})) == null) {
+            try argv.append("-rpath");
+            try argv.append(rpath);
+        }
+    }
+    if (is_dyn_lib) {
+        if ((try rpath_table.fetchPut(full_out_path, {})) == null) {
+            try argv.append("-rpath");
+            try argv.append(full_out_path);
+        }
+    }
+
+    for (self.base.options.lib_dirs) |lib_dir| {
+        try argv.append("-L");
+        try argv.append(lib_dir);
+    }
+
+    // Positional arguments to the linker such as object files.
+    try argv.appendSlice(self.base.options.objects);
+
+    for (comp.c_object_table.items()) |entry| {
+        try argv.append(entry.key.status.success.object_path);
+    }
+    if (module_obj_path) |p| {
+        try argv.append(p);
+    }
+
+    // compiler_rt on darwin is missing some stuff, so we still build it and rely on LinkOnce
+    if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) {
+        try argv.append(comp.compiler_rt_static_lib.?.full_object_path);
+    }
+
+    // Shared libraries.
+    const system_libs = self.base.options.system_libs.items();
+    try argv.ensureCapacity(argv.items.len + system_libs.len);
+    for (system_libs) |entry| {
+        const link_lib = entry.key;
+        // By this time, we depend on these libs being dynamically linked libraries and not static libraries
+        // (the check for that needs to be earlier), but they could be full paths to .dylib files, in which
+        // case we want to avoid prepending "-l".
+        const ext = Compilation.classifyFileExt(link_lib);
+        const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib});
+        argv.appendAssumeCapacity(arg);
+    }
+
+    // libc++ dep
+    if (!is_obj and self.base.options.link_libcpp) {
+        try argv.append(comp.libcxxabi_static_lib.?.full_object_path);
+        try argv.append(comp.libcxx_static_lib.?.full_object_path);
+    }
+
+    // On Darwin, libSystem has libc in it, but also you have to use it
+    // to make syscalls because the syscall numbers are not documented
+    // and change between versions. So we always link against libSystem.
+    // LLD craps out if you do -lSystem cross compiling, so until that
+    // codebase gets some love from the new maintainers we're left with
+    // this dirty hack.
+    if (self.base.options.is_native_os) {
+        try argv.append("-lSystem");
+    }
+
+    for (self.base.options.framework_dirs) |framework_dir| {
+        try argv.append("-F");
+        try argv.append(framework_dir);
+    }
+    for (self.base.options.frameworks) |framework| {
+        try argv.append("-framework");
+        try argv.append(framework);
+    }
+
+    if (allow_shlib_undefined) {
+        try argv.append("-undefined");
+        try argv.append("dynamic_lookup");
+    }
+    if (self.base.options.bind_global_refs_locally) {
+        try argv.append("-Bsymbolic");
+    }
+
+    if (self.base.options.verbose_link) {
+        Compilation.dump_argv(argv.items);
+    }
+
+    // TODO allocSentinel crashed stage1 so this is working around it.
+    const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1);
+    new_argv_with_sentinel[argv.items.len] = null;
+    const new_argv = new_argv_with_sentinel[0..argv.items.len :null];
+    for (argv.items) |arg, i| {
+        new_argv[i] = try arena.dupeZ(u8, arg);
+    }
+
+    var stderr_context: LLDContext = .{
+        .macho = self,
+        .data = std.ArrayList(u8).init(self.base.allocator),
+    };
+    defer stderr_context.data.deinit();
+    var stdout_context: LLDContext = .{
+        .macho = self,
+        .data = std.ArrayList(u8).init(self.base.allocator),
+    };
+    defer stdout_context.data.deinit();
+    const llvm = @import("../llvm.zig");
+    const ok = llvm.Link(
+        .MachO,
+        new_argv.ptr,
+        new_argv.len,
+        append_diagnostic,
+        @ptrToInt(&stdout_context),
+        @ptrToInt(&stderr_context),
+    );
+    if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory;
+    if (stdout_context.data.items.len != 0) {
+        std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items});
+    }
+    if (!ok) {
+        // TODO parse this output and surface with the Compilation API rather than
+        // directly outputting to stderr here.
+        std.debug.print("{}", .{stderr_context.data.items});
+        return error.LLDReportedFailure;
+    }
+    if (stderr_context.data.items.len != 0) {
+        std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items});
+    }
+
+    if (!self.base.options.disable_lld_caching) {
+        // Update the dangling symlink with the digest. If it fails we can continue; it only
+        // means that the next invocation will have an unnecessary cache miss.
+        directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {
+            std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});
+        };
+        // Again failure here only means an unnecessary cache miss.
+        man.writeManifest() catch |err| {
+            std.log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
+        };
+        // We hang on to this lock so that the output file path can be used without
+        // other processes clobbering it.
+        self.base.lock = man.toOwnedLock();
+    }
+}
+
+const LLDContext = struct {
+    data: std.ArrayList(u8),
+    macho: *MachO,
+    oom: bool = false,
+};
+
+fn append_diagnostic(context: usize, ptr: [*]const u8, len: usize) callconv(.C) void {
+    const lld_context = @intToPtr(*LLDContext, context);
+    const msg = ptr[0..len];
+    lld_context.data.appendSlice(msg) catch |err| switch (err) {
+        error.OutOfMemory => lld_context.oom = true,
+    };
+}
+
+fn darwinArchString(arch: std.Target.Cpu.Arch) []const u8 {
+    return switch (arch) {
+        .aarch64, .aarch64_be, .aarch64_32 => "arm64",
+        .thumb, .arm => "arm",
+        .thumbeb, .armeb => "armeb",
+        .powerpc => "ppc",
+        .powerpc64 => "ppc64",
+        .powerpc64le => "ppc64le",
+        else => @tagName(arch),
+    };
+}
+
 pub fn deinit(self: *MachO) void {
     self.offset_table.deinit(self.base.allocator);
     self.string_table.deinit(self.base.allocator);

From cfbcb4116017a27a403f8be11a32f25f6c1c8671 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 00:34:53 -0700
Subject: [PATCH 183/210] stage2: add CLI option for -fstack-report

---
 BRANCH_TODO         | 4 +---
 src/Compilation.zig | 5 ++++-
 src/main.zig        | 5 +++++
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/BRANCH_TODO b/BRANCH_TODO
index 3135689e12..51a6490ba3 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,10 +1,8 @@
  * audit the CLI options for stage2
  * audit the base cache hash
- * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process.
- * `-ftime-report`
- *  -fstack-report               print stack size diagnostics\n"
  * try building some software with zig cc to make sure it didn't regress
 
+ * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process.
  * implement proper parsing of clang stderr/stdout and exposing compile errors with the Compilation API
  * implement proper parsing of LLD stderr/stdout and exposing compile errors with the Compilation API
  * support cross compiling stage2 with `zig build`
diff --git a/src/Compilation.zig b/src/Compilation.zig
index c636ccb0b2..aa6ab94415 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -61,6 +61,7 @@ verbose_cimport: bool,
 verbose_llvm_cpu_features: bool,
 disable_c_depfile: bool,
 time_report: bool,
+stack_report: bool,
 
 c_source_files: []const CSourceFile,
 clang_argv: []const []const u8,
@@ -348,6 +349,7 @@ pub const InitOptions = struct {
     single_threaded: bool = false,
     is_native_os: bool,
     time_report: bool = false,
+    stack_report: bool = false,
     link_eh_frame_hdr: bool = false,
     linker_script: ?[]const u8 = null,
     version_script: ?[]const u8 = null,
@@ -821,6 +823,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .owned_link_dir = owned_link_dir,
             .color = options.color,
             .time_report = options.time_report,
+            .stack_report = options.stack_report,
             .test_filter = options.test_filter,
             .test_name_prefix = options.test_name_prefix,
             .test_evented_io = options.test_evented_io,
@@ -2670,7 +2673,7 @@ fn updateStage1Module(comp: *Compilation) !void {
         .function_sections = comp.bin_file.options.function_sections,
         .enable_stack_probing = comp.bin_file.options.stack_check,
         .enable_time_report = comp.time_report,
-        .enable_stack_report = false,
+        .enable_stack_report = comp.stack_report,
         .test_is_evented = comp.test_evented_io,
         .verbose_tokenize = comp.verbose_tokenize,
         .verbose_ast = comp.verbose_ast,
diff --git a/src/main.zig b/src/main.zig
index 61ef77cce7..80e92a73b0 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -284,6 +284,7 @@ const usage_build_generic =
     \\
     \\Debug Options (Zig Compiler Development):
     \\  -ftime-report                Print timing diagnostics
+    \\  -fstack-report               Print stack size diagnostics
     \\  --verbose-link               Display linker invocations
     \\  --verbose-cc                 Display C compiler invocations
     \\  --verbose-tokenize           Enable compiler debug output for tokenization
@@ -390,6 +391,7 @@ fn buildOutputType(
     var verbose_cimport = false;
     var verbose_llvm_cpu_features = false;
     var time_report = false;
+    var stack_report = false;
     var show_builtin = false;
     var emit_bin: Emit = .yes_default_path;
     var emit_asm: Emit = .no;
@@ -728,6 +730,8 @@ fn buildOutputType(
                         watch = true;
                     } else if (mem.eql(u8, arg, "-ftime-report")) {
                         time_report = true;
+                    } else if (mem.eql(u8, arg, "-fstack-report")) {
+                        stack_report = true;
                     } else if (mem.eql(u8, arg, "-fPIC")) {
                         want_pic = true;
                     } else if (mem.eql(u8, arg, "-fno-PIC")) {
@@ -1568,6 +1572,7 @@ fn buildOutputType(
         .machine_code_model = machine_code_model,
         .color = color,
         .time_report = time_report,
+        .stack_report = stack_report,
         .is_test = arg_mode == .zig_test,
         .each_lib_rpath = each_lib_rpath,
         .test_evented_io = test_evented_io,

From d3a99c7bd5396f2e458d844368ce2743934e43e2 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 00:50:20 -0700
Subject: [PATCH 184/210] add CLI options for darwin frameworks and
 -ffunction-sections

and add missing usage help text
---
 BRANCH_TODO         | 48 ++-------------------------------------------
 src/Compilation.zig |  7 +++----
 src/main.zig        | 24 +++++++++++++++++++++--
 3 files changed, 27 insertions(+), 52 deletions(-)

diff --git a/BRANCH_TODO b/BRANCH_TODO
index 51a6490ba3..f2adbb51d8 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,47 +1,3 @@
- * audit the CLI options for stage2
+ * building risc-v musl regression
+ * go ahead and use allocSentinel now that the stage1 bug is fixed
  * audit the base cache hash
- * try building some software with zig cc to make sure it didn't regress
-
- * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process.
- * implement proper parsing of clang stderr/stdout and exposing compile errors with the Compilation API
- * implement proper parsing of LLD stderr/stdout and exposing compile errors with the Compilation API
- * support cross compiling stage2 with `zig build`
- * implement proper compile errors for failing to build glibc crt files and shared libs
- * implement -fno-emit-bin
- * improve the stage2 tests to support testing with LLVM extensions enabled
- * implement emit-h in stage2
- * multi-thread building C objects
- * implement serialization/deserialization of incremental compilation metadata
- * incremental compilation - implement detection of which source files changed
- * improve the cache hash logic for c objects with respect to extra flags and file parameters
- * LLVM codegen backend: put a sub-arch in the triple in some cases
- * implement an LLVM backend for stage2
- * implement outputting dynamic libraries in self-hosted linker
- * implement outputting static libraries (archive files) in self-hosted linker
- * support linking against object files in self-hosted linker
- * avoid invoking lld when it's just 1 object file (the `zig cc -c` case)
- * `zig fmt --check` should output to stdout not stderr.
- * main.zig: If there was an argsAllocZ we could avoid this allocation
- * improve robustness of response file parsing
- * there are a couple panic("TODO") in clang options parsing
- * std.testing needs improvement to support exposing directory path for its tmp dir (look for "bogus")
- * integrate target features into building assembly code
- * libc_installation.zig: make it look for msvc only if msvc abi is chosen
- * switch the default C ABI for windows to be mingw-w64
-   - make it .obj instead of .o always for coff
- * change glibc log errors to normal exposed compile errors
- * improve Directory.join to only use 1 allocation in a clean way.
- * tracy builds with lc++
- * some kind of "zig identifier escape" function rather than unconditionally using @"" syntax
-   in builtin.zig
- * rename std.builtin.Mode to std.builtin.OptimizeMode
- * implement `zig run` and `zig test` when combined with `--watch`
- * close the --pkg-begin --pkg-end Package directory handles
- * make std.Progress support multithreaded
- * update musl.zig static data to use native path separator in static data rather than replacing '/' at runtime
- * linking hello world with LLD, lld is silently calling exit(1) instead of reporting ok=false. when run standalone the error message is: ld.lld: error: section [index 3] has a sh_offset (0x57000) + sh_size (0x68) that is greater than the file size (0x57060)
- * submit PR to godbolt and update the CLI options (see changes to test/cli.zig)
- * make proposal about log levels
- * proposal for changing fs Z/W functions to be native paths and have a way to do native path string literals
- * proposal for block { break x; }
- * generally look for the "TODO surface this as a real compile error message" and fix all that stuff
diff --git a/src/Compilation.zig b/src/Compilation.zig
index aa6ab94415..1cd0d6c099 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -347,6 +347,7 @@ pub const InitOptions = struct {
     rdynamic: bool = false,
     strip: bool = false,
     single_threaded: bool = false,
+    function_sections: bool = false,
     is_native_os: bool,
     time_report: bool = false,
     stack_report: bool = false,
@@ -355,7 +356,6 @@ pub const InitOptions = struct {
     version_script: ?[]const u8 = null,
     override_soname: ?[]const u8 = null,
     linker_gc_sections: ?bool = null,
-    function_sections: ?bool = null,
     linker_allow_shlib_undefined: ?bool = null,
     linker_bind_global_refs_locally: ?bool = null,
     each_lib_rpath: ?bool = null,
@@ -538,7 +538,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         };
 
         const single_threaded = options.single_threaded or target_util.isSingleThreaded(options.target);
-        const function_sections = options.function_sections orelse false;
 
         const llvm_cpu_features: ?[*:0]const u8 = if (build_options.have_llvm and use_llvm) blk: {
             var buf = std.ArrayList(u8).init(arena);
@@ -589,7 +588,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         cache.hash.add(pic);
         cache.hash.add(stack_check);
         cache.hash.add(link_mode);
-        cache.hash.add(function_sections);
+        cache.hash.add(options.function_sections);
         cache.hash.add(strip);
         cache.hash.add(link_libc);
         cache.hash.add(options.link_libcpp);
@@ -757,7 +756,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .rpath_list = options.rpath_list,
             .strip = strip,
             .is_native_os = options.is_native_os,
-            .function_sections = options.function_sections orelse false,
+            .function_sections = options.function_sections,
             .allow_shlib_undefined = options.linker_allow_shlib_undefined,
             .bind_global_refs_locally = options.linker_bind_global_refs_locally orelse false,
             .z_nodelete = options.linker_z_nodelete,
diff --git a/src/main.zig b/src/main.zig
index 80e92a73b0..de2f38bffb 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -260,11 +260,13 @@ const usage_build_generic =
     \\  -D[macro]=[value]         Define C [macro] to [value] (1 if [value] omitted)
     \\  --libc [file]             Provide a file which specifies libc paths
     \\  -cflags [flags] --        Set extra flags for the next positional C source files
+    \\  -ffunction-sections       Places each function in a separate section
     \\
     \\Link Options:
     \\  -l[lib], --library [lib]       Link against system library
     \\  -L[d], --library-directory [d] Add a directory to the library search path
-    \\  -T[script]                     Use a custom linker script
+    \\  -T[script], --script [script]  Use a custom linker script
+    \\  --version-script [path]        Provide a version .map file
     \\  --dynamic-linker [path]        Set the dynamic interpreter path (usually ld.so)
     \\  --each-lib-rpath               Add rpath for each used dynamic library
     \\  --version [ver]                Dynamic library semver
@@ -273,7 +275,11 @@ const usage_build_generic =
     \\  --eh-frame-hdr                 Enable C++ exception handling by passing --eh-frame-hdr to linker
     \\  -dynamic                       Force output to be dynamically linked
     \\  -static                        Force output to be statically linked
+    \\  -Bsymbolic                     Bind global references locally
     \\  --subsystem [subsystem]        (windows) /SUBSYSTEM: to the linker\n"
+    \\  --stack [size]                 Override default stack size
+    \\  -framework [name]              (darwin) link against framework
+    \\  -F[dir]                        (darwin) add search path for frameworks
     \\
     \\Test Options:
     \\  --test-filter [text]           Skip tests that do not match filter
@@ -381,6 +387,7 @@ fn buildOutputType(
     var have_version = false;
     var strip = false;
     var single_threaded = false;
+    var function_sections = false;
     var watch = false;
     var verbose_link = false;
     var verbose_cc = false;
@@ -634,7 +641,15 @@ fn buildOutputType(
                         if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
                         i += 1;
                         try lib_dirs.append(args[i]);
-                    } else if (mem.eql(u8, arg, "-T")) {
+                    } else if (mem.eql(u8, arg, "-F")) {
+                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        i += 1;
+                        try framework_dirs.append(args[i]);
+                    } else if (mem.eql(u8, arg, "-framework")) {
+                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        i += 1;
+                        try frameworks.append(args[i]);
+                    } else if (mem.eql(u8, arg, "-T") or mem.eql(u8, arg, "--script")) {
                         if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
                         i += 1;
                         linker_script = args[i];
@@ -819,6 +834,8 @@ fn buildOutputType(
                         strip = true;
                     } else if (mem.eql(u8, arg, "--single-threaded")) {
                         single_threaded = true;
+                    } else if (mem.eql(u8, arg, "-ffunction-sections")) {
+                        function_sections = true;
                     } else if (mem.eql(u8, arg, "--eh-frame-hdr")) {
                         link_eh_frame_hdr = true;
                     } else if (mem.eql(u8, arg, "-Bsymbolic")) {
@@ -843,6 +860,8 @@ fn buildOutputType(
                         linker_script = arg[2..];
                     } else if (mem.startsWith(u8, arg, "-L")) {
                         try lib_dirs.append(arg[2..]);
+                    } else if (mem.startsWith(u8, arg, "-F")) {
+                        try framework_dirs.append(arg[2..]);
                     } else if (mem.startsWith(u8, arg, "-l")) {
                         // We don't know whether this library is part of libc or libc++ until we resolve the target.
                         // So we simply append to the list for now.
@@ -1555,6 +1574,7 @@ fn buildOutputType(
         .stack_size_override = stack_size_override,
         .strip = strip,
         .single_threaded = single_threaded,
+        .function_sections = function_sections,
         .self_exe_path = self_exe_path,
         .rand = &default_prng.random,
         .clang_passthrough_mode = arg_mode != .build,

From 613f8fe83fc2db4bc39f18ad1a8190d33a4a1181 Mon Sep 17 00:00:00 2001
From: Frank Denis 
Date: Tue, 29 Sep 2020 00:41:37 +0200
Subject: [PATCH 185/210] Use mem.copy() instead of manual iterations

---
 lib/std/crypto/gimli.zig | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/lib/std/crypto/gimli.zig b/lib/std/crypto/gimli.zig
index e5f93f5833..181aa1ed53 100644
--- a/lib/std/crypto/gimli.zig
+++ b/lib/std/crypto/gimli.zig
@@ -249,17 +249,13 @@ pub const Aead = struct {
             in = in[State.RATE..];
             out = out[State.RATE..];
         }) {
-            const d = in[0..State.RATE];
-            for (d) |v, i| {
+            for (in[0..State.RATE]) |v, i| {
                 buf[i] ^= v;
             }
-            for (d) |_, i| {
-                out[i] = buf[i];
-            }
+            mem.copy(u8, out[0..State.RATE], buf[0..State.RATE]);
             state.permute();
         }
-        const d = in[0..];
-        for (d) |v, i| {
+        for (in[0..]) |v, i| {
             buf[i] ^= v;
             out[i] = buf[i];
         }
@@ -299,9 +295,7 @@ pub const Aead = struct {
             for (d) |v, i| {
                 out[i] = buf[i] ^ v;
             }
-            for (d) |v, i| {
-                buf[i] = v;
-            }
+            mem.copy(u8, buf[0..State.RATE], d[0..State.RATE]);
             state.permute();
         }
         for (buf[0..in.len]) |*p, i| {

From 3efdd7b2ad6cce5985d761240636b3d5eb3b4c84 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 01:28:48 -0700
Subject: [PATCH 186/210] don't try to build musl crti.o crtn.o when it's not
 needed

such as on RISC-V
---
 BRANCH_TODO         |  2 +-
 src/Compilation.zig | 11 ++++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/BRANCH_TODO b/BRANCH_TODO
index f2adbb51d8..c6b3c8d8b3 100644
--- a/BRANCH_TODO
+++ b/BRANCH_TODO
@@ -1,3 +1,3 @@
- * building risc-v musl regression
+ * wasi behavior tests failing
  * go ahead and use allocSentinel now that the stage1 bug is fixed
  * audit the base cache hash
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 1cd0d6c099..c3cda196de 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -855,9 +855,14 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             try comp.addBuildingGLibCJobs();
         }
         if (comp.wantBuildMuslFromSource()) {
-            try comp.work_queue.write(&[_]Job{
-                .{ .musl_crt_file = .crti_o },
-                .{ .musl_crt_file = .crtn_o },
+            try comp.work_queue.ensureUnusedCapacity(5);
+            if (target_util.libc_needs_crti_crtn(comp.getTarget())) {
+                comp.work_queue.writeAssumeCapacity(&[_]Job{
+                    .{ .musl_crt_file = .crti_o },
+                    .{ .musl_crt_file = .crtn_o },
+                });
+            }
+            comp.work_queue.writeAssumeCapacity(&[_]Job{
                 .{ .musl_crt_file = .crt1_o },
                 .{ .musl_crt_file = .scrt1_o },
                 .{ .musl_crt_file = .libc_a },

From a45a4230bc2ef67dca4bd3267863695d6b04adfa Mon Sep 17 00:00:00 2001
From: Loris Cro 
Date: Tue, 29 Sep 2020 11:18:35 +0200
Subject: [PATCH 187/210] Fix std.event.Future

Signed-off-by: Loris Cro 
---
 lib/std/event/future.zig |  2 +-
 lib/std/event/lock.zig   | 18 +++++++++++-------
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/lib/std/event/future.zig b/lib/std/event/future.zig
index c9777288e4..40c7845d53 100644
--- a/lib/std/event/future.zig
+++ b/lib/std/event/future.zig
@@ -95,7 +95,7 @@ test "std.event.Future" {
     // TODO provide a way to run tests in evented I/O mode
     if (!std.io.is_async) return error.SkipZigTest;
 
-    const handle = async testFuture();
+    testFuture();
 }
 
 fn testFuture() void {
diff --git a/lib/std/event/lock.zig b/lib/std/event/lock.zig
index a83395d7d0..6819e413d2 100644
--- a/lib/std/event/lock.zig
+++ b/lib/std/event/lock.zig
@@ -27,20 +27,24 @@ pub const Lock = struct {
 
     const Waiter = struct {
         // forced Waiter alignment to ensure it doesn't clash with LOCKED
-        next: ?*Waiter align(2), 
+        next: ?*Waiter align(2),
         tail: *Waiter,
         node: Loop.NextTickNode,
     };
 
+    pub fn initLocked() Lock {
+        return Lock{ .head = LOCKED };
+    }
+
     pub fn acquire(self: *Lock) Held {
         const held = self.mutex.acquire();
 
         // self.head transitions from multiple stages depending on the value:
-        // UNLOCKED -> LOCKED: 
+        // UNLOCKED -> LOCKED:
         //   acquire Lock ownership when theres no waiters
         // LOCKED -> :
         //   Lock is already owned, enqueue first Waiter
-        //  -> : 
+        //  -> :
         //   Lock is owned with pending waiters. Push our waiter to the queue.
 
         if (self.head == UNLOCKED) {
@@ -51,7 +55,7 @@ pub const Lock = struct {
 
         var waiter: Waiter = undefined;
         waiter.next = null;
-        waiter.tail = &waiter; 
+        waiter.tail = &waiter;
 
         const head = switch (self.head) {
             UNLOCKED => unreachable,
@@ -79,15 +83,15 @@ pub const Lock = struct {
     }
 
     pub const Held = struct {
-        lock: *Lock, 
-    
+        lock: *Lock,
+
         pub fn release(self: Held) void {
             const waiter = blk: {
                 const held = self.lock.mutex.acquire();
                 defer held.release();
 
                 // self.head goes through the reverse transition from acquire():
-                //  -> : 
+                //  -> :
                 //   pop a waiter from the queue to give Lock ownership when theres still others pending
                 //  -> LOCKED:
                 //   pop the laster waiter from the queue, while also giving it lock ownership when awaken

From 4194714965a8080e6faa87d9859cc90aab07fe54 Mon Sep 17 00:00:00 2001
From: Frank Denis 
Date: Tue, 29 Sep 2020 13:09:11 +0200
Subject: [PATCH 188/210] Don't unroll the gimli permutation on release-small

---
 lib/std/crypto/gimli.zig | 38 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/lib/std/crypto/gimli.zig b/lib/std/crypto/gimli.zig
index 181aa1ed53..847562e395 100644
--- a/lib/std/crypto/gimli.zig
+++ b/lib/std/crypto/gimli.zig
@@ -38,7 +38,7 @@ pub const State = struct {
         return mem.sliceAsBytes(self.data[0..]);
     }
 
-    pub fn permute(self: *Self) void {
+    fn _permute_unrolled(self: *Self) void {
         const state = &self.data;
         comptime var round = @as(u32, 24);
         inline while (round > 0) : (round -= 1) {
@@ -66,6 +66,42 @@ pub const State = struct {
         }
     }
 
+    fn _permute_small(self: *Self) void {
+        const state = &self.data;
+        var round = @as(u32, 24);
+        while (round > 0) : (round -= 1) {
+            var column = @as(usize, 0);
+            while (column < 4) : (column += 1) {
+                const x = math.rotl(u32, state[column], 24);
+                const y = math.rotl(u32, state[4 + column], 9);
+                const z = state[8 + column];
+                state[8 + column] = ((x ^ (z << 1)) ^ ((y & z) << 2));
+                state[4 + column] = ((y ^ x) ^ ((x | z) << 1));
+                state[column] = ((z ^ y) ^ ((x & y) << 3));
+            }
+            switch (round & 3) {
+                0 => {
+                    mem.swap(u32, &state[0], &state[1]);
+                    mem.swap(u32, &state[2], &state[3]);
+                    state[0] ^= round | 0x9e377900;
+                },
+                2 => {
+                    mem.swap(u32, &state[0], &state[2]);
+                    mem.swap(u32, &state[1], &state[3]);
+                },
+                else => {},
+            }
+        }
+    }
+
+    pub fn permute(self: *Self) void {
+        if (std.builtin.mode == .ReleaseSmall) {
+            self._permute_small();
+        } else {
+            self._permute_unrolled();
+        }
+    }
+
     pub fn squeeze(self: *Self, out: []u8) void {
         var i = @as(usize, 0);
         while (i + RATE <= out.len) : (i += RATE) {

From 56d820087d712c3b3e93e8aeed8d556509050479 Mon Sep 17 00:00:00 2001
From: Frank Denis 
Date: Tue, 29 Sep 2020 14:01:08 +0200
Subject: [PATCH 189/210] gimli: make permute a constant, remove leading
 underscore

---
 lib/std/crypto/gimli.zig | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/lib/std/crypto/gimli.zig b/lib/std/crypto/gimli.zig
index 847562e395..10e8a7dff0 100644
--- a/lib/std/crypto/gimli.zig
+++ b/lib/std/crypto/gimli.zig
@@ -38,7 +38,7 @@ pub const State = struct {
         return mem.sliceAsBytes(self.data[0..]);
     }
 
-    fn _permute_unrolled(self: *Self) void {
+    fn permute_unrolled(self: *Self) void {
         const state = &self.data;
         comptime var round = @as(u32, 24);
         inline while (round > 0) : (round -= 1) {
@@ -66,7 +66,7 @@ pub const State = struct {
         }
     }
 
-    fn _permute_small(self: *Self) void {
+    fn permute_small(self: *Self) void {
         const state = &self.data;
         var round = @as(u32, 24);
         while (round > 0) : (round -= 1) {
@@ -94,13 +94,7 @@ pub const State = struct {
         }
     }
 
-    pub fn permute(self: *Self) void {
-        if (std.builtin.mode == .ReleaseSmall) {
-            self._permute_small();
-        } else {
-            self._permute_unrolled();
-        }
-    }
+    pub const permute = if (std.builtin.mode == .ReleaseSmall) permute_small else permute_unrolled;
 
     pub fn squeeze(self: *Self, out: []u8) void {
         var i = @as(usize, 0);

From 9f274e1f7df65472b0b312cbb6a559ebbaae7e1b Mon Sep 17 00:00:00 2001
From: Frank Denis 
Date: Mon, 28 Sep 2020 00:58:59 +0200
Subject: [PATCH 190/210] std/crypto: add the AEGIS128L AEAD

Showcase that Zig can be a great option for high performance cryptography.

The AEGIS family of authenticated encryption algorithms was selected for
high-performance applications in the final portfolio of the CAESAR
competition.

They reuse the AES core function, but are substantially faster than the
CCM, GCM and OCB modes while offering a high level of security.

AEGIS algorithms are especially fast on CPUs with built-in AES support, and
the 128L variant fully takes advantage of the pipeline in modern Intel CPUs.

Performance of the Zig implementation is on par with libsodium.
---
 lib/std/crypto.zig           |   1 +
 lib/std/crypto/aegis.zig     | 197 +++++++++++++++++++++++++++++++++++
 lib/std/crypto/aes/aesni.zig |  26 +++--
 lib/std/crypto/aes/soft.zig  |  30 +++++-
 lib/std/crypto/benchmark.zig |   1 +
 5 files changed, 242 insertions(+), 13 deletions(-)
 create mode 100644 lib/std/crypto/aegis.zig

diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig
index 5763348729..73ad2ba20b 100644
--- a/lib/std/crypto.zig
+++ b/lib/std/crypto.zig
@@ -28,6 +28,7 @@ pub const aead = struct {
     pub const Gimli = @import("crypto/gimli.zig").Aead;
     pub const ChaCha20Poly1305 = chacha20.Chacha20Poly1305;
     pub const XChaCha20Poly1305 = chacha20.XChacha20Poly1305;
+    pub const AEGIS128L = @import("crypto/aegis.zig").AEGIS128L;
 };
 
 /// MAC functions requiring single-use secret keys.
diff --git a/lib/std/crypto/aegis.zig b/lib/std/crypto/aegis.zig
new file mode 100644
index 0000000000..690f016a10
--- /dev/null
+++ b/lib/std/crypto/aegis.zig
@@ -0,0 +1,197 @@
+const std = @import("std");
+const mem = std.mem;
+const assert = std.debug.assert;
+const AESBlock = std.crypto.core.aes.Block;
+
+const State = struct {
+    blocks: [8]AESBlock,
+
+    fn init(key: [16]u8, nonce: [16]u8) State {
+        const c1 = AESBlock.fromBytes(&[16]u8{ 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1, 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd });
+        const c2 = AESBlock.fromBytes(&[16]u8{ 0x0, 0x1, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d, 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62 });
+        const key_block = AESBlock.fromBytes(&key);
+        const nonce_block = AESBlock.fromBytes(&nonce);
+        const blocks = [8]AESBlock{
+            key_block.xorBlocks(nonce_block),
+            c1,
+            c2,
+            c1,
+            key_block.xorBlocks(nonce_block),
+            key_block.xorBlocks(c2),
+            key_block.xorBlocks(c1),
+            key_block.xorBlocks(c2),
+        };
+        var state = State{ .blocks = blocks };
+        var i: usize = 0;
+        while (i < 10) : (i += 1) {
+            state.update(nonce_block, key_block);
+        }
+        return state;
+    }
+
+    inline fn update(state: *State, d1: AESBlock, d2: AESBlock) void {
+        const blocks = &state.blocks;
+        const tmp = blocks[7];
+        comptime var i: usize = 7;
+        inline while (i > 0) : (i -= 1) {
+            blocks[i] = blocks[i - 1].encrypt(blocks[i]);
+        }
+        blocks[0] = tmp.encrypt(blocks[0]);
+        blocks[0] = blocks[0].xorBlocks(d1);
+        blocks[4] = blocks[4].xorBlocks(d2);
+    }
+
+    fn enc(state: *State, dst: []u8, src: []const u8) void {
+        const blocks = &state.blocks;
+        const msg0 = AESBlock.fromBytes(src[0..16]);
+        const msg1 = AESBlock.fromBytes(src[16..32]);
+        var tmp0 = msg0.xorBlocks(blocks[6]).xorBlocks(blocks[1]);
+        var tmp1 = msg1.xorBlocks(blocks[2]).xorBlocks(blocks[5]);
+        tmp0 = tmp0.xorBlocks(blocks[2].andBlocks(blocks[3]));
+        tmp1 = tmp1.xorBlocks(blocks[6].andBlocks(blocks[7]));
+        dst[0..16].* = tmp0.toBytes();
+        dst[16..32].* = tmp1.toBytes();
+        state.update(msg0, msg1);
+    }
+
+    fn dec(state: *State, dst: []u8, src: []const u8) void {
+        const blocks = &state.blocks;
+        var msg0 = AESBlock.fromBytes(src[0..16]).xorBlocks(blocks[6]).xorBlocks(blocks[1]);
+        var msg1 = AESBlock.fromBytes(src[16..32]).xorBlocks(blocks[2]).xorBlocks(blocks[5]);
+        msg0 = msg0.xorBlocks(blocks[2].andBlocks(blocks[3]));
+        msg1 = msg1.xorBlocks(blocks[6].andBlocks(blocks[7]));
+        dst[0..16].* = msg0.toBytes();
+        dst[16..32].* = msg1.toBytes();
+        state.update(msg0, msg1);
+    }
+
+    fn mac(state: *State, adlen: usize, mlen: usize) [16]u8 {
+        const blocks = &state.blocks;
+        var sizes: [16]u8 = undefined;
+        mem.writeIntLittle(u64, sizes[0..8], adlen * 8);
+        mem.writeIntLittle(u64, sizes[8..16], mlen * 8);
+        const tmp = AESBlock.fromBytes(&sizes).xorBlocks(blocks[2]);
+        var i: usize = 0;
+        while (i < 7) : (i += 1) {
+            state.update(tmp, tmp);
+        }
+        return blocks[0].xorBlocks(blocks[1]).xorBlocks(blocks[2]).xorBlocks(blocks[3]).xorBlocks(blocks[4]).
+            xorBlocks(blocks[5]).xorBlocks(blocks[6]).toBytes();
+    }
+};
+
+/// AEGIS is a very fast authenticated encryption system built on top of the core AES function.
+///
+/// The 128L variant of AEGIS has a 128 bit key, a 128 bit nonce, and processes 256 bit message blocks.
+/// It was designed to fully exploit the parallelism and built-in AES support of recent Intel and ARM CPUs.
+///
+/// https://eprint.iacr.org/2013/695.pdf
+pub const AEGIS128L = struct {
+    pub const tag_length = 16;
+    pub const nonce_length = 16;
+    pub const key_length = 16;
+
+    /// c: ciphertext: output buffer should be of size m.len
+    /// tag: authentication tag: output MAC
+    /// m: message
+    /// ad: Associated Data
+    /// npub: public nonce
+    /// k: private key
+    pub fn encrypt(c: []u8, tag: *[tag_length]u8, m: []const u8, ad: []const u8, npub: [nonce_length]u8, key: [key_length]u8) void {
+        assert(c.len == m.len);
+        var state = State.init(key, npub);
+        var src: [32]u8 align(16) = undefined;
+        var dst: [32]u8 align(16) = undefined;
+        var i: usize = 0;
+        while (i + 32 <= ad.len) : (i += 32) {
+            state.enc(&dst, ad[i..][0..32]);
+        }
+        if (ad.len % 32 != 0) {
+            mem.set(u8, src[0..], 0);
+            mem.copy(u8, src[0 .. ad.len % 32], ad[i .. i + ad.len % 32]);
+            state.enc(&dst, &src);
+        }
+        i = 0;
+        while (i + 32 <= m.len) : (i += 32) {
+            state.enc(c[i..][0..32], m[i..][0..32]);
+        }
+        if (m.len % 32 != 0) {
+            mem.set(u8, src[0..], 0);
+            mem.copy(u8, src[0 .. m.len % 32], m[i .. i + m.len % 32]);
+            state.enc(&dst, &src);
+            mem.copy(u8, c[i .. i + m.len % 32], dst[0 .. m.len % 32]);
+        }
+        tag.* = state.mac(ad.len, m.len);
+    }
+
+    /// m: message: output buffer should be of size c.len
+    /// c: ciphertext
+    /// tag: authentication tag
+    /// ad: Associated Data
+    /// npub: public nonce
+    /// k: private key
+    pub fn decrypt(m: []u8, c: []const u8, tag: [tag_length]u8, ad: []const u8, npub: [nonce_length]u8, key: [key_length]u8) !void {
+        assert(c.len == m.len);
+        var state = State.init(key, npub);
+        var src: [32]u8 align(16) = undefined;
+        var dst: [32]u8 align(16) = undefined;
+        var i: usize = 0;
+        while (i + 32 <= ad.len) : (i += 32) {
+            state.enc(&dst, ad[i..][0..32]);
+        }
+        if (ad.len % 32 != 0) {
+            mem.set(u8, src[0..], 0);
+            mem.copy(u8, src[0 .. ad.len % 32], ad[i .. i + ad.len % 32]);
+            state.enc(&dst, &src);
+        }
+        i = 0;
+        while (i + 32 <= m.len) : (i += 32) {
+            state.dec(m[i..][0..32], c[i..][0..32]);
+        }
+        if (m.len % 32 != 0) {
+            mem.set(u8, src[0..], 0);
+            mem.copy(u8, src[0 .. m.len % 32], c[i .. i + m.len % 32]);
+            state.dec(&dst, &src);
+            mem.copy(u8, m[i .. i + m.len % 32], dst[0 .. m.len % 32]);
+            mem.set(u8, dst[0 .. m.len % 32], 0);
+            const blocks = &state.blocks;
+            blocks[0] = blocks[0].xorBlocks(AESBlock.fromBytes(dst[0..16]));
+            blocks[4] = blocks[4].xorBlocks(AESBlock.fromBytes(dst[16..32]));
+        }
+        const computed_tag = state.mac(ad.len, m.len);
+        var acc: u8 = 0;
+        for (computed_tag) |_, j| {
+            acc |= (computed_tag[j] ^ tag[j]);
+        }
+        if (acc != 0) {
+            mem.set(u8, m, 0xaa);
+            return error.AuthenticationFailed;
+        }
+    }
+};
+
+const htest = @import("test.zig");
+const testing = std.testing;
+
+test "AEGIS128L" {
+    const key: [AEGIS128L.key_length]u8 = [_]u8{ 0x10, 0x01 } ++ [_]u8{0x00} ** 14;
+    const nonce: [AEGIS128L.nonce_length]u8 = [_]u8{ 0x10, 0x00, 0x02 } ++ [_]u8{0x00} ** 13;
+    const ad = [8]u8{ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 };
+    const m = [32]u8{ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f };
+    var c: [m.len]u8 = undefined;
+    var m2: [m.len]u8 = undefined;
+    var tag: [AEGIS128L.tag_length]u8 = undefined;
+
+    AEGIS128L.encrypt(&c, &tag, &m, &ad, nonce, key);
+    try AEGIS128L.decrypt(&m2, &c, tag, &ad, nonce, key);
+    testing.expectEqualSlices(u8, &m, &m2);
+
+    htest.assertEqual("79d94593d8c2119d7e8fd9b8fc77845c5c077a05b2528b6ac54b563aed8efe84", &c);
+    htest.assertEqual("cc6f3372f6aa1bb82388d695c3962d9a", &tag);
+
+    c[0] +%= 1;
+    testing.expectError(error.AuthenticationFailed, AEGIS128L.decrypt(&m2, &c, tag, &ad, nonce, key));
+    c[0] -%= 1;
+    tag[0] +%= 1;
+    testing.expectError(error.AuthenticationFailed, AEGIS128L.decrypt(&m2, &c, tag, &ad, nonce, key));
+}
diff --git a/lib/std/crypto/aes/aesni.zig b/lib/std/crypto/aes/aesni.zig
index f13a4a581c..47dd029bec 100644
--- a/lib/std/crypto/aes/aesni.zig
+++ b/lib/std/crypto/aes/aesni.zig
@@ -84,11 +84,21 @@ pub const Block = struct {
         };
     }
 
-    /// XOR the content of two blocks.
-    pub inline fn xor(block1: Block, block2: Block) Block {
+    /// Apply the bitwise XOR operation to the content of two blocks.
+    pub inline fn xorBlocks(block1: Block, block2: Block) Block {
         return Block{ .repr = block1.repr ^ block2.repr };
     }
 
+    /// Apply the bitwise AND operation to the content of two blocks.
+    pub inline fn andBlocks(block1: Block, block2: Block) Block {
+        return Block{ .repr = block1.repr & block2.repr };
+    }
+
+    /// Apply the bitwise OR operation to the content of two blocks.
+    pub inline fn orBlocks(block1: Block, block2: Block) Block {
+        return Block{ .repr = block1.repr | block2.repr };
+    }
+
     /// Perform operations on multiple blocks in parallel.
     pub const parallel = struct {
         /// The recommended number of AES encryption/decryption to perform in parallel for the chosen implementation.
@@ -261,7 +271,7 @@ pub fn AESEncryptCtx(comptime AES: type) type {
         /// Encrypt a single block.
         pub fn encrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
             const round_keys = ctx.key_schedule.round_keys;
-            var t = Block.fromBytes(src).xor(round_keys[0]);
+            var t = Block.fromBytes(src).xorBlocks(round_keys[0]);
             comptime var i = 1;
             inline while (i < rounds) : (i += 1) {
                 t = t.encrypt(round_keys[i]);
@@ -273,7 +283,7 @@ pub fn AESEncryptCtx(comptime AES: type) type {
         /// Encrypt+XOR a single block.
         pub fn xor(ctx: Self, dst: *[16]u8, src: *const [16]u8, counter: [16]u8) void {
             const round_keys = ctx.key_schedule.round_keys;
-            var t = Block.fromBytes(&counter).xor(round_keys[0]);
+            var t = Block.fromBytes(&counter).xorBlocks(round_keys[0]);
             comptime var i = 1;
             inline while (i < rounds) : (i += 1) {
                 t = t.encrypt(round_keys[i]);
@@ -288,7 +298,7 @@ pub fn AESEncryptCtx(comptime AES: type) type {
             var ts: [count]Block = undefined;
             comptime var j = 0;
             inline while (j < count) : (j += 1) {
-                ts[j] = Block.fromBytes(src[j * 16 .. j * 16 + 16][0..16]).xor(round_keys[0]);
+                ts[j] = Block.fromBytes(src[j * 16 .. j * 16 + 16][0..16]).xorBlocks(round_keys[0]);
             }
             comptime var i = 1;
             inline while (i < rounds) : (i += 1) {
@@ -310,7 +320,7 @@ pub fn AESEncryptCtx(comptime AES: type) type {
             var ts: [count]Block = undefined;
             comptime var j = 0;
             inline while (j < count) : (j += 1) {
-                ts[j] = Block.fromBytes(counters[j * 16 .. j * 16 + 16][0..16]).xor(round_keys[0]);
+                ts[j] = Block.fromBytes(counters[j * 16 .. j * 16 + 16][0..16]).xorBlocks(round_keys[0]);
             }
             comptime var i = 1;
             inline while (i < rounds) : (i += 1) {
@@ -352,7 +362,7 @@ pub fn AESDecryptCtx(comptime AES: type) type {
         /// Decrypt a single block.
         pub fn decrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
             const inv_round_keys = ctx.key_schedule.round_keys;
-            var t = Block.fromBytes(src).xor(inv_round_keys[0]);
+            var t = Block.fromBytes(src).xorBlocks(inv_round_keys[0]);
             comptime var i = 1;
             inline while (i < rounds) : (i += 1) {
                 t = t.decrypt(inv_round_keys[i]);
@@ -367,7 +377,7 @@ pub fn AESDecryptCtx(comptime AES: type) type {
             var ts: [count]Block = undefined;
             comptime var j = 0;
             inline while (j < count) : (j += 1) {
-                ts[j] = Block.fromBytes(src[j * 16 .. j * 16 + 16][0..16]).xor(inv_round_keys[0]);
+                ts[j] = Block.fromBytes(src[j * 16 .. j * 16 + 16][0..16]).xorBlocks(inv_round_keys[0]);
             }
             comptime var i = 1;
             inline while (i < rounds) : (i += 1) {
diff --git a/lib/std/crypto/aes/soft.zig b/lib/std/crypto/aes/soft.zig
index c32662fbc5..5f66f3499e 100644
--- a/lib/std/crypto/aes/soft.zig
+++ b/lib/std/crypto/aes/soft.zig
@@ -125,8 +125,8 @@ pub const Block = struct {
         return Block{ .repr = BlockVec{ s0, s1, s2, s3 } };
     }
 
-    /// XOR the content of two blocks.
-    pub inline fn xor(block1: Block, block2: Block) Block {
+    /// Apply the bitwise XOR operation to the content of two blocks.
+    pub inline fn xorBlocks(block1: Block, block2: Block) Block {
         var x: BlockVec = undefined;
         comptime var i = 0;
         inline while (i < 4) : (i += 1) {
@@ -135,6 +135,26 @@ pub const Block = struct {
         return Block{ .repr = x };
     }
 
+    /// Apply the bitwise AND operation to the content of two blocks.
+    pub inline fn andBlocks(block1: Block, block2: Block) Block {
+        var x: BlockVec = undefined;
+        comptime var i = 0;
+        inline while (i < 4) : (i += 1) {
+            x[i] = block1.repr[i] & block2.repr[i];
+        }
+        return Block{ .repr = x };
+    }
+
+    /// Apply the bitwise OR operation to the content of two blocks.
+    pub inline fn orBlocks(block1: Block, block2: Block) Block {
+        var x: BlockVec = undefined;
+        comptime var i = 0;
+        inline while (i < 4) : (i += 1) {
+            x[i] = block1.repr[i] | block2.repr[i];
+        }
+        return Block{ .repr = x };
+    }
+
     /// Perform operations on multiple blocks in parallel.
     pub const parallel = struct {
         /// The recommended number of AES encryption/decryption to perform in parallel for the chosen implementation.
@@ -283,7 +303,7 @@ pub fn AESEncryptCtx(comptime AES: type) type {
         /// Encrypt a single block.
         pub fn encrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
             const round_keys = ctx.key_schedule.round_keys;
-            var t = Block.fromBytes(src).xor(round_keys[0]);
+            var t = Block.fromBytes(src).xorBlocks(round_keys[0]);
             comptime var i = 1;
             inline while (i < rounds) : (i += 1) {
                 t = t.encrypt(round_keys[i]);
@@ -295,7 +315,7 @@ pub fn AESEncryptCtx(comptime AES: type) type {
         /// Encrypt+XOR a single block.
         pub fn xor(ctx: Self, dst: *[16]u8, src: *const [16]u8, counter: [16]u8) void {
             const round_keys = ctx.key_schedule.round_keys;
-            var t = Block.fromBytes(&counter).xor(round_keys[0]);
+            var t = Block.fromBytes(&counter).xorBlocks(round_keys[0]);
             comptime var i = 1;
             inline while (i < rounds) : (i += 1) {
                 t = t.encrypt(round_keys[i]);
@@ -349,7 +369,7 @@ pub fn AESDecryptCtx(comptime AES: type) type {
         /// Decrypt a single block.
         pub fn decrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
             const inv_round_keys = ctx.key_schedule.round_keys;
-            var t = Block.fromBytes(src).xor(inv_round_keys[0]);
+            var t = Block.fromBytes(src).xorBlocks(inv_round_keys[0]);
             comptime var i = 1;
             inline while (i < rounds) : (i += 1) {
                 t = t.decrypt(inv_round_keys[i]);
diff --git a/lib/std/crypto/benchmark.zig b/lib/std/crypto/benchmark.zig
index 860f1269f0..d5b93947c5 100644
--- a/lib/std/crypto/benchmark.zig
+++ b/lib/std/crypto/benchmark.zig
@@ -149,6 +149,7 @@ const aeads = [_]Crypto{
     Crypto{ .ty = crypto.aead.ChaCha20Poly1305, .name = "chacha20Poly1305" },
     Crypto{ .ty = crypto.aead.XChaCha20Poly1305, .name = "xchacha20Poly1305" },
     Crypto{ .ty = crypto.aead.Gimli, .name = "gimli-aead" },
+    Crypto{ .ty = crypto.aead.AEGIS128L, .name = "aegis128l" },
 };
 
 pub fn benchmarkAead(comptime Aead: anytype, comptime bytes: comptime_int) !u64 {

From bb1c6bc376f1a30d6dadd4aebcd1ebec6b4c8621 Mon Sep 17 00:00:00 2001
From: Frank Denis 
Date: Mon, 28 Sep 2020 14:50:00 +0200
Subject: [PATCH 191/210] Add AEGIS-256 as well

---
 lib/std/crypto.zig           |   1 +
 lib/std/crypto/aegis.zig     | 238 +++++++++++++++++++++++++++++++++--
 lib/std/crypto/benchmark.zig |   3 +-
 3 files changed, 231 insertions(+), 11 deletions(-)

diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig
index 73ad2ba20b..fa69d51d4d 100644
--- a/lib/std/crypto.zig
+++ b/lib/std/crypto.zig
@@ -29,6 +29,7 @@ pub const aead = struct {
     pub const ChaCha20Poly1305 = chacha20.Chacha20Poly1305;
     pub const XChaCha20Poly1305 = chacha20.XChacha20Poly1305;
     pub const AEGIS128L = @import("crypto/aegis.zig").AEGIS128L;
+    pub const AEGIS256 = @import("crypto/aegis.zig").AEGIS256;
 };
 
 /// MAC functions requiring single-use secret keys.
diff --git a/lib/std/crypto/aegis.zig b/lib/std/crypto/aegis.zig
index 690f016a10..7ea7136f1e 100644
--- a/lib/std/crypto/aegis.zig
+++ b/lib/std/crypto/aegis.zig
@@ -3,10 +3,10 @@ const mem = std.mem;
 const assert = std.debug.assert;
 const AESBlock = std.crypto.core.aes.Block;
 
-const State = struct {
+const State128L = struct {
     blocks: [8]AESBlock,
 
-    fn init(key: [16]u8, nonce: [16]u8) State {
+    fn init(key: [16]u8, nonce: [16]u8) State128L {
         const c1 = AESBlock.fromBytes(&[16]u8{ 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1, 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd });
         const c2 = AESBlock.fromBytes(&[16]u8{ 0x0, 0x1, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d, 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62 });
         const key_block = AESBlock.fromBytes(&key);
@@ -21,7 +21,7 @@ const State = struct {
             key_block.xorBlocks(c1),
             key_block.xorBlocks(c2),
         };
-        var state = State{ .blocks = blocks };
+        var state = State128L{ .blocks = blocks };
         var i: usize = 0;
         while (i < 10) : (i += 1) {
             state.update(nonce_block, key_block);
@@ -29,7 +29,7 @@ const State = struct {
         return state;
     }
 
-    inline fn update(state: *State, d1: AESBlock, d2: AESBlock) void {
+    inline fn update(state: *State128L, d1: AESBlock, d2: AESBlock) void {
         const blocks = &state.blocks;
         const tmp = blocks[7];
         comptime var i: usize = 7;
@@ -41,7 +41,7 @@ const State = struct {
         blocks[4] = blocks[4].xorBlocks(d2);
     }
 
-    fn enc(state: *State, dst: []u8, src: []const u8) void {
+    fn enc(state: *State128L, dst: *[32]u8, src: *const [32]u8) void {
         const blocks = &state.blocks;
         const msg0 = AESBlock.fromBytes(src[0..16]);
         const msg1 = AESBlock.fromBytes(src[16..32]);
@@ -54,7 +54,7 @@ const State = struct {
         state.update(msg0, msg1);
     }
 
-    fn dec(state: *State, dst: []u8, src: []const u8) void {
+    fn dec(state: *State128L, dst: *[32]u8, src: *const [32]u8) void {
         const blocks = &state.blocks;
         var msg0 = AESBlock.fromBytes(src[0..16]).xorBlocks(blocks[6]).xorBlocks(blocks[1]);
         var msg1 = AESBlock.fromBytes(src[16..32]).xorBlocks(blocks[2]).xorBlocks(blocks[5]);
@@ -65,7 +65,7 @@ const State = struct {
         state.update(msg0, msg1);
     }
 
-    fn mac(state: *State, adlen: usize, mlen: usize) [16]u8 {
+    fn mac(state: *State128L, adlen: usize, mlen: usize) [16]u8 {
         const blocks = &state.blocks;
         var sizes: [16]u8 = undefined;
         mem.writeIntLittle(u64, sizes[0..8], adlen * 8);
@@ -99,7 +99,7 @@ pub const AEGIS128L = struct {
     /// k: private key
     pub fn encrypt(c: []u8, tag: *[tag_length]u8, m: []const u8, ad: []const u8, npub: [nonce_length]u8, key: [key_length]u8) void {
         assert(c.len == m.len);
-        var state = State.init(key, npub);
+        var state = State128L.init(key, npub);
         var src: [32]u8 align(16) = undefined;
         var dst: [32]u8 align(16) = undefined;
         var i: usize = 0;
@@ -132,7 +132,7 @@ pub const AEGIS128L = struct {
     /// k: private key
     pub fn decrypt(m: []u8, c: []const u8, tag: [tag_length]u8, ad: []const u8, npub: [nonce_length]u8, key: [key_length]u8) !void {
         assert(c.len == m.len);
-        var state = State.init(key, npub);
+        var state = State128L.init(key, npub);
         var src: [32]u8 align(16) = undefined;
         var dst: [32]u8 align(16) = undefined;
         var i: usize = 0;
@@ -170,10 +170,171 @@ pub const AEGIS128L = struct {
     }
 };
 
+const State256 = struct {
+    blocks: [6]AESBlock,
+
+    fn init(key: [32]u8, nonce: [32]u8) State256 {
+        const c1 = AESBlock.fromBytes(&[16]u8{ 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1, 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd });
+        const c2 = AESBlock.fromBytes(&[16]u8{ 0x0, 0x1, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d, 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62 });
+        const key_block1 = AESBlock.fromBytes(key[0..16]);
+        const key_block2 = AESBlock.fromBytes(key[16..32]);
+        const nonce_block1 = AESBlock.fromBytes(nonce[0..16]);
+        const nonce_block2 = AESBlock.fromBytes(nonce[16..32]);
+        const kxn1 = key_block1.xorBlocks(nonce_block1);
+        const kxn2 = key_block2.xorBlocks(nonce_block2);
+        const blocks = [6]AESBlock{
+            kxn1,
+            kxn2,
+            c1,
+            c2,
+            key_block1.xorBlocks(c2),
+            key_block2.xorBlocks(c1),
+        };
+        var state = State256{ .blocks = blocks };
+        var i: usize = 0;
+        while (i < 4) : (i += 1) {
+            state.update(key_block1);
+            state.update(key_block2);
+            state.update(kxn1);
+            state.update(kxn2);
+        }
+        return state;
+    }
+
+    inline fn update(state: *State256, d: AESBlock) void {
+        const blocks = &state.blocks;
+        const tmp = blocks[5].encrypt(blocks[0]);
+        comptime var i: usize = 5;
+        inline while (i > 0) : (i -= 1) {
+            blocks[i] = blocks[i - 1].encrypt(blocks[i]);
+        }
+        blocks[0] = tmp.xorBlocks(d);
+    }
+
+    fn enc(state: *State256, dst: *[16]u8, src: *const [16]u8) void {
+        const blocks = &state.blocks;
+        const msg = AESBlock.fromBytes(src);
+        var tmp = msg.xorBlocks(blocks[5]).xorBlocks(blocks[4]).xorBlocks(blocks[1]);
+        tmp = tmp.xorBlocks(blocks[2].andBlocks(blocks[3]));
+        dst.* = tmp.toBytes();
+        state.update(msg);
+    }
+
+    fn dec(state: *State256, dst: *[16]u8, src: *const [16]u8) void {
+        const blocks = &state.blocks;
+        var msg = AESBlock.fromBytes(src).xorBlocks(blocks[5]).xorBlocks(blocks[4]).xorBlocks(blocks[1]);
+        msg = msg.xorBlocks(blocks[2].andBlocks(blocks[3]));
+        dst.* = msg.toBytes();
+        state.update(msg);
+    }
+
+    fn mac(state: *State256, adlen: usize, mlen: usize) [16]u8 {
+        const blocks = &state.blocks;
+        var sizes: [16]u8 = undefined;
+        mem.writeIntLittle(u64, sizes[0..8], adlen * 8);
+        mem.writeIntLittle(u64, sizes[8..16], mlen * 8);
+        const tmp = AESBlock.fromBytes(&sizes).xorBlocks(blocks[3]);
+        var i: usize = 0;
+        while (i < 7) : (i += 1) {
+            state.update(tmp);
+        }
+        return blocks[0].xorBlocks(blocks[1]).xorBlocks(blocks[2]).xorBlocks(blocks[3]).xorBlocks(blocks[4]).
+            xorBlocks(blocks[5]).toBytes();
+    }
+};
+
+/// AEGIS is a very fast authenticated encryption system built on top of the core AES function.
+///
+/// The 256 bit variant of AEGIS has a 256 bit key, a 256 bit nonce, and processes 128 bit message blocks.
+///
+/// https://eprint.iacr.org/2013/695.pdf
+pub const AEGIS256 = struct {
+    pub const tag_length = 16;
+    pub const nonce_length = 32;
+    pub const key_length = 32;
+
+    /// c: ciphertext: output buffer should be of size m.len
+    /// tag: authentication tag: output MAC
+    /// m: message
+    /// ad: Associated Data
+    /// npub: public nonce
+    /// k: private key
+    pub fn encrypt(c: []u8, tag: *[tag_length]u8, m: []const u8, ad: []const u8, npub: [nonce_length]u8, key: [key_length]u8) void {
+        assert(c.len == m.len);
+        var state = State256.init(key, npub);
+        var src: [16]u8 align(16) = undefined;
+        var dst: [16]u8 align(16) = undefined;
+        var i: usize = 0;
+        while (i + 16 <= ad.len) : (i += 16) {
+            state.enc(&dst, ad[i..][0..16]);
+        }
+        if (ad.len % 16 != 0) {
+            mem.set(u8, src[0..], 0);
+            mem.copy(u8, src[0 .. ad.len % 16], ad[i .. i + ad.len % 16]);
+            state.enc(&dst, &src);
+        }
+        i = 0;
+        while (i + 16 <= m.len) : (i += 16) {
+            state.enc(c[i..][0..16], m[i..][0..16]);
+        }
+        if (m.len % 16 != 0) {
+            mem.set(u8, src[0..], 0);
+            mem.copy(u8, src[0 .. m.len % 16], m[i .. i + m.len % 16]);
+            state.enc(&dst, &src);
+            mem.copy(u8, c[i .. i + m.len % 16], dst[0 .. m.len % 16]);
+        }
+        tag.* = state.mac(ad.len, m.len);
+    }
+
+    /// m: message: output buffer should be of size c.len
+    /// c: ciphertext
+    /// tag: authentication tag
+    /// ad: Associated Data
+    /// npub: public nonce
+    /// k: private key
+    pub fn decrypt(m: []u8, c: []const u8, tag: [tag_length]u8, ad: []const u8, npub: [nonce_length]u8, key: [key_length]u8) !void {
+        assert(c.len == m.len);
+        var state = State256.init(key, npub);
+        var src: [16]u8 align(16) = undefined;
+        var dst: [16]u8 align(16) = undefined;
+        var i: usize = 0;
+        while (i + 16 <= ad.len) : (i += 16) {
+            state.enc(&dst, ad[i..][0..16]);
+        }
+        if (ad.len % 16 != 0) {
+            mem.set(u8, src[0..], 0);
+            mem.copy(u8, src[0 .. ad.len % 16], ad[i .. i + ad.len % 16]);
+            state.enc(&dst, &src);
+        }
+        i = 0;
+        while (i + 16 <= m.len) : (i += 16) {
+            state.dec(m[i..][0..16], c[i..][0..16]);
+        }
+        if (m.len % 16 != 0) {
+            mem.set(u8, src[0..], 0);
+            mem.copy(u8, src[0 .. m.len % 16], c[i .. i + m.len % 16]);
+            state.dec(&dst, &src);
+            mem.copy(u8, m[i .. i + m.len % 16], dst[0 .. m.len % 16]);
+            mem.set(u8, dst[0 .. m.len % 16], 0);
+            const blocks = &state.blocks;
+            blocks[0] = blocks[0].xorBlocks(AESBlock.fromBytes(&dst));
+        }
+        const computed_tag = state.mac(ad.len, m.len);
+        var acc: u8 = 0;
+        for (computed_tag) |_, j| {
+            acc |= (computed_tag[j] ^ tag[j]);
+        }
+        if (acc != 0) {
+            mem.set(u8, m, 0xaa);
+            return error.AuthenticationFailed;
+        }
+    }
+};
+
 const htest = @import("test.zig");
 const testing = std.testing;
 
-test "AEGIS128L" {
+test "AEGIS128L test vector 1" {
     const key: [AEGIS128L.key_length]u8 = [_]u8{ 0x10, 0x01 } ++ [_]u8{0x00} ** 14;
     const nonce: [AEGIS128L.nonce_length]u8 = [_]u8{ 0x10, 0x00, 0x02 } ++ [_]u8{0x00} ** 13;
     const ad = [8]u8{ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 };
@@ -195,3 +356,60 @@ test "AEGIS128L" {
     tag[0] +%= 1;
     testing.expectError(error.AuthenticationFailed, AEGIS128L.decrypt(&m2, &c, tag, &ad, nonce, key));
 }
+
+test "AEGIS128L test vector 2" {
+    const key: [AEGIS128L.key_length]u8 = [_]u8{0x00} ** 16;
+    const nonce: [AEGIS128L.nonce_length]u8 = [_]u8{0x00} ** 16;
+    const ad = [_]u8{};
+    const m = [_]u8{0x00} ** 16;
+    var c: [m.len]u8 = undefined;
+    var m2: [m.len]u8 = undefined;
+    var tag: [AEGIS128L.tag_length]u8 = undefined;
+
+    AEGIS128L.encrypt(&c, &tag, &m, &ad, nonce, key);
+    try AEGIS128L.decrypt(&m2, &c, tag, &ad, nonce, key);
+    testing.expectEqualSlices(u8, &m, &m2);
+
+    htest.assertEqual("41de9000a7b5e40e2d68bb64d99ebb19", &c);
+    htest.assertEqual("f4d997cc9b94227ada4fe4165422b1c8", &tag);
+}
+
+test "AEGIS256 test vector 1" {
+    const key: [AEGIS256.key_length]u8 = [_]u8{ 0x10, 0x01 } ++ [_]u8{0x00} ** 30;
+    const nonce: [AEGIS256.nonce_length]u8 = [_]u8{ 0x10, 0x00, 0x02 } ++ [_]u8{0x00} ** 29;
+    const ad = [8]u8{ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 };
+    const m = [32]u8{ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f };
+    var c: [m.len]u8 = undefined;
+    var m2: [m.len]u8 = undefined;
+    var tag: [AEGIS256.tag_length]u8 = undefined;
+
+    AEGIS256.encrypt(&c, &tag, &m, &ad, nonce, key);
+    try AEGIS256.decrypt(&m2, &c, tag, &ad, nonce, key);
+    testing.expectEqualSlices(u8, &m, &m2);
+
+    htest.assertEqual("f373079ed84b2709faee373584585d60accd191db310ef5d8b11833df9dec711", &c);
+    htest.assertEqual("8d86f91ee606e9ff26a01b64ccbdd91d", &tag);
+
+    c[0] +%= 1;
+    testing.expectError(error.AuthenticationFailed, AEGIS256.decrypt(&m2, &c, tag, &ad, nonce, key));
+    c[0] -%= 1;
+    tag[0] +%= 1;
+    testing.expectError(error.AuthenticationFailed, AEGIS256.decrypt(&m2, &c, tag, &ad, nonce, key));
+}
+
+test "AEGIS256 test vector 2" {
+    const key: [AEGIS256.key_length]u8 = [_]u8{0x00} ** 32;
+    const nonce: [AEGIS256.nonce_length]u8 = [_]u8{0x00} ** 32;
+    const ad = [_]u8{};
+    const m = [_]u8{0x00} ** 16;
+    var c: [m.len]u8 = undefined;
+    var m2: [m.len]u8 = undefined;
+    var tag: [AEGIS256.tag_length]u8 = undefined;
+
+    AEGIS256.encrypt(&c, &tag, &m, &ad, nonce, key);
+    try AEGIS256.decrypt(&m2, &c, tag, &ad, nonce, key);
+    testing.expectEqualSlices(u8, &m, &m2);
+
+    htest.assertEqual("b98f03a947807713d75a4fff9fc277a6", &c);
+    htest.assertEqual("478f3b50dc478ef7d5cf2d0f7cc13180", &tag);
+}
diff --git a/lib/std/crypto/benchmark.zig b/lib/std/crypto/benchmark.zig
index d5b93947c5..6578961902 100644
--- a/lib/std/crypto/benchmark.zig
+++ b/lib/std/crypto/benchmark.zig
@@ -149,7 +149,8 @@ const aeads = [_]Crypto{
     Crypto{ .ty = crypto.aead.ChaCha20Poly1305, .name = "chacha20Poly1305" },
     Crypto{ .ty = crypto.aead.XChaCha20Poly1305, .name = "xchacha20Poly1305" },
     Crypto{ .ty = crypto.aead.Gimli, .name = "gimli-aead" },
-    Crypto{ .ty = crypto.aead.AEGIS128L, .name = "aegis128l" },
+    Crypto{ .ty = crypto.aead.AEGIS128L, .name = "aegis-128l" },
+    Crypto{ .ty = crypto.aead.AEGIS256, .name = "aegis-256" },
 };
 
 pub fn benchmarkAead(comptime Aead: anytype, comptime bytes: comptime_int) !u64 {

From 8d67f15d36bdd1a094596876796c77606a5e4a83 Mon Sep 17 00:00:00 2001
From: Frank Denis 
Date: Mon, 28 Sep 2020 15:02:11 +0200
Subject: [PATCH 192/210] aegis: add test vectors, and link to the latest
 version of the spec

---
 lib/std/crypto/aegis.zig | 36 ++++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/lib/std/crypto/aegis.zig b/lib/std/crypto/aegis.zig
index 7ea7136f1e..cb9e4cabe9 100644
--- a/lib/std/crypto/aegis.zig
+++ b/lib/std/crypto/aegis.zig
@@ -85,7 +85,7 @@ const State128L = struct {
 /// The 128L variant of AEGIS has a 128 bit key, a 128 bit nonce, and processes 256 bit message blocks.
 /// It was designed to fully exploit the parallelism and built-in AES support of recent Intel and ARM CPUs.
 ///
-/// https://eprint.iacr.org/2013/695.pdf
+/// https://competitions.cr.yp.to/round3/aegisv11.pdf
 pub const AEGIS128L = struct {
     pub const tag_length = 16;
     pub const nonce_length = 16;
@@ -247,7 +247,7 @@ const State256 = struct {
 ///
 /// The 256 bit variant of AEGIS has a 256 bit key, a 256 bit nonce, and processes 128 bit message blocks.
 ///
-/// https://eprint.iacr.org/2013/695.pdf
+/// https://competitions.cr.yp.to/round3/aegisv11.pdf
 pub const AEGIS256 = struct {
     pub const tag_length = 16;
     pub const nonce_length = 32;
@@ -374,6 +374,22 @@ test "AEGIS128L test vector 2" {
     htest.assertEqual("f4d997cc9b94227ada4fe4165422b1c8", &tag);
 }
 
+test "AEGIS128L test vector 3" {
+    const key: [AEGIS128L.key_length]u8 = [_]u8{0x00} ** 16;
+    const nonce: [AEGIS128L.nonce_length]u8 = [_]u8{0x00} ** 16;
+    const ad = [_]u8{};
+    const m = [_]u8{};
+    var c: [m.len]u8 = undefined;
+    var m2: [m.len]u8 = undefined;
+    var tag: [AEGIS128L.tag_length]u8 = undefined;
+
+    AEGIS128L.encrypt(&c, &tag, &m, &ad, nonce, key);
+    try AEGIS128L.decrypt(&m2, &c, tag, &ad, nonce, key);
+    testing.expectEqualSlices(u8, &m, &m2);
+
+    htest.assertEqual("83cc600dc4e3e7e62d4055826174f149", &tag);
+}
+
 test "AEGIS256 test vector 1" {
     const key: [AEGIS256.key_length]u8 = [_]u8{ 0x10, 0x01 } ++ [_]u8{0x00} ** 30;
     const nonce: [AEGIS256.nonce_length]u8 = [_]u8{ 0x10, 0x00, 0x02 } ++ [_]u8{0x00} ** 29;
@@ -413,3 +429,19 @@ test "AEGIS256 test vector 2" {
     htest.assertEqual("b98f03a947807713d75a4fff9fc277a6", &c);
     htest.assertEqual("478f3b50dc478ef7d5cf2d0f7cc13180", &tag);
 }
+
+test "AEGIS256 test vector 3" {
+    const key: [AEGIS256.key_length]u8 = [_]u8{0x00} ** 32;
+    const nonce: [AEGIS256.nonce_length]u8 = [_]u8{0x00} ** 32;
+    const ad = [_]u8{};
+    const m = [_]u8{};
+    var c: [m.len]u8 = undefined;
+    var m2: [m.len]u8 = undefined;
+    var tag: [AEGIS256.tag_length]u8 = undefined;
+
+    AEGIS256.encrypt(&c, &tag, &m, &ad, nonce, key);
+    try AEGIS256.decrypt(&m2, &c, tag, &ad, nonce, key);
+    testing.expectEqualSlices(u8, &m, &m2);
+
+    htest.assertEqual("f7a0878f68bd083e8065354071fc27c3", &tag);
+}

From d8fa8b5455058d235793a8b9ecdf252fb8dd8782 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 12:06:35 -0700
Subject: [PATCH 193/210] use Allocator.allocSentinel now that the stage1 bug
 is fixed

Thanks @LemonBoy!
---
 BRANCH_TODO               | 3 ---
 lib/std/mem/Allocator.zig | 2 --
 src/link/Coff.zig         | 4 +---
 src/link/Elf.zig          | 5 +----
 src/link/MachO.zig        | 5 +----
 src/link/Wasm.zig         | 5 +----
 6 files changed, 4 insertions(+), 20 deletions(-)
 delete mode 100644 BRANCH_TODO

diff --git a/BRANCH_TODO b/BRANCH_TODO
deleted file mode 100644
index c6b3c8d8b3..0000000000
--- a/BRANCH_TODO
+++ /dev/null
@@ -1,3 +0,0 @@
- * wasi behavior tests failing
- * go ahead and use allocSentinel now that the stage1 bug is fixed
- * audit the base cache hash
diff --git a/lib/std/mem/Allocator.zig b/lib/std/mem/Allocator.zig
index 326a73b915..4511acb275 100644
--- a/lib/std/mem/Allocator.zig
+++ b/lib/std/mem/Allocator.zig
@@ -231,8 +231,6 @@ fn AllocWithOptionsPayload(comptime Elem: type, comptime alignment: ?u29, compti
 /// call `free` when done.
 ///
 /// For allocating a single item, see `create`.
-///
-/// Deprecated; use `allocWithOptions`.
 pub fn allocSentinel(
     self: *Allocator,
     comptime Elem: type,
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 0356a327ba..424ece511b 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -1118,9 +1118,7 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
         Compilation.dump_argv(argv.items);
     }
 
-    const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1);
-    new_argv_with_sentinel[argv.items.len] = null;
-    const new_argv = new_argv_with_sentinel[0..argv.items.len :null];
+    const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null);
     for (argv.items) |arg, i| {
         new_argv[i] = try arena.dupeZ(u8, arg);
     }
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index dc664b14be..38b9b1acca 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -1589,10 +1589,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
     }
 
     // Oh, snapplesauce! We need null terminated argv.
-    // TODO allocSentinel crashed stage1 so this is working around it.
-    const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1);
-    new_argv_with_sentinel[argv.items.len] = null;
-    const new_argv = new_argv_with_sentinel[0..argv.items.len: null];
+    const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null);
     for (argv.items) |arg, i| {
         new_argv[i] = try arena.dupeZ(u8, arg);
     }
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 146f2616c2..2570a92daa 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -567,10 +567,7 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
         Compilation.dump_argv(argv.items);
     }
 
-    // TODO allocSentinel crashed stage1 so this is working around it.
-    const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1);
-    new_argv_with_sentinel[argv.items.len] = null;
-    const new_argv = new_argv_with_sentinel[0..argv.items.len :null];
+    const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null);
     for (argv.items) |arg, i| {
         new_argv[i] = try arena.dupeZ(u8, arg);
     }
diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig
index 509544c94f..3f879a3b32 100644
--- a/src/link/Wasm.zig
+++ b/src/link/Wasm.zig
@@ -385,10 +385,7 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
         Compilation.dump_argv(argv.items);
     }
 
-    // TODO allocSentinel crashed stage1 so this is working around it.
-    const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1);
-    new_argv_with_sentinel[argv.items.len] = null;
-    const new_argv = new_argv_with_sentinel[0..argv.items.len :null];
+    const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null);
     for (argv.items) |arg, i| {
         new_argv[i] = try arena.dupeZ(u8, arg);
     }

From 7c0ee423859739b5484de8de867006ead787fb66 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 12:10:38 -0700
Subject: [PATCH 194/210] CI: TERM=dumb to source hut

---
 ci/srht/freebsd_script | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ci/srht/freebsd_script b/ci/srht/freebsd_script
index 31aea6c3dc..3d9eb73735 100755
--- a/ci/srht/freebsd_script
+++ b/ci/srht/freebsd_script
@@ -21,6 +21,11 @@ cd $ZIGDIR
 # This will affect the cmake command below.
 git config core.abbrev 9
 
+# SourceHut reports that it is a terminal that supports escape codes, but it
+# is a filthy liar. Here we tell Zig to not try to send any terminal escape
+# codes to show progress.
+export TERM=dumb
+
 mkdir build
 cd build
 cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=$PREFIX "-DCMAKE_INSTALL_PREFIX=$(pwd)/release" -DZIG_STATIC=ON

From 0da7c4b0c8a2a2fe0862f7757bc5976342d51dc8 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 14:46:40 -0700
Subject: [PATCH 195/210] improve stage2 COFF LLD linking

 * change some {} to be {s} to gain type safety
 * fix libraries being libfoo.lib instead of foo.lib for COFF
 * when linking mingw-w64, add the "always link" libs so that we
   generate DLL import .lib files for them as the linker code relies on.
 * COFF LLD linker does not support -r so we do a file copy as an
   alternative to the -r thing that ELF linking does.
   I will file an issue for the corresponding TODO upon merging this
   branch, to look into an optimization that possibly elides this copy
   when the source and destination are both cache directories.
 * add a CLI error message when trying to link multiple objects into one
   and using COFF object format.
---
 lib/std/zig.zig     |  26 +--
 src/Compilation.zig |   5 +
 src/link.zig        |   2 +-
 src/link/Coff.zig   | 501 +++++++++++++++++++++++---------------------
 src/main.zig        |  13 +-
 5 files changed, 288 insertions(+), 259 deletions(-)

diff --git a/lib/std/zig.zig b/lib/std/zig.zig
index 59855f62d4..66226e8dd1 100644
--- a/lib/std/zig.zig
+++ b/lib/std/zig.zig
@@ -84,36 +84,36 @@ pub fn binNameAlloc(allocator: *std.mem.Allocator, options: BinNameOptions) erro
                     .uefi => ".efi",
                     else => ".exe",
                 };
-                return std.fmt.allocPrint(allocator, "{}{}", .{ root_name, suffix });
+                return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, suffix });
             },
             .Lib => {
                 const suffix = switch (options.link_mode orelse .Static) {
                     .Static => ".lib",
                     .Dynamic => ".dll",
                 };
-                return std.fmt.allocPrint(allocator, "{}{}{}", .{ target.libPrefix(), root_name, suffix });
+                return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, suffix });
             },
-            .Obj => return std.fmt.allocPrint(allocator, "{}{}", .{ root_name, target.abi.oFileExt() }),
+            .Obj => return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, target.abi.oFileExt() }),
         },
         .elf => switch (options.output_mode) {
             .Exe => return allocator.dupe(u8, root_name),
             .Lib => {
                 switch (options.link_mode orelse .Static) {
-                    .Static => return std.fmt.allocPrint(allocator, "{}{}.a", .{
+                    .Static => return std.fmt.allocPrint(allocator, "{s}{s}.a", .{
                         target.libPrefix(), root_name,
                     }),
                     .Dynamic => {
                         if (options.version) |ver| {
-                            return std.fmt.allocPrint(allocator, "{}{}.so.{}.{}.{}", .{
+                            return std.fmt.allocPrint(allocator, "{s}{s}.so.{d}.{d}.{d}", .{
                                 target.libPrefix(), root_name, ver.major, ver.minor, ver.patch,
                             });
                         } else {
-                            return std.fmt.allocPrint(allocator, "{}{}.so", .{ target.libPrefix(), root_name });
+                            return std.fmt.allocPrint(allocator, "{s}{s}.so", .{ target.libPrefix(), root_name });
                         }
                     },
                 }
             },
-            .Obj => return std.fmt.allocPrint(allocator, "{}.o", .{root_name}),
+            .Obj => return std.fmt.allocPrint(allocator, "{s}.o", .{root_name}),
         },
         .macho => switch (options.output_mode) {
             .Exe => return allocator.dupe(u8, root_name),
@@ -122,14 +122,14 @@ pub fn binNameAlloc(allocator: *std.mem.Allocator, options: BinNameOptions) erro
                     .Static => ".a",
                     .Dynamic => ".dylib",
                 };
-                return std.fmt.allocPrint(allocator, "{}{}{}", .{ target.libPrefix(), root_name, suffix });
+                return std.fmt.allocPrint(allocator, "{s}{s}{s}", .{ target.libPrefix(), root_name, suffix });
             },
-            .Obj => return std.fmt.allocPrint(allocator, "{}.o", .{root_name}),
+            .Obj => return std.fmt.allocPrint(allocator, "{s}.o", .{root_name}),
         },
-        .wasm => return std.fmt.allocPrint(allocator, "{}.wasm", .{root_name}),
-        .c => return std.fmt.allocPrint(allocator, "{}.c", .{root_name}),
-        .hex => return std.fmt.allocPrint(allocator, "{}.ihex", .{root_name}),
-        .raw => return std.fmt.allocPrint(allocator, "{}.bin", .{root_name}),
+        .wasm => return std.fmt.allocPrint(allocator, "{s}.wasm", .{root_name}),
+        .c => return std.fmt.allocPrint(allocator, "{s}.c", .{root_name}),
+        .hex => return std.fmt.allocPrint(allocator, "{s}.ihex", .{root_name}),
+        .raw => return std.fmt.allocPrint(allocator, "{s}.bin", .{root_name}),
     }
 }
 
diff --git a/src/Compilation.zig b/src/Compilation.zig
index c3cda196de..9cc2c3aa5c 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -879,6 +879,11 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             try comp.work_queue.ensureUnusedCapacity(static_lib_jobs.len + 1);
             comp.work_queue.writeAssumeCapacity(&static_lib_jobs);
             comp.work_queue.writeItemAssumeCapacity(crt_job);
+
+            // When linking mingw-w64 there are some import libs we always need.
+            for (mingw.always_link_libs) |name| {
+                try comp.bin_file.options.system_libs.put(comp.gpa, name, .{});
+            }
         }
         // Generate Windows import libs.
         if (comp.getTarget().os.tag == .windows) {
diff --git a/src/link.zig b/src/link.zig
index 9112bd8d1d..51bf8a64ae 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -321,7 +321,7 @@ pub const File = struct {
             // TODO: avoid extra link step when it's just 1 object file (the `zig cc -c` case)
             // Until then, we do `lld -r -o output.o input.o` even though the output is the same
             // as the input. For the preprocessing case (`zig cc -E -o foo`) we copy the file
-            // to the final location.
+            // to the final location. See also the corresponding TODO in Coff linking.
             const full_out_path = try emit.directory.join(comp.gpa, &[_][]const u8{emit.sub_path});
             defer comp.gpa.free(full_out_path);
             assert(comp.c_object_table.count() == 1);
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 424ece511b..2290560509 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -873,287 +873,302 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
         };
     }
 
-    const is_obj = self.base.options.output_mode == .Obj;
-
-    // Create an LLD command line and invoke it.
-    var argv = std.ArrayList([]const u8).init(self.base.allocator);
-    defer argv.deinit();
-    // Even though we're calling LLD as a library it thinks the first argument is its own exe name.
-    try argv.append("lld");
-    if (is_obj) {
-        try argv.append("-r");
-    }
-
-    try argv.append("-ERRORLIMIT:0");
-    try argv.append("-NOLOGO");
-    if (!self.base.options.strip) {
-        try argv.append("-DEBUG");
-    }
-    if (self.base.options.output_mode == .Exe) {
-        const stack_size = self.base.options.stack_size_override orelse 16777216;
-        try argv.append(try allocPrint(arena, "-STACK:{d}", .{stack_size}));
-    }
-
-    if (target.cpu.arch == .i386) {
-        try argv.append("-MACHINE:X86");
-    } else if (target.cpu.arch == .x86_64) {
-        try argv.append("-MACHINE:X64");
-    } else if (target.cpu.arch.isARM()) {
-        if (target.cpu.arch.ptrBitWidth() == 32) {
-            try argv.append("-MACHINE:ARM");
-        } else {
-            try argv.append("-MACHINE:ARM64");
-        }
-    }
-
-    if (is_dyn_lib) {
-        try argv.append("-DLL");
-    }
-
     const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
-    try argv.append(try allocPrint(arena, "-OUT:{s}", .{full_out_path}));
 
-    if (self.base.options.link_libc) {
-        if (self.base.options.libc_installation) |libc_installation| {
-            try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.crt_dir.?}));
+    if (self.base.options.output_mode == .Obj) {
+        // LLD's COFF driver does not support the equvialent of `-r` so we do a simple file copy
+        // here. TODO: think carefully about how we can avoid this redundant operation when doing
+        // build-obj. See also the corresponding TODO in linkAsArchive.
+        const the_object_path = blk: {
+            if (self.base.options.objects.len != 0)
+                break :blk self.base.options.objects[0];
 
-            if (target.abi == .msvc) {
-                try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.msvc_lib_dir.?}));
-                try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.kernel32_lib_dir.?}));
+            if (comp.c_object_table.count() != 0)
+                break :blk comp.c_object_table.items()[0].key.status.success.object_path;
+
+            if (module_obj_path) |p|
+                break :blk p;
+
+            // TODO I think this is unreachable. Audit this situation when solving the above TODO
+            // regarding eliding redundant object -> object transformations.
+            return error.NoObjectsToLink;
+        };
+        try fs.cwd().copyFile(the_object_path, fs.cwd(), full_out_path, .{});
+    } else {
+        // Create an LLD command line and invoke it.
+        var argv = std.ArrayList([]const u8).init(self.base.allocator);
+        defer argv.deinit();
+        // Even though we're calling LLD as a library it thinks the first argument is its own exe name.
+        try argv.append("lld");
+
+        try argv.append("-ERRORLIMIT:0");
+        try argv.append("-NOLOGO");
+        if (!self.base.options.strip) {
+            try argv.append("-DEBUG");
+        }
+        if (self.base.options.output_mode == .Exe) {
+            const stack_size = self.base.options.stack_size_override orelse 16777216;
+            try argv.append(try allocPrint(arena, "-STACK:{d}", .{stack_size}));
+        }
+
+        if (target.cpu.arch == .i386) {
+            try argv.append("-MACHINE:X86");
+        } else if (target.cpu.arch == .x86_64) {
+            try argv.append("-MACHINE:X64");
+        } else if (target.cpu.arch.isARM()) {
+            if (target.cpu.arch.ptrBitWidth() == 32) {
+                try argv.append("-MACHINE:ARM");
+            } else {
+                try argv.append("-MACHINE:ARM64");
             }
         }
-    }
 
-    for (self.base.options.lib_dirs) |lib_dir| {
-        try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{lib_dir}));
-    }
+        if (is_dyn_lib) {
+            try argv.append("-DLL");
+        }
 
-    try argv.appendSlice(self.base.options.objects);
+        try argv.append(try allocPrint(arena, "-OUT:{s}", .{full_out_path}));
 
-    for (comp.c_object_table.items()) |entry| {
-        try argv.append(entry.key.status.success.object_path);
-    }
+        if (self.base.options.link_libc) {
+            if (self.base.options.libc_installation) |libc_installation| {
+                try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.crt_dir.?}));
 
-    if (module_obj_path) |p| {
-        try argv.append(p);
-    }
-
-    const resolved_subsystem: ?std.Target.SubSystem = blk: {
-        if (self.base.options.subsystem) |explicit| break :blk explicit;
-        switch (target.os.tag) {
-            .windows => {
-                if (self.base.options.module) |module| {
-                    if (module.stage1_flags.have_dllmain_crt_startup or is_dyn_lib)
-                        break :blk null;
-                    if (module.stage1_flags.have_c_main or self.base.options.is_test or
-                        module.stage1_flags.have_winmain_crt_startup or
-                        module.stage1_flags.have_wwinmain_crt_startup)
-                    {
-                        break :blk .Console;
-                    }
-                    if (module.stage1_flags.have_winmain or module.stage1_flags.have_wwinmain)
-                        break :blk .Windows;
+                if (target.abi == .msvc) {
+                    try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.msvc_lib_dir.?}));
+                    try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.kernel32_lib_dir.?}));
                 }
-            },
-            .uefi => break :blk .EfiApplication,
-            else => {},
+            }
         }
-        break :blk null;
-    };
-    const Mode = enum { uefi, win32 };
-    const mode: Mode = mode: {
-        if (resolved_subsystem) |subsystem| switch (subsystem) {
-            .Console => {
-                try argv.append("-SUBSYSTEM:console");
-                break :mode .win32;
-            },
-            .EfiApplication => {
-                try argv.append("-SUBSYSTEM:efi_application");
-                break :mode .uefi;
-            },
-            .EfiBootServiceDriver => {
-                try argv.append("-SUBSYSTEM:efi_boot_service_driver");
-                break :mode .uefi;
-            },
-            .EfiRom => {
-                try argv.append("-SUBSYSTEM:efi_rom");
-                break :mode .uefi;
-            },
-            .EfiRuntimeDriver => {
-                try argv.append("-SUBSYSTEM:efi_runtime_driver");
-                break :mode .uefi;
-            },
-            .Native => {
-                try argv.append("-SUBSYSTEM:native");
-                break :mode .win32;
-            },
-            .Posix => {
-                try argv.append("-SUBSYSTEM:posix");
-                break :mode .win32;
-            },
-            .Windows => {
-                try argv.append("-SUBSYSTEM:windows");
-                break :mode .win32;
-            },
-        } else if (target.os.tag == .uefi) {
-            break :mode .uefi;
-        } else {
-            break :mode .win32;
+
+        for (self.base.options.lib_dirs) |lib_dir| {
+            try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{lib_dir}));
         }
-    };
 
-    switch (mode) {
-        .uefi => try argv.appendSlice(&[_][]const u8{
-            "-BASE:0",
-            "-ENTRY:EfiMain",
-            "-OPT:REF",
-            "-SAFESEH:NO",
-            "-MERGE:.rdata=.data",
-            "-ALIGN:32",
-            "-NODEFAULTLIB",
-            "-SECTION:.xdata,D",
-        }),
-        .win32 => {
-            if (link_in_crt) {
-                if (target.abi.isGnu()) {
-                    try argv.append("-lldmingw");
+        try argv.appendSlice(self.base.options.objects);
 
-                    if (target.cpu.arch == .i386) {
-                        try argv.append("-ALTERNATENAME:__image_base__=___ImageBase");
-                    } else {
-                        try argv.append("-ALTERNATENAME:__image_base__=__ImageBase");
-                    }
+        for (comp.c_object_table.items()) |entry| {
+            try argv.append(entry.key.status.success.object_path);
+        }
 
-                    if (is_dyn_lib) {
-                        try argv.append(try comp.get_libc_crt_file(arena, "dllcrt2.o"));
-                    } else {
-                        try argv.append(try comp.get_libc_crt_file(arena, "crt2.o"));
-                    }
+        if (module_obj_path) |p| {
+            try argv.append(p);
+        }
 
-                    try argv.append(try comp.get_libc_crt_file(arena, "mingw32.lib"));
-                    try argv.append(try comp.get_libc_crt_file(arena, "mingwex.lib"));
-                    try argv.append(try comp.get_libc_crt_file(arena, "msvcrt-os.lib"));
-
-                    for (mingw.always_link_libs) |name| {
-                        if (!self.base.options.system_libs.contains(name)) {
-                            const lib_basename = try allocPrint(arena, "{s}.lib", .{name});
-                            try argv.append(try comp.get_libc_crt_file(arena, lib_basename));
+        const resolved_subsystem: ?std.Target.SubSystem = blk: {
+            if (self.base.options.subsystem) |explicit| break :blk explicit;
+            switch (target.os.tag) {
+                .windows => {
+                    if (self.base.options.module) |module| {
+                        if (module.stage1_flags.have_dllmain_crt_startup or is_dyn_lib)
+                            break :blk null;
+                        if (module.stage1_flags.have_c_main or self.base.options.is_test or
+                            module.stage1_flags.have_winmain_crt_startup or
+                            module.stage1_flags.have_wwinmain_crt_startup)
+                        {
+                            break :blk .Console;
                         }
+                        if (module.stage1_flags.have_winmain or module.stage1_flags.have_wwinmain)
+                            break :blk .Windows;
+                    }
+                },
+                .uefi => break :blk .EfiApplication,
+                else => {},
+            }
+            break :blk null;
+        };
+        const Mode = enum { uefi, win32 };
+        const mode: Mode = mode: {
+            if (resolved_subsystem) |subsystem| switch (subsystem) {
+                .Console => {
+                    try argv.append("-SUBSYSTEM:console");
+                    break :mode .win32;
+                },
+                .EfiApplication => {
+                    try argv.append("-SUBSYSTEM:efi_application");
+                    break :mode .uefi;
+                },
+                .EfiBootServiceDriver => {
+                    try argv.append("-SUBSYSTEM:efi_boot_service_driver");
+                    break :mode .uefi;
+                },
+                .EfiRom => {
+                    try argv.append("-SUBSYSTEM:efi_rom");
+                    break :mode .uefi;
+                },
+                .EfiRuntimeDriver => {
+                    try argv.append("-SUBSYSTEM:efi_runtime_driver");
+                    break :mode .uefi;
+                },
+                .Native => {
+                    try argv.append("-SUBSYSTEM:native");
+                    break :mode .win32;
+                },
+                .Posix => {
+                    try argv.append("-SUBSYSTEM:posix");
+                    break :mode .win32;
+                },
+                .Windows => {
+                    try argv.append("-SUBSYSTEM:windows");
+                    break :mode .win32;
+                },
+            } else if (target.os.tag == .uefi) {
+                break :mode .uefi;
+            } else {
+                break :mode .win32;
+            }
+        };
+
+        switch (mode) {
+            .uefi => try argv.appendSlice(&[_][]const u8{
+                "-BASE:0",
+                "-ENTRY:EfiMain",
+                "-OPT:REF",
+                "-SAFESEH:NO",
+                "-MERGE:.rdata=.data",
+                "-ALIGN:32",
+                "-NODEFAULTLIB",
+                "-SECTION:.xdata,D",
+            }),
+            .win32 => {
+                if (link_in_crt) {
+                    if (target.abi.isGnu()) {
+                        try argv.append("-lldmingw");
+
+                        if (target.cpu.arch == .i386) {
+                            try argv.append("-ALTERNATENAME:__image_base__=___ImageBase");
+                        } else {
+                            try argv.append("-ALTERNATENAME:__image_base__=__ImageBase");
+                        }
+
+                        if (is_dyn_lib) {
+                            try argv.append(try comp.get_libc_crt_file(arena, "dllcrt2.o"));
+                        } else {
+                            try argv.append(try comp.get_libc_crt_file(arena, "crt2.o"));
+                        }
+
+                        try argv.append(try comp.get_libc_crt_file(arena, "mingw32.lib"));
+                        try argv.append(try comp.get_libc_crt_file(arena, "mingwex.lib"));
+                        try argv.append(try comp.get_libc_crt_file(arena, "msvcrt-os.lib"));
+
+                        for (mingw.always_link_libs) |name| {
+                            if (!self.base.options.system_libs.contains(name)) {
+                                const lib_basename = try allocPrint(arena, "{s}.lib", .{name});
+                                try argv.append(try comp.get_libc_crt_file(arena, lib_basename));
+                            }
+                        }
+                    } else {
+                        const lib_str = switch (self.base.options.link_mode) {
+                            .Dynamic => "",
+                            .Static => "lib",
+                        };
+                        const d_str = switch (self.base.options.optimize_mode) {
+                            .Debug => "d",
+                            else => "",
+                        };
+                        switch (self.base.options.link_mode) {
+                            .Static => try argv.append(try allocPrint(arena, "libcmt{s}.lib", .{d_str})),
+                            .Dynamic => try argv.append(try allocPrint(arena, "msvcrt{s}.lib", .{d_str})),
+                        }
+
+                        try argv.append(try allocPrint(arena, "{s}vcruntime{s}.lib", .{ lib_str, d_str }));
+                        try argv.append(try allocPrint(arena, "{s}ucrt{s}.lib", .{ lib_str, d_str }));
+
+                        //Visual C++ 2015 Conformance Changes
+                        //https://msdn.microsoft.com/en-us/library/bb531344.aspx
+                        try argv.append("legacy_stdio_definitions.lib");
+
+                        // msvcrt depends on kernel32 and ntdll
+                        try argv.append("kernel32.lib");
+                        try argv.append("ntdll.lib");
                     }
                 } else {
-                    const lib_str = switch (self.base.options.link_mode) {
-                        .Dynamic => "",
-                        .Static => "lib",
-                    };
-                    const d_str = switch (self.base.options.optimize_mode) {
-                        .Debug => "d",
-                        else => "",
-                    };
-                    switch (self.base.options.link_mode) {
-                        .Static => try argv.append(try allocPrint(arena, "libcmt{s}.lib", .{d_str})),
-                        .Dynamic => try argv.append(try allocPrint(arena, "msvcrt{s}.lib", .{d_str})),
-                    }
-
-                    try argv.append(try allocPrint(arena, "{s}vcruntime{s}.lib", .{ lib_str, d_str }));
-                    try argv.append(try allocPrint(arena, "{s}ucrt{s}.lib", .{ lib_str, d_str }));
-
-                    //Visual C++ 2015 Conformance Changes
-                    //https://msdn.microsoft.com/en-us/library/bb531344.aspx
-                    try argv.append("legacy_stdio_definitions.lib");
-
-                    // msvcrt depends on kernel32 and ntdll
-                    try argv.append("kernel32.lib");
-                    try argv.append("ntdll.lib");
-                }
-            } else {
-                try argv.append("-NODEFAULTLIB");
-                if (!is_lib) {
-                    if (self.base.options.module) |module| {
-                        if (module.stage1_flags.have_winmain) {
-                            try argv.append("-ENTRY:WinMain");
-                        } else if (module.stage1_flags.have_wwinmain) {
-                            try argv.append("-ENTRY:wWinMain");
-                        } else if (module.stage1_flags.have_wwinmain_crt_startup) {
-                            try argv.append("-ENTRY:wWinMainCRTStartup");
+                    try argv.append("-NODEFAULTLIB");
+                    if (!is_lib) {
+                        if (self.base.options.module) |module| {
+                            if (module.stage1_flags.have_winmain) {
+                                try argv.append("-ENTRY:WinMain");
+                            } else if (module.stage1_flags.have_wwinmain) {
+                                try argv.append("-ENTRY:wWinMain");
+                            } else if (module.stage1_flags.have_wwinmain_crt_startup) {
+                                try argv.append("-ENTRY:wWinMainCRTStartup");
+                            } else {
+                                try argv.append("-ENTRY:WinMainCRTStartup");
+                            }
                         } else {
                             try argv.append("-ENTRY:WinMainCRTStartup");
                         }
-                    } else {
-                        try argv.append("-ENTRY:WinMainCRTStartup");
                     }
                 }
-            }
-        },
-    }
+            },
+        }
 
-    if (!is_obj) {
         // libc++ dep
         if (self.base.options.link_libcpp) {
             try argv.append(comp.libcxxabi_static_lib.?.full_object_path);
             try argv.append(comp.libcxx_static_lib.?.full_object_path);
             try argv.append(comp.libunwind_static_lib.?.full_object_path);
         }
-    }
 
-    // compiler-rt and libc
-    if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) {
-        if (!self.base.options.link_libc) {
-            try argv.append(comp.libc_static_lib.?.full_object_path);
+        // compiler-rt and libc
+        if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) {
+            if (!self.base.options.link_libc) {
+                try argv.append(comp.libc_static_lib.?.full_object_path);
+            }
+            // MSVC compiler_rt is missing some stuff, so we build it unconditionally but
+            // and rely on weak linkage to allow MSVC compiler_rt functions to override ours.
+            try argv.append(comp.compiler_rt_static_lib.?.full_object_path);
         }
-        // MSVC compiler_rt is missing some stuff, so we build it unconditionally but
-        // and rely on weak linkage to allow MSVC compiler_rt functions to override ours.
-        try argv.append(comp.compiler_rt_static_lib.?.full_object_path);
-    }
 
-    for (self.base.options.system_libs.items()) |entry| {
-        const lib_basename = try allocPrint(arena, "{s}.lib", .{entry.key});
-        if (comp.crt_files.get(lib_basename)) |crt_file| {
-            try argv.append(crt_file.full_object_path);
-        } else {
-            try argv.append(lib_basename);
+        for (self.base.options.system_libs.items()) |entry| {
+            const lib_basename = try allocPrint(arena, "{s}.lib", .{entry.key});
+            if (comp.crt_files.get(lib_basename)) |crt_file| {
+                try argv.append(crt_file.full_object_path);
+            } else {
+                try argv.append(lib_basename);
+            }
         }
-    }
 
-    if (self.base.options.verbose_link) {
-        Compilation.dump_argv(argv.items);
-    }
+        if (self.base.options.verbose_link) {
+            Compilation.dump_argv(argv.items);
+        }
 
-    const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null);
-    for (argv.items) |arg, i| {
-        new_argv[i] = try arena.dupeZ(u8, arg);
-    }
+        const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null);
+        for (argv.items) |arg, i| {
+            new_argv[i] = try arena.dupeZ(u8, arg);
+        }
 
-    var stderr_context: LLDContext = .{
-        .coff = self,
-        .data = std.ArrayList(u8).init(self.base.allocator),
-    };
-    defer stderr_context.data.deinit();
-    var stdout_context: LLDContext = .{
-        .coff = self,
-        .data = std.ArrayList(u8).init(self.base.allocator),
-    };
-    defer stdout_context.data.deinit();
-    const llvm = @import("../llvm.zig");
-    const ok = llvm.Link(
-        .COFF,
-        new_argv.ptr,
-        new_argv.len,
-        append_diagnostic,
-        @ptrToInt(&stdout_context),
-        @ptrToInt(&stderr_context),
-    );
-    if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory;
-    if (stdout_context.data.items.len != 0) {
-        std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items});
-    }
-    if (!ok) {
-        // TODO parse this output and surface with the Compilation API rather than
-        // directly outputting to stderr here.
-        std.debug.print("{}", .{stderr_context.data.items});
-        return error.LLDReportedFailure;
-    }
-    if (stderr_context.data.items.len != 0) {
-        std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items});
+        var stderr_context: LLDContext = .{
+            .coff = self,
+            .data = std.ArrayList(u8).init(self.base.allocator),
+        };
+        defer stderr_context.data.deinit();
+        var stdout_context: LLDContext = .{
+            .coff = self,
+            .data = std.ArrayList(u8).init(self.base.allocator),
+        };
+        defer stdout_context.data.deinit();
+        const llvm = @import("../llvm.zig");
+        const ok = llvm.Link(
+            .COFF,
+            new_argv.ptr,
+            new_argv.len,
+            append_diagnostic,
+            @ptrToInt(&stdout_context),
+            @ptrToInt(&stderr_context),
+        );
+        if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory;
+        if (stdout_context.data.items.len != 0) {
+            std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items});
+        }
+        if (!ok) {
+            // TODO parse this output and surface with the Compilation API rather than
+            // directly outputting to stderr here.
+            std.debug.print("{}", .{stderr_context.data.items});
+            return error.LLDReportedFailure;
+        }
+        if (stderr_context.data.items.len != 0) {
+            std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items});
+        }
     }
 
     if (!self.base.options.disable_lld_caching) {
diff --git a/src/main.zig b/src/main.zig
index de2f38bffb..822e777d68 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -1314,8 +1314,8 @@ fn buildOutputType(
         }
     }
 
-    const object_format: ?std.Target.ObjectFormat = blk: {
-        const ofmt = target_ofmt orelse break :blk null;
+    const object_format: std.Target.ObjectFormat = blk: {
+        const ofmt = target_ofmt orelse break :blk target_info.target.getObjectFormat();
         if (mem.eql(u8, ofmt, "elf")) {
             break :blk .elf;
         } else if (mem.eql(u8, ofmt, "c")) {
@@ -1337,6 +1337,15 @@ fn buildOutputType(
         }
     };
 
+    if (output_mode == .Obj and object_format == .coff) {
+        const total_obj_count = c_source_files.items.len +
+            @boolToInt(root_src_file != null) +
+            link_objects.items.len;
+        if (total_obj_count > 1) {
+            fatal("COFF does not support linking multiple objects into one", .{});
+        }
+    }
+
     var cleanup_emit_bin_dir: ?fs.Dir = null;
     defer if (cleanup_emit_bin_dir) |*dir| dir.close();
 

From 63685190da9415ff7d27080e0f33cbeab6a80156 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 16:01:20 -0700
Subject: [PATCH 196/210] move std.rb to the standard library orphanage

https://github.com/ziglang/std-lib-orphanage/

This code is not used by anything else in the standard library or by the
compiler or any of its tools and therefore it's a great candidate
to be maintained by a third party.
---
 lib/std/rb.zig  | 633 ------------------------------------------------
 lib/std/std.zig |   1 -
 2 files changed, 634 deletions(-)
 delete mode 100644 lib/std/rb.zig

diff --git a/lib/std/rb.zig b/lib/std/rb.zig
deleted file mode 100644
index 8cf90a1eea..0000000000
--- a/lib/std/rb.zig
+++ /dev/null
@@ -1,633 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2015-2020 Zig Contributors
-// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
-// The MIT license requires this copyright notice to be included in all copies
-// and substantial portions of the software.
-const std = @import("std");
-const assert = std.debug.assert;
-const testing = std.testing;
-const Order = std.math.Order;
-
-const Color = enum(u1) {
-    Black,
-    Red,
-};
-const Red = Color.Red;
-const Black = Color.Black;
-
-const ReplaceError = error{NotEqual};
-const SortError = error{NotUnique}; // The new comparison function results in duplicates.
-
-/// Insert this into your struct that you want to add to a red-black tree.
-/// Do not use a pointer. Turn the *rb.Node results of the functions in rb
-/// (after resolving optionals) to your structure using @fieldParentPtr(). Example:
-///
-/// const Number = struct {
-///     node: rb.Node,
-///     value: i32,
-/// };
-/// fn number(node: *rb.Node) Number {
-///     return @fieldParentPtr(Number, "node", node);
-/// }
-pub const Node = struct {
-    left: ?*Node,
-    right: ?*Node,
-
-    /// parent | color
-    parent_and_color: usize,
-
-    pub fn next(constnode: *Node) ?*Node {
-        var node = constnode;
-
-        if (node.right) |right| {
-            var n = right;
-            while (n.left) |left|
-                n = left;
-            return n;
-        }
-
-        while (true) {
-            var parent = node.getParent();
-            if (parent) |p| {
-                if (node != p.right)
-                    return p;
-                node = p;
-            } else
-                return null;
-        }
-    }
-
-    pub fn prev(constnode: *Node) ?*Node {
-        var node = constnode;
-
-        if (node.left) |left| {
-            var n = left;
-            while (n.right) |right|
-                n = right;
-            return n;
-        }
-
-        while (true) {
-            var parent = node.getParent();
-            if (parent) |p| {
-                if (node != p.left)
-                    return p;
-                node = p;
-            } else
-                return null;
-        }
-    }
-
-    pub fn isRoot(node: *Node) bool {
-        return node.getParent() == null;
-    }
-
-    fn isRed(node: *Node) bool {
-        return node.getColor() == Red;
-    }
-
-    fn isBlack(node: *Node) bool {
-        return node.getColor() == Black;
-    }
-
-    fn setParent(node: *Node, parent: ?*Node) void {
-        node.parent_and_color = @ptrToInt(parent) | (node.parent_and_color & 1);
-    }
-
-    fn getParent(node: *Node) ?*Node {
-        const mask: usize = 1;
-        comptime {
-            assert(@alignOf(*Node) >= 2);
-        }
-        const maybe_ptr = node.parent_and_color & ~mask;
-        return if (maybe_ptr == 0) null else @intToPtr(*Node, maybe_ptr);
-    }
-
-    fn setColor(node: *Node, color: Color) void {
-        const mask: usize = 1;
-        node.parent_and_color = (node.parent_and_color & ~mask) | @enumToInt(color);
-    }
-
-    fn getColor(node: *Node) Color {
-        return @intToEnum(Color, @intCast(u1, node.parent_and_color & 1));
-    }
-
-    fn setChild(node: *Node, child: ?*Node, is_left: bool) void {
-        if (is_left) {
-            node.left = child;
-        } else {
-            node.right = child;
-        }
-    }
-
-    fn getFirst(nodeconst: *Node) *Node {
-        var node = nodeconst;
-        while (node.left) |left| {
-            node = left;
-        }
-        return node;
-    }
-
-    fn getLast(nodeconst: *Node) *Node {
-        var node = nodeconst;
-        while (node.right) |right| {
-            node = right;
-        }
-        return node;
-    }
-};
-
-pub const Tree = struct {
-    root: ?*Node,
-    compareFn: fn (*Node, *Node, *Tree) Order,
-
-    /// Re-sorts a tree with a new compare function
-    pub fn sort(tree: *Tree, newCompareFn: fn (*Node, *Node, *Tree) Order) SortError!void {
-        var newTree = Tree.init(newCompareFn);
-        var node: *Node = undefined;
-        while (true) {
-            node = tree.first() orelse break;
-            tree.remove(node);
-            if (newTree.insert(node) != null) {
-                return error.NotUnique; // EEXISTS
-            }
-        }
-        tree.* = newTree;
-    }
-
-    /// If you have a need for a version that caches this, please file a bug.
-    pub fn first(tree: *Tree) ?*Node {
-        var node: *Node = tree.root orelse return null;
-
-        while (node.left) |left| {
-            node = left;
-        }
-
-        return node;
-    }
-
-    pub fn last(tree: *Tree) ?*Node {
-        var node: *Node = tree.root orelse return null;
-
-        while (node.right) |right| {
-            node = right;
-        }
-
-        return node;
-    }
-
-    /// Duplicate keys are not allowed. The item with the same key already in the
-    /// tree will be returned, and the item will not be inserted.
-    pub fn insert(tree: *Tree, node_const: *Node) ?*Node {
-        var node = node_const;
-        var maybe_key: ?*Node = undefined;
-        var maybe_parent: ?*Node = undefined;
-        var is_left: bool = undefined;
-
-        maybe_key = doLookup(node, tree, &maybe_parent, &is_left);
-        if (maybe_key) |key| {
-            return key;
-        }
-
-        node.left = null;
-        node.right = null;
-        node.setColor(Red);
-        node.setParent(maybe_parent);
-
-        if (maybe_parent) |parent| {
-            parent.setChild(node, is_left);
-        } else {
-            tree.root = node;
-        }
-
-        while (node.getParent()) |*parent| {
-            if (parent.*.isBlack())
-                break;
-            // the root is always black
-            var grandpa = parent.*.getParent() orelse unreachable;
-
-            if (parent.* == grandpa.left) {
-                var maybe_uncle = grandpa.right;
-
-                if (maybe_uncle) |uncle| {
-                    if (uncle.isBlack())
-                        break;
-
-                    parent.*.setColor(Black);
-                    uncle.setColor(Black);
-                    grandpa.setColor(Red);
-                    node = grandpa;
-                } else {
-                    if (node == parent.*.right) {
-                        rotateLeft(parent.*, tree);
-                        node = parent.*;
-                        parent.* = node.getParent().?; // Just rotated
-                    }
-                    parent.*.setColor(Black);
-                    grandpa.setColor(Red);
-                    rotateRight(grandpa, tree);
-                }
-            } else {
-                var maybe_uncle = grandpa.left;
-
-                if (maybe_uncle) |uncle| {
-                    if (uncle.isBlack())
-                        break;
-
-                    parent.*.setColor(Black);
-                    uncle.setColor(Black);
-                    grandpa.setColor(Red);
-                    node = grandpa;
-                } else {
-                    if (node == parent.*.left) {
-                        rotateRight(parent.*, tree);
-                        node = parent.*;
-                        parent.* = node.getParent().?; // Just rotated
-                    }
-                    parent.*.setColor(Black);
-                    grandpa.setColor(Red);
-                    rotateLeft(grandpa, tree);
-                }
-            }
-        }
-        // This was an insert, there is at least one node.
-        tree.root.?.setColor(Black);
-        return null;
-    }
-
-    /// lookup searches for the value of key, using binary search. It will
-    /// return a pointer to the node if it is there, otherwise it will return null.
-    /// Complexity guaranteed O(log n), where n is the number of nodes book-kept
-    /// by tree.
-    pub fn lookup(tree: *Tree, key: *Node) ?*Node {
-        var parent: ?*Node = undefined;
-        var is_left: bool = undefined;
-        return doLookup(key, tree, &parent, &is_left);
-    }
-
-    /// If node is not part of tree, behavior is undefined.
-    pub fn remove(tree: *Tree, nodeconst: *Node) void {
-        var node = nodeconst;
-        // as this has the same value as node, it is unsafe to access node after newnode
-        var newnode: ?*Node = nodeconst;
-        var maybe_parent: ?*Node = node.getParent();
-        var color: Color = undefined;
-        var next: *Node = undefined;
-
-        // This clause is to avoid optionals
-        if (node.left == null and node.right == null) {
-            if (maybe_parent) |parent| {
-                parent.setChild(null, parent.left == node);
-            } else
-                tree.root = null;
-            color = node.getColor();
-            newnode = null;
-        } else {
-            if (node.left == null) {
-                next = node.right.?; // Not both null as per above
-            } else if (node.right == null) {
-                next = node.left.?; // Not both null as per above
-            } else
-                next = node.right.?.getFirst(); // Just checked for null above
-
-            if (maybe_parent) |parent| {
-                parent.setChild(next, parent.left == node);
-            } else
-                tree.root = next;
-
-            if (node.left != null and node.right != null) {
-                const left = node.left.?;
-                const right = node.right.?;
-
-                color = next.getColor();
-                next.setColor(node.getColor());
-
-                next.left = left;
-                left.setParent(next);
-
-                if (next != right) {
-                    var parent = next.getParent().?; // Was traversed via child node (right/left)
-                    next.setParent(node.getParent());
-
-                    newnode = next.right;
-                    parent.left = node;
-
-                    next.right = right;
-                    right.setParent(next);
-                } else {
-                    next.setParent(maybe_parent);
-                    maybe_parent = next;
-                    newnode = next.right;
-                }
-            } else {
-                color = node.getColor();
-                newnode = next;
-            }
-        }
-
-        if (newnode) |n|
-            n.setParent(maybe_parent);
-
-        if (color == Red)
-            return;
-        if (newnode) |n| {
-            n.setColor(Black);
-            return;
-        }
-
-        while (node == tree.root) {
-            // If not root, there must be parent
-            var parent = maybe_parent.?;
-            if (node == parent.left) {
-                var sibling = parent.right.?; // Same number of black nodes.
-
-                if (sibling.isRed()) {
-                    sibling.setColor(Black);
-                    parent.setColor(Red);
-                    rotateLeft(parent, tree);
-                    sibling = parent.right.?; // Just rotated
-                }
-                if ((if (sibling.left) |n| n.isBlack() else true) and
-                    (if (sibling.right) |n| n.isBlack() else true))
-                {
-                    sibling.setColor(Red);
-                    node = parent;
-                    maybe_parent = parent.getParent();
-                    continue;
-                }
-                if (if (sibling.right) |n| n.isBlack() else true) {
-                    sibling.left.?.setColor(Black); // Same number of black nodes.
-                    sibling.setColor(Red);
-                    rotateRight(sibling, tree);
-                    sibling = parent.right.?; // Just rotated
-                }
-                sibling.setColor(parent.getColor());
-                parent.setColor(Black);
-                sibling.right.?.setColor(Black); // Same number of black nodes.
-                rotateLeft(parent, tree);
-                newnode = tree.root;
-                break;
-            } else {
-                var sibling = parent.left.?; // Same number of black nodes.
-
-                if (sibling.isRed()) {
-                    sibling.setColor(Black);
-                    parent.setColor(Red);
-                    rotateRight(parent, tree);
-                    sibling = parent.left.?; // Just rotated
-                }
-                if ((if (sibling.left) |n| n.isBlack() else true) and
-                    (if (sibling.right) |n| n.isBlack() else true))
-                {
-                    sibling.setColor(Red);
-                    node = parent;
-                    maybe_parent = parent.getParent();
-                    continue;
-                }
-                if (if (sibling.left) |n| n.isBlack() else true) {
-                    sibling.right.?.setColor(Black); // Same number of black nodes
-                    sibling.setColor(Red);
-                    rotateLeft(sibling, tree);
-                    sibling = parent.left.?; // Just rotated
-                }
-                sibling.setColor(parent.getColor());
-                parent.setColor(Black);
-                sibling.left.?.setColor(Black); // Same number of black nodes
-                rotateRight(parent, tree);
-                newnode = tree.root;
-                break;
-            }
-
-            if (node.isRed())
-                break;
-        }
-
-        if (newnode) |n|
-            n.setColor(Black);
-    }
-
-    /// This is a shortcut to avoid removing and re-inserting an item with the same key.
-    pub fn replace(tree: *Tree, old: *Node, newconst: *Node) !void {
-        var new = newconst;
-
-        // I assume this can get optimized out if the caller already knows.
-        if (tree.compareFn(old, new, tree) != .eq) return ReplaceError.NotEqual;
-
-        if (old.getParent()) |parent| {
-            parent.setChild(new, parent.left == old);
-        } else
-            tree.root = new;
-
-        if (old.left) |left|
-            left.setParent(new);
-        if (old.right) |right|
-            right.setParent(new);
-
-        new.* = old.*;
-    }
-
-    pub fn init(f: fn (*Node, *Node, *Tree) Order) Tree {
-        return Tree{
-            .root = null,
-            .compareFn = f,
-        };
-    }
-};
-
-fn rotateLeft(node: *Node, tree: *Tree) void {
-    var p: *Node = node;
-    var q: *Node = node.right orelse unreachable;
-    var parent: *Node = undefined;
-
-    if (!p.isRoot()) {
-        parent = p.getParent().?;
-        if (parent.left == p) {
-            parent.left = q;
-        } else {
-            parent.right = q;
-        }
-        q.setParent(parent);
-    } else {
-        tree.root = q;
-        q.setParent(null);
-    }
-    p.setParent(q);
-
-    p.right = q.left;
-    if (p.right) |right| {
-        right.setParent(p);
-    }
-    q.left = p;
-}
-
-fn rotateRight(node: *Node, tree: *Tree) void {
-    var p: *Node = node;
-    var q: *Node = node.left orelse unreachable;
-    var parent: *Node = undefined;
-
-    if (!p.isRoot()) {
-        parent = p.getParent().?;
-        if (parent.left == p) {
-            parent.left = q;
-        } else {
-            parent.right = q;
-        }
-        q.setParent(parent);
-    } else {
-        tree.root = q;
-        q.setParent(null);
-    }
-    p.setParent(q);
-
-    p.left = q.right;
-    if (p.left) |left| {
-        left.setParent(p);
-    }
-    q.right = p;
-}
-
-fn doLookup(key: *Node, tree: *Tree, pparent: *?*Node, is_left: *bool) ?*Node {
-    var maybe_node: ?*Node = tree.root;
-
-    pparent.* = null;
-    is_left.* = false;
-
-    while (maybe_node) |node| {
-        const res = tree.compareFn(node, key, tree);
-        if (res == .eq) {
-            return node;
-        }
-        pparent.* = node;
-        switch (res) {
-            .gt => {
-                is_left.* = true;
-                maybe_node = node.left;
-            },
-            .lt => {
-                is_left.* = false;
-                maybe_node = node.right;
-            },
-            .eq => unreachable, // handled above
-        }
-    }
-    return null;
-}
-
-const testNumber = struct {
-    node: Node,
-    value: usize,
-};
-
-fn testGetNumber(node: *Node) *testNumber {
-    return @fieldParentPtr(testNumber, "node", node);
-}
-
-fn testCompare(l: *Node, r: *Node, contextIgnored: *Tree) Order {
-    var left = testGetNumber(l);
-    var right = testGetNumber(r);
-
-    if (left.value < right.value) {
-        return .lt;
-    } else if (left.value == right.value) {
-        return .eq;
-    } else if (left.value > right.value) {
-        return .gt;
-    }
-    unreachable;
-}
-
-fn testCompareReverse(l: *Node, r: *Node, contextIgnored: *Tree) Order {
-    return testCompare(r, l, contextIgnored);
-}
-
-test "rb" {
-    if (@import("builtin").arch == .aarch64) {
-        // TODO https://github.com/ziglang/zig/issues/3288
-        return error.SkipZigTest;
-    }
-
-    var tree = Tree.init(testCompare);
-    var ns: [10]testNumber = undefined;
-    ns[0].value = 42;
-    ns[1].value = 41;
-    ns[2].value = 40;
-    ns[3].value = 39;
-    ns[4].value = 38;
-    ns[5].value = 39;
-    ns[6].value = 3453;
-    ns[7].value = 32345;
-    ns[8].value = 392345;
-    ns[9].value = 4;
-
-    var dup: testNumber = undefined;
-    dup.value = 32345;
-
-    _ = tree.insert(&ns[1].node);
-    _ = tree.insert(&ns[2].node);
-    _ = tree.insert(&ns[3].node);
-    _ = tree.insert(&ns[4].node);
-    _ = tree.insert(&ns[5].node);
-    _ = tree.insert(&ns[6].node);
-    _ = tree.insert(&ns[7].node);
-    _ = tree.insert(&ns[8].node);
-    _ = tree.insert(&ns[9].node);
-    tree.remove(&ns[3].node);
-    testing.expect(tree.insert(&dup.node) == &ns[7].node);
-    try tree.replace(&ns[7].node, &dup.node);
-
-    var num: *testNumber = undefined;
-    num = testGetNumber(tree.first().?);
-    while (num.node.next() != null) {
-        testing.expect(testGetNumber(num.node.next().?).value > num.value);
-        num = testGetNumber(num.node.next().?);
-    }
-}
-
-test "inserting and looking up" {
-    var tree = Tree.init(testCompare);
-    var number: testNumber = undefined;
-    number.value = 1000;
-    _ = tree.insert(&number.node);
-    var dup: testNumber = undefined;
-    //Assert that tuples with identical value fields finds the same pointer
-    dup.value = 1000;
-    assert(tree.lookup(&dup.node) == &number.node);
-    //Assert that tuples with identical values do not clobber when inserted.
-    _ = tree.insert(&dup.node);
-    assert(tree.lookup(&dup.node) == &number.node);
-    assert(tree.lookup(&number.node) != &dup.node);
-    assert(testGetNumber(tree.lookup(&dup.node).?).value == testGetNumber(&dup.node).value);
-    //Assert that if looking for a non-existing value, return null.
-    var non_existing_value: testNumber = undefined;
-    non_existing_value.value = 1234;
-    assert(tree.lookup(&non_existing_value.node) == null);
-}
-
-test "multiple inserts, followed by calling first and last" {
-    if (@import("builtin").arch == .aarch64) {
-        // TODO https://github.com/ziglang/zig/issues/3288
-        return error.SkipZigTest;
-    }
-    var tree = Tree.init(testCompare);
-    var zeroth: testNumber = undefined;
-    zeroth.value = 0;
-    var first: testNumber = undefined;
-    first.value = 1;
-    var second: testNumber = undefined;
-    second.value = 2;
-    var third: testNumber = undefined;
-    third.value = 3;
-    _ = tree.insert(&zeroth.node);
-    _ = tree.insert(&first.node);
-    _ = tree.insert(&second.node);
-    _ = tree.insert(&third.node);
-    assert(testGetNumber(tree.first().?).value == 0);
-    assert(testGetNumber(tree.last().?).value == 3);
-    var lookupNode: testNumber = undefined;
-    lookupNode.value = 3;
-    assert(tree.lookup(&lookupNode.node) == &third.node);
-    tree.sort(testCompareReverse) catch unreachable;
-    assert(testGetNumber(tree.first().?).value == 3);
-    assert(testGetNumber(tree.last().?).value == 0);
-    assert(tree.lookup(&lookupNode.node) == &third.node);
-}
diff --git a/lib/std/std.zig b/lib/std/std.zig
index 4236b29298..1ee3944d3b 100644
--- a/lib/std/std.zig
+++ b/lib/std/std.zig
@@ -78,7 +78,6 @@ pub const packed_int_array = @import("packed_int_array.zig");
 pub const pdb = @import("pdb.zig");
 pub const process = @import("process.zig");
 pub const rand = @import("rand.zig");
-pub const rb = @import("rb.zig");
 pub const sort = @import("sort.zig");
 pub const ascii = @import("ascii.zig");
 pub const testing = @import("testing.zig");

From bb636cb3bffadf5a03e592b97a1968492f16f8cd Mon Sep 17 00:00:00 2001
From: Jakub Konka 
Date: Wed, 30 Sep 2020 00:25:35 +0200
Subject: [PATCH 197/210] Revert "Fix for Windows: std.os.windows.DeleteFile()"

---
 lib/std/fs/test.zig    | 23 -----------------------
 lib/std/os/windows.zig | 18 +-----------------
 2 files changed, 1 insertion(+), 40 deletions(-)

diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig
index 8d7ef5172e..b3cc1fe569 100644
--- a/lib/std/fs/test.zig
+++ b/lib/std/fs/test.zig
@@ -813,26 +813,3 @@ fn run_lock_file_test(contexts: []FileLockTestContext) !void {
         try threads.append(try std.Thread.spawn(ctx, FileLockTestContext.run));
     }
 }
-
-test "deleteDir" {
-    var tmp_dir = tmpDir(.{});
-    defer tmp_dir.cleanup();
-
-    // deleting a non-existent directory
-    testing.expectError(error.FileNotFound, tmp_dir.dir.deleteDir("test_dir"));
-
-    var dir = try tmp_dir.dir.makeOpenPath("test_dir", .{});
-    var file = try dir.createFile("test_file", .{});
-    file.close();
-    dir.close();
-
-    // deleting a non-empty directory
-    testing.expectError(error.DirNotEmpty, tmp_dir.dir.deleteDir("test_dir"));
-
-    dir = try tmp_dir.dir.openDir("test_dir", .{});
-    try dir.deleteFile("test_file");
-    dir.close();
-
-    // deleting an empty directory
-    try tmp_dir.dir.deleteDir("test_dir");
-}
diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig
index 2aa222414f..de0d0ea45f 100644
--- a/lib/std/os/windows.zig
+++ b/lib/std/os/windows.zig
@@ -764,7 +764,6 @@ pub const DeleteFileError = error{
     Unexpected,
     NotDir,
     IsDir,
-    DirNotEmpty,
 };
 
 pub const DeleteFileOptions = struct {
@@ -819,7 +818,7 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil
         0,
     );
     switch (rc) {
-        .SUCCESS => CloseHandle(tmp_handle),
+        .SUCCESS => return CloseHandle(tmp_handle),
         .OBJECT_NAME_INVALID => unreachable,
         .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
         .INVALID_PARAMETER => unreachable,
@@ -827,21 +826,6 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil
         .NOT_A_DIRECTORY => return error.NotDir,
         else => return unexpectedStatus(rc),
     }
-
-    // If a directory fails to be deleted, CloseHandle will still report success
-    // Check if the directory still exists and return error.DirNotEmpty if true
-    if (options.remove_dir) {
-        var basic_info: FILE_BASIC_INFORMATION = undefined;
-        switch (ntdll.NtQueryAttributesFile(&attr, &basic_info)) {
-            .SUCCESS => return error.DirNotEmpty,
-            .OBJECT_NAME_NOT_FOUND => return,
-            .OBJECT_PATH_NOT_FOUND => return,
-            .INVALID_PARAMETER => unreachable,
-            .ACCESS_DENIED => return error.AccessDenied,
-            .OBJECT_PATH_SYNTAX_BAD => unreachable,
-            else => |urc| return unexpectedStatus(urc),
-        }
-    }
 }
 
 pub const MoveFileError = error{ FileNotFound, Unexpected };

From d1cea16f5cd29eb143ff9b3302e7ec56731647ea Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 16:39:05 -0700
Subject: [PATCH 198/210] move std.BloomFilter to the standard library
 orphanage

---
 lib/std/bloom_filter.zig | 265 ---------------------------------------
 lib/std/std.zig          |   1 -
 2 files changed, 266 deletions(-)
 delete mode 100644 lib/std/bloom_filter.zig

diff --git a/lib/std/bloom_filter.zig b/lib/std/bloom_filter.zig
deleted file mode 100644
index 0e251f548f..0000000000
--- a/lib/std/bloom_filter.zig
+++ /dev/null
@@ -1,265 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2015-2020 Zig Contributors
-// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
-// The MIT license requires this copyright notice to be included in all copies
-// and substantial portions of the software.
-const builtin = @import("builtin");
-const std = @import("std.zig");
-const math = std.math;
-const debug = std.debug;
-const assert = std.debug.assert;
-const testing = std.testing;
-
-/// There is a trade off of how quickly to fill a bloom filter;
-/// the number of items is:
-///     n_items / K * ln(2)
-/// the rate of false positives is:
-///     (1-e^(-K*N/n_items))^K
-/// where N is the number of items
-pub fn BloomFilter(
-    /// Size of bloom filter in cells, must be a power of two.
-    comptime n_items: usize,
-    /// Number of cells to set per item
-    comptime K: usize,
-    /// Cell type, should be:
-    ///  - `bool` for a standard bloom filter
-    ///  - an unsigned integer type for a counting bloom filter
-    comptime Cell: type,
-    /// endianess of the Cell
-    comptime endian: builtin.Endian,
-    /// Hash function to use
-    comptime hash: fn (out: []u8, Ki: usize, in: []const u8) void,
-) type {
-    assert(n_items > 0);
-    assert(math.isPowerOfTwo(n_items));
-    assert(K > 0);
-    const cellEmpty = if (Cell == bool) false else @as(Cell, 0);
-    const cellMax = if (Cell == bool) true else math.maxInt(Cell);
-    const n_bytes = (n_items * comptime std.meta.bitCount(Cell)) / 8;
-    assert(n_bytes > 0);
-    const Io = std.packed_int_array.PackedIntIo(Cell, endian);
-
-    return struct {
-        const Self = @This();
-        pub const items = n_items;
-        pub const Index = math.IntFittingRange(0, n_items - 1);
-
-        data: [n_bytes]u8 = [_]u8{0} ** n_bytes,
-
-        pub fn reset(self: *Self) void {
-            std.mem.set(u8, self.data[0..], 0);
-        }
-
-        pub fn @"union"(x: Self, y: Self) Self {
-            var r = Self{ .data = undefined };
-            inline for (x.data) |v, i| {
-                r.data[i] = v | y.data[i];
-            }
-            return r;
-        }
-
-        pub fn intersection(x: Self, y: Self) Self {
-            var r = Self{ .data = undefined };
-            inline for (x.data) |v, i| {
-                r.data[i] = v & y.data[i];
-            }
-            return r;
-        }
-
-        pub fn getCell(self: Self, cell: Index) Cell {
-            return Io.get(&self.data, cell, 0);
-        }
-
-        pub fn incrementCell(self: *Self, cell: Index) void {
-            if (Cell == bool or Cell == u1) {
-                // skip the 'get' operation
-                Io.set(&self.data, cell, 0, cellMax);
-            } else {
-                const old = Io.get(&self.data, cell, 0);
-                if (old != cellMax) {
-                    Io.set(&self.data, cell, 0, old + 1);
-                }
-            }
-        }
-
-        pub fn clearCell(self: *Self, cell: Index) void {
-            Io.set(&self.data, cell, 0, cellEmpty);
-        }
-
-        pub fn add(self: *Self, item: []const u8) void {
-            comptime var i = 0;
-            inline while (i < K) : (i += 1) {
-                var K_th_bit: packed struct {
-                    x: Index,
-                } = undefined;
-                hash(std.mem.asBytes(&K_th_bit), i, item);
-                incrementCell(self, K_th_bit.x);
-            }
-        }
-
-        pub fn contains(self: Self, item: []const u8) bool {
-            comptime var i = 0;
-            inline while (i < K) : (i += 1) {
-                var K_th_bit: packed struct {
-                    x: Index,
-                } = undefined;
-                hash(std.mem.asBytes(&K_th_bit), i, item);
-                if (getCell(self, K_th_bit.x) == cellEmpty)
-                    return false;
-            }
-            return true;
-        }
-
-        pub fn resize(self: Self, comptime newsize: usize) BloomFilter(newsize, K, Cell, endian, hash) {
-            var r: BloomFilter(newsize, K, Cell, endian, hash) = undefined;
-            if (newsize < n_items) {
-                std.mem.copy(u8, r.data[0..], self.data[0..r.data.len]);
-                var copied: usize = r.data.len;
-                while (copied < self.data.len) : (copied += r.data.len) {
-                    for (self.data[copied .. copied + r.data.len]) |s, i| {
-                        r.data[i] |= s;
-                    }
-                }
-            } else if (newsize == n_items) {
-                r = self;
-            } else if (newsize > n_items) {
-                var copied: usize = 0;
-                while (copied < r.data.len) : (copied += self.data.len) {
-                    std.mem.copy(u8, r.data[copied .. copied + self.data.len], &self.data);
-                }
-            }
-            return r;
-        }
-
-        /// Returns number of non-zero cells
-        pub fn popCount(self: Self) Index {
-            var n: Index = 0;
-            if (Cell == bool or Cell == u1) {
-                for (self.data) |b, i| {
-                    n += @popCount(u8, b);
-                }
-            } else {
-                var i: usize = 0;
-                while (i < n_items) : (i += 1) {
-                    const cell = self.getCell(@intCast(Index, i));
-                    n += if (if (Cell == bool) cell else cell > 0) @as(Index, 1) else @as(Index, 0);
-                }
-            }
-            return n;
-        }
-
-        pub fn estimateItems(self: Self) f64 {
-            const m = comptime @intToFloat(f64, n_items);
-            const k = comptime @intToFloat(f64, K);
-            const X = @intToFloat(f64, self.popCount());
-            return (comptime (-m / k)) * math.log1p(X * comptime (-1 / m));
-        }
-    };
-}
-
-fn hashFunc(out: []u8, Ki: usize, in: []const u8) void {
-    var st = std.crypto.hash.Gimli.init(.{});
-    st.update(std.mem.asBytes(&Ki));
-    st.update(in);
-    st.final(out);
-}
-
-test "std.BloomFilter" {
-    // https://github.com/ziglang/zig/issues/5127
-    if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
-
-    inline for ([_]type{ bool, u1, u2, u3, u4 }) |Cell| {
-        const emptyCell = if (Cell == bool) false else @as(Cell, 0);
-        const BF = BloomFilter(128 * 8, 8, Cell, builtin.endian, hashFunc);
-        var bf = BF{};
-        var i: usize = undefined;
-        // confirm that it is initialised to the empty filter
-        i = 0;
-        while (i < BF.items) : (i += 1) {
-            testing.expectEqual(emptyCell, bf.getCell(@intCast(BF.Index, i)));
-        }
-        testing.expectEqual(@as(BF.Index, 0), bf.popCount());
-        testing.expectEqual(@as(f64, 0), bf.estimateItems());
-        // fill in a few items
-        bf.incrementCell(42);
-        bf.incrementCell(255);
-        bf.incrementCell(256);
-        bf.incrementCell(257);
-        // check that they were set
-        testing.expectEqual(true, bf.getCell(42) != emptyCell);
-        testing.expectEqual(true, bf.getCell(255) != emptyCell);
-        testing.expectEqual(true, bf.getCell(256) != emptyCell);
-        testing.expectEqual(true, bf.getCell(257) != emptyCell);
-        // clear just one of them; make sure the rest are still set
-        bf.clearCell(256);
-        testing.expectEqual(true, bf.getCell(42) != emptyCell);
-        testing.expectEqual(true, bf.getCell(255) != emptyCell);
-        testing.expectEqual(false, bf.getCell(256) != emptyCell);
-        testing.expectEqual(true, bf.getCell(257) != emptyCell);
-        // reset any of the ones we've set and confirm we're back to the empty filter
-        bf.clearCell(42);
-        bf.clearCell(255);
-        bf.clearCell(257);
-        i = 0;
-        while (i < BF.items) : (i += 1) {
-            testing.expectEqual(emptyCell, bf.getCell(@intCast(BF.Index, i)));
-        }
-        testing.expectEqual(@as(BF.Index, 0), bf.popCount());
-        testing.expectEqual(@as(f64, 0), bf.estimateItems());
-
-        // Lets add a string
-        bf.add("foo");
-        testing.expectEqual(true, bf.contains("foo"));
-        {
-            // try adding same string again. make sure popcount is the same
-            const old_popcount = bf.popCount();
-            testing.expect(old_popcount > 0);
-            bf.add("foo");
-            testing.expectEqual(true, bf.contains("foo"));
-            testing.expectEqual(old_popcount, bf.popCount());
-        }
-
-        // Get back to empty filter via .reset
-        bf.reset();
-        // Double check that .reset worked
-        i = 0;
-        while (i < BF.items) : (i += 1) {
-            testing.expectEqual(emptyCell, bf.getCell(@intCast(BF.Index, i)));
-        }
-        testing.expectEqual(@as(BF.Index, 0), bf.popCount());
-        testing.expectEqual(@as(f64, 0), bf.estimateItems());
-
-        comptime var teststrings = [_][]const u8{
-            "foo",
-            "bar",
-            "a longer string",
-            "some more",
-            "the quick brown fox",
-            "unique string",
-        };
-        inline for (teststrings) |str| {
-            bf.add(str);
-        }
-        inline for (teststrings) |str| {
-            testing.expectEqual(true, bf.contains(str));
-        }
-
-        { // estimate should be close for low packing
-            const est = bf.estimateItems();
-            testing.expect(est > @intToFloat(f64, teststrings.len) - 1);
-            testing.expect(est < @intToFloat(f64, teststrings.len) + 1);
-        }
-
-        const larger_bf = bf.resize(4096);
-        inline for (teststrings) |str| {
-            testing.expectEqual(true, larger_bf.contains(str));
-        }
-        testing.expectEqual(@as(u12, bf.popCount()) * (4096 / 1024), larger_bf.popCount());
-
-        const smaller_bf = bf.resize(64);
-        inline for (teststrings) |str| {
-            testing.expectEqual(true, smaller_bf.contains(str));
-        }
-        testing.expect(bf.popCount() <= @as(u10, smaller_bf.popCount()) * (1024 / 64));
-    }
-}
diff --git a/lib/std/std.zig b/lib/std/std.zig
index 1ee3944d3b..5b12046ed8 100644
--- a/lib/std/std.zig
+++ b/lib/std/std.zig
@@ -14,7 +14,6 @@ pub const AutoArrayHashMap = array_hash_map.AutoArrayHashMap;
 pub const AutoArrayHashMapUnmanaged = array_hash_map.AutoArrayHashMapUnmanaged;
 pub const AutoHashMap = hash_map.AutoHashMap;
 pub const AutoHashMapUnmanaged = hash_map.AutoHashMapUnmanaged;
-pub const BloomFilter = @import("bloom_filter.zig").BloomFilter;
 pub const BufMap = @import("buf_map.zig").BufMap;
 pub const BufSet = @import("buf_set.zig").BufSet;
 pub const ChildProcess = @import("child_process.zig").ChildProcess;

From f69650a4788c57ec20348e9c002884ba39339d1c Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 16:59:35 -0700
Subject: [PATCH 199/210] update wasm to use ".o.wasm" extension for objects

This is convenient for debugging purposes, as well as simplifying the
caching system since executable basenames will not conflict with their
corresponding object files.
---
 lib/std/build.zig            |  1 +
 lib/std/target.zig           | 19 +++++++++++--------
 lib/std/zig.zig              | 20 +++++++++-----------
 lib/std/zig/cross_target.zig |  2 +-
 src/Compilation.zig          |  2 +-
 src/link.zig                 |  2 +-
 6 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/lib/std/build.zig b/lib/std/build.zig
index b2da9da332..7e3c75bc78 100644
--- a/lib/std/build.zig
+++ b/lib/std/build.zig
@@ -1384,6 +1384,7 @@ pub const LibExeObjStep = struct {
     }
 
     fn computeOutFileNames(self: *LibExeObjStep) void {
+        // TODO make this call std.zig.binNameAlloc
         switch (self.kind) {
             .Obj => {
                 self.out_filename = self.builder.fmt("{}{}", .{ self.name, self.target.oFileExt() });
diff --git a/lib/std/target.zig b/lib/std/target.zig
index e1a7e1a2bf..99617c6d0e 100644
--- a/lib/std/target.zig
+++ b/lib/std/target.zig
@@ -483,13 +483,6 @@ pub const Target = struct {
                 else => false,
             };
         }
-
-        pub fn oFileExt(abi: Abi) [:0]const u8 {
-            return switch (abi) {
-                .msvc => ".obj",
-                else => ".o",
-            };
-        }
     };
 
     pub const ObjectFormat = enum {
@@ -1142,8 +1135,18 @@ pub const Target = struct {
         return linuxTripleSimple(allocator, self.cpu.arch, self.os.tag, self.abi);
     }
 
+    pub fn oFileExt_cpu_arch_abi(cpu_arch: Cpu.Arch, abi: Abi) [:0]const u8 {
+        if (cpu_arch.isWasm()) {
+            return ".o.wasm";
+        }
+        switch (abi) {
+            .msvc => return ".obj",
+            else => return ".o",
+        }
+    }
+
     pub fn oFileExt(self: Target) [:0]const u8 {
-        return self.abi.oFileExt();
+        return oFileExt_cpu_arch_abi(self.cpu.arch, self.abi);
     }
 
     pub fn exeFileExtSimple(cpu_arch: Cpu.Arch, os_tag: Os.Tag) [:0]const u8 {
diff --git a/lib/std/zig.zig b/lib/std/zig.zig
index 66226e8dd1..42a33383b7 100644
--- a/lib/std/zig.zig
+++ b/lib/std/zig.zig
@@ -79,13 +79,7 @@ pub fn binNameAlloc(allocator: *std.mem.Allocator, options: BinNameOptions) erro
     const target = options.target;
     switch (options.object_format orelse target.getObjectFormat()) {
         .coff, .pe => switch (options.output_mode) {
-            .Exe => {
-                const suffix = switch (target.os.tag) {
-                    .uefi => ".efi",
-                    else => ".exe",
-                };
-                return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, suffix });
-            },
+            .Exe => return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, target.exeFileExt() }),
             .Lib => {
                 const suffix = switch (options.link_mode orelse .Static) {
                     .Static => ".lib",
@@ -93,7 +87,7 @@ pub fn binNameAlloc(allocator: *std.mem.Allocator, options: BinNameOptions) erro
                 };
                 return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, suffix });
             },
-            .Obj => return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, target.abi.oFileExt() }),
+            .Obj => return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, target.oFileExt() }),
         },
         .elf => switch (options.output_mode) {
             .Exe => return allocator.dupe(u8, root_name),
@@ -113,7 +107,7 @@ pub fn binNameAlloc(allocator: *std.mem.Allocator, options: BinNameOptions) erro
                     },
                 }
             },
-            .Obj => return std.fmt.allocPrint(allocator, "{s}.o", .{root_name}),
+            .Obj => return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, target.oFileExt() }),
         },
         .macho => switch (options.output_mode) {
             .Exe => return allocator.dupe(u8, root_name),
@@ -124,9 +118,13 @@ pub fn binNameAlloc(allocator: *std.mem.Allocator, options: BinNameOptions) erro
                 };
                 return std.fmt.allocPrint(allocator, "{s}{s}{s}", .{ target.libPrefix(), root_name, suffix });
             },
-            .Obj => return std.fmt.allocPrint(allocator, "{s}.o", .{root_name}),
+            .Obj => return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, target.oFileExt() }),
+        },
+        .wasm => switch (options.output_mode) {
+            .Exe => return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, target.exeFileExt() }),
+            .Obj => return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, target.oFileExt() }),
+            .Lib => return std.fmt.allocPrint(allocator, "{s}.wasm", .{root_name}),
         },
-        .wasm => return std.fmt.allocPrint(allocator, "{s}.wasm", .{root_name}),
         .c => return std.fmt.allocPrint(allocator, "{s}.c", .{root_name}),
         .hex => return std.fmt.allocPrint(allocator, "{s}.ihex", .{root_name}),
         .raw => return std.fmt.allocPrint(allocator, "{s}.bin", .{root_name}),
diff --git a/lib/std/zig/cross_target.zig b/lib/std/zig/cross_target.zig
index f94598c30c..f1ae3457a5 100644
--- a/lib/std/zig/cross_target.zig
+++ b/lib/std/zig/cross_target.zig
@@ -466,7 +466,7 @@ pub const CrossTarget = struct {
     }
 
     pub fn oFileExt(self: CrossTarget) [:0]const u8 {
-        return self.getAbi().oFileExt();
+        return Target.oFileExt_cpu_arch_abi(self.getCpuArch(), self.getAbi());
     }
 
     pub fn exeFileExt(self: CrossTarget) [:0]const u8 {
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 9cc2c3aa5c..d60b323216 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1520,7 +1520,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void {
         comp.bin_file.options.root_name
     else
         mem.split(c_source_basename, ".").next().?;
-    const o_basename = try std.fmt.allocPrint(arena, "{}{}", .{ o_basename_noext, comp.getTarget().oFileExt() });
+    const o_basename = try std.fmt.allocPrint(arena, "{s}{s}", .{ o_basename_noext, comp.getTarget().oFileExt() });
 
     const digest = if (!comp.disable_c_depfile and try man.hit()) man.final() else blk: {
         var argv = std.ArrayList([]const u8).init(comp.gpa);
diff --git a/src/link.zig b/src/link.zig
index 51bf8a64ae..2d215ea0dc 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -176,7 +176,7 @@ pub const File = struct {
                 };
             }
             // Open a temporary object file, not the final output file because we want to link with LLD.
-            break :blk try std.fmt.allocPrint(allocator, "{}{}", .{ emit.sub_path, options.target.oFileExt() });
+            break :blk try std.fmt.allocPrint(allocator, "{s}{s}", .{ emit.sub_path, options.target.oFileExt() });
         } else emit.sub_path;
         errdefer if (use_lld) allocator.free(sub_path);
 

From 27e008eb292038c5a6b9a13b64c7b69d1525f690 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 17:24:21 -0700
Subject: [PATCH 200/210] fix not passing std.builtin.link_libc to
 compiler_rt.zig and c.zig

comment reproduced here:

This is so that compiler_rt and libc.zig libraries know whether they
will eventually be linked with libc. They make different decisions
about what to export depending on whether another libc will be linked
in. For example, compiler_rt will not export the __chkstk symbol if it
knows libc will provide it, and likewise c.zig will not export memcpy.
---
 src/Compilation.zig | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index d60b323216..1cb02d6f1c 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2842,6 +2842,12 @@ pub fn build_crt_file(
         .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
         .clang_passthrough_mode = comp.clang_passthrough_mode,
         .is_compiler_rt_or_libc = true,
+        // This is so that compiler_rt and libc.zig libraries know whether they
+        // will eventually be linked with libc. They make different decisions
+        // about what to export depending on whether another libc will be linked
+        // in. For example, compiler_rt will not export the __chkstk symbol if it
+        // knows libc will provide it, and likewise c.zig will not export memcpy.
+        .link_libc = comp.bin_file.options.link_libc,
     });
     defer sub_compilation.destroy();
 

From 402f967ed5339fa3d828b7fe1d57cdb5bf38dbf2 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 17:39:35 -0700
Subject: [PATCH 201/210] move std.http to the standard library orphanage

I want to take the design of this in a different direction. I think this
abstraction is too high level. I want to start bottom-up.

std-lib-orphanage commit 179ae67d61455758d71037434704fd4a17a635a9
---
 lib/std/http.zig         |  10 -
 lib/std/http/headers.zig | 597 ---------------------------------------
 lib/std/std.zig          |   1 -
 3 files changed, 608 deletions(-)
 delete mode 100644 lib/std/http.zig
 delete mode 100644 lib/std/http/headers.zig

diff --git a/lib/std/http.zig b/lib/std/http.zig
deleted file mode 100644
index 2b8495db71..0000000000
--- a/lib/std/http.zig
+++ /dev/null
@@ -1,10 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2015-2020 Zig Contributors
-// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
-// The MIT license requires this copyright notice to be included in all copies
-// and substantial portions of the software.
-test "std.http" {
-    _ = @import("http/headers.zig");
-}
-
-pub const Headers = @import("http/headers.zig").Headers;
diff --git a/lib/std/http/headers.zig b/lib/std/http/headers.zig
deleted file mode 100644
index 0ce642865c..0000000000
--- a/lib/std/http/headers.zig
+++ /dev/null
@@ -1,597 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2015-2020 Zig Contributors
-// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
-// The MIT license requires this copyright notice to be included in all copies
-// and substantial portions of the software.
-// HTTP Header data structure/type
-// Based on lua-http's http.header module
-//
-// Design criteria:
-//   - the same header field is allowed more than once
-//       - must be able to fetch separate occurrences (important for some headers e.g. Set-Cookie)
-//       - optionally available as comma separated list
-//   - http2 adds flag to headers that they should never be indexed
-//   - header order should be recoverable
-//
-// Headers are implemented as an array of entries.
-// An index of field name => array indices is kept.
-
-const std = @import("../std.zig");
-const debug = std.debug;
-const assert = debug.assert;
-const testing = std.testing;
-const mem = std.mem;
-const Allocator = mem.Allocator;
-
-fn never_index_default(name: []const u8) bool {
-    if (mem.eql(u8, "authorization", name)) return true;
-    if (mem.eql(u8, "proxy-authorization", name)) return true;
-    if (mem.eql(u8, "cookie", name)) return true;
-    if (mem.eql(u8, "set-cookie", name)) return true;
-    return false;
-}
-
-const HeaderEntry = struct {
-    name: []const u8,
-    value: []u8,
-    never_index: bool,
-
-    const Self = @This();
-
-    fn init(allocator: *Allocator, name: []const u8, value: []const u8, never_index: ?bool) !Self {
-        return Self{
-            .name = name, // takes reference
-            .value = try allocator.dupe(u8, value),
-            .never_index = never_index orelse never_index_default(name),
-        };
-    }
-
-    fn deinit(self: Self, allocator: *Allocator) void {
-        allocator.free(self.value);
-    }
-
-    pub fn modify(self: *Self, allocator: *Allocator, value: []const u8, never_index: ?bool) !void {
-        const old_len = self.value.len;
-        if (value.len > old_len) {
-            self.value = try allocator.realloc(self.value, value.len);
-        } else if (value.len < old_len) {
-            self.value = allocator.shrink(self.value, value.len);
-        }
-        mem.copy(u8, self.value, value);
-        self.never_index = never_index orelse never_index_default(self.name);
-    }
-
-    fn compare(context: void, a: HeaderEntry, b: HeaderEntry) bool {
-        if (a.name.ptr != b.name.ptr and a.name.len != b.name.len) {
-            // Things beginning with a colon *must* be before others
-            const a_is_colon = a.name[0] == ':';
-            const b_is_colon = b.name[0] == ':';
-            if (a_is_colon and !b_is_colon) {
-                return true;
-            } else if (!a_is_colon and b_is_colon) {
-                return false;
-            }
-
-            // Sort lexicographically on header name
-            return mem.order(u8, a.name, b.name) == .lt;
-        }
-
-        // Sort lexicographically on header value
-        if (!mem.eql(u8, a.value, b.value)) {
-            return mem.order(u8, a.value, b.value) == .lt;
-        }
-
-        // Doesn't matter here; need to pick something for sort consistency
-        return a.never_index;
-    }
-};
-
-test "HeaderEntry" {
-    var e = try HeaderEntry.init(testing.allocator, "foo", "bar", null);
-    defer e.deinit(testing.allocator);
-    testing.expectEqualSlices(u8, "foo", e.name);
-    testing.expectEqualSlices(u8, "bar", e.value);
-    testing.expectEqual(false, e.never_index);
-
-    try e.modify(testing.allocator, "longer value", null);
-    testing.expectEqualSlices(u8, "longer value", e.value);
-
-    // shorter value
-    try e.modify(testing.allocator, "x", null);
-    testing.expectEqualSlices(u8, "x", e.value);
-}
-
-const HeaderList = std.ArrayListUnmanaged(HeaderEntry);
-const HeaderIndexList = std.ArrayListUnmanaged(usize);
-const HeaderIndex = std.StringHashMapUnmanaged(HeaderIndexList);
-
-pub const Headers = struct {
-    // the owned header field name is stored in the index as part of the key
-    allocator: *Allocator,
-    data: HeaderList,
-    index: HeaderIndex,
-
-    const Self = @This();
-
-    pub fn init(allocator: *Allocator) Self {
-        return Self{
-            .allocator = allocator,
-            .data = HeaderList{},
-            .index = HeaderIndex{},
-        };
-    }
-
-    pub fn deinit(self: *Self) void {
-        {
-            var it = self.index.iterator();
-            while (it.next()) |entry| {
-                entry.value.deinit(self.allocator);
-                self.allocator.free(entry.key);
-            }
-            self.index.deinit(self.allocator);
-        }
-        {
-            for (self.data.items) |entry| {
-                entry.deinit(self.allocator);
-            }
-            self.data.deinit(self.allocator);
-        }
-        self.* = undefined;
-    }
-
-    pub fn clone(self: Self, allocator: *Allocator) !Self {
-        var other = Headers.init(allocator);
-        errdefer other.deinit();
-        try other.data.ensureCapacity(allocator, self.data.items.len);
-        try other.index.initCapacity(allocator, self.index.entries.len);
-        for (self.data.items) |entry| {
-            try other.append(entry.name, entry.value, entry.never_index);
-        }
-        return other;
-    }
-
-    pub fn toSlice(self: Self) []const HeaderEntry {
-        return self.data.items;
-    }
-
-    pub fn append(self: *Self, name: []const u8, value: []const u8, never_index: ?bool) !void {
-        const n = self.data.items.len + 1;
-        try self.data.ensureCapacity(self.allocator, n);
-        var entry: HeaderEntry = undefined;
-        if (self.index.getEntry(name)) |kv| {
-            entry = try HeaderEntry.init(self.allocator, kv.key, value, never_index);
-            errdefer entry.deinit(self.allocator);
-            const dex = &kv.value;
-            try dex.append(self.allocator, n - 1);
-        } else {
-            const name_dup = try self.allocator.dupe(u8, name);
-            errdefer self.allocator.free(name_dup);
-            entry = try HeaderEntry.init(self.allocator, name_dup, value, never_index);
-            errdefer entry.deinit(self.allocator);
-            var dex = HeaderIndexList{};
-            try dex.append(self.allocator, n - 1);
-            errdefer dex.deinit(self.allocator);
-            _ = try self.index.put(self.allocator, name_dup, dex);
-        }
-        self.data.appendAssumeCapacity(entry);
-    }
-
-    /// If the header already exists, replace the current value, otherwise append it to the list of headers.
-    /// If the header has multiple entries then returns an error.
-    pub fn upsert(self: *Self, name: []const u8, value: []const u8, never_index: ?bool) !void {
-        if (self.index.get(name)) |kv| {
-            const dex = kv.value;
-            if (dex.len != 1)
-                return error.CannotUpsertMultiValuedField;
-            var e = &self.data.at(dex.at(0));
-            try e.modify(value, never_index);
-        } else {
-            try self.append(name, value, never_index);
-        }
-    }
-
-    /// Returns boolean indicating if the field is present.
-    pub fn contains(self: Self, name: []const u8) bool {
-        return self.index.contains(name);
-    }
-
-    /// Returns boolean indicating if something was deleted.
-    pub fn delete(self: *Self, name: []const u8) bool {
-        if (self.index.remove(name)) |*kv| {
-            const dex = &kv.value;
-            // iterate backwards
-            var i = dex.items.len;
-            while (i > 0) {
-                i -= 1;
-                const data_index = dex.items[i];
-                const removed = self.data.orderedRemove(data_index);
-                assert(mem.eql(u8, removed.name, name));
-                removed.deinit(self.allocator);
-            }
-            dex.deinit(self.allocator);
-            self.allocator.free(kv.key);
-            self.rebuildIndex();
-            return true;
-        } else {
-            return false;
-        }
-    }
-
-    /// Removes the element at the specified index.
-    /// Moves items down to fill the empty space.
-    /// TODO this implementation can be replaced by adding
-    /// orderedRemove to the new hash table implementation as an
-    /// alternative to swapRemove.
-    pub fn orderedRemove(self: *Self, i: usize) void {
-        const removed = self.data.orderedRemove(i);
-        const kv = self.index.getEntry(removed.name).?;
-        const dex = &kv.value;
-        if (dex.items.len == 1) {
-            // was last item; delete the index
-            dex.deinit(self.allocator);
-            removed.deinit(self.allocator);
-            const key = kv.key;
-            _ = self.index.remove(key); // invalidates `kv` and `dex`
-            self.allocator.free(key);
-        } else {
-            dex.shrink(self.allocator, dex.items.len - 1);
-            removed.deinit(self.allocator);
-        }
-        // if it was the last item; no need to rebuild index
-        if (i != self.data.items.len) {
-            self.rebuildIndex();
-        }
-    }
-
-    /// Removes the element at the specified index.
-    /// The empty slot is filled from the end of the list.
-    /// TODO this implementation can be replaced by simply using the
-    /// new hash table which does swap removal.
-    pub fn swapRemove(self: *Self, i: usize) void {
-        const removed = self.data.swapRemove(i);
-        const kv = self.index.getEntry(removed.name).?;
-        const dex = &kv.value;
-        if (dex.items.len == 1) {
-            // was last item; delete the index
-            dex.deinit(self.allocator);
-            removed.deinit(self.allocator);
-            const key = kv.key;
-            _ = self.index.remove(key); // invalidates `kv` and `dex`
-            self.allocator.free(key);
-        } else {
-            dex.shrink(self.allocator, dex.items.len - 1);
-            removed.deinit(self.allocator);
-        }
-        // if it was the last item; no need to rebuild index
-        if (i != self.data.items.len) {
-            self.rebuildIndex();
-        }
-    }
-
-    /// Access the header at the specified index.
-    pub fn at(self: Self, i: usize) HeaderEntry {
-        return self.data.items[i];
-    }
-
-    /// Returns a list of indices containing headers with the given name.
-    /// The returned list should not be modified by the caller.
-    pub fn getIndices(self: Self, name: []const u8) ?HeaderIndexList {
-        return self.index.get(name);
-    }
-
-    /// Returns a slice containing each header with the given name.
-    pub fn get(self: Self, allocator: *Allocator, name: []const u8) !?[]const HeaderEntry {
-        const dex = self.getIndices(name) orelse return null;
-
-        const buf = try allocator.alloc(HeaderEntry, dex.items.len);
-        var n: usize = 0;
-        for (dex.items) |idx| {
-            buf[n] = self.data.items[idx];
-            n += 1;
-        }
-        return buf;
-    }
-
-    /// Returns all headers with the given name as a comma separated string.
-    ///
-    /// Useful for HTTP headers that follow RFC-7230 section 3.2.2:
-    ///   A recipient MAY combine multiple header fields with the same field
-    ///   name into one "field-name: field-value" pair, without changing the
-    ///   semantics of the message, by appending each subsequent field value to
-    ///   the combined field value in order, separated by a comma.  The order
-    ///   in which header fields with the same field name are received is
-    ///   therefore significant to the interpretation of the combined field
-    ///   value
-    pub fn getCommaSeparated(self: Self, allocator: *Allocator, name: []const u8) !?[]u8 {
-        const dex = self.getIndices(name) orelse return null;
-
-        // adapted from mem.join
-        const total_len = blk: {
-            var sum: usize = dex.items.len - 1; // space for separator(s)
-            for (dex.items) |idx|
-                sum += self.data.items[idx].value.len;
-            break :blk sum;
-        };
-
-        const buf = try allocator.alloc(u8, total_len);
-        errdefer allocator.free(buf);
-
-        const first_value = self.data.items[dex.items[0]].value;
-        mem.copy(u8, buf, first_value);
-        var buf_index: usize = first_value.len;
-        for (dex.items[1..]) |idx| {
-            const value = self.data.items[idx].value;
-            buf[buf_index] = ',';
-            buf_index += 1;
-            mem.copy(u8, buf[buf_index..], value);
-            buf_index += value.len;
-        }
-
-        // No need for shrink since buf is exactly the correct size.
-        return buf;
-    }
-
-    fn rebuildIndex(self: *Self) void {
-        // clear out the indexes
-        var it = self.index.iterator();
-        while (it.next()) |entry| {
-            entry.value.shrinkRetainingCapacity(0);
-        }
-        // fill up indexes again; we know capacity is fine from before
-        for (self.data.items) |entry, i| {
-            self.index.getEntry(entry.name).?.value.appendAssumeCapacity(i);
-        }
-    }
-
-    pub fn sort(self: *Self) void {
-        std.sort.sort(HeaderEntry, self.data.items, {}, HeaderEntry.compare);
-        self.rebuildIndex();
-    }
-
-    pub fn format(
-        self: Self,
-        comptime fmt: []const u8,
-        options: std.fmt.FormatOptions,
-        out_stream: anytype,
-    ) !void {
-        for (self.toSlice()) |entry| {
-            try out_stream.writeAll(entry.name);
-            try out_stream.writeAll(": ");
-            try out_stream.writeAll(entry.value);
-            try out_stream.writeAll("\n");
-        }
-    }
-};
-
-test "Headers.iterator" {
-    var h = Headers.init(testing.allocator);
-    defer h.deinit();
-    try h.append("foo", "bar", null);
-    try h.append("cookie", "somevalue", null);
-
-    var count: i32 = 0;
-    for (h.toSlice()) |e| {
-        if (count == 0) {
-            testing.expectEqualSlices(u8, "foo", e.name);
-            testing.expectEqualSlices(u8, "bar", e.value);
-            testing.expectEqual(false, e.never_index);
-        } else if (count == 1) {
-            testing.expectEqualSlices(u8, "cookie", e.name);
-            testing.expectEqualSlices(u8, "somevalue", e.value);
-            testing.expectEqual(true, e.never_index);
-        }
-        count += 1;
-    }
-    testing.expectEqual(@as(i32, 2), count);
-}
-
-test "Headers.contains" {
-    var h = Headers.init(testing.allocator);
-    defer h.deinit();
-    try h.append("foo", "bar", null);
-    try h.append("cookie", "somevalue", null);
-
-    testing.expectEqual(true, h.contains("foo"));
-    testing.expectEqual(false, h.contains("flooble"));
-}
-
-test "Headers.delete" {
-    var h = Headers.init(testing.allocator);
-    defer h.deinit();
-    try h.append("foo", "bar", null);
-    try h.append("baz", "qux", null);
-    try h.append("cookie", "somevalue", null);
-
-    testing.expectEqual(false, h.delete("not-present"));
-    testing.expectEqual(@as(usize, 3), h.toSlice().len);
-
-    testing.expectEqual(true, h.delete("foo"));
-    testing.expectEqual(@as(usize, 2), h.toSlice().len);
-    {
-        const e = h.at(0);
-        testing.expectEqualSlices(u8, "baz", e.name);
-        testing.expectEqualSlices(u8, "qux", e.value);
-        testing.expectEqual(false, e.never_index);
-    }
-    {
-        const e = h.at(1);
-        testing.expectEqualSlices(u8, "cookie", e.name);
-        testing.expectEqualSlices(u8, "somevalue", e.value);
-        testing.expectEqual(true, e.never_index);
-    }
-
-    testing.expectEqual(false, h.delete("foo"));
-}
-
-test "Headers.orderedRemove" {
-    var h = Headers.init(testing.allocator);
-    defer h.deinit();
-    try h.append("foo", "bar", null);
-    try h.append("baz", "qux", null);
-    try h.append("cookie", "somevalue", null);
-
-    h.orderedRemove(0);
-    testing.expectEqual(@as(usize, 2), h.toSlice().len);
-    {
-        const e = h.at(0);
-        testing.expectEqualSlices(u8, "baz", e.name);
-        testing.expectEqualSlices(u8, "qux", e.value);
-        testing.expectEqual(false, e.never_index);
-    }
-    {
-        const e = h.at(1);
-        testing.expectEqualSlices(u8, "cookie", e.name);
-        testing.expectEqualSlices(u8, "somevalue", e.value);
-        testing.expectEqual(true, e.never_index);
-    }
-}
-
-test "Headers.swapRemove" {
-    var h = Headers.init(testing.allocator);
-    defer h.deinit();
-    try h.append("foo", "bar", null);
-    try h.append("baz", "qux", null);
-    try h.append("cookie", "somevalue", null);
-
-    h.swapRemove(0);
-    testing.expectEqual(@as(usize, 2), h.toSlice().len);
-    {
-        const e = h.at(0);
-        testing.expectEqualSlices(u8, "cookie", e.name);
-        testing.expectEqualSlices(u8, "somevalue", e.value);
-        testing.expectEqual(true, e.never_index);
-    }
-    {
-        const e = h.at(1);
-        testing.expectEqualSlices(u8, "baz", e.name);
-        testing.expectEqualSlices(u8, "qux", e.value);
-        testing.expectEqual(false, e.never_index);
-    }
-}
-
-test "Headers.at" {
-    var h = Headers.init(testing.allocator);
-    defer h.deinit();
-    try h.append("foo", "bar", null);
-    try h.append("cookie", "somevalue", null);
-
-    {
-        const e = h.at(0);
-        testing.expectEqualSlices(u8, "foo", e.name);
-        testing.expectEqualSlices(u8, "bar", e.value);
-        testing.expectEqual(false, e.never_index);
-    }
-    {
-        const e = h.at(1);
-        testing.expectEqualSlices(u8, "cookie", e.name);
-        testing.expectEqualSlices(u8, "somevalue", e.value);
-        testing.expectEqual(true, e.never_index);
-    }
-}
-
-test "Headers.getIndices" {
-    var h = Headers.init(testing.allocator);
-    defer h.deinit();
-    try h.append("foo", "bar", null);
-    try h.append("set-cookie", "x=1", null);
-    try h.append("set-cookie", "y=2", null);
-
-    testing.expect(null == h.getIndices("not-present"));
-    testing.expectEqualSlices(usize, &[_]usize{0}, h.getIndices("foo").?.items);
-    testing.expectEqualSlices(usize, &[_]usize{ 1, 2 }, h.getIndices("set-cookie").?.items);
-}
-
-test "Headers.get" {
-    var h = Headers.init(testing.allocator);
-    defer h.deinit();
-    try h.append("foo", "bar", null);
-    try h.append("set-cookie", "x=1", null);
-    try h.append("set-cookie", "y=2", null);
-
-    {
-        const v = try h.get(testing.allocator, "not-present");
-        testing.expect(null == v);
-    }
-    {
-        const v = (try h.get(testing.allocator, "foo")).?;
-        defer testing.allocator.free(v);
-        const e = v[0];
-        testing.expectEqualSlices(u8, "foo", e.name);
-        testing.expectEqualSlices(u8, "bar", e.value);
-        testing.expectEqual(false, e.never_index);
-    }
-    {
-        const v = (try h.get(testing.allocator, "set-cookie")).?;
-        defer testing.allocator.free(v);
-        {
-            const e = v[0];
-            testing.expectEqualSlices(u8, "set-cookie", e.name);
-            testing.expectEqualSlices(u8, "x=1", e.value);
-            testing.expectEqual(true, e.never_index);
-        }
-        {
-            const e = v[1];
-            testing.expectEqualSlices(u8, "set-cookie", e.name);
-            testing.expectEqualSlices(u8, "y=2", e.value);
-            testing.expectEqual(true, e.never_index);
-        }
-    }
-}
-
-test "Headers.getCommaSeparated" {
-    var h = Headers.init(testing.allocator);
-    defer h.deinit();
-    try h.append("foo", "bar", null);
-    try h.append("set-cookie", "x=1", null);
-    try h.append("set-cookie", "y=2", null);
-
-    {
-        const v = try h.getCommaSeparated(testing.allocator, "not-present");
-        testing.expect(null == v);
-    }
-    {
-        const v = (try h.getCommaSeparated(testing.allocator, "foo")).?;
-        defer testing.allocator.free(v);
-        testing.expectEqualSlices(u8, "bar", v);
-    }
-    {
-        const v = (try h.getCommaSeparated(testing.allocator, "set-cookie")).?;
-        defer testing.allocator.free(v);
-        testing.expectEqualSlices(u8, "x=1,y=2", v);
-    }
-}
-
-test "Headers.sort" {
-    var h = Headers.init(testing.allocator);
-    defer h.deinit();
-    try h.append("foo", "bar", null);
-    try h.append("cookie", "somevalue", null);
-
-    h.sort();
-    {
-        const e = h.at(0);
-        testing.expectEqualSlices(u8, "cookie", e.name);
-        testing.expectEqualSlices(u8, "somevalue", e.value);
-        testing.expectEqual(true, e.never_index);
-    }
-    {
-        const e = h.at(1);
-        testing.expectEqualSlices(u8, "foo", e.name);
-        testing.expectEqualSlices(u8, "bar", e.value);
-        testing.expectEqual(false, e.never_index);
-    }
-}
-
-test "Headers.format" {
-    var h = Headers.init(testing.allocator);
-    defer h.deinit();
-    try h.append("foo", "bar", null);
-    try h.append("cookie", "somevalue", null);
-
-    var buf: [100]u8 = undefined;
-    testing.expectEqualSlices(u8,
-        \\foo: bar
-        \\cookie: somevalue
-        \\
-    , try std.fmt.bufPrint(buf[0..], "{}", .{h}));
-}
diff --git a/lib/std/std.zig b/lib/std/std.zig
index 5b12046ed8..e21e428c77 100644
--- a/lib/std/std.zig
+++ b/lib/std/std.zig
@@ -62,7 +62,6 @@ pub const fs = @import("fs.zig");
 pub const hash = @import("hash.zig");
 pub const hash_map = @import("hash_map.zig");
 pub const heap = @import("heap.zig");
-pub const http = @import("http.zig");
 pub const io = @import("io.zig");
 pub const json = @import("json.zig");
 pub const log = @import("log.zig");

From 254ee89def29f67fdf4dd4c6a433ae876f76d79e Mon Sep 17 00:00:00 2001
From: Ryan Liptak 
Date: Tue, 29 Sep 2020 17:44:28 -0700
Subject: [PATCH 202/210] Windows: Handle ACCESS_DENIED in DeviceIoControl

This was causing the Dir.readLink test to fail for me locally with error.Unexpected NTSTATUS=0xc0000022. Not sure if PRIVILEGE_NOT_HELD is actually possible or not.
---
 lib/std/os/windows.zig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig
index de0d0ea45f..a308ee76fc 100644
--- a/lib/std/os/windows.zig
+++ b/lib/std/os/windows.zig
@@ -217,6 +217,7 @@ pub fn DeviceIoControl(
     switch (rc) {
         .SUCCESS => {},
         .PRIVILEGE_NOT_HELD => return error.AccessDenied,
+        .ACCESS_DENIED => return error.AccessDenied,
         .INVALID_PARAMETER => unreachable,
         else => return unexpectedStatus(rc),
     }

From 1572cd4d76dfa35e6a05bbb1405982d7fef6f833 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 18:32:41 -0700
Subject: [PATCH 203/210] different strategy to fix compiler_rt.zig and c.zig

with respect to std.builtin.link_libc.

The commit 27e008eb292038c5a6b9a13b64c7b69d1525f690 did not solve the
problem because although it got std.builtin.link_libc to be true for
compiler_rt.zig and c.zig, it had other unintentional side effects which
broke the build for -lc -target foo-linux-musl.

This commit introduces a new flag to Compilation to allow setting this
comptime flag to true without introducing other side effects to
compilation and linking.
---
 src/Compilation.zig | 29 ++++++++++++++++++-----------
 src/link.zig        |  1 +
 src/link/MachO.zig  |  3 ---
 3 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 1cb02d6f1c..cc1c77e5fa 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -374,6 +374,7 @@ pub const InitOptions = struct {
     is_test: bool = false,
     test_evented_io: bool = false,
     is_compiler_rt_or_libc: bool = false,
+    parent_compilation_link_libc: bool = false,
     stack_size_override: ?u64 = null,
     self_exe_path: ?[]const u8 = null,
     version: ?std.builtin.Version = null,
@@ -614,6 +615,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             hash.add(options.target.os.getVersionRange());
             hash.add(dll_export_fns);
             hash.add(options.is_test);
+            hash.add(options.is_compiler_rt_or_libc);
+            hash.add(options.parent_compilation_link_libc);
 
             const digest = hash.final();
             const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });
@@ -781,6 +784,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .error_return_tracing = error_return_tracing,
             .llvm_cpu_features = llvm_cpu_features,
             .is_compiler_rt_or_libc = options.is_compiler_rt_or_libc,
+            .parent_compilation_link_libc = options.parent_compilation_link_libc,
             .each_lib_rpath = options.each_lib_rpath orelse false,
             .disable_lld_caching = options.disable_lld_caching,
             .subsystem = options.subsystem,
@@ -848,7 +852,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         comp.c_object_table.putAssumeCapacityNoClobber(c_object, {});
     }
 
-    if (comp.bin_file.options.emit != null) {
+    if (comp.bin_file.options.emit != null and !comp.bin_file.options.is_compiler_rt_or_libc) {
         // If we need to build glibc for the target, add work items for it.
         // We go through the work queue so that building can be done in parallel.
         if (comp.wantBuildGLibCFromSource()) {
@@ -903,9 +907,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             try comp.work_queue.writeItem(.libcxx);
             try comp.work_queue.writeItem(.libcxxabi);
         }
-        if (is_exe_or_dyn_lib and !comp.bin_file.options.is_compiler_rt_or_libc and
-            build_options.is_stage1)
-        {
+        if (is_exe_or_dyn_lib and build_options.is_stage1) {
             try comp.work_queue.writeItem(.{ .libcompiler_rt = {} });
             if (!comp.bin_file.options.link_libc) {
                 try comp.work_queue.writeItem(.{ .zig_libc = {} });
@@ -2345,6 +2347,15 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         ),
     }
     try buffer.appendSlice("};\n");
+
+    // This is so that compiler_rt and libc.zig libraries know whether they
+    // will eventually be linked with libc. They make different decisions
+    // about what to export depending on whether another libc will be linked
+    // in. For example, compiler_rt will not export the __chkstk symbol if it
+    // knows libc will provide it, and likewise c.zig will not export memcpy.
+    const link_libc = comp.bin_file.options.link_libc or
+        (comp.bin_file.options.is_compiler_rt_or_libc and comp.bin_file.options.parent_compilation_link_libc);
+
     try buffer.writer().print(
         \\pub const object_format = ObjectFormat.{};
         \\pub const mode = Mode.{};
@@ -2359,7 +2370,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
     , .{
         @tagName(comp.bin_file.options.object_format),
         @tagName(comp.bin_file.options.optimize_mode),
-        comp.bin_file.options.link_libc,
+        link_libc,
         comp.bin_file.options.link_libcpp,
         comp.bin_file.options.error_return_tracing,
         comp.bin_file.options.valgrind,
@@ -2481,6 +2492,7 @@ fn buildStaticLibFromZig(comp: *Compilation, src_basename: []const u8, out: *?CR
         .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
         .clang_passthrough_mode = comp.clang_passthrough_mode,
         .is_compiler_rt_or_libc = true,
+        .parent_compilation_link_libc = comp.bin_file.options.link_libc,
     });
     defer sub_compilation.destroy();
 
@@ -2842,12 +2854,7 @@ pub fn build_crt_file(
         .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
         .clang_passthrough_mode = comp.clang_passthrough_mode,
         .is_compiler_rt_or_libc = true,
-        // This is so that compiler_rt and libc.zig libraries know whether they
-        // will eventually be linked with libc. They make different decisions
-        // about what to export depending on whether another libc will be linked
-        // in. For example, compiler_rt will not export the __chkstk symbol if it
-        // knows libc will provide it, and likewise c.zig will not export memcpy.
-        .link_libc = comp.bin_file.options.link_libc,
+        .parent_compilation_link_libc = comp.bin_file.options.link_libc,
     });
     defer sub_compilation.destroy();
 
diff --git a/src/link.zig b/src/link.zig
index 2d215ea0dc..4d28c8a1a7 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -73,6 +73,7 @@ pub const Options = struct {
     dll_export_fns: bool,
     error_return_tracing: bool,
     is_compiler_rt_or_libc: bool,
+    parent_compilation_link_libc: bool,
     each_lib_rpath: bool,
     disable_lld_caching: bool,
     is_test: bool,
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 2570a92daa..4c0be32922 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -323,9 +323,6 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
     const is_lib = self.base.options.output_mode == .Lib;
     const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib;
     const is_exe_or_dyn_lib = is_dyn_lib or self.base.options.output_mode == .Exe;
-    const have_dynamic_linker = self.base.options.link_libc and
-        self.base.options.link_mode == .Dynamic and is_exe_or_dyn_lib;
-    const link_in_crt = self.base.options.link_libc and self.base.options.output_mode == .Exe;
     const target = self.base.options.target;
     const stack_size = self.base.options.stack_size_override orelse 16777216;
     const allow_shlib_undefined = self.base.options.allow_shlib_undefined orelse !self.base.options.is_native_os;

From 7bffa6f1fb9148aed65f5ca55156dbb0e766c92e Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 21:47:37 -0700
Subject: [PATCH 204/210] clean up call to std.fs.rename

Thanks Ryan Liptak!
---
 src/Compilation.zig | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/Compilation.zig b/src/Compilation.zig
index cc1c77e5fa..623635a6b0 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1627,9 +1627,8 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void {
         const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });
         var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{});
         defer o_dir.close();
-        // TODO https://github.com/ziglang/zig/issues/6344
         const tmp_basename = std.fs.path.basename(out_obj_path);
-        try std.os.renameat(zig_cache_tmp_dir.fd, tmp_basename, o_dir.fd, o_basename);
+        try std.fs.rename(zig_cache_tmp_dir, tmp_basename, o_dir, o_basename);
 
         man.writeManifest() catch |err| {
             log.warn("failed to write cache manifest when compiling '{}': {}", .{ c_object.src.src_path, @errorName(err) });

From 839bdfdc93b1d2a2a4ed6f7c9ca3155323242d82 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 29 Sep 2020 21:48:50 -0700
Subject: [PATCH 205/210] link.Coff: skip redundant (and invalid) copy to same
 path

---
 src/link/Coff.zig | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 2290560509..3f462582e7 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -893,7 +893,11 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
             // regarding eliding redundant object -> object transformations.
             return error.NoObjectsToLink;
         };
-        try fs.cwd().copyFile(the_object_path, fs.cwd(), full_out_path, .{});
+        // This can happen when using --enable-cache and using the stage1 backend. In this case
+        // we can skip the file copy.
+        if (!mem.eql(u8, the_object_path, full_out_path)) {
+            try fs.cwd().copyFile(the_object_path, fs.cwd(), full_out_path, .{});
+        }
     } else {
         // Create an LLD command line and invoke it.
         var argv = std.ArrayList([]const u8).init(self.base.allocator);

From 14d6d07af2ce37569bce8b148599e31664de5f12 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 30 Sep 2020 01:31:08 -0400
Subject: [PATCH 206/210] fix the cli test expected string to support native
 path separators

---
 lib/std/testing.zig | 2 +-
 test/cli.zig        | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/std/testing.zig b/lib/std/testing.zig
index 658d31bb82..ccbd1d6324 100644
--- a/lib/std/testing.zig
+++ b/lib/std/testing.zig
@@ -38,7 +38,7 @@ pub fn expectError(expected_error: anyerror, actual_error_union: anytype) void {
 /// This function is intended to be used only in tests. When the two values are not
 /// equal, prints diagnostics to stderr to show exactly how they are not equal,
 /// then aborts.
-/// The types must match exactly.
+/// `actual` is casted to the type of `expected`.
 pub fn expectEqual(expected: anytype, actual: @TypeOf(expected)) void {
     switch (@typeInfo(@TypeOf(actual))) {
         .NoReturn,
diff --git a/test/cli.zig b/test/cli.zig
index edb87cc750..7a0a7d6459 100644
--- a/test/cli.zig
+++ b/test/cli.zig
@@ -145,7 +145,9 @@ fn testMissingOutputPath(zig_exe: []const u8, dir_path: []const u8) !void {
     const output_arg = try std.fmt.allocPrint(a, "-femit-bin={s}", .{output_path});
     const source_path = try fs.path.join(a, &[_][]const u8{ "src", "main.zig" });
     const result = try exec(dir_path, false, &[_][]const u8{ zig_exe, "build-exe", source_path, output_arg });
-    testing.expect(std.mem.eql(u8, result.stderr, "error: unable to open output directory 'does/not/exist': FileNotFound\n"));
+    const s = std.fs.path.sep_str;
+    const expected: []const u8 = "error: unable to open output directory 'does" ++ s ++ "not" ++ s ++ "exist': FileNotFound\n";
+    testing.expectEqualStrings(expected, result.stderr);
 }
 
 fn testZigFmt(zig_exe: []const u8, dir_path: []const u8) !void {

From bd449b184a0c9fb824184672b12f90ed2698b77a Mon Sep 17 00:00:00 2001
From: Ryan Liptak 
Date: Tue, 29 Sep 2020 17:47:24 -0700
Subject: [PATCH 207/210] Add deleteDir test with exception for not-empty
 directory on Windows

Re-adds the test that was added and then reverted in https://github.com/ziglang/zig/pull/6397, but with the test for https://github.com/ziglang/zig/issues/5537 skipped for now since that issue is no longer fixed.
---
 lib/std/fs/test.zig | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig
index b3cc1fe569..60cd380444 100644
--- a/lib/std/fs/test.zig
+++ b/lib/std/fs/test.zig
@@ -274,6 +274,32 @@ test "file operations on directories" {
     dir.close();
 }
 
+test "deleteDir" {
+    var tmp_dir = tmpDir(.{});
+    defer tmp_dir.cleanup();
+
+    // deleting a non-existent directory
+    testing.expectError(error.FileNotFound, tmp_dir.dir.deleteDir("test_dir"));
+
+    var dir = try tmp_dir.dir.makeOpenPath("test_dir", .{});
+    var file = try dir.createFile("test_file", .{});
+    file.close();
+    dir.close();
+
+    // deleting a non-empty directory
+    // TODO: Re-enable this check on Windows, see https://github.com/ziglang/zig/issues/5537
+    if (builtin.os.tag != .windows) {
+        testing.expectError(error.DirNotEmpty, tmp_dir.dir.deleteDir("test_dir"));
+    }
+
+    dir = try tmp_dir.dir.openDir("test_dir", .{});
+    try dir.deleteFile("test_file");
+    dir.close();
+
+    // deleting an empty directory
+    try tmp_dir.dir.deleteDir("test_dir");
+}
+
 test "Dir.rename files" {
     var tmp_dir = tmpDir(.{});
     defer tmp_dir.cleanup();

From 870af4907437cb55ffe7f0bfaa7b1ab025d928d3 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 30 Sep 2020 00:36:20 -0700
Subject: [PATCH 208/210] std.zig.binNameAlloc: take into account version for
 macos dylibs

---
 lib/std/zig.zig | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/lib/std/zig.zig b/lib/std/zig.zig
index 42a33383b7..28193392a1 100644
--- a/lib/std/zig.zig
+++ b/lib/std/zig.zig
@@ -102,7 +102,9 @@ pub fn binNameAlloc(allocator: *std.mem.Allocator, options: BinNameOptions) erro
                                 target.libPrefix(), root_name, ver.major, ver.minor, ver.patch,
                             });
                         } else {
-                            return std.fmt.allocPrint(allocator, "{s}{s}.so", .{ target.libPrefix(), root_name });
+                            return std.fmt.allocPrint(allocator, "{s}{s}.so", .{
+                                target.libPrefix(), root_name,
+                            });
                         }
                     },
                 }
@@ -112,10 +114,22 @@ pub fn binNameAlloc(allocator: *std.mem.Allocator, options: BinNameOptions) erro
         .macho => switch (options.output_mode) {
             .Exe => return allocator.dupe(u8, root_name),
             .Lib => {
-                const suffix = switch (options.link_mode orelse .Static) {
-                    .Static => ".a",
-                    .Dynamic => ".dylib",
-                };
+                switch (options.link_mode orelse .Static) {
+                    .Static => return std.fmt.allocPrint(allocator, "{s}{s}.a", .{
+                        target.libPrefix(), root_name,
+                    }),
+                    .Dynamic => {
+                        if (options.version) |ver| {
+                            return std.fmt.allocPrint(allocator, "{s}{s}.dylib.{d}.{d}.{d}", .{
+                                target.libPrefix(), root_name, ver.major, ver.minor, ver.patch,
+                            });
+                        } else {
+                            return std.fmt.allocPrint(allocator, "{s}{s}.dylib", .{
+                                target.libPrefix(), root_name,
+                            });
+                        }
+                    },
+                }
                 return std.fmt.allocPrint(allocator, "{s}{s}{s}", .{ target.libPrefix(), root_name, suffix });
             },
             .Obj => return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, target.oFileExt() }),

From 2a893efae12b2f80d0ff5fd883fdaeb00d9425e4 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 30 Sep 2020 00:53:55 -0700
Subject: [PATCH 209/210] fix incorrect dylib filename pattern in the previous
 commit

---
 lib/std/zig.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/std/zig.zig b/lib/std/zig.zig
index 28193392a1..cc1815e8e5 100644
--- a/lib/std/zig.zig
+++ b/lib/std/zig.zig
@@ -120,7 +120,7 @@ pub fn binNameAlloc(allocator: *std.mem.Allocator, options: BinNameOptions) erro
                     }),
                     .Dynamic => {
                         if (options.version) |ver| {
-                            return std.fmt.allocPrint(allocator, "{s}{s}.dylib.{d}.{d}.{d}", .{
+                            return std.fmt.allocPrint(allocator, "{s}{s}.{d}.{d}.{d}.dylib", .{
                                 target.libPrefix(), root_name, ver.major, ver.minor, ver.patch,
                             });
                         } else {

From 3249e5d952cfcecca999391ffc02cce92ff8fcc4 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 30 Sep 2020 01:00:06 -0700
Subject: [PATCH 210/210] MachO: add the same workaround for no -r LLD flag
 support

This is the MachO equivalent for the code added to COFF for doing the
file copy when the input and output are both just one object file.
---
 src/link/MachO.zig | 410 ++++++++++++++++++++++++---------------------
 src/main.zig       |   4 +-
 2 files changed, 218 insertions(+), 196 deletions(-)

diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 4c0be32922..961b64a840 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -319,7 +319,6 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
         break :blk full_obj_path;
     } else null;
 
-    const is_obj = self.base.options.output_mode == .Obj;
     const is_lib = self.base.options.output_mode == .Lib;
     const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib;
     const is_exe_or_dyn_lib = is_dyn_lib or self.base.options.output_mode == .Exe;
@@ -391,215 +390,238 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
         };
     }
 
-    // Create an LLD command line and invoke it.
-    var argv = std.ArrayList([]const u8).init(self.base.allocator);
-    defer argv.deinit();
-    // Even though we're calling LLD as a library it thinks the first argument is its own exe name.
-    try argv.append("lld");
-    if (is_obj) {
-        try argv.append("-r");
-    }
-
-    try argv.append("-error-limit");
-    try argv.append("0");
-
-    try argv.append("-demangle");
-
-    if (self.base.options.rdynamic) {
-        try argv.append("--export-dynamic");
-    }
-
-    try argv.appendSlice(self.base.options.extra_lld_args);
-
-    if (self.base.options.z_nodelete) {
-        try argv.append("-z");
-        try argv.append("nodelete");
-    }
-    if (self.base.options.z_defs) {
-        try argv.append("-z");
-        try argv.append("defs");
-    }
-
-    if (is_dyn_lib) {
-        try argv.append("-static");
-    } else {
-        try argv.append("-dynamic");
-    }
-
-    if (is_dyn_lib) {
-        try argv.append("-dylib");
-
-        if (self.base.options.version) |ver| {
-            const compat_vers = try std.fmt.allocPrint(arena, "{d}.0.0", .{ver.major});
-            try argv.append("-compatibility_version");
-            try argv.append(compat_vers);
-
-            const cur_vers = try std.fmt.allocPrint(arena, "{d}.{d}.{d}", .{ ver.major, ver.minor, ver.patch });
-            try argv.append("-current_version");
-            try argv.append(cur_vers);
-        }
-
-        // TODO getting an error when running an executable when doing this rpath thing
-        //Buf *dylib_install_name = buf_sprintf("@rpath/lib%s.%" ZIG_PRI_usize ".dylib",
-        //    buf_ptr(g->root_out_name), g->version_major);
-        //try argv.append("-install_name");
-        //try argv.append(buf_ptr(dylib_install_name));
-    }
-
-    try argv.append("-arch");
-    try argv.append(darwinArchString(target.cpu.arch));
-
-    switch (target.os.tag) {
-        .macosx => {
-            try argv.append("-macosx_version_min");
-        },
-        .ios, .tvos, .watchos => switch (target.cpu.arch) {
-            .i386, .x86_64 => {
-                try argv.append("-ios_simulator_version_min");
-            },
-            else => {
-                try argv.append("-iphoneos_version_min");
-            },
-        },
-        else => unreachable,
-    }
-    const ver = target.os.version_range.semver.min;
-    const version_string = try std.fmt.allocPrint(arena, "{d}.{d}.{d}", .{ ver.major, ver.minor, ver.patch });
-    try argv.append(version_string);
-
-    try argv.append("-sdk_version");
-    try argv.append(version_string);
-
-    if (target_util.requiresPIE(target) and self.base.options.output_mode == .Exe) {
-        try argv.append("-pie");
-    }
-
     const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
-    try argv.append("-o");
-    try argv.append(full_out_path);
 
-    // rpaths
-    var rpath_table = std.StringHashMap(void).init(self.base.allocator);
-    defer rpath_table.deinit();
-    for (self.base.options.rpath_list) |rpath| {
-        if ((try rpath_table.fetchPut(rpath, {})) == null) {
-            try argv.append("-rpath");
-            try argv.append(rpath);
+    if (self.base.options.output_mode == .Obj) {
+        // LLD's MachO driver does not support the equvialent of `-r` so we do a simple file copy
+        // here. TODO: think carefully about how we can avoid this redundant operation when doing
+        // build-obj. See also the corresponding TODO in linkAsArchive.
+        const the_object_path = blk: {
+            if (self.base.options.objects.len != 0)
+                break :blk self.base.options.objects[0];
+
+            if (comp.c_object_table.count() != 0)
+                break :blk comp.c_object_table.items()[0].key.status.success.object_path;
+
+            if (module_obj_path) |p|
+                break :blk p;
+
+            // TODO I think this is unreachable. Audit this situation when solving the above TODO
+            // regarding eliding redundant object -> object transformations.
+            return error.NoObjectsToLink;
+        };
+        // This can happen when using --enable-cache and using the stage1 backend. In this case
+        // we can skip the file copy.
+        if (!mem.eql(u8, the_object_path, full_out_path)) {
+            try fs.cwd().copyFile(the_object_path, fs.cwd(), full_out_path, .{});
         }
-    }
-    if (is_dyn_lib) {
-        if ((try rpath_table.fetchPut(full_out_path, {})) == null) {
-            try argv.append("-rpath");
-            try argv.append(full_out_path);
+    } else {
+        // Create an LLD command line and invoke it.
+        var argv = std.ArrayList([]const u8).init(self.base.allocator);
+        defer argv.deinit();
+        // Even though we're calling LLD as a library it thinks the first argument is its own exe name.
+        try argv.append("lld");
+
+        try argv.append("-error-limit");
+        try argv.append("0");
+
+        try argv.append("-demangle");
+
+        if (self.base.options.rdynamic) {
+            try argv.append("--export-dynamic");
         }
-    }
 
-    for (self.base.options.lib_dirs) |lib_dir| {
-        try argv.append("-L");
-        try argv.append(lib_dir);
-    }
+        try argv.appendSlice(self.base.options.extra_lld_args);
 
-    // Positional arguments to the linker such as object files.
-    try argv.appendSlice(self.base.options.objects);
+        if (self.base.options.z_nodelete) {
+            try argv.append("-z");
+            try argv.append("nodelete");
+        }
+        if (self.base.options.z_defs) {
+            try argv.append("-z");
+            try argv.append("defs");
+        }
 
-    for (comp.c_object_table.items()) |entry| {
-        try argv.append(entry.key.status.success.object_path);
-    }
-    if (module_obj_path) |p| {
-        try argv.append(p);
-    }
+        if (is_dyn_lib) {
+            try argv.append("-static");
+        } else {
+            try argv.append("-dynamic");
+        }
 
-    // compiler_rt on darwin is missing some stuff, so we still build it and rely on LinkOnce
-    if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) {
-        try argv.append(comp.compiler_rt_static_lib.?.full_object_path);
-    }
+        if (is_dyn_lib) {
+            try argv.append("-dylib");
 
-    // Shared libraries.
-    const system_libs = self.base.options.system_libs.items();
-    try argv.ensureCapacity(argv.items.len + system_libs.len);
-    for (system_libs) |entry| {
-        const link_lib = entry.key;
-        // By this time, we depend on these libs being dynamically linked libraries and not static libraries
-        // (the check for that needs to be earlier), but they could be full paths to .dylib files, in which
-        // case we want to avoid prepending "-l".
-        const ext = Compilation.classifyFileExt(link_lib);
-        const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib});
-        argv.appendAssumeCapacity(arg);
-    }
+            if (self.base.options.version) |ver| {
+                const compat_vers = try std.fmt.allocPrint(arena, "{d}.0.0", .{ver.major});
+                try argv.append("-compatibility_version");
+                try argv.append(compat_vers);
 
-    // libc++ dep
-    if (!is_obj and self.base.options.link_libcpp) {
-        try argv.append(comp.libcxxabi_static_lib.?.full_object_path);
-        try argv.append(comp.libcxx_static_lib.?.full_object_path);
-    }
+                const cur_vers = try std.fmt.allocPrint(arena, "{d}.{d}.{d}", .{ ver.major, ver.minor, ver.patch });
+                try argv.append("-current_version");
+                try argv.append(cur_vers);
+            }
 
-    // On Darwin, libSystem has libc in it, but also you have to use it
-    // to make syscalls because the syscall numbers are not documented
-    // and change between versions. So we always link against libSystem.
-    // LLD craps out if you do -lSystem cross compiling, so until that
-    // codebase gets some love from the new maintainers we're left with
-    // this dirty hack.
-    if (self.base.options.is_native_os) {
-        try argv.append("-lSystem");
-    }
+            // TODO getting an error when running an executable when doing this rpath thing
+            //Buf *dylib_install_name = buf_sprintf("@rpath/lib%s.%" ZIG_PRI_usize ".dylib",
+            //    buf_ptr(g->root_out_name), g->version_major);
+            //try argv.append("-install_name");
+            //try argv.append(buf_ptr(dylib_install_name));
+        }
 
-    for (self.base.options.framework_dirs) |framework_dir| {
-        try argv.append("-F");
-        try argv.append(framework_dir);
-    }
-    for (self.base.options.frameworks) |framework| {
-        try argv.append("-framework");
-        try argv.append(framework);
-    }
+        try argv.append("-arch");
+        try argv.append(darwinArchString(target.cpu.arch));
 
-    if (allow_shlib_undefined) {
-        try argv.append("-undefined");
-        try argv.append("dynamic_lookup");
-    }
-    if (self.base.options.bind_global_refs_locally) {
-        try argv.append("-Bsymbolic");
-    }
+        switch (target.os.tag) {
+            .macosx => {
+                try argv.append("-macosx_version_min");
+            },
+            .ios, .tvos, .watchos => switch (target.cpu.arch) {
+                .i386, .x86_64 => {
+                    try argv.append("-ios_simulator_version_min");
+                },
+                else => {
+                    try argv.append("-iphoneos_version_min");
+                },
+            },
+            else => unreachable,
+        }
+        const ver = target.os.version_range.semver.min;
+        const version_string = try std.fmt.allocPrint(arena, "{d}.{d}.{d}", .{ ver.major, ver.minor, ver.patch });
+        try argv.append(version_string);
 
-    if (self.base.options.verbose_link) {
-        Compilation.dump_argv(argv.items);
-    }
+        try argv.append("-sdk_version");
+        try argv.append(version_string);
 
-    const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null);
-    for (argv.items) |arg, i| {
-        new_argv[i] = try arena.dupeZ(u8, arg);
-    }
+        if (target_util.requiresPIE(target) and self.base.options.output_mode == .Exe) {
+            try argv.append("-pie");
+        }
 
-    var stderr_context: LLDContext = .{
-        .macho = self,
-        .data = std.ArrayList(u8).init(self.base.allocator),
-    };
-    defer stderr_context.data.deinit();
-    var stdout_context: LLDContext = .{
-        .macho = self,
-        .data = std.ArrayList(u8).init(self.base.allocator),
-    };
-    defer stdout_context.data.deinit();
-    const llvm = @import("../llvm.zig");
-    const ok = llvm.Link(
-        .MachO,
-        new_argv.ptr,
-        new_argv.len,
-        append_diagnostic,
-        @ptrToInt(&stdout_context),
-        @ptrToInt(&stderr_context),
-    );
-    if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory;
-    if (stdout_context.data.items.len != 0) {
-        std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items});
-    }
-    if (!ok) {
-        // TODO parse this output and surface with the Compilation API rather than
-        // directly outputting to stderr here.
-        std.debug.print("{}", .{stderr_context.data.items});
-        return error.LLDReportedFailure;
-    }
-    if (stderr_context.data.items.len != 0) {
-        std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items});
+        try argv.append("-o");
+        try argv.append(full_out_path);
+
+        // rpaths
+        var rpath_table = std.StringHashMap(void).init(self.base.allocator);
+        defer rpath_table.deinit();
+        for (self.base.options.rpath_list) |rpath| {
+            if ((try rpath_table.fetchPut(rpath, {})) == null) {
+                try argv.append("-rpath");
+                try argv.append(rpath);
+            }
+        }
+        if (is_dyn_lib) {
+            if ((try rpath_table.fetchPut(full_out_path, {})) == null) {
+                try argv.append("-rpath");
+                try argv.append(full_out_path);
+            }
+        }
+
+        for (self.base.options.lib_dirs) |lib_dir| {
+            try argv.append("-L");
+            try argv.append(lib_dir);
+        }
+
+        // Positional arguments to the linker such as object files.
+        try argv.appendSlice(self.base.options.objects);
+
+        for (comp.c_object_table.items()) |entry| {
+            try argv.append(entry.key.status.success.object_path);
+        }
+        if (module_obj_path) |p| {
+            try argv.append(p);
+        }
+
+        // compiler_rt on darwin is missing some stuff, so we still build it and rely on LinkOnce
+        if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) {
+            try argv.append(comp.compiler_rt_static_lib.?.full_object_path);
+        }
+
+        // Shared libraries.
+        const system_libs = self.base.options.system_libs.items();
+        try argv.ensureCapacity(argv.items.len + system_libs.len);
+        for (system_libs) |entry| {
+            const link_lib = entry.key;
+            // By this time, we depend on these libs being dynamically linked libraries and not static libraries
+            // (the check for that needs to be earlier), but they could be full paths to .dylib files, in which
+            // case we want to avoid prepending "-l".
+            const ext = Compilation.classifyFileExt(link_lib);
+            const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib});
+            argv.appendAssumeCapacity(arg);
+        }
+
+        // libc++ dep
+        if (self.base.options.link_libcpp) {
+            try argv.append(comp.libcxxabi_static_lib.?.full_object_path);
+            try argv.append(comp.libcxx_static_lib.?.full_object_path);
+        }
+
+        // On Darwin, libSystem has libc in it, but also you have to use it
+        // to make syscalls because the syscall numbers are not documented
+        // and change between versions. So we always link against libSystem.
+        // LLD craps out if you do -lSystem cross compiling, so until that
+        // codebase gets some love from the new maintainers we're left with
+        // this dirty hack.
+        if (self.base.options.is_native_os) {
+            try argv.append("-lSystem");
+        }
+
+        for (self.base.options.framework_dirs) |framework_dir| {
+            try argv.append("-F");
+            try argv.append(framework_dir);
+        }
+        for (self.base.options.frameworks) |framework| {
+            try argv.append("-framework");
+            try argv.append(framework);
+        }
+
+        if (allow_shlib_undefined) {
+            try argv.append("-undefined");
+            try argv.append("dynamic_lookup");
+        }
+        if (self.base.options.bind_global_refs_locally) {
+            try argv.append("-Bsymbolic");
+        }
+
+        if (self.base.options.verbose_link) {
+            Compilation.dump_argv(argv.items);
+        }
+
+        const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null);
+        for (argv.items) |arg, i| {
+            new_argv[i] = try arena.dupeZ(u8, arg);
+        }
+
+        var stderr_context: LLDContext = .{
+            .macho = self,
+            .data = std.ArrayList(u8).init(self.base.allocator),
+        };
+        defer stderr_context.data.deinit();
+        var stdout_context: LLDContext = .{
+            .macho = self,
+            .data = std.ArrayList(u8).init(self.base.allocator),
+        };
+        defer stdout_context.data.deinit();
+        const llvm = @import("../llvm.zig");
+        const ok = llvm.Link(
+            .MachO,
+            new_argv.ptr,
+            new_argv.len,
+            append_diagnostic,
+            @ptrToInt(&stdout_context),
+            @ptrToInt(&stderr_context),
+        );
+        if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory;
+        if (stdout_context.data.items.len != 0) {
+            std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items});
+        }
+        if (!ok) {
+            // TODO parse this output and surface with the Compilation API rather than
+            // directly outputting to stderr here.
+            std.debug.print("{}", .{stderr_context.data.items});
+            return error.LLDReportedFailure;
+        }
+        if (stderr_context.data.items.len != 0) {
+            std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items});
+        }
     }
 
     if (!self.base.options.disable_lld_caching) {
diff --git a/src/main.zig b/src/main.zig
index 822e777d68..d421322c17 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -1337,12 +1337,12 @@ fn buildOutputType(
         }
     };
 
-    if (output_mode == .Obj and object_format == .coff) {
+    if (output_mode == .Obj and (object_format == .coff or object_format == .macho)) {
         const total_obj_count = c_source_files.items.len +
             @boolToInt(root_src_file != null) +
             link_objects.items.len;
         if (total_obj_count > 1) {
-            fatal("COFF does not support linking multiple objects into one", .{});
+            fatal("{s} does not support linking multiple objects into one", .{@tagName(object_format)});
         }
     }