From 91955dee587214722daa09e6f3dbff059ac3752e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 8 Mar 2019 22:53:35 -0500 Subject: [PATCH 1/5] breaking changes to zig build API and improved caching * in Zig build scripts, getOutputPath() is no longer a valid function to call, unless setOutputDir() was used, or within a custom make() function. Instead there is more convenient API to use which takes advantage of the caching system. Search this commit diff for `exe.run()` for an example. * Zig build by default enables caching. All build artifacts will go into zig-cache. If you want to access build artifacts in a convenient location, it is recommended to add an `install` step. Otherwise you can use the `run()` API mentioned above to execute programs directly from their location in the cache. Closes #330. `addSystemCommand` is available for programs not built with Zig build. * Please note that Zig does no cache evicting yet. You may have to manually delete zig-cache directories periodically to keep disk usage down. It's planned for this to be a simple Least Recently Used eviction system eventually. * `--output`, `--output-lib`, and `--output-h` are removed. Instead, use `--output-dir` which defaults to the current working directory. Or take advantage of `--cache on`, which will print the main output path to stdout, and the other artifacts will be in the same directory with predictable file names. `--disable-gen-h` is available when one wants to prevent .h file generation. * `@cImport` is always independently cached now. Closes #2015. It always writes the generated Zig code to disk which makes debug info and compile errors better. No more "TODO: remember C source location to display here" * Fix .d file parsing. (Fixes the MacOS CI failure) * Zig no longer creates "temporary files" other than inside a zig-cache directory. This breaks the CLI API that Godbolt uses. The suggested new invocation can be found in this commit diff, in the changes to `test/cli.zig`. --- build.zig | 4 +- doc/langref.html.in | 3 +- example/mix_o_files/build.zig | 5 +- example/shared_library/build.zig | 3 +- src/all_types.hpp | 11 +- src/analyze.cpp | 31 +- src/analyze.hpp | 5 +- src/ast_render.cpp | 3 + src/cache_hash.cpp | 49 +- src/cache_hash.hpp | 6 + src/codegen.cpp | 249 +++++----- src/codegen.hpp | 6 +- src/ir.cpp | 164 +++++-- src/link.cpp | 28 +- src/main.cpp | 86 ++-- src/os.cpp | 88 +--- src/os.hpp | 6 +- src/target.cpp | 6 +- src/translate_c.cpp | 48 +- src/translate_c.hpp | 7 +- std/build.zig | 453 +++++++++++------- std/special/init-exe/build.zig | 4 +- test/cli.zig | 6 +- .../standalone/load_dynamic_library/build.zig | 8 +- test/standalone/pkg_import/build.zig | 3 +- test/tests.zig | 82 ++-- 26 files changed, 697 insertions(+), 667 deletions(-) diff --git a/build.zig b/build.zig index 6acf08a0e5..2dc9c671ec 100644 --- a/build.zig +++ b/build.zig @@ -20,8 +20,8 @@ pub fn build(b: *Builder) !void { b.allocator, [][]const u8{ b.cache_root, "langref.html" }, ) catch unreachable; - var docgen_cmd = b.addCommand(null, b.env_map, [][]const u8{ - docgen_exe.getOutputPath(), + var docgen_cmd = docgen_exe.run(); + docgen_cmd.addArgs([][]const u8{ rel_zig_exe, "doc" ++ os.path.sep_str ++ "langref.html.in", langref_out_path, diff --git a/doc/langref.html.in b/doc/langref.html.in index b816d65289..39d7e4449c 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -7914,8 +7914,7 @@ pub fn build(b: *Builder) void { b.default_step.dependOn(&exe.step); - const run_cmd = b.addCommand(".", b.env_map, [][]const u8{exe.getOutputPath()}); - run_cmd.step.dependOn(&exe.step); + const run_cmd = exe.run(); const test_step = b.step("test", "Test the program"); test_step.dependOn(&run_cmd.step); diff --git a/example/mix_o_files/build.zig b/example/mix_o_files/build.zig index 623ec63de9..c945ce6bd6 100644 --- a/example/mix_o_files/build.zig +++ b/example/mix_o_files/build.zig @@ -4,14 +4,13 @@ pub fn build(b: *Builder) void { const obj = b.addObject("base64", "base64.zig"); const exe = b.addExecutable("test", null); - exe.addCSourceFile("test.c",[][]const u8{"-std=c99"}); + exe.addCSourceFile("test.c", [][]const u8{"-std=c99"}); exe.addObject(obj); exe.linkSystemLibrary("c"); b.default_step.dependOn(&exe.step); - const run_cmd = b.addCommand(".", b.env_map, [][]const u8{exe.getOutputPath()}); - run_cmd.step.dependOn(&exe.step); + const run_cmd = exe.run(); const test_step = b.step("test", "Test the program"); test_step.dependOn(&run_cmd.step); diff --git a/example/shared_library/build.zig b/example/shared_library/build.zig index 5eaa4f4402..37af059ae0 100644 --- a/example/shared_library/build.zig +++ b/example/shared_library/build.zig @@ -10,8 +10,7 @@ pub fn build(b: *Builder) void { b.default_step.dependOn(&exe.step); - const run_cmd = b.addCommand(".", b.env_map, [][]const u8{exe.getOutputPath()}); - run_cmd.step.dependOn(&exe.step); + const run_cmd = exe.run(); const test_step = b.step("test", "Test the program"); test_step.dependOn(&run_cmd.step); diff --git a/src/all_types.hpp b/src/all_types.hpp index c6826664ef..ec14382a2b 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1087,7 +1087,6 @@ struct RootStruct { Buf *path; // relative to root_package->root_src_dir ZigList *line_offsets; Buf *source_code; - AstNode *c_import_node; ZigLLVMDIFile *di_file; }; @@ -1746,13 +1745,12 @@ struct CodeGen { Buf triple_str; Buf global_asm; - Buf *out_h_path; - Buf *out_lib_path; - Buf artifact_dir; Buf output_file_path; Buf o_file_output_path; - Buf *wanted_output_file_path; Buf *cache_dir; + // As an input parameter, mutually exclusive with enable_cache. But it gets + // populated in codegen_build_and_link. + Buf *output_dir; Buf **libc_include_dir_list; size_t libc_include_dir_len; @@ -1804,7 +1802,7 @@ struct CodeGen { bool verbose_cc; bool error_during_imports; bool generate_error_name_table; - bool enable_cache; + bool enable_cache; // mutually exclusive with output_dir bool enable_time_report; bool system_linker_hack; bool reported_bad_link_libc_error; @@ -1844,6 +1842,7 @@ struct CodeGen { bool each_lib_rpath; bool disable_pic; bool is_dummy_so; + bool disable_gen_h; Buf *mmacosx_version_min; Buf *mios_version_min; diff --git a/src/analyze.cpp b/src/analyze.cpp index 366bb2963e..6cbcc311ef 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -35,20 +35,6 @@ static bool is_top_level_struct(ZigType *import) { static ErrorMsg *add_error_note_token(CodeGen *g, ErrorMsg *parent_msg, ZigType *owner, Token *token, Buf *msg) { assert(is_top_level_struct(owner)); RootStruct *root_struct = owner->data.structure.root_struct; - if (root_struct->c_import_node != nullptr) { - // if this happens, then translate_c generated code that - // failed semantic analysis, which isn't supposed to happen - - Buf *note_path = buf_create_from_str("?.c"); - Buf *note_source = buf_create_from_str("TODO: remember C source location to display here "); - ZigList note_line_offsets = {0}; - note_line_offsets.append(0); - ErrorMsg *note = err_msg_create_with_line(note_path, 0, 0, - note_source, ¬e_line_offsets, msg); - - err_msg_add_note(parent_msg, note); - return note; - } ErrorMsg *err = err_msg_create_with_line(root_struct->path, token->start_line, token->start_column, root_struct->source_code, root_struct->line_offsets, msg); @@ -60,17 +46,6 @@ static ErrorMsg *add_error_note_token(CodeGen *g, ErrorMsg *parent_msg, ZigType ErrorMsg *add_token_error(CodeGen *g, ZigType *owner, Token *token, Buf *msg) { assert(is_top_level_struct(owner)); RootStruct *root_struct = owner->data.structure.root_struct; - if (root_struct->c_import_node != nullptr) { - // if this happens, then translate_c generated code that - // failed semantic analysis, which isn't supposed to happen - ErrorMsg *err = add_node_error(g, root_struct->c_import_node, - buf_sprintf("compiler bug: @cImport generated invalid zig code")); - - add_error_note_token(g, err, owner, token, msg); - - g->errors.append(err); - return err; - } ErrorMsg *err = err_msg_create_with_line(root_struct->path, token->start_line, token->start_column, root_struct->source_code, root_struct->line_offsets, msg); @@ -1300,7 +1275,7 @@ static ZigTypeId container_to_type(ContainerKind kind) { } // This is like get_partial_container_type except it's for the implicit root struct of files. -ZigType *get_root_container_type(CodeGen *g, const char *full_name, Buf *bare_name, +static ZigType *get_root_container_type(CodeGen *g, const char *full_name, Buf *bare_name, RootStruct *root_struct) { ZigType *entry = new_type_table_entry(ZigTypeIdStruct); @@ -4503,11 +4478,11 @@ ZigType *add_source_file(CodeGen *g, ZigPackage *package, Buf *resolved_path, Bu Buf *pkg_root_src_dir = &package->root_src_dir; Buf resolved_root_src_dir = os_path_resolve(&pkg_root_src_dir, 1); - assert(buf_starts_with_buf(resolved_path, &resolved_root_src_dir)); - Buf namespace_name = BUF_INIT; buf_init_from_buf(&namespace_name, &package->pkg_path); if (source_kind == SourceKindNonRoot) { + assert(buf_starts_with_buf(resolved_path, &resolved_root_src_dir)); + if (buf_len(&namespace_name) != 0) buf_append_char(&namespace_name, NAMESPACE_SEP_CHAR); buf_append_mem(&namespace_name, buf_ptr(&noextname) + buf_len(&resolved_root_src_dir) + 1, buf_len(&noextname) - (buf_len(&resolved_root_src_dir) + 1)); diff --git a/src/analyze.hpp b/src/analyze.hpp index aa94f0c2c5..22e44ebd3f 100644 --- a/src/analyze.hpp +++ b/src/analyze.hpp @@ -31,8 +31,6 @@ ZigType *get_array_type(CodeGen *g, ZigType *child_type, uint64_t array_size); ZigType *get_slice_type(CodeGen *g, ZigType *ptr_type); ZigType *get_partial_container_type(CodeGen *g, Scope *scope, ContainerKind kind, AstNode *decl_node, const char *full_name, Buf *bare_name, ContainerLayout layout); -ZigType *get_root_container_type(CodeGen *g, const char *full_name, Buf *bare_name, - RootStruct *root_struct); ZigType *get_smallest_unsigned_int_type(CodeGen *g, uint64_t x); ZigType *get_error_union_type(CodeGen *g, ZigType *err_set_type, ZigType *payload_type); ZigType *get_bound_fn_type(CodeGen *g, ZigFn *fn_entry); @@ -53,6 +51,7 @@ enum SourceKind { SourceKindRoot, SourceKindPkgMain, SourceKindNonRoot, + SourceKindCImport, }; ZigType *add_source_file(CodeGen *g, ZigPackage *package, Buf *abs_full_path, Buf *source_code, SourceKind source_kind); @@ -242,4 +241,6 @@ Error ensure_const_val_repr(IrAnalyze *ira, CodeGen *codegen, AstNode *source_no void typecheck_panic_fn(CodeGen *g, TldFn *tld_fn, ZigFn *panic_fn); Buf *type_bare_name(ZigType *t); Buf *type_h_name(ZigType *t); +Error create_c_object_cache(CodeGen *g, CacheHash **out_cache_hash, bool verbose); + #endif diff --git a/src/ast_render.cpp b/src/ast_render.cpp index 7b57841205..76a0622058 100644 --- a/src/ast_render.cpp +++ b/src/ast_render.cpp @@ -470,6 +470,9 @@ static void render_node_extra(AstRender *ar, AstNode *node, bool grouped) { fprintf(ar->f, ", "); } } + if (node->data.fn_proto.is_var_args) { + fprintf(ar->f, ", ..."); + } fprintf(ar->f, ")"); if (node->data.fn_proto.align_expr) { fprintf(ar->f, " align("); diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index e9d4bdc671..cd336b71ae 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -19,6 +19,8 @@ void cache_init(CacheHash *ch, Buf *manifest_dir) { ch->manifest_dir = manifest_dir; ch->manifest_file_path = nullptr; ch->manifest_dirty = false; + ch->force_check_manifest = false; + ch->b64_digest = BUF_INIT; } void cache_str(CacheHash *ch, const char *ptr) { @@ -243,22 +245,21 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { int rc = blake2b_final(&ch->blake, bin_digest, 48); assert(rc == 0); - if (ch->files.length == 0) { + buf_resize(&ch->b64_digest, 64); + base64_encode(buf_to_slice(&ch->b64_digest), {bin_digest, 48}); + + if (ch->files.length == 0 && !ch->force_check_manifest) { buf_resize(out_digest, 64); base64_encode(buf_to_slice(out_digest), {bin_digest, 48}); return ErrorNone; } - Buf b64_digest = BUF_INIT; - buf_resize(&b64_digest, 64); - base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); - rc = blake2b_init(&ch->blake, 48); assert(rc == 0); blake2b_update(&ch->blake, bin_digest, 48); ch->manifest_file_path = buf_alloc(); - os_path_join(ch->manifest_dir, &b64_digest, ch->manifest_file_path); + os_path_join(ch->manifest_dir, &ch->b64_digest, ch->manifest_file_path); buf_append_str(ch->manifest_file_path, ".txt"); @@ -380,7 +381,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { blake2b_update(&ch->blake, chf->bin_digest, 48); } } - if (file_i < input_file_count) { + if (file_i < input_file_count || file_i == 0) { // manifest file is empty or missing entries, so this is a cache miss ch->manifest_dirty = true; for (; file_i < input_file_count; file_i += 1) { @@ -442,6 +443,7 @@ Error cache_add_dep_file(CacheHash *ch, Buf *dep_file_path, bool verbose) { } if (opt_line.value.len == 0) continue; + if (opt_line.value.ptr[0] == '"') { if (opt_line.value.len < 2) { if (verbose) { @@ -460,21 +462,28 @@ Error cache_add_dep_file(CacheHash *ch, Buf *dep_file_path, bool verbose) { } return ErrorInvalidDepFile; } + Buf *filename_buf = buf_create_from_slice(opt_line.value); + if ((err = cache_add_file(ch, filename_buf))) { + if (verbose) { + fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(filename_buf), err_str(err)); + fprintf(stderr, "when processing .d file: %s\n", buf_ptr(dep_file_path)); + } + return err; + } } else { - if (opt_line.value.ptr[opt_line.value.len - 1] == '\\') { - opt_line.value.len -= 2; // cut off ` \` + // sometimes there are multiple files on the same line; we actually need space tokenization. + SplitIterator line_it = memSplit(opt_line.value, str(" \t\\")); + Slice filename; + while (SplitIterator_next(&line_it).unwrap(&filename)) { + Buf *filename_buf = buf_create_from_slice(filename); + if ((err = cache_add_file(ch, filename_buf))) { + if (verbose) { + fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(filename_buf), err_str(err)); + fprintf(stderr, "when processing .d file: %s\n", buf_ptr(dep_file_path)); + } + return err; + } } - if (opt_line.value.len == 0) - continue; - } - - Buf *filename_buf = buf_create_from_slice(opt_line.value); - if ((err = cache_add_file(ch, filename_buf))) { - if (verbose) { - fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(filename_buf), err_str(err)); - fprintf(stderr, "when processing .d file: %s\n", buf_ptr(dep_file_path)); - } - return err; } } return ErrorNone; diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp index d74c8623ce..5be02ea405 100644 --- a/src/cache_hash.hpp +++ b/src/cache_hash.hpp @@ -25,8 +25,10 @@ struct CacheHash { ZigList files; Buf *manifest_dir; Buf *manifest_file_path; + Buf b64_digest; OsFile manifest_file; bool manifest_dirty; + bool force_check_manifest; }; // Always call this first to set up. @@ -51,6 +53,10 @@ void cache_file_opt(CacheHash *ch, Buf *path); // If you got a cache hit, the next step is cache_release. // From this point on, there is a lock on the input params. Release // the lock with cache_release. +// Set force_check_manifest if you plan to add files later, but have not +// added any files before calling cache_hit. CacheHash::b64_digest becomes +// available for use after this call, even in the case of a miss, and it +// is a hash of the input parameters only. Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest); // If you did not get a cache hit, call this function for every file diff --git a/src/codegen.cpp b/src/codegen.cpp index 87974b7778..30b1c87662 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -23,6 +23,9 @@ #include #include +#define CACHE_OUT_SUBDIR "o" +#define CACHE_HASH_SUBDIR "h" + static void init_darwin_native(CodeGen *g) { char *osx_target = getenv("MACOSX_DEPLOYMENT_TARGET"); char *ios_target = getenv("IPHONEOS_DEPLOYMENT_TARGET"); @@ -196,18 +199,6 @@ CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget return g; } -void codegen_set_output_h_path(CodeGen *g, Buf *h_path) { - g->out_h_path = h_path; -} - -void codegen_set_output_lib_path(CodeGen *g, Buf *lib_path) { - g->out_lib_path = lib_path; -} - -void codegen_set_output_path(CodeGen *g, Buf *path) { - g->wanted_output_file_path = path; -} - void codegen_set_clang_argv(CodeGen *g, const char **args, size_t len) { g->clang_argv = args; g->clang_argv_len = len; @@ -256,10 +247,6 @@ void codegen_set_out_name(CodeGen *g, Buf *out_name) { g->root_out_name = out_name; } -void codegen_set_dynamic_linker(CodeGen *g, Buf *dynamic_linker_path) { - g->dynamic_linker_path = dynamic_linker_path; -} - void codegen_add_lib_dir(CodeGen *g, const char *dir) { g->lib_dirs.append(dir); } @@ -7766,7 +7753,10 @@ static Error define_builtin_compile_vars(CodeGen *g) { } } else { contents = codegen_generate_builtin_source(g); - os_write_file(builtin_zig_path, contents); + if ((err = os_write_file(builtin_zig_path, contents))) { + fprintf(stderr, "Unable to write file '%s': %s\n", buf_ptr(builtin_zig_path), err_str(err)); + exit(1); + } } assert(g->root_package); @@ -8030,7 +8020,7 @@ static void detect_libc(CodeGen *g) { } } -void codegen_translate_c(CodeGen *g, Buf *full_path) { +AstNode *codegen_translate_c(CodeGen *g, Buf *full_path) { Buf *src_basename = buf_alloc(); Buf *src_dirname = buf_alloc(); os_path_split(full_path, src_dirname, src_basename); @@ -8042,16 +8032,9 @@ void codegen_translate_c(CodeGen *g, Buf *full_path) { init(g); - RootStruct *root_struct = allocate(1); - root_struct->source_code = nullptr; - root_struct->path = full_path; - root_struct->di_file = ZigLLVMCreateFile(g->dbuilder, buf_ptr(src_basename), buf_ptr(src_dirname)); - - ZigType *import = get_root_container_type(g, buf_ptr(&noextname), &noextname, root_struct); - g->root_import = import; - ZigList errors = {0}; - Error err = parse_h_file(import, &errors, buf_ptr(full_path), g, nullptr); + AstNode *root_node; + Error err = parse_h_file(&root_node, &errors, buf_ptr(full_path), g, nullptr); if (err == ErrorCCompileErrors && errors.length > 0) { for (size_t i = 0; i < errors.length; i += 1) { @@ -8065,6 +8048,8 @@ void codegen_translate_c(CodeGen *g, Buf *full_path) { fprintf(stderr, "unable to parse C file: %s\n", err_str(err)); exit(1); } + + return root_node; } static ZigType *add_special_code(CodeGen *g, ZigPackage *package, const char *basename) { @@ -8282,32 +8267,18 @@ static Error get_tmp_filename(CodeGen *g, Buf *out, Buf *suffix) { return ErrorNone; } -// returns true if it was a cache miss -static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) { +Error create_c_object_cache(CodeGen *g, CacheHash **out_cache_hash, bool verbose) { Error err; - - Buf *artifact_dir; - Buf *o_final_path; - - Buf *o_dir = buf_alloc(); - os_path_join(g->cache_dir, buf_create_from_str("o"), o_dir); - - Buf *c_source_file = buf_create_from_str(c_file->source_path); - Buf *c_source_basename = buf_alloc(); - os_path_split(c_source_file, nullptr, c_source_basename); - Buf *final_o_basename = buf_alloc(); - os_path_extname(c_source_basename, final_o_basename, nullptr); - buf_append_str(final_o_basename, target_o_file_ext(g->zig_target)); - CacheHash *cache_hash = allocate(1); - Buf *manifest_dir = buf_alloc(); - os_path_join(g->cache_dir, buf_create_from_str("c"), manifest_dir); + Buf *manifest_dir = buf_sprintf("%s" OS_SEP CACHE_HASH_SUBDIR, buf_ptr(g->cache_dir)); cache_init(cache_hash, manifest_dir); Buf *compiler_id; if ((err = get_compiler_id(&compiler_id))) { - fprintf(stderr, "unable to get compiler id: %s\n", err_str(err)); - exit(1); + if (verbose) { + fprintf(stderr, "unable to get compiler id: %s\n", err_str(err)); + } + return err; } cache_buf(cache_hash, compiler_id); cache_int(cache_hash, g->err_color); @@ -8321,11 +8292,38 @@ static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) { cache_int(cache_hash, g->zig_target->abi); cache_bool(cache_hash, g->strip_debug_symbols); cache_int(cache_hash, g->build_mode); - cache_file(cache_hash, c_source_file); cache_bool(cache_hash, g->disable_pic); for (size_t arg_i = 0; arg_i < g->clang_argv_len; arg_i += 1) { cache_str(cache_hash, g->clang_argv[arg_i]); } + + *out_cache_hash = cache_hash; + return ErrorNone; +} + +// returns true if it was a cache miss +static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) { + Error err; + + Buf *artifact_dir; + Buf *o_final_path; + + Buf *o_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR, buf_ptr(g->cache_dir)); + + Buf *c_source_file = buf_create_from_str(c_file->source_path); + Buf *c_source_basename = buf_alloc(); + os_path_split(c_source_file, nullptr, c_source_basename); + Buf *final_o_basename = buf_alloc(); + os_path_extname(c_source_basename, final_o_basename, nullptr); + buf_append_str(final_o_basename, target_o_file_ext(g->zig_target)); + + CacheHash *cache_hash; + if ((err = create_c_object_cache(g, &cache_hash, true))) { + // Already printed error; verbose = true + exit(1); + } + cache_file(cache_hash, c_source_file); + // Note: not directory args, just args that always have a file next static const char *file_args[] = { "-include", @@ -8787,11 +8785,13 @@ static void gen_h_file(CodeGen *g) { GenH *gen_h = &gen_h_data; assert(!g->is_test_build); - assert(g->out_h_path != nullptr); + assert(!g->disable_gen_h); - FILE *out_h = fopen(buf_ptr(g->out_h_path), "wb"); + Buf *out_h_path = buf_sprintf("%s" OS_SEP "%s.h", buf_ptr(g->output_dir), buf_ptr(g->root_out_name)); + + FILE *out_h = fopen(buf_ptr(out_h_path), "wb"); if (!out_h) - zig_panic("unable to open %s: %s\n", buf_ptr(g->out_h_path), strerror(errno)); + zig_panic("unable to open %s: %s\n", buf_ptr(out_h_path), strerror(errno)); Buf *export_macro = preprocessor_mangle(buf_sprintf("%s_EXPORT", buf_ptr(g->root_out_name))); buf_upcase(export_macro); @@ -9065,6 +9065,7 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { cache_bool(ch, g->linker_rdynamic); cache_bool(ch, g->each_lib_rpath); cache_bool(ch, g->disable_pic); + cache_bool(ch, g->disable_gen_h); cache_bool(ch, g->valgrind_support); cache_bool(ch, g->is_dummy_so); cache_buf_opt(ch, g->mmacosx_version_min); @@ -9103,99 +9104,83 @@ static bool need_llvm_module(CodeGen *g) { return g->assembly_files.length != 0 || buf_len(&g->root_package->root_src_path) != 0; } -static bool compilation_is_already_done(CodeGen *g) { - return !need_llvm_module(g) && g->link_objects.length == 1 && g->out_type == OutTypeObj; -} - static void resolve_out_paths(CodeGen *g) { - Buf *o_basename = buf_create_from_buf(g->root_out_name); + assert(g->output_dir != nullptr); + assert(g->root_out_name != nullptr); + Buf *out_basename = buf_create_from_buf(g->root_out_name); + Buf *o_basename = buf_create_from_buf(g->root_out_name); switch (g->emit_file_type) { - case EmitFileTypeBinary: - { - const char *o_ext = target_o_file_ext(g->zig_target); - buf_append_str(o_basename, o_ext); + case EmitFileTypeBinary: { + switch (g->out_type) { + case OutTypeUnknown: + zig_unreachable(); + case OutTypeObj: + if (g->enable_cache && g->link_objects.length == 1 && !need_llvm_module(g)) { + buf_init_from_buf(&g->output_file_path, g->link_objects.at(0)); + return; + } + if (!need_llvm_module(g) || (g->enable_cache && g->link_objects.length == 0)) { + // Either we're not creating an object file from our LLVM Module, + // or we have caching enabled and do not need to link objects together. + // In both cases, the output file path and object file path basename can match. + buf_append_str(o_basename, target_o_file_ext(g->zig_target)); + buf_append_str(out_basename, target_o_file_ext(g->zig_target)); + } else { + // make it not collide with main output object + buf_append_str(o_basename, ".root"); + buf_append_str(o_basename, target_o_file_ext(g->zig_target)); + buf_append_str(out_basename, target_o_file_ext(g->zig_target)); + } + break; + case OutTypeExe: + buf_append_str(o_basename, target_o_file_ext(g->zig_target)); + buf_append_str(out_basename, target_exe_file_ext(g->zig_target)); + break; + case OutTypeLib: + buf_append_str(o_basename, target_o_file_ext(g->zig_target)); + buf_resize(out_basename, 0); + buf_append_str(out_basename, target_lib_file_prefix(g->zig_target)); + buf_append_buf(out_basename, g->root_out_name); + buf_append_str(out_basename, target_lib_file_ext(g->zig_target, g->is_static, + g->version_major, g->version_minor, g->version_patch)); + break; + } break; } - case EmitFileTypeAssembly: - { + case EmitFileTypeAssembly: { const char *asm_ext = target_asm_file_ext(g->zig_target); buf_append_str(o_basename, asm_ext); + buf_append_str(out_basename, asm_ext); break; } - case EmitFileTypeLLVMIr: - { + case EmitFileTypeLLVMIr: { const char *llvm_ir_ext = target_llvm_ir_file_ext(g->zig_target); buf_append_str(o_basename, llvm_ir_ext); + buf_append_str(out_basename, llvm_ir_ext); break; } - default: - zig_unreachable(); } - if (compilation_is_already_done(g)) { - buf_init_from_str(&g->o_file_output_path, buf_ptr(g->link_objects.at(0))); - } else if (g->enable_cache || g->out_type != OutTypeObj) { - os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path); - } else if (g->wanted_output_file_path != nullptr && g->out_type == OutTypeObj) { - buf_init_from_buf(&g->o_file_output_path, g->wanted_output_file_path); - } else { - buf_init_from_buf(&g->o_file_output_path, o_basename); - } - - if (!g->enable_cache && g->wanted_output_file_path != nullptr) { - buf_init_from_buf(&g->output_file_path, g->wanted_output_file_path); - return; - } - - if (compilation_is_already_done(g)) { - buf_init_from_buf(&g->output_file_path, &g->o_file_output_path); - return; - } - - const char *prefix = ""; - const char *extname; - switch (g->out_type) { - case OutTypeUnknown: - zig_unreachable(); - case OutTypeObj: - extname = target_o_file_ext(g->zig_target); - break; - case OutTypeExe: - extname = target_exe_file_ext(g->zig_target); - break; - case OutTypeLib: - prefix = target_lib_file_prefix(g->zig_target); - extname = target_lib_file_ext(g->zig_target, g->is_static, - g->version_major, g->version_minor, g->version_patch); - break; - } - - assert(g->root_out_name); - - Buf basename = BUF_INIT; - buf_init_from_str(&basename, prefix); - buf_append_buf(&basename, g->root_out_name); - buf_append_str(&basename, extname); - if (g->enable_cache || g->is_test_build) { - os_path_join(&g->artifact_dir, &basename, &g->output_file_path); - } else { - buf_init_from_buf(&g->output_file_path, &basename); - } + os_path_join(g->output_dir, o_basename, &g->o_file_output_path); + os_path_join(g->output_dir, out_basename, &g->output_file_path); } void codegen_build_and_link(CodeGen *g) { Error err; assert(g->out_type != OutTypeUnknown); + if (!g->enable_cache && g->output_dir == nullptr) { + g->output_dir = buf_create_from_str("."); + } + detect_libc(g); detect_dynamic_linker(g); - Buf *artifact_dir = buf_alloc(); Buf digest = BUF_INIT; if (g->enable_cache) { Buf *manifest_dir = buf_alloc(); - os_path_join(g->cache_dir, buf_create_from_str("h"), manifest_dir); + os_path_join(g->cache_dir, buf_create_from_str(CACHE_HASH_SUBDIR), manifest_dir); if ((err = check_cache(g, manifest_dir, &digest))) { if (err == ErrorCacheUnavailable) { @@ -9208,15 +9193,14 @@ void codegen_build_and_link(CodeGen *g) { } exit(1); } - - os_path_join(g->cache_dir, buf_create_from_str("artifact"), artifact_dir); } else { // There is a call to this in check_cache gen_c_objects(g); } if (g->enable_cache && buf_len(&digest) != 0) { - os_path_join(artifact_dir, &digest, &g->artifact_dir); + g->output_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR OS_SEP "%s", + buf_ptr(g->cache_dir), buf_ptr(&digest)); resolve_out_paths(g); } else { if (need_llvm_module(g)) { @@ -9235,13 +9219,13 @@ void codegen_build_and_link(CodeGen *g) { exit(1); } } - os_path_join(artifact_dir, &digest, &g->artifact_dir); - } else { - buf_init_from_buf(&g->artifact_dir, g->cache_dir); - } - if ((err = os_make_path(&g->artifact_dir))) { - fprintf(stderr, "Unable to create artifact directory: %s\n", err_str(err)); - exit(1); + g->output_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR OS_SEP "%s", + buf_ptr(g->cache_dir), buf_ptr(&digest)); + + if ((err = os_make_path(g->output_dir))) { + fprintf(stderr, "Unable to create output directory: %s\n", err_str(err)); + exit(1); + } } resolve_out_paths(g); @@ -9252,14 +9236,19 @@ void codegen_build_and_link(CodeGen *g) { codegen_add_time_event(g, "LLVM Emit Output"); zig_llvm_emit_output(g); - if (g->out_h_path != nullptr) { + if (!g->disable_gen_h && (g->out_type == OutTypeObj || g->out_type == OutTypeLib)) { codegen_add_time_event(g, "Generate .h"); gen_h_file(g); } } - if (g->emit_file_type == EmitFileTypeBinary && - (g->link_objects.length > 1 || g->out_type != OutTypeObj)) + // If we're outputting assembly or llvm IR we skip linking. + // If we're making a library or executable we must link. + // If there is more than one object, we have to link them (with -r). + // Finally, if we didn't make an object from zig source, and we don't have caching enabled, + // then we have an object from C source that we must copy to the output dir which we do with a -r link. + if (g->emit_file_type == EmitFileTypeBinary && (g->out_type != OutTypeObj || g->link_objects.length > 1 || + (!need_llvm_module(g) && !g->enable_cache))) { codegen_link(g); } diff --git a/src/codegen.hpp b/src/codegen.hpp index 120b68bb11..3befca2de5 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -28,7 +28,6 @@ void codegen_set_emit_file_type(CodeGen *g, EmitFileType emit_file_type); void codegen_set_strip(CodeGen *codegen, bool strip); void codegen_set_errmsg_color(CodeGen *codegen, ErrColor err_color); void codegen_set_out_name(CodeGen *codegen, Buf *out_name); -void codegen_set_dynamic_linker(CodeGen *g, Buf *dynamic_linker); void codegen_add_lib_dir(CodeGen *codegen, const char *dir); void codegen_add_forbidden_lib(CodeGen *codegen, Buf *lib); LinkLib *codegen_add_link_lib(CodeGen *codegen, Buf *lib); @@ -41,9 +40,6 @@ void codegen_set_linker_script(CodeGen *g, const char *linker_script); void codegen_set_test_filter(CodeGen *g, Buf *filter); void codegen_set_test_name_prefix(CodeGen *g, Buf *prefix); void codegen_set_lib_version(CodeGen *g, size_t major, size_t minor, size_t patch); -void codegen_set_output_h_path(CodeGen *g, Buf *h_path); -void codegen_set_output_lib_path(CodeGen *g, Buf *lib_path); -void codegen_set_output_path(CodeGen *g, Buf *path); void codegen_add_time_event(CodeGen *g, const char *name); void codegen_print_timing_report(CodeGen *g, FILE *f); void codegen_link(CodeGen *g); @@ -54,7 +50,7 @@ ZigPackage *codegen_create_package(CodeGen *g, const char *root_src_dir, const c void codegen_add_assembly(CodeGen *g, Buf *path); void codegen_add_object(CodeGen *g, Buf *object_path); -void codegen_translate_c(CodeGen *g, Buf *path); +AstNode *codegen_translate_c(CodeGen *g, Buf *path); Buf *codegen_generate_builtin_source(CodeGen *g); diff --git a/src/ir.cpp b/src/ir.cpp index aeb03397ca..a8771285f6 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -16,6 +16,8 @@ #include "translate_c.hpp" #include "util.hpp" +#include + struct IrExecContext { ZigList mem_slot_list; }; @@ -18687,7 +18689,15 @@ static IrInstruction *ir_analyze_instruction_type_name(IrAnalyze *ira, IrInstruc return result; } +static void ir_cimport_cache_paths(Buf *cache_dir, Buf *tmp_c_file_digest, Buf *out_zig_dir, Buf *out_zig_path) { + buf_resize(out_zig_dir, 0); + buf_resize(out_zig_path, 0); + buf_appendf(out_zig_dir, "%s" OS_SEP "o" OS_SEP "%s", + buf_ptr(cache_dir), buf_ptr(tmp_c_file_digest)); + buf_appendf(out_zig_path, "%s" OS_SEP "cimport.zig", buf_ptr(out_zig_dir)); +} static IrInstruction *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstructionCImport *instruction) { + Error err; AstNode *node = instruction->base.source_node; assert(node->type == NodeTypeFnCallExpr); AstNode *block_node = node->data.fn_call_expr.params.at(0); @@ -18706,50 +18716,128 @@ static IrInstruction *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstruct Buf *namespace_name = buf_sprintf("%s.cimport:%" ZIG_PRI_usize ":%" ZIG_PRI_usize, buf_ptr(&cur_scope_pkg->pkg_path), node->line + 1, node->column + 1); - RootStruct *root_struct = allocate(1); - root_struct->package = new_anonymous_package(); - root_struct->package->package_table.put(buf_create_from_str("builtin"), ira->codegen->compile_var_package); - root_struct->package->package_table.put(buf_create_from_str("std"), ira->codegen->std_package); - root_struct->c_import_node = node; - // TODO create namespace_name file in zig-cache instead of /tmp and use it - // for this DIFile - root_struct->di_file = ZigLLVMCreateFile(ira->codegen->dbuilder, - buf_ptr(buf_create_from_str("cimport.h")), buf_ptr(buf_create_from_str("."))); - ZigType *child_import = get_root_container_type(ira->codegen, buf_ptr(namespace_name), - namespace_name, root_struct); - - ZigList errors = {0}; - - Error err; - if ((err = parse_h_buf(child_import, &errors, &cimport_scope->buf, ira->codegen, node))) { - if (err != ErrorCCompileErrors) { - ir_add_error_node(ira, node, buf_sprintf("C import failed: %s", err_str(err))); - return ira->codegen->invalid_instruction; - } - } - - if (errors.length > 0) { - ErrorMsg *parent_err_msg = ir_add_error_node(ira, node, buf_sprintf("C import failed")); - if (ira->codegen->libc_link_lib == nullptr) { - add_error_note(ira->codegen, parent_err_msg, node, - buf_sprintf("libc headers not available; compilation does not link against libc")); - } - for (size_t i = 0; i < errors.length; i += 1) { - ErrorMsg *err_msg = errors.at(i); - err_msg_add_note(parent_err_msg, err_msg); - } + ZigPackage *cimport_pkg = new_anonymous_package(); + cimport_pkg->package_table.put(buf_create_from_str("builtin"), ira->codegen->compile_var_package); + cimport_pkg->package_table.put(buf_create_from_str("std"), ira->codegen->std_package); + buf_init_from_buf(&cimport_pkg->pkg_path, namespace_name); + CacheHash *cache_hash; + if ((err = create_c_object_cache(ira->codegen, &cache_hash, false))) { + ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to create cache: %s", err_str(err))); return ira->codegen->invalid_instruction; } + cache_buf(cache_hash, &cimport_scope->buf); - if (ira->codegen->verbose_cimport) { - fprintf(stderr, "\nC imports:\n"); - fprintf(stderr, "-----------\n"); - ast_render(ira->codegen, stderr, child_import->data.structure.decl_node, 4); + // Set this because we're not adding any files before checking for a hit. + cache_hash->force_check_manifest = true; + + Buf tmp_c_file_digest = BUF_INIT; + buf_resize(&tmp_c_file_digest, 0); + if ((err = cache_hit(cache_hash, &tmp_c_file_digest))) { + ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to check cache: %s", err_str(err))); + return ira->codegen->invalid_instruction; + } + ira->codegen->caches_to_release.append(cache_hash); + + Buf *out_zig_dir = buf_alloc(); + Buf *out_zig_path = buf_alloc(); + if (buf_len(&tmp_c_file_digest) == 0 || cache_hash->files.length == 0) { + // Cache Miss + Buf *tmp_c_file_dir = buf_sprintf("%s" OS_SEP "o" OS_SEP "%s", + buf_ptr(ira->codegen->cache_dir), buf_ptr(&cache_hash->b64_digest)); + Buf *resolve_paths[] = { + tmp_c_file_dir, + buf_create_from_str("cimport.h"), + }; + Buf tmp_c_file_path = os_path_resolve(resolve_paths, 2); + + if ((err = os_make_path(tmp_c_file_dir))) { + ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to make dir: %s", err_str(err))); + return ira->codegen->invalid_instruction; + } + + if ((err = os_write_file(&tmp_c_file_path, &cimport_scope->buf))) { + ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to write .h file: %s", err_str(err))); + return ira->codegen->invalid_instruction; + } + if (ira->codegen->verbose_cimport) { + fprintf(stderr, "@cImport source: %s\n", buf_ptr(&tmp_c_file_path)); + } + + ZigList errors = {0}; + + Buf *tmp_dep_file = buf_sprintf("%s.d", buf_ptr(&tmp_c_file_path)); + AstNode *root_node; + if ((err = parse_h_file(&root_node, &errors, buf_ptr(&tmp_c_file_path), ira->codegen, tmp_dep_file))) { + if (err != ErrorCCompileErrors) { + ir_add_error_node(ira, node, buf_sprintf("C import failed: %s", err_str(err))); + return ira->codegen->invalid_instruction; + } + assert(errors.length > 0); + + ErrorMsg *parent_err_msg = ir_add_error_node(ira, node, buf_sprintf("C import failed")); + if (ira->codegen->libc_link_lib == nullptr) { + add_error_note(ira->codegen, parent_err_msg, node, + buf_sprintf("libc headers not available; compilation does not link against libc")); + } + for (size_t i = 0; i < errors.length; i += 1) { + ErrorMsg *err_msg = errors.at(i); + err_msg_add_note(parent_err_msg, err_msg); + } + + return ira->codegen->invalid_instruction; + } + if (ira->codegen->verbose_cimport) { + fprintf(stderr, "@cImport .d file: %s\n", buf_ptr(tmp_dep_file)); + } + + if ((err = cache_add_dep_file(cache_hash, tmp_dep_file, false))) { + ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to parse .d file: %s", err_str(err))); + return ira->codegen->invalid_instruction; + } + if ((err = cache_final(cache_hash, &tmp_c_file_digest))) { + ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to finalize cache: %s", err_str(err))); + return ira->codegen->invalid_instruction; + } + + ir_cimport_cache_paths(ira->codegen->cache_dir, &tmp_c_file_digest, out_zig_dir, out_zig_path); + if ((err = os_make_path(out_zig_dir))) { + ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to make output dir: %s", err_str(err))); + return ira->codegen->invalid_instruction; + } + FILE *out_file = fopen(buf_ptr(out_zig_path), "wb"); + if (out_file == nullptr) { + ir_add_error_node(ira, node, + buf_sprintf("C import failed: unable to open output file: %s", strerror(errno))); + return ira->codegen->invalid_instruction; + } + ast_render(ira->codegen, out_file, root_node, 4); + if (fclose(out_file) != 0) { + ir_add_error_node(ira, node, + buf_sprintf("C import failed: unable to write to output file: %s", strerror(errno))); + return ira->codegen->invalid_instruction; + } + + if (ira->codegen->verbose_cimport) { + fprintf(stderr, "@cImport output: %s\n", buf_ptr(out_zig_path)); + } + + } else { + // Cache Hit + ir_cimport_cache_paths(ira->codegen->cache_dir, &tmp_c_file_digest, out_zig_dir, out_zig_path); + if (ira->codegen->verbose_cimport) { + fprintf(stderr, "@cImport cache hit: %s\n", buf_ptr(out_zig_path)); + } } - scan_decls(ira->codegen, get_container_scope(child_import), child_import->data.structure.decl_node); - + Buf *import_code = buf_alloc(); + if ((err = file_fetch(ira->codegen, out_zig_path, import_code))) { + ir_add_error_node(ira, node, + buf_sprintf("unable to open '%s': %s", buf_ptr(out_zig_path), err_str(err))); + return ira->codegen->invalid_instruction; + } + ZigType *child_import = add_source_file(ira->codegen, cimport_pkg, out_zig_path, + import_code, SourceKindCImport); return ir_const_type(ira, &instruction->base, child_import); } diff --git a/src/link.cpp b/src/link.cpp index a8d01303aa..eee2224a48 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -24,7 +24,7 @@ static CodeGen *create_child_codegen(CodeGen *parent_gen, Buf *root_src_path, Ou { CodeGen *child_gen = codegen_create(nullptr, root_src_path, parent_gen->zig_target, out_type, parent_gen->build_mode, parent_gen->zig_lib_dir, parent_gen->zig_std_dir, libc, get_stage1_cache_path()); - child_gen->out_h_path = nullptr; + child_gen->disable_gen_h = true; child_gen->verbose_tokenize = parent_gen->verbose_tokenize; child_gen->verbose_ast = parent_gen->verbose_ast; child_gen->verbose_link = parent_gen->verbose_link; @@ -700,11 +700,8 @@ static void construct_linker_job_elf(LinkJob *lj) { } else if (is_dyn_lib) { lj->args.append("-shared"); - if (buf_len(&g->output_file_path) == 0) { - buf_appendf(&g->output_file_path, "lib%s.so.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize "", - buf_ptr(g->root_out_name), g->version_major, g->version_minor, g->version_patch); - } - soname = buf_sprintf("lib%s.so.%" ZIG_PRI_usize "", buf_ptr(g->root_out_name), g->version_major); + assert(buf_len(&g->output_file_path) != 0); + soname = buf_sprintf("lib%s.so.%" ZIG_PRI_usize, buf_ptr(g->root_out_name), g->version_major); } lj->args.append("-o"); @@ -884,11 +881,6 @@ static void construct_linker_job_wasm(LinkJob *lj) { } } -//static bool is_target_cyg_mingw(const ZigTarget *target) { -// return (target->os == ZigLLVM_Win32 && target->abi == ZigLLVM_Cygnus) || -// (target->os == ZigLLVM_Win32 && target->abi == ZigLLVM_GNU); -//} - static void coff_append_machine_arg(CodeGen *g, ZigList *list) { if (g->zig_target->arch == ZigLLVM_x86) { list->append("-MACHINE:X86"); @@ -960,6 +952,7 @@ static void add_nt_link_args(LinkJob *lj, bool is_library) { } static void construct_linker_job_coff(LinkJob *lj) { + Error err; CodeGen *g = lj->codegen; lj->args.append("/ERRORLIMIT:0"); @@ -1079,11 +1072,13 @@ static void construct_linker_job_coff(LinkJob *lj) { buf_appendf(def_contents, "\n"); Buf *def_path = buf_alloc(); - os_path_join(&g->artifact_dir, buf_sprintf("%s.def", buf_ptr(link_lib->name)), def_path); - os_write_file(def_path, def_contents); + os_path_join(g->output_dir, buf_sprintf("%s.def", buf_ptr(link_lib->name)), def_path); + if ((err = os_write_file(def_path, def_contents))) { + zig_panic("error writing def file: %s", err_str(err)); + } Buf *generated_lib_path = buf_alloc(); - os_path_join(&g->artifact_dir, buf_sprintf("%s.lib", buf_ptr(link_lib->name)), generated_lib_path); + os_path_join(g->output_dir, buf_sprintf("%s.lib", buf_ptr(link_lib->name)), generated_lib_path); gen_lib_args.resize(0); gen_lib_args.append("link"); @@ -1245,10 +1240,7 @@ static void construct_linker_job_macho(LinkJob *lj) { //lj->args.append("-install_name"); //lj->args.append(buf_ptr(dylib_install_name)); - if (buf_len(&g->output_file_path) == 0) { - buf_appendf(&g->output_file_path, "lib%s.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".dylib", - buf_ptr(g->root_out_name), g->version_major, g->version_minor, g->version_patch); - } + assert(buf_len(&g->output_file_path) != 0); } lj->args.append("-arch"); diff --git a/src/main.cpp b/src/main.cpp index cd1b9d2891..28da1cbbe7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -48,9 +48,10 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) { "Compile Options:\n" " --assembly [source] add assembly file to build\n" " --c-source [options] [file] compile C source code\n" - " --cache-dir [path] override the cache directory\n" - " --cache [auto|off|on] build in global cache, print out paths to stdout\n" + " --cache-dir [path] override the local cache directory\n" + " --cache [auto|off|on] build in cache, print output path to stdout\n" " --color [auto|off|on] enable or disable colored error messages\n" + " --disable-gen-h do not generate a C header file (.h)\n" " --disable-pic disable Position Independent Code for libraries\n" " --disable-valgrind omit valgrind client requests in debug builds\n" " --enable-valgrind include valgrind client requests release builds\n" @@ -58,9 +59,7 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) { " -ftime-report print timing diagnostics\n" " --libc [file] Provide a file which specifies libc paths\n" " --name [name] override output name\n" - " --output [file] override destination path\n" - " --output-h [file] generate header file\n" - " --output-lib [file] override import library path\n" + " --output-dir [dir] override output directory (defaults to cwd)\n" " --pkg-begin [name] [path] make pkg available to import and push current pkg\n" " --pkg-end pop current pkg\n" " --main-pkg-path set the directory of the root package\n" @@ -371,8 +370,14 @@ int main(int argc, char **argv) { fprintf(stderr, "Unable to make directory: %s: %s\n", buf_ptr(out_src_dir_path), err_str(err)); return EXIT_FAILURE; } - os_write_file(out_build_zig_path, modified_build_zig_contents); - os_write_file(out_main_zig_path, main_zig_contents); + if ((err = os_write_file(out_build_zig_path, modified_build_zig_contents))) { + fprintf(stderr, "Unable to write file: %s: %s\n", buf_ptr(out_build_zig_path), err_str(err)); + return EXIT_FAILURE; + } + if ((err = os_write_file(out_main_zig_path, main_zig_contents))) { + fprintf(stderr, "Unable to write file: %s: %s\n", buf_ptr(out_main_zig_path), err_str(err)); + return EXIT_FAILURE; + } fprintf(stderr, "Created %s\n", buf_ptr(out_build_zig_path)); fprintf(stderr, "Created %s\n", buf_ptr(out_main_zig_path)); if (init_kind == InitKindExe) { @@ -390,9 +395,7 @@ int main(int argc, char **argv) { Cmd cmd = CmdNone; EmitFileType emit_file_type = EmitFileTypeBinary; const char *in_file = nullptr; - const char *out_file = nullptr; - const char *out_file_h = nullptr; - const char *out_file_lib = nullptr; + Buf *output_dir = nullptr; bool strip = false; bool is_static = false; OutType out_type = OutTypeUnknown; @@ -406,7 +409,7 @@ int main(int argc, char **argv) { bool verbose_cc = false; ErrColor color = ErrColorAuto; CacheOpt enable_cache = CacheOptAuto; - const char *dynamic_linker = nullptr; + Buf *dynamic_linker = nullptr; const char *libc_txt = nullptr; ZigList clang_argv = {0}; ZigList lib_dirs = {0}; @@ -438,6 +441,7 @@ int main(int argc, char **argv) { bool system_linker_hack = false; TargetSubsystem subsystem = TargetSubsystemAuto; bool is_single_threaded = false; + bool disable_gen_h = false; Buf *override_std_dir = nullptr; Buf *main_pkg_path = nullptr; ValgrindSupport valgrind_support = ValgrindSupportAuto; @@ -648,6 +652,8 @@ int main(int argc, char **argv) { system_linker_hack = true; } else if (strcmp(arg, "--single-threaded") == 0) { is_single_threaded = true; + } else if (strcmp(arg, "--disable-gen-h") == 0) { + disable_gen_h = true; } else if (strcmp(arg, "--test-cmd-bin") == 0) { test_exec_args.append(nullptr); } else if (arg[1] == 'L' && arg[2] != 0) { @@ -677,12 +683,8 @@ int main(int argc, char **argv) { return print_error_usage(arg0); } else { i += 1; - if (strcmp(arg, "--output") == 0) { - out_file = argv[i]; - } else if (strcmp(arg, "--output-h") == 0) { - out_file_h = argv[i]; - } else if (strcmp(arg, "--output-lib") == 0) { - out_file_lib = argv[i]; + if (strcmp(arg, "--output-dir") == 0) { + output_dir = buf_create_from_str(argv[i]); } else if (strcmp(arg, "--color") == 0) { if (strcmp(argv[i], "auto") == 0) { color = ErrColorAuto; @@ -719,7 +721,7 @@ int main(int argc, char **argv) { } else if (strcmp(arg, "--name") == 0) { out_name = argv[i]; } else if (strcmp(arg, "--dynamic-linker") == 0) { - dynamic_linker = argv[i]; + dynamic_linker = buf_create_from_str(argv[i]); } else if (strcmp(arg, "--libc") == 0) { libc_txt = argv[i]; } else if (strcmp(arg, "-isystem") == 0) { @@ -901,6 +903,21 @@ int main(int argc, char **argv) { } } + if (output_dir != nullptr && enable_cache == CacheOptOn) { + fprintf(stderr, "The --output-dir argument is incompatible with --cache on.\n"); + return print_error_usage(arg0); + } + + if (emit_file_type != EmitFileTypeBinary && in_file == nullptr) { + fprintf(stderr, "A root source file is required when using --emit asm or --emit llvm-ir"); + return print_error_usage(arg0); + } + + if (llvm_argv.length > 1) { + llvm_argv.append(nullptr); + ZigLLVMParseCommandLineOptions(llvm_argv.length - 1, llvm_argv.items); + } + switch (cmd) { case CmdLibC: { if (in_file) { @@ -1008,6 +1025,7 @@ int main(int argc, char **argv) { } CodeGen *g = codegen_create(main_pkg_path, zig_root_source_file, &target, out_type, build_mode, get_zig_lib_dir(), override_std_dir, libc, cache_dir_buf); + if (llvm_argv.length >= 2) codegen_set_llvm_argv(g, llvm_argv.items + 1, llvm_argv.length - 2); g->valgrind_support = valgrind_support; g->subsystem = subsystem; @@ -1030,16 +1048,9 @@ int main(int argc, char **argv) { codegen_set_clang_argv(g, clang_argv.items, clang_argv.length); - if (llvm_argv.length > 1) { - llvm_argv.append(nullptr); - ZigLLVMParseCommandLineOptions(llvm_argv.length - 1, llvm_argv.items); - } - - codegen_set_llvm_argv(g, llvm_argv.items + 1, llvm_argv.length - 2); codegen_set_strip(g, strip); g->is_static = is_static; - if (dynamic_linker != nullptr) - codegen_set_dynamic_linker(g, buf_create_from_str(dynamic_linker)); + g->dynamic_linker_path = dynamic_linker; g->verbose_tokenize = verbose_tokenize; g->verbose_ast = verbose_ast; g->verbose_link = verbose_link; @@ -1047,6 +1058,8 @@ int main(int argc, char **argv) { g->verbose_llvm_ir = verbose_llvm_ir; g->verbose_cimport = verbose_cimport; g->verbose_cc = verbose_cc; + g->output_dir = output_dir; + g->disable_gen_h = disable_gen_h; codegen_set_errmsg_color(g, color); g->system_linker_hack = system_linker_hack; @@ -1090,13 +1103,6 @@ int main(int argc, char **argv) { codegen_set_test_name_prefix(g, buf_create_from_str(test_name_prefix)); } - if (out_file) - codegen_set_output_path(g, buf_create_from_str(out_file)); - if (out_file_h != nullptr && (out_type == OutTypeObj || out_type == OutTypeLib)) - codegen_set_output_h_path(g, buf_create_from_str(out_file_h)); - if (out_file_lib != nullptr && out_type == OutTypeLib && !is_static) - codegen_set_output_lib_path(g, buf_create_from_str(out_file_lib)); - add_package(g, cur_pkg, g->root_package); if (cmd == CmdBuild || cmd == CmdRun || cmd == CmdTest) { @@ -1135,20 +1141,18 @@ int main(int argc, char **argv) { return term.code; } else if (cmd == CmdBuild) { if (g->enable_cache) { - printf("%s\n", buf_ptr(&g->output_file_path)); - if (g->out_h_path != nullptr) { - printf("%s\n", buf_ptr(g->out_h_path)); - } + if (printf("%s\n", buf_ptr(&g->output_file_path)) < 0) + return EXIT_FAILURE; } return EXIT_SUCCESS; } else { zig_unreachable(); } } else if (cmd == CmdTranslateC) { - codegen_translate_c(g, in_file_buf); - ast_render(g, stdout, g->root_import->data.structure.decl_node, 4); + AstNode *root_node = codegen_translate_c(g, in_file_buf); + ast_render(g, stdout, root_node, 4); if (timing_info) - codegen_print_timing_report(g, stdout); + codegen_print_timing_report(g, stderr); return EXIT_SUCCESS; } else if (cmd == CmdTest) { codegen_set_emit_file_type(g, emit_file_type); @@ -1156,7 +1160,7 @@ int main(int argc, char **argv) { ZigTarget native; get_native_target(&native); - g->enable_cache = get_cache_opt(enable_cache, false); + g->enable_cache = get_cache_opt(enable_cache, true); codegen_build_and_link(g); if (timing_info) { diff --git a/src/os.cpp b/src/os.cpp index c50fb71884..d5195c92d8 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -1031,7 +1031,7 @@ Error os_exec_process(const char *exe, ZigList &args, #endif } -void os_write_file(Buf *full_path, Buf *contents) { +Error os_write_file(Buf *full_path, Buf *contents) { FILE *f = fopen(buf_ptr(full_path), "wb"); if (!f) { zig_panic("os_write_file failed for %s", buf_ptr(full_path)); @@ -1041,6 +1041,7 @@ void os_write_file(Buf *full_path, Buf *contents) { zig_panic("write failed: %s", strerror(errno)); if (fclose(f)) zig_panic("close failed"); + return ErrorNone; } Error os_copy_file(Buf *src_path, Buf *dest_path) { @@ -1208,91 +1209,6 @@ bool os_stderr_tty(void) { #endif } -#if defined(ZIG_OS_POSIX) -static Error os_buf_to_tmp_file_posix(Buf *contents, Buf *suffix, Buf *out_tmp_path) { - const char *tmp_dir = getenv("TMPDIR"); - if (!tmp_dir) { - tmp_dir = P_tmpdir; - } - buf_resize(out_tmp_path, 0); - buf_appendf(out_tmp_path, "%s/XXXXXX%s", tmp_dir, buf_ptr(suffix)); - - int fd = mkstemps(buf_ptr(out_tmp_path), (int)buf_len(suffix)); - if (fd < 0) { - return ErrorFileSystem; - } - - FILE *f = fdopen(fd, "wb"); - if (!f) { - zig_panic("fdopen failed"); - } - - size_t amt_written = fwrite(buf_ptr(contents), 1, buf_len(contents), f); - if (amt_written != (size_t)buf_len(contents)) - zig_panic("write failed: %s", strerror(errno)); - if (fclose(f)) - zig_panic("close failed"); - - return ErrorNone; -} -#endif - -Buf *os_tmp_filename(Buf *prefix, Buf *suffix) { - Buf *result = buf_create_from_buf(prefix); - - const char base64[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"; - assert(array_length(base64) == 64 + 1); - for (size_t i = 0; i < 12; i += 1) { - buf_append_char(result, base64[rand() % 64]); - } - buf_append_buf(result, suffix); - return result; -} - -#if defined(ZIG_OS_WINDOWS) -static Error os_buf_to_tmp_file_windows(Buf *contents, Buf *suffix, Buf *out_tmp_path) { - char tmp_dir[MAX_PATH + 1]; - if (GetTempPath(MAX_PATH, tmp_dir) == 0) { - zig_panic("GetTempPath failed"); - } - buf_init_from_str(out_tmp_path, tmp_dir); - - const char base64[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"; - assert(array_length(base64) == 64 + 1); - for (size_t i = 0; i < 8; i += 1) { - buf_append_char(out_tmp_path, base64[rand() % 64]); - } - - buf_append_buf(out_tmp_path, suffix); - - FILE *f = fopen(buf_ptr(out_tmp_path), "wb"); - - if (!f) { - zig_panic("unable to open %s: %s", buf_ptr(out_tmp_path), strerror(errno)); - } - - size_t amt_written = fwrite(buf_ptr(contents), 1, buf_len(contents), f); - if (amt_written != (size_t)buf_len(contents)) { - zig_panic("write failed: %s", strerror(errno)); - } - - if (fclose(f)) { - zig_panic("fclose failed"); - } - return ErrorNone; -} -#endif - -Error os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path) { -#if defined(ZIG_OS_WINDOWS) - return os_buf_to_tmp_file_windows(contents, suffix, out_tmp_path); -#elif defined(ZIG_OS_POSIX) - return os_buf_to_tmp_file_posix(contents, suffix, out_tmp_path); -#else -#error "missing os_buf_to_tmp_file implementation" -#endif -} - Error os_delete_file(Buf *path) { if (remove(buf_ptr(path))) { return ErrorFileSystem; diff --git a/src/os.hpp b/src/os.hpp index 2e0b1ea88f..916158b4f3 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -110,8 +110,8 @@ Error ATTRIBUTE_MUST_USE os_file_read_all(OsFile file, Buf *contents); Error ATTRIBUTE_MUST_USE os_file_overwrite(OsFile file, Buf *contents); void os_file_close(OsFile file); -void os_write_file(Buf *full_path, Buf *contents); -Error os_copy_file(Buf *src_path, Buf *dest_path); +Error ATTRIBUTE_MUST_USE os_write_file(Buf *full_path, Buf *contents); +Error ATTRIBUTE_MUST_USE os_copy_file(Buf *src_path, Buf *dest_path); Error ATTRIBUTE_MUST_USE os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); Error ATTRIBUTE_MUST_USE os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); @@ -121,8 +121,6 @@ Error ATTRIBUTE_MUST_USE os_get_cwd(Buf *out_cwd); bool os_stderr_tty(void); void os_stderr_set_color(TermColor color); -Buf *os_tmp_filename(Buf *prefix, Buf *suffix); -Error os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path); Error os_delete_file(Buf *path); Error ATTRIBUTE_MUST_USE os_file_exists(Buf *full_path, bool *result); diff --git a/src/target.cpp b/src/target.cpp index 5264184454..7eb4998f57 100644 --- a/src/target.cpp +++ b/src/target.cpp @@ -942,8 +942,12 @@ const char *target_lib_file_ext(const ZigTarget *target, bool is_static, } else { if (is_static) { return ".a"; + } else if (target_is_darwin(target)) { + return buf_ptr(buf_sprintf(".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".dylib", + version_major, version_minor, version_patch)); } else { - return buf_ptr(buf_sprintf(".so.%" ZIG_PRI_usize, version_major)); + return buf_ptr(buf_sprintf(".so.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize, + version_major, version_minor, version_patch)); } } } diff --git a/src/translate_c.cpp b/src/translate_c.cpp index bdedd78b35..c56d31eb2d 100644 --- a/src/translate_c.cpp +++ b/src/translate_c.cpp @@ -76,7 +76,6 @@ struct TransScopeWhile { }; struct Context { - ZigType *import; ZigList *errors; VisibMod visib_mod; bool want_export; @@ -86,7 +85,6 @@ struct Context { HashMap global_table; ZigClangSourceManager *source_manager; ZigList aliases; - AstNode *source_node; bool warnings_on; CodeGen *codegen; @@ -190,7 +188,6 @@ static Buf *trans_lookup_zig_symbol(Context *c, TransScope *scope, Buf *c_symbol static AstNode * trans_create_node(Context *c, NodeType id) { AstNode *node = allocate(1); node->type = id; - node->owner = c->import; // TODO line/column. mapping to C file?? return node; } @@ -4732,29 +4729,12 @@ static void process_preprocessor_entities(Context *c, ZigClangASTUnit *zunit) { } } -Error parse_h_buf(ZigType *import, ZigList *errors, Buf *source, - CodeGen *codegen, AstNode *source_node) -{ - Error err; - Buf tmp_file_path = BUF_INIT; - if ((err = os_buf_to_tmp_file(source, buf_create_from_str(".h"), &tmp_file_path))) { - return err; - } - - err = parse_h_file(import, errors, buf_ptr(&tmp_file_path), codegen, source_node); - - os_delete_file(&tmp_file_path); - - return err; -} - -Error parse_h_file(ZigType *import, ZigList *errors, const char *target_file, - CodeGen *codegen, AstNode *source_node) +Error parse_h_file(AstNode **out_root_node, ZigList *errors, const char *target_file, + CodeGen *codegen, Buf *tmp_dep_file) { Context context = {0}; Context *c = &context; c->warnings_on = codegen->verbose_cimport; - c->import = import; c->errors = errors; if (buf_ends_with_str(buf_create_from_str(target_file), ".h")) { c->visib_mod = VisibModPub; @@ -4768,7 +4748,6 @@ Error parse_h_file(ZigType *import, ZigList *errors, const char *tar c->global_table.init(8); c->ptr_params.init(8); c->codegen = codegen; - c->source_node = source_node; c->global_scope = trans_scope_root_create(c); ZigList clang_argv = {0}; @@ -4776,14 +4755,11 @@ Error parse_h_file(ZigType *import, ZigList *errors, const char *tar clang_argv.append("-x"); clang_argv.append("c"); - Buf *out_dep_path = nullptr; - if (codegen->enable_cache) { - Buf *prefix = buf_sprintf("%s" OS_SEP, buf_ptr(codegen->cache_dir)); - out_dep_path = os_tmp_filename(prefix, buf_create_from_str(".d")); + if (tmp_dep_file != nullptr) { clang_argv.append("-MD"); clang_argv.append("-MV"); clang_argv.append("-MF"); - clang_argv.append(buf_ptr(out_dep_path)); + clang_argv.append(buf_ptr(tmp_dep_file)); } if (c->codegen->zig_target->is_native) { @@ -4847,7 +4823,7 @@ Error parse_h_file(ZigType *import, ZigList *errors, const char *tar clang_argv.append(target_file); - if (codegen->verbose_cimport) { + if (codegen->verbose_cc) { fprintf(stderr, "clang"); for (size_t i = 0; i < clang_argv.length; i += 1) { fprintf(stderr, " %s", clang_argv.at(i)); @@ -4932,17 +4908,6 @@ Error parse_h_file(ZigType *import, ZigList *errors, const char *tar return ErrorCCompileErrors; } - if (codegen->enable_cache) { - Error err; - assert(out_dep_path != nullptr); - if ((err = cache_add_dep_file(&codegen->cache_hash, out_dep_path, codegen->verbose_cimport))) { - if (codegen->verbose_cimport) { - fprintf(stderr, "translate-c: aborting due to failed cache operation: %s\n", err_str(err)); - } - return err; - } - } - c->ctx = ZigClangASTUnit_getASTContext(ast_unit); c->source_manager = ZigClangASTUnit_getSourceManager(ast_unit); c->root = trans_create_node(c, NodeTypeContainerDecl); @@ -4955,8 +4920,7 @@ Error parse_h_file(ZigType *import, ZigList *errors, const char *tar render_macros(c); render_aliases(c); - import->data.structure.decl_node = c->root; - import->data.structure.decls_scope->base.source_node = c->root; + *out_root_node = c->root; return ErrorNone; } diff --git a/src/translate_c.hpp b/src/translate_c.hpp index c4bd270c21..21461d2813 100644 --- a/src/translate_c.hpp +++ b/src/translate_c.hpp @@ -11,10 +11,7 @@ #include "all_types.hpp" -Error parse_h_file(ZigType *import, ZigList *errors, const char *target_file, - CodeGen *codegen, AstNode *source_node); - -Error parse_h_buf(ZigType *import, ZigList *errors, Buf *source, - CodeGen *codegen, AstNode *source_node); +Error parse_h_file(AstNode **out_root_node, ZigList *errors, const char *target_file, + CodeGen *codegen, Buf *tmp_dep_file); #endif diff --git a/std/build.zig b/std/build.zig index 2316a87b31..8ff742d5a2 100644 --- a/std/build.zig +++ b/std/build.zig @@ -183,9 +183,20 @@ pub const Builder = struct { return obj_step; } - /// ::argv is copied. - pub fn addCommand(self: *Builder, cwd: ?[]const u8, env_map: *const BufMap, argv: []const []const u8) *CommandStep { - return CommandStep.create(self, cwd, env_map, argv); + /// Initializes a RunStep with argv, which must at least have the path to the + /// executable. More command line arguments can be added with `addArg`, + /// `addArgs`, and `addArtifactArg`. + /// Be careful using this function, as it introduces a system dependency. + /// To run an executable built with zig build, see `LibExeObjStep.run`. + pub fn addSystemCommand(self: *Builder, argv: []const []const u8) *RunStep { + assert(argv.len >= 1); + const run_step = RunStep.create(self, self.fmt("run {}", argv[0])); + run_step.addArgs(argv); + return run_step; + } + + fn dupe(self: *Builder, bytes: []const u8) []u8 { + return mem.dupe(self.allocator, u8, bytes) catch unreachable; } pub fn addWriteFile(self: *Builder, file_path: []const u8, data: []const u8) *WriteFileStep { @@ -702,25 +713,42 @@ pub const Builder = struct { } pub fn exec(self: *Builder, argv: []const []const u8) ![]u8 { + assert(argv.len != 0); + const max_output_size = 100 * 1024; - const result = try os.ChildProcess.exec(self.allocator, argv, null, null, max_output_size); - switch (result.term) { + const child = try os.ChildProcess.init(argv, self.allocator); + defer child.deinit(); + + child.stdin_behavior = os.ChildProcess.StdIo.Ignore; + child.stdout_behavior = os.ChildProcess.StdIo.Pipe; + child.stderr_behavior = os.ChildProcess.StdIo.Inherit; + + try child.spawn(); + + var stdout = std.Buffer.initNull(self.allocator); + defer std.Buffer.deinit(&stdout); + + var stdout_file_in_stream = child.stdout.?.inStream(); + try stdout_file_in_stream.stream.readAllBuffer(&stdout, max_output_size); + + const term = child.wait() catch |err| std.debug.panic("unable to spawn {}: {}", argv[0], err); + switch (term) { os.ChildProcess.Term.Exited => |code| { if (code != 0) { warn("The following command exited with error code {}:\n", code); printCmd(null, argv); - warn("stderr:{}\n", result.stderr); - std.debug.panic("command failed"); + std.debug.panic("exec failed"); } - return result.stdout; + return stdout.toOwnedSlice(); }, else => { warn("The following command terminated unexpectedly:\n"); printCmd(null, argv); - warn("stderr:{}\n", result.stderr); - std.debug.panic("command failed"); + std.debug.panic("exec failed"); }, } + + return stdout.toOwnedSlice(); } pub fn addSearchPrefix(self: *Builder, search_prefix: []const u8) void { @@ -837,26 +865,20 @@ pub const LibExeObjStep = struct { builder: *Builder, name: []const u8, target: Target, - link_libs: BufSet, linker_script: ?[]const u8, out_filename: []const u8, - output_path: ?[]const u8, - output_lib_path: ?[]const u8, static: bool, version: Version, - object_files: ArrayList([]const u8), build_mode: builtin.Mode, kind: Kind, major_only_filename: []const u8, name_only_filename: []const u8, strip: bool, - full_path_libs: ArrayList([]const u8), - need_flat_namespace_hack: bool, - include_dirs: ArrayList([]const u8), lib_paths: ArrayList([]const u8), frameworks: BufSet, verbose_link: bool, verbose_cc: bool, + disable_gen_h: bool, c_std: Builder.CStd, override_std_dir: ?[]const u8, main_pkg_path: ?[]const u8, @@ -865,17 +887,31 @@ pub const LibExeObjStep = struct { filter: ?[]const u8, root_src: ?[]const u8, - output_h_path: ?[]const u8, out_h_filename: []const u8, out_lib_filename: []const u8, - assembly_files: ArrayList([]const u8), packages: ArrayList(Pkg), build_options_contents: std.Buffer, system_linker_hack: bool, - c_source_files: ArrayList(*CSourceFile), object_src: []const u8, + link_objects: ArrayList(LinkObject), + include_dirs: ArrayList(IncludeDir), + output_dir: ?[]const u8, + + const LinkObject = union(enum) { + StaticPath: []const u8, + OtherStep: *LibExeObjStep, + SystemLib: []const u8, + AssemblyFile: []const u8, + CSourceFile: *CSourceFile, + }; + + const IncludeDir = union(enum) { + RawPath: []const u8, + OtherStep: *LibExeObjStep, + }; + const Kind = enum { Exe, Lib, @@ -926,25 +962,17 @@ pub const LibExeObjStep = struct { .name = name, .target = Target.Native, .linker_script = null, - .link_libs = BufSet.init(builder.allocator), .frameworks = BufSet.init(builder.allocator), .step = Step.init(name, builder.allocator, make), - .output_path = null, - .output_lib_path = null, - .output_h_path = null, .version = ver, .out_filename = undefined, .out_h_filename = builder.fmt("{}.h", name), .out_lib_filename = undefined, .major_only_filename = undefined, .name_only_filename = undefined, - .object_files = ArrayList([]const u8).init(builder.allocator), - .assembly_files = ArrayList([]const u8).init(builder.allocator), .packages = ArrayList(Pkg).init(builder.allocator), - .full_path_libs = ArrayList([]const u8).init(builder.allocator), - .need_flat_namespace_hack = false, - .c_source_files = ArrayList(*CSourceFile).init(builder.allocator), - .include_dirs = ArrayList([]const u8).init(builder.allocator), + .include_dirs = ArrayList(IncludeDir).init(builder.allocator), + .link_objects = ArrayList(LinkObject).init(builder.allocator), .lib_paths = ArrayList([]const u8).init(builder.allocator), .object_src = undefined, .build_options_contents = std.Buffer.initSize(builder.allocator, 0) catch unreachable, @@ -955,6 +983,8 @@ pub const LibExeObjStep = struct { .exec_cmd_args = null, .name_prefix = "", .filter = null, + .disable_gen_h = false, + .output_dir = null, }; self.computeOutFileNames(); return self; @@ -1022,7 +1052,19 @@ pub const LibExeObjStep = struct { self.computeOutFileNames(); } - // TODO respect this in the C args + pub fn setOutputDir(self: *LibExeObjStep, dir: []const u8) void { + self.output_dir = self.builder.dupe(dir); + } + + /// Creates a `RunStep` with an executable built with `addExecutable`. + /// Add command line arguments with `addArg`. + pub fn run(exe: *LibExeObjStep) *RunStep { + assert(exe.kind == Kind.Exe); + const run_step = RunStep.create(exe.builder, exe.builder.fmt("run {}", exe.step.name)); + run_step.addArtifactArg(exe); + return run_step; + } + pub fn setLinkerScriptPath(self: *LibExeObjStep, path: []const u8) void { self.linker_script = path; } @@ -1032,37 +1074,28 @@ pub const LibExeObjStep = struct { self.frameworks.put(framework_name) catch unreachable; } - pub fn linkLibrary(self: *LibExeObjStep, lib: *LibExeObjStep) void { - assert(self.kind != Kind.Obj); - assert(lib.kind == Kind.Lib); - - self.step.dependOn(&lib.step); - - if (lib.static or self.target.isWindows()) { - self.object_files.append(lib.getOutputLibPath()) catch unreachable; - } else { - self.full_path_libs.append(lib.getOutputPath()) catch unreachable; - } - - if (lib.link_libs.exists("c")) { - self.link_libs.put("c") catch unreachable; - } - - // TODO should be some kind of isolated directory that only has this header in it - self.include_dirs.append(self.builder.cache_root) catch unreachable; - self.need_flat_namespace_hack = true; - - // inherit the object's frameworks - if (self.target.isDarwin() and lib.static) { - var it = lib.frameworks.iterator(); - while (it.next()) |entry| { - self.frameworks.put(entry.key) catch unreachable; + /// Returns whether the library, executable, or object depends on a particular system library. + pub fn dependsOnSystemLibrary(self: LibExeObjStep, name: []const u8) bool { + for (self.link_objects.toSliceConst()) |link_object| { + switch (link_object) { + LinkObject.SystemLib => |n| if (mem.eql(u8, n, name)) return true, + else => continue, } } + return false; + } + + pub fn linkLibrary(self: *LibExeObjStep, lib: *LibExeObjStep) void { + assert(lib.kind == Kind.Lib); + self.linkLibraryOrObject(lib); + } + + pub fn isDynamicLibrary(self: *LibExeObjStep) bool { + return self.kind == Kind.Lib and !self.static; } pub fn linkSystemLibrary(self: *LibExeObjStep, name: []const u8) void { - self.link_libs.put(name) catch unreachable; + self.link_objects.append(LinkObject{ .SystemLib = self.builder.dupe(name) }) catch unreachable; } pub fn setNamePrefix(self: *LibExeObjStep, text: []const u8) void { @@ -1077,11 +1110,15 @@ pub const LibExeObjStep = struct { pub fn addCSourceFile(self: *LibExeObjStep, file: []const u8, args: []const []const u8) void { const c_source_file = self.builder.allocator.create(CSourceFile) catch unreachable; + const args_copy = self.builder.allocator.alloc([]u8, args.len) catch unreachable; + for (args) |arg, i| { + args_copy[i] = self.builder.dupe(arg); + } c_source_file.* = CSourceFile{ - .source_path = file, - .args = args, + .source_path = self.builder.dupe(file), + .args = args_copy, }; - self.c_source_files.append(c_source_file) catch unreachable; + self.link_objects.append(LinkObject{ .CSourceFile = c_source_file }) catch unreachable; } pub fn setVerboseLink(self: *LibExeObjStep, value: bool) void { @@ -1104,78 +1141,48 @@ pub const LibExeObjStep = struct { self.main_pkg_path = dir_path; } - pub fn setOutputPath(self: *LibExeObjStep, file_path: []const u8) void { - self.output_path = file_path; - - // catch a common mistake - if (mem.eql(u8, self.builder.pathFromRoot(file_path), self.builder.pathFromRoot("."))) { - debug.panic("setOutputPath wants a file path, not a directory\n"); - } - } - + /// Unless setOutputDir was called, this function must be called only in + /// the make step, from a step that has declared a dependency on this one. + /// To run an executable built with zig build, use `run`, or create an install step and invoke it. pub fn getOutputPath(self: *LibExeObjStep) []const u8 { - return if (self.output_path) |output_path| output_path else os.path.join( + return os.path.join( self.builder.allocator, - [][]const u8{ self.builder.cache_root, self.out_filename }, + [][]const u8{ self.output_dir.?, self.out_filename }, ) catch unreachable; } - pub fn setOutputLibPath(self: *LibExeObjStep, file_path: []const u8) void { - assert(self.kind == Kind.Lib); - if (self.static) - return self.setOutputPath(file_path); - - self.output_lib_path = file_path; - } - + /// Unless setOutputDir was called, this function must be called only in + /// the make step, from a step that has declared a dependency on this one. pub fn getOutputLibPath(self: *LibExeObjStep) []const u8 { assert(self.kind == Kind.Lib); - return if (self.output_lib_path) |output_lib_path| output_lib_path else os.path.join( + return os.path.join( self.builder.allocator, - [][]const u8{ self.builder.cache_root, self.out_lib_filename }, + [][]const u8{ self.output_dir.?, self.out_lib_filename }, ) catch unreachable; } - pub fn setOutputHPath(self: *LibExeObjStep, file_path: []const u8) void { - self.output_h_path = file_path; - - // catch a common mistake - if (mem.eql(u8, self.builder.pathFromRoot(file_path), self.builder.pathFromRoot("."))) { - debug.panic("setOutputHPath wants a file path, not a directory\n"); - } - } - + /// Unless setOutputDir was called, this function must be called only in + /// the make step, from a step that has declared a dependency on this one. pub fn getOutputHPath(self: *LibExeObjStep) []const u8 { - return if (self.output_h_path) |output_h_path| output_h_path else os.path.join( + assert(self.kind != Kind.Exe); + assert(!self.disable_gen_h); + return os.path.join( self.builder.allocator, - [][]const u8{ self.builder.cache_root, self.out_h_filename }, + [][]const u8{ self.output_dir.?, self.out_h_filename }, ) catch unreachable; } pub fn addAssemblyFile(self: *LibExeObjStep, path: []const u8) void { - self.assembly_files.append(path) catch unreachable; + self.link_objects.append(LinkObject{ .AssemblyFile = self.builder.dupe(path) }) catch unreachable; } pub fn addObjectFile(self: *LibExeObjStep, path: []const u8) void { - assert(self.kind != Kind.Obj); - - self.object_files.append(path) catch unreachable; + self.link_objects.append(LinkObject{ .StaticPath = self.builder.dupe(path) }) catch unreachable; } pub fn addObject(self: *LibExeObjStep, obj: *LibExeObjStep) void { assert(obj.kind == Kind.Obj); - assert(self.kind != Kind.Obj); - - self.step.dependOn(&obj.step); - - self.object_files.append(obj.getOutputPath()) catch unreachable; - - // TODO should be some kind of isolated directory that only has this header in it - self.include_dirs.append(self.builder.cache_root) catch unreachable; - - if (obj.link_libs.exists("c")) { - self.link_libs.put("c") catch unreachable; - } + self.linkLibraryOrObject(obj); } pub fn addBuildOption(self: *LibExeObjStep, comptime T: type, name: []const u8, value: T) void { @@ -1184,7 +1191,7 @@ pub const LibExeObjStep = struct { } pub fn addIncludeDir(self: *LibExeObjStep, path: []const u8) void { - self.include_dirs.append(path) catch unreachable; + self.include_dirs.append(IncludeDir{ .RawPath = self.builder.dupe(path) }) catch unreachable; } pub fn addLibPath(self: *LibExeObjStep, path: []const u8) void { @@ -1207,13 +1214,30 @@ pub const LibExeObjStep = struct { self.system_linker_hack = true; } + fn linkLibraryOrObject(self: *LibExeObjStep, other: *LibExeObjStep) void { + self.step.dependOn(&other.step); + self.link_objects.append(LinkObject{ .OtherStep = other }) catch unreachable; + self.include_dirs.append(IncludeDir{ .OtherStep = other }) catch unreachable; + + // Inherit dependency on libc + if (other.dependsOnSystemLibrary("c")) { + self.linkSystemLibrary("c"); + } + + // Inherit dependencies on darwin frameworks + if (self.target.isDarwin() and !other.isDynamicLibrary()) { + var it = other.frameworks.iterator(); + while (it.next()) |entry| { + self.frameworks.put(entry.key) catch unreachable; + } + } + } + fn make(step: *Step) !void { const self = @fieldParentPtr(LibExeObjStep, "step", step); const builder = self.builder; - if (self.root_src == null and self.object_files.len == 0 and - self.assembly_files.len == 0 and self.c_source_files.len == 0) - { + if (self.root_src == null and self.link_objects.len == 0) { warn("{}: linker needs 1 or more objects to link\n", self.step.name); return error.NeedAnObject; } @@ -1235,12 +1259,52 @@ pub const LibExeObjStep = struct { zig_args.append(builder.pathFromRoot(root_src)) catch unreachable; } - for (self.c_source_files.toSliceConst()) |c_source_file| { - try zig_args.append("--c-source"); - for (c_source_file.args) |arg| { - try zig_args.append(arg); + for (self.link_objects.toSlice()) |link_object| { + switch (link_object) { + LinkObject.StaticPath => |static_path| { + try zig_args.append("--object"); + try zig_args.append(builder.pathFromRoot(static_path)); + }, + + LinkObject.OtherStep => |other| switch (other.kind) { + LibExeObjStep.Kind.Exe => unreachable, + LibExeObjStep.Kind.Test => unreachable, + LibExeObjStep.Kind.Obj => { + try zig_args.append("--object"); + try zig_args.append(other.getOutputPath()); + }, + LibExeObjStep.Kind.Lib => { + if (other.static or self.target.isWindows()) { + try zig_args.append("--object"); + try zig_args.append(other.getOutputPath()); + } else { + const full_path_lib = other.getOutputPath(); + try zig_args.append("--library"); + try zig_args.append(full_path_lib); + + if (os.path.dirname(full_path_lib)) |dirname| { + try zig_args.append("-rpath"); + try zig_args.append(dirname); + } + } + }, + }, + LinkObject.SystemLib => |name| { + try zig_args.append("--library"); + try zig_args.append(name); + }, + LinkObject.AssemblyFile => |asm_file| { + try zig_args.append("--assembly"); + try zig_args.append(builder.pathFromRoot(asm_file)); + }, + LinkObject.CSourceFile => |c_source_file| { + try zig_args.append("--c-source"); + for (c_source_file.args) |arg| { + try zig_args.append(arg); + } + try zig_args.append(self.builder.pathFromRoot(c_source_file.source_path)); + }, } - try zig_args.append(self.builder.pathFromRoot(c_source_file.source_path)); } if (self.build_options_contents.len() > 0) { @@ -1265,16 +1329,6 @@ pub const LibExeObjStep = struct { try zig_args.append(self.name_prefix); } - for (self.object_files.toSliceConst()) |object_file| { - zig_args.append("--object") catch unreachable; - zig_args.append(builder.pathFromRoot(object_file)) catch unreachable; - } - - for (self.assembly_files.toSliceConst()) |asm_file| { - zig_args.append("--assembly") catch unreachable; - zig_args.append(builder.pathFromRoot(asm_file)) catch unreachable; - } - if (builder.verbose_tokenize) zig_args.append("--verbose-tokenize") catch unreachable; if (builder.verbose_ast) zig_args.append("--verbose-ast") catch unreachable; if (builder.verbose_cimport) zig_args.append("--verbose-cimport") catch unreachable; @@ -1294,24 +1348,8 @@ pub const LibExeObjStep = struct { builtin.Mode.ReleaseSmall => zig_args.append("--release-small") catch unreachable, } - zig_args.append("--cache-dir") catch unreachable; - zig_args.append(builder.pathFromRoot(builder.cache_root)) catch unreachable; - - const output_path = builder.pathFromRoot(self.getOutputPath()); - zig_args.append("--output") catch unreachable; - zig_args.append(output_path) catch unreachable; - - if (self.kind == Kind.Lib and !self.static) { - const output_lib_path = builder.pathFromRoot(self.getOutputLibPath()); - zig_args.append("--output-lib") catch unreachable; - zig_args.append(output_lib_path) catch unreachable; - } - - if (self.kind != Kind.Exe and self.root_src != null) { - const output_h_path = self.getOutputHPath(); - zig_args.append("--output-h") catch unreachable; - zig_args.append(builder.pathFromRoot(output_h_path)) catch unreachable; - } + try zig_args.append("--cache-dir"); + try zig_args.append(builder.pathFromRoot(builder.cache_root)); zig_args.append("--name") catch unreachable; zig_args.append(self.name) catch unreachable; @@ -1351,15 +1389,6 @@ pub const LibExeObjStep = struct { zig_args.append(linker_script) catch unreachable; } - { - var it = self.link_libs.iterator(); - while (true) { - const entry = it.next() orelse break; - zig_args.append("--library") catch unreachable; - zig_args.append(entry.key) catch unreachable; - } - } - if (self.exec_cmd_args) |exec_cmd_args| { for (exec_cmd_args) |cmd_arg| { if (cmd_arg) |arg| { @@ -1377,9 +1406,18 @@ pub const LibExeObjStep = struct { zig_args.append("--pkg-end") catch unreachable; } - for (self.include_dirs.toSliceConst()) |include_path| { - zig_args.append("-isystem") catch unreachable; - zig_args.append(self.builder.pathFromRoot(include_path)) catch unreachable; + for (self.include_dirs.toSliceConst()) |include_dir| { + switch (include_dir) { + IncludeDir.RawPath => |include_path| { + try zig_args.append("-isystem"); + try zig_args.append(self.builder.pathFromRoot(include_path)); + }, + IncludeDir.OtherStep => |other| { + const h_path = other.getOutputHPath(); + try zig_args.append("-isystem"); + try zig_args.append(os.path.dirname(h_path).?); + }, + } } for (builder.include_paths.toSliceConst()) |include_path| { @@ -1402,17 +1440,6 @@ pub const LibExeObjStep = struct { zig_args.append(lib_path) catch unreachable; } - for (self.full_path_libs.toSliceConst()) |full_path_lib| { - try zig_args.append("--library"); - try zig_args.append(builder.pathFromRoot(full_path_lib)); - - const full_path_lib_abs = builder.pathFromRoot(full_path_lib); - if (os.path.dirname(full_path_lib_abs)) |dirname| { - try zig_args.append("-rpath"); - try zig_args.append(dirname); - } - } - if (self.target.isDarwin()) { var it = self.frameworks.iterator(); while (it.next()) |entry| { @@ -1435,42 +1462,96 @@ pub const LibExeObjStep = struct { try zig_args.append(builder.pathFromRoot(dir)); } - try builder.spawnChild(zig_args.toSliceConst()); + if (self.output_dir) |output_dir| { + try zig_args.append("--output-dir"); + try zig_args.append(output_dir); + + try builder.spawnChild(zig_args.toSliceConst()); + } else if (self.kind == Kind.Test) { + try builder.spawnChild(zig_args.toSliceConst()); + } else { + try zig_args.append("--cache"); + try zig_args.append("on"); + + const output_path_nl = try builder.exec(zig_args.toSliceConst()); + const output_path = mem.trimRight(u8, output_path_nl, "\r\n"); + self.output_dir = os.path.dirname(output_path).?; + } if (self.kind == Kind.Lib and !self.static and self.target.wantSharedLibSymLinks()) { - try doAtomicSymLinks(builder.allocator, output_path, self.major_only_filename, self.name_only_filename); + try doAtomicSymLinks(builder.allocator, self.getOutputPath(), self.major_only_filename, self.name_only_filename); } } }; -pub const CommandStep = struct { +pub const RunStep = struct { step: Step, builder: *Builder, - argv: [][]const u8, + + /// See also addArg and addArgs to modifying this directly + argv: ArrayList(Arg), + + /// Set this to modify the current working directory cwd: ?[]const u8, - env_map: *const BufMap, - /// ::argv is copied. - pub fn create(builder: *Builder, cwd: ?[]const u8, env_map: *const BufMap, argv: []const []const u8) *CommandStep { - const self = builder.allocator.create(CommandStep) catch unreachable; - self.* = CommandStep{ + /// Override this field to modify the environment, or use setEnvironmentVariable + env_map: ?*const BufMap, + + pub const Arg = union(enum) { + Artifact: *LibExeObjStep, + Bytes: []u8, + }; + + pub fn create(builder: *Builder, name: []const u8) *RunStep { + const self = builder.allocator.create(RunStep) catch unreachable; + self.* = RunStep{ .builder = builder, - .step = Step.init(argv[0], builder.allocator, make), - .argv = builder.allocator.alloc([]u8, argv.len) catch unreachable, - .cwd = cwd, - .env_map = env_map, + .step = Step.init(name, builder.allocator, make), + .argv = ArrayList(Arg).init(builder.allocator), + .cwd = null, + .env_map = null, }; - - mem.copy([]const u8, self.argv, argv); - self.step.name = self.argv[0]; return self; } + pub fn addArtifactArg(self: *RunStep, artifact: *LibExeObjStep) void { + self.argv.append(Arg{ .Artifact = artifact }) catch unreachable; + self.step.dependOn(&artifact.step); + } + + pub fn addArg(self: *RunStep, arg: []const u8) void { + self.argv.append(Arg{ .Bytes = self.builder.dupe(arg) }) catch unreachable; + } + + pub fn addArgs(self: *RunStep, args: []const []const u8) void { + for (args) |arg| { + self.addArg(arg); + } + } + + pub fn setEnvironmentVariable(self: *RunStep, key: []const u8, value: []const u8) void { + const env_map = self.env_map orelse blk: { + const env_map = os.getEnvMap(allocator) catch unreachable; + self.env_map = env_map; + break :blk env_map; + }; + env_map.set(key, value) catch unreachable; + } + fn make(step: *Step) !void { - const self = @fieldParentPtr(CommandStep, "step", step); + const self = @fieldParentPtr(RunStep, "step", step); const cwd = if (self.cwd) |cwd| self.builder.pathFromRoot(cwd) else self.builder.build_root; - return self.builder.spawnChildEnvMap(cwd, self.env_map, self.argv); + + var argv = ArrayList([]const u8).init(self.builder.allocator); + for (self.argv.toSlice()) |arg| { + switch (arg) { + Arg.Bytes => |bytes| try argv.append(bytes), + Arg.Artifact => |artifact| try argv.append(artifact.getOutputPath()), + } + } + + return self.builder.spawnChildEnvMap(cwd, self.env_map orelse self.builder.env_map, argv.toSliceConst()); } }; diff --git a/std/special/init-exe/build.zig b/std/special/init-exe/build.zig index 21a2600562..1ee3289643 100644 --- a/std/special/init-exe/build.zig +++ b/std/special/init-exe/build.zig @@ -5,10 +5,10 @@ pub fn build(b: *Builder) void { const exe = b.addExecutable("$", "src/main.zig"); exe.setBuildMode(mode); + const run_cmd = exe.run(); + const run_step = b.step("run", "Run the app"); - const run_cmd = b.addCommand(".", b.env_map, [][]const u8{exe.getOutputPath()}); run_step.dependOn(&run_cmd.step); - run_cmd.step.dependOn(&exe.step); b.default_step.dependOn(&exe.step); b.installArtifact(exe); diff --git a/test/cli.zig b/test/cli.zig index 1520b3bde0..a36c810cd8 100644 --- a/test/cli.zig +++ b/test/cli.zig @@ -116,12 +116,12 @@ fn testGodboltApi(zig_exe: []const u8, dir_path: []const u8) anyerror!void { const args = [][]const u8{ zig_exe, "build-obj", "--cache-dir", dir_path, - "--output", example_s_path, - "--output-h", "/dev/null", + "--name", "example", + "--output-dir", dir_path, "--emit", "asm", "-mllvm", "--x86-asm-syntax=intel", "--strip", "--release-fast", - example_zig_path, + example_zig_path, "--disable-gen-h", }; _ = try exec(dir_path, args); diff --git a/test/standalone/load_dynamic_library/build.zig b/test/standalone/load_dynamic_library/build.zig index 2d47a893f2..109c742c6f 100644 --- a/test/standalone/load_dynamic_library/build.zig +++ b/test/standalone/load_dynamic_library/build.zig @@ -9,12 +9,8 @@ pub fn build(b: *Builder) void { const main = b.addExecutable("main", "main.zig"); main.setBuildMode(opts); - const run = b.addCommand(".", b.env_map, [][]const u8{ - main.getOutputPath(), - lib.getOutputPath(), - }); - run.step.dependOn(&lib.step); - run.step.dependOn(&main.step); + const run = main.run(); + run.addArtifactArg(lib); const test_step = b.step("test", "Test the program"); test_step.dependOn(&run.step); diff --git a/test/standalone/pkg_import/build.zig b/test/standalone/pkg_import/build.zig index e0b3885dc3..7529d106f9 100644 --- a/test/standalone/pkg_import/build.zig +++ b/test/standalone/pkg_import/build.zig @@ -9,8 +9,7 @@ pub fn build(b: *Builder) void { exe.setBuildMode(b.standardReleaseOptions()); exe.setBuildMode(b.standardReleaseOptions()); - const run = b.addCommand(".", b.env_map, [][]const u8{exe.getOutputPath()}); - run.step.dependOn(&exe.step); + const run = exe.run(); const test_step = b.step("test", "Test it"); test_step.dependOn(&run.step); diff --git a/test/tests.zig b/test/tests.zig index d3a27f3e94..ba713902ba 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -12,6 +12,7 @@ const fmt = std.fmt; const ArrayList = std.ArrayList; const builtin = @import("builtin"); const Mode = builtin.Mode; +const LibExeObjStep = build.LibExeObjStep; const compare_output = @import("compare_output.zig"); const build_examples = @import("build_examples.zig"); @@ -111,12 +112,11 @@ pub fn addCliTests(b: *build.Builder, test_filter: ?[]const u8, modes: []const M const step = b.step("test-cli", "Test the command line interface"); const exe = b.addExecutable("test-cli", "test/cli.zig"); - const run_cmd = b.addCommand(null, b.env_map, [][]const u8{ - b.pathFromRoot(exe.getOutputPath()), + const run_cmd = exe.run(); + run_cmd.addArgs([][]const u8{ os.path.realAlloc(b.allocator, b.zig_exe) catch unreachable, b.pathFromRoot(b.cache_root), }); - run_cmd.step.dependOn(&exe.step); step.dependOn(&run_cmd.step); return step; @@ -246,24 +246,31 @@ pub const CompareOutputContext = struct { const RunCompareOutputStep = struct { step: build.Step, context: *CompareOutputContext, - exe_path: []const u8, + exe: *LibExeObjStep, name: []const u8, expected_output: []const u8, test_index: usize, cli_args: []const []const u8, - pub fn create(context: *CompareOutputContext, exe_path: []const u8, name: []const u8, expected_output: []const u8, cli_args: []const []const u8) *RunCompareOutputStep { + pub fn create( + context: *CompareOutputContext, + exe: *LibExeObjStep, + name: []const u8, + expected_output: []const u8, + cli_args: []const []const u8, + ) *RunCompareOutputStep { const allocator = context.b.allocator; const ptr = allocator.create(RunCompareOutputStep) catch unreachable; ptr.* = RunCompareOutputStep{ .context = context, - .exe_path = exe_path, + .exe = exe, .name = name, .expected_output = expected_output, .test_index = context.test_index, .step = build.Step.init("RunCompareOutput", allocator, make), .cli_args = cli_args, }; + ptr.step.dependOn(&exe.step); context.test_index += 1; return ptr; } @@ -272,7 +279,7 @@ pub const CompareOutputContext = struct { const self = @fieldParentPtr(RunCompareOutputStep, "step", step); const b = self.context.b; - const full_exe_path = b.pathFromRoot(self.exe_path); + const full_exe_path = self.exe.getOutputPath(); var args = ArrayList([]const u8).init(b.allocator); defer args.deinit(); @@ -336,21 +343,21 @@ pub const CompareOutputContext = struct { const RuntimeSafetyRunStep = struct { step: build.Step, context: *CompareOutputContext, - exe_path: []const u8, + exe: *LibExeObjStep, name: []const u8, test_index: usize, - pub fn create(context: *CompareOutputContext, exe_path: []const u8, name: []const u8) *RuntimeSafetyRunStep { + pub fn create(context: *CompareOutputContext, exe: *LibExeObjStep, name: []const u8) *RuntimeSafetyRunStep { const allocator = context.b.allocator; const ptr = allocator.create(RuntimeSafetyRunStep) catch unreachable; ptr.* = RuntimeSafetyRunStep{ .context = context, - .exe_path = exe_path, + .exe = exe, .name = name, .test_index = context.test_index, .step = build.Step.init("RuntimeSafetyRun", allocator, make), }; - + ptr.step.dependOn(&exe.step); context.test_index += 1; return ptr; } @@ -359,11 +366,11 @@ pub const CompareOutputContext = struct { const self = @fieldParentPtr(RuntimeSafetyRunStep, "step", step); const b = self.context.b; - const full_exe_path = b.pathFromRoot(self.exe_path); + const full_exe_path = self.exe.getOutputPath(); warn("Test {}/{} {}...", self.test_index + 1, self.context.test_index, self.name); - const child = os.ChildProcess.init([][]u8{full_exe_path}, b.allocator) catch unreachable; + const child = os.ChildProcess.init([][]const u8{full_exe_path}, b.allocator) catch unreachable; defer child.deinit(); child.env_map = b.env_map; @@ -463,8 +470,13 @@ pub const CompareOutputContext = struct { exe.step.dependOn(&write_src.step); } - const run_and_cmp_output = RunCompareOutputStep.create(self, exe.getOutputPath(), annotated_case_name, case.expected_output, case.cli_args); - run_and_cmp_output.step.dependOn(&exe.step); + const run_and_cmp_output = RunCompareOutputStep.create( + self, + exe, + annotated_case_name, + case.expected_output, + case.cli_args, + ); self.step.dependOn(&run_and_cmp_output.step); }, @@ -490,8 +502,13 @@ pub const CompareOutputContext = struct { exe.step.dependOn(&write_src.step); } - const run_and_cmp_output = RunCompareOutputStep.create(self, exe.getOutputPath(), annotated_case_name, case.expected_output, case.cli_args); - run_and_cmp_output.step.dependOn(&exe.step); + const run_and_cmp_output = RunCompareOutputStep.create( + self, + exe, + annotated_case_name, + case.expected_output, + case.cli_args, + ); self.step.dependOn(&run_and_cmp_output.step); } @@ -516,8 +533,7 @@ pub const CompareOutputContext = struct { exe.step.dependOn(&write_src.step); } - const run_and_cmp_output = RuntimeSafetyRunStep.create(self, exe.getOutputPath(), annotated_case_name); - run_and_cmp_output.step.dependOn(&exe.step); + const run_and_cmp_output = RuntimeSafetyRunStep.create(self, exe, annotated_case_name); self.step.dependOn(&run_and_cmp_output.step); }, @@ -608,10 +624,6 @@ pub const CompileErrorContext = struct { b.allocator, [][]const u8{ b.cache_root, self.case.sources.items[0].filename }, ) catch unreachable; - const obj_path = os.path.join( - b.allocator, - [][]const u8{ b.cache_root, "test.o" }, - ) catch unreachable; var zig_args = ArrayList([]const u8).init(b.allocator); zig_args.append(b.zig_exe) catch unreachable; @@ -628,8 +640,8 @@ pub const CompileErrorContext = struct { zig_args.append("--name") catch unreachable; zig_args.append("test") catch unreachable; - zig_args.append("--output") catch unreachable; - zig_args.append(b.pathFromRoot(obj_path)) catch unreachable; + zig_args.append("--output-dir") catch unreachable; + zig_args.append(b.pathFromRoot(b.cache_root)) catch unreachable; switch (self.build_mode) { Mode.Debug => {}, @@ -851,7 +863,7 @@ pub const BuildExamplesContext = struct { zig_args.append("--verbose") catch unreachable; } - const run_cmd = b.addCommand(null, b.env_map, zig_args.toSliceConst()); + const run_cmd = b.addSystemCommand(zig_args.toSliceConst()); const log_step = b.addLog("PASS {}\n", annotated_case_name); log_step.step.dependOn(&run_cmd.step); @@ -1118,23 +1130,28 @@ pub const GenHContext = struct { const GenHCmpOutputStep = struct { step: build.Step, context: *GenHContext, - h_path: []const u8, + obj: *LibExeObjStep, name: []const u8, test_index: usize, case: *const TestCase, - pub fn create(context: *GenHContext, h_path: []const u8, name: []const u8, case: *const TestCase) *GenHCmpOutputStep { + pub fn create( + context: *GenHContext, + obj: *LibExeObjStep, + name: []const u8, + case: *const TestCase, + ) *GenHCmpOutputStep { const allocator = context.b.allocator; const ptr = allocator.create(GenHCmpOutputStep) catch unreachable; ptr.* = GenHCmpOutputStep{ .step = build.Step.init("ParseCCmpOutput", allocator, make), .context = context, - .h_path = h_path, + .obj = obj, .name = name, .test_index = context.test_index, .case = case, }; - + ptr.step.dependOn(&obj.step); context.test_index += 1; return ptr; } @@ -1145,7 +1162,7 @@ pub const GenHContext = struct { warn("Test {}/{} {}...", self.test_index + 1, self.context.test_index, self.name); - const full_h_path = b.pathFromRoot(self.h_path); + const full_h_path = self.obj.getOutputHPath(); const actual_h = try io.readFileAlloc(b.allocator, full_h_path); for (self.case.expected_lines.toSliceConst()) |expected_line| { @@ -1218,8 +1235,7 @@ pub const GenHContext = struct { obj.step.dependOn(&write_src.step); } - const cmp_h = GenHCmpOutputStep.create(self, obj.getOutputHPath(), annotated_case_name, case); - cmp_h.step.dependOn(&obj.step); + const cmp_h = GenHCmpOutputStep.create(self, obj, annotated_case_name, case); self.step.dependOn(&cmp_h.step); } From 85d23e68eecc8b2af89a3836296ac0689ec7a5b9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 8 Mar 2019 23:40:39 -0500 Subject: [PATCH 2/5] fix .d file parsing and string literal ast rendering --- src/ast_render.cpp | 8 ++++---- src/cache_hash.cpp | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ast_render.cpp b/src/ast_render.cpp index 76a0622058..84a952fa99 100644 --- a/src/ast_render.cpp +++ b/src/ast_render.cpp @@ -317,7 +317,7 @@ static bool is_digit(uint8_t c) { static bool is_printable(uint8_t c) { static const uint8_t printables[] = - " abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.~`!@#$%^&*()_-+=\\{}[];'\"?/<>,"; + " abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.~`!@#$%^&*()_-+=\\{}[];'\"?/<>,:"; for (size_t i = 0; i < array_length(printables); i += 1) { if (c == printables[i]) return true; } @@ -328,9 +328,7 @@ static void string_literal_escape(Buf *source, Buf *dest) { buf_resize(dest, 0); for (size_t i = 0; i < buf_len(source); i += 1) { uint8_t c = *((uint8_t*)buf_ptr(source) + i); - if (is_printable(c)) { - buf_append_char(dest, c); - } else if (c == '\'') { + if (c == '\'') { buf_append_str(dest, "\\'"); } else if (c == '"') { buf_append_str(dest, "\\\""); @@ -350,6 +348,8 @@ static void string_literal_escape(Buf *source, Buf *dest) { buf_append_str(dest, "\\t"); } else if (c == '\v') { buf_append_str(dest, "\\v"); + } else if (is_printable(c)) { + buf_append_char(dest, c); } else { buf_appendf(dest, "\\x%x", (int)c); } diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index cd336b71ae..6c9cf12962 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -472,10 +472,11 @@ Error cache_add_dep_file(CacheHash *ch, Buf *dep_file_path, bool verbose) { } } else { // sometimes there are multiple files on the same line; we actually need space tokenization. - SplitIterator line_it = memSplit(opt_line.value, str(" \t\\")); + SplitIterator line_it = memSplit(opt_line.value, str(" \t")); Slice filename; while (SplitIterator_next(&line_it).unwrap(&filename)) { Buf *filename_buf = buf_create_from_slice(filename); + if (buf_eql_str(filename_buf, "\\")) continue; if ((err = cache_add_file(ch, filename_buf))) { if (verbose) { fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(filename_buf), err_str(err)); From 5046aa9403e98fec7a744f35faa9d17925c9079d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 9 Mar 2019 00:42:14 -0500 Subject: [PATCH 3/5] fix running things with zig build on Windows Windows doesn't have rpaths for DLLs so we instead add search paths to Path environment variable when running an executable that depends on DLLs built with zig build. --- std/build.zig | 53 ++++++++++++++++++++++++++++++++++------- std/os/windows/util.zig | 8 ++++++- 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/std/build.zig b/std/build.zig index 8ff742d5a2..e7d73b37c0 100644 --- a/std/build.zig +++ b/std/build.zig @@ -1060,6 +1060,7 @@ pub const LibExeObjStep = struct { /// Add command line arguments with `addArg`. pub fn run(exe: *LibExeObjStep) *RunStep { assert(exe.kind == Kind.Exe); + assert(exe.target == Target.Native); const run_step = RunStep.create(exe.builder, exe.builder.fmt("run {}", exe.step.name)); run_step.addArtifactArg(exe); return run_step; @@ -1276,7 +1277,7 @@ pub const LibExeObjStep = struct { LibExeObjStep.Kind.Lib => { if (other.static or self.target.isWindows()) { try zig_args.append("--object"); - try zig_args.append(other.getOutputPath()); + try zig_args.append(other.getOutputLibPath()); } else { const full_path_lib = other.getOutputPath(); try zig_args.append("--library"); @@ -1495,7 +1496,7 @@ pub const RunStep = struct { cwd: ?[]const u8, /// Override this field to modify the environment, or use setEnvironmentVariable - env_map: ?*const BufMap, + env_map: ?*BufMap, pub const Arg = union(enum) { Artifact: *LibExeObjStep, @@ -1529,12 +1530,28 @@ pub const RunStep = struct { } } - pub fn setEnvironmentVariable(self: *RunStep, key: []const u8, value: []const u8) void { - const env_map = self.env_map orelse blk: { - const env_map = os.getEnvMap(allocator) catch unreachable; - self.env_map = env_map; - break :blk env_map; + pub fn addPathDir(self: *RunStep, search_path: []const u8) void { + const PATH = if (builtin.os == builtin.Os.windows) "Path" else "PATH"; + const env_map = self.getEnvMap(); + const prev_path = env_map.get(PATH) orelse { + env_map.set(PATH, search_path) catch unreachable; + return; }; + const new_path = self.builder.fmt("{}" ++ [1]u8{os.path.delimiter} ++ "{}", prev_path, search_path); + env_map.set(PATH, new_path) catch unreachable; + } + + pub fn getEnvMap(self: *RunStep) *BufMap { + return self.env_map orelse { + const env_map = self.builder.allocator.create(BufMap) catch unreachable; + env_map.* = os.getEnvMap(self.builder.allocator) catch unreachable; + self.env_map = env_map; + return env_map; + }; + } + + pub fn setEnvironmentVariable(self: *RunStep, key: []const u8, value: []const u8) void { + const env_map = self.getEnvMap(); env_map.set(key, value) catch unreachable; } @@ -1547,12 +1564,32 @@ pub const RunStep = struct { for (self.argv.toSlice()) |arg| { switch (arg) { Arg.Bytes => |bytes| try argv.append(bytes), - Arg.Artifact => |artifact| try argv.append(artifact.getOutputPath()), + Arg.Artifact => |artifact| { + if (artifact.target.isWindows()) { + // On Windows we don't have rpaths so we have to add .dll search paths to PATH + self.addPathForDynLibs(artifact); + } + try argv.append(artifact.getOutputPath()); + }, } } return self.builder.spawnChildEnvMap(cwd, self.env_map orelse self.builder.env_map, argv.toSliceConst()); } + + fn addPathForDynLibs(self: *RunStep, artifact: *LibExeObjStep) void { + for (artifact.link_objects.toSliceConst()) |link_object| { + switch (link_object) { + LibExeObjStep.LinkObject.OtherStep => |other| { + if (other.target.isWindows() and other.isDynamicLibrary()) { + self.addPathDir(os.path.dirname(other.getOutputPath()).?); + self.addPathForDynLibs(other); + } + }, + else => {}, + } + } + } }; const InstallArtifactStep = struct { diff --git a/std/os/windows/util.zig b/std/os/windows/util.zig index 0952343051..24039967a1 100644 --- a/std/os/windows/util.zig +++ b/std/os/windows/util.zig @@ -167,7 +167,7 @@ pub fn windowsOpen( pub fn createWindowsEnvBlock(allocator: *mem.Allocator, env_map: *const BufMap) ![]u16 { // count bytes needed const max_chars_needed = x: { - var max_chars_needed: usize = 1; // 1 for the final null byte + var max_chars_needed: usize = 4; // 4 for the final 4 null bytes var it = env_map.iterator(); while (it.next()) |pair| { // +1 for '=' @@ -191,6 +191,12 @@ pub fn createWindowsEnvBlock(allocator: *mem.Allocator, env_map: *const BufMap) } result[i] = 0; i += 1; + result[i] = 0; + i += 1; + result[i] = 0; + i += 1; + result[i] = 0; + i += 1; return allocator.shrink(u16, result, i); } From 94e52dba85e4b81f5c4f2576c91df039604d06fb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 9 Mar 2019 01:00:45 -0500 Subject: [PATCH 4/5] fix docgen and fix unnecessarily adding .root suffix to objects --- doc/docgen.zig | 26 ++++++++++++++------------ src/codegen.cpp | 14 +++++--------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index cea456a98d..527802ec0d 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -1011,8 +1011,10 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var zig_exe, "build-exe", tmp_source_file_name, - "--output", - tmp_bin_file_name, + "--output-dir", + tmp_dir_name, + "--name", + code.name, }); try out.print("
$ zig build-exe {}.zig", code.name);
                         switch (code.mode) {
@@ -1085,8 +1087,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
                             zig_exe,
                             "test",
                             tmp_source_file_name,
-                            "--output",
-                            test_out_path,
+                            "--output-dir",
+                            tmp_dir_name,
                         });
                         try out.print("
$ zig test {}.zig", code.name);
                         switch (code.mode) {
@@ -1122,8 +1124,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
                             "--color",
                             "on",
                             tmp_source_file_name,
-                            "--output",
-                            test_out_path,
+                            "--output-dir",
+                            tmp_dir_name,
                         });
                         try out.print("
$ zig test {}.zig", code.name);
                         switch (code.mode) {
@@ -1179,8 +1181,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
                             zig_exe,
                             "test",
                             tmp_source_file_name,
-                            "--output",
-                            test_out_path,
+                            "--output-dir",
+                            tmp_dir_name,
                         });
                         switch (code.mode) {
                             builtin.Mode.Debug => {},
@@ -1239,10 +1241,10 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
                             tmp_source_file_name,
                             "--color",
                             "on",
-                            "--output",
-                            tmp_obj_file_name,
-                            "--output-h",
-                            output_h_file_name,
+                            "--name",
+                            code.name,
+                            "--output-dir",
+                            tmp_dir_name,
                         });
 
                         if (!code.is_inline) {
diff --git a/src/codegen.cpp b/src/codegen.cpp
index 30b1c87662..719eead7f2 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -9120,18 +9120,14 @@ static void resolve_out_paths(CodeGen *g) {
                         buf_init_from_buf(&g->output_file_path, g->link_objects.at(0));
                         return;
                     }
-                    if (!need_llvm_module(g) || (g->enable_cache && g->link_objects.length == 0)) {
-                        // Either we're not creating an object file from our LLVM Module,
-                        // or we have caching enabled and do not need to link objects together.
-                        // In both cases, the output file path and object file path basename can match.
-                        buf_append_str(o_basename, target_o_file_ext(g->zig_target));
-                        buf_append_str(out_basename, target_o_file_ext(g->zig_target));
-                    } else {
+                    if (need_llvm_module(g) && g->link_objects.length != 0 && !g->enable_cache &&
+                        buf_eql_buf(o_basename, out_basename))
+                    {
                         // make it not collide with main output object
                         buf_append_str(o_basename, ".root");
-                        buf_append_str(o_basename, target_o_file_ext(g->zig_target));
-                        buf_append_str(out_basename, target_o_file_ext(g->zig_target));
                     }
+                    buf_append_str(o_basename, target_o_file_ext(g->zig_target));
+                    buf_append_str(out_basename, target_o_file_ext(g->zig_target));
                     break;
                 case OutTypeExe:
                     buf_append_str(o_basename, target_o_file_ext(g->zig_target));

From a45807db22cf92167734d2055b836fa0aba8cf30 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Sat, 9 Mar 2019 01:15:18 -0500
Subject: [PATCH 5/5] disable flaky event test until post coroutine rewrite

---
 std/event/future.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/std/event/future.zig b/std/event/future.zig
index 0e1b928898..e288e1ba5a 100644
--- a/std/event/future.zig
+++ b/std/event/future.zig
@@ -86,7 +86,7 @@ pub fn Future(comptime T: type) type {
 
 test "std.event.Future" {
     // https://github.com/ziglang/zig/issues/1908
-    if (builtin.single_threaded) return error.SkipZigTest;
+    if (builtin.single_threaded or builtin.os != builtin.Os.linux) return error.SkipZigTest;
 
     var da = std.heap.DirectAllocator.init();
     defer da.deinit();