From d2602b442e7cdea2e74584f9529917d7a53625fd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 6 Feb 2019 14:32:20 -0500 Subject: [PATCH] require running std lib tests coherently this should actually improve CI times a bit too See the description at the top of std/os/startup.zig (deleted in this commit) for a more detailed understanding of what this commit does. --- CMakeLists.txt | 1 - src/codegen.cpp | 18 +++++++++++++----- src/codegen.hpp | 2 +- src/link.cpp | 2 +- src/main.cpp | 12 +++++++++--- std/build.zig | 10 ++++++++++ std/index.zig | 1 - std/os/index.zig | 31 +++++++++++++++++++++---------- std/os/startup.zig | 26 -------------------------- std/special/bootstrap.zig | 14 +++++++------- test/tests.zig | 3 +++ 11 files changed, 65 insertions(+), 55 deletions(-) delete mode 100644 std/os/startup.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index a093bfbfd9..4dd6a1dcfa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -587,7 +587,6 @@ set(ZIG_STD_FILES "os/linux/vdso.zig" "os/linux/x86_64.zig" "os/path.zig" - "os/startup.zig" "os/time.zig" "os/uefi.zig" "os/windows/advapi32.zig" diff --git a/src/codegen.cpp b/src/codegen.cpp index d8fc077efc..5e282160d6 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -88,7 +88,7 @@ static const char *symbols_that_llvm_depends_on[] = { }; CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir) + Buf *zig_lib_dir, Buf *override_std_dir) { CodeGen *g = allocate(1); @@ -96,8 +96,12 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out g->zig_lib_dir = zig_lib_dir; - g->zig_std_dir = buf_alloc(); - os_path_join(zig_lib_dir, buf_create_from_str("std"), g->zig_std_dir); + if (override_std_dir == nullptr) { + g->zig_std_dir = buf_alloc(); + os_path_join(zig_lib_dir, buf_create_from_str("std"), g->zig_std_dir); + } else { + g->zig_std_dir = override_std_dir; + } g->zig_c_headers_dir = buf_alloc(); os_path_join(zig_lib_dir, buf_create_from_str("include"), g->zig_c_headers_dir); @@ -8356,8 +8360,12 @@ static void add_cache_pkg(CodeGen *g, CacheHash *ch, PackageTableEntry *pkg) { if (!entry) break; - cache_buf(ch, entry->key); - add_cache_pkg(g, ch, entry->value); + // TODO: I think we need a more sophisticated detection of + // packages we have already seen + if (entry->value != pkg) { + cache_buf(ch, entry->key); + add_cache_pkg(g, ch, entry->value); + } } } diff --git a/src/codegen.hpp b/src/codegen.hpp index 6f1cdfb677..4bd8f2dcca 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -15,7 +15,7 @@ #include CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir); + Buf *zig_lib_dir, Buf *override_std_dir); void codegen_set_clang_argv(CodeGen *codegen, const char **args, size_t len); void codegen_set_llvm_argv(CodeGen *codegen, const char **args, size_t len); diff --git a/src/link.cpp b/src/link.cpp index 593f7f309f..58221a99ea 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -42,7 +42,7 @@ static Buf *build_a_raw(CodeGen *parent_gen, const char *aname, Buf *full_path) } CodeGen *child_gen = codegen_create(full_path, child_target, child_out_type, - parent_gen->build_mode, parent_gen->zig_lib_dir); + parent_gen->build_mode, parent_gen->zig_lib_dir, parent_gen->zig_std_dir); child_gen->out_h_path = nullptr; child_gen->verbose_tokenize = parent_gen->verbose_tokenize; diff --git a/src/main.cpp b/src/main.cpp index 81f49089be..73a1d75d42 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -74,6 +74,7 @@ static int print_full_usage(const char *arg0) { " -dirafter [dir] same as -isystem but do it last\n" " -isystem [dir] add additional search path for other .h files\n" " -mllvm [arg] forward an arg to LLVM's option processing\n" + " --override-std-dir [arg] use an alternate Zig standard library\n" "\n" "Link Options:\n" " --dynamic-linker [path] set the path to ld.so\n" @@ -395,6 +396,7 @@ int main(int argc, char **argv) { bool system_linker_hack = false; TargetSubsystem subsystem = TargetSubsystemAuto; bool is_single_threaded = false; + Buf *override_std_dir = nullptr; if (argc >= 2 && strcmp(argv[1], "build") == 0) { Buf zig_exe_path_buf = BUF_INIT; @@ -430,7 +432,8 @@ int main(int argc, char **argv) { Buf *build_runner_path = buf_alloc(); os_path_join(get_zig_special_dir(), buf_create_from_str("build_runner.zig"), build_runner_path); - CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, get_zig_lib_dir()); + CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, get_zig_lib_dir(), + override_std_dir); g->enable_time_report = timing_info; buf_init_from_str(&g->cache_dir, cache_dir ? cache_dir : default_zig_cache_name); codegen_set_out_name(g, buf_create_from_str("build")); @@ -645,6 +648,8 @@ int main(int argc, char **argv) { clang_argv.append(argv[i]); llvm_argv.append(argv[i]); + } else if (strcmp(arg, "--override-std-dir") == 0) { + override_std_dir = buf_create_from_str(argv[i]); } else if (strcmp(arg, "--library-path") == 0 || strcmp(arg, "-L") == 0) { lib_dirs.append(argv[i]); } else if (strcmp(arg, "--library") == 0) { @@ -819,7 +824,7 @@ int main(int argc, char **argv) { switch (cmd) { case CmdBuiltin: { - CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, get_zig_lib_dir()); + CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, get_zig_lib_dir(), override_std_dir); g->is_single_threaded = is_single_threaded; Buf *builtin_source = codegen_generate_builtin_source(g); if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) { @@ -878,7 +883,8 @@ int main(int argc, char **argv) { if (cmd == CmdRun && buf_out_name == nullptr) { buf_out_name = buf_create_from_str("run"); } - CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, get_zig_lib_dir()); + CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, get_zig_lib_dir(), + override_std_dir); g->subsystem = subsystem; if (disable_pic) { diff --git a/std/build.zig b/std/build.zig index 5246d97339..07efcec30d 100644 --- a/std/build.zig +++ b/std/build.zig @@ -1686,6 +1686,7 @@ pub const TestStep = struct { no_rosegment: bool, output_path: ?[]const u8, system_linker_hack: bool, + override_std_dir: ?[]const u8, pub fn init(builder: *Builder, root_src: []const u8) TestStep { const step_name = builder.fmt("test {}", root_src); @@ -1707,6 +1708,7 @@ pub const TestStep = struct { .no_rosegment = false, .output_path = null, .system_linker_hack = false, + .override_std_dir = null, }; } @@ -1737,6 +1739,10 @@ pub const TestStep = struct { self.build_mode = mode; } + pub fn overrideStdDir(self: *TestStep, dir_path: []const u8) void { + self.override_std_dir = dir_path; + } + pub fn setOutputPath(self: *TestStep, file_path: []const u8) void { self.output_path = file_path; @@ -1914,6 +1920,10 @@ pub const TestStep = struct { if (self.system_linker_hack) { try zig_args.append("--system-linker-hack"); } + if (self.override_std_dir) |dir| { + try zig_args.append("--override-std-dir"); + try zig_args.append(builder.pathFromRoot(dir)); + } try builder.spawnChild(zig_args.toSliceConst()); } diff --git a/std/index.zig b/std/index.zig index ef3988460f..2a63244004 100644 --- a/std/index.zig +++ b/std/index.zig @@ -45,7 +45,6 @@ pub const unicode = @import("unicode.zig"); pub const zig = @import("zig/index.zig"); pub const lazyInit = @import("lazy_init.zig").lazyInit; -pub const startup = @import("os/startup.zig"); test "std" { // run tests from these diff --git a/std/os/index.zig b/std/os/index.zig index 68b3409757..d17b6f4f40 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -8,8 +8,9 @@ const is_posix = switch (builtin.os) { }; const os = @This(); -// See the comment in startup.zig for why this does not use the `std` global above. -const startup = @import("std").startup; +comptime { + assert(@import("std") == std); // You have to run the std lib tests with --override-std-dir +} test "std.os" { _ = @import("child_process.zig"); @@ -670,11 +671,14 @@ fn posixExecveErrnoToErr(err: usize) PosixExecveError { } } +pub var linux_elf_aux_maybe: ?[*]std.elf.Auxv = null; +pub var posix_environ_raw: [][*]u8 = undefined; + /// See std.elf for the constants. pub fn linuxGetAuxVal(index: usize) usize { if (builtin.link_libc) { return usize(std.c.getauxval(index)); - } else if (startup.linux_elf_aux_maybe) |auxv| { + } else if (linux_elf_aux_maybe) |auxv| { var i: usize = 0; while (auxv[i].a_type != std.elf.AT_NULL) : (i += 1) { if (auxv[i].a_type == index) @@ -734,7 +738,7 @@ pub fn getEnvMap(allocator: *Allocator) !BufMap { try result.setMove(key, value); } } else { - for (startup.posix_environ_raw) |ptr| { + for (posix_environ_raw) |ptr| { var line_i: usize = 0; while (ptr[line_i] != 0 and ptr[line_i] != '=') : (line_i += 1) {} const key = ptr[0..line_i]; @@ -756,7 +760,7 @@ test "os.getEnvMap" { /// TODO make this go through libc when we have it pub fn getEnvPosix(key: []const u8) ?[]const u8 { - for (startup.posix_environ_raw) |ptr| { + for (posix_environ_raw) |ptr| { var line_i: usize = 0; while (ptr[line_i] != 0 and ptr[line_i] != '=') : (line_i += 1) {} const this_key = ptr[0..line_i]; @@ -1937,14 +1941,14 @@ pub const ArgIteratorPosix = struct { pub fn init() ArgIteratorPosix { return ArgIteratorPosix{ .index = 0, - .count = startup.posix_argv_raw.len, + .count = raw.len, }; } pub fn next(self: *ArgIteratorPosix) ?[]const u8 { if (self.index == self.count) return null; - const s = startup.posix_argv_raw[self.index]; + const s = raw[self.index]; self.index += 1; return cstr.toSlice(s); } @@ -1955,6 +1959,10 @@ pub const ArgIteratorPosix = struct { self.index += 1; return true; } + + /// This is marked as public but actually it's only meant to be used + /// internally by zig's startup code. + pub var raw: [][*]u8 = undefined; }; pub const ArgIteratorWindows = struct { @@ -3000,6 +3008,9 @@ pub const SpawnThreadError = error{ Unexpected, }; +pub var linux_tls_phdr: ?*std.elf.Phdr = null; +pub var linux_tls_img_src: [*]const u8 = undefined; // defined if linux_tls_phdr is + /// caller must call wait on the returned thread /// fn startFn(@typeOf(context)) T /// where T is u8, noreturn, void, or !void @@ -3109,7 +3120,7 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread } // Finally, the Thread Local Storage, if any. if (!Thread.use_pthreads) { - if (startup.linux_tls_phdr) |tls_phdr| { + if (linux_tls_phdr) |tls_phdr| { l = mem.alignForward(l, tls_phdr.p_align); tls_start_offset = l; l += tls_phdr.p_memsz; @@ -3153,8 +3164,8 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread posix.CLONE_THREAD | posix.CLONE_SYSVSEM | posix.CLONE_PARENT_SETTID | posix.CLONE_CHILD_CLEARTID | posix.CLONE_DETACHED; var newtls: usize = undefined; - if (startup.linux_tls_phdr) |tls_phdr| { - @memcpy(@intToPtr([*]u8, mmap_addr + tls_start_offset), startup.linux_tls_img_src, tls_phdr.p_filesz); + if (linux_tls_phdr) |tls_phdr| { + @memcpy(@intToPtr([*]u8, mmap_addr + tls_start_offset), linux_tls_img_src, tls_phdr.p_filesz); thread_ptr.data.tls_end_addr = mmap_addr + mmap_len; newtls = @ptrToInt(&thread_ptr.data.tls_end_addr); flags |= posix.CLONE_SETTLS; diff --git a/std/os/startup.zig b/std/os/startup.zig deleted file mode 100644 index c54d274c5d..0000000000 --- a/std/os/startup.zig +++ /dev/null @@ -1,26 +0,0 @@ -// This file contains global variables that are initialized on startup from -// std/special/bootstrap.zig. There are a few things to be aware of here. -// -// First, when building an object or library, and no entry point is defined -// (such as pub fn main), std/special/bootstrap.zig is not included in the -// compilation. And so these global variables will remain set to the values -// you see here. -// -// Second, when using `zig test` to test the standard library, note that -// `zig test` is self-hosted. This means that it uses std/special/bootstrap.zig -// and an @import("std") from the install directory, which is distinct from -// the standard library files that we are directly testing with `zig test`. -// This means that these global variables would not get set. So the workaround -// here is that references to these globals from the standard library must -// use `@import("std").startup` rather than -// `@import("path/to/std/index.zig").startup` (and rather than the file path of -// this file directly). We also put "std" as a reference to itself in the -// standard library package so that this can work. - -const std = @import("../index.zig"); - -pub var linux_tls_phdr: ?*std.elf.Phdr = null; -pub var linux_tls_img_src: [*]const u8 = undefined; // defined when linux_tls_phdr is non-null -pub var linux_elf_aux_maybe: ?[*]std.elf.Auxv = null; -pub var posix_environ_raw: [][*]u8 = undefined; -pub var posix_argv_raw: [][*]u8 = undefined; diff --git a/std/special/bootstrap.zig b/std/special/bootstrap.zig index 0e84f67481..97699e0cc5 100644 --- a/std/special/bootstrap.zig +++ b/std/special/bootstrap.zig @@ -64,7 +64,7 @@ fn posixCallMainAndExit() noreturn { if (builtin.os == builtin.Os.linux) { // Scan auxiliary vector. const auxv = @ptrCast([*]std.elf.Auxv, envp.ptr + envp_count + 1); - std.startup.linux_elf_aux_maybe = auxv; + std.os.linux_elf_aux_maybe = auxv; var i: usize = 0; var at_phdr: usize = 0; var at_phnum: usize = 0; @@ -87,8 +87,8 @@ fn posixCallMainAndExit() noreturn { // This is marked inline because for some reason LLVM in release mode fails to inline it, // and we want fewer call frames in stack traces. inline fn callMainWithArgs(argc: usize, argv: [*][*]u8, envp: [][*]u8) u8 { - std.startup.posix_argv_raw = argv[0..argc]; - std.startup.posix_environ_raw = envp; + std.os.ArgIteratorPosix.raw = argv[0..argc]; + std.os.posix_environ_raw = envp; return callMain(); } @@ -145,15 +145,15 @@ fn linuxInitializeThreadLocalStorage(at_phdr: usize, at_phnum: usize, at_phent: // TODO look for PT_DYNAMIC when we have https://github.com/ziglang/zig/issues/1917 switch (phdr.p_type) { std.elf.PT_PHDR => base = at_phdr - phdr.p_vaddr, - std.elf.PT_TLS => std.startup.linux_tls_phdr = phdr, + std.elf.PT_TLS => std.os.linux_tls_phdr = phdr, else => continue, } } - const tls_phdr = std.startup.linux_tls_phdr orelse return; - std.startup.linux_tls_img_src = @intToPtr([*]const u8, base + tls_phdr.p_vaddr); + const tls_phdr = std.os.linux_tls_phdr orelse return; + std.os.linux_tls_img_src = @intToPtr([*]const u8, base + tls_phdr.p_vaddr); assert(main_thread_tls_bytes.len >= tls_phdr.p_memsz); // not enough preallocated Thread Local Storage assert(main_thread_tls_align >= tls_phdr.p_align); // preallocated Thread Local Storage not aligned enough - @memcpy(&main_thread_tls_bytes, std.startup.linux_tls_img_src, tls_phdr.p_filesz); + @memcpy(&main_thread_tls_bytes, std.os.linux_tls_img_src, tls_phdr.p_filesz); tls_end_addr = @ptrToInt(&main_thread_tls_bytes) + tls_phdr.p_memsz; linuxSetThreadArea(@ptrToInt(&tls_end_addr)); } diff --git a/test/tests.zig b/test/tests.zig index 548496fa2f..fc188f5550 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -194,6 +194,9 @@ pub fn addPkgTests(b: *build.Builder, test_filter: ?[]const u8, root_src: []cons if (link_libc) { these_tests.linkSystemLibrary("c"); } + if (mem.eql(u8, name, "std")) { + these_tests.overrideStdDir("std"); + } step.dependOn(&these_tests.step); } }