From 23058d8b435266852d4ca69ee89f8c3d27f52e24 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 11 Dec 2017 23:34:59 -0500 Subject: [PATCH] self-hosted: link with LLVM --- build.zig | 181 ++++++++++++++++++++++++++++++++++++- src-self-hosted/c.zig | 7 ++ src-self-hosted/main.zig | 97 ++------------------ src-self-hosted/parser.zig | 95 +++++++++++++++++++ src-self-hosted/target.zig | 9 ++ std/build.zig | 18 +++- 6 files changed, 313 insertions(+), 94 deletions(-) create mode 100644 src-self-hosted/c.zig create mode 100644 src-self-hosted/target.zig diff --git a/build.zig b/build.zig index 84c3b05eda..a97142b090 100644 --- a/build.zig +++ b/build.zig @@ -1,6 +1,13 @@ -const Builder = @import("std").build.Builder; +const std = @import("std"); +const Builder = std.build.Builder; const tests = @import("test/tests.zig"); -const os = @import("std").os; +const os = std.os; +const BufMap = std.BufMap; +const warn = std.debug.warn; +const mem = std.mem; +const ArrayList = std.ArrayList; +const Buffer = std.Buffer; +const io = std.io; pub fn build(b: &Builder) { const mode = b.standardReleaseOptions(); @@ -28,6 +35,7 @@ pub fn build(b: &Builder) { var exe = b.addExecutable("zig", "src-self-hosted/main.zig"); exe.setBuildMode(mode); exe.linkSystemLibrary("c"); + dependOnLib(exe, findLLVM(b)); b.default_step.dependOn(&exe.step); b.default_step.dependOn(docs_step); @@ -64,3 +72,172 @@ pub fn build(b: &Builder) { test_step.dependOn(tests.addDebugSafetyTests(b, test_filter)); test_step.dependOn(tests.addTranslateCTests(b, test_filter)); } + +fn dependOnLib(lib_exe_obj: &std.build.LibExeObjStep, dep: &const LibraryDep) { + for (dep.libdirs.toSliceConst()) |lib_dir| { + lib_exe_obj.addLibPath(lib_dir); + } + for (dep.libs.toSliceConst()) |lib| { + lib_exe_obj.linkSystemLibrary(lib); + } + for (dep.includes.toSliceConst()) |include_path| { + lib_exe_obj.addIncludeDir(include_path); + } +} + +const LibraryDep = struct { + libdirs: ArrayList([]const u8), + libs: ArrayList([]const u8), + includes: ArrayList([]const u8), +}; + +fn findLLVM(b: &Builder) -> LibraryDep { + const libs_output = { + const args1 = [][]const u8{"llvm-config-5.0", "--libs", "--system-libs"}; + const args2 = [][]const u8{"llvm-config", "--libs", "--system-libs"}; + const max_output_size = 10 * 1024; + const good_result = exec(b.allocator, args1, null, null, max_output_size) %% |err| { + if (err == error.FileNotFound) { + exec(b.allocator, args2, null, null, max_output_size) %% |err2| { + std.debug.panic("unable to spawn {}: {}\n", args2[0], err2); + } + } else { + std.debug.panic("unable to spawn {}: {}\n", args1[0], err); + } + }; + switch (good_result.term) { + os.ChildProcess.Term.Exited => |code| { + if (code != 0) { + std.debug.panic("llvm-config exited with {}:\n{}\n", code, good_result.stderr); + } + }, + else => { + std.debug.panic("llvm-config failed:\n{}\n", good_result.stderr); + }, + } + good_result.stdout + }; + const includes_output = { + const args1 = [][]const u8{"llvm-config-5.0", "--includedir"}; + const args2 = [][]const u8{"llvm-config", "--includedir"}; + const max_output_size = 10 * 1024; + const good_result = exec(b.allocator, args1, null, null, max_output_size) %% |err| { + if (err == error.FileNotFound) { + exec(b.allocator, args2, null, null, max_output_size) %% |err2| { + std.debug.panic("unable to spawn {}: {}\n", args2[0], err2); + } + } else { + std.debug.panic("unable to spawn {}: {}\n", args1[0], err); + } + }; + switch (good_result.term) { + os.ChildProcess.Term.Exited => |code| { + if (code != 0) { + std.debug.panic("llvm-config --includedir exited with {}:\n{}\n", code, good_result.stderr); + } + }, + else => { + std.debug.panic("llvm-config failed:\n{}\n", good_result.stderr); + }, + } + good_result.stdout + }; + const libdir_output = { + const args1 = [][]const u8{"llvm-config-5.0", "--libdir"}; + const args2 = [][]const u8{"llvm-config", "--libdir"}; + const max_output_size = 10 * 1024; + const good_result = exec(b.allocator, args1, null, null, max_output_size) %% |err| { + if (err == error.FileNotFound) { + exec(b.allocator, args2, null, null, max_output_size) %% |err2| { + std.debug.panic("unable to spawn {}: {}\n", args2[0], err2); + } + } else { + std.debug.panic("unable to spawn {}: {}\n", args1[0], err); + } + }; + switch (good_result.term) { + os.ChildProcess.Term.Exited => |code| { + if (code != 0) { + std.debug.panic("llvm-config --libdir exited with {}:\n{}\n", code, good_result.stderr); + } + }, + else => { + std.debug.panic("llvm-config failed:\n{}\n", good_result.stderr); + }, + } + good_result.stdout + }; + + var result = LibraryDep { + .libs = ArrayList([]const u8).init(b.allocator), + .includes = ArrayList([]const u8).init(b.allocator), + .libdirs = ArrayList([]const u8).init(b.allocator), + }; + { + var it = mem.split(libs_output, " \n"); + while (it.next()) |lib_arg| { + if (mem.startsWith(u8, lib_arg, "-l")) { + %%result.libs.append(lib_arg[2..]); + } + } + } + { + var it = mem.split(includes_output, " \n"); + while (it.next()) |include_arg| { + if (mem.startsWith(u8, include_arg, "-I")) { + %%result.includes.append(include_arg[2..]); + } else { + %%result.includes.append(include_arg); + } + } + } + { + var it = mem.split(libdir_output, " \n"); + while (it.next()) |libdir| { + if (mem.startsWith(u8, libdir, "-L")) { + %%result.libdirs.append(libdir[2..]); + } else { + %%result.libdirs.append(libdir); + } + } + } + return result; +} + + +// TODO move to std lib +const ExecResult = struct { + term: os.ChildProcess.Term, + stdout: []u8, + stderr: []u8, +}; + +fn exec(allocator: &std.mem.Allocator, argv: []const []const u8, cwd: ?[]const u8, env_map: ?&const BufMap, max_output_size: usize) -> %ExecResult { + const child = %%os.ChildProcess.init(argv, allocator); + defer child.deinit(); + + child.stdin_behavior = os.ChildProcess.StdIo.Ignore; + child.stdout_behavior = os.ChildProcess.StdIo.Pipe; + child.stderr_behavior = os.ChildProcess.StdIo.Pipe; + child.cwd = cwd; + child.env_map = env_map; + + %return child.spawn(); + + var stdout = Buffer.initNull(allocator); + var stderr = Buffer.initNull(allocator); + defer Buffer.deinit(&stdout); + defer Buffer.deinit(&stderr); + + var stdout_file_in_stream = io.FileInStream.init(&??child.stdout); + var stderr_file_in_stream = io.FileInStream.init(&??child.stderr); + + %return stdout_file_in_stream.stream.readAllBuffer(&stdout, max_output_size); + %return stderr_file_in_stream.stream.readAllBuffer(&stderr, max_output_size); + + return ExecResult { + .term = %return child.wait(), + .stdout = stdout.toOwnedSlice(), + .stderr = stderr.toOwnedSlice(), + }; +} diff --git a/src-self-hosted/c.zig b/src-self-hosted/c.zig new file mode 100644 index 0000000000..b7e057b941 --- /dev/null +++ b/src-self-hosted/c.zig @@ -0,0 +1,7 @@ +pub use @cImport({ + @cInclude("llvm-c/Core.h"); + @cInclude("llvm-c/Analysis.h"); + @cInclude("llvm-c/Target.h"); + @cInclude("llvm-c/Initialization.h"); + @cInclude("llvm-c/TargetMachine.h"); +}); diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 8a7ce3786a..6c40d2216f 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -1,6 +1,5 @@ const std = @import("std"); const mem = std.mem; -const builtin = @import("builtin"); const io = std.io; const os = std.os; const heap = std.heap; @@ -9,6 +8,7 @@ const Tokenizer = @import("tokenizer.zig").Tokenizer; const Token = @import("tokenizer.zig").Token; const Parser = @import("parser.zig").Parser; const assert = std.debug.assert; +const target = @import("target.zig"); pub fn main() -> %void { main2() %% |err| { @@ -26,6 +26,8 @@ pub fn main2() -> %void { const args = %return os.argsAlloc(allocator); defer os.argsFree(allocator, args); + target.initializeAll(); + const target_file = args[1]; const target_file_buf = %return io.readFileAlloc(target_file, allocator); @@ -66,94 +68,7 @@ pub fn main2() -> %void { %return parser.renderSource(out_stream, root_node); } - -var fixed_buffer_mem: [100 * 1024]u8 = undefined; - -fn testParse(source: []const u8, allocator: &mem.Allocator) -> %[]u8 { - var tokenizer = Tokenizer.init(source); - var parser = Parser.init(&tokenizer, allocator, "(memory buffer)"); - defer parser.deinit(); - - const root_node = %return parser.parse(); - defer parser.freeAst(root_node); - - var buffer = %return std.Buffer.initSize(allocator, 0); - var buffer_out_stream = io.BufferOutStream.init(&buffer); - %return parser.renderSource(&buffer_out_stream.stream, root_node); - return buffer.toOwnedSlice(); -} - -fn testCanonical(source: []const u8) { - const needed_alloc_count = { - // Try it once with unlimited memory, make sure it works - var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]); - var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, @maxValue(usize)); - const result_source = testParse(source, &failing_allocator.allocator) %% @panic("test failed"); - if (!mem.eql(u8, result_source, source)) { - warn("\n====== expected this output: =========\n"); - warn("{}", source); - warn("\n======== instead found this: =========\n"); - warn("{}", result_source); - warn("\n======================================\n"); - @panic("test failed"); - } - failing_allocator.allocator.free(result_source); - failing_allocator.index - }; - - var fail_index = needed_alloc_count; - while (fail_index != 0) { - fail_index -= 1; - var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]); - var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, fail_index); - if (testParse(source, &failing_allocator.allocator)) |_| { - @panic("non-deterministic memory usage"); - } else |err| { - assert(err == error.OutOfMemory); - } - } -} - -test "zig fmt" { - if (builtin.os == builtin.Os.windows and builtin.arch == builtin.Arch.i386) { - // TODO get this test passing - // https://github.com/zig-lang/zig/issues/537 - return; - } - - testCanonical( - \\extern fn puts(s: &const u8) -> c_int; - \\ - ); - - testCanonical( - \\const a = b; - \\pub const a = b; - \\var a = b; - \\pub var a = b; - \\const a: i32 = b; - \\pub const a: i32 = b; - \\var a: i32 = b; - \\pub var a: i32 = b; - \\ - ); - - testCanonical( - \\extern var foo: c_int; - \\ - ); - - testCanonical( - \\fn main(argc: c_int, argv: &&u8) -> c_int { - \\ const a = b; - \\} - \\ - ); - - testCanonical( - \\fn foo(argc: c_int, argv: &&u8) -> c_int { - \\ return 0; - \\} - \\ - ); +test "import other tests" { + _ = @import("parser.zig"); + _ = @import("tokenizer.zig"); } diff --git a/src-self-hosted/parser.zig b/src-self-hosted/parser.zig index 6f1f00646a..4492d8259a 100644 --- a/src-self-hosted/parser.zig +++ b/src-self-hosted/parser.zig @@ -5,6 +5,8 @@ const mem = std.mem; const ast = @import("ast.zig"); const Tokenizer = @import("tokenizer.zig").Tokenizer; const Token = @import("tokenizer.zig").Token; +const builtin = @import("builtin"); +const io = std.io; // TODO when we make parse errors into error types instead of printing directly, // get rid of this @@ -1095,3 +1097,96 @@ pub const Parser = struct { }; +var fixed_buffer_mem: [100 * 1024]u8 = undefined; + +fn testParse(source: []const u8, allocator: &mem.Allocator) -> %[]u8 { + var tokenizer = Tokenizer.init(source); + var parser = Parser.init(&tokenizer, allocator, "(memory buffer)"); + defer parser.deinit(); + + const root_node = %return parser.parse(); + defer parser.freeAst(root_node); + + var buffer = %return std.Buffer.initSize(allocator, 0); + var buffer_out_stream = io.BufferOutStream.init(&buffer); + %return parser.renderSource(&buffer_out_stream.stream, root_node); + return buffer.toOwnedSlice(); +} + +// TODO test for memory leaks +// TODO test for valid frees +fn testCanonical(source: []const u8) { + const needed_alloc_count = { + // Try it once with unlimited memory, make sure it works + var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]); + var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, @maxValue(usize)); + const result_source = testParse(source, &failing_allocator.allocator) %% @panic("test failed"); + if (!mem.eql(u8, result_source, source)) { + warn("\n====== expected this output: =========\n"); + warn("{}", source); + warn("\n======== instead found this: =========\n"); + warn("{}", result_source); + warn("\n======================================\n"); + @panic("test failed"); + } + failing_allocator.allocator.free(result_source); + failing_allocator.index + }; + + // TODO make this pass + //var fail_index = needed_alloc_count; + //while (fail_index != 0) { + // fail_index -= 1; + // var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]); + // var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, fail_index); + // if (testParse(source, &failing_allocator.allocator)) |_| { + // @panic("non-deterministic memory usage"); + // } else |err| { + // assert(err == error.OutOfMemory); + // } + //} +} + +test "zig fmt" { + if (builtin.os == builtin.Os.windows and builtin.arch == builtin.Arch.i386) { + // TODO get this test passing + // https://github.com/zig-lang/zig/issues/537 + return; + } + + testCanonical( + \\extern fn puts(s: &const u8) -> c_int; + \\ + ); + + testCanonical( + \\const a = b; + \\pub const a = b; + \\var a = b; + \\pub var a = b; + \\const a: i32 = b; + \\pub const a: i32 = b; + \\var a: i32 = b; + \\pub var a: i32 = b; + \\ + ); + + testCanonical( + \\extern var foo: c_int; + \\ + ); + + testCanonical( + \\fn main(argc: c_int, argv: &&u8) -> c_int { + \\ const a = b; + \\} + \\ + ); + + testCanonical( + \\fn foo(argc: c_int, argv: &&u8) -> c_int { + \\ return 0; + \\} + \\ + ); +} diff --git a/src-self-hosted/target.zig b/src-self-hosted/target.zig new file mode 100644 index 0000000000..90e2132ac5 --- /dev/null +++ b/src-self-hosted/target.zig @@ -0,0 +1,9 @@ +const c = @import("c.zig"); + +pub fn initializeAll() { + c.LLVMInitializeAllTargets(); + c.LLVMInitializeAllTargetInfos(); + c.LLVMInitializeAllTargetMCs(); + c.LLVMInitializeAllAsmPrinters(); + c.LLVMInitializeAllAsmParsers(); +} diff --git a/std/build.zig b/std/build.zig index 9bdc4b3076..532eca1cc9 100644 --- a/std/build.zig +++ b/std/build.zig @@ -755,6 +755,7 @@ pub const LibExeObjStep = struct { is_zig: bool, cflags: ArrayList([]const u8), include_dirs: ArrayList([]const u8), + lib_paths: ArrayList([]const u8), disable_libc: bool, frameworks: BufSet, @@ -865,6 +866,7 @@ pub const LibExeObjStep = struct { .cflags = ArrayList([]const u8).init(builder.allocator), .source_files = undefined, .include_dirs = ArrayList([]const u8).init(builder.allocator), + .lib_paths = ArrayList([]const u8).init(builder.allocator), .object_src = undefined, .disable_libc = true, }; @@ -888,6 +890,7 @@ pub const LibExeObjStep = struct { .frameworks = BufSet.init(builder.allocator), .full_path_libs = ArrayList([]const u8).init(builder.allocator), .include_dirs = ArrayList([]const u8).init(builder.allocator), + .lib_paths = ArrayList([]const u8).init(builder.allocator), .output_path = null, .out_filename = undefined, .major_only_filename = undefined, @@ -1069,11 +1072,14 @@ pub const LibExeObjStep = struct { %%self.include_dirs.append(self.builder.cache_root); } - // TODO put include_dirs in zig command line pub fn addIncludeDir(self: &LibExeObjStep, path: []const u8) { %%self.include_dirs.append(path); } + pub fn addLibPath(self: &LibExeObjStep, path: []const u8) { + %%self.lib_paths.append(path); + } + pub fn addPackagePath(self: &LibExeObjStep, name: []const u8, pkg_index_path: []const u8) { assert(self.is_zig); @@ -1222,6 +1228,11 @@ pub const LibExeObjStep = struct { %%zig_args.append("--pkg-end"); } + for (self.include_dirs.toSliceConst()) |include_path| { + %%zig_args.append("-isystem"); + %%zig_args.append(self.builder.pathFromRoot(include_path)); + } + for (builder.include_paths.toSliceConst()) |include_path| { %%zig_args.append("-isystem"); %%zig_args.append(builder.pathFromRoot(include_path)); @@ -1232,6 +1243,11 @@ pub const LibExeObjStep = struct { %%zig_args.append(rpath); } + for (self.lib_paths.toSliceConst()) |lib_path| { + %%zig_args.append("--library-path"); + %%zig_args.append(lib_path); + } + for (builder.lib_paths.toSliceConst()) |lib_path| { %%zig_args.append("--library-path"); %%zig_args.append(lib_path);