From 64bd134818b77ba56deb613112ec8cdb0c967703 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 30 Jun 2020 17:43:00 +0200 Subject: [PATCH 01/29] Add Dir.Iterator tests This commit adds some `std.fs.Dir.Iterator` tests. --- lib/std/fs/test.zig | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index 52f823a32f..ea08b8f064 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -3,10 +3,47 @@ const testing = std.testing; const builtin = std.builtin; const fs = std.fs; const mem = std.mem; +const wasi = std.os.wasi; +const Dir = std.fs.Dir; const File = std.fs.File; const tmpDir = testing.tmpDir; +test "Dir.Iterator" { + var tmp_dir = tmpDir(.{ .iterate = true }); + defer tmp_dir.cleanup(); + + // First, create a couple of entries to iterate over. + const file = try tmp_dir.dir.createFile("some_file", .{}); + file.close(); + + try tmp_dir.dir.makeDir("some_dir"); + + // Create iterator. + var iter = tmp_dir.dir.iterate(); + var entries = std.ArrayList(Dir.Entry).init(testing.allocator); + defer entries.deinit(); + + while (try iter.next()) |entry| { + try entries.append(entry); + } + + testing.expect(entries.items.len == 2); // note that the Iterator skips '.' and '..' + testing.expect(contains(&entries, Dir.Entry{ .name = "some_file", .kind = Dir.Entry.Kind.File })); + testing.expect(contains(&entries, Dir.Entry{ .name = "some_dir", .kind = Dir.Entry.Kind.Directory })); +} + +fn entry_eql(lhs: Dir.Entry, rhs: Dir.Entry) bool { + return mem.eql(u8, lhs.name, rhs.name) and lhs.kind == rhs.kind; +} + +fn contains(entries: *const std.ArrayList(Dir.Entry), el: Dir.Entry) bool { + for (entries.items) |entry| { + if (entry_eql(entry, el)) return true; + } + return false; +} + test "readAllAlloc" { var tmp_dir = tmpDir(.{}); defer tmp_dir.cleanup(); @@ -237,7 +274,7 @@ test "fs.copyFile" { try expectFileContents(tmp.dir, dest_file2, data); } -fn expectFileContents(dir: fs.Dir, file_path: []const u8, data: []const u8) !void { +fn expectFileContents(dir: Dir, file_path: []const u8, data: []const u8) !void { const contents = try dir.readFileAlloc(testing.allocator, file_path, 1000); defer testing.allocator.free(contents); From b5badd112288b528d75085d82d5f7f1b109e87dc Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 1 Jul 2020 14:15:58 +0200 Subject: [PATCH 02/29] Fix memory corruption in Dir.Iterator test --- lib/std/fs/test.zig | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index ea08b8f064..ddf4a28213 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -5,6 +5,7 @@ const fs = std.fs; const mem = std.mem; const wasi = std.os.wasi; +const ArenaAllocator = std.heap.ArenaAllocator; const Dir = std.fs.Dir; const File = std.fs.File; const tmpDir = testing.tmpDir; @@ -19,13 +20,18 @@ test "Dir.Iterator" { try tmp_dir.dir.makeDir("some_dir"); + var arena = ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + var entries = std.ArrayList(Dir.Entry).init(&arena.allocator); + // Create iterator. var iter = tmp_dir.dir.iterate(); - var entries = std.ArrayList(Dir.Entry).init(testing.allocator); - defer entries.deinit(); - while (try iter.next()) |entry| { - try entries.append(entry); + // We cannot just store `entry` as on Windows, we're re-using the name buffer + // which means we'll actually share the `name` pointer between entries! + const name = try mem.dupe(&arena.allocator, u8, entry.name); + try entries.append(Dir.Entry{ .name = name, .kind = entry.kind }); } testing.expect(entries.items.len == 2); // note that the Iterator skips '.' and '..' From e7d02eae4d99fbae11e91ddd7601e5ac2a392292 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 2 Jul 2020 20:54:57 +0200 Subject: [PATCH 03/29] Update lib/std/fs/test.zig Co-authored-by: Joachim Schmidt --- lib/std/fs/test.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index ddf4a28213..a3cf2e8002 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -30,7 +30,7 @@ test "Dir.Iterator" { while (try iter.next()) |entry| { // We cannot just store `entry` as on Windows, we're re-using the name buffer // which means we'll actually share the `name` pointer between entries! - const name = try mem.dupe(&arena.allocator, u8, entry.name); + const name = try arena.allocator.dupe(u8, entry.name); try entries.append(Dir.Entry{ .name = name, .kind = entry.kind }); } From 485231deaef9b12e013d423facdef1624f16014b Mon Sep 17 00:00:00 2001 From: Vexu Date: Mon, 6 Jul 2020 16:51:53 +0300 Subject: [PATCH 04/29] fix HashMap.clone() --- lib/std/hash_map.zig | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/lib/std/hash_map.zig b/lib/std/hash_map.zig index aaec9a4d58..05550a289a 100644 --- a/lib/std/hash_map.zig +++ b/lib/std/hash_map.zig @@ -529,12 +529,13 @@ pub fn HashMapUnmanaged( } pub fn clone(self: Self, allocator: *Allocator) !Self { - // TODO this can be made more efficient by directly allocating - // the memory slices and memcpying the elements. - var other = Self.init(); - try other.initCapacity(allocator, self.entries.len); - for (self.entries.items) |entry| { - other.putAssumeCapacityNoClobber(entry.key, entry.value); + var other: Self = .{}; + try other.entries.appendSlice(allocator, self.entries.items); + + if (self.index_header) |header| { + const new_header = try IndexHeader.alloc(allocator, header.indexes_len); + other.insertAllEntriesIntoNewHeader(new_header); + other.index_header = new_header; } return other; } @@ -976,6 +977,25 @@ test "ensure capacity" { testing.expect(initial_capacity == map.capacity()); } +test "clone" { + var original = AutoHashMap(i32, i32).init(std.testing.allocator); + defer original.deinit(); + + // put more than `linear_scan_max` so we can test that the index header is properly cloned + var i: u8 = 0; + while (i < 10) : (i += 1) { + try original.putNoClobber(i, i * 10); + } + + var copy = try original.clone(); + defer copy.deinit(); + + i = 0; + while (i < 10) : (i += 1) { + testing.expect(copy.get(i).? == i * 10); + } +} + pub fn getHashPtrAddrFn(comptime K: type) (fn (K) u32) { return struct { fn hash(key: K) u32 { From 0db0258fb2fed196791d3579fb3b893736d28d0a Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Mon, 6 Jul 2020 17:54:06 -0400 Subject: [PATCH 05/29] Remove old comment --- src-self-hosted/test.zig | 3 --- 1 file changed, 3 deletions(-) diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 38392b8aa7..ae5cecce20 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -415,9 +415,6 @@ pub const TestContext = struct { var module = try Module.init(allocator, .{ .target = target, - // This is an Executable, as opposed to e.g. a *library*. This does - // not mean no ZIR is generated. - // // TODO: support tests for object file building, and library builds // and linking. This will require a rework to support multi-file // tests. From b4c571301be7dd2174b2d067d643a2a093797e7e Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 14:55:44 -0400 Subject: [PATCH 06/29] Stage2: Refactor in preparation for C backend --- src-self-hosted/Module.zig | 60 +- src-self-hosted/codegen.zig | 4 +- src-self-hosted/link.zig | 2467 ++++++++++++++++++----------------- src-self-hosted/test.zig | 25 +- test/stage2/cbe.zig | 18 + test/stage2/test.zig | 1 + 6 files changed, 1356 insertions(+), 1219 deletions(-) create mode 100644 test/stage2/cbe.zig diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index f4d65ab7c0..ac7bd00170 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -26,7 +26,7 @@ root_pkg: *Package, /// Module owns this resource. /// The `Scope` is either a `Scope.ZIRModule` or `Scope.File`. root_scope: *Scope, -bin_file: link.ElfFile, +bin_file: *link.File, bin_file_dir: std.fs.Dir, bin_file_path: []const u8, /// It's rare for a decl to be exported, so we save memory by having a sparse map of @@ -45,7 +45,7 @@ export_owners: std.AutoHashMap(*Decl, []*Export), decl_table: DeclTable, optimize_mode: std.builtin.Mode, -link_error_flags: link.ElfFile.ErrorFlags = .{}, +link_error_flags: link.File.ErrorFlags = .{}, work_queue: std.fifo.LinearFifo(WorkItem, .Dynamic), @@ -91,7 +91,7 @@ pub const Export = struct { /// Byte offset into the file that contains the export directive. src: usize, /// Represents the position of the export, if any, in the output file. - link: link.ElfFile.Export, + link: link.File.Elf.Export, /// The Decl that performs the export. Note that this is *not* the Decl being exported. owner_decl: *Decl, /// The Decl being exported. Note this is *not* the Decl performing the export. @@ -169,7 +169,7 @@ pub const Decl = struct { /// Represents the position of the code in the output file. /// This is populated regardless of semantic analysis and code generation. - link: link.ElfFile.TextBlock = link.ElfFile.TextBlock.empty, + link: link.File.Elf.TextBlock = link.File.Elf.TextBlock.empty, contents_hash: std.zig.SrcHash, @@ -722,6 +722,12 @@ pub const AllErrors = struct { } }; +pub const CStandard = enum { + C99, + GNU99, + C11, +}; + pub const InitOptions = struct { target: std.Target, root_pkg: *Package, @@ -732,17 +738,19 @@ pub const InitOptions = struct { object_format: ?std.builtin.ObjectFormat = null, optimize_mode: std.builtin.Mode = .Debug, keep_source_files_loaded: bool = false, + c_standard: ?CStandard = null, }; pub fn init(gpa: *Allocator, options: InitOptions) !Module { const bin_file_dir = options.bin_file_dir orelse std.fs.cwd(); - var bin_file = try link.openBinFilePath(gpa, bin_file_dir, options.bin_file_path, .{ + const bin_file = try link.openBinFilePath(gpa, bin_file_dir, options.bin_file_path, .{ .target = options.target, .output_mode = options.output_mode, .link_mode = options.link_mode orelse .Static, .object_format = options.object_format orelse options.target.getObjectFormat(), + .c_standard = options.c_standard, }); - errdefer bin_file.deinit(); + errdefer bin_file.*.deinit(); const root_scope = blk: { if (mem.endsWith(u8, options.root_pkg.root_src_path, ".zig")) { @@ -793,6 +801,7 @@ pub fn init(gpa: *Allocator, options: InitOptions) !Module { pub fn deinit(self: *Module) void { self.bin_file.deinit(); const allocator = self.allocator; + allocator.destroy(self.bin_file); self.deletion_set.deinit(allocator); self.work_queue.deinit(); @@ -840,7 +849,7 @@ fn freeExportList(allocator: *Allocator, export_list: []*Export) void { } pub fn target(self: Module) std.Target { - return self.bin_file.options.target; + return self.bin_file.options().target; } /// Detect changes to source files, perform semantic analysis, and update the output files. @@ -882,7 +891,7 @@ pub fn update(self: *Module) !void { try self.deleteDecl(decl); } - self.link_error_flags = self.bin_file.error_flags; + self.link_error_flags = self.bin_file.errorFlags(); // If there are any errors, we anticipate the source files being loaded // to report error messages. Otherwise we unload all source files to save memory. @@ -1898,8 +1907,9 @@ fn deleteDeclExports(self: *Module, decl: *Decl) void { self.decl_exports.removeAssertDiscard(exp.exported_decl); } } - - self.bin_file.deleteExport(exp.link); + if (self.bin_file.cast(link.File.Elf)) |elf| { + elf.deleteExport(exp.link); + } if (self.failed_exports.remove(exp)) |entry| { entry.value.destroy(self.allocator); } @@ -1961,7 +1971,7 @@ fn allocateNewDecl( .analysis = .unreferenced, .deletion_flag = false, .contents_hash = contents_hash, - .link = link.ElfFile.TextBlock.empty, + .link = link.File.Elf.TextBlock.empty, .generation = 0, }; return new_decl; @@ -2189,19 +2199,21 @@ fn analyzeExport(self: *Module, scope: *Scope, src: usize, symbol_name: []const } try self.symbol_exports.putNoClobber(symbol_name, new_export); - self.bin_file.updateDeclExports(self, exported_decl, de_gop.entry.value) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - else => { - try self.failed_exports.ensureCapacity(self.failed_exports.items().len + 1); - self.failed_exports.putAssumeCapacityNoClobber(new_export, try ErrorMsg.create( - self.allocator, - src, - "unable to export: {}", - .{@errorName(err)}, - )); - new_export.status = .failed_retryable; - }, - }; + if (self.bin_file.cast(link.File.Elf)) |elf| { + elf.updateDeclExports(self, exported_decl, de_gop.entry.value) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => { + try self.failed_exports.ensureCapacity(self.failed_exports.items().len + 1); + self.failed_exports.putAssumeCapacityNoClobber(new_export, try ErrorMsg.create( + self.allocator, + src, + "unable to export: {}", + .{@errorName(err)}, + )); + new_export.status = .failed_retryable; + }, + }; + } } fn addNewInstArgs( diff --git a/src-self-hosted/codegen.zig b/src-self-hosted/codegen.zig index 8885ed2825..46715b762b 100644 --- a/src-self-hosted/codegen.zig +++ b/src-self-hosted/codegen.zig @@ -21,7 +21,7 @@ pub const Result = union(enum) { }; pub fn generateSymbol( - bin_file: *link.ElfFile, + bin_file: *link.File.Elf, src: usize, typed_value: TypedValue, code: *std.ArrayList(u8), @@ -211,7 +211,7 @@ pub fn generateSymbol( } const Function = struct { - bin_file: *link.ElfFile, + bin_file: *link.File.Elf, target: *const std.Target, mod_fn: *const Module.Fn, code: *std.ArrayList(u8), diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index c615ad35fd..9e273a2b9e 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -21,6 +21,7 @@ pub const Options = struct { /// Used for calculating how much space to reserve for executable program code in case /// the binary file deos not already have such a section. program_code_size_hint: u64 = 256 * 1024, + c_standard: ?Module.CStandard = null, }; /// Attempts incremental linking, if the file already exists. @@ -32,13 +33,19 @@ pub fn openBinFilePath( dir: fs.Dir, sub_path: []const u8, options: Options, -) !ElfFile { +) !*File { const file = try dir.createFile(sub_path, .{ .truncate = false, .read = true, .mode = determineMode(options) }); errdefer file.close(); - var bin_file = try openBinFile(allocator, file, options); - bin_file.owns_file_handle = true; - return bin_file; + if (options.c_standard) |cstd| { + return error.Unimplemented; + } else { + var bin_file = try allocator.create(File.Elf); + errdefer allocator.destroy(bin_file); + bin_file.* = try openBinFile(allocator, file, options); + bin_file.owns_file_handle = true; + return &bin_file.base; + } } /// Atomically overwrites the old file, if present. @@ -80,7 +87,7 @@ pub fn writeFilePath( /// Returns an error if `file` is not already open with +read +write +seek abilities. /// A malicious file is detected as incremental link failure and does not cause Illegal Behavior. /// This operation is not atomic. -pub fn openBinFile(allocator: *Allocator, file: fs.File, options: Options) !ElfFile { +pub fn openBinFile(allocator: *Allocator, file: fs.File, options: Options) !File.Elf { return openBinFileInner(allocator, file, options) catch |err| switch (err) { error.IncrFailed => { return createElfFile(allocator, file, options); @@ -89,514 +96,484 @@ pub fn openBinFile(allocator: *Allocator, file: fs.File, options: Options) !ElfF }; } -pub const ElfFile = struct { - allocator: *Allocator, - file: ?fs.File, - owns_file_handle: bool, - options: Options, - ptr_width: enum { p32, p64 }, +pub const File = struct { + tag: Tag, + pub fn cast(base: *File, comptime T: type) ?*T { + if (base.tag != T.base_tag) + return null; - /// Stored in native-endian format, depending on target endianness needs to be bswapped on read/write. - /// Same order as in the file. - sections: std.ArrayListUnmanaged(elf.Elf64_Shdr) = std.ArrayListUnmanaged(elf.Elf64_Shdr){}, - shdr_table_offset: ?u64 = null, + return @fieldParentPtr(T, "base", base); + } - /// Stored in native-endian format, depending on target endianness needs to be bswapped on read/write. - /// Same order as in the file. - program_headers: std.ArrayListUnmanaged(elf.Elf64_Phdr) = std.ArrayListUnmanaged(elf.Elf64_Phdr){}, - phdr_table_offset: ?u64 = null, - /// The index into the program headers of a PT_LOAD program header with Read and Execute flags - phdr_load_re_index: ?u16 = null, - /// The index into the program headers of the global offset table. - /// It needs PT_LOAD and Read flags. - phdr_got_index: ?u16 = null, - entry_addr: ?u64 = null, + pub fn makeWritable(base: *File, dir: fs.Dir, sub_path: []const u8) !void { + try switch (base.tag) { + .Elf => @fieldParentPtr(Elf, "base", base).makeWritable(dir, sub_path), + else => unreachable, + }; + } - shstrtab: std.ArrayListUnmanaged(u8) = std.ArrayListUnmanaged(u8){}, - shstrtab_index: ?u16 = null, + pub fn makeExecutable(base: *File) !void { + try switch (base.tag) { + .Elf => @fieldParentPtr(Elf, "base", base).makeExecutable(), + else => unreachable, + }; + } - text_section_index: ?u16 = null, - symtab_section_index: ?u16 = null, - got_section_index: ?u16 = null, + pub fn updateDecl(base: *File, module: *Module, decl: *Module.Decl) !void { + try switch (base.tag) { + .Elf => @fieldParentPtr(Elf, "base", base).updateDecl(module, decl), + else => unreachable, + }; + } - /// The same order as in the file. ELF requires global symbols to all be after the - /// local symbols, they cannot be mixed. So we must buffer all the global symbols and - /// write them at the end. These are only the local symbols. The length of this array - /// is the value used for sh_info in the .symtab section. - local_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = std.ArrayListUnmanaged(elf.Elf64_Sym){}, - global_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = std.ArrayListUnmanaged(elf.Elf64_Sym){}, + pub fn allocateDeclIndexes(base: *File, decl: *Module.Decl) !void { + try switch (base.tag) { + .Elf => @fieldParentPtr(Elf, "base", base).allocateDeclIndexes(decl), + else => unreachable, + }; + } - local_symbol_free_list: std.ArrayListUnmanaged(u32) = std.ArrayListUnmanaged(u32){}, - global_symbol_free_list: std.ArrayListUnmanaged(u32) = std.ArrayListUnmanaged(u32){}, - offset_table_free_list: std.ArrayListUnmanaged(u32) = std.ArrayListUnmanaged(u32){}, + pub fn deinit(base: *File) void { + switch (base.tag) { + .Elf => @fieldParentPtr(Elf, "base", base).deinit(), + else => unreachable, + } + } - /// Same order as in the file. The value is the absolute vaddr value. - /// If the vaddr of the executable program header changes, the entire - /// offset table needs to be rewritten. - offset_table: std.ArrayListUnmanaged(u64) = std.ArrayListUnmanaged(u64){}, + pub fn flush(base: *File) !void { + try switch (base.tag) { + .Elf => @fieldParentPtr(Elf, "base", base).flush(), + else => unreachable, + }; + } - phdr_table_dirty: bool = false, - shdr_table_dirty: bool = false, - shstrtab_dirty: bool = false, - offset_table_count_dirty: bool = false, + pub fn freeDecl(base: *File, decl: *Module.Decl) void { + switch (base.tag) { + .Elf => @fieldParentPtr(Elf, "base", base).freeDecl(decl), + else => unreachable, + } + } - error_flags: ErrorFlags = ErrorFlags{}, + pub fn errorFlags(base: *File) ErrorFlags { + return switch (base.tag) { + .Elf => @fieldParentPtr(Elf, "base", base).error_flags, + else => unreachable, + }; + } - /// A list of text blocks that have surplus capacity. This list can have false - /// positives, as functions grow and shrink over time, only sometimes being added - /// or removed from the freelist. - /// - /// A text block has surplus capacity when its overcapacity value is greater than - /// minimum_text_block_size * alloc_num / alloc_den. That is, when it has so - /// much extra capacity, that we could fit a small new symbol in it, itself with - /// ideal_capacity or more. - /// - /// Ideal capacity is defined by size * alloc_num / alloc_den. - /// - /// Overcapacity is measured by actual_capacity - ideal_capacity. Note that - /// overcapacity can be negative. A simple way to have negative overcapacity is to - /// allocate a fresh text block, which will have ideal capacity, and then grow it - /// by 1 byte. It will then have -1 overcapacity. - text_block_free_list: std.ArrayListUnmanaged(*TextBlock) = std.ArrayListUnmanaged(*TextBlock){}, - last_text_block: ?*TextBlock = null, + pub fn options(base: *File) Options { + return switch (base.tag) { + .Elf => @fieldParentPtr(Elf, "base", base).options, + else => unreachable, + }; + } - /// `alloc_num / alloc_den` is the factor of padding when allocating. - const alloc_num = 4; - const alloc_den = 3; - - /// In order for a slice of bytes to be considered eligible to keep metadata pointing at - /// it as a possible place to put new symbols, it must have enough room for this many bytes - /// (plus extra for reserved capacity). - const minimum_text_block_size = 64; - const min_text_capacity = minimum_text_block_size * alloc_num / alloc_den; + pub const Tag = enum { + Elf, + C, + }; pub const ErrorFlags = struct { no_entry_point_found: bool = false, }; + pub const Elf = struct { + pub const base_tag: Tag = .Elf; + base: File = File{ .tag = base_tag }, - pub const TextBlock = struct { - /// Each decl always gets a local symbol with the fully qualified name. - /// The vaddr and size are found here directly. - /// The file offset is found by computing the vaddr offset from the section vaddr - /// the symbol references, and adding that to the file offset of the section. - /// If this field is 0, it means the codegen size = 0 and there is no symbol or - /// offset table entry. - local_sym_index: u32, - /// This field is undefined for symbols with size = 0. - offset_table_index: u32, - /// Points to the previous and next neighbors, based on the `text_offset`. - /// This can be used to find, for example, the capacity of this `TextBlock`. - prev: ?*TextBlock, - next: ?*TextBlock, + allocator: *Allocator, + file: ?fs.File, + owns_file_handle: bool, + options: Options, + ptr_width: enum { p32, p64 }, - pub const empty = TextBlock{ - .local_sym_index = 0, - .offset_table_index = undefined, - .prev = null, - .next = null, - }; + /// Stored in native-endian format, depending on target endianness needs to be bswapped on read/write. + /// Same order as in the file. + sections: std.ArrayListUnmanaged(elf.Elf64_Shdr) = std.ArrayListUnmanaged(elf.Elf64_Shdr){}, + shdr_table_offset: ?u64 = null, - /// Returns how much room there is to grow in virtual address space. - /// File offset relocation happens transparently, so it is not included in - /// this calculation. - fn capacity(self: TextBlock, elf_file: ElfFile) u64 { - const self_sym = elf_file.local_symbols.items[self.local_sym_index]; - if (self.next) |next| { + /// Stored in native-endian format, depending on target endianness needs to be bswapped on read/write. + /// Same order as in the file. + program_headers: std.ArrayListUnmanaged(elf.Elf64_Phdr) = std.ArrayListUnmanaged(elf.Elf64_Phdr){}, + phdr_table_offset: ?u64 = null, + /// The index into the program headers of a PT_LOAD program header with Read and Execute flags + phdr_load_re_index: ?u16 = null, + /// The index into the program headers of the global offset table. + /// It needs PT_LOAD and Read flags. + phdr_got_index: ?u16 = null, + entry_addr: ?u64 = null, + + shstrtab: std.ArrayListUnmanaged(u8) = std.ArrayListUnmanaged(u8){}, + shstrtab_index: ?u16 = null, + + text_section_index: ?u16 = null, + symtab_section_index: ?u16 = null, + got_section_index: ?u16 = null, + + /// The same order as in the file. ELF requires global symbols to all be after the + /// local symbols, they cannot be mixed. So we must buffer all the global symbols and + /// write them at the end. These are only the local symbols. The length of this array + /// is the value used for sh_info in the .symtab section. + local_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = std.ArrayListUnmanaged(elf.Elf64_Sym){}, + global_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = std.ArrayListUnmanaged(elf.Elf64_Sym){}, + + local_symbol_free_list: std.ArrayListUnmanaged(u32) = std.ArrayListUnmanaged(u32){}, + global_symbol_free_list: std.ArrayListUnmanaged(u32) = std.ArrayListUnmanaged(u32){}, + offset_table_free_list: std.ArrayListUnmanaged(u32) = std.ArrayListUnmanaged(u32){}, + + /// Same order as in the file. The value is the absolute vaddr value. + /// If the vaddr of the executable program header changes, the entire + /// offset table needs to be rewritten. + offset_table: std.ArrayListUnmanaged(u64) = std.ArrayListUnmanaged(u64){}, + + phdr_table_dirty: bool = false, + shdr_table_dirty: bool = false, + shstrtab_dirty: bool = false, + offset_table_count_dirty: bool = false, + + error_flags: ErrorFlags = ErrorFlags{}, + + /// A list of text blocks that have surplus capacity. This list can have false + /// positives, as functions grow and shrink over time, only sometimes being added + /// or removed from the freelist. + /// + /// A text block has surplus capacity when its overcapacity value is greater than + /// minimum_text_block_size * alloc_num / alloc_den. That is, when it has so + /// much extra capacity, that we could fit a small new symbol in it, itself with + /// ideal_capacity or more. + /// + /// Ideal capacity is defined by size * alloc_num / alloc_den. + /// + /// Overcapacity is measured by actual_capacity - ideal_capacity. Note that + /// overcapacity can be negative. A simple way to have negative overcapacity is to + /// allocate a fresh text block, which will have ideal capacity, and then grow it + /// by 1 byte. It will then have -1 overcapacity. + text_block_free_list: std.ArrayListUnmanaged(*TextBlock) = std.ArrayListUnmanaged(*TextBlock){}, + last_text_block: ?*TextBlock = null, + + /// `alloc_num / alloc_den` is the factor of padding when allocating. + const alloc_num = 4; + const alloc_den = 3; + + /// In order for a slice of bytes to be considered eligible to keep metadata pointing at + /// it as a possible place to put new symbols, it must have enough room for this many bytes + /// (plus extra for reserved capacity). + const minimum_text_block_size = 64; + const min_text_capacity = minimum_text_block_size * alloc_num / alloc_den; + + pub const TextBlock = struct { + /// Each decl always gets a local symbol with the fully qualified name. + /// The vaddr and size are found here directly. + /// The file offset is found by computing the vaddr offset from the section vaddr + /// the symbol references, and adding that to the file offset of the section. + /// If this field is 0, it means the codegen size = 0 and there is no symbol or + /// offset table entry. + local_sym_index: u32, + /// This field is undefined for symbols with size = 0. + offset_table_index: u32, + /// Points to the previous and next neighbors, based on the `text_offset`. + /// This can be used to find, for example, the capacity of this `TextBlock`. + prev: ?*TextBlock, + next: ?*TextBlock, + + pub const empty = TextBlock{ + .local_sym_index = 0, + .offset_table_index = undefined, + .prev = null, + .next = null, + }; + + /// Returns how much room there is to grow in virtual address space. + /// File offset relocation happens transparently, so it is not included in + /// this calculation. + fn capacity(self: TextBlock, elf_file: File.Elf) u64 { + const self_sym = elf_file.local_symbols.items[self.local_sym_index]; + if (self.next) |next| { + const next_sym = elf_file.local_symbols.items[next.local_sym_index]; + return next_sym.st_value - self_sym.st_value; + } else { + // We are the last block. The capacity is limited only by virtual address space. + return std.math.maxInt(u32) - self_sym.st_value; + } + } + + fn freeListEligible(self: TextBlock, elf_file: File.Elf) bool { + // No need to keep a free list node for the last block. + const next = self.next orelse return false; + const self_sym = elf_file.local_symbols.items[self.local_sym_index]; const next_sym = elf_file.local_symbols.items[next.local_sym_index]; - return next_sym.st_value - self_sym.st_value; - } else { - // We are the last block. The capacity is limited only by virtual address space. - return std.math.maxInt(u32) - self_sym.st_value; + const cap = next_sym.st_value - self_sym.st_value; + const ideal_cap = self_sym.st_size * alloc_num / alloc_den; + if (cap <= ideal_cap) return false; + const surplus = cap - ideal_cap; + return surplus >= min_text_capacity; } - } - - fn freeListEligible(self: TextBlock, elf_file: ElfFile) bool { - // No need to keep a free list node for the last block. - const next = self.next orelse return false; - const self_sym = elf_file.local_symbols.items[self.local_sym_index]; - const next_sym = elf_file.local_symbols.items[next.local_sym_index]; - const cap = next_sym.st_value - self_sym.st_value; - const ideal_cap = self_sym.st_size * alloc_num / alloc_den; - if (cap <= ideal_cap) return false; - const surplus = cap - ideal_cap; - return surplus >= min_text_capacity; - } - }; - - pub const Export = struct { - sym_index: ?u32 = null, - }; - - pub fn deinit(self: *ElfFile) void { - self.sections.deinit(self.allocator); - self.program_headers.deinit(self.allocator); - self.shstrtab.deinit(self.allocator); - self.local_symbols.deinit(self.allocator); - self.global_symbols.deinit(self.allocator); - self.global_symbol_free_list.deinit(self.allocator); - self.local_symbol_free_list.deinit(self.allocator); - self.offset_table_free_list.deinit(self.allocator); - self.text_block_free_list.deinit(self.allocator); - self.offset_table.deinit(self.allocator); - if (self.owns_file_handle) { - if (self.file) |f| f.close(); - } - } - - pub fn makeExecutable(self: *ElfFile) !void { - assert(self.owns_file_handle); - if (self.file) |f| { - f.close(); - self.file = null; - } - } - - pub fn makeWritable(self: *ElfFile, dir: fs.Dir, sub_path: []const u8) !void { - assert(self.owns_file_handle); - if (self.file != null) return; - self.file = try dir.createFile(sub_path, .{ - .truncate = false, - .read = true, - .mode = determineMode(self.options), - }); - } - - /// Returns end pos of collision, if any. - fn detectAllocCollision(self: *ElfFile, start: u64, size: u64) ?u64 { - const small_ptr = self.options.target.cpu.arch.ptrBitWidth() == 32; - const ehdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Ehdr) else @sizeOf(elf.Elf64_Ehdr); - if (start < ehdr_size) - return ehdr_size; - - const end = start + satMul(size, alloc_num) / alloc_den; - - if (self.shdr_table_offset) |off| { - const shdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Shdr) else @sizeOf(elf.Elf64_Shdr); - const tight_size = self.sections.items.len * shdr_size; - const increased_size = satMul(tight_size, alloc_num) / alloc_den; - const test_end = off + increased_size; - if (end > off and start < test_end) { - return test_end; - } - } - - if (self.phdr_table_offset) |off| { - const phdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Phdr) else @sizeOf(elf.Elf64_Phdr); - const tight_size = self.sections.items.len * phdr_size; - const increased_size = satMul(tight_size, alloc_num) / alloc_den; - const test_end = off + increased_size; - if (end > off and start < test_end) { - return test_end; - } - } - - for (self.sections.items) |section| { - const increased_size = satMul(section.sh_size, alloc_num) / alloc_den; - const test_end = section.sh_offset + increased_size; - if (end > section.sh_offset and start < test_end) { - return test_end; - } - } - for (self.program_headers.items) |program_header| { - const increased_size = satMul(program_header.p_filesz, alloc_num) / alloc_den; - const test_end = program_header.p_offset + increased_size; - if (end > program_header.p_offset and start < test_end) { - return test_end; - } - } - return null; - } - - fn allocatedSize(self: *ElfFile, start: u64) u64 { - var min_pos: u64 = std.math.maxInt(u64); - if (self.shdr_table_offset) |off| { - if (off > start and off < min_pos) min_pos = off; - } - if (self.phdr_table_offset) |off| { - if (off > start and off < min_pos) min_pos = off; - } - for (self.sections.items) |section| { - if (section.sh_offset <= start) continue; - if (section.sh_offset < min_pos) min_pos = section.sh_offset; - } - for (self.program_headers.items) |program_header| { - if (program_header.p_offset <= start) continue; - if (program_header.p_offset < min_pos) min_pos = program_header.p_offset; - } - return min_pos - start; - } - - fn findFreeSpace(self: *ElfFile, object_size: u64, min_alignment: u16) u64 { - var start: u64 = 0; - while (self.detectAllocCollision(start, object_size)) |item_end| { - start = mem.alignForwardGeneric(u64, item_end, min_alignment); - } - return start; - } - - fn makeString(self: *ElfFile, bytes: []const u8) !u32 { - try self.shstrtab.ensureCapacity(self.allocator, self.shstrtab.items.len + bytes.len + 1); - const result = self.shstrtab.items.len; - self.shstrtab.appendSliceAssumeCapacity(bytes); - self.shstrtab.appendAssumeCapacity(0); - return @intCast(u32, result); - } - - fn getString(self: *ElfFile, str_off: u32) []const u8 { - assert(str_off < self.shstrtab.items.len); - return mem.spanZ(@ptrCast([*:0]const u8, self.shstrtab.items.ptr + str_off)); - } - - fn updateString(self: *ElfFile, old_str_off: u32, new_name: []const u8) !u32 { - const existing_name = self.getString(old_str_off); - if (mem.eql(u8, existing_name, new_name)) { - return old_str_off; - } - return self.makeString(new_name); - } - - pub fn populateMissingMetadata(self: *ElfFile) !void { - const small_ptr = switch (self.ptr_width) { - .p32 => true, - .p64 => false, }; - const ptr_size: u8 = switch (self.ptr_width) { - .p32 => 4, - .p64 => 8, - }; - if (self.phdr_load_re_index == null) { - self.phdr_load_re_index = @intCast(u16, self.program_headers.items.len); - const file_size = self.options.program_code_size_hint; - const p_align = 0x1000; - const off = self.findFreeSpace(file_size, p_align); - //std.log.debug(.link, "found PT_LOAD free space 0x{x} to 0x{x}\n", .{ off, off + file_size }); - try self.program_headers.append(self.allocator, .{ - .p_type = elf.PT_LOAD, - .p_offset = off, - .p_filesz = file_size, - .p_vaddr = default_entry_addr, - .p_paddr = default_entry_addr, - .p_memsz = file_size, - .p_align = p_align, - .p_flags = elf.PF_X | elf.PF_R, - }); - self.entry_addr = null; - self.phdr_table_dirty = true; - } - if (self.phdr_got_index == null) { - self.phdr_got_index = @intCast(u16, self.program_headers.items.len); - const file_size = @as(u64, ptr_size) * self.options.symbol_count_hint; - // We really only need ptr alignment but since we are using PROGBITS, linux requires - // page align. - const p_align = 0x1000; - const off = self.findFreeSpace(file_size, p_align); - //std.log.debug(.link, "found PT_LOAD free space 0x{x} to 0x{x}\n", .{ off, off + file_size }); - // TODO instead of hard coding the vaddr, make a function to find a vaddr to put things at. - // we'll need to re-use that function anyway, in case the GOT grows and overlaps something - // else in virtual memory. - const default_got_addr = 0x4000000; - try self.program_headers.append(self.allocator, .{ - .p_type = elf.PT_LOAD, - .p_offset = off, - .p_filesz = file_size, - .p_vaddr = default_got_addr, - .p_paddr = default_got_addr, - .p_memsz = file_size, - .p_align = p_align, - .p_flags = elf.PF_R, - }); - self.phdr_table_dirty = true; - } - if (self.shstrtab_index == null) { - self.shstrtab_index = @intCast(u16, self.sections.items.len); - assert(self.shstrtab.items.len == 0); - try self.shstrtab.append(self.allocator, 0); // need a 0 at position 0 - const off = self.findFreeSpace(self.shstrtab.items.len, 1); - //std.log.debug(.link, "found shstrtab free space 0x{x} to 0x{x}\n", .{ off, off + self.shstrtab.items.len }); - try self.sections.append(self.allocator, .{ - .sh_name = try self.makeString(".shstrtab"), - .sh_type = elf.SHT_STRTAB, - .sh_flags = 0, - .sh_addr = 0, - .sh_offset = off, - .sh_size = self.shstrtab.items.len, - .sh_link = 0, - .sh_info = 0, - .sh_addralign = 1, - .sh_entsize = 0, - }); - self.shstrtab_dirty = true; - self.shdr_table_dirty = true; - } - if (self.text_section_index == null) { - self.text_section_index = @intCast(u16, self.sections.items.len); - const phdr = &self.program_headers.items[self.phdr_load_re_index.?]; - try self.sections.append(self.allocator, .{ - .sh_name = try self.makeString(".text"), - .sh_type = elf.SHT_PROGBITS, - .sh_flags = elf.SHF_ALLOC | elf.SHF_EXECINSTR, - .sh_addr = phdr.p_vaddr, - .sh_offset = phdr.p_offset, - .sh_size = phdr.p_filesz, - .sh_link = 0, - .sh_info = 0, - .sh_addralign = phdr.p_align, - .sh_entsize = 0, - }); - self.shdr_table_dirty = true; - } - if (self.got_section_index == null) { - self.got_section_index = @intCast(u16, self.sections.items.len); - const phdr = &self.program_headers.items[self.phdr_got_index.?]; + pub const Export = struct { + sym_index: ?u32 = null, + }; - try self.sections.append(self.allocator, .{ - .sh_name = try self.makeString(".got"), - .sh_type = elf.SHT_PROGBITS, - .sh_flags = elf.SHF_ALLOC, - .sh_addr = phdr.p_vaddr, - .sh_offset = phdr.p_offset, - .sh_size = phdr.p_filesz, - .sh_link = 0, - .sh_info = 0, - .sh_addralign = phdr.p_align, - .sh_entsize = 0, - }); - self.shdr_table_dirty = true; - } - if (self.symtab_section_index == null) { - self.symtab_section_index = @intCast(u16, self.sections.items.len); - const min_align: u16 = if (small_ptr) @alignOf(elf.Elf32_Sym) else @alignOf(elf.Elf64_Sym); - const each_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Sym) else @sizeOf(elf.Elf64_Sym); - const file_size = self.options.symbol_count_hint * each_size; - const off = self.findFreeSpace(file_size, min_align); - //std.log.debug(.link, "found symtab free space 0x{x} to 0x{x}\n", .{ off, off + file_size }); - - try self.sections.append(self.allocator, .{ - .sh_name = try self.makeString(".symtab"), - .sh_type = elf.SHT_SYMTAB, - .sh_flags = 0, - .sh_addr = 0, - .sh_offset = off, - .sh_size = file_size, - // The section header index of the associated string table. - .sh_link = self.shstrtab_index.?, - .sh_info = @intCast(u32, self.local_symbols.items.len), - .sh_addralign = min_align, - .sh_entsize = each_size, - }); - self.shdr_table_dirty = true; - try self.writeSymbol(0); - } - const shsize: u64 = switch (self.ptr_width) { - .p32 => @sizeOf(elf.Elf32_Shdr), - .p64 => @sizeOf(elf.Elf64_Shdr), - }; - const shalign: u16 = switch (self.ptr_width) { - .p32 => @alignOf(elf.Elf32_Shdr), - .p64 => @alignOf(elf.Elf64_Shdr), - }; - if (self.shdr_table_offset == null) { - self.shdr_table_offset = self.findFreeSpace(self.sections.items.len * shsize, shalign); - self.shdr_table_dirty = true; - } - const phsize: u64 = switch (self.ptr_width) { - .p32 => @sizeOf(elf.Elf32_Phdr), - .p64 => @sizeOf(elf.Elf64_Phdr), - }; - const phalign: u16 = switch (self.ptr_width) { - .p32 => @alignOf(elf.Elf32_Phdr), - .p64 => @alignOf(elf.Elf64_Phdr), - }; - if (self.phdr_table_offset == null) { - self.phdr_table_offset = self.findFreeSpace(self.program_headers.items.len * phsize, phalign); - self.phdr_table_dirty = true; - } - { - // Iterate over symbols, populating free_list and last_text_block. - if (self.local_symbols.items.len != 1) { - @panic("TODO implement setting up free_list and last_text_block from existing ELF file"); + pub fn deinit(self: *File.Elf) void { + self.sections.deinit(self.allocator); + self.program_headers.deinit(self.allocator); + self.shstrtab.deinit(self.allocator); + self.local_symbols.deinit(self.allocator); + self.global_symbols.deinit(self.allocator); + self.global_symbol_free_list.deinit(self.allocator); + self.local_symbol_free_list.deinit(self.allocator); + self.offset_table_free_list.deinit(self.allocator); + self.text_block_free_list.deinit(self.allocator); + self.offset_table.deinit(self.allocator); + if (self.owns_file_handle) { + if (self.file) |f| f.close(); } - // We are starting with an empty file. The default values are correct, null and empty list. - } - } - - /// Commit pending changes and write headers. - pub fn flush(self: *ElfFile) !void { - const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); - - // Unfortunately these have to be buffered and done at the end because ELF does not allow - // mixing local and global symbols within a symbol table. - try self.writeAllGlobalSymbols(); - - if (self.phdr_table_dirty) { - const phsize: u64 = switch (self.ptr_width) { - .p32 => @sizeOf(elf.Elf32_Phdr), - .p64 => @sizeOf(elf.Elf64_Phdr), - }; - const phalign: u16 = switch (self.ptr_width) { - .p32 => @alignOf(elf.Elf32_Phdr), - .p64 => @alignOf(elf.Elf64_Phdr), - }; - const allocated_size = self.allocatedSize(self.phdr_table_offset.?); - const needed_size = self.program_headers.items.len * phsize; - - if (needed_size > allocated_size) { - self.phdr_table_offset = null; // free the space - self.phdr_table_offset = self.findFreeSpace(needed_size, phalign); - } - - switch (self.ptr_width) { - .p32 => { - const buf = try self.allocator.alloc(elf.Elf32_Phdr, self.program_headers.items.len); - defer self.allocator.free(buf); - - for (buf) |*phdr, i| { - phdr.* = progHeaderTo32(self.program_headers.items[i]); - if (foreign_endian) { - bswapAllFields(elf.Elf32_Phdr, phdr); - } - } - try self.file.?.pwriteAll(mem.sliceAsBytes(buf), self.phdr_table_offset.?); - }, - .p64 => { - const buf = try self.allocator.alloc(elf.Elf64_Phdr, self.program_headers.items.len); - defer self.allocator.free(buf); - - for (buf) |*phdr, i| { - phdr.* = self.program_headers.items[i]; - if (foreign_endian) { - bswapAllFields(elf.Elf64_Phdr, phdr); - } - } - try self.file.?.pwriteAll(mem.sliceAsBytes(buf), self.phdr_table_offset.?); - }, - } - self.phdr_table_dirty = false; } - { - const shstrtab_sect = &self.sections.items[self.shstrtab_index.?]; - if (self.shstrtab_dirty or self.shstrtab.items.len != shstrtab_sect.sh_size) { - const allocated_size = self.allocatedSize(shstrtab_sect.sh_offset); - const needed_size = self.shstrtab.items.len; + pub fn makeExecutable(self: *File.Elf) !void { + assert(self.owns_file_handle); + if (self.file) |f| { + f.close(); + self.file = null; + } + } - if (needed_size > allocated_size) { - shstrtab_sect.sh_size = 0; // free the space - shstrtab_sect.sh_offset = self.findFreeSpace(needed_size, 1); + pub fn makeWritable(self: *File.Elf, dir: fs.Dir, sub_path: []const u8) !void { + assert(self.owns_file_handle); + if (self.file != null) return; + self.file = try dir.createFile(sub_path, .{ + .truncate = false, + .read = true, + .mode = determineMode(self.options), + }); + } + + /// Returns end pos of collision, if any. + fn detectAllocCollision(self: *File.Elf, start: u64, size: u64) ?u64 { + const small_ptr = self.options.target.cpu.arch.ptrBitWidth() == 32; + const ehdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Ehdr) else @sizeOf(elf.Elf64_Ehdr); + if (start < ehdr_size) + return ehdr_size; + + const end = start + satMul(size, alloc_num) / alloc_den; + + if (self.shdr_table_offset) |off| { + const shdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Shdr) else @sizeOf(elf.Elf64_Shdr); + const tight_size = self.sections.items.len * shdr_size; + const increased_size = satMul(tight_size, alloc_num) / alloc_den; + const test_end = off + increased_size; + if (end > off and start < test_end) { + return test_end; } - shstrtab_sect.sh_size = needed_size; - //std.log.debug(.link, "shstrtab start=0x{x} end=0x{x}\n", .{ shstrtab_sect.sh_offset, shstrtab_sect.sh_offset + needed_size }); - - try self.file.?.pwriteAll(self.shstrtab.items, shstrtab_sect.sh_offset); - if (!self.shdr_table_dirty) { - // Then it won't get written with the others and we need to do it. - try self.writeSectHeader(self.shstrtab_index.?); - } - self.shstrtab_dirty = false; } + + if (self.phdr_table_offset) |off| { + const phdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Phdr) else @sizeOf(elf.Elf64_Phdr); + const tight_size = self.sections.items.len * phdr_size; + const increased_size = satMul(tight_size, alloc_num) / alloc_den; + const test_end = off + increased_size; + if (end > off and start < test_end) { + return test_end; + } + } + + for (self.sections.items) |section| { + const increased_size = satMul(section.sh_size, alloc_num) / alloc_den; + const test_end = section.sh_offset + increased_size; + if (end > section.sh_offset and start < test_end) { + return test_end; + } + } + for (self.program_headers.items) |program_header| { + const increased_size = satMul(program_header.p_filesz, alloc_num) / alloc_den; + const test_end = program_header.p_offset + increased_size; + if (end > program_header.p_offset and start < test_end) { + return test_end; + } + } + return null; } - if (self.shdr_table_dirty) { + + fn allocatedSize(self: *File.Elf, start: u64) u64 { + var min_pos: u64 = std.math.maxInt(u64); + if (self.shdr_table_offset) |off| { + if (off > start and off < min_pos) min_pos = off; + } + if (self.phdr_table_offset) |off| { + if (off > start and off < min_pos) min_pos = off; + } + for (self.sections.items) |section| { + if (section.sh_offset <= start) continue; + if (section.sh_offset < min_pos) min_pos = section.sh_offset; + } + for (self.program_headers.items) |program_header| { + if (program_header.p_offset <= start) continue; + if (program_header.p_offset < min_pos) min_pos = program_header.p_offset; + } + return min_pos - start; + } + + fn findFreeSpace(self: *File.Elf, object_size: u64, min_alignment: u16) u64 { + var start: u64 = 0; + while (self.detectAllocCollision(start, object_size)) |item_end| { + start = mem.alignForwardGeneric(u64, item_end, min_alignment); + } + return start; + } + + fn makeString(self: *File.Elf, bytes: []const u8) !u32 { + try self.shstrtab.ensureCapacity(self.allocator, self.shstrtab.items.len + bytes.len + 1); + const result = self.shstrtab.items.len; + self.shstrtab.appendSliceAssumeCapacity(bytes); + self.shstrtab.appendAssumeCapacity(0); + return @intCast(u32, result); + } + + fn getString(self: *File.Elf, str_off: u32) []const u8 { + assert(str_off < self.shstrtab.items.len); + return mem.spanZ(@ptrCast([*:0]const u8, self.shstrtab.items.ptr + str_off)); + } + + fn updateString(self: *File.Elf, old_str_off: u32, new_name: []const u8) !u32 { + const existing_name = self.getString(old_str_off); + if (mem.eql(u8, existing_name, new_name)) { + return old_str_off; + } + return self.makeString(new_name); + } + + pub fn populateMissingMetadata(self: *File.Elf) !void { + const small_ptr = switch (self.ptr_width) { + .p32 => true, + .p64 => false, + }; + const ptr_size: u8 = switch (self.ptr_width) { + .p32 => 4, + .p64 => 8, + }; + if (self.phdr_load_re_index == null) { + self.phdr_load_re_index = @intCast(u16, self.program_headers.items.len); + const file_size = self.options.program_code_size_hint; + const p_align = 0x1000; + const off = self.findFreeSpace(file_size, p_align); + //std.log.debug(.link, "found PT_LOAD free space 0x{x} to 0x{x}\n", .{ off, off + file_size }); + try self.program_headers.append(self.allocator, .{ + .p_type = elf.PT_LOAD, + .p_offset = off, + .p_filesz = file_size, + .p_vaddr = default_entry_addr, + .p_paddr = default_entry_addr, + .p_memsz = file_size, + .p_align = p_align, + .p_flags = elf.PF_X | elf.PF_R, + }); + self.entry_addr = null; + self.phdr_table_dirty = true; + } + if (self.phdr_got_index == null) { + self.phdr_got_index = @intCast(u16, self.program_headers.items.len); + const file_size = @as(u64, ptr_size) * self.options.symbol_count_hint; + // We really only need ptr alignment but since we are using PROGBITS, linux requires + // page align. + const p_align = 0x1000; + const off = self.findFreeSpace(file_size, p_align); + //std.log.debug(.link, "found PT_LOAD free space 0x{x} to 0x{x}\n", .{ off, off + file_size }); + // TODO instead of hard coding the vaddr, make a function to find a vaddr to put things at. + // we'll need to re-use that function anyway, in case the GOT grows and overlaps something + // else in virtual memory. + const default_got_addr = 0x4000000; + try self.program_headers.append(self.allocator, .{ + .p_type = elf.PT_LOAD, + .p_offset = off, + .p_filesz = file_size, + .p_vaddr = default_got_addr, + .p_paddr = default_got_addr, + .p_memsz = file_size, + .p_align = p_align, + .p_flags = elf.PF_R, + }); + self.phdr_table_dirty = true; + } + if (self.shstrtab_index == null) { + self.shstrtab_index = @intCast(u16, self.sections.items.len); + assert(self.shstrtab.items.len == 0); + try self.shstrtab.append(self.allocator, 0); // need a 0 at position 0 + const off = self.findFreeSpace(self.shstrtab.items.len, 1); + //std.log.debug(.link, "found shstrtab free space 0x{x} to 0x{x}\n", .{ off, off + self.shstrtab.items.len }); + try self.sections.append(self.allocator, .{ + .sh_name = try self.makeString(".shstrtab"), + .sh_type = elf.SHT_STRTAB, + .sh_flags = 0, + .sh_addr = 0, + .sh_offset = off, + .sh_size = self.shstrtab.items.len, + .sh_link = 0, + .sh_info = 0, + .sh_addralign = 1, + .sh_entsize = 0, + }); + self.shstrtab_dirty = true; + self.shdr_table_dirty = true; + } + if (self.text_section_index == null) { + self.text_section_index = @intCast(u16, self.sections.items.len); + const phdr = &self.program_headers.items[self.phdr_load_re_index.?]; + + try self.sections.append(self.allocator, .{ + .sh_name = try self.makeString(".text"), + .sh_type = elf.SHT_PROGBITS, + .sh_flags = elf.SHF_ALLOC | elf.SHF_EXECINSTR, + .sh_addr = phdr.p_vaddr, + .sh_offset = phdr.p_offset, + .sh_size = phdr.p_filesz, + .sh_link = 0, + .sh_info = 0, + .sh_addralign = phdr.p_align, + .sh_entsize = 0, + }); + self.shdr_table_dirty = true; + } + if (self.got_section_index == null) { + self.got_section_index = @intCast(u16, self.sections.items.len); + const phdr = &self.program_headers.items[self.phdr_got_index.?]; + + try self.sections.append(self.allocator, .{ + .sh_name = try self.makeString(".got"), + .sh_type = elf.SHT_PROGBITS, + .sh_flags = elf.SHF_ALLOC, + .sh_addr = phdr.p_vaddr, + .sh_offset = phdr.p_offset, + .sh_size = phdr.p_filesz, + .sh_link = 0, + .sh_info = 0, + .sh_addralign = phdr.p_align, + .sh_entsize = 0, + }); + self.shdr_table_dirty = true; + } + if (self.symtab_section_index == null) { + self.symtab_section_index = @intCast(u16, self.sections.items.len); + const min_align: u16 = if (small_ptr) @alignOf(elf.Elf32_Sym) else @alignOf(elf.Elf64_Sym); + const each_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Sym) else @sizeOf(elf.Elf64_Sym); + const file_size = self.options.symbol_count_hint * each_size; + const off = self.findFreeSpace(file_size, min_align); + //std.log.debug(.link, "found symtab free space 0x{x} to 0x{x}\n", .{ off, off + file_size }); + + try self.sections.append(self.allocator, .{ + .sh_name = try self.makeString(".symtab"), + .sh_type = elf.SHT_SYMTAB, + .sh_flags = 0, + .sh_addr = 0, + .sh_offset = off, + .sh_size = file_size, + // The section header index of the associated string table. + .sh_link = self.shstrtab_index.?, + .sh_info = @intCast(u32, self.local_symbols.items.len), + .sh_addralign = min_align, + .sh_entsize = each_size, + }); + self.shdr_table_dirty = true; + try self.writeSymbol(0); + } const shsize: u64 = switch (self.ptr_width) { .p32 => @sizeOf(elf.Elf32_Shdr), .p64 => @sizeOf(elf.Elf64_Shdr), @@ -605,756 +582,866 @@ pub const ElfFile = struct { .p32 => @alignOf(elf.Elf32_Shdr), .p64 => @alignOf(elf.Elf64_Shdr), }; - const allocated_size = self.allocatedSize(self.shdr_table_offset.?); - const needed_size = self.sections.items.len * shsize; - - if (needed_size > allocated_size) { - self.shdr_table_offset = null; // free the space - self.shdr_table_offset = self.findFreeSpace(needed_size, shalign); + if (self.shdr_table_offset == null) { + self.shdr_table_offset = self.findFreeSpace(self.sections.items.len * shsize, shalign); + self.shdr_table_dirty = true; } + const phsize: u64 = switch (self.ptr_width) { + .p32 => @sizeOf(elf.Elf32_Phdr), + .p64 => @sizeOf(elf.Elf64_Phdr), + }; + const phalign: u16 = switch (self.ptr_width) { + .p32 => @alignOf(elf.Elf32_Phdr), + .p64 => @alignOf(elf.Elf64_Phdr), + }; + if (self.phdr_table_offset == null) { + self.phdr_table_offset = self.findFreeSpace(self.program_headers.items.len * phsize, phalign); + self.phdr_table_dirty = true; + } + { + // Iterate over symbols, populating free_list and last_text_block. + if (self.local_symbols.items.len != 1) { + @panic("TODO implement setting up free_list and last_text_block from existing ELF file"); + } + // We are starting with an empty file. The default values are correct, null and empty list. + } + } + + /// Commit pending changes and write headers. + pub fn flush(self: *File.Elf) !void { + const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); + + // Unfortunately these have to be buffered and done at the end because ELF does not allow + // mixing local and global symbols within a symbol table. + try self.writeAllGlobalSymbols(); + + if (self.phdr_table_dirty) { + const phsize: u64 = switch (self.ptr_width) { + .p32 => @sizeOf(elf.Elf32_Phdr), + .p64 => @sizeOf(elf.Elf64_Phdr), + }; + const phalign: u16 = switch (self.ptr_width) { + .p32 => @alignOf(elf.Elf32_Phdr), + .p64 => @alignOf(elf.Elf64_Phdr), + }; + const allocated_size = self.allocatedSize(self.phdr_table_offset.?); + const needed_size = self.program_headers.items.len * phsize; + + if (needed_size > allocated_size) { + self.phdr_table_offset = null; // free the space + self.phdr_table_offset = self.findFreeSpace(needed_size, phalign); + } + + switch (self.ptr_width) { + .p32 => { + const buf = try self.allocator.alloc(elf.Elf32_Phdr, self.program_headers.items.len); + defer self.allocator.free(buf); + + for (buf) |*phdr, i| { + phdr.* = progHeaderTo32(self.program_headers.items[i]); + if (foreign_endian) { + bswapAllFields(elf.Elf32_Phdr, phdr); + } + } + try self.file.?.pwriteAll(mem.sliceAsBytes(buf), self.phdr_table_offset.?); + }, + .p64 => { + const buf = try self.allocator.alloc(elf.Elf64_Phdr, self.program_headers.items.len); + defer self.allocator.free(buf); + + for (buf) |*phdr, i| { + phdr.* = self.program_headers.items[i]; + if (foreign_endian) { + bswapAllFields(elf.Elf64_Phdr, phdr); + } + } + try self.file.?.pwriteAll(mem.sliceAsBytes(buf), self.phdr_table_offset.?); + }, + } + self.phdr_table_dirty = false; + } + + { + const shstrtab_sect = &self.sections.items[self.shstrtab_index.?]; + if (self.shstrtab_dirty or self.shstrtab.items.len != shstrtab_sect.sh_size) { + const allocated_size = self.allocatedSize(shstrtab_sect.sh_offset); + const needed_size = self.shstrtab.items.len; + + if (needed_size > allocated_size) { + shstrtab_sect.sh_size = 0; // free the space + shstrtab_sect.sh_offset = self.findFreeSpace(needed_size, 1); + } + shstrtab_sect.sh_size = needed_size; + //std.log.debug(.link, "shstrtab start=0x{x} end=0x{x}\n", .{ shstrtab_sect.sh_offset, shstrtab_sect.sh_offset + needed_size }); + + try self.file.?.pwriteAll(self.shstrtab.items, shstrtab_sect.sh_offset); + if (!self.shdr_table_dirty) { + // Then it won't get written with the others and we need to do it. + try self.writeSectHeader(self.shstrtab_index.?); + } + self.shstrtab_dirty = false; + } + } + if (self.shdr_table_dirty) { + const shsize: u64 = switch (self.ptr_width) { + .p32 => @sizeOf(elf.Elf32_Shdr), + .p64 => @sizeOf(elf.Elf64_Shdr), + }; + const shalign: u16 = switch (self.ptr_width) { + .p32 => @alignOf(elf.Elf32_Shdr), + .p64 => @alignOf(elf.Elf64_Shdr), + }; + const allocated_size = self.allocatedSize(self.shdr_table_offset.?); + const needed_size = self.sections.items.len * shsize; + + if (needed_size > allocated_size) { + self.shdr_table_offset = null; // free the space + self.shdr_table_offset = self.findFreeSpace(needed_size, shalign); + } + + switch (self.ptr_width) { + .p32 => { + const buf = try self.allocator.alloc(elf.Elf32_Shdr, self.sections.items.len); + defer self.allocator.free(buf); + + for (buf) |*shdr, i| { + shdr.* = sectHeaderTo32(self.sections.items[i]); + if (foreign_endian) { + bswapAllFields(elf.Elf32_Shdr, shdr); + } + } + try self.file.?.pwriteAll(mem.sliceAsBytes(buf), self.shdr_table_offset.?); + }, + .p64 => { + const buf = try self.allocator.alloc(elf.Elf64_Shdr, self.sections.items.len); + defer self.allocator.free(buf); + + for (buf) |*shdr, i| { + shdr.* = self.sections.items[i]; + //std.log.debug(.link, "writing section {}\n", .{shdr.*}); + if (foreign_endian) { + bswapAllFields(elf.Elf64_Shdr, shdr); + } + } + try self.file.?.pwriteAll(mem.sliceAsBytes(buf), self.shdr_table_offset.?); + }, + } + self.shdr_table_dirty = false; + } + if (self.entry_addr == null and self.options.output_mode == .Exe) { + self.error_flags.no_entry_point_found = true; + } else { + self.error_flags.no_entry_point_found = false; + try self.writeElfHeader(); + } + + // The point of flush() is to commit changes, so nothing should be dirty after this. + assert(!self.phdr_table_dirty); + assert(!self.shdr_table_dirty); + assert(!self.shstrtab_dirty); + assert(!self.offset_table_count_dirty); + const syms_sect = &self.sections.items[self.symtab_section_index.?]; + assert(syms_sect.sh_info == self.local_symbols.items.len); + } + + fn writeElfHeader(self: *File.Elf) !void { + var hdr_buf: [@sizeOf(elf.Elf64_Ehdr)]u8 = undefined; + + var index: usize = 0; + hdr_buf[0..4].* = "\x7fELF".*; + index += 4; + + hdr_buf[index] = switch (self.ptr_width) { + .p32 => elf.ELFCLASS32, + .p64 => elf.ELFCLASS64, + }; + index += 1; + + const endian = self.options.target.cpu.arch.endian(); + hdr_buf[index] = switch (endian) { + .Little => elf.ELFDATA2LSB, + .Big => elf.ELFDATA2MSB, + }; + index += 1; + + hdr_buf[index] = 1; // ELF version + index += 1; + + // OS ABI, often set to 0 regardless of target platform + // ABI Version, possibly used by glibc but not by static executables + // padding + mem.set(u8, hdr_buf[index..][0..9], 0); + index += 9; + + assert(index == 16); + + const elf_type = switch (self.options.output_mode) { + .Exe => elf.ET.EXEC, + .Obj => elf.ET.REL, + .Lib => switch (self.options.link_mode) { + .Static => elf.ET.REL, + .Dynamic => elf.ET.DYN, + }, + }; + mem.writeInt(u16, hdr_buf[index..][0..2], @enumToInt(elf_type), endian); + index += 2; + + const machine = self.options.target.cpu.arch.toElfMachine(); + mem.writeInt(u16, hdr_buf[index..][0..2], @enumToInt(machine), endian); + index += 2; + + // ELF Version, again + mem.writeInt(u32, hdr_buf[index..][0..4], 1, endian); + index += 4; + + const e_entry = if (elf_type == .REL) 0 else self.entry_addr.?; switch (self.ptr_width) { .p32 => { - const buf = try self.allocator.alloc(elf.Elf32_Shdr, self.sections.items.len); - defer self.allocator.free(buf); + mem.writeInt(u32, hdr_buf[index..][0..4], @intCast(u32, e_entry), endian); + index += 4; - for (buf) |*shdr, i| { - shdr.* = sectHeaderTo32(self.sections.items[i]); - if (foreign_endian) { - bswapAllFields(elf.Elf32_Shdr, shdr); - } - } - try self.file.?.pwriteAll(mem.sliceAsBytes(buf), self.shdr_table_offset.?); + // e_phoff + mem.writeInt(u32, hdr_buf[index..][0..4], @intCast(u32, self.phdr_table_offset.?), endian); + index += 4; + + // e_shoff + mem.writeInt(u32, hdr_buf[index..][0..4], @intCast(u32, self.shdr_table_offset.?), endian); + index += 4; }, .p64 => { - const buf = try self.allocator.alloc(elf.Elf64_Shdr, self.sections.items.len); - defer self.allocator.free(buf); + // e_entry + mem.writeInt(u64, hdr_buf[index..][0..8], e_entry, endian); + index += 8; - for (buf) |*shdr, i| { - shdr.* = self.sections.items[i]; - //std.log.debug(.link, "writing section {}\n", .{shdr.*}); - if (foreign_endian) { - bswapAllFields(elf.Elf64_Shdr, shdr); - } - } - try self.file.?.pwriteAll(mem.sliceAsBytes(buf), self.shdr_table_offset.?); + // e_phoff + mem.writeInt(u64, hdr_buf[index..][0..8], self.phdr_table_offset.?, endian); + index += 8; + + // e_shoff + mem.writeInt(u64, hdr_buf[index..][0..8], self.shdr_table_offset.?, endian); + index += 8; }, } - self.shdr_table_dirty = false; - } - if (self.entry_addr == null and self.options.output_mode == .Exe) { - self.error_flags.no_entry_point_found = true; - } else { - self.error_flags.no_entry_point_found = false; - try self.writeElfHeader(); + + const e_flags = 0; + mem.writeInt(u32, hdr_buf[index..][0..4], e_flags, endian); + index += 4; + + const e_ehsize: u16 = switch (self.ptr_width) { + .p32 => @sizeOf(elf.Elf32_Ehdr), + .p64 => @sizeOf(elf.Elf64_Ehdr), + }; + mem.writeInt(u16, hdr_buf[index..][0..2], e_ehsize, endian); + index += 2; + + const e_phentsize: u16 = switch (self.ptr_width) { + .p32 => @sizeOf(elf.Elf32_Phdr), + .p64 => @sizeOf(elf.Elf64_Phdr), + }; + mem.writeInt(u16, hdr_buf[index..][0..2], e_phentsize, endian); + index += 2; + + const e_phnum = @intCast(u16, self.program_headers.items.len); + mem.writeInt(u16, hdr_buf[index..][0..2], e_phnum, endian); + index += 2; + + const e_shentsize: u16 = switch (self.ptr_width) { + .p32 => @sizeOf(elf.Elf32_Shdr), + .p64 => @sizeOf(elf.Elf64_Shdr), + }; + mem.writeInt(u16, hdr_buf[index..][0..2], e_shentsize, endian); + index += 2; + + const e_shnum = @intCast(u16, self.sections.items.len); + mem.writeInt(u16, hdr_buf[index..][0..2], e_shnum, endian); + index += 2; + + mem.writeInt(u16, hdr_buf[index..][0..2], self.shstrtab_index.?, endian); + index += 2; + + assert(index == e_ehsize); + + try self.file.?.pwriteAll(hdr_buf[0..index], 0); } - // The point of flush() is to commit changes, so nothing should be dirty after this. - assert(!self.phdr_table_dirty); - assert(!self.shdr_table_dirty); - assert(!self.shstrtab_dirty); - assert(!self.offset_table_count_dirty); - const syms_sect = &self.sections.items[self.symtab_section_index.?]; - assert(syms_sect.sh_info == self.local_symbols.items.len); - } - - fn writeElfHeader(self: *ElfFile) !void { - var hdr_buf: [@sizeOf(elf.Elf64_Ehdr)]u8 = undefined; - - var index: usize = 0; - hdr_buf[0..4].* = "\x7fELF".*; - index += 4; - - hdr_buf[index] = switch (self.ptr_width) { - .p32 => elf.ELFCLASS32, - .p64 => elf.ELFCLASS64, - }; - index += 1; - - const endian = self.options.target.cpu.arch.endian(); - hdr_buf[index] = switch (endian) { - .Little => elf.ELFDATA2LSB, - .Big => elf.ELFDATA2MSB, - }; - index += 1; - - hdr_buf[index] = 1; // ELF version - index += 1; - - // OS ABI, often set to 0 regardless of target platform - // ABI Version, possibly used by glibc but not by static executables - // padding - mem.set(u8, hdr_buf[index..][0..9], 0); - index += 9; - - assert(index == 16); - - const elf_type = switch (self.options.output_mode) { - .Exe => elf.ET.EXEC, - .Obj => elf.ET.REL, - .Lib => switch (self.options.link_mode) { - .Static => elf.ET.REL, - .Dynamic => elf.ET.DYN, - }, - }; - mem.writeInt(u16, hdr_buf[index..][0..2], @enumToInt(elf_type), endian); - index += 2; - - const machine = self.options.target.cpu.arch.toElfMachine(); - mem.writeInt(u16, hdr_buf[index..][0..2], @enumToInt(machine), endian); - index += 2; - - // ELF Version, again - mem.writeInt(u32, hdr_buf[index..][0..4], 1, endian); - index += 4; - - const e_entry = if (elf_type == .REL) 0 else self.entry_addr.?; - - switch (self.ptr_width) { - .p32 => { - mem.writeInt(u32, hdr_buf[index..][0..4], @intCast(u32, e_entry), endian); - index += 4; - - // e_phoff - mem.writeInt(u32, hdr_buf[index..][0..4], @intCast(u32, self.phdr_table_offset.?), endian); - index += 4; - - // e_shoff - mem.writeInt(u32, hdr_buf[index..][0..4], @intCast(u32, self.shdr_table_offset.?), endian); - index += 4; - }, - .p64 => { - // e_entry - mem.writeInt(u64, hdr_buf[index..][0..8], e_entry, endian); - index += 8; - - // e_phoff - mem.writeInt(u64, hdr_buf[index..][0..8], self.phdr_table_offset.?, endian); - index += 8; - - // e_shoff - mem.writeInt(u64, hdr_buf[index..][0..8], self.shdr_table_offset.?, endian); - index += 8; - }, - } - - const e_flags = 0; - mem.writeInt(u32, hdr_buf[index..][0..4], e_flags, endian); - index += 4; - - const e_ehsize: u16 = switch (self.ptr_width) { - .p32 => @sizeOf(elf.Elf32_Ehdr), - .p64 => @sizeOf(elf.Elf64_Ehdr), - }; - mem.writeInt(u16, hdr_buf[index..][0..2], e_ehsize, endian); - index += 2; - - const e_phentsize: u16 = switch (self.ptr_width) { - .p32 => @sizeOf(elf.Elf32_Phdr), - .p64 => @sizeOf(elf.Elf64_Phdr), - }; - mem.writeInt(u16, hdr_buf[index..][0..2], e_phentsize, endian); - index += 2; - - const e_phnum = @intCast(u16, self.program_headers.items.len); - mem.writeInt(u16, hdr_buf[index..][0..2], e_phnum, endian); - index += 2; - - const e_shentsize: u16 = switch (self.ptr_width) { - .p32 => @sizeOf(elf.Elf32_Shdr), - .p64 => @sizeOf(elf.Elf64_Shdr), - }; - mem.writeInt(u16, hdr_buf[index..][0..2], e_shentsize, endian); - index += 2; - - const e_shnum = @intCast(u16, self.sections.items.len); - mem.writeInt(u16, hdr_buf[index..][0..2], e_shnum, endian); - index += 2; - - mem.writeInt(u16, hdr_buf[index..][0..2], self.shstrtab_index.?, endian); - index += 2; - - assert(index == e_ehsize); - - try self.file.?.pwriteAll(hdr_buf[0..index], 0); - } - - fn freeTextBlock(self: *ElfFile, text_block: *TextBlock) void { - var already_have_free_list_node = false; - { - var i: usize = 0; - while (i < self.text_block_free_list.items.len) { - if (self.text_block_free_list.items[i] == text_block) { - _ = self.text_block_free_list.swapRemove(i); - continue; - } - if (self.text_block_free_list.items[i] == text_block.prev) { - already_have_free_list_node = true; - } - i += 1; - } - } - - if (self.last_text_block == text_block) { - // TODO shrink the .text section size here - self.last_text_block = text_block.prev; - } - - if (text_block.prev) |prev| { - prev.next = text_block.next; - - if (!already_have_free_list_node and prev.freeListEligible(self.*)) { - // The free list is heuristics, it doesn't have to be perfect, so we can - // ignore the OOM here. - self.text_block_free_list.append(self.allocator, prev) catch {}; - } - } else { - text_block.prev = null; - } - - if (text_block.next) |next| { - next.prev = text_block.prev; - } else { - text_block.next = null; - } - } - - fn shrinkTextBlock(self: *ElfFile, text_block: *TextBlock, new_block_size: u64) void { - // TODO check the new capacity, and if it crosses the size threshold into a big enough - // capacity, insert a free list node for it. - } - - fn growTextBlock(self: *ElfFile, text_block: *TextBlock, new_block_size: u64, alignment: u64) !u64 { - const sym = self.local_symbols.items[text_block.local_sym_index]; - const align_ok = mem.alignBackwardGeneric(u64, sym.st_value, alignment) == sym.st_value; - const need_realloc = !align_ok or new_block_size > text_block.capacity(self.*); - if (!need_realloc) return sym.st_value; - return self.allocateTextBlock(text_block, new_block_size, alignment); - } - - fn allocateTextBlock(self: *ElfFile, text_block: *TextBlock, new_block_size: u64, alignment: u64) !u64 { - const phdr = &self.program_headers.items[self.phdr_load_re_index.?]; - const shdr = &self.sections.items[self.text_section_index.?]; - const new_block_ideal_capacity = new_block_size * alloc_num / alloc_den; - - // We use these to indicate our intention to update metadata, placing the new block, - // and possibly removing a free list node. - // It would be simpler to do it inside the for loop below, but that would cause a - // problem if an error was returned later in the function. So this action - // is actually carried out at the end of the function, when errors are no longer possible. - var block_placement: ?*TextBlock = null; - var free_list_removal: ?usize = null; - - // First we look for an appropriately sized free list node. - // The list is unordered. We'll just take the first thing that works. - const vaddr = blk: { - var i: usize = 0; - while (i < self.text_block_free_list.items.len) { - const big_block = self.text_block_free_list.items[i]; - // We now have a pointer to a live text block that has too much capacity. - // Is it enough that we could fit this new text block? - const sym = self.local_symbols.items[big_block.local_sym_index]; - const capacity = big_block.capacity(self.*); - const ideal_capacity = capacity * alloc_num / alloc_den; - const ideal_capacity_end_vaddr = sym.st_value + ideal_capacity; - const capacity_end_vaddr = sym.st_value + capacity; - const new_start_vaddr_unaligned = capacity_end_vaddr - new_block_ideal_capacity; - const new_start_vaddr = mem.alignBackwardGeneric(u64, new_start_vaddr_unaligned, alignment); - if (new_start_vaddr < ideal_capacity_end_vaddr) { - // Additional bookkeeping here to notice if this free list node - // should be deleted because the block that it points to has grown to take up - // more of the extra capacity. - if (!big_block.freeListEligible(self.*)) { + fn freeTextBlock(self: *File.Elf, text_block: *TextBlock) void { + var already_have_free_list_node = false; + { + var i: usize = 0; + while (i < self.text_block_free_list.items.len) { + if (self.text_block_free_list.items[i] == text_block) { _ = self.text_block_free_list.swapRemove(i); - } else { - i += 1; + continue; } - continue; + if (self.text_block_free_list.items[i] == text_block.prev) { + already_have_free_list_node = true; + } + i += 1; } - // At this point we know that we will place the new block here. But the - // remaining question is whether there is still yet enough capacity left - // over for there to still be a free list node. - const remaining_capacity = new_start_vaddr - ideal_capacity_end_vaddr; - const keep_free_list_node = remaining_capacity >= min_text_capacity; - - // Set up the metadata to be updated, after errors are no longer possible. - block_placement = big_block; - if (!keep_free_list_node) { - free_list_removal = i; - } - break :blk new_start_vaddr; - } else if (self.last_text_block) |last| { - const sym = self.local_symbols.items[last.local_sym_index]; - const ideal_capacity = sym.st_size * alloc_num / alloc_den; - const ideal_capacity_end_vaddr = sym.st_value + ideal_capacity; - const new_start_vaddr = mem.alignForwardGeneric(u64, ideal_capacity_end_vaddr, alignment); - // Set up the metadata to be updated, after errors are no longer possible. - block_placement = last; - break :blk new_start_vaddr; - } else { - break :blk phdr.p_vaddr; } - }; - const expand_text_section = block_placement == null or block_placement.?.next == null; - if (expand_text_section) { - const text_capacity = self.allocatedSize(shdr.sh_offset); - const needed_size = (vaddr + new_block_size) - phdr.p_vaddr; - if (needed_size > text_capacity) { - // Must move the entire text section. - const new_offset = self.findFreeSpace(needed_size, 0x1000); - const text_size = if (self.last_text_block) |last| blk: { + if (self.last_text_block == text_block) { + // TODO shrink the .text section size here + self.last_text_block = text_block.prev; + } + + if (text_block.prev) |prev| { + prev.next = text_block.next; + + if (!already_have_free_list_node and prev.freeListEligible(self.*)) { + // The free list is heuristics, it doesn't have to be perfect, so we can + // ignore the OOM here. + self.text_block_free_list.append(self.allocator, prev) catch {}; + } + } else { + text_block.prev = null; + } + + if (text_block.next) |next| { + next.prev = text_block.prev; + } else { + text_block.next = null; + } + } + + fn shrinkTextBlock(self: *File.Elf, text_block: *TextBlock, new_block_size: u64) void { + // TODO check the new capacity, and if it crosses the size threshold into a big enough + // capacity, insert a free list node for it. + } + + fn growTextBlock(self: *File.Elf, text_block: *TextBlock, new_block_size: u64, alignment: u64) !u64 { + const sym = self.local_symbols.items[text_block.local_sym_index]; + const align_ok = mem.alignBackwardGeneric(u64, sym.st_value, alignment) == sym.st_value; + const need_realloc = !align_ok or new_block_size > text_block.capacity(self.*); + if (!need_realloc) return sym.st_value; + return self.allocateTextBlock(text_block, new_block_size, alignment); + } + + fn allocateTextBlock(self: *File.Elf, text_block: *TextBlock, new_block_size: u64, alignment: u64) !u64 { + const phdr = &self.program_headers.items[self.phdr_load_re_index.?]; + const shdr = &self.sections.items[self.text_section_index.?]; + const new_block_ideal_capacity = new_block_size * alloc_num / alloc_den; + + // We use these to indicate our intention to update metadata, placing the new block, + // and possibly removing a free list node. + // It would be simpler to do it inside the for loop below, but that would cause a + // problem if an error was returned later in the function. So this action + // is actually carried out at the end of the function, when errors are no longer possible. + var block_placement: ?*TextBlock = null; + var free_list_removal: ?usize = null; + + // First we look for an appropriately sized free list node. + // The list is unordered. We'll just take the first thing that works. + const vaddr = blk: { + var i: usize = 0; + while (i < self.text_block_free_list.items.len) { + const big_block = self.text_block_free_list.items[i]; + // We now have a pointer to a live text block that has too much capacity. + // Is it enough that we could fit this new text block? + const sym = self.local_symbols.items[big_block.local_sym_index]; + const capacity = big_block.capacity(self.*); + const ideal_capacity = capacity * alloc_num / alloc_den; + const ideal_capacity_end_vaddr = sym.st_value + ideal_capacity; + const capacity_end_vaddr = sym.st_value + capacity; + const new_start_vaddr_unaligned = capacity_end_vaddr - new_block_ideal_capacity; + const new_start_vaddr = mem.alignBackwardGeneric(u64, new_start_vaddr_unaligned, alignment); + if (new_start_vaddr < ideal_capacity_end_vaddr) { + // Additional bookkeeping here to notice if this free list node + // should be deleted because the block that it points to has grown to take up + // more of the extra capacity. + if (!big_block.freeListEligible(self.*)) { + _ = self.text_block_free_list.swapRemove(i); + } else { + i += 1; + } + continue; + } + // At this point we know that we will place the new block here. But the + // remaining question is whether there is still yet enough capacity left + // over for there to still be a free list node. + const remaining_capacity = new_start_vaddr - ideal_capacity_end_vaddr; + const keep_free_list_node = remaining_capacity >= min_text_capacity; + + // Set up the metadata to be updated, after errors are no longer possible. + block_placement = big_block; + if (!keep_free_list_node) { + free_list_removal = i; + } + break :blk new_start_vaddr; + } else if (self.last_text_block) |last| { const sym = self.local_symbols.items[last.local_sym_index]; - break :blk (sym.st_value + sym.st_size) - phdr.p_vaddr; - } else 0; - const amt = try self.file.?.copyRangeAll(shdr.sh_offset, self.file.?, new_offset, text_size); - if (amt != text_size) return error.InputOutput; - shdr.sh_offset = new_offset; - phdr.p_offset = new_offset; - } - self.last_text_block = text_block; - - shdr.sh_size = needed_size; - phdr.p_memsz = needed_size; - phdr.p_filesz = needed_size; - - self.phdr_table_dirty = true; // TODO look into making only the one program header dirty - self.shdr_table_dirty = true; // TODO look into making only the one section dirty - } - - // This function can also reallocate a text block. - // In this case we need to "unplug" it from its previous location before - // plugging it in to its new location. - if (text_block.prev) |prev| { - prev.next = text_block.next; - } - if (text_block.next) |next| { - next.prev = text_block.prev; - } - - if (block_placement) |big_block| { - text_block.prev = big_block; - text_block.next = big_block.next; - big_block.next = text_block; - } else { - text_block.prev = null; - text_block.next = null; - } - if (free_list_removal) |i| { - _ = self.text_block_free_list.swapRemove(i); - } - return vaddr; - } - - pub fn allocateDeclIndexes(self: *ElfFile, decl: *Module.Decl) !void { - if (decl.link.local_sym_index != 0) return; - - // Here we also ensure capacity for the free lists so that they can be appended to without fail. - try self.local_symbols.ensureCapacity(self.allocator, self.local_symbols.items.len + 1); - try self.local_symbol_free_list.ensureCapacity(self.allocator, self.local_symbols.items.len); - try self.offset_table.ensureCapacity(self.allocator, self.offset_table.items.len + 1); - try self.offset_table_free_list.ensureCapacity(self.allocator, self.local_symbols.items.len); - - if (self.local_symbol_free_list.popOrNull()) |i| { - //std.log.debug(.link, "reusing symbol index {} for {}\n", .{i, decl.name}); - decl.link.local_sym_index = i; - } else { - //std.log.debug(.link, "allocating symbol index {} for {}\n", .{self.local_symbols.items.len, decl.name}); - decl.link.local_sym_index = @intCast(u32, self.local_symbols.items.len); - _ = self.local_symbols.addOneAssumeCapacity(); - } - - if (self.offset_table_free_list.popOrNull()) |i| { - decl.link.offset_table_index = i; - } else { - decl.link.offset_table_index = @intCast(u32, self.offset_table.items.len); - _ = self.offset_table.addOneAssumeCapacity(); - self.offset_table_count_dirty = true; - } - - const phdr = &self.program_headers.items[self.phdr_load_re_index.?]; - - self.local_symbols.items[decl.link.local_sym_index] = .{ - .st_name = 0, - .st_info = 0, - .st_other = 0, - .st_shndx = 0, - .st_value = phdr.p_vaddr, - .st_size = 0, - }; - self.offset_table.items[decl.link.offset_table_index] = 0; - } - - pub fn freeDecl(self: *ElfFile, decl: *Module.Decl) void { - self.freeTextBlock(&decl.link); - if (decl.link.local_sym_index != 0) { - self.local_symbol_free_list.appendAssumeCapacity(decl.link.local_sym_index); - self.offset_table_free_list.appendAssumeCapacity(decl.link.offset_table_index); - - self.local_symbols.items[decl.link.local_sym_index].st_info = 0; - - decl.link.local_sym_index = 0; - } - } - - pub fn updateDecl(self: *ElfFile, module: *Module, decl: *Module.Decl) !void { - var code_buffer = std.ArrayList(u8).init(self.allocator); - defer code_buffer.deinit(); - - const typed_value = decl.typed_value.most_recent.typed_value; - const code = switch (try codegen.generateSymbol(self, decl.src(), typed_value, &code_buffer)) { - .externally_managed => |x| x, - .appended => code_buffer.items, - .fail => |em| { - decl.analysis = .codegen_failure; - _ = try module.failed_decls.put(decl, em); - return; - }, - }; - - const required_alignment = typed_value.ty.abiAlignment(self.options.target); - - const stt_bits: u8 = switch (typed_value.ty.zigTypeTag()) { - .Fn => elf.STT_FUNC, - else => elf.STT_OBJECT, - }; - - assert(decl.link.local_sym_index != 0); // Caller forgot to allocateDeclIndexes() - const local_sym = &self.local_symbols.items[decl.link.local_sym_index]; - if (local_sym.st_size != 0) { - const capacity = decl.link.capacity(self.*); - const need_realloc = code.len > capacity or - !mem.isAlignedGeneric(u64, local_sym.st_value, required_alignment); - if (need_realloc) { - const vaddr = try self.growTextBlock(&decl.link, code.len, required_alignment); - //std.log.debug(.link, "growing {} from 0x{x} to 0x{x}\n", .{ decl.name, local_sym.st_value, vaddr }); - if (vaddr != local_sym.st_value) { - local_sym.st_value = vaddr; - - //std.log.debug(.link, " (writing new offset table entry)\n", .{}); - self.offset_table.items[decl.link.offset_table_index] = vaddr; - try self.writeOffsetTableEntry(decl.link.offset_table_index); + const ideal_capacity = sym.st_size * alloc_num / alloc_den; + const ideal_capacity_end_vaddr = sym.st_value + ideal_capacity; + const new_start_vaddr = mem.alignForwardGeneric(u64, ideal_capacity_end_vaddr, alignment); + // Set up the metadata to be updated, after errors are no longer possible. + block_placement = last; + break :blk new_start_vaddr; + } else { + break :blk phdr.p_vaddr; } - } else if (code.len < local_sym.st_size) { - self.shrinkTextBlock(&decl.link, code.len); - } - local_sym.st_size = code.len; - local_sym.st_name = try self.updateString(local_sym.st_name, mem.spanZ(decl.name)); - local_sym.st_info = (elf.STB_LOCAL << 4) | stt_bits; - local_sym.st_other = 0; - local_sym.st_shndx = self.text_section_index.?; - // TODO this write could be avoided if no fields of the symbol were changed. - try self.writeSymbol(decl.link.local_sym_index); - } else { - const decl_name = mem.spanZ(decl.name); - const name_str_index = try self.makeString(decl_name); - const vaddr = try self.allocateTextBlock(&decl.link, code.len, required_alignment); - //std.log.debug(.link, "allocated text block for {} at 0x{x}\n", .{ decl_name, vaddr }); - errdefer self.freeTextBlock(&decl.link); - - local_sym.* = .{ - .st_name = name_str_index, - .st_info = (elf.STB_LOCAL << 4) | stt_bits, - .st_other = 0, - .st_shndx = self.text_section_index.?, - .st_value = vaddr, - .st_size = code.len, }; - self.offset_table.items[decl.link.offset_table_index] = vaddr; - try self.writeSymbol(decl.link.local_sym_index); - try self.writeOffsetTableEntry(decl.link.offset_table_index); - } - - const section_offset = local_sym.st_value - self.program_headers.items[self.phdr_load_re_index.?].p_vaddr; - const file_offset = self.sections.items[self.text_section_index.?].sh_offset + section_offset; - try self.file.?.pwriteAll(code, file_offset); - - // Since we updated the vaddr and the size, each corresponding export symbol also needs to be updated. - const decl_exports = module.decl_exports.get(decl) orelse &[0]*Module.Export{}; - return self.updateDeclExports(module, decl, decl_exports); - } - - /// Must be called only after a successful call to `updateDecl`. - pub fn updateDeclExports( - self: *ElfFile, - module: *Module, - decl: *const Module.Decl, - exports: []const *Module.Export, - ) !void { - // In addition to ensuring capacity for global_symbols, we also ensure capacity for freeing all of - // them, so that deleting exports is guaranteed to succeed. - try self.global_symbols.ensureCapacity(self.allocator, self.global_symbols.items.len + exports.len); - try self.global_symbol_free_list.ensureCapacity(self.allocator, self.global_symbols.items.len); - const typed_value = decl.typed_value.most_recent.typed_value; - if (decl.link.local_sym_index == 0) return; - const decl_sym = self.local_symbols.items[decl.link.local_sym_index]; - - for (exports) |exp| { - if (exp.options.section) |section_name| { - if (!mem.eql(u8, section_name, ".text")) { - try module.failed_exports.ensureCapacity(module.failed_exports.items().len + 1); - module.failed_exports.putAssumeCapacityNoClobber( - exp, - try Module.ErrorMsg.create(self.allocator, 0, "Unimplemented: ExportOptions.section", .{}), - ); - continue; + const expand_text_section = block_placement == null or block_placement.?.next == null; + if (expand_text_section) { + const text_capacity = self.allocatedSize(shdr.sh_offset); + const needed_size = (vaddr + new_block_size) - phdr.p_vaddr; + if (needed_size > text_capacity) { + // Must move the entire text section. + const new_offset = self.findFreeSpace(needed_size, 0x1000); + const text_size = if (self.last_text_block) |last| blk: { + const sym = self.local_symbols.items[last.local_sym_index]; + break :blk (sym.st_value + sym.st_size) - phdr.p_vaddr; + } else 0; + const amt = try self.file.?.copyRangeAll(shdr.sh_offset, self.file.?, new_offset, text_size); + if (amt != text_size) return error.InputOutput; + shdr.sh_offset = new_offset; + phdr.p_offset = new_offset; } + self.last_text_block = text_block; + + shdr.sh_size = needed_size; + phdr.p_memsz = needed_size; + phdr.p_filesz = needed_size; + + self.phdr_table_dirty = true; // TODO look into making only the one program header dirty + self.shdr_table_dirty = true; // TODO look into making only the one section dirty } - const stb_bits: u8 = switch (exp.options.linkage) { - .Internal => elf.STB_LOCAL, - .Strong => blk: { - if (mem.eql(u8, exp.options.name, "_start")) { - self.entry_addr = decl_sym.st_value; - } - break :blk elf.STB_GLOBAL; - }, - .Weak => elf.STB_WEAK, - .LinkOnce => { - try module.failed_exports.ensureCapacity(module.failed_exports.items().len + 1); - module.failed_exports.putAssumeCapacityNoClobber( - exp, - try Module.ErrorMsg.create(self.allocator, 0, "Unimplemented: GlobalLinkage.LinkOnce", .{}), - ); - continue; - }, - }; - const stt_bits: u8 = @truncate(u4, decl_sym.st_info); - if (exp.link.sym_index) |i| { - const sym = &self.global_symbols.items[i]; - sym.* = .{ - .st_name = try self.updateString(sym.st_name, exp.options.name), - .st_info = (stb_bits << 4) | stt_bits, - .st_other = 0, - .st_shndx = self.text_section_index.?, - .st_value = decl_sym.st_value, - .st_size = decl_sym.st_size, - }; + + // This function can also reallocate a text block. + // In this case we need to "unplug" it from its previous location before + // plugging it in to its new location. + if (text_block.prev) |prev| { + prev.next = text_block.next; + } + if (text_block.next) |next| { + next.prev = text_block.prev; + } + + if (block_placement) |big_block| { + text_block.prev = big_block; + text_block.next = big_block.next; + big_block.next = text_block; } else { - const name = try self.makeString(exp.options.name); - const i = if (self.global_symbol_free_list.popOrNull()) |i| i else blk: { - _ = self.global_symbols.addOneAssumeCapacity(); - break :blk self.global_symbols.items.len - 1; - }; - self.global_symbols.items[i] = .{ - .st_name = name, - .st_info = (stb_bits << 4) | stt_bits, + text_block.prev = null; + text_block.next = null; + } + if (free_list_removal) |i| { + _ = self.text_block_free_list.swapRemove(i); + } + return vaddr; + } + + pub fn allocateDeclIndexes(self: *File.Elf, decl: *Module.Decl) !void { + if (decl.link.local_sym_index != 0) return; + + // Here we also ensure capacity for the free lists so that they can be appended to without fail. + try self.local_symbols.ensureCapacity(self.allocator, self.local_symbols.items.len + 1); + try self.local_symbol_free_list.ensureCapacity(self.allocator, self.local_symbols.items.len); + try self.offset_table.ensureCapacity(self.allocator, self.offset_table.items.len + 1); + try self.offset_table_free_list.ensureCapacity(self.allocator, self.local_symbols.items.len); + + if (self.local_symbol_free_list.popOrNull()) |i| { + //std.log.debug(.link, "reusing symbol index {} for {}\n", .{i, decl.name}); + decl.link.local_sym_index = i; + } else { + //std.log.debug(.link, "allocating symbol index {} for {}\n", .{self.local_symbols.items.len, decl.name}); + decl.link.local_sym_index = @intCast(u32, self.local_symbols.items.len); + _ = self.local_symbols.addOneAssumeCapacity(); + } + + if (self.offset_table_free_list.popOrNull()) |i| { + decl.link.offset_table_index = i; + } else { + decl.link.offset_table_index = @intCast(u32, self.offset_table.items.len); + _ = self.offset_table.addOneAssumeCapacity(); + self.offset_table_count_dirty = true; + } + + const phdr = &self.program_headers.items[self.phdr_load_re_index.?]; + + self.local_symbols.items[decl.link.local_sym_index] = .{ + .st_name = 0, + .st_info = 0, + .st_other = 0, + .st_shndx = 0, + .st_value = phdr.p_vaddr, + .st_size = 0, + }; + self.offset_table.items[decl.link.offset_table_index] = 0; + } + + pub fn freeDecl(self: *File.Elf, decl: *Module.Decl) void { + self.freeTextBlock(&decl.link); + if (decl.link.local_sym_index != 0) { + self.local_symbol_free_list.appendAssumeCapacity(decl.link.local_sym_index); + self.offset_table_free_list.appendAssumeCapacity(decl.link.offset_table_index); + + self.local_symbols.items[decl.link.local_sym_index].st_info = 0; + + decl.link.local_sym_index = 0; + } + } + + pub fn updateDecl(self: *File.Elf, module: *Module, decl: *Module.Decl) !void { + var code_buffer = std.ArrayList(u8).init(self.allocator); + defer code_buffer.deinit(); + + const typed_value = decl.typed_value.most_recent.typed_value; + const code = switch (try codegen.generateSymbol(self, decl.src(), typed_value, &code_buffer)) { + .externally_managed => |x| x, + .appended => code_buffer.items, + .fail => |em| { + decl.analysis = .codegen_failure; + _ = try module.failed_decls.put(decl, em); + return; + }, + }; + + const required_alignment = typed_value.ty.abiAlignment(self.options.target); + + const stt_bits: u8 = switch (typed_value.ty.zigTypeTag()) { + .Fn => elf.STT_FUNC, + else => elf.STT_OBJECT, + }; + + assert(decl.link.local_sym_index != 0); // Caller forgot to allocateDeclIndexes() + const local_sym = &self.local_symbols.items[decl.link.local_sym_index]; + if (local_sym.st_size != 0) { + const capacity = decl.link.capacity(self.*); + const need_realloc = code.len > capacity or + !mem.isAlignedGeneric(u64, local_sym.st_value, required_alignment); + if (need_realloc) { + const vaddr = try self.growTextBlock(&decl.link, code.len, required_alignment); + //std.log.debug(.link, "growing {} from 0x{x} to 0x{x}\n", .{ decl.name, local_sym.st_value, vaddr }); + if (vaddr != local_sym.st_value) { + local_sym.st_value = vaddr; + + //std.log.debug(.link, " (writing new offset table entry)\n", .{}); + self.offset_table.items[decl.link.offset_table_index] = vaddr; + try self.writeOffsetTableEntry(decl.link.offset_table_index); + } + } else if (code.len < local_sym.st_size) { + self.shrinkTextBlock(&decl.link, code.len); + } + local_sym.st_size = code.len; + local_sym.st_name = try self.updateString(local_sym.st_name, mem.spanZ(decl.name)); + local_sym.st_info = (elf.STB_LOCAL << 4) | stt_bits; + local_sym.st_other = 0; + local_sym.st_shndx = self.text_section_index.?; + // TODO this write could be avoided if no fields of the symbol were changed. + try self.writeSymbol(decl.link.local_sym_index); + } else { + const decl_name = mem.spanZ(decl.name); + const name_str_index = try self.makeString(decl_name); + const vaddr = try self.allocateTextBlock(&decl.link, code.len, required_alignment); + //std.log.debug(.link, "allocated text block for {} at 0x{x}\n", .{ decl_name, vaddr }); + errdefer self.freeTextBlock(&decl.link); + + local_sym.* = .{ + .st_name = name_str_index, + .st_info = (elf.STB_LOCAL << 4) | stt_bits, .st_other = 0, .st_shndx = self.text_section_index.?, - .st_value = decl_sym.st_value, - .st_size = decl_sym.st_size, + .st_value = vaddr, + .st_size = code.len, }; + self.offset_table.items[decl.link.offset_table_index] = vaddr; - exp.link.sym_index = @intCast(u32, i); + try self.writeSymbol(decl.link.local_sym_index); + try self.writeOffsetTableEntry(decl.link.offset_table_index); + } + + const section_offset = local_sym.st_value - self.program_headers.items[self.phdr_load_re_index.?].p_vaddr; + const file_offset = self.sections.items[self.text_section_index.?].sh_offset + section_offset; + try self.file.?.pwriteAll(code, file_offset); + + // Since we updated the vaddr and the size, each corresponding export symbol also needs to be updated. + const decl_exports = module.decl_exports.get(decl) orelse &[0]*Module.Export{}; + return self.updateDeclExports(module, decl, decl_exports); + } + + /// Must be called only after a successful call to `updateDecl`. + pub fn updateDeclExports( + self: *File.Elf, + module: *Module, + decl: *const Module.Decl, + exports: []const *Module.Export, + ) !void { + // In addition to ensuring capacity for global_symbols, we also ensure capacity for freeing all of + // them, so that deleting exports is guaranteed to succeed. + try self.global_symbols.ensureCapacity(self.allocator, self.global_symbols.items.len + exports.len); + try self.global_symbol_free_list.ensureCapacity(self.allocator, self.global_symbols.items.len); + const typed_value = decl.typed_value.most_recent.typed_value; + if (decl.link.local_sym_index == 0) return; + const decl_sym = self.local_symbols.items[decl.link.local_sym_index]; + + for (exports) |exp| { + if (exp.options.section) |section_name| { + if (!mem.eql(u8, section_name, ".text")) { + try module.failed_exports.ensureCapacity(module.failed_exports.items().len + 1); + module.failed_exports.putAssumeCapacityNoClobber( + exp, + try Module.ErrorMsg.create(self.allocator, 0, "Unimplemented: ExportOptions.section", .{}), + ); + continue; + } + } + const stb_bits: u8 = switch (exp.options.linkage) { + .Internal => elf.STB_LOCAL, + .Strong => blk: { + if (mem.eql(u8, exp.options.name, "_start")) { + self.entry_addr = decl_sym.st_value; + } + break :blk elf.STB_GLOBAL; + }, + .Weak => elf.STB_WEAK, + .LinkOnce => { + try module.failed_exports.ensureCapacity(module.failed_exports.items().len + 1); + module.failed_exports.putAssumeCapacityNoClobber( + exp, + try Module.ErrorMsg.create(self.allocator, 0, "Unimplemented: GlobalLinkage.LinkOnce", .{}), + ); + continue; + }, + }; + const stt_bits: u8 = @truncate(u4, decl_sym.st_info); + if (exp.link.sym_index) |i| { + const sym = &self.global_symbols.items[i]; + sym.* = .{ + .st_name = try self.updateString(sym.st_name, exp.options.name), + .st_info = (stb_bits << 4) | stt_bits, + .st_other = 0, + .st_shndx = self.text_section_index.?, + .st_value = decl_sym.st_value, + .st_size = decl_sym.st_size, + }; + } else { + const name = try self.makeString(exp.options.name); + const i = if (self.global_symbol_free_list.popOrNull()) |i| i else blk: { + _ = self.global_symbols.addOneAssumeCapacity(); + break :blk self.global_symbols.items.len - 1; + }; + self.global_symbols.items[i] = .{ + .st_name = name, + .st_info = (stb_bits << 4) | stt_bits, + .st_other = 0, + .st_shndx = self.text_section_index.?, + .st_value = decl_sym.st_value, + .st_size = decl_sym.st_size, + }; + + exp.link.sym_index = @intCast(u32, i); + } } } - } - pub fn deleteExport(self: *ElfFile, exp: Export) void { - const sym_index = exp.sym_index orelse return; - self.global_symbol_free_list.appendAssumeCapacity(sym_index); - self.global_symbols.items[sym_index].st_info = 0; - } - - fn writeProgHeader(self: *ElfFile, index: usize) !void { - const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); - const offset = self.program_headers.items[index].p_offset; - switch (self.options.target.cpu.arch.ptrBitWidth()) { - 32 => { - var phdr = [1]elf.Elf32_Phdr{progHeaderTo32(self.program_headers.items[index])}; - if (foreign_endian) { - bswapAllFields(elf.Elf32_Phdr, &phdr[0]); - } - return self.file.?.pwriteAll(mem.sliceAsBytes(&phdr), offset); - }, - 64 => { - var phdr = [1]elf.Elf64_Phdr{self.program_headers.items[index]}; - if (foreign_endian) { - bswapAllFields(elf.Elf64_Phdr, &phdr[0]); - } - return self.file.?.pwriteAll(mem.sliceAsBytes(&phdr), offset); - }, - else => return error.UnsupportedArchitecture, + pub fn deleteExport(self: *File.Elf, exp: Export) void { + const sym_index = exp.sym_index orelse return; + self.global_symbol_free_list.appendAssumeCapacity(sym_index); + self.global_symbols.items[sym_index].st_info = 0; } - } - fn writeSectHeader(self: *ElfFile, index: usize) !void { - const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); - const offset = self.sections.items[index].sh_offset; - switch (self.options.target.cpu.arch.ptrBitWidth()) { - 32 => { - var shdr: [1]elf.Elf32_Shdr = undefined; - shdr[0] = sectHeaderTo32(self.sections.items[index]); - if (foreign_endian) { - bswapAllFields(elf.Elf32_Shdr, &shdr[0]); - } - return self.file.?.pwriteAll(mem.sliceAsBytes(&shdr), offset); - }, - 64 => { - var shdr = [1]elf.Elf64_Shdr{self.sections.items[index]}; - if (foreign_endian) { - bswapAllFields(elf.Elf64_Shdr, &shdr[0]); - } - return self.file.?.pwriteAll(mem.sliceAsBytes(&shdr), offset); - }, - else => return error.UnsupportedArchitecture, - } - } - - fn writeOffsetTableEntry(self: *ElfFile, index: usize) !void { - const shdr = &self.sections.items[self.got_section_index.?]; - const phdr = &self.program_headers.items[self.phdr_got_index.?]; - const entry_size: u16 = switch (self.ptr_width) { - .p32 => 4, - .p64 => 8, - }; - if (self.offset_table_count_dirty) { - // TODO Also detect virtual address collisions. - const allocated_size = self.allocatedSize(shdr.sh_offset); - const needed_size = self.local_symbols.items.len * entry_size; - if (needed_size > allocated_size) { - // Must move the entire got section. - const new_offset = self.findFreeSpace(needed_size, entry_size); - const amt = try self.file.?.copyRangeAll(shdr.sh_offset, self.file.?, new_offset, shdr.sh_size); - if (amt != shdr.sh_size) return error.InputOutput; - shdr.sh_offset = new_offset; - phdr.p_offset = new_offset; + fn writeProgHeader(self: *File.Elf, index: usize) !void { + const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); + const offset = self.program_headers.items[index].p_offset; + switch (self.options.target.cpu.arch.ptrBitWidth()) { + 32 => { + var phdr = [1]elf.Elf32_Phdr{progHeaderTo32(self.program_headers.items[index])}; + if (foreign_endian) { + bswapAllFields(elf.Elf32_Phdr, &phdr[0]); + } + return self.file.?.pwriteAll(mem.sliceAsBytes(&phdr), offset); + }, + 64 => { + var phdr = [1]elf.Elf64_Phdr{self.program_headers.items[index]}; + if (foreign_endian) { + bswapAllFields(elf.Elf64_Phdr, &phdr[0]); + } + return self.file.?.pwriteAll(mem.sliceAsBytes(&phdr), offset); + }, + else => return error.UnsupportedArchitecture, } - shdr.sh_size = needed_size; - phdr.p_memsz = needed_size; - phdr.p_filesz = needed_size; - - self.shdr_table_dirty = true; // TODO look into making only the one section dirty - self.phdr_table_dirty = true; // TODO look into making only the one program header dirty - - self.offset_table_count_dirty = false; } - const endian = self.options.target.cpu.arch.endian(); - const off = shdr.sh_offset + @as(u64, entry_size) * index; - switch (self.ptr_width) { - .p32 => { - var buf: [4]u8 = undefined; - mem.writeInt(u32, &buf, @intCast(u32, self.offset_table.items[index]), endian); - try self.file.?.pwriteAll(&buf, off); - }, - .p64 => { - var buf: [8]u8 = undefined; - mem.writeInt(u64, &buf, self.offset_table.items[index], endian); - try self.file.?.pwriteAll(&buf, off); - }, - } - } - fn writeSymbol(self: *ElfFile, index: usize) !void { - const syms_sect = &self.sections.items[self.symtab_section_index.?]; - // Make sure we are not pointlessly writing symbol data that will have to get relocated - // due to running out of space. - if (self.local_symbols.items.len != syms_sect.sh_info) { + fn writeSectHeader(self: *File.Elf, index: usize) !void { + const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); + const offset = self.sections.items[index].sh_offset; + switch (self.options.target.cpu.arch.ptrBitWidth()) { + 32 => { + var shdr: [1]elf.Elf32_Shdr = undefined; + shdr[0] = sectHeaderTo32(self.sections.items[index]); + if (foreign_endian) { + bswapAllFields(elf.Elf32_Shdr, &shdr[0]); + } + return self.file.?.pwriteAll(mem.sliceAsBytes(&shdr), offset); + }, + 64 => { + var shdr = [1]elf.Elf64_Shdr{self.sections.items[index]}; + if (foreign_endian) { + bswapAllFields(elf.Elf64_Shdr, &shdr[0]); + } + return self.file.?.pwriteAll(mem.sliceAsBytes(&shdr), offset); + }, + else => return error.UnsupportedArchitecture, + } + } + + fn writeOffsetTableEntry(self: *File.Elf, index: usize) !void { + const shdr = &self.sections.items[self.got_section_index.?]; + const phdr = &self.program_headers.items[self.phdr_got_index.?]; + const entry_size: u16 = switch (self.ptr_width) { + .p32 => 4, + .p64 => 8, + }; + if (self.offset_table_count_dirty) { + // TODO Also detect virtual address collisions. + const allocated_size = self.allocatedSize(shdr.sh_offset); + const needed_size = self.local_symbols.items.len * entry_size; + if (needed_size > allocated_size) { + // Must move the entire got section. + const new_offset = self.findFreeSpace(needed_size, entry_size); + const amt = try self.file.?.copyRangeAll(shdr.sh_offset, self.file.?, new_offset, shdr.sh_size); + if (amt != shdr.sh_size) return error.InputOutput; + shdr.sh_offset = new_offset; + phdr.p_offset = new_offset; + } + shdr.sh_size = needed_size; + phdr.p_memsz = needed_size; + phdr.p_filesz = needed_size; + + self.shdr_table_dirty = true; // TODO look into making only the one section dirty + self.phdr_table_dirty = true; // TODO look into making only the one program header dirty + + self.offset_table_count_dirty = false; + } + const endian = self.options.target.cpu.arch.endian(); + const off = shdr.sh_offset + @as(u64, entry_size) * index; + switch (self.ptr_width) { + .p32 => { + var buf: [4]u8 = undefined; + mem.writeInt(u32, &buf, @intCast(u32, self.offset_table.items[index]), endian); + try self.file.?.pwriteAll(&buf, off); + }, + .p64 => { + var buf: [8]u8 = undefined; + mem.writeInt(u64, &buf, self.offset_table.items[index], endian); + try self.file.?.pwriteAll(&buf, off); + }, + } + } + + fn writeSymbol(self: *File.Elf, index: usize) !void { + const syms_sect = &self.sections.items[self.symtab_section_index.?]; + // Make sure we are not pointlessly writing symbol data that will have to get relocated + // due to running out of space. + if (self.local_symbols.items.len != syms_sect.sh_info) { + const sym_size: u64 = switch (self.ptr_width) { + .p32 => @sizeOf(elf.Elf32_Sym), + .p64 => @sizeOf(elf.Elf64_Sym), + }; + const sym_align: u16 = switch (self.ptr_width) { + .p32 => @alignOf(elf.Elf32_Sym), + .p64 => @alignOf(elf.Elf64_Sym), + }; + const needed_size = (self.local_symbols.items.len + self.global_symbols.items.len) * sym_size; + if (needed_size > self.allocatedSize(syms_sect.sh_offset)) { + // Move all the symbols to a new file location. + const new_offset = self.findFreeSpace(needed_size, sym_align); + const existing_size = @as(u64, syms_sect.sh_info) * sym_size; + const amt = try self.file.?.copyRangeAll(syms_sect.sh_offset, self.file.?, new_offset, existing_size); + if (amt != existing_size) return error.InputOutput; + syms_sect.sh_offset = new_offset; + } + syms_sect.sh_info = @intCast(u32, self.local_symbols.items.len); + syms_sect.sh_size = needed_size; // anticipating adding the global symbols later + self.shdr_table_dirty = true; // TODO look into only writing one section + } + const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); + switch (self.ptr_width) { + .p32 => { + var sym = [1]elf.Elf32_Sym{ + .{ + .st_name = self.local_symbols.items[index].st_name, + .st_value = @intCast(u32, self.local_symbols.items[index].st_value), + .st_size = @intCast(u32, self.local_symbols.items[index].st_size), + .st_info = self.local_symbols.items[index].st_info, + .st_other = self.local_symbols.items[index].st_other, + .st_shndx = self.local_symbols.items[index].st_shndx, + }, + }; + if (foreign_endian) { + bswapAllFields(elf.Elf32_Sym, &sym[0]); + } + const off = syms_sect.sh_offset + @sizeOf(elf.Elf32_Sym) * index; + try self.file.?.pwriteAll(mem.sliceAsBytes(sym[0..1]), off); + }, + .p64 => { + var sym = [1]elf.Elf64_Sym{self.local_symbols.items[index]}; + if (foreign_endian) { + bswapAllFields(elf.Elf64_Sym, &sym[0]); + } + const off = syms_sect.sh_offset + @sizeOf(elf.Elf64_Sym) * index; + try self.file.?.pwriteAll(mem.sliceAsBytes(sym[0..1]), off); + }, + } + } + + fn writeAllGlobalSymbols(self: *File.Elf) !void { + const syms_sect = &self.sections.items[self.symtab_section_index.?]; const sym_size: u64 = switch (self.ptr_width) { .p32 => @sizeOf(elf.Elf32_Sym), .p64 => @sizeOf(elf.Elf64_Sym), }; - const sym_align: u16 = switch (self.ptr_width) { - .p32 => @alignOf(elf.Elf32_Sym), - .p64 => @alignOf(elf.Elf64_Sym), - }; - const needed_size = (self.local_symbols.items.len + self.global_symbols.items.len) * sym_size; - if (needed_size > self.allocatedSize(syms_sect.sh_offset)) { - // Move all the symbols to a new file location. - const new_offset = self.findFreeSpace(needed_size, sym_align); - const existing_size = @as(u64, syms_sect.sh_info) * sym_size; - const amt = try self.file.?.copyRangeAll(syms_sect.sh_offset, self.file.?, new_offset, existing_size); - if (amt != existing_size) return error.InputOutput; - syms_sect.sh_offset = new_offset; + const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); + const global_syms_off = syms_sect.sh_offset + self.local_symbols.items.len * sym_size; + switch (self.ptr_width) { + .p32 => { + const buf = try self.allocator.alloc(elf.Elf32_Sym, self.global_symbols.items.len); + defer self.allocator.free(buf); + + for (buf) |*sym, i| { + sym.* = .{ + .st_name = self.global_symbols.items[i].st_name, + .st_value = @intCast(u32, self.global_symbols.items[i].st_value), + .st_size = @intCast(u32, self.global_symbols.items[i].st_size), + .st_info = self.global_symbols.items[i].st_info, + .st_other = self.global_symbols.items[i].st_other, + .st_shndx = self.global_symbols.items[i].st_shndx, + }; + if (foreign_endian) { + bswapAllFields(elf.Elf32_Sym, sym); + } + } + try self.file.?.pwriteAll(mem.sliceAsBytes(buf), global_syms_off); + }, + .p64 => { + const buf = try self.allocator.alloc(elf.Elf64_Sym, self.global_symbols.items.len); + defer self.allocator.free(buf); + + for (buf) |*sym, i| { + sym.* = .{ + .st_name = self.global_symbols.items[i].st_name, + .st_value = self.global_symbols.items[i].st_value, + .st_size = self.global_symbols.items[i].st_size, + .st_info = self.global_symbols.items[i].st_info, + .st_other = self.global_symbols.items[i].st_other, + .st_shndx = self.global_symbols.items[i].st_shndx, + }; + if (foreign_endian) { + bswapAllFields(elf.Elf64_Sym, sym); + } + } + try self.file.?.pwriteAll(mem.sliceAsBytes(buf), global_syms_off); + }, } - syms_sect.sh_info = @intCast(u32, self.local_symbols.items.len); - syms_sect.sh_size = needed_size; // anticipating adding the global symbols later - self.shdr_table_dirty = true; // TODO look into only writing one section } - const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); - switch (self.ptr_width) { - .p32 => { - var sym = [1]elf.Elf32_Sym{ - .{ - .st_name = self.local_symbols.items[index].st_name, - .st_value = @intCast(u32, self.local_symbols.items[index].st_value), - .st_size = @intCast(u32, self.local_symbols.items[index].st_size), - .st_info = self.local_symbols.items[index].st_info, - .st_other = self.local_symbols.items[index].st_other, - .st_shndx = self.local_symbols.items[index].st_shndx, - }, - }; - if (foreign_endian) { - bswapAllFields(elf.Elf32_Sym, &sym[0]); - } - const off = syms_sect.sh_offset + @sizeOf(elf.Elf32_Sym) * index; - try self.file.?.pwriteAll(mem.sliceAsBytes(sym[0..1]), off); - }, - .p64 => { - var sym = [1]elf.Elf64_Sym{self.local_symbols.items[index]}; - if (foreign_endian) { - bswapAllFields(elf.Elf64_Sym, &sym[0]); - } - const off = syms_sect.sh_offset + @sizeOf(elf.Elf64_Sym) * index; - try self.file.?.pwriteAll(mem.sliceAsBytes(sym[0..1]), off); - }, - } - } - - fn writeAllGlobalSymbols(self: *ElfFile) !void { - const syms_sect = &self.sections.items[self.symtab_section_index.?]; - const sym_size: u64 = switch (self.ptr_width) { - .p32 => @sizeOf(elf.Elf32_Sym), - .p64 => @sizeOf(elf.Elf64_Sym), - }; - const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); - const global_syms_off = syms_sect.sh_offset + self.local_symbols.items.len * sym_size; - switch (self.ptr_width) { - .p32 => { - const buf = try self.allocator.alloc(elf.Elf32_Sym, self.global_symbols.items.len); - defer self.allocator.free(buf); - - for (buf) |*sym, i| { - sym.* = .{ - .st_name = self.global_symbols.items[i].st_name, - .st_value = @intCast(u32, self.global_symbols.items[i].st_value), - .st_size = @intCast(u32, self.global_symbols.items[i].st_size), - .st_info = self.global_symbols.items[i].st_info, - .st_other = self.global_symbols.items[i].st_other, - .st_shndx = self.global_symbols.items[i].st_shndx, - }; - if (foreign_endian) { - bswapAllFields(elf.Elf32_Sym, sym); - } - } - try self.file.?.pwriteAll(mem.sliceAsBytes(buf), global_syms_off); - }, - .p64 => { - const buf = try self.allocator.alloc(elf.Elf64_Sym, self.global_symbols.items.len); - defer self.allocator.free(buf); - - for (buf) |*sym, i| { - sym.* = .{ - .st_name = self.global_symbols.items[i].st_name, - .st_value = self.global_symbols.items[i].st_value, - .st_size = self.global_symbols.items[i].st_size, - .st_info = self.global_symbols.items[i].st_info, - .st_other = self.global_symbols.items[i].st_other, - .st_shndx = self.global_symbols.items[i].st_shndx, - }; - if (foreign_endian) { - bswapAllFields(elf.Elf64_Sym, sym); - } - } - try self.file.?.pwriteAll(mem.sliceAsBytes(buf), global_syms_off); - }, - } - } + }; }; /// Truncates the existing file contents and overwrites the contents. /// Returns an error if `file` is not already open with +read +write +seek abilities. -pub fn createElfFile(allocator: *Allocator, file: fs.File, options: Options) !ElfFile { +pub fn createElfFile(allocator: *Allocator, file: fs.File, options: Options) !File.Elf { switch (options.output_mode) { .Exe => {}, .Obj => {}, @@ -1368,7 +1455,7 @@ pub fn createElfFile(allocator: *Allocator, file: fs.File, options: Options) !El .wasm => return error.TODOImplementWritingWasmObjects, } - var self: ElfFile = .{ + var self: File.Elf = .{ .allocator = allocator, .file = file, .options = options, @@ -1412,7 +1499,7 @@ pub fn createElfFile(allocator: *Allocator, file: fs.File, options: Options) !El } /// Returns error.IncrFailed if incremental update could not be performed. -fn openBinFileInner(allocator: *Allocator, file: fs.File, options: Options) !ElfFile { +fn openBinFileInner(allocator: *Allocator, file: fs.File, options: Options) !File.Elf { switch (options.output_mode) { .Exe => {}, .Obj => {}, @@ -1425,7 +1512,7 @@ fn openBinFileInner(allocator: *Allocator, file: fs.File, options: Options) !Elf .macho => return error.IncrFailed, .wasm => return error.IncrFailed, } - var self: ElfFile = .{ + var self: File.Elf = .{ .allocator = allocator, .file = file, .owns_file_handle = false, diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index ae5cecce20..a1dd659e75 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -64,10 +64,11 @@ pub const TestContext = struct { /// such as QEMU is required for tests to complete. target: std.zig.CrossTarget, /// In order to be able to run e.g. Execution updates, this must be set - /// to Executable. + /// to Executable. This is ignored when generating C output. output_mode: std.builtin.OutputMode, updates: std.ArrayList(Update), extension: TestType, + c_standard: ?Module.CStandard = null, /// Adds a subcase in which the module is updated with `src`, and the /// resulting ZIR is validated against `result`. @@ -187,6 +188,22 @@ pub const TestContext = struct { return ctx.addObj(name, target, .ZIR); } + pub fn addC(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget, T: TestType, standard: Module.CStandard) *Case { + ctx.cases.append(Case{ + .name = name, + .target = target, + .updates = std.ArrayList(Update).init(ctx.cases.allocator), + .output_mode = .Obj, + .extension = T, + .c_standard = standard, + }) catch unreachable; + return &ctx.cases.items[ctx.cases.items.len - 1]; + } + + pub fn c11(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget, src: [:0]const u8, c: [:0]const u8) void { + ctx.addC(name, target, .Zig, .C11).addTransform(src, c); + } + pub fn addCompareOutput( ctx: *TestContext, name: []const u8, @@ -425,6 +442,7 @@ pub const TestContext = struct { .bin_file_path = bin_name, .root_pkg = root_pkg, .keep_source_files_loaded = true, + .c_standard = case.c_standard, }); defer module.deinit(); @@ -463,14 +481,15 @@ pub const TestContext = struct { var test_node = update_node.start("assert", null); test_node.activate(); defer test_node.end(); + const label = if (case.c_standard) |_| "C" else "ZIR"; if (expected_output.len != out_zir.items.len) { - std.debug.warn("{}\nTransformed ZIR length differs:\n================\nExpected:\n================\n{}\n================\nFound: {}\n================\nTest failed.\n", .{ case.name, expected_output, out_zir.items }); + std.debug.warn("{}\nTransformed {} length differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out_zir.items }); std.process.exit(1); } for (expected_output) |e, i| { if (out_zir.items[i] != e) { if (expected_output.len != out_zir.items.len) { - std.debug.warn("{}\nTransformed ZIR differs:\n================\nExpected:\n================\n{}\n================\nFound: {}\n================\nTest failed.\n", .{ case.name, expected_output, out_zir.items }); + std.debug.warn("{}\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out_zir.items }); std.process.exit(1); } } diff --git a/test/stage2/cbe.zig b/test/stage2/cbe.zig new file mode 100644 index 0000000000..884f5c927f --- /dev/null +++ b/test/stage2/cbe.zig @@ -0,0 +1,18 @@ +const std = @import("std"); +const TestContext = @import("../../src-self-hosted/test.zig").TestContext; + +// These tests should work with all platforms, but we're using linux_x64 for +// now for consistency. Will be expanded eventually. +const linux_x64 = std.zig.CrossTarget{ + .cpu_arch = .x86_64, + .os_tag = .linux, +}; + +pub fn addCases(ctx: *TestContext) !void { + // // These tests should work on every platform + // ctx.c11("empty start function", linux_x64, + // \\export fn start() void {} + // , + // \\void start(void) {} + // ); +} diff --git a/test/stage2/test.zig b/test/stage2/test.zig index e0ef291588..f5acc72f93 100644 --- a/test/stage2/test.zig +++ b/test/stage2/test.zig @@ -4,4 +4,5 @@ pub fn addCases(ctx: *TestContext) !void { try @import("compile_errors.zig").addCases(ctx); try @import("compare_output.zig").addCases(ctx); try @import("zir.zig").addCases(ctx); + try @import("cbe.zig").addCases(ctx); } From a17200dab1b0b373e85c6c2cc266078012a81948 Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 16:40:14 -0400 Subject: [PATCH 07/29] CBE skeleton --- src-self-hosted/link.zig | 59 +++++++++++++++++++++++++++++++- src-self-hosted/test.zig | 73 ++++++++++++++++++++++++++-------------- test/stage2/cbe.zig | 12 +++---- 3 files changed, 112 insertions(+), 32 deletions(-) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 9e273a2b9e..e37f567d92 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -38,7 +38,11 @@ pub fn openBinFilePath( errdefer file.close(); if (options.c_standard) |cstd| { - return error.Unimplemented; + var bin_file = try allocator.create(File.C); + errdefer allocator.destroy(bin_file); + bin_file.* = try openCFile(allocator, file, options); + bin_file.owns_file_handle = true; + return &bin_file.base; } else { var bin_file = try allocator.create(File.Elf); errdefer allocator.destroy(bin_file); @@ -82,6 +86,17 @@ pub fn writeFilePath( return result; } +pub fn openCFile(allocator: *Allocator, file: fs.File, options: Options) !File.C { + var self: File.C = .{ + .allocator = allocator, + .file = file, + .options = options, + .owns_file_handle = false, + }; + errdefer self.deinit(); + return self; +} + /// Attempts incremental linking, if the file already exists. /// If incremental linking fails, falls back to truncating the file and rewriting it. /// Returns an error if `file` is not already open with +read +write +seek abilities. @@ -108,6 +123,7 @@ pub const File = struct { pub fn makeWritable(base: *File, dir: fs.Dir, sub_path: []const u8) !void { try switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).makeWritable(dir, sub_path), + .C => @fieldParentPtr(C, "base", base).makeWritable(dir, sub_path), else => unreachable, }; } @@ -122,6 +138,7 @@ pub const File = struct { pub fn updateDecl(base: *File, module: *Module, decl: *Module.Decl) !void { try switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).updateDecl(module, decl), + .C => @fieldParentPtr(C, "base", base).updateDecl(module, decl), else => unreachable, }; } @@ -129,6 +146,9 @@ pub const File = struct { pub fn allocateDeclIndexes(base: *File, decl: *Module.Decl) !void { try switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).allocateDeclIndexes(decl), + .C => { + //TODO + }, else => unreachable, }; } @@ -136,6 +156,7 @@ pub const File = struct { pub fn deinit(base: *File) void { switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).deinit(), + .C => @fieldParentPtr(C, "base", base).deinit(), else => unreachable, } } @@ -143,6 +164,9 @@ pub const File = struct { pub fn flush(base: *File) !void { try switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).flush(), + .C => { + //TODO + }, else => unreachable, }; } @@ -157,6 +181,7 @@ pub const File = struct { pub fn errorFlags(base: *File) ErrorFlags { return switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).error_flags, + .C => return .{ .no_entry_point_found = false }, else => unreachable, }; } @@ -176,6 +201,38 @@ pub const File = struct { pub const ErrorFlags = struct { no_entry_point_found: bool = false, }; + + pub const C = struct { + pub const base_tag: Tag = .C; + base: File = File{ .tag = base_tag }, + + allocator: *Allocator, + file: ?fs.File, + owns_file_handle: bool, + options: Options, + + pub fn makeWritable(self: *File.C, dir: fs.Dir, sub_path: []const u8) !void { + assert(self.owns_file_handle); + if (self.file != null) return; + self.file = try dir.createFile(sub_path, .{ + .truncate = false, + .read = true, + .mode = determineMode(self.options), + }); + } + + pub fn deinit(self: *File.C) void { + if (self.owns_file_handle) { + if (self.file) |f| + f.close(); + } + } + + pub fn updateDecl(self: *File.C, module: *Module, decl: *Module.Decl) !void { + return error.Unimplemented; + } + }; + pub const Elf = struct { pub const base_tag: Tag = .Elf; base: File = File{ .tag = base_tag }, diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index a1dd659e75..ce20c172cf 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -464,33 +464,54 @@ pub const TestContext = struct { switch (update.case) { .Transformation => |expected_output| { - update_node.estimated_total_items = 5; - var emit_node = update_node.start("emit", null); - emit_node.activate(); - var new_zir_module = try zir.emit(allocator, module); - defer new_zir_module.deinit(allocator); - emit_node.end(); + var label: []const u8 = "ZIR"; + if (case.c_standard) |cstd| { + label = @tagName(cstd); + var c: *link.File.C = module.bin_file.cast(link.File.C).?; + var out = c.file.?.reader().readAllAlloc(allocator, 1024 * 1024) catch @panic("Unable to read C output!"); + defer allocator.free(out); - var write_node = update_node.start("write", null); - write_node.activate(); - var out_zir = std.ArrayList(u8).init(allocator); - defer out_zir.deinit(); - try new_zir_module.writeToStream(allocator, out_zir.outStream()); - write_node.end(); + if (expected_output.len != out.len) { + std.debug.warn("{}\nTransformed {} length differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out }); + std.process.exit(1); + } + for (expected_output) |e, i| { + if (out[i] != e) { + if (expected_output.len != out.len) { + std.debug.warn("{}\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out }); + std.process.exit(1); + } + } + } + } else { + update_node.estimated_total_items = 5; + var emit_node = update_node.start("emit", null); + emit_node.activate(); + var new_zir_module = try zir.emit(allocator, module); + defer new_zir_module.deinit(allocator); + emit_node.end(); - var test_node = update_node.start("assert", null); - test_node.activate(); - defer test_node.end(); - const label = if (case.c_standard) |_| "C" else "ZIR"; - if (expected_output.len != out_zir.items.len) { - std.debug.warn("{}\nTransformed {} length differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out_zir.items }); - std.process.exit(1); - } - for (expected_output) |e, i| { - if (out_zir.items[i] != e) { - if (expected_output.len != out_zir.items.len) { - std.debug.warn("{}\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out_zir.items }); - std.process.exit(1); + var write_node = update_node.start("write", null); + write_node.activate(); + var out_zir = std.ArrayList(u8).init(allocator); + defer out_zir.deinit(); + try new_zir_module.writeToStream(allocator, out_zir.outStream()); + write_node.end(); + + var test_node = update_node.start("assert", null); + test_node.activate(); + defer test_node.end(); + + if (expected_output.len != out_zir.items.len) { + std.debug.warn("{}\nTransformed {} length differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out_zir.items }); + std.process.exit(1); + } + for (expected_output) |e, i| { + if (out_zir.items[i] != e) { + if (expected_output.len != out_zir.items.len) { + std.debug.warn("{}\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out_zir.items }); + std.process.exit(1); + } } } } @@ -527,6 +548,8 @@ pub const TestContext = struct { } }, .Execution => |expected_stdout| { + std.debug.assert(case.c_standard == null); + update_node.estimated_total_items = 4; var exec_result = x: { var exec_node = update_node.start("execute", null); diff --git a/test/stage2/cbe.zig b/test/stage2/cbe.zig index 884f5c927f..a599367aec 100644 --- a/test/stage2/cbe.zig +++ b/test/stage2/cbe.zig @@ -9,10 +9,10 @@ const linux_x64 = std.zig.CrossTarget{ }; pub fn addCases(ctx: *TestContext) !void { - // // These tests should work on every platform - // ctx.c11("empty start function", linux_x64, - // \\export fn start() void {} - // , - // \\void start(void) {} - // ); + // These tests should work on every platform + ctx.c11("empty start function", linux_x64, + \\export fn start() void {} + , + \\void start(void) {} + ); } From aaaebfe97fa876356c22469001c6511dbdb8354d Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 16:47:39 -0400 Subject: [PATCH 08/29] Detect unexpected compilation errors in tests --- src-self-hosted/test.zig | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index ce20c172cf..0a4e9708ac 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -462,6 +462,19 @@ pub const TestContext = struct { try module.update(); module_node.end(); + if (update.case != .Error) { + var all_errors = try module.getAllErrorsAlloc(); + defer all_errors.deinit(allocator); + if (all_errors.list.len != 0) { + std.debug.warn("\nErrors occurred updating the module:\n================\n", .{}); + for (all_errors.list) |err| { + std.debug.warn(":{}:{}: error: {}\n================\n", .{ err.line + 1, err.column + 1, err.msg }); + } + std.debug.warn("Test failed.\n", .{}); + std.process.exit(1); + } + } + switch (update.case) { .Transformation => |expected_output| { var label: []const u8 = "ZIR"; @@ -472,13 +485,13 @@ pub const TestContext = struct { defer allocator.free(out); if (expected_output.len != out.len) { - std.debug.warn("{}\nTransformed {} length differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out }); + std.debug.warn("\nTransformed {} length differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ label, expected_output, out }); std.process.exit(1); } for (expected_output) |e, i| { if (out[i] != e) { if (expected_output.len != out.len) { - std.debug.warn("{}\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out }); + std.debug.warn("\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ label, expected_output, out }); std.process.exit(1); } } From 2f28ecf946b566f40e648145af18d207d0c5770d Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 17:06:07 -0400 Subject: [PATCH 09/29] CBE: Get test more useful --- src-self-hosted/cgen.zig | 11 +++++++++++ src-self-hosted/link.zig | 3 ++- src-self-hosted/test.zig | 5 ++++- 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 src-self-hosted/cgen.zig diff --git a/src-self-hosted/cgen.zig b/src-self-hosted/cgen.zig new file mode 100644 index 0000000000..f06f9df6ac --- /dev/null +++ b/src-self-hosted/cgen.zig @@ -0,0 +1,11 @@ +const link = @import("link.zig"); +const Module = @import("Module.zig"); + +const C = link.File.C; +const Decl = Module.Decl; +const CStandard = Module.CStandard; + +pub fn generate(file: *C, decl: *Decl, standard: CStandard) !void { + const writer = file.file.?.writer(); + try writer.print("Generating decl '{}', targeting {}", .{ decl.name, @tagName(standard) }); +} diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index e37f567d92..98133a4f42 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -7,6 +7,7 @@ const Module = @import("Module.zig"); const fs = std.fs; const elf = std.elf; const codegen = @import("codegen.zig"); +const cgen = @import("cgen.zig"); const default_entry_addr = 0x8000000; @@ -229,7 +230,7 @@ pub const File = struct { } pub fn updateDecl(self: *File.C, module: *Module, decl: *Module.Decl) !void { - return error.Unimplemented; + try cgen.generate(self, decl, self.options.c_standard.?); } }; diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 0a4e9708ac..153c77ce47 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -481,7 +481,10 @@ pub const TestContext = struct { if (case.c_standard) |cstd| { label = @tagName(cstd); var c: *link.File.C = module.bin_file.cast(link.File.C).?; - var out = c.file.?.reader().readAllAlloc(allocator, 1024 * 1024) catch @panic("Unable to read C output!"); + c.file.?.close(); + var file = try tmp.dir.openFile(bin_name, .{ .read = true }); + defer file.close(); + var out = file.reader().readAllAlloc(allocator, 1024 * 1024) catch @panic("Unable to read C output!"); defer allocator.free(out); if (expected_output.len != out.len) { From 6ece36a051f5cf2bd2329d3f6512a7e9d55486a3 Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 17:51:59 -0400 Subject: [PATCH 10/29] Working translation of empty function --- src-self-hosted/cgen.zig | 41 +++++++++++++++++++++++++++++++++++++++- src-self-hosted/test.zig | 1 + test/stage2/cbe.zig | 7 +++++-- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src-self-hosted/cgen.zig b/src-self-hosted/cgen.zig index f06f9df6ac..48b725c988 100644 --- a/src-self-hosted/cgen.zig +++ b/src-self-hosted/cgen.zig @@ -1,11 +1,50 @@ const link = @import("link.zig"); const Module = @import("Module.zig"); +const std = @import("std"); +const Value = @import("value.zig").Value; const C = link.File.C; const Decl = Module.Decl; const CStandard = Module.CStandard; +const mem = std.mem; + +/// Maps a name from Zig source to C. This will always give the same output for +/// any given input. +fn map(name: []const u8) ![]const u8 { + return name; +} pub fn generate(file: *C, decl: *Decl, standard: CStandard) !void { const writer = file.file.?.writer(); - try writer.print("Generating decl '{}', targeting {}", .{ decl.name, @tagName(standard) }); + const tv = decl.typed_value.most_recent.typed_value; + switch (tv.ty.zigTypeTag()) { + .Fn => { + const return_type = tv.ty.fnReturnType(); + switch (return_type.zigTypeTag()) { + .NoReturn => try writer.writeAll("_Noreturn void "), + else => return error.Unimplemented, + } + + const name = try map(mem.spanZ(decl.name)); + try writer.print("{} (", .{name}); + if (tv.ty.fnParamLen() == 0) { + try writer.writeAll("void){"); + } else { + return error.Unimplemented; + } + + const func: *Module.Fn = tv.val.cast(Value.Payload.Function).?.func; + const instructions = func.analysis.success.instructions; + if (instructions.len > 0) { + try writer.writeAll("\n\t"); + for (instructions) |inst| { + std.debug.warn("\nTranslating {}\n", .{inst.*}); + } + try writer.writeAll("\n"); + } + + try writer.writeAll("}\n"); + }, + else => return error.Unimplemented, + } } diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 153c77ce47..9e7a7bbf50 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -482,6 +482,7 @@ pub const TestContext = struct { label = @tagName(cstd); var c: *link.File.C = module.bin_file.cast(link.File.C).?; c.file.?.close(); + c.file = null; var file = try tmp.dir.openFile(bin_name, .{ .read = true }); defer file.close(); var out = file.reader().readAllAlloc(allocator, 1024 * 1024) catch @panic("Unable to read C output!"); diff --git a/test/stage2/cbe.zig b/test/stage2/cbe.zig index a599367aec..86f966af82 100644 --- a/test/stage2/cbe.zig +++ b/test/stage2/cbe.zig @@ -11,8 +11,11 @@ const linux_x64 = std.zig.CrossTarget{ pub fn addCases(ctx: *TestContext) !void { // These tests should work on every platform ctx.c11("empty start function", linux_x64, - \\export fn start() void {} + \\export fn _start() noreturn {} , - \\void start(void) {} + // A newline is always generated after every function; this ensures, among + // other things, that there is always a newline at the end of the file + \\_Noreturn void _start(void) {} + \\ ); } From 417c92895263250ca399bcf7a1a2abaee6067624 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 8 Jul 2020 00:03:45 +0200 Subject: [PATCH 11/29] Add comment about memory invalidation in Iterator.next on Win --- lib/std/fs.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index e932253bdb..f8777f28e4 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -453,6 +453,8 @@ pub const Dir = struct { pub const Error = IteratorError; + /// Memory such as file names referenced in this returned entry becomes invalid + /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. pub fn next(self: *Self) Error!?Entry { start_over: while (true) { const w = os.windows; From cf86aa8772f266cb475711aeee7b0c1b2083e6aa Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 18:43:25 -0400 Subject: [PATCH 12/29] Fix a dumb in tests --- src-self-hosted/test.zig | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 9e7a7bbf50..80e40900a0 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -494,10 +494,8 @@ pub const TestContext = struct { } for (expected_output) |e, i| { if (out[i] != e) { - if (expected_output.len != out.len) { - std.debug.warn("\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ label, expected_output, out }); - std.process.exit(1); - } + std.debug.warn("\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ label, expected_output, out }); + std.process.exit(1); } } } else { @@ -525,10 +523,8 @@ pub const TestContext = struct { } for (expected_output) |e, i| { if (out_zir.items[i] != e) { - if (expected_output.len != out_zir.items.len) { - std.debug.warn("{}\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out_zir.items }); - std.process.exit(1); - } + std.debug.warn("{}\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out_zir.items }); + std.process.exit(1); } } } From cf09b335d8b7ad5b43ceefd169b4c2e6be8ed249 Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 19:35:33 -0400 Subject: [PATCH 13/29] CBE: Working function call w/ no args or return value --- src-self-hosted/cgen.zig | 71 ++++++++++++++++++++++++++++++---------- src-self-hosted/link.zig | 28 ++++++++++++++-- test/stage2/cbe.zig | 21 +++++++++--- 3 files changed, 95 insertions(+), 25 deletions(-) diff --git a/src-self-hosted/cgen.zig b/src-self-hosted/cgen.zig index 48b725c988..99828ee362 100644 --- a/src-self-hosted/cgen.zig +++ b/src-self-hosted/cgen.zig @@ -1,7 +1,9 @@ const link = @import("link.zig"); const Module = @import("Module.zig"); -const std = @import("std"); +const ir = @import("ir.zig"); const Value = @import("value.zig").Value; +const Type = @import("type.zig").Type; +const std = @import("std"); const C = link.File.C; const Decl = Module.Decl; @@ -14,36 +16,69 @@ fn map(name: []const u8) ![]const u8 { return name; } +fn renderFunctionSignature(writer: std.ArrayList(u8).Writer, decl: *Decl) !void { + const tv = decl.typed_value.most_recent.typed_value; + switch (tv.ty.fnReturnType().zigTypeTag()) { + .NoReturn => { + try writer.writeAll("_Noreturn void "); + }, + else => return error.Unimplemented, + } + const name = try map(mem.spanZ(decl.name)); + try writer.print("{}(", .{name}); + if (tv.ty.fnParamLen() == 0) { + try writer.writeAll("void)"); + } else { + return error.Unimplemented; + } +} + pub fn generate(file: *C, decl: *Decl, standard: CStandard) !void { - const writer = file.file.?.writer(); + const writer = file.main.writer(); + const header = file.header.writer(); const tv = decl.typed_value.most_recent.typed_value; switch (tv.ty.zigTypeTag()) { .Fn => { - const return_type = tv.ty.fnReturnType(); - switch (return_type.zigTypeTag()) { - .NoReturn => try writer.writeAll("_Noreturn void "), - else => return error.Unimplemented, - } - - const name = try map(mem.spanZ(decl.name)); - try writer.print("{} (", .{name}); - if (tv.ty.fnParamLen() == 0) { - try writer.writeAll("void){"); - } else { - return error.Unimplemented; - } + try renderFunctionSignature(writer, decl); + try writer.writeAll(" {"); const func: *Module.Fn = tv.val.cast(Value.Payload.Function).?.func; const instructions = func.analysis.success.instructions; if (instructions.len > 0) { - try writer.writeAll("\n\t"); for (instructions) |inst| { - std.debug.warn("\nTranslating {}\n", .{inst.*}); + try writer.writeAll("\n\t"); + switch (inst.tag) { + .call => { + const call = inst.cast(ir.Inst.Call).?.args; + if (call.func.cast(ir.Inst.Constant)) |func_inst| { + if (func_inst.val.cast(Value.Payload.Function)) |func_val| { + const target = func_val.func.owner_decl; + const tname = mem.spanZ(target.name); + if (file.called.get(tname) == null) { + try file.called.put(tname, void{}); + try renderFunctionSignature(header, target); + try header.writeAll(";\n"); + } + try writer.print("{}();", .{tname}); + } else { + return error.Unimplemented; + } + if (call.args.len != 0) { + return error.Unimplemented; + } + } else { + return error.Unimplemented; + } + }, + else => { + std.debug.warn("\nTranslating {}\n", .{inst.*}); + }, + } } try writer.writeAll("\n"); } - try writer.writeAll("}\n"); + try writer.writeAll("}\n\n"); }, else => return error.Unimplemented, } diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 98133a4f42..c47e1597c6 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -93,6 +93,9 @@ pub fn openCFile(allocator: *Allocator, file: fs.File, options: Options) !File.C .file = file, .options = options, .owns_file_handle = false, + .main = std.ArrayList(u8).init(allocator), + .header = std.ArrayList(u8).init(allocator), + .called = std.StringHashMap(void).init(allocator), }; errdefer self.deinit(); return self; @@ -165,9 +168,7 @@ pub const File = struct { pub fn flush(base: *File) !void { try switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).flush(), - .C => { - //TODO - }, + .C => @fieldParentPtr(C, "base", base).flush(), else => unreachable, }; } @@ -208,9 +209,12 @@ pub const File = struct { base: File = File{ .tag = base_tag }, allocator: *Allocator, + header: std.ArrayList(u8), + main: std.ArrayList(u8), file: ?fs.File, owns_file_handle: bool, options: Options, + called: std.StringHashMap(void), pub fn makeWritable(self: *File.C, dir: fs.Dir, sub_path: []const u8) !void { assert(self.owns_file_handle); @@ -223,6 +227,9 @@ pub const File = struct { } pub fn deinit(self: *File.C) void { + self.main.deinit(); + self.header.deinit(); + self.called.deinit(); if (self.owns_file_handle) { if (self.file) |f| f.close(); @@ -232,6 +239,21 @@ pub const File = struct { pub fn updateDecl(self: *File.C, module: *Module, decl: *Module.Decl) !void { try cgen.generate(self, decl, self.options.c_standard.?); } + + pub fn flush(self: *File.C) !void { + const writer = self.file.?.writer(); + if (self.header.items.len > 0) { + try self.header.append('\n'); + } + try writer.writeAll(self.header.items); + if (self.main.items.len > 1) { + const last_two = self.main.items[self.main.items.len - 2 ..]; + if (std.mem.eql(u8, last_two, "\n\n")) { + self.main.items.len -= 1; + } + } + try writer.writeAll(self.main.items); + } }; pub const Elf = struct { diff --git a/test/stage2/cbe.zig b/test/stage2/cbe.zig index 86f966af82..88eb6c5953 100644 --- a/test/stage2/cbe.zig +++ b/test/stage2/cbe.zig @@ -9,13 +9,26 @@ const linux_x64 = std.zig.CrossTarget{ }; pub fn addCases(ctx: *TestContext) !void { - // These tests should work on every platform ctx.c11("empty start function", linux_x64, \\export fn _start() noreturn {} , - // A newline is always generated after every function; this ensures, among - // other things, that there is always a newline at the end of the file - \\_Noreturn void _start(void) {} + \\_Noreturn void _start(void) {} + \\ + ); + ctx.c11("less empty start function", linux_x64, + \\fn main() noreturn {} + \\ + \\export fn _start() noreturn { + \\ main(); + \\} + , + \\_Noreturn void main(void); + \\ + \\_Noreturn void _start(void) { + \\ main(); + \\} + \\ + \\_Noreturn void main(void) {} \\ ); } From 64bf1301822866904f52a70213dd71eefce4b99a Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 21:35:00 -0400 Subject: [PATCH 14/29] CBE: working asm Inputs and Outputs; std{int,def}.h auto-inclusion --- src-self-hosted/cgen.zig | 97 +++++++++++++++++++++++++++++++++++----- src-self-hosted/link.zig | 21 +++++++-- test/stage2/cbe.zig | 51 +++++++++++++++++++++ 3 files changed, 154 insertions(+), 15 deletions(-) diff --git a/src-self-hosted/cgen.zig b/src-self-hosted/cgen.zig index 99828ee362..5b04d98119 100644 --- a/src-self-hosted/cgen.zig +++ b/src-self-hosted/cgen.zig @@ -16,16 +16,24 @@ fn map(name: []const u8) ![]const u8 { return name; } -fn renderFunctionSignature(writer: std.ArrayList(u8).Writer, decl: *Decl) !void { - const tv = decl.typed_value.most_recent.typed_value; - switch (tv.ty.fnReturnType().zigTypeTag()) { - .NoReturn => { - try writer.writeAll("_Noreturn void "); - }, - else => return error.Unimplemented, +fn renderType(file: *C, writer: std.ArrayList(u8).Writer, T: Type) !void { + if (T.tag() == .usize) { + file.need_stddef = true; + try writer.writeAll("size_t"); + } else { + switch (T.zigTypeTag()) { + .NoReturn => try writer.writeAll("_Noreturn void"), + .Void => try writer.writeAll("void"), + else => return error.Unimplemented, + } } +} + +fn renderFunctionSignature(file: *C, writer: std.ArrayList(u8).Writer, decl: *Decl) !void { + const tv = decl.typed_value.most_recent.typed_value; + try renderType(file, writer, tv.ty.fnReturnType()); const name = try map(mem.spanZ(decl.name)); - try writer.print("{}(", .{name}); + try writer.print(" {}(", .{name}); if (tv.ty.fnParamLen() == 0) { try writer.writeAll("void)"); } else { @@ -39,7 +47,7 @@ pub fn generate(file: *C, decl: *Decl, standard: CStandard) !void { const tv = decl.typed_value.most_recent.typed_value; switch (tv.ty.zigTypeTag()) { .Fn => { - try renderFunctionSignature(writer, decl); + try renderFunctionSignature(file, writer, decl); try writer.writeAll(" {"); const func: *Module.Fn = tv.val.cast(Value.Payload.Function).?.func; @@ -48,6 +56,55 @@ pub fn generate(file: *C, decl: *Decl, standard: CStandard) !void { for (instructions) |inst| { try writer.writeAll("\n\t"); switch (inst.tag) { + .assembly => { + const as = inst.cast(ir.Inst.Assembly).?.args; + for (as.inputs) |i, index| { + if (i[0] == '{' and i[i.len - 1] == '}') { + const reg = i[1 .. i.len - 1]; + const arg = as.args[index]; + if (arg.cast(ir.Inst.Constant)) |c| { + if (c.val.tag() == .int_u64) { + try writer.writeAll("register "); + try renderType(file, writer, arg.ty); + try writer.print(" {}_constant __asm__(\"{}\") = {};\n\t", .{ reg, reg, c.val.toUnsignedInt() }); + } else { + return error.Unimplemented; + } + } else { + return error.Unimplemented; + } + } else { + return error.Unimplemented; + } + } + try writer.print("__asm {} (\"{}\"", .{ if (as.is_volatile) @as([]const u8, "volatile") else "", as.asm_source }); + if (as.output) |o| { + return error.Unimplemented; + } + if (as.inputs.len > 0) { + if (as.output == null) { + try writer.writeAll(" :"); + } + try writer.writeAll(": "); + for (as.inputs) |i, index| { + if (i[0] == '{' and i[i.len - 1] == '}') { + const reg = i[1 .. i.len - 1]; + const arg = as.args[index]; + if (index > 0) { + try writer.writeAll(", "); + } + if (arg.cast(ir.Inst.Constant)) |c| { + try writer.print("\"\"({}_constant)", .{reg}); + } else { + return error.Unimplemented; + } + } else { + return error.Unimplemented; + } + } + } + try writer.writeAll(");"); + }, .call => { const call = inst.cast(ir.Inst.Call).?.args; if (call.func.cast(ir.Inst.Constant)) |func_inst| { @@ -56,17 +113,20 @@ pub fn generate(file: *C, decl: *Decl, standard: CStandard) !void { const tname = mem.spanZ(target.name); if (file.called.get(tname) == null) { try file.called.put(tname, void{}); - try renderFunctionSignature(header, target); + try renderFunctionSignature(file, header, target); try header.writeAll(";\n"); } try writer.print("{}();", .{tname}); } else { + std.debug.warn("non-function call target?\n", .{}); return error.Unimplemented; } if (call.args.len != 0) { + std.debug.warn("parameters\n", .{}); return error.Unimplemented; } } else { + std.debug.warn("non-constant call inst?\n", .{}); return error.Unimplemented; } }, @@ -80,6 +140,21 @@ pub fn generate(file: *C, decl: *Decl, standard: CStandard) !void { try writer.writeAll("}\n\n"); }, - else => return error.Unimplemented, + .Array => { + if (mem.indexOf(u8, mem.span(decl.name), "$") == null) { + // TODO: prevent inline asm constants from being emitted + if (tv.val.cast(Value.Payload.Bytes)) |payload| { + try writer.print("const char *const {} = \"{}\";\n", .{ decl.name, payload.data }); + std.debug.warn("\n\nARRAYTRANS\n", .{}); + if (tv.ty.arraySentinel()) |sentinel| {} + } else { + return error.Unimplemented; + } + } + }, + else => |e| { + std.debug.warn("\nTODO implement {}\n", .{e}); + return error.Unimplemented; + }, } } diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index c47e1597c6..8352f64fca 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -191,7 +191,7 @@ pub const File = struct { pub fn options(base: *File) Options { return switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).options, - else => unreachable, + .C => @fieldParentPtr(C, "base", base).options, }; } @@ -215,6 +215,8 @@ pub const File = struct { owns_file_handle: bool, options: Options, called: std.StringHashMap(void), + need_stddef: bool = false, + need_stdint: bool = false, pub fn makeWritable(self: *File.C, dir: fs.Dir, sub_path: []const u8) !void { assert(self.owns_file_handle); @@ -242,10 +244,21 @@ pub const File = struct { pub fn flush(self: *File.C) !void { const writer = self.file.?.writer(); - if (self.header.items.len > 0) { - try self.header.append('\n'); + var includes = false; + if (self.need_stddef) { + try writer.writeAll("#include \n"); + includes = true; + } + if (self.need_stdint) { + try writer.writeAll("#include \n"); + includes = true; + } + if (includes) { + try writer.writeByte('\n'); + } + if (self.header.items.len > 0) { + try writer.print("{}\n", .{self.header.items}); } - try writer.writeAll(self.header.items); if (self.main.items.len > 1) { const last_two = self.main.items[self.main.items.len - 2 ..]; if (std.mem.eql(u8, last_two, "\n\n")) { diff --git a/test/stage2/cbe.zig b/test/stage2/cbe.zig index 88eb6c5953..9a28c25e27 100644 --- a/test/stage2/cbe.zig +++ b/test/stage2/cbe.zig @@ -31,4 +31,55 @@ pub fn addCases(ctx: *TestContext) !void { \\_Noreturn void main(void) {} \\ ); + // TODO: implement return values + ctx.c11("inline asm", linux_x64, + \\fn exitGood() void { + \\ asm volatile ("syscall" + \\ : + \\ : [number] "{rax}" (231), + \\ [arg1] "{rdi}" (0) + \\ ); + \\} + \\ + \\export fn _start() noreturn { + \\ exitGood(); + \\} + , + \\#include + \\ + \\void exitGood(void); + \\ + \\_Noreturn void _start(void) { + \\ exitGood(); + \\} + \\ + \\void exitGood(void) { + \\ register size_t rax_constant __asm__("rax") = 231; + \\ register size_t rdi_constant __asm__("rdi") = 0; + \\ __asm volatile ("syscall" :: ""(rax_constant), ""(rdi_constant)); + \\} + \\ + ); + //ctx.c11("basic return", linux_x64, + // \\fn main() u8 { + // \\ return 103; + // \\} + // \\ + // \\export fn _start() noreturn { + // \\ _ = main(); + // \\} + //, + // \\#include + // \\ + // \\uint8_t main(void); + // \\ + // \\_Noreturn void _start(void) { + // \\ (void)main(); + // \\} + // \\ + // \\uint8_t main(void) { + // \\ return 103; + // \\} + // \\ + //); } From 5461c482d0643abd83ac834be0a95da066c53c69 Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 21:54:34 -0400 Subject: [PATCH 15/29] CBE: Integrate into stage2 via --c-standard --- src-self-hosted/main.zig | 106 +++++++++++++++++++++++---------------- 1 file changed, 63 insertions(+), 43 deletions(-) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 33f422692c..fdbc5fc1ae 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -71,7 +71,7 @@ pub fn main() !void { const args = try process.argsAlloc(arena); if (args.len <= 1) { - std.debug.warn("expected command argument\n\n{}", .{usage}); + std.debug.print("expected command argument\n\n{}", .{usage}); process.exit(1); } @@ -91,14 +91,14 @@ pub fn main() !void { return @import("print_targets.zig").cmdTargets(arena, cmd_args, stdout, info.target); } else if (mem.eql(u8, cmd, "version")) { // Need to set up the build script to give the version as a comptime value. - std.debug.warn("TODO version command not implemented yet\n", .{}); + std.debug.print("TODO version command not implemented yet\n", .{}); return error.Unimplemented; } else if (mem.eql(u8, cmd, "zen")) { try io.getStdOut().writeAll(info_zen); } else if (mem.eql(u8, cmd, "help")) { try io.getStdOut().writeAll(usage); } else { - std.debug.warn("unknown command: {}\n\n{}", .{ args[1], usage }); + std.debug.print("unknown command: {}\n\n{}", .{ args[1], usage }); process.exit(1); } } @@ -191,6 +191,7 @@ fn buildOutputType( var emit_zir: Emit = .no; var target_arch_os_abi: []const u8 = "native"; var target_mcpu: ?[]const u8 = null; + var target_c_standard: ?Module.CStandard = null; var target_dynamic_linker: ?[]const u8 = null; var system_libs = std.ArrayList([]const u8).init(gpa); @@ -206,7 +207,7 @@ fn buildOutputType( process.exit(0); } else if (mem.eql(u8, arg, "--color")) { if (i + 1 >= args.len) { - std.debug.warn("expected [auto|on|off] after --color\n", .{}); + std.debug.print("expected [auto|on|off] after --color\n", .{}); process.exit(1); } i += 1; @@ -218,12 +219,12 @@ fn buildOutputType( } else if (mem.eql(u8, next_arg, "off")) { color = .Off; } else { - std.debug.warn("expected [auto|on|off] after --color, found '{}'\n", .{next_arg}); + std.debug.print("expected [auto|on|off] after --color, found '{}'\n", .{next_arg}); process.exit(1); } } else if (mem.eql(u8, arg, "--mode")) { if (i + 1 >= args.len) { - std.debug.warn("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode\n", .{}); + std.debug.print("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode\n", .{}); process.exit(1); } i += 1; @@ -237,52 +238,64 @@ fn buildOutputType( } else if (mem.eql(u8, next_arg, "ReleaseSmall")) { build_mode = .ReleaseSmall; } else { - std.debug.warn("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode, found '{}'\n", .{next_arg}); + std.debug.print("expected [Debug|ReleaseSafe|ReleaseFast|ReleaseSmall] after --mode, found '{}'\n", .{next_arg}); process.exit(1); } } else if (mem.eql(u8, arg, "--name")) { if (i + 1 >= args.len) { - std.debug.warn("expected parameter after --name\n", .{}); + std.debug.print("expected parameter after --name\n", .{}); process.exit(1); } i += 1; provided_name = args[i]; } else if (mem.eql(u8, arg, "--library")) { if (i + 1 >= args.len) { - std.debug.warn("expected parameter after --library\n", .{}); + std.debug.print("expected parameter after --library\n", .{}); process.exit(1); } i += 1; try system_libs.append(args[i]); } else if (mem.eql(u8, arg, "--version")) { if (i + 1 >= args.len) { - std.debug.warn("expected parameter after --version\n", .{}); + std.debug.print("expected parameter after --version\n", .{}); process.exit(1); } i += 1; version = std.builtin.Version.parse(args[i]) catch |err| { - std.debug.warn("unable to parse --version '{}': {}\n", .{ args[i], @errorName(err) }); + std.debug.print("unable to parse --version '{}': {}\n", .{ args[i], @errorName(err) }); process.exit(1); }; } else if (mem.eql(u8, arg, "-target")) { if (i + 1 >= args.len) { - std.debug.warn("expected parameter after -target\n", .{}); + std.debug.print("expected parameter after -target\n", .{}); process.exit(1); } i += 1; target_arch_os_abi = args[i]; } else if (mem.eql(u8, arg, "-mcpu")) { if (i + 1 >= args.len) { - std.debug.warn("expected parameter after -mcpu\n", .{}); + std.debug.print("expected parameter after -mcpu\n", .{}); process.exit(1); } i += 1; target_mcpu = args[i]; + } else if (mem.eql(u8, arg, "--c-standard")) { + if (i + 1 >= args.len) { + std.debug.print("expected parameter after --c-standard\n", .{}); + process.exit(1); + } + i += 1; + if (std.meta.stringToEnum(Module.CStandard, args[i])) |cstd| { + target_c_standard = cstd; + } else { + std.debug.print("Invalid C standard: {}\n", .{args[i]}); + process.exit(1); + } } else if (mem.startsWith(u8, arg, "-mcpu=")) { target_mcpu = arg["-mcpu=".len..]; } else if (mem.eql(u8, arg, "--dynamic-linker")) { if (i + 1 >= args.len) { - std.debug.warn("expected parameter after --dynamic-linker\n", .{}); + std.debug.print("expected parameter after --dynamic-linker\n", .{}); process.exit(1); } i += 1; @@ -324,56 +337,61 @@ fn buildOutputType( } else if (mem.startsWith(u8, arg, "-l")) { try system_libs.append(arg[2..]); } else { - std.debug.warn("unrecognized parameter: '{}'", .{arg}); + std.debug.print("unrecognized parameter: '{}'", .{arg}); process.exit(1); } } else if (mem.endsWith(u8, arg, ".s") or mem.endsWith(u8, arg, ".S")) { - std.debug.warn("assembly files not supported yet", .{}); + std.debug.print("assembly files not supported yet", .{}); process.exit(1); } else if (mem.endsWith(u8, arg, ".o") or mem.endsWith(u8, arg, ".obj") or mem.endsWith(u8, arg, ".a") or mem.endsWith(u8, arg, ".lib")) { - std.debug.warn("object files and static libraries not supported yet", .{}); + std.debug.print("object files and static libraries not supported yet", .{}); process.exit(1); } else if (mem.endsWith(u8, arg, ".c") or mem.endsWith(u8, arg, ".cpp")) { - std.debug.warn("compilation of C and C++ source code requires LLVM extensions which are not implemented yet", .{}); + std.debug.print("compilation of C and C++ source code requires LLVM extensions which are not implemented yet", .{}); process.exit(1); } else if (mem.endsWith(u8, arg, ".so") or mem.endsWith(u8, arg, ".dylib") or mem.endsWith(u8, arg, ".dll")) { - std.debug.warn("linking against dynamic libraries not yet supported", .{}); + std.debug.print("linking against dynamic libraries not yet supported", .{}); process.exit(1); } else if (mem.endsWith(u8, arg, ".zig") or mem.endsWith(u8, arg, ".zir")) { if (root_src_file) |other| { - std.debug.warn("found another zig file '{}' after root source file '{}'", .{ arg, other }); + std.debug.print("found another zig file '{}' after root source file '{}'", .{ arg, other }); process.exit(1); } else { root_src_file = arg; } } else { - std.debug.warn("unrecognized file extension of parameter '{}'", .{arg}); + std.debug.print("unrecognized file extension of parameter '{}'", .{arg}); } } } + if (target_c_standard != null and output_mode != .Obj) { + std.debug.print("The C backend must be used with build-obj\n", .{}); + process.exit(1); + } + const root_name = if (provided_name) |n| n else blk: { if (root_src_file) |file| { const basename = fs.path.basename(file); var it = mem.split(basename, "."); break :blk it.next() orelse basename; } else { - std.debug.warn("--name [name] not provided and unable to infer\n", .{}); + std.debug.print("--name [name] not provided and unable to infer\n", .{}); process.exit(1); } }; if (system_libs.items.len != 0) { - std.debug.warn("linking against system libraries not yet supported", .{}); + std.debug.print("linking against system libraries not yet supported", .{}); process.exit(1); } @@ -385,17 +403,17 @@ fn buildOutputType( .diagnostics = &diags, }) catch |err| switch (err) { error.UnknownCpuModel => { - std.debug.warn("Unknown CPU: '{}'\nAvailable CPUs for architecture '{}':\n", .{ + std.debug.print("Unknown CPU: '{}'\nAvailable CPUs for architecture '{}':\n", .{ diags.cpu_name.?, @tagName(diags.arch.?), }); for (diags.arch.?.allCpuModels()) |cpu| { - std.debug.warn(" {}\n", .{cpu.name}); + std.debug.print(" {}\n", .{cpu.name}); } process.exit(1); }, error.UnknownCpuFeature => { - std.debug.warn( + std.debug.print( \\Unknown CPU feature: '{}' \\Available CPU features for architecture '{}': \\ @@ -404,7 +422,7 @@ fn buildOutputType( @tagName(diags.arch.?), }); for (diags.arch.?.allFeaturesList()) |feature| { - std.debug.warn(" {}: {}\n", .{ feature.name, feature.description }); + std.debug.print(" {}: {}\n", .{ feature.name, feature.description }); } process.exit(1); }, @@ -416,21 +434,22 @@ fn buildOutputType( if (target_info.cpu_detection_unimplemented) { // TODO We want to just use detected_info.target but implementing // CPU model & feature detection is todo so here we rely on LLVM. - std.debug.warn("CPU features detection is not yet available for this system without LLVM extensions\n", .{}); + std.debug.print("CPU features detection is not yet available for this system without LLVM extensions\n", .{}); process.exit(1); } const src_path = root_src_file orelse { - std.debug.warn("expected at least one file argument", .{}); + std.debug.print("expected at least one file argument", .{}); process.exit(1); }; const bin_path = switch (emit_bin) { .no => { - std.debug.warn("-fno-emit-bin not supported yet", .{}); + std.debug.print("-fno-emit-bin not supported yet", .{}); process.exit(1); }, - .yes_default_path => try std.zig.binNameAlloc(arena, root_name, target_info.target, output_mode, link_mode), + .yes_default_path => try std.fmt.allocPrint(arena, "{}.c", .{root_name}), + .yes => |p| p, }; @@ -460,6 +479,7 @@ fn buildOutputType( .object_format = object_format, .optimize_mode = build_mode, .keep_source_files_loaded = zir_out_path != null, + .c_standard = target_c_standard, }); defer module.deinit(); @@ -506,7 +526,7 @@ fn updateModule(gpa: *Allocator, module: *Module, zir_out_path: ?[]const u8) !vo if (errors.list.len != 0) { for (errors.list) |full_err_msg| { - std.debug.warn("{}:{}:{}: error: {}\n", .{ + std.debug.print("{}:{}:{}: error: {}\n", .{ full_err_msg.src_path, full_err_msg.line + 1, full_err_msg.column + 1, @@ -583,7 +603,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { process.exit(0); } else if (mem.eql(u8, arg, "--color")) { if (i + 1 >= args.len) { - std.debug.warn("expected [auto|on|off] after --color\n", .{}); + std.debug.print("expected [auto|on|off] after --color\n", .{}); process.exit(1); } i += 1; @@ -595,7 +615,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { } else if (mem.eql(u8, next_arg, "off")) { color = .Off; } else { - std.debug.warn("expected [auto|on|off] after --color, found '{}'\n", .{next_arg}); + std.debug.print("expected [auto|on|off] after --color, found '{}'\n", .{next_arg}); process.exit(1); } } else if (mem.eql(u8, arg, "--stdin")) { @@ -603,7 +623,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { } else if (mem.eql(u8, arg, "--check")) { check_flag = true; } else { - std.debug.warn("unrecognized parameter: '{}'", .{arg}); + std.debug.print("unrecognized parameter: '{}'", .{arg}); process.exit(1); } } else { @@ -614,7 +634,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { if (stdin_flag) { if (input_files.items.len != 0) { - std.debug.warn("cannot use --stdin with positional arguments\n", .{}); + std.debug.print("cannot use --stdin with positional arguments\n", .{}); process.exit(1); } @@ -624,7 +644,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { defer gpa.free(source_code); const tree = std.zig.parse(gpa, source_code) catch |err| { - std.debug.warn("error parsing stdin: {}\n", .{err}); + std.debug.print("error parsing stdin: {}\n", .{err}); process.exit(1); }; defer tree.deinit(); @@ -647,7 +667,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { } if (input_files.items.len == 0) { - std.debug.warn("expected at least one source file argument\n", .{}); + std.debug.print("expected at least one source file argument\n", .{}); process.exit(1); } @@ -664,7 +684,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { for (input_files.span()) |file_path| { // Get the real path here to avoid Windows failing on relative file paths with . or .. in them. const real_path = fs.realpathAlloc(gpa, file_path) catch |err| { - std.debug.warn("unable to open '{}': {}\n", .{ file_path, err }); + std.debug.print("unable to open '{}': {}\n", .{ file_path, err }); process.exit(1); }; defer gpa.free(real_path); @@ -702,7 +722,7 @@ fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool, dir: fs.Dir, sub_ fmtPathFile(fmt, file_path, check_mode, dir, sub_path) catch |err| switch (err) { error.IsDir, error.AccessDenied => return fmtPathDir(fmt, file_path, check_mode, dir, sub_path), else => { - std.debug.warn("unable to format '{}': {}\n", .{ file_path, err }); + std.debug.print("unable to format '{}': {}\n", .{ file_path, err }); fmt.any_error = true; return; }, @@ -733,7 +753,7 @@ fn fmtPathDir( try fmtPathDir(fmt, full_path, check_mode, dir, entry.name); } else { fmtPathFile(fmt, full_path, check_mode, dir, entry.name) catch |err| { - std.debug.warn("unable to format '{}': {}\n", .{ full_path, err }); + std.debug.print("unable to format '{}': {}\n", .{ full_path, err }); fmt.any_error = true; return; }; @@ -784,7 +804,7 @@ fn fmtPathFile( if (check_mode) { const anything_changed = try std.zig.render(fmt.gpa, io.null_out_stream, tree); if (anything_changed) { - std.debug.warn("{}\n", .{file_path}); + std.debug.print("{}\n", .{file_path}); fmt.any_error = true; } } else { @@ -800,7 +820,7 @@ fn fmtPathFile( try af.file.writeAll(fmt.out_buffer.items); try af.finish(); - std.debug.warn("{}\n", .{file_path}); + std.debug.print("{}\n", .{file_path}); } } From b91cf1597267275dbbf57551acbe0461216ff08e Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 22:57:34 -0400 Subject: [PATCH 16/29] CBE: Move standards determination to generated code --- src-self-hosted/Module.zig | 10 ++------- src-self-hosted/cbe.h | 8 ++++++++ src-self-hosted/cgen.zig | 9 +++++--- src-self-hosted/link.zig | 8 +++++--- src-self-hosted/main.zig | 20 +++++------------- src-self-hosted/test.zig | 42 +++++++++++++++++++------------------- test/stage2/cbe.zig | 20 +++++++++--------- 7 files changed, 57 insertions(+), 60 deletions(-) create mode 100644 src-self-hosted/cbe.h diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index ac7bd00170..2ef85c4d50 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -722,12 +722,6 @@ pub const AllErrors = struct { } }; -pub const CStandard = enum { - C99, - GNU99, - C11, -}; - pub const InitOptions = struct { target: std.Target, root_pkg: *Package, @@ -738,7 +732,7 @@ pub const InitOptions = struct { object_format: ?std.builtin.ObjectFormat = null, optimize_mode: std.builtin.Mode = .Debug, keep_source_files_loaded: bool = false, - c_standard: ?CStandard = null, + cbe: bool = false, }; pub fn init(gpa: *Allocator, options: InitOptions) !Module { @@ -748,7 +742,7 @@ pub fn init(gpa: *Allocator, options: InitOptions) !Module { .output_mode = options.output_mode, .link_mode = options.link_mode orelse .Static, .object_format = options.object_format orelse options.target.getObjectFormat(), - .c_standard = options.c_standard, + .cbe = options.cbe, }); errdefer bin_file.*.deinit(); diff --git a/src-self-hosted/cbe.h b/src-self-hosted/cbe.h new file mode 100644 index 0000000000..85b07eb48a --- /dev/null +++ b/src-self-hosted/cbe.h @@ -0,0 +1,8 @@ +#if __STDC_VERSION__ >= 201112L +#define noreturn _Noreturn +#elif !__STRICT_ANSI__ +#define noreturn __attribute__ ((noreturn)) +#else +#define noreturn +#endif + diff --git a/src-self-hosted/cgen.zig b/src-self-hosted/cgen.zig index 5b04d98119..872fa3e830 100644 --- a/src-self-hosted/cgen.zig +++ b/src-self-hosted/cgen.zig @@ -7,7 +7,6 @@ const std = @import("std"); const C = link.File.C; const Decl = Module.Decl; -const CStandard = Module.CStandard; const mem = std.mem; /// Maps a name from Zig source to C. This will always give the same output for @@ -22,7 +21,10 @@ fn renderType(file: *C, writer: std.ArrayList(u8).Writer, T: Type) !void { try writer.writeAll("size_t"); } else { switch (T.zigTypeTag()) { - .NoReturn => try writer.writeAll("_Noreturn void"), + .NoReturn => { + file.need_noreturn = true; + try writer.writeAll("noreturn void"); + }, .Void => try writer.writeAll("void"), else => return error.Unimplemented, } @@ -41,13 +43,14 @@ fn renderFunctionSignature(file: *C, writer: std.ArrayList(u8).Writer, decl: *De } } -pub fn generate(file: *C, decl: *Decl, standard: CStandard) !void { +pub fn generate(file: *C, decl: *Decl) !void { const writer = file.main.writer(); const header = file.header.writer(); const tv = decl.typed_value.most_recent.typed_value; switch (tv.ty.zigTypeTag()) { .Fn => { try renderFunctionSignature(file, writer, decl); + try writer.writeAll(" {"); const func: *Module.Fn = tv.val.cast(Value.Payload.Function).?.func; diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 8352f64fca..0efab9657f 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -22,7 +22,7 @@ pub const Options = struct { /// Used for calculating how much space to reserve for executable program code in case /// the binary file deos not already have such a section. program_code_size_hint: u64 = 256 * 1024, - c_standard: ?Module.CStandard = null, + cbe: bool = false, }; /// Attempts incremental linking, if the file already exists. @@ -38,7 +38,7 @@ pub fn openBinFilePath( const file = try dir.createFile(sub_path, .{ .truncate = false, .read = true, .mode = determineMode(options) }); errdefer file.close(); - if (options.c_standard) |cstd| { + if (options.cbe) { var bin_file = try allocator.create(File.C); errdefer allocator.destroy(bin_file); bin_file.* = try openCFile(allocator, file, options); @@ -217,6 +217,7 @@ pub const File = struct { called: std.StringHashMap(void), need_stddef: bool = false, need_stdint: bool = false, + need_noreturn: bool = false, pub fn makeWritable(self: *File.C, dir: fs.Dir, sub_path: []const u8) !void { assert(self.owns_file_handle); @@ -239,11 +240,12 @@ pub const File = struct { } pub fn updateDecl(self: *File.C, module: *Module, decl: *Module.Decl) !void { - try cgen.generate(self, decl, self.options.c_standard.?); + try cgen.generate(self, decl); } pub fn flush(self: *File.C) !void { const writer = self.file.?.writer(); + try writer.writeAll(@embedFile("cbe.h")); var includes = false; if (self.need_stddef) { try writer.writeAll("#include \n"); diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index fdbc5fc1ae..fc02a57cc8 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -191,7 +191,7 @@ fn buildOutputType( var emit_zir: Emit = .no; var target_arch_os_abi: []const u8 = "native"; var target_mcpu: ?[]const u8 = null; - var target_c_standard: ?Module.CStandard = null; + var cbe: bool = false; var target_dynamic_linker: ?[]const u8 = null; var system_libs = std.ArrayList([]const u8).init(gpa); @@ -279,18 +279,8 @@ fn buildOutputType( } i += 1; target_mcpu = args[i]; - } else if (mem.eql(u8, arg, "--c-standard")) { - if (i + 1 >= args.len) { - std.debug.print("expected parameter after --c-standard\n", .{}); - process.exit(1); - } - i += 1; - if (std.meta.stringToEnum(Module.CStandard, args[i])) |cstd| { - target_c_standard = cstd; - } else { - std.debug.print("Invalid C standard: {}\n", .{args[i]}); - process.exit(1); - } + } else if (mem.eql(u8, arg, "--c")) { + cbe = true; } else if (mem.startsWith(u8, arg, "-mcpu=")) { target_mcpu = arg["-mcpu=".len..]; } else if (mem.eql(u8, arg, "--dynamic-linker")) { @@ -374,7 +364,7 @@ fn buildOutputType( } } - if (target_c_standard != null and output_mode != .Obj) { + if (cbe and output_mode != .Obj) { std.debug.print("The C backend must be used with build-obj\n", .{}); process.exit(1); } @@ -479,7 +469,7 @@ fn buildOutputType( .object_format = object_format, .optimize_mode = build_mode, .keep_source_files_loaded = zir_out_path != null, - .c_standard = target_c_standard, + .cbe = cbe, }); defer module.deinit(); diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 80e40900a0..6eca7ab651 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -5,6 +5,8 @@ const Allocator = std.mem.Allocator; const zir = @import("zir.zig"); const Package = @import("Package.zig"); +const cheader = @embedFile("cbe.h"); + test "self-hosted" { var ctx = TestContext.init(); defer ctx.deinit(); @@ -68,7 +70,7 @@ pub const TestContext = struct { output_mode: std.builtin.OutputMode, updates: std.ArrayList(Update), extension: TestType, - c_standard: ?Module.CStandard = null, + cbe: bool = false, /// Adds a subcase in which the module is updated with `src`, and the /// resulting ZIR is validated against `result`. @@ -188,20 +190,20 @@ pub const TestContext = struct { return ctx.addObj(name, target, .ZIR); } - pub fn addC(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget, T: TestType, standard: Module.CStandard) *Case { + pub fn addC(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget, T: TestType) *Case { ctx.cases.append(Case{ .name = name, .target = target, .updates = std.ArrayList(Update).init(ctx.cases.allocator), .output_mode = .Obj, .extension = T, - .c_standard = standard, + .cbe = true, }) catch unreachable; return &ctx.cases.items[ctx.cases.items.len - 1]; } - pub fn c11(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget, src: [:0]const u8, c: [:0]const u8) void { - ctx.addC(name, target, .Zig, .C11).addTransform(src, c); + pub fn c(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget, src: [:0]const u8, comptime out: [:0]const u8) void { + ctx.addC(name, target, .Zig).addTransform(src, cheader ++ out); } pub fn addCompareOutput( @@ -382,13 +384,13 @@ pub const TestContext = struct { } fn deinit(self: *TestContext) void { - for (self.cases.items) |c| { - for (c.updates.items) |u| { + for (self.cases.items) |case| { + for (case.updates.items) |u| { if (u.case == .Error) { - c.updates.allocator.free(u.case.Error); + case.updates.allocator.free(u.case.Error); } } - c.updates.deinit(); + case.updates.deinit(); } self.cases.deinit(); self.* = undefined; @@ -442,7 +444,7 @@ pub const TestContext = struct { .bin_file_path = bin_name, .root_pkg = root_pkg, .keep_source_files_loaded = true, - .c_standard = case.c_standard, + .cbe = case.cbe, }); defer module.deinit(); @@ -477,24 +479,22 @@ pub const TestContext = struct { switch (update.case) { .Transformation => |expected_output| { - var label: []const u8 = "ZIR"; - if (case.c_standard) |cstd| { - label = @tagName(cstd); - var c: *link.File.C = module.bin_file.cast(link.File.C).?; - c.file.?.close(); - c.file = null; + if (case.cbe) { + var cfile: *link.File.C = module.bin_file.cast(link.File.C).?; + cfile.file.?.close(); + cfile.file = null; var file = try tmp.dir.openFile(bin_name, .{ .read = true }); defer file.close(); var out = file.reader().readAllAlloc(allocator, 1024 * 1024) catch @panic("Unable to read C output!"); defer allocator.free(out); if (expected_output.len != out.len) { - std.debug.warn("\nTransformed {} length differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ label, expected_output, out }); + std.debug.warn("\nTransformed C length differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ expected_output, out }); std.process.exit(1); } for (expected_output) |e, i| { if (out[i] != e) { - std.debug.warn("\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ label, expected_output, out }); + std.debug.warn("\nTransformed C differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ expected_output, out }); std.process.exit(1); } } @@ -518,12 +518,12 @@ pub const TestContext = struct { defer test_node.end(); if (expected_output.len != out_zir.items.len) { - std.debug.warn("{}\nTransformed {} length differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out_zir.items }); + std.debug.warn("{}\nTransformed ZIR length differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, expected_output, out_zir.items }); std.process.exit(1); } for (expected_output) |e, i| { if (out_zir.items[i] != e) { - std.debug.warn("{}\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out_zir.items }); + std.debug.warn("{}\nTransformed ZIR differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, expected_output, out_zir.items }); std.process.exit(1); } } @@ -561,7 +561,7 @@ pub const TestContext = struct { } }, .Execution => |expected_stdout| { - std.debug.assert(case.c_standard == null); + std.debug.assert(!case.cbe); update_node.estimated_total_items = 4; var exec_result = x: { diff --git a/test/stage2/cbe.zig b/test/stage2/cbe.zig index 9a28c25e27..e56507bd84 100644 --- a/test/stage2/cbe.zig +++ b/test/stage2/cbe.zig @@ -9,30 +9,30 @@ const linux_x64 = std.zig.CrossTarget{ }; pub fn addCases(ctx: *TestContext) !void { - ctx.c11("empty start function", linux_x64, + ctx.c("empty start function", linux_x64, \\export fn _start() noreturn {} , - \\_Noreturn void _start(void) {} + \\noreturn void _start(void) {} \\ ); - ctx.c11("less empty start function", linux_x64, + ctx.c("less empty start function", linux_x64, \\fn main() noreturn {} \\ \\export fn _start() noreturn { \\ main(); \\} , - \\_Noreturn void main(void); + \\noreturn void main(void); \\ - \\_Noreturn void _start(void) { + \\noreturn void _start(void) { \\ main(); \\} \\ - \\_Noreturn void main(void) {} + \\noreturn void main(void) {} \\ ); // TODO: implement return values - ctx.c11("inline asm", linux_x64, + ctx.c("inline asm", linux_x64, \\fn exitGood() void { \\ asm volatile ("syscall" \\ : @@ -49,7 +49,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ \\void exitGood(void); \\ - \\_Noreturn void _start(void) { + \\noreturn void _start(void) { \\ exitGood(); \\} \\ @@ -60,7 +60,7 @@ pub fn addCases(ctx: *TestContext) !void { \\} \\ ); - //ctx.c11("basic return", linux_x64, + //ctx.c("basic return", linux_x64, // \\fn main() u8 { // \\ return 103; // \\} @@ -73,7 +73,7 @@ pub fn addCases(ctx: *TestContext) !void { // \\ // \\uint8_t main(void); // \\ - // \\_Noreturn void _start(void) { + // \\noreturn void _start(void) { // \\ (void)main(); // \\} // \\ From 5667a21b1e7b8aa2dfcefed81d2b594029a116db Mon Sep 17 00:00:00 2001 From: Vexu Date: Tue, 7 Jul 2020 23:31:38 +0300 Subject: [PATCH 17/29] fix missing check on extern variables with no type --- src/analyze.cpp | 4 ++++ test/compile_errors.zig | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/src/analyze.cpp b/src/analyze.cpp index 699b121276..afe0fe6849 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -4012,6 +4012,10 @@ static void resolve_decl_var(CodeGen *g, TldVar *tld_var, bool allow_lazy) { } else if (!is_extern) { add_node_error(g, source_node, buf_sprintf("variables must be initialized")); implicit_type = g->builtin_types.entry_invalid; + } else if (explicit_type == nullptr) { + // extern variable without explicit type + add_node_error(g, source_node, buf_sprintf("unable to infer variable type")); + implicit_type = g->builtin_types.entry_invalid; } ZigType *type = explicit_type ? explicit_type : implicit_type; diff --git a/test/compile_errors.zig b/test/compile_errors.zig index adf92800fe..611094c050 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -2,6 +2,15 @@ const tests = @import("tests.zig"); const std = @import("std"); pub fn addCases(cases: *tests.CompileErrorContext) void { + cases.add("extern variable has no type", + \\extern var foo; + \\pub export fn entry() void { + \\ foo; + \\} + , &[_][]const u8{ + "tmp.zig:1:1: error: unable to infer variable type", + }); + cases.add("@src outside function", \\comptime { \\ @src(); From 173e6712417c8102ff6bbd28fd964b89eda14f36 Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 23:11:17 -0400 Subject: [PATCH 18/29] CBE: Some cleanup --- src-self-hosted/link.zig | 88 ++++++++++++++++++++-------------------- src-self-hosted/main.zig | 5 --- src-self-hosted/test.zig | 2 +- 3 files changed, 44 insertions(+), 51 deletions(-) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 0efab9657f..6625598d11 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -125,36 +125,34 @@ pub const File = struct { } pub fn makeWritable(base: *File, dir: fs.Dir, sub_path: []const u8) !void { - try switch (base.tag) { - .Elf => @fieldParentPtr(Elf, "base", base).makeWritable(dir, sub_path), - .C => @fieldParentPtr(C, "base", base).makeWritable(dir, sub_path), + switch (base.tag) { + .Elf => return @fieldParentPtr(Elf, "base", base).makeWritable(dir, sub_path), + .C => return @fieldParentPtr(C, "base", base).makeWritable(dir, sub_path), else => unreachable, - }; + } } pub fn makeExecutable(base: *File) !void { - try switch (base.tag) { - .Elf => @fieldParentPtr(Elf, "base", base).makeExecutable(), + switch (base.tag) { + .Elf => return @fieldParentPtr(Elf, "base", base).makeExecutable(), else => unreachable, - }; + } } pub fn updateDecl(base: *File, module: *Module, decl: *Module.Decl) !void { - try switch (base.tag) { - .Elf => @fieldParentPtr(Elf, "base", base).updateDecl(module, decl), - .C => @fieldParentPtr(C, "base", base).updateDecl(module, decl), + switch (base.tag) { + .Elf => return @fieldParentPtr(Elf, "base", base).updateDecl(module, decl), + .C => return @fieldParentPtr(C, "base", base).updateDecl(module, decl), else => unreachable, - }; + } } pub fn allocateDeclIndexes(base: *File, decl: *Module.Decl) !void { - try switch (base.tag) { - .Elf => @fieldParentPtr(Elf, "base", base).allocateDeclIndexes(decl), - .C => { - //TODO - }, + switch (base.tag) { + .Elf => return @fieldParentPtr(Elf, "base", base).allocateDeclIndexes(decl), + .C => {}, else => unreachable, - }; + } } pub fn deinit(base: *File) void { @@ -380,7 +378,7 @@ pub const File = struct { /// Returns how much room there is to grow in virtual address space. /// File offset relocation happens transparently, so it is not included in /// this calculation. - fn capacity(self: TextBlock, elf_file: File.Elf) u64 { + fn capacity(self: TextBlock, elf_file: Elf) u64 { const self_sym = elf_file.local_symbols.items[self.local_sym_index]; if (self.next) |next| { const next_sym = elf_file.local_symbols.items[next.local_sym_index]; @@ -391,7 +389,7 @@ pub const File = struct { } } - fn freeListEligible(self: TextBlock, elf_file: File.Elf) bool { + fn freeListEligible(self: TextBlock, elf_file: Elf) bool { // No need to keep a free list node for the last block. const next = self.next orelse return false; const self_sym = elf_file.local_symbols.items[self.local_sym_index]; @@ -408,7 +406,7 @@ pub const File = struct { sym_index: ?u32 = null, }; - pub fn deinit(self: *File.Elf) void { + pub fn deinit(self: *Elf) void { self.sections.deinit(self.allocator); self.program_headers.deinit(self.allocator); self.shstrtab.deinit(self.allocator); @@ -424,7 +422,7 @@ pub const File = struct { } } - pub fn makeExecutable(self: *File.Elf) !void { + pub fn makeExecutable(self: *Elf) !void { assert(self.owns_file_handle); if (self.file) |f| { f.close(); @@ -432,7 +430,7 @@ pub const File = struct { } } - pub fn makeWritable(self: *File.Elf, dir: fs.Dir, sub_path: []const u8) !void { + pub fn makeWritable(self: *Elf, dir: fs.Dir, sub_path: []const u8) !void { assert(self.owns_file_handle); if (self.file != null) return; self.file = try dir.createFile(sub_path, .{ @@ -443,7 +441,7 @@ pub const File = struct { } /// Returns end pos of collision, if any. - fn detectAllocCollision(self: *File.Elf, start: u64, size: u64) ?u64 { + fn detectAllocCollision(self: *Elf, start: u64, size: u64) ?u64 { const small_ptr = self.options.target.cpu.arch.ptrBitWidth() == 32; const ehdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Ehdr) else @sizeOf(elf.Elf64_Ehdr); if (start < ehdr_size) @@ -488,7 +486,7 @@ pub const File = struct { return null; } - fn allocatedSize(self: *File.Elf, start: u64) u64 { + fn allocatedSize(self: *Elf, start: u64) u64 { var min_pos: u64 = std.math.maxInt(u64); if (self.shdr_table_offset) |off| { if (off > start and off < min_pos) min_pos = off; @@ -507,7 +505,7 @@ pub const File = struct { return min_pos - start; } - fn findFreeSpace(self: *File.Elf, object_size: u64, min_alignment: u16) u64 { + fn findFreeSpace(self: *Elf, object_size: u64, min_alignment: u16) u64 { var start: u64 = 0; while (self.detectAllocCollision(start, object_size)) |item_end| { start = mem.alignForwardGeneric(u64, item_end, min_alignment); @@ -515,7 +513,7 @@ pub const File = struct { return start; } - fn makeString(self: *File.Elf, bytes: []const u8) !u32 { + fn makeString(self: *Elf, bytes: []const u8) !u32 { try self.shstrtab.ensureCapacity(self.allocator, self.shstrtab.items.len + bytes.len + 1); const result = self.shstrtab.items.len; self.shstrtab.appendSliceAssumeCapacity(bytes); @@ -523,12 +521,12 @@ pub const File = struct { return @intCast(u32, result); } - fn getString(self: *File.Elf, str_off: u32) []const u8 { + fn getString(self: *Elf, str_off: u32) []const u8 { assert(str_off < self.shstrtab.items.len); return mem.spanZ(@ptrCast([*:0]const u8, self.shstrtab.items.ptr + str_off)); } - fn updateString(self: *File.Elf, old_str_off: u32, new_name: []const u8) !u32 { + fn updateString(self: *Elf, old_str_off: u32, new_name: []const u8) !u32 { const existing_name = self.getString(old_str_off); if (mem.eql(u8, existing_name, new_name)) { return old_str_off; @@ -536,7 +534,7 @@ pub const File = struct { return self.makeString(new_name); } - pub fn populateMissingMetadata(self: *File.Elf) !void { + pub fn populateMissingMetadata(self: *Elf) !void { const small_ptr = switch (self.ptr_width) { .p32 => true, .p64 => false, @@ -703,7 +701,7 @@ pub const File = struct { } /// Commit pending changes and write headers. - pub fn flush(self: *File.Elf) !void { + pub fn flush(self: *Elf) !void { const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); // Unfortunately these have to be buffered and done at the end because ELF does not allow @@ -839,7 +837,7 @@ pub const File = struct { assert(syms_sect.sh_info == self.local_symbols.items.len); } - fn writeElfHeader(self: *File.Elf) !void { + fn writeElfHeader(self: *Elf) !void { var hdr_buf: [@sizeOf(elf.Elf64_Ehdr)]u8 = undefined; var index: usize = 0; @@ -960,7 +958,7 @@ pub const File = struct { try self.file.?.pwriteAll(hdr_buf[0..index], 0); } - fn freeTextBlock(self: *File.Elf, text_block: *TextBlock) void { + fn freeTextBlock(self: *Elf, text_block: *TextBlock) void { var already_have_free_list_node = false; { var i: usize = 0; @@ -1000,12 +998,12 @@ pub const File = struct { } } - fn shrinkTextBlock(self: *File.Elf, text_block: *TextBlock, new_block_size: u64) void { + fn shrinkTextBlock(self: *Elf, text_block: *TextBlock, new_block_size: u64) void { // TODO check the new capacity, and if it crosses the size threshold into a big enough // capacity, insert a free list node for it. } - fn growTextBlock(self: *File.Elf, text_block: *TextBlock, new_block_size: u64, alignment: u64) !u64 { + fn growTextBlock(self: *Elf, text_block: *TextBlock, new_block_size: u64, alignment: u64) !u64 { const sym = self.local_symbols.items[text_block.local_sym_index]; const align_ok = mem.alignBackwardGeneric(u64, sym.st_value, alignment) == sym.st_value; const need_realloc = !align_ok or new_block_size > text_block.capacity(self.*); @@ -1013,7 +1011,7 @@ pub const File = struct { return self.allocateTextBlock(text_block, new_block_size, alignment); } - fn allocateTextBlock(self: *File.Elf, text_block: *TextBlock, new_block_size: u64, alignment: u64) !u64 { + fn allocateTextBlock(self: *Elf, text_block: *TextBlock, new_block_size: u64, alignment: u64) !u64 { const phdr = &self.program_headers.items[self.phdr_load_re_index.?]; const shdr = &self.sections.items[self.text_section_index.?]; const new_block_ideal_capacity = new_block_size * alloc_num / alloc_den; @@ -1127,7 +1125,7 @@ pub const File = struct { return vaddr; } - pub fn allocateDeclIndexes(self: *File.Elf, decl: *Module.Decl) !void { + pub fn allocateDeclIndexes(self: *Elf, decl: *Module.Decl) !void { if (decl.link.local_sym_index != 0) return; // Here we also ensure capacity for the free lists so that they can be appended to without fail. @@ -1166,7 +1164,7 @@ pub const File = struct { self.offset_table.items[decl.link.offset_table_index] = 0; } - pub fn freeDecl(self: *File.Elf, decl: *Module.Decl) void { + pub fn freeDecl(self: *Elf, decl: *Module.Decl) void { self.freeTextBlock(&decl.link); if (decl.link.local_sym_index != 0) { self.local_symbol_free_list.appendAssumeCapacity(decl.link.local_sym_index); @@ -1178,7 +1176,7 @@ pub const File = struct { } } - pub fn updateDecl(self: *File.Elf, module: *Module, decl: *Module.Decl) !void { + pub fn updateDecl(self: *Elf, module: *Module, decl: *Module.Decl) !void { var code_buffer = std.ArrayList(u8).init(self.allocator); defer code_buffer.deinit(); @@ -1258,7 +1256,7 @@ pub const File = struct { /// Must be called only after a successful call to `updateDecl`. pub fn updateDeclExports( - self: *File.Elf, + self: *Elf, module: *Module, decl: *const Module.Decl, exports: []const *Module.Export, @@ -1331,13 +1329,13 @@ pub const File = struct { } } - pub fn deleteExport(self: *File.Elf, exp: Export) void { + pub fn deleteExport(self: *Elf, exp: Export) void { const sym_index = exp.sym_index orelse return; self.global_symbol_free_list.appendAssumeCapacity(sym_index); self.global_symbols.items[sym_index].st_info = 0; } - fn writeProgHeader(self: *File.Elf, index: usize) !void { + fn writeProgHeader(self: *Elf, index: usize) !void { const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); const offset = self.program_headers.items[index].p_offset; switch (self.options.target.cpu.arch.ptrBitWidth()) { @@ -1359,7 +1357,7 @@ pub const File = struct { } } - fn writeSectHeader(self: *File.Elf, index: usize) !void { + fn writeSectHeader(self: *Elf, index: usize) !void { const foreign_endian = self.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); const offset = self.sections.items[index].sh_offset; switch (self.options.target.cpu.arch.ptrBitWidth()) { @@ -1382,7 +1380,7 @@ pub const File = struct { } } - fn writeOffsetTableEntry(self: *File.Elf, index: usize) !void { + fn writeOffsetTableEntry(self: *Elf, index: usize) !void { const shdr = &self.sections.items[self.got_section_index.?]; const phdr = &self.program_headers.items[self.phdr_got_index.?]; const entry_size: u16 = switch (self.ptr_width) { @@ -1426,7 +1424,7 @@ pub const File = struct { } } - fn writeSymbol(self: *File.Elf, index: usize) !void { + fn writeSymbol(self: *Elf, index: usize) !void { const syms_sect = &self.sections.items[self.symtab_section_index.?]; // Make sure we are not pointlessly writing symbol data that will have to get relocated // due to running out of space. @@ -1482,7 +1480,7 @@ pub const File = struct { } } - fn writeAllGlobalSymbols(self: *File.Elf) !void { + fn writeAllGlobalSymbols(self: *Elf) !void { const syms_sect = &self.sections.items[self.symtab_section_index.?]; const sym_size: u64 = switch (self.ptr_width) { .p32 => @sizeOf(elf.Elf32_Sym), diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index fc02a57cc8..3d859483a5 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -364,11 +364,6 @@ fn buildOutputType( } } - if (cbe and output_mode != .Obj) { - std.debug.print("The C backend must be used with build-obj\n", .{}); - process.exit(1); - } - const root_name = if (provided_name) |n| n else blk: { if (root_src_file) |file| { const basename = fs.path.basename(file); diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 6eca7ab651..0a32008369 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -66,7 +66,7 @@ pub const TestContext = struct { /// such as QEMU is required for tests to complete. target: std.zig.CrossTarget, /// In order to be able to run e.g. Execution updates, this must be set - /// to Executable. This is ignored when generating C output. + /// to Executable. output_mode: std.builtin.OutputMode, updates: std.ArrayList(Update), extension: TestType, From 7a6104929b2d140235597d866dc0fabc93df036f Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 23:19:25 -0400 Subject: [PATCH 19/29] CBE: truncate output file --- src-self-hosted/link.zig | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 6625598d11..297b3d8800 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -35,14 +35,13 @@ pub fn openBinFilePath( sub_path: []const u8, options: Options, ) !*File { - const file = try dir.createFile(sub_path, .{ .truncate = false, .read = true, .mode = determineMode(options) }); + const file = try dir.createFile(sub_path, .{ .truncate = options.cbe, .read = true, .mode = determineMode(options) }); errdefer file.close(); if (options.cbe) { var bin_file = try allocator.create(File.C); errdefer allocator.destroy(bin_file); bin_file.* = try openCFile(allocator, file, options); - bin_file.owns_file_handle = true; return &bin_file.base; } else { var bin_file = try allocator.create(File.Elf); @@ -88,17 +87,14 @@ pub fn writeFilePath( } pub fn openCFile(allocator: *Allocator, file: fs.File, options: Options) !File.C { - var self: File.C = .{ + return File.C{ .allocator = allocator, .file = file, .options = options, - .owns_file_handle = false, .main = std.ArrayList(u8).init(allocator), .header = std.ArrayList(u8).init(allocator), .called = std.StringHashMap(void).init(allocator), }; - errdefer self.deinit(); - return self; } /// Attempts incremental linking, if the file already exists. @@ -127,7 +123,7 @@ pub const File = struct { pub fn makeWritable(base: *File, dir: fs.Dir, sub_path: []const u8) !void { switch (base.tag) { .Elf => return @fieldParentPtr(Elf, "base", base).makeWritable(dir, sub_path), - .C => return @fieldParentPtr(C, "base", base).makeWritable(dir, sub_path), + .C => {}, else => unreachable, } } @@ -210,31 +206,18 @@ pub const File = struct { header: std.ArrayList(u8), main: std.ArrayList(u8), file: ?fs.File, - owns_file_handle: bool, options: Options, called: std.StringHashMap(void), need_stddef: bool = false, need_stdint: bool = false, need_noreturn: bool = false, - pub fn makeWritable(self: *File.C, dir: fs.Dir, sub_path: []const u8) !void { - assert(self.owns_file_handle); - if (self.file != null) return; - self.file = try dir.createFile(sub_path, .{ - .truncate = false, - .read = true, - .mode = determineMode(self.options), - }); - } - pub fn deinit(self: *File.C) void { self.main.deinit(); self.header.deinit(); self.called.deinit(); - if (self.owns_file_handle) { - if (self.file) |f| - f.close(); - } + if (self.file) |f| + f.close(); } pub fn updateDecl(self: *File.C, module: *Module, decl: *Module.Decl) !void { From 089c056dbe1fdbb1da9b1a5bab970ca688d3aa01 Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 23:24:30 -0400 Subject: [PATCH 20/29] CBE: Improve resource cleanup --- src-self-hosted/Module.zig | 5 ++--- src-self-hosted/link.zig | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 2ef85c4d50..9df02dd186 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -744,7 +744,7 @@ pub fn init(gpa: *Allocator, options: InitOptions) !Module { .object_format = options.object_format orelse options.target.getObjectFormat(), .cbe = options.cbe, }); - errdefer bin_file.*.deinit(); + errdefer bin_file.destroy(); const root_scope = blk: { if (mem.endsWith(u8, options.root_pkg.root_src_path, ".zig")) { @@ -793,9 +793,8 @@ pub fn init(gpa: *Allocator, options: InitOptions) !Module { } pub fn deinit(self: *Module) void { - self.bin_file.deinit(); + self.bin_file.destroy(); const allocator = self.allocator; - allocator.destroy(self.bin_file); self.deletion_set.deinit(allocator); self.work_queue.deinit(); diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 297b3d8800..da01282995 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -159,6 +159,22 @@ pub const File = struct { } } + pub fn destroy(base: *File) void { + switch (base.tag) { + .Elf => { + const parent = @fieldParentPtr(Elf, "base", base); + parent.deinit(); + parent.allocator.destroy(parent); + }, + .C => { + const parent = @fieldParentPtr(C, "base", base); + parent.deinit(); + parent.allocator.destroy(parent); + }, + else => unreachable, + } + } + pub fn flush(base: *File) !void { try switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).flush(), From 9aaffe00d3450d2fabd5fbb6a099b3cdc56d04d2 Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Tue, 7 Jul 2020 23:59:55 -0400 Subject: [PATCH 21/29] CBE: Cleanup unimplementeds --- src-self-hosted/cgen.zig | 42 +++++++++++++++++++--------------------- src-self-hosted/link.zig | 15 ++++++++++++-- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src-self-hosted/cgen.zig b/src-self-hosted/cgen.zig index 872fa3e830..0de28748c6 100644 --- a/src-self-hosted/cgen.zig +++ b/src-self-hosted/cgen.zig @@ -15,7 +15,7 @@ fn map(name: []const u8) ![]const u8 { return name; } -fn renderType(file: *C, writer: std.ArrayList(u8).Writer, T: Type) !void { +fn renderType(file: *C, writer: std.ArrayList(u8).Writer, T: Type, src: usize) !void { if (T.tag() == .usize) { file.need_stddef = true; try writer.writeAll("size_t"); @@ -26,20 +26,20 @@ fn renderType(file: *C, writer: std.ArrayList(u8).Writer, T: Type) !void { try writer.writeAll("noreturn void"); }, .Void => try writer.writeAll("void"), - else => return error.Unimplemented, + else => |e| return file.fail(src, "TODO implement type {}", .{e}), } } } fn renderFunctionSignature(file: *C, writer: std.ArrayList(u8).Writer, decl: *Decl) !void { const tv = decl.typed_value.most_recent.typed_value; - try renderType(file, writer, tv.ty.fnReturnType()); + try renderType(file, writer, tv.ty.fnReturnType(), decl.src()); const name = try map(mem.spanZ(decl.name)); try writer.print(" {}(", .{name}); if (tv.ty.fnParamLen() == 0) { try writer.writeAll("void)"); } else { - return error.Unimplemented; + return file.fail(decl.src(), "TODO implement parameters", .{}); } } @@ -68,21 +68,21 @@ pub fn generate(file: *C, decl: *Decl) !void { if (arg.cast(ir.Inst.Constant)) |c| { if (c.val.tag() == .int_u64) { try writer.writeAll("register "); - try renderType(file, writer, arg.ty); + try renderType(file, writer, arg.ty, decl.src()); try writer.print(" {}_constant __asm__(\"{}\") = {};\n\t", .{ reg, reg, c.val.toUnsignedInt() }); } else { - return error.Unimplemented; + return file.fail(decl.src(), "TODO inline asm {} args", .{c.val.tag()}); } } else { - return error.Unimplemented; + return file.fail(decl.src(), "TODO non-constant inline asm args", .{}); } } else { - return error.Unimplemented; + return file.fail(decl.src(), "TODO non-explicit inline asm regs", .{}); } } try writer.print("__asm {} (\"{}\"", .{ if (as.is_volatile) @as([]const u8, "volatile") else "", as.asm_source }); if (as.output) |o| { - return error.Unimplemented; + return file.fail(decl.src(), "TODO inline asm output", .{}); } if (as.inputs.len > 0) { if (as.output == null) { @@ -99,10 +99,12 @@ pub fn generate(file: *C, decl: *Decl) !void { if (arg.cast(ir.Inst.Constant)) |c| { try writer.print("\"\"({}_constant)", .{reg}); } else { - return error.Unimplemented; + // This is blocked by the earlier test + unreachable; } } else { - return error.Unimplemented; + // This is blocked by the earlier test + unreachable; } } } @@ -121,20 +123,17 @@ pub fn generate(file: *C, decl: *Decl) !void { } try writer.print("{}();", .{tname}); } else { - std.debug.warn("non-function call target?\n", .{}); - return error.Unimplemented; + return file.fail(decl.src(), "TODO non-function call target?", .{}); } if (call.args.len != 0) { - std.debug.warn("parameters\n", .{}); - return error.Unimplemented; + return file.fail(decl.src(), "TODO function arguments", .{}); } } else { - std.debug.warn("non-constant call inst?\n", .{}); - return error.Unimplemented; + return file.fail(decl.src(), "TODO non-constant call inst?", .{}); } }, - else => { - std.debug.warn("\nTranslating {}\n", .{inst.*}); + else => |e| { + return file.fail(decl.src(), "TODO {}", .{e}); }, } } @@ -151,13 +150,12 @@ pub fn generate(file: *C, decl: *Decl) !void { std.debug.warn("\n\nARRAYTRANS\n", .{}); if (tv.ty.arraySentinel()) |sentinel| {} } else { - return error.Unimplemented; + return file.fail(decl.src(), "TODO non-byte arrays", .{}); } } }, else => |e| { - std.debug.warn("\nTODO implement {}\n", .{e}); - return error.Unimplemented; + return file.fail(decl.src(), "TODO {}", .{e}); }, } } diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index da01282995..0683a741c5 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -227,6 +227,12 @@ pub const File = struct { need_stddef: bool = false, need_stdint: bool = false, need_noreturn: bool = false, + error_msg: *Module.ErrorMsg = undefined, + + pub fn fail(self: *C, src: usize, comptime format: []const u8, args: var) !void { + self.error_msg = try Module.ErrorMsg.create(self.allocator, src, format, args); + return error.CGenFailure; + } pub fn deinit(self: *File.C) void { self.main.deinit(); @@ -237,7 +243,12 @@ pub const File = struct { } pub fn updateDecl(self: *File.C, module: *Module, decl: *Module.Decl) !void { - try cgen.generate(self, decl); + cgen.generate(self, decl) catch |err| { + if (err == error.CGenFailure) { + try module.failed_decls.put(decl, self.error_msg); + } + return err; + }; } pub fn flush(self: *File.C) !void { @@ -1185,7 +1196,7 @@ pub const File = struct { .appended => code_buffer.items, .fail => |em| { decl.analysis = .codegen_failure; - _ = try module.failed_decls.put(decl, em); + try module.failed_decls.put(decl, em); return; }, }; From e2aad33d4eafe482fa145dc971a423307ae83a39 Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Wed, 8 Jul 2020 00:11:41 -0400 Subject: [PATCH 22/29] Stage2: facepalm. --- src-self-hosted/main.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 3d859483a5..518be8c023 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -433,7 +433,10 @@ fn buildOutputType( std.debug.print("-fno-emit-bin not supported yet", .{}); process.exit(1); }, - .yes_default_path => try std.fmt.allocPrint(arena, "{}.c", .{root_name}), + .yes_default_path => if (cbe) + try std.fmt.allocPrint(arena, "{}.c", .{root_name}) + else + try std.zig.binNameAlloc(arena, root_name, target_info.target, output_mode, link_mode), .yes => |p| p, }; From 9d92d62525324453694190528021ab0216af0ed5 Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Wed, 8 Jul 2020 00:33:44 -0400 Subject: [PATCH 23/29] CBE: Only try to use GNU attribute when __GNUC__is set --- src-self-hosted/cbe.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-self-hosted/cbe.h b/src-self-hosted/cbe.h index 85b07eb48a..66e7b8bd3e 100644 --- a/src-self-hosted/cbe.h +++ b/src-self-hosted/cbe.h @@ -1,6 +1,6 @@ #if __STDC_VERSION__ >= 201112L #define noreturn _Noreturn -#elif !__STRICT_ANSI__ +#elif __GNUC__ && !__STRICT_ANSI__ #define noreturn __attribute__ ((noreturn)) #else #define noreturn From 6b4863416611253c68539136ae9e4b83359efe74 Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Wed, 8 Jul 2020 14:05:07 -0400 Subject: [PATCH 24/29] CBE: Emit asm decls for now, but rename to make them valid --- src-self-hosted/Module.zig | 2 +- src-self-hosted/cgen.zig | 27 +++++++++++++++++---------- src-self-hosted/link.zig | 6 ++++++ test/stage2/cbe.zig | 5 +++++ test/stage2/zir.zig | 18 +++++++++--------- 5 files changed, 38 insertions(+), 20 deletions(-) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 9df02dd186..733aac7c71 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -2456,7 +2456,7 @@ fn createAnonymousDecl( ) !*Decl { const name_index = self.getNextAnonNameIndex(); const scope_decl = scope.decl().?; - const name = try std.fmt.allocPrint(self.allocator, "{}${}", .{ scope_decl.name, name_index }); + const name = try std.fmt.allocPrint(self.allocator, "{}__anon_{}", .{ scope_decl.name, name_index }); defer self.allocator.free(name); const name_hash = scope.namespace().fullyQualifiedNameHash(name); const src_hash: std.zig.SrcHash = undefined; diff --git a/src-self-hosted/cgen.zig b/src-self-hosted/cgen.zig index 0de28748c6..75caddc131 100644 --- a/src-self-hosted/cgen.zig +++ b/src-self-hosted/cgen.zig @@ -11,8 +11,8 @@ const mem = std.mem; /// Maps a name from Zig source to C. This will always give the same output for /// any given input. -fn map(name: []const u8) ![]const u8 { - return name; +fn map(allocator: *std.mem.Allocator, name: []const u8) ![]const u8 { + return allocator.dupe(u8, name); } fn renderType(file: *C, writer: std.ArrayList(u8).Writer, T: Type, src: usize) !void { @@ -34,7 +34,8 @@ fn renderType(file: *C, writer: std.ArrayList(u8).Writer, T: Type, src: usize) ! fn renderFunctionSignature(file: *C, writer: std.ArrayList(u8).Writer, decl: *Decl) !void { const tv = decl.typed_value.most_recent.typed_value; try renderType(file, writer, tv.ty.fnReturnType(), decl.src()); - const name = try map(mem.spanZ(decl.name)); + const name = try map(file.allocator, mem.spanZ(decl.name)); + defer file.allocator.free(name); try writer.print(" {}(", .{name}); if (tv.ty.fnParamLen() == 0) { try writer.writeAll("void)"); @@ -143,15 +144,21 @@ pub fn generate(file: *C, decl: *Decl) !void { try writer.writeAll("}\n\n"); }, .Array => { - if (mem.indexOf(u8, mem.span(decl.name), "$") == null) { - // TODO: prevent inline asm constants from being emitted - if (tv.val.cast(Value.Payload.Bytes)) |payload| { - try writer.print("const char *const {} = \"{}\";\n", .{ decl.name, payload.data }); - std.debug.warn("\n\nARRAYTRANS\n", .{}); - if (tv.ty.arraySentinel()) |sentinel| {} + // TODO: prevent inline asm constants from being emitted + const name = try map(file.allocator, mem.span(decl.name)); + defer file.allocator.free(name); + if (tv.val.cast(Value.Payload.Bytes)) |payload| { + if (tv.ty.arraySentinel()) |sentinel| { + if (sentinel.toUnsignedInt() == 0) { + try file.constants.writer().print("const char *const {} = \"{}\";\n", .{ name, payload.data }); + } else { + return file.fail(decl.src(), "TODO byte arrays with non-zero sentinels", .{}); + } } else { - return file.fail(decl.src(), "TODO non-byte arrays", .{}); + return file.fail(decl.src(), "TODO byte arrays without sentinels", .{}); } + } else { + return file.fail(decl.src(), "TODO non-byte arrays", .{}); } }, else => |e| { diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 0683a741c5..2f21dcbfde 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -93,6 +93,7 @@ pub fn openCFile(allocator: *Allocator, file: fs.File, options: Options) !File.C .options = options, .main = std.ArrayList(u8).init(allocator), .header = std.ArrayList(u8).init(allocator), + .constants = std.ArrayList(u8).init(allocator), .called = std.StringHashMap(void).init(allocator), }; } @@ -220,6 +221,7 @@ pub const File = struct { allocator: *Allocator, header: std.ArrayList(u8), + constants: std.ArrayList(u8), main: std.ArrayList(u8), file: ?fs.File, options: Options, @@ -237,6 +239,7 @@ pub const File = struct { pub fn deinit(self: *File.C) void { self.main.deinit(); self.header.deinit(); + self.constants.deinit(); self.called.deinit(); if (self.file) |f| f.close(); @@ -269,6 +272,9 @@ pub const File = struct { if (self.header.items.len > 0) { try writer.print("{}\n", .{self.header.items}); } + if (self.constants.items.len > 0) { + try writer.print("{}\n", .{self.constants.items}); + } if (self.main.items.len > 1) { const last_two = self.main.items[self.main.items.len - 2 ..]; if (std.mem.eql(u8, last_two, "\n\n")) { diff --git a/test/stage2/cbe.zig b/test/stage2/cbe.zig index e56507bd84..5961ab4c0c 100644 --- a/test/stage2/cbe.zig +++ b/test/stage2/cbe.zig @@ -32,6 +32,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ ); // TODO: implement return values + // TODO: figure out a way to prevent asm constants from being generated ctx.c("inline asm", linux_x64, \\fn exitGood() void { \\ asm volatile ("syscall" @@ -49,6 +50,10 @@ pub fn addCases(ctx: *TestContext) !void { \\ \\void exitGood(void); \\ + \\const char *const exitGood__anon_0 = "{rax}"; + \\const char *const exitGood__anon_1 = "{rdi}"; + \\const char *const exitGood__anon_2 = "syscall"; + \\ \\noreturn void _start(void) { \\ exitGood(); \\} diff --git a/test/stage2/zir.zig b/test/stage2/zir.zig index 052ada667e..1a0aed85cf 100644 --- a/test/stage2/zir.zig +++ b/test/stage2/zir.zig @@ -22,8 +22,8 @@ pub fn addCases(ctx: *TestContext) !void { , \\@void = primitive(void) \\@fnty = fntype([], @void, cc=C) - \\@9 = declref("9$0") - \\@9$0 = str("entry") + \\@9 = declref("9__anon_0") + \\@9__anon_0 = str("entry") \\@unnamed$4 = str("entry") \\@unnamed$5 = export(@unnamed$4, "entry") \\@unnamed$6 = fntype([], @void, cc=C) @@ -77,9 +77,9 @@ pub fn addCases(ctx: *TestContext) !void { \\@entry = fn(@unnamed$6, { \\ %0 = returnvoid() \\}) - \\@entry$1 = str("2\x08\x01\n") - \\@9 = declref("9$0") - \\@9$0 = str("entry") + \\@entry__anon_1 = str("2\x08\x01\n") + \\@9 = declref("9__anon_0") + \\@9__anon_0 = str("entry") \\@unnamed$11 = str("entry") \\@unnamed$12 = export(@unnamed$11, "entry") \\ @@ -111,8 +111,8 @@ pub fn addCases(ctx: *TestContext) !void { , \\@void = primitive(void) \\@fnty = fntype([], @void, cc=C) - \\@9 = declref("9$0") - \\@9$0 = str("entry") + \\@9 = declref("9__anon_0") + \\@9__anon_0 = str("entry") \\@unnamed$4 = str("entry") \\@unnamed$5 = export(@unnamed$4, "entry") \\@unnamed$6 = fntype([], @void, cc=C) @@ -187,8 +187,8 @@ pub fn addCases(ctx: *TestContext) !void { , \\@void = primitive(void) \\@fnty = fntype([], @void, cc=C) - \\@9 = declref("9$2") - \\@9$2 = str("entry") + \\@9 = declref("9__anon_2") + \\@9__anon_2 = str("entry") \\@unnamed$4 = str("entry") \\@unnamed$5 = export(@unnamed$4, "entry") \\@unnamed$6 = fntype([], @void, cc=C) From d060be880460598dcf89cb3d59bfcdf5059a923d Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Wed, 8 Jul 2020 14:10:11 -0400 Subject: [PATCH 25/29] CBE: Don't expose openCFile, always close file after an update --- src-self-hosted/link.zig | 4 +++- src-self-hosted/test.zig | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 2f21dcbfde..98a2578822 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -86,7 +86,7 @@ pub fn writeFilePath( return result; } -pub fn openCFile(allocator: *Allocator, file: fs.File, options: Options) !File.C { +fn openCFile(allocator: *Allocator, file: fs.File, options: Options) !File.C { return File.C{ .allocator = allocator, .file = file, @@ -282,6 +282,8 @@ pub const File = struct { } } try writer.writeAll(self.main.items); + self.file.?.close(); + self.file = null; } }; diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 0a32008369..3e4958cc95 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -480,9 +480,8 @@ pub const TestContext = struct { switch (update.case) { .Transformation => |expected_output| { if (case.cbe) { - var cfile: *link.File.C = module.bin_file.cast(link.File.C).?; - cfile.file.?.close(); - cfile.file = null; + // The C file is always closed after an update, because we don't support + // incremental updates var file = try tmp.dir.openFile(bin_name, .{ .read = true }); defer file.close(); var out = file.reader().readAllAlloc(allocator, 1024 * 1024) catch @panic("Unable to read C output!"); From eeae3a8f9df49e53392bcd7b04b463f6bffcec9c Mon Sep 17 00:00:00 2001 From: Paul Espinosa Date: Wed, 8 Jul 2020 11:42:02 +0700 Subject: [PATCH 26/29] Rename langref's Index to Contents (TOC) The language reference's Index is a list of the documentation's contents in order of appearance. This commit renames "Index" to "Contents" as in table of contents. It also renames the HTML/CSS identifiers from "index" to "toc". --- doc/langref.html.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index d7b9de3c6e..1e4d993fe9 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -97,7 +97,7 @@ margin: auto; } - #index { + #toc { padding: 0 1em; } @@ -105,7 +105,7 @@ #main-wrapper { flex-direction: row; } - #contents-wrapper, #index { + #contents-wrapper, #toc { overflow: auto; } } @@ -181,7 +181,7 @@
-
+
0.1.1 | 0.2.0 | 0.3.0 | @@ -189,7 +189,7 @@ 0.5.0 | 0.6.0 | master -

Index

+

Contents

{#nav#}
From f77c968cf841046238eb37f54600cda201f09ce2 Mon Sep 17 00:00:00 2001 From: Ryan Liptak Date: Wed, 8 Jul 2020 00:53:19 -0700 Subject: [PATCH 27/29] langref: Add test case for "if error union with optional" This is an edge case that isn't too uncommon but is rather confusing to try to deduce without documentation, since it feels like `else` is being overloaded in this scenario and there's no obvious 'correct' behavior here. This just adds a test demonstrating how Zig currently behaves in this scenario. --- doc/langref.html.in | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/doc/langref.html.in b/doc/langref.html.in index 1e4d993fe9..032e05d600 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -3861,6 +3861,32 @@ test "if error union" { unreachable; } } + +test "if error union with optional" { + // If expressions test for errors before unwrapping optionals. + // The |optional_value| capture's type is ?u32. + + const a: anyerror!?u32 = 0; + if (a) |optional_value| { + assert(optional_value.? == 0); + } else |err| { + unreachable; + } + + const b: anyerror!?u32 = null; + if (b) |optional_value| { + assert(optional_value == null); + } else |err| { + unreachable; + } + + const c: anyerror!?u32 = error.BadValue; + if (c) |optional_value| { + unreachable; + } else |err| { + assert(err == error.BadValue); + } +} {#code_end#} {#see_also|Optionals|Errors#} {#header_close#} From 2064e84cdd596b5bc63c8d3e7e5aa4130221faee Mon Sep 17 00:00:00 2001 From: xackus <14938807+xackus@users.noreply.github.com> Date: Tue, 7 Jul 2020 21:20:33 +0200 Subject: [PATCH 28/29] ci: check langref.html for html errors --- ci/azure/linux_script | 6 +++++- doc/langref.html.in | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/azure/linux_script b/ci/azure/linux_script index b313a76161..a55de24496 100755 --- a/ci/azure/linux_script +++ b/ci/azure/linux_script @@ -12,7 +12,7 @@ sudo apt-get update -q sudo apt-get remove -y llvm-* sudo rm -rf /usr/local/* -sudo apt-get install -y libxml2-dev libclang-10-dev llvm-10 llvm-10-dev liblld-10-dev cmake s3cmd gcc-7 g++-7 ninja-build +sudo apt-get install -y libxml2-dev libclang-10-dev llvm-10 llvm-10-dev liblld-10-dev cmake s3cmd gcc-7 g++-7 ninja-build tidy QEMUBASE="qemu-linux-x86_64-5.0.0-49ee115552" wget https://ziglang.org/deps/$QEMUBASE.tar.xz @@ -51,6 +51,10 @@ cd build cmake .. -DCMAKE_BUILD_TYPE=Release -GNinja ninja install ./zig build test -Denable-qemu -Denable-wasmtime + +# look for HTML errors +tidy -qe ../zig-cache/langref.html + VERSION="$(./zig version)" if [ "${BUILD_REASON}" != "PullRequest" ]; then diff --git a/doc/langref.html.in b/doc/langref.html.in index 032e05d600..78dcd537b8 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -8419,6 +8419,7 @@ fn foo(comptime T: type, ptr: *T) T { {#header_close#} {#header_open|Opaque Types#} +

{#syntax#}@Type(.Opaque){#endsyntax#} creates a new type with an unknown (but non-zero) size and alignment.

From 12a7dedb1f1ce34b16993d33aa97d7b78d5d5ca2 Mon Sep 17 00:00:00 2001 From: Ryan Liptak Date: Wed, 8 Jul 2020 13:40:16 -0700 Subject: [PATCH 29/29] langref: Expand "if error union with optional" test case Follow-up to #5818, closes #5819 --- doc/langref.html.in | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/langref.html.in b/doc/langref.html.in index 78dcd537b8..f64170817f 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -3886,6 +3886,22 @@ test "if error union with optional" { } else |err| { assert(err == error.BadValue); } + + // Access the value by reference by using a pointer capture each time. + var d: anyerror!?u32 = 3; + if (d) |*optional_value| { + if (optional_value.*) |*value| { + value.* = 9; + } + } else |err| { + unreachable; + } + + if (d) |optional_value| { + assert(optional_value.? == 9); + } else |err| { + unreachable; + } } {#code_end#} {#see_also|Optionals|Errors#}