From ba6e5cbfd279583dcc724f6a74a5593fa26ad5df Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 31 Jul 2020 23:22:35 -0700 Subject: [PATCH 01/21] stage2: add the .debug_line header and associated data types * the .debug_line header is written properly * link.File.Elf gains: - SrcFn, which is now a field in Module.Fn - SrcFile, which is now a field in Module.Scope.File * link.File.Elf gets a whole *Package field rather than only root_src_dir_path. * the fields first_dbg_line_file and last_dbg_line_file tell where the Line Number Program begins and ends, which alows moving files when the header gets too big, and allows appending files to the end. * codegen is passed a buffer for emitting .debug_line Line Number Program opcodes for functions. See #5963 There is some work-in-progress code here, but I need to go make some experimental changes to changing how to represent source locations and I want to do that in a separate commit. --- lib/std/zig/ast.zig | 8 +- src-self-hosted/Module.zig | 9 +- src-self-hosted/codegen.zig | 3 +- src-self-hosted/link.zig | 296 ++++++++++++++++++++++++++++++++---- 4 files changed, 284 insertions(+), 32 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 02d1fd6ba2..6b2b8b4cf2 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -1299,6 +1299,10 @@ pub const Node = struct { }); } + pub fn body(self: *const FnProto) ?*Node { + return self.getTrailer("body_node"); + } + pub fn getTrailer(self: *const FnProto, comptime name: []const u8) ?TrailerFlags.Field(name) { const trailers_start = @alignCast( @alignOf(ParamDecl), @@ -1381,7 +1385,7 @@ pub const Node = struct { .Invalid => {}, } - if (self.getTrailer("body_node")) |body_node| { + if (self.body()) |body_node| { if (i < 1) return body_node; i -= 1; } @@ -1397,7 +1401,7 @@ pub const Node = struct { } pub fn lastToken(self: *const FnProto) TokenIndex { - if (self.getTrailer("body_node")) |body_node| return body_node.lastToken(); + if (self.body()) |body_node| return body_node.lastToken(); switch (self.return_type) { .Explicit, .InferErrorSet => |node| return node.lastToken(), .Invalid => |tok| return tok, diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 06dc77938e..d9560dc425 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -279,6 +279,9 @@ pub const Fn = struct { }, owner_decl: *Decl, + /// Represents the function in the linked output file. + link: link.File.Elf.SrcFn = link.File.Elf.SrcFn.empty, + /// This memory is temporary and points to stack memory for the duration /// of Fn analysis. pub const Analysis = struct { @@ -503,6 +506,10 @@ pub const Scope = struct { /// Direct children of the file. decls: ArrayListUnmanaged(*Decl), + /// Represents the file in the linker code. The linker code + /// uses this field to store data relevant to its purposes. + link: link.File.Elf.SrcFile = link.File.Elf.SrcFile.empty, + pub fn unload(self: *File, gpa: *Allocator) void { switch (self.status) { .never_loaded, @@ -792,7 +799,7 @@ pub fn init(gpa: *Allocator, options: InitOptions) !Module { const bin_file_dir = options.bin_file_dir orelse std.fs.cwd(); const bin_file = try link.File.openPath(gpa, bin_file_dir, options.bin_file_path, .{ .root_name = root_name, - .root_src_dir_path = options.root_pkg.root_src_dir_path, + .root_pkg = options.root_pkg, .target = options.target, .output_mode = options.output_mode, .link_mode = options.link_mode orelse .Static, diff --git a/src-self-hosted/codegen.zig b/src-self-hosted/codegen.zig index 2ea255bf7f..0114f66329 100644 --- a/src-self-hosted/codegen.zig +++ b/src-self-hosted/codegen.zig @@ -44,6 +44,7 @@ pub fn generateSymbol( src: usize, typed_value: TypedValue, code: *std.ArrayList(u8), + dbg_line: *std.ArrayList(u8), ) GenerateSymbolError!Result { const tracy = trace(@src()); defer tracy.end(); @@ -114,7 +115,7 @@ pub fn generateSymbol( switch (try generateSymbol(bin_file, src, .{ .ty = typed_value.ty.elemType(), .val = sentinel, - }, code)) { + }, code, dbg_line)) { .appended => return Result{ .appended = {} }, .externally_managed => |slice| { code.appendSliceAssumeCapacity(slice); diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 533cccbd42..af18b12097 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -11,6 +11,9 @@ const c_codegen = @import("codegen/c.zig"); const log = std.log; const DW = std.dwarf; const trace = @import("tracy.zig").trace; +const leb128 = std.debug.leb; +const Package = @import("Package.zig"); +const Value = @import("value.zig").Value; // TODO Turn back on zig fmt when https://github.com/ziglang/zig/issues/5948 is implemented. // zig fmt: off @@ -24,7 +27,7 @@ pub const Options = struct { object_format: std.builtin.ObjectFormat, optimize_mode: std.builtin.Mode, root_name: []const u8, - root_src_dir_path: []const u8, + root_pkg: *const Package, /// Used for calculating how much space to reserve for symbols in case the binary file /// does not already have a symbol table. symbol_count_hint: u64 = 32, @@ -291,6 +294,7 @@ pub const File = struct { debug_abbrev_section_index: ?u16 = null, debug_str_section_index: ?u16 = null, debug_aranges_section_index: ?u16 = null, + debug_line_section_index: ?u16 = null, debug_abbrev_table_offset: ?u64 = null, @@ -318,6 +322,7 @@ pub const File = struct { debug_info_section_dirty: bool = false, debug_abbrev_section_dirty: bool = false, debug_aranges_section_dirty: bool = false, + debug_line_header_dirty: bool = false, error_flags: ErrorFlags = ErrorFlags{}, @@ -339,6 +344,9 @@ pub const File = struct { text_block_free_list: std.ArrayListUnmanaged(*TextBlock) = std.ArrayListUnmanaged(*TextBlock){}, last_text_block: ?*TextBlock = null, + first_dbg_line_file: ?*SrcFile = null, + last_dbg_line_file: ?*SrcFile = null, + /// `alloc_num / alloc_den` is the factor of padding when allocating. const alloc_num = 4; const alloc_den = 3; @@ -402,6 +410,45 @@ pub const File = struct { sym_index: ?u32 = null, }; + pub const SrcFn = struct { + /// Offset from the `SrcFile` that contains this function. + dbg_line_off: u32, + /// Size of the line number program component belonging to this function, not + /// including padding. + dbg_line_len: u32, + + pub const empty: SrcFn = .{ + .dbg_line_off = 0, + .dbg_line_len = 0, + }; + }; + + pub const SrcFile = struct { + /// Byte offset from the start of the Line Number Program that contains this file. + off: u32, + /// Length in bytes, not including padding, of this file component within the + /// Line Number Program that contains it. + len: u32, + + /// A list of `SrcFn` that have surplus capacity. + /// This is the same concept as `text_block_free_list` (see the doc comments there) + /// but it's for the function's component of the Line Number program. + free_list: std.ArrayListUnmanaged(*SrcFn), + + /// Points to the previous and next neighbors, based on the offset from .debug_line. + /// This can be used to find, for example, the capacity of this `SrcFile`. + prev: ?*SrcFile, + next: ?*SrcFile, + + pub const empty: SrcFile = .{ + .off = 0, + .len = 0, + .free_list = .{}, + .prev = null, + .next = null, + }; + }; + pub fn openPath(allocator: *Allocator, dir: fs.Dir, sub_path: []const u8, options: Options) !*File { assert(options.object_format == .elf); @@ -538,6 +585,14 @@ pub const File = struct { }); } + fn getDebugLineProgramOff(self: Elf) u32 { + return self.first_dbg_line_file.?.off; + } + + fn getDebugLineProgramLen(self: Elf) u32 { + return self.last_dbg_line_file.?.off + self.last_dbg_line_file.?.len; + } + /// Returns end pos of collision, if any. fn detectAllocCollision(self: *Elf, start: u64, size: u64) ?u64 { const small_ptr = self.base.options.target.cpu.arch.ptrBitWidth() == 32; @@ -611,6 +666,7 @@ pub const File = struct { return start; } + /// TODO Improve this to use a table. fn makeString(self: *Elf, bytes: []const u8) !u32 { try self.shstrtab.ensureCapacity(self.allocator, self.shstrtab.items.len + bytes.len + 1); const result = self.shstrtab.items.len; @@ -619,6 +675,7 @@ pub const File = struct { return @intCast(u32, result); } + /// TODO Improve this to use a table. fn makeDebugString(self: *Elf, bytes: []const u8) !u32 { try self.debug_strtab.ensureCapacity(self.allocator, self.debug_strtab.items.len + bytes.len + 1); const result = self.debug_strtab.items.len; @@ -645,10 +702,7 @@ pub const File = struct { .p32 => true, .p64 => false, }; - const ptr_size: u8 = switch (self.ptr_width) { - .p32 => 4, - .p64 => 8, - }; + const ptr_size: u8 = self.ptrWidthBytes(); if (self.phdr_load_re_index == null) { self.phdr_load_re_index = @intCast(u16, self.program_headers.items.len); const file_size = self.base.options.program_code_size_hint; @@ -869,6 +923,31 @@ pub const File = struct { self.shdr_table_dirty = true; self.debug_aranges_section_dirty = true; } + if (self.debug_line_section_index == null) { + self.debug_line_section_index = @intCast(u16, self.sections.items.len); + + const file_size_hint = 250; + const p_align = 1; + const off = self.findFreeSpace(file_size_hint, p_align); + log.debug(.link, "found .debug_line free space 0x{x} to 0x{x}\n", .{ + off, + off + file_size_hint, + }); + try self.sections.append(self.allocator, .{ + .sh_name = try self.makeString(".debug_line"), + .sh_type = elf.SHT_PROGBITS, + .sh_flags = 0, + .sh_addr = 0, + .sh_offset = off, + .sh_size = file_size_hint, + .sh_link = 0, + .sh_info = 0, + .sh_addralign = p_align, + .sh_entsize = 0, + }); + self.shdr_table_dirty = true; + self.debug_line_header_dirty = true; + } const shsize: u64 = switch (self.ptr_width) { .p32 => @sizeOf(elf.Elf32_Shdr), .p64 => @sizeOf(elf.Elf64_Shdr), @@ -906,9 +985,10 @@ pub const File = struct { pub fn flush(self: *Elf) !void { const target_endian = self.base.options.target.cpu.arch.endian(); const foreign_endian = target_endian != std.Target.current.cpu.arch.endian(); - const ptr_width_bytes: u8 = switch (self.ptr_width) { + const ptr_width_bytes: u8 = self.ptrWidthBytes(); + const init_len_size: usize = switch (self.ptr_width) { .p32 => 4, - .p64 => 8, + .p64 => 12, }; // Unfortunately these have to be buffered and done at the end because ELF does not allow @@ -922,7 +1002,7 @@ pub const File = struct { // we can simply append these bytes. const abbrev_buf = [_]u8{ 1, DW.TAG_compile_unit, DW.CHILDREN_no, // header - //DW.AT_stmt_list, DW.FORM_data4, TODO + DW.AT_stmt_list, DW.FORM_data1, DW.AT_low_pc , DW.FORM_addr, DW.AT_high_pc , DW.FORM_addr, DW.AT_name , DW.FORM_strp, @@ -969,10 +1049,7 @@ pub const File = struct { // not including the initial length itself. // We have to come back and write it later after we know the size. const init_len_index = di_buf.items.len; - switch (self.ptr_width) { - .p32 => di_buf.items.len += 4, - .p64 => di_buf.items.len += 12, - } + di_buf.items.len += init_len_size; const after_init_len = di_buf.items.len; mem.writeInt(u16, di_buf.addManyAsArrayAssumeCapacity(2), 5, target_endian); // DWARF version di_buf.appendAssumeCapacity(DW.UT_compile); @@ -989,7 +1066,7 @@ pub const File = struct { } // Write the form for the compile unit, which must match the abbrev table above. const name_strp = try self.makeDebugString(self.base.options.root_name); - const comp_dir_strp = try self.makeDebugString(self.base.options.root_src_dir_path); + const comp_dir_strp = try self.makeDebugString(self.base.options.root_pkg.root_src_dir_path); const producer_strp = try self.makeDebugString("zig (TODO version here)"); // Currently only one compilation unit is supported, so the address range is simply // identical to the main program header virtual address and memory size. @@ -997,8 +1074,10 @@ pub const File = struct { const low_pc = text_phdr.p_vaddr; const high_pc = text_phdr.p_vaddr + text_phdr.p_memsz; - di_buf.appendAssumeCapacity(1); // abbrev tag, matching the value from the abbrev table header - //DW.AT_stmt_list, DW.FORM_data4, TODO line information + di_buf.appendSliceAssumeCapacity(&[_]u8{ + 1, // abbrev tag, matching the value from the abbrev table header + 0, // DW.AT_stmt_list, DW.FORM_data1: offset to corresponding .debug_line header + }); self.writeDwarfAddrAssumeCapacity(&di_buf, low_pc); self.writeDwarfAddrAssumeCapacity(&di_buf, high_pc); self.writeDwarfAddrAssumeCapacity(&di_buf, name_strp); @@ -1056,10 +1135,7 @@ pub const File = struct { // not including the initial length itself. // We have to come back and write it later after we know the size. const init_len_index = di_buf.items.len; - switch (self.ptr_width) { - .p32 => di_buf.items.len += 4, - .p64 => di_buf.items.len += 12, - } + di_buf.items.len += init_len_size; const after_init_len = di_buf.items.len; mem.writeInt(u16, di_buf.addManyAsArrayAssumeCapacity(2), 2, target_endian); // version // When more than one compilation unit is supported, this will be the offset to it. @@ -1116,6 +1192,99 @@ pub const File = struct { self.debug_aranges_section_dirty = false; } + if (self.debug_line_header_dirty) { + const dbg_line_prg_off = self.getDebugLineProgramOff(); + const dbg_line_prg_len = self.getDebugLineProgramLen(); + assert(dbg_line_prg_len != 0); + + const debug_line_sect = &self.sections.items[self.debug_line_section_index.?]; + + var di_buf = std.ArrayList(u8).init(self.allocator); + defer di_buf.deinit(); + + // This is a heuristic. The size of this header is variable, depending on + // the number of directories, files, and padding. + try di_buf.ensureCapacity(100); + + // initial length - length of the .debug_line contribution for this compilation unit, + // not including the initial length itself. + const after_init_len = di_buf.items.len + init_len_size; + const init_len = (dbg_line_prg_off + dbg_line_prg_len) - after_init_len; + switch (self.ptr_width) { + .p32 => { + mem.writeInt(u32, di_buf.addManyAsArrayAssumeCapacity(4), @intCast(u32, init_len), target_endian); + }, + .p64 => { + di_buf.appendNTimesAssumeCapacity(0xff, 4); + mem.writeInt(u64, di_buf.addManyAsArrayAssumeCapacity(8), init_len, target_endian); + }, + } + + mem.writeInt(u16, di_buf.addManyAsArrayAssumeCapacity(2), 5, target_endian); // version + di_buf.appendSliceAssumeCapacity(&[_]u8{ + ptr_width_bytes, // address_size + 0, // segment_selector_size + }); + + const header_length = dbg_line_prg_off - (di_buf.items.len + ptr_width_bytes); + self.writeDwarfAddrAssumeCapacity(&di_buf, header_length); + + const opcode_base = DW.LNS_set_isa + 1; + di_buf.appendSliceAssumeCapacity(&[_]u8{ + 1, // minimum_instruction_length + 1, // maximum_operations_per_instruction + 1, // default_is_stmt + 1, // line_base (signed) + 1, // line_range + opcode_base, + + // Standard opcode lengths. The number of items here is based on `opcode_base`. + // The value is the number of LEB128 operands the instruction takes. + 0, // `DW.LNS_copy` + 1, // `DW.LNS_advance_pc` + 1, // `DW.LNS_advance_line` + 1, // `DW.LNS_set_file` + 1, // `DW.LNS_set_column` + 0, // `DW.LNS_negate_stmt` + 0, // `DW.LNS_set_basic_block` + 0, // `DW.LNS_const_add_pc` + 0, // `DW.LNS_fixed_advance_pc` + 0, // `DW.LNS_set_prologue_end` + 0, // `DW.LNS_set_epilogue_begin` + 1, // `DW.LNS_set_isa` + + 1, // directory_entry_format_count + DW.LNCT_path, DW.FORM_strp, // directory_entry_format + + // For now we only support one compilation unit, which has one directory. + 1, // directories_count (this is a ULEB128) + }); + const comp_dir_strp = try self.makeDebugString(self.base.options.root_pkg.root_src_dir_path); + self.writeDwarfAddrAssumeCapacity(&di_buf, comp_dir_strp); + + di_buf.appendSliceAssumeCapacity(&[_]u8{ + 2, // file_name_entry_format_count + DW.LNCT_path, DW.FORM_strp, // file_name_entry_format[0] + DW.LNCT_directory_index, DW.FORM_data1, // file_name_entry_format[1] + // TODO Look into adding the file size here. Maybe even the mtime and MD5. + //DW.LNCT_size, DW.FORM_udata, // file_name_entry_format[2] + + // For now we only put the root file name here. Once more source files + // are supported, this will need to be improved. + 1, // file_names_count (this is a ULEB128) + }); + const root_src_file_strp = try self.makeDebugString(self.base.options.root_pkg.root_src_path); + self.writeDwarfAddrAssumeCapacity(&di_buf, root_src_file_strp); // DW.LNCT_path, DW.FORM_strp + di_buf.appendAssumeCapacity(0); // LNCT_directory_index, FORM_data1 + + if (di_buf.items.len > dbg_line_prg_off) { + // Move the first N files to the end to make more padding for the header. + @panic("TODO: handle .debug_line header exceeding its padding"); + } + + try self.file.?.pwriteAll(di_buf.items, debug_line_sect.sh_offset); + self.debug_line_header_dirty = false; + } if (self.phdr_table_dirty) { const phsize: u64 = switch (self.ptr_width) { @@ -1263,6 +1432,7 @@ pub const File = struct { assert(!self.debug_info_section_dirty); assert(!self.debug_abbrev_section_dirty); assert(!self.debug_aranges_section_dirty); + assert(!self.debug_line_header_dirty); assert(!self.phdr_table_dirty); assert(!self.shdr_table_dirty); assert(!self.shstrtab_dirty); @@ -1635,8 +1805,52 @@ pub const File = struct { var code_buffer = std.ArrayList(u8).init(self.allocator); defer code_buffer.deinit(); + var dbg_line_buffer = std.ArrayList(u8).init(self.allocator); + defer dbg_line_buffer.deinit(); + const typed_value = decl.typed_value.most_recent.typed_value; - const code = switch (try codegen.generateSymbol(self, decl.src(), typed_value, &code_buffer)) { + const is_fn: bool = switch (typed_value.ty.zigTypeTag()) { + .Fn => true, + else => false, + }; + const dbg_line_vaddr_reloc_index = 1; + if (is_fn) { + const scope_file = decl.scope.cast(Module.Scope.File).?; + const line_off: u28 = blk: { + const file_ast_decls = scope_file.contents.tree.root_node.decls(); + if (decl.src_index == 0) { + // Then it's the line number of the open curly. + const block = file_ast_decls[decl.src_index].castTag(.Block).?; + @panic("TODO implement this"); + } else { + const prev_decl = file_ast_decls[decl.src_index - 1]; + // Find the difference between prev decl end curly and this decl begin curly. + @panic("TODO implement this"); + } + }; + + // For functions we need to add a prologue to the debug line program. + try dbg_line_buffer.ensureCapacity(24); + + dbg_line_buffer.appendAssumeCapacity(DW.LNE_set_address); + // This is the "relocatable" vaddr, corresponding to `code_buffer` index `0`. + assert(dbg_line_vaddr_reloc_index == dbg_line_buffer.items.len); + dbg_line_buffer.items.len += self.ptrWidthBytes(); + + dbg_line_buffer.appendAssumeCapacity(DW.LNS_advance_line); + // This is the "relocatable" relative line offset from the previous function's end curly + // to this function's begin curly. + assert(self.getRelocDbgLineOff() == dbg_line_buffer.items.len); + // Here we use a ULEB128 but we write 4 bytes regardless (possibly wasting space) + // so that we can patch this later as a fixed width field. + leb128.writeUnsignedFixed(4, dbg_line_buffer.addManyAsArrayAssumeCapacity(4), line_off); + + // Emit a line for the begin curly with prologue_end=false. The codegen will + // do the work of setting prologue_end=true and epilogue_begin=true. + dbg_line_buffer.appendAssumeCapacity(DW.LNS_copy); + } + const res = try codegen.generateSymbol(self, decl.src(), typed_value, &code_buffer, &dbg_line_buffer); + const code = switch (res) { .externally_managed => |x| x, .appended => code_buffer.items, .fail => |em| { @@ -1648,10 +1862,7 @@ pub const File = struct { const required_alignment = typed_value.ty.abiAlignment(self.base.options.target); - const stt_bits: u8 = switch (typed_value.ty.zigTypeTag()) { - .Fn => elf.STT_FUNC, - else => elf.STT_OBJECT, - }; + const stt_bits: u8 = if (is_fn) elf.STT_FUNC else elf.STT_OBJECT; assert(decl.link.local_sym_index != 0); // Caller forgot to allocateDeclIndexes() const local_sym = &self.local_symbols.items[decl.link.local_sym_index]; @@ -1704,6 +1915,30 @@ pub const File = struct { const file_offset = self.sections.items[self.text_section_index.?].sh_offset + section_offset; try self.file.?.pwriteAll(code, file_offset); + // If the Decl is a function, we need to update the .debug_line program. + if (is_fn) { + // Perform the relocation based on vaddr. + const target_endian = self.base.options.target.cpu.arch.endian(); + switch (self.ptr_width) { + .p32 => { + const ptr = dbg_line_buffer.items[dbg_line_vaddr_reloc_index..][0..4]; + mem.writeInt(u32, ptr, @intCast(u32, local_sym.st_value), target_endian); + }, + .p64 => { + const ptr = dbg_line_buffer.items[dbg_line_vaddr_reloc_index..][0..8]; + mem.writeInt(u64, ptr, local_sym.st_value, target_endian); + }, + } + + const src_file = &decl.scope.cast(Module.Scope.File).?.link; + const src_fn = &typed_value.val.cast(Value.Payload.Function).?.func.link; + if (src_file.next == null and src_file.prev == null) { + @panic("TODO updateDecl for .debug_line: add new SrcFile"); + } else { + @panic("TODO updateDecl for .debug_line: update existing SrcFile"); + } + } + // Since we updated the vaddr and the size, each corresponding export symbol also needs to be updated. const decl_exports = module.decl_exports.get(decl) orelse &[0]*Module.Export{}; return self.updateDeclExports(module, decl, decl_exports); @@ -1841,10 +2076,7 @@ pub const File = struct { fn writeOffsetTableEntry(self: *Elf, index: usize) !void { const shdr = &self.sections.items[self.got_section_index.?]; const phdr = &self.program_headers.items[self.phdr_got_index.?]; - const entry_size: u16 = switch (self.ptr_width) { - .p32 => 4, - .p64 => 8, - }; + const entry_size: u16 = self.ptrWidthBytes(); if (self.offset_table_count_dirty) { // TODO Also detect virtual address collisions. const allocated_size = self.allocatedSize(shdr.sh_offset); @@ -1987,6 +2219,14 @@ pub const File = struct { }, } } + + fn ptrWidthBytes(self: Elf) u8 { + return switch (self.ptr_width) { + .p32 => 4, + .p64 => 8, + }; + } + }; }; From 1a3f250f195d4ed5455795d4fa6e4b0cf97cb6ce Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 2 Aug 2020 19:21:01 -0700 Subject: [PATCH 02/21] .debug_line incremental compilation initial support Supports writing the first function. Still TODO is: * handling the .debug_line header growing too large * adding a new file to an existing compilation * adding an additional function to an existing file * handling incremental updates * adding the main IR debug ops for IR instructions There are also issues to work out: * readelf --debug-dump=rawline is saying there is no .debug_str section even though there is * readelf --debug-dump=decodedline is saying the file index 0 is bad and reporting some other kind of corruption. --- lib/std/hash_map.zig | 18 ++- src-self-hosted/link.zig | 277 +++++++++++++++++++++++++++++---------- 2 files changed, 223 insertions(+), 72 deletions(-) diff --git a/lib/std/hash_map.zig b/lib/std/hash_map.zig index c81def3a00..d52a337422 100644 --- a/lib/std/hash_map.zig +++ b/lib/std/hash_map.zig @@ -196,6 +196,10 @@ pub fn HashMap( return self.unmanaged.getEntry(key); } + pub fn getIndex(self: Self, key: K) ?usize { + return self.unmanaged.getIndex(key); + } + pub fn get(self: Self, key: K) ?V { return self.unmanaged.get(key); } @@ -479,17 +483,21 @@ pub fn HashMapUnmanaged( } pub fn getEntry(self: Self, key: K) ?*Entry { + const index = self.getIndex(key) orelse return null; + return &self.entries.items[index]; + } + + pub fn getIndex(self: Self, key: K) ?usize { const header = self.index_header orelse { // Linear scan. const h = if (store_hash) hash(key) else {}; - for (self.entries.items) |*item| { + for (self.entries.items) |*item, i| { if (item.hash == h and eql(key, item.key)) { - return item; + return i; } } return null; }; - switch (header.capacityIndexType()) { .u8 => return self.getInternal(key, header, u8), .u16 => return self.getInternal(key, header, u16), @@ -711,7 +719,7 @@ pub fn HashMapUnmanaged( unreachable; } - fn getInternal(self: Self, key: K, header: *IndexHeader, comptime I: type) ?*Entry { + fn getInternal(self: Self, key: K, header: *IndexHeader, comptime I: type) ?usize { const indexes = header.indexes(I); const h = hash(key); const start_index = header.constrainIndex(h); @@ -725,7 +733,7 @@ pub fn HashMapUnmanaged( const entry = &self.entries.items[index.entry_index]; const hash_match = if (store_hash) h == entry.hash else true; if (hash_match and eql(key, entry.key)) - return entry; + return index.entry_index; } return null; } diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index af18b12097..ca12e59ee6 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -412,14 +412,14 @@ pub const File = struct { pub const SrcFn = struct { /// Offset from the `SrcFile` that contains this function. - dbg_line_off: u32, + off: u32, /// Size of the line number program component belonging to this function, not /// including padding. - dbg_line_len: u32, + len: u32, pub const empty: SrcFn = .{ - .dbg_line_off = 0, - .dbg_line_len = 0, + .off = 0, + .len = 0, }; }; @@ -430,10 +430,17 @@ pub const File = struct { /// Line Number Program that contains it. len: u32, - /// A list of `SrcFn` that have surplus capacity. - /// This is the same concept as `text_block_free_list` (see the doc comments there) - /// but it's for the function's component of the Line Number program. - free_list: std.ArrayListUnmanaged(*SrcFn), + /// An ordered list of all the `SrcFn` in this file. This list is not redundant with + /// the source Decl list, for two reasons: + /// * Lazy decl analysis: some source functions do not correspond to any compiled functions. + /// * Generic functions: some source functions correspond to many compiled functions. + /// This list corresponds to the file data in the Line Number Program. When a new `SrcFn` + /// is inserted, the list must be shifted to accomodate it, and likewise the Line + /// Number Program data must be shifted within the ELF file to accomodate (if there is + /// not enough padding). + /// It is a hash map so that we can look up the index based on the `*SrcFn` and therefore + /// find the next and previous functions. + fns: std.AutoHashMapUnmanaged(*SrcFn, void), /// Points to the previous and next neighbors, based on the offset from .debug_line. /// This can be used to find, for example, the capacity of this `SrcFile`. @@ -443,7 +450,7 @@ pub const File = struct { pub const empty: SrcFile = .{ .off = 0, .len = 0, - .free_list = .{}, + .fns = .{}, .prev = null, .next = null, }; @@ -589,7 +596,7 @@ pub const File = struct { return self.first_dbg_line_file.?.off; } - fn getDebugLineProgramLen(self: Elf) u32 { + fn getDebugLineProgramEnd(self: Elf) u32 { return self.last_dbg_line_file.?.off + self.last_dbg_line_file.?.len; } @@ -767,27 +774,6 @@ pub const File = struct { self.shstrtab_dirty = true; self.shdr_table_dirty = true; } - if (self.debug_str_section_index == null) { - self.debug_str_section_index = @intCast(u16, self.sections.items.len); - assert(self.debug_strtab.items.len == 0); - try self.debug_strtab.append(self.allocator, 0); // need a 0 at position 0 - const off = self.findFreeSpace(self.debug_strtab.items.len, 1); - log.debug(.link, "found debug_strtab free space 0x{x} to 0x{x}\n", .{ off, off + self.debug_strtab.items.len }); - try self.sections.append(self.allocator, .{ - .sh_name = try self.makeString(".debug_str"), - .sh_type = elf.SHT_PROGBITS, - .sh_flags = elf.SHF_MERGE | elf.SHF_STRINGS, - .sh_addr = 0, - .sh_offset = off, - .sh_size = self.debug_strtab.items.len, - .sh_link = 0, - .sh_info = 0, - .sh_addralign = 1, - .sh_entsize = 1, - }); - self.debug_strtab_dirty = true; - self.shdr_table_dirty = true; - } if (self.text_section_index == null) { self.text_section_index = @intCast(u16, self.sections.items.len); const phdr = &self.program_headers.items[self.phdr_load_re_index.?]; @@ -848,6 +834,24 @@ pub const File = struct { self.shdr_table_dirty = true; try self.writeSymbol(0); } + if (self.debug_str_section_index == null) { + self.debug_str_section_index = @intCast(u16, self.sections.items.len); + assert(self.debug_strtab.items.len == 0); + try self.sections.append(self.allocator, .{ + .sh_name = try self.makeString(".debug_str"), + .sh_type = elf.SHT_PROGBITS, + .sh_flags = elf.SHF_MERGE | elf.SHF_STRINGS, + .sh_addr = 0, + .sh_offset = 0, + .sh_size = self.debug_strtab.items.len, + .sh_link = 0, + .sh_info = 0, + .sh_addralign = 1, + .sh_entsize = 1, + }); + self.debug_strtab_dirty = true; + self.shdr_table_dirty = true; + } if (self.debug_info_section_index == null) { self.debug_info_section_index = @intCast(u16, self.sections.items.len); @@ -1002,7 +1006,7 @@ pub const File = struct { // we can simply append these bytes. const abbrev_buf = [_]u8{ 1, DW.TAG_compile_unit, DW.CHILDREN_no, // header - DW.AT_stmt_list, DW.FORM_data1, + DW.AT_stmt_list, DW.FORM_sec_offset, DW.AT_low_pc , DW.FORM_addr, DW.AT_high_pc , DW.FORM_addr, DW.AT_name , DW.FORM_strp, @@ -1074,10 +1078,8 @@ pub const File = struct { const low_pc = text_phdr.p_vaddr; const high_pc = text_phdr.p_vaddr + text_phdr.p_memsz; - di_buf.appendSliceAssumeCapacity(&[_]u8{ - 1, // abbrev tag, matching the value from the abbrev table header - 0, // DW.AT_stmt_list, DW.FORM_data1: offset to corresponding .debug_line header - }); + di_buf.appendAssumeCapacity(1); // abbrev tag, matching the value from the abbrev table header + self.writeDwarfAddrAssumeCapacity(&di_buf, 0); // DW.AT_stmt_list, DW.FORM_sec_offset self.writeDwarfAddrAssumeCapacity(&di_buf, low_pc); self.writeDwarfAddrAssumeCapacity(&di_buf, high_pc); self.writeDwarfAddrAssumeCapacity(&di_buf, name_strp); @@ -1194,22 +1196,23 @@ pub const File = struct { } if (self.debug_line_header_dirty) { const dbg_line_prg_off = self.getDebugLineProgramOff(); - const dbg_line_prg_len = self.getDebugLineProgramLen(); - assert(dbg_line_prg_len != 0); + const dbg_line_prg_end = self.getDebugLineProgramEnd(); + assert(dbg_line_prg_end != 0); const debug_line_sect = &self.sections.items[self.debug_line_section_index.?]; var di_buf = std.ArrayList(u8).init(self.allocator); defer di_buf.deinit(); - // This is a heuristic. The size of this header is variable, depending on - // the number of directories, files, and padding. - try di_buf.ensureCapacity(100); + // The size of this header is variable, depending on the number of directories, + // files, and padding. We have a function to compute the upper bound size, however, + // because it's needed for determining where to put the offset of the first `SrcFile`. + try di_buf.ensureCapacity(self.dbgLineNeededHeaderBytes()); // initial length - length of the .debug_line contribution for this compilation unit, // not including the initial length itself. const after_init_len = di_buf.items.len + init_len_size; - const init_len = (dbg_line_prg_off + dbg_line_prg_len) - after_init_len; + const init_len = dbg_line_prg_end - after_init_len; switch (self.ptr_width) { .p32 => { mem.writeInt(u32, di_buf.addManyAsArrayAssumeCapacity(4), @intCast(u32, init_len), target_endian); @@ -1277,10 +1280,16 @@ pub const File = struct { self.writeDwarfAddrAssumeCapacity(&di_buf, root_src_file_strp); // DW.LNCT_path, DW.FORM_strp di_buf.appendAssumeCapacity(0); // LNCT_directory_index, FORM_data1 - if (di_buf.items.len > dbg_line_prg_off) { + // Add a redundant NOP in case the consumer ignores header_length. + const after_jmp = di_buf.items.len + 6; + if (after_jmp > dbg_line_prg_off) { // Move the first N files to the end to make more padding for the header. @panic("TODO: handle .debug_line header exceeding its padding"); } + const jmp_amt = dbg_line_prg_off - after_jmp + 1; + di_buf.appendAssumeCapacity(DW.LNS_extended_op); + leb128.writeUnsignedFixed(4, di_buf.addManyAsArrayAssumeCapacity(4), @intCast(u28, jmp_amt)); + di_buf.appendAssumeCapacity(DW.LNE_hi_user); try self.file.?.pwriteAll(di_buf.items, debug_line_sect.sh_offset); self.debug_line_header_dirty = false; @@ -1813,37 +1822,27 @@ pub const File = struct { .Fn => true, else => false, }; - const dbg_line_vaddr_reloc_index = 1; if (is_fn) { - const scope_file = decl.scope.cast(Module.Scope.File).?; - const line_off: u28 = blk: { - const file_ast_decls = scope_file.contents.tree.root_node.decls(); - if (decl.src_index == 0) { - // Then it's the line number of the open curly. - const block = file_ast_decls[decl.src_index].castTag(.Block).?; - @panic("TODO implement this"); - } else { - const prev_decl = file_ast_decls[decl.src_index - 1]; - // Find the difference between prev decl end curly and this decl begin curly. - @panic("TODO implement this"); - } - }; - // For functions we need to add a prologue to the debug line program. - try dbg_line_buffer.ensureCapacity(24); + try dbg_line_buffer.ensureCapacity(26); - dbg_line_buffer.appendAssumeCapacity(DW.LNE_set_address); + const ptr_width_bytes = self.ptrWidthBytes(); + dbg_line_buffer.appendSliceAssumeCapacity(&[_]u8{ + DW.LNS_extended_op, + ptr_width_bytes + 1, + DW.LNE_set_address, + }); // This is the "relocatable" vaddr, corresponding to `code_buffer` index `0`. assert(dbg_line_vaddr_reloc_index == dbg_line_buffer.items.len); - dbg_line_buffer.items.len += self.ptrWidthBytes(); + dbg_line_buffer.items.len += ptr_width_bytes; dbg_line_buffer.appendAssumeCapacity(DW.LNS_advance_line); // This is the "relocatable" relative line offset from the previous function's end curly // to this function's begin curly. assert(self.getRelocDbgLineOff() == dbg_line_buffer.items.len); - // Here we use a ULEB128 but we write 4 bytes regardless (possibly wasting space) - // so that we can patch this later as a fixed width field. - leb128.writeUnsignedFixed(4, dbg_line_buffer.addManyAsArrayAssumeCapacity(4), line_off); + // Here we allocate 4 bytes for the relocation. This field is a ULEB128, however, + // it is possible to encode small values as still taking up 4 bytes. + dbg_line_buffer.items.len += 4; // Emit a line for the begin curly with prologue_end=false. The codegen will // do the work of setting prologue_end=true and epilogue_begin=true. @@ -1917,6 +1916,14 @@ pub const File = struct { // If the Decl is a function, we need to update the .debug_line program. if (is_fn) { + // For padding between functions, we terminate with `LNS_extended_op` with sub-op + // `LNE_hi_user`, using a fixed 4-byte ULEB128 for the opcode size. This is always + // found at the very end of the SrcFile's Line Number Program component. + try dbg_line_buffer.ensureCapacity(dbg_line_buffer.items.len + 6); + dbg_line_buffer.appendAssumeCapacity(DW.LNS_extended_op); + leb128.writeUnsignedFixed(4, dbg_line_buffer.addManyAsArrayAssumeCapacity(4), 1); + dbg_line_buffer.appendAssumeCapacity(DW.LNE_hi_user); + // Perform the relocation based on vaddr. const target_endian = self.base.options.target.cpu.arch.endian(); switch (self.ptr_width) { @@ -1930,13 +1937,91 @@ pub const File = struct { }, } - const src_file = &decl.scope.cast(Module.Scope.File).?.link; + // Now we want to write the line offset relocation, however, first we must + // "plug in" the SrcFn into its parent SrcFile, so that we know what function the line + // number is offset from. It must go in the same order as the functions are found + // in the Zig source. When we insert a function before another one, the latter one + // must have its line offset relocation updated. + + const debug_line_sect = &self.sections.items[self.debug_line_section_index.?]; + const scope_file = decl.scope.cast(Module.Scope.File).?; + const src_file = &scope_file.link; const src_fn = &typed_value.val.cast(Value.Payload.Function).?.func.link; - if (src_file.next == null and src_file.prev == null) { - @panic("TODO updateDecl for .debug_line: add new SrcFile"); + var src_fn_index: usize = undefined; + if (src_file.len == 0) { + // This is the first function of the SrcFile. + assert(src_file.fns.entries.items.len == 0); + src_fn_index = 0; + try src_file.fns.put(self.allocator, src_fn, {}); + + if (self.last_dbg_line_file) |last| { + src_file.prev = last; + self.last_dbg_line_file = src_file; + + // Update the previous last SrcFile's terminating NOP to skip to the start + // of the new last SrcFile's start. + @panic("TODO updateDecl for .debug_line: add new SrcFile: append"); + } else { + // This is the first file (and function) of the Line Number Program. + self.first_dbg_line_file = src_file; + self.last_dbg_line_file = src_file; + + src_fn.off = dbg_line_file_header_len; + src_fn.len = @intCast(u32, dbg_line_buffer.items.len); + + src_file.off = self.dbgLineNeededHeaderBytes() * alloc_num / alloc_den; + src_file.len = src_fn.off + src_fn.len + dbg_line_file_trailer_len; + + const needed_size = src_file.off + src_file.len; + if (needed_size > debug_line_sect.sh_size) { + debug_line_sect.sh_offset = self.findFreeSpace(needed_size, 1); + } + debug_line_sect.sh_size = needed_size; + self.shdr_table_dirty = true; // TODO look into making only the one section dirty + self.debug_line_header_dirty = true; + + try self.updateDbgLineFile(src_file); + } } else { @panic("TODO updateDecl for .debug_line: update existing SrcFile"); + //src_fn_index = @panic("TODO"); } + const line_off: u28 = blk: { + const tree = scope_file.contents.tree; + const file_ast_decls = tree.root_node.decls(); + // TODO Look into improving the performance here by adding a token-index-to-line + // lookup table. Currently this involves scanning over the source code for newlines + // (but only from the previous decl to the current one). + if (src_fn_index == 0) { + // Since it's the first function in the file, the line number delta is just the + // line number of the open curly from the beginning of the file. + const fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?; + const block = fn_proto.body().?.castTag(.Block).?; + const loc = tree.tokenLocation(0, block.lbrace); + // No need to add one; this is a delta from DWARF's starting line number (1). + break :blk @intCast(u28, loc.line); + } else { + const prev_src_fn = src_file.fns.entries.items[src_fn_index - 1].key; + const mod_fn = @fieldParentPtr(Module.Fn, "link", prev_src_fn); + const prev_fn_proto = file_ast_decls[mod_fn.owner_decl.src_index].castTag(.FnProto).?; + const this_fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?; + const prev_block = prev_fn_proto.body().?.castTag(.Block).?; + const this_block = this_fn_proto.body().?.castTag(.Block).?; + // Find the difference between prev decl end curly and this decl begin curly. + const loc = tree.tokenLocation(tree.token_locs[prev_block.rbrace].start, this_block.lbrace); + // No need to add one; this is a delta from the previous line number. + break :blk @intCast(u28, loc.line); + } + }; + + // Here we use a ULEB128 but we write 4 bytes regardless (possibly wasting space) because + // that is the amount of space we allocated for this field. + leb128.writeUnsignedFixed(4, dbg_line_buffer.items[self.getRelocDbgLineOff()..][0..4], line_off); + + // We only have support for one compilation unit so far, so the offsets are directly + // from the .debug_line section. + const file_pos = debug_line_sect.sh_offset + src_file.off + src_fn.off; + try self.file.?.pwriteAll(dbg_line_buffer.items, file_pos); } // Since we updated the vaddr and the size, each corresponding export symbol also needs to be updated. @@ -2028,6 +2113,46 @@ pub const File = struct { self.global_symbols.items[sym_index].st_info = 0; } + const dbg_line_file_header_len = 5; // DW.LNS_set_file + ULEB128-fixed-4 file_index + const dbg_line_file_trailer_len = 9; // DW.LNE_end_sequence + 6-byte terminating NOP + + fn updateDbgLineFile(self: *Elf, src_file: *SrcFile) !void { + const target_endian = self.base.options.target.cpu.arch.endian(); + const shdr = &self.sections.items[self.debug_line_section_index.?]; + const header_off = shdr.sh_offset + src_file.off; + { + var header: [dbg_line_file_header_len]u8 = undefined; + header[0] = DW.LNS_set_file; + // Once we support more than one source file, this will have the ability to be non-zero. + const file_index = 0; + leb128.writeUnsignedFixed(4, header[1..5], file_index); + try self.file.?.pwriteAll(&header, header_off); + } + { + const last_src_fn = src_file.fns.entries.items[src_file.fns.entries.items.len - 1].key; + const trailer_off = header_off + last_src_fn.off + last_src_fn.len; + const padding_to_next = blk: { + if (src_file.next) |next| { + break :blk next.off - (src_file.off + src_file.len); + } else { + // No need for padding after this one; we will add padding to it when a SrcFile + // is added after it. + break :blk 0; + } + }; + var trailer: [dbg_line_file_trailer_len]u8 = undefined; + + trailer[0] = DW.LNS_extended_op; + trailer[1] = 1; + trailer[2] = DW.LNE_end_sequence; + + trailer[3] = DW.LNS_extended_op; + leb128.writeUnsignedFixed(4, trailer[4..8], @intCast(u28, padding_to_next + 1)); + trailer[8] = DW.LNE_hi_user; + try self.file.?.pwriteAll(&trailer, trailer_off); + } + } + fn writeProgHeader(self: *Elf, index: usize) !void { const foreign_endian = self.base.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); const offset = self.program_headers.items[index].p_offset; @@ -2227,6 +2352,24 @@ pub const File = struct { }; } + /// The reloc offset for the virtual address of a function in its Line Number Program. + /// Size is a virtual address integer. + const dbg_line_vaddr_reloc_index = 3; + + /// The reloc offset for the line offset of a function from the previous function's line. + /// It's a fixed-size 4-byte ULEB128. + fn getRelocDbgLineOff(self: Elf) usize { + return dbg_line_vaddr_reloc_index + self.ptrWidthBytes() + 1; + } + + fn dbgLineNeededHeaderBytes(self: Elf) u32 { + const directory_entry_format_count = 1; + const file_name_entry_format_count = 1; + const directory_count = 1; + const file_name_count = 1; + return 53 + directory_entry_format_count * 2 + file_name_entry_format_count * 2 + + directory_count * 8 + file_name_count * 8; + } }; }; From 1ce6e201aa818b05f6637438d1c0b460ea817e3f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 2 Aug 2020 19:57:42 -0700 Subject: [PATCH 03/21] .debug_line: don't rely on header_length field Empirically, debug info consumers do not respect this field, or otherwise consider it to be an error when it does not point exactly to the end of the header. Therefore we rely on the NOP jump at the beginning of the Line Number Program for padding rather than this field. llvm-dwarfdump says the line number data is fine; gdb and binutils-readelf crap out. --- src-self-hosted/link.zig | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index ca12e59ee6..96a15cadee 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -647,6 +647,8 @@ pub const File = struct { } fn allocatedSize(self: *Elf, start: u64) u64 { + if (start == 0) + return 0; var min_pos: u64 = std.math.maxInt(u64); if (self.shdr_table_offset) |off| { if (off > start and off < min_pos) min_pos = off; @@ -1229,8 +1231,13 @@ pub const File = struct { 0, // segment_selector_size }); - const header_length = dbg_line_prg_off - (di_buf.items.len + ptr_width_bytes); - self.writeDwarfAddrAssumeCapacity(&di_buf, header_length); + // Empirically, debug info consumers do not respect this field, or otherwise + // consider it to be an error when it does not point exactly to the end of the header. + // Therefore we rely on the NOP jump at the beginning of the Line Number Program for + // padding rather than this field. + const before_header_len = di_buf.items.len; + di_buf.items.len += ptr_width_bytes; // We will come back and write this. + const after_header_len = di_buf.items.len; const opcode_base = DW.LNS_set_isa + 1; di_buf.appendSliceAssumeCapacity(&[_]u8{ @@ -1280,7 +1287,17 @@ pub const File = struct { self.writeDwarfAddrAssumeCapacity(&di_buf, root_src_file_strp); // DW.LNCT_path, DW.FORM_strp di_buf.appendAssumeCapacity(0); // LNCT_directory_index, FORM_data1 - // Add a redundant NOP in case the consumer ignores header_length. + const header_len = di_buf.items.len - after_header_len; + switch (self.ptr_width) { + .p32 => { + mem.writeInt(u32, di_buf.items[before_header_len..][0..4], @intCast(u32, header_len), target_endian); + }, + .p64 => { + mem.writeInt(u64, di_buf.items[before_header_len..][0..8], header_len, target_endian); + }, + } + + // We use a NOP jmp because consumers empirically do not respect the header length field. const after_jmp = di_buf.items.len + 6; if (after_jmp > dbg_line_prg_off) { // Move the first N files to the end to make more padding for the header. From 42d331b58aa874420d2515071e94f046adaf0689 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 2 Aug 2020 20:22:41 -0700 Subject: [PATCH 04/21] .debug_line: avoid DW_FORM_strp to work around readelf/gdb These tools do not support DWARFv5 yet apparently. --- src-self-hosted/link.zig | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 96a15cadee..284b881028 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1264,27 +1264,37 @@ pub const File = struct { 1, // `DW.LNS_set_isa` 1, // directory_entry_format_count - DW.LNCT_path, DW.FORM_strp, // directory_entry_format + DW.LNCT_path, DW.FORM_string, // directory_entry_format // For now we only support one compilation unit, which has one directory. 1, // directories_count (this is a ULEB128) }); - const comp_dir_strp = try self.makeDebugString(self.base.options.root_pkg.root_src_dir_path); - self.writeDwarfAddrAssumeCapacity(&di_buf, comp_dir_strp); + // Empirically, some tools do not understand DW.FORM_strp yet. readelf 2.31.1 gives the bogus + // error and gdb 8.3.1 crashes. Both programs seem to work fine with + // DW.FORM_string however. + di_buf.appendSliceAssumeCapacity(self.base.options.root_pkg.root_src_dir_path); + di_buf.appendAssumeCapacity(0); di_buf.appendSliceAssumeCapacity(&[_]u8{ 2, // file_name_entry_format_count - DW.LNCT_path, DW.FORM_strp, // file_name_entry_format[0] + DW.LNCT_path, DW.FORM_string, // file_name_entry_format[0] DW.LNCT_directory_index, DW.FORM_data1, // file_name_entry_format[1] // TODO Look into adding the file size here. Maybe even the mtime and MD5. //DW.LNCT_size, DW.FORM_udata, // file_name_entry_format[2] // For now we only put the root file name here. Once more source files // are supported, this will need to be improved. - 1, // file_names_count (this is a ULEB128) + 2, // file_names_count (this is a ULEB128) }); - const root_src_file_strp = try self.makeDebugString(self.base.options.root_pkg.root_src_path); - self.writeDwarfAddrAssumeCapacity(&di_buf, root_src_file_strp); // DW.LNCT_path, DW.FORM_strp + // See note above with directories about why we use DW.FORM_string here. + di_buf.appendSliceAssumeCapacity(self.base.options.root_pkg.root_src_path); + di_buf.appendAssumeCapacity(0); + di_buf.appendAssumeCapacity(0); // LNCT_directory_index, FORM_data1 + + // We add the root file twice because according to DWARF, the state machine + // starts out with file index 1. + di_buf.appendSliceAssumeCapacity(self.base.options.root_pkg.root_src_path); + di_buf.appendAssumeCapacity(0); di_buf.appendAssumeCapacity(0); // LNCT_directory_index, FORM_data1 const header_len = di_buf.items.len - after_header_len; @@ -2140,8 +2150,9 @@ pub const File = struct { { var header: [dbg_line_file_header_len]u8 = undefined; header[0] = DW.LNS_set_file; - // Once we support more than one source file, this will have the ability to be non-zero. - const file_index = 0; + // Once we support more than one source file, this will have the ability to be more + // than one possible value. + const file_index = 1; leb128.writeUnsignedFixed(4, header[1..5], file_index); try self.file.?.pwriteAll(&header, header_off); } @@ -2384,8 +2395,13 @@ pub const File = struct { const file_name_entry_format_count = 1; const directory_count = 1; const file_name_count = 1; - return 53 + directory_entry_format_count * 2 + file_name_entry_format_count * 2 + - directory_count * 8 + file_name_count * 8; + return @intCast(u32, 53 + directory_entry_format_count * 2 + file_name_entry_format_count * 2 + + directory_count * 8 + file_name_count * 8 + + // These are encoded as DW.FORM_string rather than DW.FORM_strp as we would like + // because of a workaround for readelf and gdb failing to understand DWARFv5 correctly. + self.base.options.root_pkg.root_src_dir_path.len + + self.base.options.root_pkg.root_src_path.len * 2); + } }; }; From 659603c6211e5628db0ab0dd9b1e8454ed1b69c6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 2 Aug 2020 21:28:06 -0700 Subject: [PATCH 05/21] codegen: emit .debug_line ops for IR instructions --- lib/std/zig.zig | 9 ++ src-self-hosted/astgen.zig | 3 +- src-self-hosted/codegen.zig | 189 ++++++++++++++++++++++++----------- src-self-hosted/ir.zig | 2 + src-self-hosted/link.zig | 11 +- src-self-hosted/zir.zig | 5 + src-self-hosted/zir_sema.zig | 6 ++ 7 files changed, 159 insertions(+), 66 deletions(-) diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 841827cc19..057b79f11c 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -43,6 +43,15 @@ pub fn findLineColumn(source: []const u8, byte_offset: usize) struct { line: usi return .{ .line = line, .column = column }; } +pub fn lineDelta(source: []const u8, start: usize, end: usize) usize { + var line: usize = 0; + for (source[start..end]) |byte| switch (byte) { + '\n' => line += 1, + else => continue, + }; + return line; +} + /// Returns the standard file system basename of a binary generated by the Zig compiler. pub fn binNameAlloc( allocator: *std.mem.Allocator, diff --git a/src-self-hosted/astgen.zig b/src-self-hosted/astgen.zig index 3f08d1272d..029e7d4b8c 100644 --- a/src-self-hosted/astgen.zig +++ b/src-self-hosted/astgen.zig @@ -120,6 +120,8 @@ pub fn blockExpr(mod: *Module, parent_scope: *Scope, block_node: *ast.Node.Block var scope = parent_scope; for (block_node.statements()) |statement| { + const src = scope.tree().token_locs[statement.firstToken()].start; + _ = try addZIRNoOp(mod, scope, src, .dbg_stmt); switch (statement.tag) { .VarDecl => { const var_decl_node = statement.castTag(.VarDecl).?; @@ -146,7 +148,6 @@ pub fn blockExpr(mod: *Module, parent_scope: *Scope, block_node: *ast.Node.Block else => { const possibly_unused_result = try expr(mod, scope, .none, statement); if (!possibly_unused_result.tag.isNoReturn()) { - const src = scope.tree().token_locs[statement.firstToken()].start; _ = try addZIRUnOp(mod, scope, src, .ensure_result_used, possibly_unused_result); } }, diff --git a/src-self-hosted/codegen.zig b/src-self-hosted/codegen.zig index 0114f66329..1c441b9179 100644 --- a/src-self-hosted/codegen.zig +++ b/src-self-hosted/codegen.zig @@ -12,6 +12,11 @@ const ErrorMsg = Module.ErrorMsg; const Target = std.Target; const Allocator = mem.Allocator; const trace = @import("tracy.zig").trace; +const DW = std.dwarf; +const leb128 = std.debug.leb; + +// TODO Turn back on zig fmt when https://github.com/ziglang/zig/issues/5948 is implemented. +// zig fmt: off /// The codegen-related data that is stored in `ir.Inst.Block` instructions. pub const BlockData = struct { @@ -52,57 +57,57 @@ pub fn generateSymbol( switch (typed_value.ty.zigTypeTag()) { .Fn => { switch (bin_file.base.options.target.cpu.arch) { - //.arm => return Function(.arm).generateSymbol(bin_file, src, typed_value, code), - //.armeb => return Function(.armeb).generateSymbol(bin_file, src, typed_value, code), - //.aarch64 => return Function(.aarch64).generateSymbol(bin_file, src, typed_value, code), - //.aarch64_be => return Function(.aarch64_be).generateSymbol(bin_file, src, typed_value, code), - //.aarch64_32 => return Function(.aarch64_32).generateSymbol(bin_file, src, typed_value, code), - //.arc => return Function(.arc).generateSymbol(bin_file, src, typed_value, code), - //.avr => return Function(.avr).generateSymbol(bin_file, src, typed_value, code), - //.bpfel => return Function(.bpfel).generateSymbol(bin_file, src, typed_value, code), - //.bpfeb => return Function(.bpfeb).generateSymbol(bin_file, src, typed_value, code), - //.hexagon => return Function(.hexagon).generateSymbol(bin_file, src, typed_value, code), - //.mips => return Function(.mips).generateSymbol(bin_file, src, typed_value, code), - //.mipsel => return Function(.mipsel).generateSymbol(bin_file, src, typed_value, code), - //.mips64 => return Function(.mips64).generateSymbol(bin_file, src, typed_value, code), - //.mips64el => return Function(.mips64el).generateSymbol(bin_file, src, typed_value, code), - //.msp430 => return Function(.msp430).generateSymbol(bin_file, src, typed_value, code), - //.powerpc => return Function(.powerpc).generateSymbol(bin_file, src, typed_value, code), - //.powerpc64 => return Function(.powerpc64).generateSymbol(bin_file, src, typed_value, code), - //.powerpc64le => return Function(.powerpc64le).generateSymbol(bin_file, src, typed_value, code), - //.r600 => return Function(.r600).generateSymbol(bin_file, src, typed_value, code), - //.amdgcn => return Function(.amdgcn).generateSymbol(bin_file, src, typed_value, code), - //.riscv32 => return Function(.riscv32).generateSymbol(bin_file, src, typed_value, code), - //.riscv64 => return Function(.riscv64).generateSymbol(bin_file, src, typed_value, code), - //.sparc => return Function(.sparc).generateSymbol(bin_file, src, typed_value, code), - //.sparcv9 => return Function(.sparcv9).generateSymbol(bin_file, src, typed_value, code), - //.sparcel => return Function(.sparcel).generateSymbol(bin_file, src, typed_value, code), - //.s390x => return Function(.s390x).generateSymbol(bin_file, src, typed_value, code), - //.tce => return Function(.tce).generateSymbol(bin_file, src, typed_value, code), - //.tcele => return Function(.tcele).generateSymbol(bin_file, src, typed_value, code), - //.thumb => return Function(.thumb).generateSymbol(bin_file, src, typed_value, code), - //.thumbeb => return Function(.thumbeb).generateSymbol(bin_file, src, typed_value, code), - //.i386 => return Function(.i386).generateSymbol(bin_file, src, typed_value, code), - .x86_64 => return Function(.x86_64).generateSymbol(bin_file, src, typed_value, code), - //.xcore => return Function(.xcore).generateSymbol(bin_file, src, typed_value, code), - //.nvptx => return Function(.nvptx).generateSymbol(bin_file, src, typed_value, code), - //.nvptx64 => return Function(.nvptx64).generateSymbol(bin_file, src, typed_value, code), - //.le32 => return Function(.le32).generateSymbol(bin_file, src, typed_value, code), - //.le64 => return Function(.le64).generateSymbol(bin_file, src, typed_value, code), - //.amdil => return Function(.amdil).generateSymbol(bin_file, src, typed_value, code), - //.amdil64 => return Function(.amdil64).generateSymbol(bin_file, src, typed_value, code), - //.hsail => return Function(.hsail).generateSymbol(bin_file, src, typed_value, code), - //.hsail64 => return Function(.hsail64).generateSymbol(bin_file, src, typed_value, code), - //.spir => return Function(.spir).generateSymbol(bin_file, src, typed_value, code), - //.spir64 => return Function(.spir64).generateSymbol(bin_file, src, typed_value, code), - //.kalimba => return Function(.kalimba).generateSymbol(bin_file, src, typed_value, code), - //.shave => return Function(.shave).generateSymbol(bin_file, src, typed_value, code), - //.lanai => return Function(.lanai).generateSymbol(bin_file, src, typed_value, code), - //.wasm32 => return Function(.wasm32).generateSymbol(bin_file, src, typed_value, code), - //.wasm64 => return Function(.wasm64).generateSymbol(bin_file, src, typed_value, code), - //.renderscript32 => return Function(.renderscript32).generateSymbol(bin_file, src, typed_value, code), - //.renderscript64 => return Function(.renderscript64).generateSymbol(bin_file, src, typed_value, code), - //.ve => return Function(.ve).generateSymbol(bin_file, src, typed_value, code), + //.arm => return Function(.arm).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.armeb => return Function(.armeb).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.aarch64 => return Function(.aarch64).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.aarch64_be => return Function(.aarch64_be).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.aarch64_32 => return Function(.aarch64_32).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.arc => return Function(.arc).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.avr => return Function(.avr).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.bpfel => return Function(.bpfel).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.bpfeb => return Function(.bpfeb).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.hexagon => return Function(.hexagon).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.mips => return Function(.mips).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.mipsel => return Function(.mipsel).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.mips64 => return Function(.mips64).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.mips64el => return Function(.mips64el).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.msp430 => return Function(.msp430).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.powerpc => return Function(.powerpc).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.powerpc64 => return Function(.powerpc64).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.powerpc64le => return Function(.powerpc64le).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.r600 => return Function(.r600).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.amdgcn => return Function(.amdgcn).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.riscv32 => return Function(.riscv32).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.riscv64 => return Function(.riscv64).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.sparc => return Function(.sparc).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.sparcv9 => return Function(.sparcv9).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.sparcel => return Function(.sparcel).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.s390x => return Function(.s390x).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.tce => return Function(.tce).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.tcele => return Function(.tcele).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.thumb => return Function(.thumb).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.thumbeb => return Function(.thumbeb).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.i386 => return Function(.i386).generateSymbol(bin_file, src, typed_value, code, dbg_line), + .x86_64 => return Function(.x86_64).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.xcore => return Function(.xcore).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.nvptx => return Function(.nvptx).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.nvptx64 => return Function(.nvptx64).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.le32 => return Function(.le32).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.le64 => return Function(.le64).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.amdil => return Function(.amdil).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.amdil64 => return Function(.amdil64).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.hsail => return Function(.hsail).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.hsail64 => return Function(.hsail64).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.spir => return Function(.spir).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.spir64 => return Function(.spir64).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.kalimba => return Function(.kalimba).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.shave => return Function(.shave).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.lanai => return Function(.lanai).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.wasm32 => return Function(.wasm32).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.wasm64 => return Function(.wasm64).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.renderscript32 => return Function(.renderscript32).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.renderscript64 => return Function(.renderscript64).generateSymbol(bin_file, src, typed_value, code, dbg_line), + //.ve => return Function(.ve).generateSymbol(bin_file, src, typed_value, code, dbg_line), else => @panic("Backend architectures that don't have good support yet are commented out, to improve compilation performance. If you are interested in one of these other backends feel free to uncomment them. Eventually these will be completed, but stage1 is slow and a memory hog."), } }, @@ -207,6 +212,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { target: *const std.Target, mod_fn: *const Module.Fn, code: *std.ArrayList(u8), + dbg_line: *std.ArrayList(u8), err_msg: ?*ErrorMsg, args: []MCValue, ret_mcv: MCValue, @@ -215,6 +221,15 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { src: usize, stack_align: u32, + /// Byte offset within the source file. + prev_di_src: usize, + /// Relative to the beginning of `code`. + prev_di_pc: usize, + /// Used to find newlines and count line deltas. + source: []const u8, + /// Byte offset within the source file of the ending curly. + rbrace_src: usize, + /// The value is an offset into the `Function` `code` from the beginning. /// To perform the reloc, write 32-bit signed little-endian integer /// which is a relative jump, based on the address following the reloc. @@ -366,6 +381,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { src: usize, typed_value: TypedValue, code: *std.ArrayList(u8), + dbg_line: *std.ArrayList(u8), ) GenerateSymbolError!Result { const module_fn = typed_value.val.cast(Value.Payload.Function).?.func; @@ -380,12 +396,20 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { const branch = try branch_stack.addOne(); branch.* = .{}; + const scope_file = module_fn.owner_decl.scope.cast(Module.Scope.File).?; + const tree = scope_file.contents.tree; + const fn_proto = tree.root_node.decls()[module_fn.owner_decl.src_index].castTag(.FnProto).?; + const block = fn_proto.body().?.castTag(.Block).?; + const lbrace_src = tree.token_locs[block.lbrace].start; + const rbrace_src = tree.token_locs[block.rbrace].start; + var function = Self{ .gpa = bin_file.allocator, .target = &bin_file.base.options.target, .bin_file = bin_file, .mod_fn = module_fn, .code = code, + .dbg_line = dbg_line, .err_msg = null, .args = undefined, // populated after `resolveCallingConventionValues` .ret_mcv = undefined, // populated after `resolveCallingConventionValues` @@ -394,6 +418,10 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .branch_stack = &branch_stack, .src = src, .stack_align = undefined, + .prev_di_pc = 0, + .prev_di_src = lbrace_src, + .rbrace_src = rbrace_src, + .source = tree.source, }; defer function.exitlude_jump_relocs.deinit(bin_file.allocator); @@ -432,21 +460,16 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { // TODO During semantic analysis, check if there are no function calls. If there // are none, here we can omit the part where we subtract and then add rsp. self.code.appendSliceAssumeCapacity(&[_]u8{ - // push rbp - 0x55, - // mov rbp, rsp - 0x48, - 0x89, - 0xe5, - // sub rsp, imm32 (with reloc) - 0x48, - 0x81, - 0xec, + 0x55, // push rbp + 0x48, 0x89, 0xe5, // mov rbp, rsp + 0x48, 0x81, 0xec, // sub rsp, imm32 (with reloc) }); const reloc_index = self.code.items.len; self.code.items.len += 4; + try self.dbgSetPrologueEnd(); try self.genBody(self.mod_fn.analysis.success); + try self.dbgSetEpilogueBegin(); const stack_end = self.branch_stack.items[0].max_end_stack; if (stack_end > math.maxInt(i32)) @@ -486,13 +509,19 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { 0xc3, // ret }); } else { + try self.dbgSetPrologueEnd(); try self.genBody(self.mod_fn.analysis.success); + try self.dbgSetEpilogueBegin(); } }, else => { + try self.dbgSetPrologueEnd(); try self.genBody(self.mod_fn.analysis.success); + try self.dbgSetEpilogueBegin(); }, } + // Drop them off at the rbrace. + try self.dbgAdvancePCAndLine(self.rbrace_src); } fn genBody(self: *Self, body: ir.Body) InnerError!void { @@ -509,6 +538,38 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } + fn dbgSetPrologueEnd(self: *Self) InnerError!void { + try self.dbg_line.append(DW.LNS_set_prologue_end); + try self.dbgAdvancePCAndLine(self.prev_di_src); + } + + fn dbgSetEpilogueBegin(self: *Self) InnerError!void { + try self.dbg_line.append(DW.LNS_set_epilogue_begin); + try self.dbgAdvancePCAndLine(self.prev_di_src); + } + + fn dbgAdvancePCAndLine(self: *Self, src: usize) InnerError!void { + // TODO Look into improving the performance here by adding a token-index-to-line + // lookup table, and changing ir.Inst from storing byte offset to token. Currently + // this involves scanning over the source code for newlines + // (but only from the previous byte offset to the new one). + const delta_line = std.zig.lineDelta(self.source, self.prev_di_src, src); + const delta_pc = self.code.items.len - self.prev_di_pc; + self.prev_di_src = src; + self.prev_di_pc = self.code.items.len; + // TODO Look into using the DWARF special opcodes to compress this data. It lets you emit + // single-byte opcodes that add different numbers to both the PC and the line number + // at the same time. + try self.dbg_line.ensureCapacity(self.dbg_line.items.len + 11); + self.dbg_line.appendAssumeCapacity(DW.LNS_advance_pc); + leb128.writeULEB128(self.dbg_line.writer(), delta_pc) catch unreachable; + if (delta_line != 0) { + self.dbg_line.appendAssumeCapacity(DW.LNS_advance_line); + leb128.writeULEB128(self.dbg_line.writer(), delta_line) catch unreachable; + } + self.dbg_line.appendAssumeCapacity(DW.LNS_copy); + } + fn processDeath(self: *Self, inst: *ir.Inst) void { const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; const entry = branch.inst_table.getEntry(inst) orelse return; @@ -544,6 +605,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .cmp_neq => return self.genCmp(inst.castTag(.cmp_neq).?, .neq), .condbr => return self.genCondBr(inst.castTag(.condbr).?), .constant => unreachable, // excluded from function bodies + .dbg_stmt => return self.genDbgStmt(inst.castTag(.dbg_stmt).?), .floatcast => return self.genFloatCast(inst.castTag(.floatcast).?), .intcast => return self.genIntCast(inst.castTag(.intcast).?), .isnonnull => return self.genIsNonNull(inst.castTag(.isnonnull).?), @@ -1107,6 +1169,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } + fn genDbgStmt(self: *Self, inst: *ir.Inst.NoOp) !MCValue { + try self.dbgAdvancePCAndLine(inst.base.src); + return MCValue.none; + } + fn genCondBr(self: *Self, inst: *ir.Inst.CondBr) !MCValue { switch (arch) { .x86_64 => { diff --git a/src-self-hosted/ir.zig b/src-self-hosted/ir.zig index deb0a91cec..1188230a54 100644 --- a/src-self-hosted/ir.zig +++ b/src-self-hosted/ir.zig @@ -65,6 +65,7 @@ pub const Inst = struct { cmp_neq, condbr, constant, + dbg_stmt, isnonnull, isnull, /// Read a value from a pointer. @@ -88,6 +89,7 @@ pub const Inst = struct { .unreach, .arg, .breakpoint, + .dbg_stmt, => NoOp, .ref, diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 284b881028..c680bb47d5 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -2024,9 +2024,9 @@ pub const File = struct { // line number of the open curly from the beginning of the file. const fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?; const block = fn_proto.body().?.castTag(.Block).?; - const loc = tree.tokenLocation(0, block.lbrace); + const line_delta = std.zig.lineDelta(tree.source, 0, tree.token_locs[block.lbrace].start); // No need to add one; this is a delta from DWARF's starting line number (1). - break :blk @intCast(u28, loc.line); + break :blk @intCast(u28, line_delta); } else { const prev_src_fn = src_file.fns.entries.items[src_fn_index - 1].key; const mod_fn = @fieldParentPtr(Module.Fn, "link", prev_src_fn); @@ -2035,9 +2035,12 @@ pub const File = struct { const prev_block = prev_fn_proto.body().?.castTag(.Block).?; const this_block = this_fn_proto.body().?.castTag(.Block).?; // Find the difference between prev decl end curly and this decl begin curly. - const loc = tree.tokenLocation(tree.token_locs[prev_block.rbrace].start, this_block.lbrace); + const line_delta = std.zig.lineDelta(tree.source, + tree.token_locs[prev_block.rbrace].start, + tree.token_locs[this_block.lbrace].start, + ); // No need to add one; this is a delta from the previous line number. - break :blk @intCast(u28, loc.line); + break :blk @intCast(u28, line_delta); } }; diff --git a/src-self-hosted/zir.zig b/src-self-hosted/zir.zig index 318c4bdc8e..5fb81df051 100644 --- a/src-self-hosted/zir.zig +++ b/src-self-hosted/zir.zig @@ -107,6 +107,8 @@ pub const Inst = struct { condbr, /// Special case, has no textual representation. @"const", + /// Declares the beginning of a statement. Used for debug info. + dbg_stmt, /// Represents a pointer to a global decl by name. declref, /// Represents a pointer to a global decl by string name. @@ -211,6 +213,7 @@ pub const Inst = struct { return switch (tag) { .arg, .breakpoint, + .dbg_stmt, .returnvoid, .alloc_inferred, .ret_ptr, @@ -324,6 +327,7 @@ pub const Inst = struct { .coerce_result_block_ptr, .coerce_to_ptr_elem, .@"const", + .dbg_stmt, .declref, .declref_str, .declval, @@ -1843,6 +1847,7 @@ const EmitZIR = struct { .breakpoint => try self.emitNoOp(inst.src, .breakpoint), .unreach => try self.emitNoOp(inst.src, .@"unreachable"), .retvoid => try self.emitNoOp(inst.src, .returnvoid), + .dbg_stmt => try self.emitNoOp(inst.src, .dbg_stmt), .not => try self.emitUnOp(inst.src, new_body, inst.castTag(.not).?, .boolnot), .ret => try self.emitUnOp(inst.src, new_body, inst.castTag(.ret).?, .@"return"), diff --git a/src-self-hosted/zir_sema.zig b/src-self-hosted/zir_sema.zig index 6bd4159e36..f2ed8abeac 100644 --- a/src-self-hosted/zir_sema.zig +++ b/src-self-hosted/zir_sema.zig @@ -41,6 +41,7 @@ pub fn analyzeInst(mod: *Module, scope: *Scope, old_inst: *zir.Inst) InnerError! .coerce_to_ptr_elem => return analyzeInstCoerceToPtrElem(mod, scope, old_inst.castTag(.coerce_to_ptr_elem).?), .compileerror => return analyzeInstCompileError(mod, scope, old_inst.castTag(.compileerror).?), .@"const" => return analyzeInstConst(mod, scope, old_inst.castTag(.@"const").?), + .dbg_stmt => return analyzeInstDbgStmt(mod, scope, old_inst.castTag(.dbg_stmt).?), .declref => return analyzeInstDeclRef(mod, scope, old_inst.castTag(.declref).?), .declref_str => return analyzeInstDeclRefStr(mod, scope, old_inst.castTag(.declref_str).?), .declval => return analyzeInstDeclVal(mod, scope, old_inst.castTag(.declval).?), @@ -487,6 +488,11 @@ fn analyzeInstBreakVoid(mod: *Module, scope: *Scope, inst: *zir.Inst.BreakVoid) return analyzeBreak(mod, scope, inst.base.src, block, void_inst); } +fn analyzeInstDbgStmt(mod: *Module, scope: *Scope, inst: *zir.Inst.NoOp) InnerError!*Inst { + const b = try mod.requireRuntimeBlock(scope, inst.base.src); + return mod.addNoOp(b, inst.base.src, Type.initTag(.void), .dbg_stmt); +} + fn analyzeInstDeclRefStr(mod: *Module, scope: *Scope, inst: *zir.Inst.DeclRefStr) InnerError!*Inst { const decl_name = try resolveConstString(mod, scope, inst.positionals.name); return mod.analyzeDeclRefByName(scope, inst.base.src, decl_name); From bf85d3db3f281971a5a274e25c90e782a8b692a7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Aug 2020 00:34:12 -0700 Subject: [PATCH 06/21] downgrade .debug_line to DWARFv4 apparently gdb 8.3.1 which is still a commonly distributed version of gdb, does not support v5. --- src-self-hosted/link.zig | 46 +++++++++------------------------------- 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index c680bb47d5..b49c17e09d 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1071,7 +1071,7 @@ pub const File = struct { }, } // Write the form for the compile unit, which must match the abbrev table above. - const name_strp = try self.makeDebugString(self.base.options.root_name); + const name_strp = try self.makeDebugString(self.base.options.root_pkg.root_src_path); const comp_dir_strp = try self.makeDebugString(self.base.options.root_pkg.root_src_dir_path); const producer_strp = try self.makeDebugString("zig (TODO version here)"); // Currently only one compilation unit is supported, so the address range is simply @@ -1225,11 +1225,7 @@ pub const File = struct { }, } - mem.writeInt(u16, di_buf.addManyAsArrayAssumeCapacity(2), 5, target_endian); // version - di_buf.appendSliceAssumeCapacity(&[_]u8{ - ptr_width_bytes, // address_size - 0, // segment_selector_size - }); + mem.writeInt(u16, di_buf.addManyAsArrayAssumeCapacity(2), 4, target_endian); // version // Empirically, debug info consumers do not respect this field, or otherwise // consider it to be an error when it does not point exactly to the end of the header. @@ -1263,39 +1259,17 @@ pub const File = struct { 0, // `DW.LNS_set_epilogue_begin` 1, // `DW.LNS_set_isa` - 1, // directory_entry_format_count - DW.LNCT_path, DW.FORM_string, // directory_entry_format - - // For now we only support one compilation unit, which has one directory. - 1, // directories_count (this is a ULEB128) + 0, // include_directories (none except the compilation unit cwd) }); - // Empirically, some tools do not understand DW.FORM_strp yet. readelf 2.31.1 gives the bogus - // error and gdb 8.3.1 crashes. Both programs seem to work fine with - // DW.FORM_string however. - di_buf.appendSliceAssumeCapacity(self.base.options.root_pkg.root_src_dir_path); - di_buf.appendAssumeCapacity(0); - + // file_names[0] + di_buf.appendSliceAssumeCapacity(self.base.options.root_pkg.root_src_path); // relative path name di_buf.appendSliceAssumeCapacity(&[_]u8{ - 2, // file_name_entry_format_count - DW.LNCT_path, DW.FORM_string, // file_name_entry_format[0] - DW.LNCT_directory_index, DW.FORM_data1, // file_name_entry_format[1] - // TODO Look into adding the file size here. Maybe even the mtime and MD5. - //DW.LNCT_size, DW.FORM_udata, // file_name_entry_format[2] - - // For now we only put the root file name here. Once more source files - // are supported, this will need to be improved. - 2, // file_names_count (this is a ULEB128) + 0, // null byte for the relative path name + 0, // directory_index + 0, // mtime (TODO supply this) + 0, // file size bytes (TODO supply this) + 0, // file_names sentinel }); - // See note above with directories about why we use DW.FORM_string here. - di_buf.appendSliceAssumeCapacity(self.base.options.root_pkg.root_src_path); - di_buf.appendAssumeCapacity(0); - di_buf.appendAssumeCapacity(0); // LNCT_directory_index, FORM_data1 - - // We add the root file twice because according to DWARF, the state machine - // starts out with file index 1. - di_buf.appendSliceAssumeCapacity(self.base.options.root_pkg.root_src_path); - di_buf.appendAssumeCapacity(0); - di_buf.appendAssumeCapacity(0); // LNCT_directory_index, FORM_data1 const header_len = di_buf.items.len - after_header_len; switch (self.ptr_width) { From eccbb03063b5a491d923e41d7cb82c6e71534327 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Aug 2020 00:42:08 -0700 Subject: [PATCH 07/21] downgrade .debug_info to DWARFv4 gdb, notice me senpai --- src-self-hosted/link.zig | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index b49c17e09d..72e059743b 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1057,17 +1057,16 @@ pub const File = struct { const init_len_index = di_buf.items.len; di_buf.items.len += init_len_size; const after_init_len = di_buf.items.len; - mem.writeInt(u16, di_buf.addManyAsArrayAssumeCapacity(2), 5, target_endian); // DWARF version - di_buf.appendAssumeCapacity(DW.UT_compile); + mem.writeInt(u16, di_buf.addManyAsArrayAssumeCapacity(2), 4, target_endian); // DWARF version const abbrev_offset = self.debug_abbrev_table_offset.?; switch (self.ptr_width) { .p32 => { - di_buf.appendAssumeCapacity(4); // address size mem.writeInt(u32, di_buf.addManyAsArrayAssumeCapacity(4), @intCast(u32, abbrev_offset), target_endian); + di_buf.appendAssumeCapacity(4); // address size }, .p64 => { - di_buf.appendAssumeCapacity(8); // address size mem.writeInt(u64, di_buf.addManyAsArrayAssumeCapacity(8), abbrev_offset, target_endian); + di_buf.appendAssumeCapacity(8); // address size }, } // Write the form for the compile unit, which must match the abbrev table above. From 4e023c6fa85c3263dbf388be5ef84dae7f1b0022 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Aug 2020 00:54:30 -0700 Subject: [PATCH 08/21] stage2: dwarf: fix standard opcode length of LNS_fixed_advance_pc --- src-self-hosted/link.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 72e059743b..ed2b92e5d6 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1253,7 +1253,7 @@ pub const File = struct { 0, // `DW.LNS_negate_stmt` 0, // `DW.LNS_set_basic_block` 0, // `DW.LNS_const_add_pc` - 0, // `DW.LNS_fixed_advance_pc` + 1, // `DW.LNS_fixed_advance_pc` 0, // `DW.LNS_set_prologue_end` 0, // `DW.LNS_set_epilogue_begin` 1, // `DW.LNS_set_isa` From ac10841fa9321c71fa0e682521dd39872d43c132 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Aug 2020 19:14:09 -0700 Subject: [PATCH 09/21] stage2 .debug_line: simpler strategy for incremental compilation See #5963 --- src-self-hosted/Module.zig | 4 - src-self-hosted/codegen.zig | 4 +- src-self-hosted/link.zig | 295 +++++++++++++++--------------------- 3 files changed, 122 insertions(+), 181 deletions(-) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index d9560dc425..e84cfe5c14 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -506,10 +506,6 @@ pub const Scope = struct { /// Direct children of the file. decls: ArrayListUnmanaged(*Decl), - /// Represents the file in the linker code. The linker code - /// uses this field to store data relevant to its purposes. - link: link.File.Elf.SrcFile = link.File.Elf.SrcFile.empty, - pub fn unload(self: *File, gpa: *Allocator) void { switch (self.status) { .never_loaded, diff --git a/src-self-hosted/codegen.zig b/src-self-hosted/codegen.zig index 1c441b9179..d45226b4cb 100644 --- a/src-self-hosted/codegen.zig +++ b/src-self-hosted/codegen.zig @@ -469,7 +469,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { try self.dbgSetPrologueEnd(); try self.genBody(self.mod_fn.analysis.success); - try self.dbgSetEpilogueBegin(); const stack_end = self.branch_stack.items[0].max_end_stack; if (stack_end > math.maxInt(i32)) @@ -491,6 +490,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { mem.writeIntLittle(i32, self.code.items[jmp_reloc..][0..4], s32_amt); } + // Important to be after the possible self.code.items.len -= 5 above. + try self.dbgSetEpilogueBegin(); + try self.code.ensureCapacity(self.code.items.len + 9); // add rsp, x if (aligned_stack_end > math.maxInt(i8)) { diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index ed2b92e5d6..353baf72d1 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -344,8 +344,11 @@ pub const File = struct { text_block_free_list: std.ArrayListUnmanaged(*TextBlock) = std.ArrayListUnmanaged(*TextBlock){}, last_text_block: ?*TextBlock = null, - first_dbg_line_file: ?*SrcFile = null, - last_dbg_line_file: ?*SrcFile = null, + /// A list of `SrcFn` whose Line Number Programs have surplus capacity. + /// This is the same concept as `text_block_free_list`; see those doc comments. + dbg_line_fn_free_list: std.AutoHashMapUnmanaged(*SrcFn, void) = .{}, + dbg_line_fn_first: ?*SrcFn = null, + dbg_line_fn_last: ?*SrcFn = null, /// `alloc_num / alloc_den` is the factor of padding when allocating. const alloc_num = 4; @@ -411,46 +414,20 @@ pub const File = struct { }; pub const SrcFn = struct { - /// Offset from the `SrcFile` that contains this function. + /// Offset from the beginning of the Debug Line Program header that contains this function. off: u32, /// Size of the line number program component belonging to this function, not /// including padding. len: u32, + /// Points to the previous and next neighbors, based on the offset from .debug_line. + /// This can be used to find, for example, the capacity of this `SrcFn`. + prev: ?*SrcFn, + next: ?*SrcFn, + pub const empty: SrcFn = .{ .off = 0, .len = 0, - }; - }; - - pub const SrcFile = struct { - /// Byte offset from the start of the Line Number Program that contains this file. - off: u32, - /// Length in bytes, not including padding, of this file component within the - /// Line Number Program that contains it. - len: u32, - - /// An ordered list of all the `SrcFn` in this file. This list is not redundant with - /// the source Decl list, for two reasons: - /// * Lazy decl analysis: some source functions do not correspond to any compiled functions. - /// * Generic functions: some source functions correspond to many compiled functions. - /// This list corresponds to the file data in the Line Number Program. When a new `SrcFn` - /// is inserted, the list must be shifted to accomodate it, and likewise the Line - /// Number Program data must be shifted within the ELF file to accomodate (if there is - /// not enough padding). - /// It is a hash map so that we can look up the index based on the `*SrcFn` and therefore - /// find the next and previous functions. - fns: std.AutoHashMapUnmanaged(*SrcFn, void), - - /// Points to the previous and next neighbors, based on the offset from .debug_line. - /// This can be used to find, for example, the capacity of this `SrcFile`. - prev: ?*SrcFile, - next: ?*SrcFile, - - pub const empty: SrcFile = .{ - .off = 0, - .len = 0, - .fns = .{}, .prev = null, .next = null, }; @@ -593,11 +570,11 @@ pub const File = struct { } fn getDebugLineProgramOff(self: Elf) u32 { - return self.first_dbg_line_file.?.off; + return self.dbg_line_fn_first.?.off; } fn getDebugLineProgramEnd(self: Elf) u32 { - return self.last_dbg_line_file.?.off + self.last_dbg_line_file.?.len; + return self.dbg_line_fn_last.?.off + self.dbg_line_fn_last.?.len; } /// Returns end pos of collision, if any. @@ -1207,7 +1184,7 @@ pub const File = struct { // The size of this header is variable, depending on the number of directories, // files, and padding. We have a function to compute the upper bound size, however, - // because it's needed for determining where to put the offset of the first `SrcFile`. + // because it's needed for determining where to put the offset of the first `SrcFn`. try di_buf.ensureCapacity(self.dbgLineNeededHeaderBytes()); // initial length - length of the .debug_line contribution for this compilation unit, @@ -1280,18 +1257,13 @@ pub const File = struct { }, } - // We use a NOP jmp because consumers empirically do not respect the header length field. - const after_jmp = di_buf.items.len + 6; - if (after_jmp > dbg_line_prg_off) { + // We use NOPs because consumers empirically do not respect the header length field. + if (di_buf.items.len > dbg_line_prg_off) { // Move the first N files to the end to make more padding for the header. @panic("TODO: handle .debug_line header exceeding its padding"); } - const jmp_amt = dbg_line_prg_off - after_jmp + 1; - di_buf.appendAssumeCapacity(DW.LNS_extended_op); - leb128.writeUnsignedFixed(4, di_buf.addManyAsArrayAssumeCapacity(4), @intCast(u28, jmp_amt)); - di_buf.appendAssumeCapacity(DW.LNE_hi_user); - - try self.file.?.pwriteAll(di_buf.items, debug_line_sect.sh_offset); + const jmp_amt = dbg_line_prg_off - di_buf.items.len; + try self.pwriteWithNops(di_buf.items, jmp_amt, debug_line_sect.sh_offset); self.debug_line_header_dirty = false; } @@ -1826,6 +1798,16 @@ pub const File = struct { // For functions we need to add a prologue to the debug line program. try dbg_line_buffer.ensureCapacity(26); + const scope_file = decl.scope.cast(Module.Scope.File).?; + const tree = scope_file.contents.tree; + const file_ast_decls = tree.root_node.decls(); + // TODO Look into improving the performance here by adding a token-index-to-line + // lookup table. Currently this involves scanning over the source code for newlines. + const fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?; + const block = fn_proto.body().?.castTag(.Block).?; + const line_delta = std.zig.lineDelta(tree.source, 0, tree.token_locs[block.lbrace].start); + const casted_line_off = @intCast(u28, line_delta); + const ptr_width_bytes = self.ptrWidthBytes(); dbg_line_buffer.appendSliceAssumeCapacity(&[_]u8{ DW.LNS_extended_op, @@ -1840,9 +1822,15 @@ pub const File = struct { // This is the "relocatable" relative line offset from the previous function's end curly // to this function's begin curly. assert(self.getRelocDbgLineOff() == dbg_line_buffer.items.len); - // Here we allocate 4 bytes for the relocation. This field is a ULEB128, however, - // it is possible to encode small values as still taking up 4 bytes. - dbg_line_buffer.items.len += 4; + // Here we use a ULEB128-fixed-4 to make sure this field can be overwritten later. + leb128.writeUnsignedFixed(4, dbg_line_buffer.addManyAsArrayAssumeCapacity(4), casted_line_off); + + dbg_line_buffer.appendAssumeCapacity(DW.LNS_set_file); + assert(self.getRelocDbgFileIndex() == dbg_line_buffer.items.len); + // Once we support more than one source file, this will have the ability to be more + // than one possible value. + const file_index = 1; + leb128.writeUnsignedFixed(4, dbg_line_buffer.addManyAsArrayAssumeCapacity(4), file_index); // Emit a line for the begin curly with prologue_end=false. The codegen will // do the work of setting prologue_end=true and epilogue_begin=true. @@ -1916,14 +1904,6 @@ pub const File = struct { // If the Decl is a function, we need to update the .debug_line program. if (is_fn) { - // For padding between functions, we terminate with `LNS_extended_op` with sub-op - // `LNE_hi_user`, using a fixed 4-byte ULEB128 for the opcode size. This is always - // found at the very end of the SrcFile's Line Number Program component. - try dbg_line_buffer.ensureCapacity(dbg_line_buffer.items.len + 6); - dbg_line_buffer.appendAssumeCapacity(DW.LNS_extended_op); - leb128.writeUnsignedFixed(4, dbg_line_buffer.addManyAsArrayAssumeCapacity(4), 1); - dbg_line_buffer.appendAssumeCapacity(DW.LNE_hi_user); - // Perform the relocation based on vaddr. const target_endian = self.base.options.target.cpu.arch.endian(); switch (self.ptr_width) { @@ -1937,94 +1917,53 @@ pub const File = struct { }, } - // Now we want to write the line offset relocation, however, first we must - // "plug in" the SrcFn into its parent SrcFile, so that we know what function the line - // number is offset from. It must go in the same order as the functions are found - // in the Zig source. When we insert a function before another one, the latter one - // must have its line offset relocation updated. + try dbg_line_buffer.appendSlice(&[_]u8{ DW.LNS_extended_op, 1, DW.LNE_end_sequence }); + + // Now we have the full contents and may allocate a region to store it. const debug_line_sect = &self.sections.items[self.debug_line_section_index.?]; - const scope_file = decl.scope.cast(Module.Scope.File).?; - const src_file = &scope_file.link; const src_fn = &typed_value.val.cast(Value.Payload.Function).?.func.link; - var src_fn_index: usize = undefined; - if (src_file.len == 0) { - // This is the first function of the SrcFile. - assert(src_file.fns.entries.items.len == 0); - src_fn_index = 0; - try src_file.fns.put(self.allocator, src_fn, {}); + if (self.dbg_line_fn_last) |last| { + if (src_fn.prev == null and src_fn.next == null) { + // Append new function. + src_fn.prev = last; + last.next = src_fn; + self.dbg_line_fn_last = src_fn; - if (self.last_dbg_line_file) |last| { - src_file.prev = last; - self.last_dbg_line_file = src_file; - - // Update the previous last SrcFile's terminating NOP to skip to the start - // of the new last SrcFile's start. - @panic("TODO updateDecl for .debug_line: add new SrcFile: append"); - } else { - // This is the first file (and function) of the Line Number Program. - self.first_dbg_line_file = src_file; - self.last_dbg_line_file = src_file; - - src_fn.off = dbg_line_file_header_len; + src_fn.off = last.off + (last.len * alloc_num / alloc_den); src_fn.len = @intCast(u32, dbg_line_buffer.items.len); - - src_file.off = self.dbgLineNeededHeaderBytes() * alloc_num / alloc_den; - src_file.len = src_fn.off + src_fn.len + dbg_line_file_trailer_len; - - const needed_size = src_file.off + src_file.len; - if (needed_size > debug_line_sect.sh_size) { - debug_line_sect.sh_offset = self.findFreeSpace(needed_size, 1); - } - debug_line_sect.sh_size = needed_size; - self.shdr_table_dirty = true; // TODO look into making only the one section dirty - self.debug_line_header_dirty = true; - - try self.updateDbgLineFile(src_file); + } else { + // Update existing function. + @panic("TODO updateDecl for .debug_line: add new SrcFn: update"); } } else { - @panic("TODO updateDecl for .debug_line: update existing SrcFile"); - //src_fn_index = @panic("TODO"); - } - const line_off: u28 = blk: { - const tree = scope_file.contents.tree; - const file_ast_decls = tree.root_node.decls(); - // TODO Look into improving the performance here by adding a token-index-to-line - // lookup table. Currently this involves scanning over the source code for newlines - // (but only from the previous decl to the current one). - if (src_fn_index == 0) { - // Since it's the first function in the file, the line number delta is just the - // line number of the open curly from the beginning of the file. - const fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?; - const block = fn_proto.body().?.castTag(.Block).?; - const line_delta = std.zig.lineDelta(tree.source, 0, tree.token_locs[block.lbrace].start); - // No need to add one; this is a delta from DWARF's starting line number (1). - break :blk @intCast(u28, line_delta); - } else { - const prev_src_fn = src_file.fns.entries.items[src_fn_index - 1].key; - const mod_fn = @fieldParentPtr(Module.Fn, "link", prev_src_fn); - const prev_fn_proto = file_ast_decls[mod_fn.owner_decl.src_index].castTag(.FnProto).?; - const this_fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?; - const prev_block = prev_fn_proto.body().?.castTag(.Block).?; - const this_block = this_fn_proto.body().?.castTag(.Block).?; - // Find the difference between prev decl end curly and this decl begin curly. - const line_delta = std.zig.lineDelta(tree.source, - tree.token_locs[prev_block.rbrace].start, - tree.token_locs[this_block.lbrace].start, - ); - // No need to add one; this is a delta from the previous line number. - break :blk @intCast(u28, line_delta); - } - }; + // This is the first function of the Line Number Program. + self.dbg_line_fn_first = src_fn; + self.dbg_line_fn_last = src_fn; - // Here we use a ULEB128 but we write 4 bytes regardless (possibly wasting space) because - // that is the amount of space we allocated for this field. - leb128.writeUnsignedFixed(4, dbg_line_buffer.items[self.getRelocDbgLineOff()..][0..4], line_off); + src_fn.off = self.dbgLineNeededHeaderBytes() * alloc_num / alloc_den; + src_fn.len = @intCast(u32, dbg_line_buffer.items.len); + } + + const needed_size = src_fn.off + src_fn.len; + if (needed_size != debug_line_sect.sh_size) { + if (needed_size > self.allocatedSize(debug_line_sect.sh_offset)) { + const new_offset = self.findFreeSpace(needed_size, 1); + const existing_size = src_fn.off; + const amt = try self.file.?.copyRangeAll(debug_line_sect.sh_offset, self.file.?, new_offset, existing_size); + if (amt != existing_size) return error.InputOutput; + debug_line_sect.sh_offset = new_offset; + } + debug_line_sect.sh_size = needed_size; + self.shdr_table_dirty = true; // TODO look into making only the one section dirty + self.debug_line_header_dirty = true; + } + const padding_size: u32 = if (src_fn.next) |next| next.off - (src_fn.off + src_fn.len) else 0; // We only have support for one compilation unit so far, so the offsets are directly // from the .debug_line section. - const file_pos = debug_line_sect.sh_offset + src_file.off + src_fn.off; - try self.file.?.pwriteAll(dbg_line_buffer.items, file_pos); + const file_pos = debug_line_sect.sh_offset + src_fn.off; + try self.pwriteWithNops(dbg_line_buffer.items, padding_size, file_pos); } // Since we updated the vaddr and the size, each corresponding export symbol also needs to be updated. @@ -2116,47 +2055,6 @@ pub const File = struct { self.global_symbols.items[sym_index].st_info = 0; } - const dbg_line_file_header_len = 5; // DW.LNS_set_file + ULEB128-fixed-4 file_index - const dbg_line_file_trailer_len = 9; // DW.LNE_end_sequence + 6-byte terminating NOP - - fn updateDbgLineFile(self: *Elf, src_file: *SrcFile) !void { - const target_endian = self.base.options.target.cpu.arch.endian(); - const shdr = &self.sections.items[self.debug_line_section_index.?]; - const header_off = shdr.sh_offset + src_file.off; - { - var header: [dbg_line_file_header_len]u8 = undefined; - header[0] = DW.LNS_set_file; - // Once we support more than one source file, this will have the ability to be more - // than one possible value. - const file_index = 1; - leb128.writeUnsignedFixed(4, header[1..5], file_index); - try self.file.?.pwriteAll(&header, header_off); - } - { - const last_src_fn = src_file.fns.entries.items[src_file.fns.entries.items.len - 1].key; - const trailer_off = header_off + last_src_fn.off + last_src_fn.len; - const padding_to_next = blk: { - if (src_file.next) |next| { - break :blk next.off - (src_file.off + src_file.len); - } else { - // No need for padding after this one; we will add padding to it when a SrcFile - // is added after it. - break :blk 0; - } - }; - var trailer: [dbg_line_file_trailer_len]u8 = undefined; - - trailer[0] = DW.LNS_extended_op; - trailer[1] = 1; - trailer[2] = DW.LNE_end_sequence; - - trailer[3] = DW.LNS_extended_op; - leb128.writeUnsignedFixed(4, trailer[4..8], @intCast(u28, padding_to_next + 1)); - trailer[8] = DW.LNE_hi_user; - try self.file.?.pwriteAll(&trailer, trailer_off); - } - } - fn writeProgHeader(self: *Elf, index: usize) !void { const foreign_endian = self.base.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); const offset = self.program_headers.items[index].p_offset; @@ -2366,6 +2264,10 @@ pub const File = struct { return dbg_line_vaddr_reloc_index + self.ptrWidthBytes() + 1; } + fn getRelocDbgFileIndex(self: Elf) usize { + return self.getRelocDbgLineOff() + 5; + } + fn dbgLineNeededHeaderBytes(self: Elf) u32 { const directory_entry_format_count = 1; const file_name_entry_format_count = 1; @@ -2376,9 +2278,50 @@ pub const File = struct { // These are encoded as DW.FORM_string rather than DW.FORM_strp as we would like // because of a workaround for readelf and gdb failing to understand DWARFv5 correctly. self.base.options.root_pkg.root_src_dir_path.len + - self.base.options.root_pkg.root_src_path.len * 2); + self.base.options.root_pkg.root_src_path.len); } + + /// Writes to the file a buffer, followed by the specified number of bytes of NOPs. + /// Asserts `padding_size >= 2` and less than 126,976 bytes (if this limit is ever + /// reached, this function can be improved to make more than one pwritev call). + fn pwriteWithNops(self: *Elf, buf: []const u8, padding_size: usize, offset: usize) !void { + const page_of_nops = [1]u8{DW.LNS_negate_stmt} ** 4096; + const three_byte_nop = [3]u8{DW.LNS_advance_pc, 0b1000_0000, 0}; + var vecs: [32]std.os.iovec_const = undefined; + var vec_index: usize = 0; + vecs[vec_index] = .{ + .iov_base = buf.ptr, + .iov_len = buf.len, + }; + vec_index += 1; + var padding_left = padding_size; + if (padding_left % 2 != 0) { + vecs[vec_index] = .{ + .iov_base = &three_byte_nop, + .iov_len = three_byte_nop.len, + }; + vec_index += 1; + padding_left -= three_byte_nop.len; + } + while (padding_left > page_of_nops.len) { + vecs[vec_index] = .{ + .iov_base = &page_of_nops, + .iov_len = page_of_nops.len, + }; + vec_index += 1; + padding_left -= page_of_nops.len; + } + if (padding_left > 0) { + vecs[vec_index] = .{ + .iov_base = &page_of_nops, + .iov_len = padding_left, + }; + vec_index += 1; + } + try self.file.?.pwritevAll(vecs[0..vec_index], offset); + } + }; }; From d624bf8059fa3a86a740f40a1ef763756edd91ce Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Aug 2020 21:01:06 -0700 Subject: [PATCH 10/21] stage2 .debug_line stepping with gdb is working --- lib/std/zig.zig | 19 ++++--- src-self-hosted/Module.zig | 3 ++ src-self-hosted/codegen.zig | 21 +++++--- src-self-hosted/link.zig | 101 +++++++++++++++++++++++++----------- 4 files changed, 100 insertions(+), 44 deletions(-) diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 057b79f11c..b070fbdcd5 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -43,12 +43,19 @@ pub fn findLineColumn(source: []const u8, byte_offset: usize) struct { line: usi return .{ .line = line, .column = column }; } -pub fn lineDelta(source: []const u8, start: usize, end: usize) usize { - var line: usize = 0; - for (source[start..end]) |byte| switch (byte) { - '\n' => line += 1, - else => continue, - }; +pub fn lineDelta(source: []const u8, start: usize, end: usize) isize { + var line: isize = 0; + if (end >= start) { + for (source[start..end]) |byte| switch (byte) { + '\n' => line += 1, + else => continue, + }; + } else { + for (source[end..start]) |byte| switch (byte) { + '\n' => line -= 1, + else => continue, + }; + } return line; } diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index e84cfe5c14..d6cc35b2b6 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -1484,6 +1484,9 @@ fn getAstTree(self: *Module, root_scope: *Scope.File) !*ast.Tree { } fn analyzeRootSrcFile(self: *Module, root_scope: *Scope.File) !void { + const tracy = trace(@src()); + defer tracy.end(); + // We may be analyzing it for the first time, or this may be // an incremental update. This code handles both cases. const tree = try self.getAstTree(root_scope); diff --git a/src-self-hosted/codegen.zig b/src-self-hosted/codegen.zig index d45226b4cb..06a76b8f3c 100644 --- a/src-self-hosted/codegen.zig +++ b/src-self-hosted/codegen.zig @@ -225,6 +225,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { prev_di_src: usize, /// Relative to the beginning of `code`. prev_di_pc: usize, + /// The is_stmt register value, used to avoid redundant LNS_negate_stmt ops. + prev_di_is_stmt: bool, /// Used to find newlines and count line deltas. source: []const u8, /// Byte offset within the source file of the ending curly. @@ -420,6 +422,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .stack_align = undefined, .prev_di_pc = 0, .prev_di_src = lbrace_src, + .prev_di_is_stmt = true, .rbrace_src = rbrace_src, .source = tree.source, }; @@ -523,7 +526,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { }, } // Drop them off at the rbrace. - try self.dbgAdvancePCAndLine(self.rbrace_src); + try self.dbgAdvancePCAndLine(self.rbrace_src, true); } fn genBody(self: *Self, body: ir.Body) InnerError!void { @@ -542,15 +545,15 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { fn dbgSetPrologueEnd(self: *Self) InnerError!void { try self.dbg_line.append(DW.LNS_set_prologue_end); - try self.dbgAdvancePCAndLine(self.prev_di_src); + try self.dbgAdvancePCAndLine(self.prev_di_src, true); } fn dbgSetEpilogueBegin(self: *Self) InnerError!void { try self.dbg_line.append(DW.LNS_set_epilogue_begin); - try self.dbgAdvancePCAndLine(self.prev_di_src); + try self.dbgAdvancePCAndLine(self.prev_di_src, true); } - fn dbgAdvancePCAndLine(self: *Self, src: usize) InnerError!void { + fn dbgAdvancePCAndLine(self: *Self, src: usize, is_stmt: bool) InnerError!void { // TODO Look into improving the performance here by adding a token-index-to-line // lookup table, and changing ir.Inst from storing byte offset to token. Currently // this involves scanning over the source code for newlines @@ -562,12 +565,16 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { // TODO Look into using the DWARF special opcodes to compress this data. It lets you emit // single-byte opcodes that add different numbers to both the PC and the line number // at the same time. - try self.dbg_line.ensureCapacity(self.dbg_line.items.len + 11); + try self.dbg_line.ensureCapacity(self.dbg_line.items.len + 12); + if (self.prev_di_is_stmt != is_stmt) { + self.dbg_line.appendAssumeCapacity(DW.LNS_negate_stmt); + self.prev_di_is_stmt = is_stmt; + } self.dbg_line.appendAssumeCapacity(DW.LNS_advance_pc); leb128.writeULEB128(self.dbg_line.writer(), delta_pc) catch unreachable; if (delta_line != 0) { self.dbg_line.appendAssumeCapacity(DW.LNS_advance_line); - leb128.writeULEB128(self.dbg_line.writer(), delta_line) catch unreachable; + leb128.writeILEB128(self.dbg_line.writer(), delta_line) catch unreachable; } self.dbg_line.appendAssumeCapacity(DW.LNS_copy); } @@ -1172,7 +1179,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } fn genDbgStmt(self: *Self, inst: *ir.Inst.NoOp) !MCValue { - try self.dbgAdvancePCAndLine(inst.base.src); + try self.dbgAdvancePCAndLine(inst.base.src, true); return MCValue.none; } diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 353baf72d1..bd572019c0 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1263,7 +1263,7 @@ pub const File = struct { @panic("TODO: handle .debug_line header exceeding its padding"); } const jmp_amt = dbg_line_prg_off - di_buf.items.len; - try self.pwriteWithNops(di_buf.items, jmp_amt, debug_line_sect.sh_offset); + try self.pwriteWithNops(0, di_buf.items, jmp_amt, debug_line_sect.sh_offset); self.debug_line_header_dirty = false; } @@ -1958,12 +1958,13 @@ pub const File = struct { self.shdr_table_dirty = true; // TODO look into making only the one section dirty self.debug_line_header_dirty = true; } - const padding_size: u32 = if (src_fn.next) |next| next.off - (src_fn.off + src_fn.len) else 0; + const prev_padding_size: u32 = if (src_fn.prev) |prev| src_fn.off - (prev.off + prev.len) else 0; + const next_padding_size: u32 = if (src_fn.next) |next| next.off - (src_fn.off + src_fn.len) else 0; // We only have support for one compilation unit so far, so the offsets are directly // from the .debug_line section. const file_pos = debug_line_sect.sh_offset + src_fn.off; - try self.pwriteWithNops(dbg_line_buffer.items, padding_size, file_pos); + try self.pwriteWithNops(prev_padding_size, dbg_line_buffer.items, next_padding_size, file_pos); } // Since we updated the vaddr and the size, each corresponding export symbol also needs to be updated. @@ -2282,44 +2283,82 @@ pub const File = struct { } - /// Writes to the file a buffer, followed by the specified number of bytes of NOPs. - /// Asserts `padding_size >= 2` and less than 126,976 bytes (if this limit is ever - /// reached, this function can be improved to make more than one pwritev call). - fn pwriteWithNops(self: *Elf, buf: []const u8, padding_size: usize, offset: usize) !void { + /// Writes to the file a buffer, prefixed and suffixed by the specified number of + /// bytes of NOPs. Asserts each padding size is at least two bytes and total padding bytes + /// are less than 126,976 bytes (if this limit is ever reached, this function can be + /// improved to make more than one pwritev call, or the limit can be raised by a fixed + /// amount by increasing the length of `vecs`). + fn pwriteWithNops( + self: *Elf, + prev_padding_size: usize, + buf: []const u8, + next_padding_size: usize, + offset: usize, + ) !void { const page_of_nops = [1]u8{DW.LNS_negate_stmt} ** 4096; const three_byte_nop = [3]u8{DW.LNS_advance_pc, 0b1000_0000, 0}; var vecs: [32]std.os.iovec_const = undefined; var vec_index: usize = 0; + { + var padding_left = prev_padding_size; + if (padding_left % 2 != 0) { + vecs[vec_index] = .{ + .iov_base = &three_byte_nop, + .iov_len = three_byte_nop.len, + }; + vec_index += 1; + padding_left -= three_byte_nop.len; + } + while (padding_left > page_of_nops.len) { + vecs[vec_index] = .{ + .iov_base = &page_of_nops, + .iov_len = page_of_nops.len, + }; + vec_index += 1; + padding_left -= page_of_nops.len; + } + if (padding_left > 0) { + vecs[vec_index] = .{ + .iov_base = &page_of_nops, + .iov_len = padding_left, + }; + vec_index += 1; + } + } + vecs[vec_index] = .{ .iov_base = buf.ptr, .iov_len = buf.len, }; vec_index += 1; - var padding_left = padding_size; - if (padding_left % 2 != 0) { - vecs[vec_index] = .{ - .iov_base = &three_byte_nop, - .iov_len = three_byte_nop.len, - }; - vec_index += 1; - padding_left -= three_byte_nop.len; + + { + var padding_left = next_padding_size; + if (padding_left % 2 != 0) { + vecs[vec_index] = .{ + .iov_base = &three_byte_nop, + .iov_len = three_byte_nop.len, + }; + vec_index += 1; + padding_left -= three_byte_nop.len; + } + while (padding_left > page_of_nops.len) { + vecs[vec_index] = .{ + .iov_base = &page_of_nops, + .iov_len = page_of_nops.len, + }; + vec_index += 1; + padding_left -= page_of_nops.len; + } + if (padding_left > 0) { + vecs[vec_index] = .{ + .iov_base = &page_of_nops, + .iov_len = padding_left, + }; + vec_index += 1; + } } - while (padding_left > page_of_nops.len) { - vecs[vec_index] = .{ - .iov_base = &page_of_nops, - .iov_len = page_of_nops.len, - }; - vec_index += 1; - padding_left -= page_of_nops.len; - } - if (padding_left > 0) { - vecs[vec_index] = .{ - .iov_base = &page_of_nops, - .iov_len = padding_left, - }; - vec_index += 1; - } - try self.file.?.pwritevAll(vecs[0..vec_index], offset); + try self.file.?.pwritevAll(vecs[0..vec_index], offset - prev_padding_size); } }; From a33efc74ed303517aa458109acfcd5c40dc97703 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Aug 2020 21:09:58 -0700 Subject: [PATCH 11/21] stage2 codegen: revert the unneeded is_stmt stuff --- src-self-hosted/codegen.zig | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src-self-hosted/codegen.zig b/src-self-hosted/codegen.zig index 06a76b8f3c..4bd126402b 100644 --- a/src-self-hosted/codegen.zig +++ b/src-self-hosted/codegen.zig @@ -225,8 +225,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { prev_di_src: usize, /// Relative to the beginning of `code`. prev_di_pc: usize, - /// The is_stmt register value, used to avoid redundant LNS_negate_stmt ops. - prev_di_is_stmt: bool, /// Used to find newlines and count line deltas. source: []const u8, /// Byte offset within the source file of the ending curly. @@ -422,7 +420,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .stack_align = undefined, .prev_di_pc = 0, .prev_di_src = lbrace_src, - .prev_di_is_stmt = true, .rbrace_src = rbrace_src, .source = tree.source, }; @@ -526,7 +523,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { }, } // Drop them off at the rbrace. - try self.dbgAdvancePCAndLine(self.rbrace_src, true); + try self.dbgAdvancePCAndLine(self.rbrace_src); } fn genBody(self: *Self, body: ir.Body) InnerError!void { @@ -545,15 +542,15 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { fn dbgSetPrologueEnd(self: *Self) InnerError!void { try self.dbg_line.append(DW.LNS_set_prologue_end); - try self.dbgAdvancePCAndLine(self.prev_di_src, true); + try self.dbgAdvancePCAndLine(self.prev_di_src); } fn dbgSetEpilogueBegin(self: *Self) InnerError!void { try self.dbg_line.append(DW.LNS_set_epilogue_begin); - try self.dbgAdvancePCAndLine(self.prev_di_src, true); + try self.dbgAdvancePCAndLine(self.prev_di_src); } - fn dbgAdvancePCAndLine(self: *Self, src: usize, is_stmt: bool) InnerError!void { + fn dbgAdvancePCAndLine(self: *Self, src: usize) InnerError!void { // TODO Look into improving the performance here by adding a token-index-to-line // lookup table, and changing ir.Inst from storing byte offset to token. Currently // this involves scanning over the source code for newlines @@ -565,11 +562,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { // TODO Look into using the DWARF special opcodes to compress this data. It lets you emit // single-byte opcodes that add different numbers to both the PC and the line number // at the same time. - try self.dbg_line.ensureCapacity(self.dbg_line.items.len + 12); - if (self.prev_di_is_stmt != is_stmt) { - self.dbg_line.appendAssumeCapacity(DW.LNS_negate_stmt); - self.prev_di_is_stmt = is_stmt; - } + try self.dbg_line.ensureCapacity(self.dbg_line.items.len + 11); self.dbg_line.appendAssumeCapacity(DW.LNS_advance_pc); leb128.writeULEB128(self.dbg_line.writer(), delta_pc) catch unreachable; if (delta_line != 0) { @@ -1179,7 +1172,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } fn genDbgStmt(self: *Self, inst: *ir.Inst.NoOp) !MCValue { - try self.dbgAdvancePCAndLine(inst.base.src, true); + try self.dbgAdvancePCAndLine(inst.base.src); return MCValue.none; } From edfede575c3113c3611b14a868e8d1e956ca9ef5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Aug 2020 21:56:21 -0700 Subject: [PATCH 12/21] self-hosted: add build option for log scopes Now you can enable a set of log scopes by passing -Dlog= --- build.zig | 3 +++ lib/std/build.zig | 50 +++++++++++++++++++++++--------------- src-self-hosted/Module.zig | 33 ++++++++++++------------- src-self-hosted/main.zig | 22 +++++++---------- 4 files changed, 59 insertions(+), 49 deletions(-) diff --git a/build.zig b/build.zig index a5e430fb01..51089664ad 100644 --- a/build.zig +++ b/build.zig @@ -77,6 +77,9 @@ pub fn build(b: *Builder) !void { const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse false; if (link_libc) exe.linkLibC(); + const log_scopes = b.option([]const []const u8, "log", "Which log scopes to enable") orelse &[0][]const u8{}; + + exe.addBuildOption([]const []const u8, "log_scopes", log_scopes); exe.addBuildOption(bool, "enable_tracy", tracy != null); if (tracy) |tracy_path| { const client_cpp = fs.path.join( diff --git a/lib/std/build.zig b/lib/std/build.zig index 103d3a3193..c85534ba7d 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -430,9 +430,9 @@ pub const Builder = struct { const entry = self.user_input_options.getEntry(name) orelse return null; entry.value.used = true; switch (type_id) { - TypeId.Bool => switch (entry.value.value) { - UserValue.Flag => return true, - UserValue.Scalar => |s| { + .Bool => switch (entry.value.value) { + .Flag => return true, + .Scalar => |s| { if (mem.eql(u8, s, "true")) { return true; } else if (mem.eql(u8, s, "false")) { @@ -443,21 +443,21 @@ pub const Builder = struct { return null; } }, - UserValue.List => { + .List => { warn("Expected -D{} to be a boolean, but received a list.\n", .{name}); self.markInvalidUserInput(); return null; }, }, - TypeId.Int => panic("TODO integer options to build script", .{}), - TypeId.Float => panic("TODO float options to build script", .{}), - TypeId.Enum => switch (entry.value.value) { - UserValue.Flag => { + .Int => panic("TODO integer options to build script", .{}), + .Float => panic("TODO float options to build script", .{}), + .Enum => switch (entry.value.value) { + .Flag => { warn("Expected -D{} to be a string, but received a boolean.\n", .{name}); self.markInvalidUserInput(); return null; }, - UserValue.Scalar => |s| { + .Scalar => |s| { if (std.meta.stringToEnum(T, s)) |enum_lit| { return enum_lit; } else { @@ -466,33 +466,35 @@ pub const Builder = struct { return null; } }, - UserValue.List => { + .List => { warn("Expected -D{} to be a string, but received a list.\n", .{name}); self.markInvalidUserInput(); return null; }, }, - TypeId.String => switch (entry.value.value) { - UserValue.Flag => { + .String => switch (entry.value.value) { + .Flag => { warn("Expected -D{} to be a string, but received a boolean.\n", .{name}); self.markInvalidUserInput(); return null; }, - UserValue.List => { + .List => { warn("Expected -D{} to be a string, but received a list.\n", .{name}); self.markInvalidUserInput(); return null; }, - UserValue.Scalar => |s| return s, + .Scalar => |s| return s, }, - TypeId.List => switch (entry.value.value) { - UserValue.Flag => { + .List => switch (entry.value.value) { + .Flag => { warn("Expected -D{} to be a list, but received a boolean.\n", .{name}); self.markInvalidUserInput(); return null; }, - UserValue.Scalar => |s| return &[_][]const u8{s}, - UserValue.List => |lst| return lst.span(), + .Scalar => |s| { + return self.allocator.dupe([]const u8, &[_][]const u8{s}) catch unreachable; + }, + .List => |lst| return lst.span(), }, } } @@ -1706,9 +1708,19 @@ pub const LibExeObjStep = struct { pub fn addBuildOption(self: *LibExeObjStep, comptime T: type, name: []const u8, value: T) void { const out = self.build_options_contents.outStream(); + if (T == []const []const u8) { + out.print("pub const {}: []const []const u8 = &[_][]const u8{{\n", .{name}) catch unreachable; + for (value) |slice| { + out.writeAll(" ") catch unreachable; + std.zig.renderStringLiteral(slice, out) catch unreachable; + out.writeAll(",\n") catch unreachable; + } + out.writeAll("};\n") catch unreachable; + return; + } switch (@typeInfo(T)) { .Enum => |enum_info| { - out.print("const {} = enum {{\n", .{@typeName(T)}) catch unreachable; + out.print("pub const {} = enum {{\n", .{@typeName(T)}) catch unreachable; inline for (enum_info.fields) |field| { out.print(" {},\n", .{field.name}) catch unreachable; } diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index d6cc35b2b6..0eb79e92ed 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -6,6 +6,7 @@ const Value = @import("value.zig").Value; const Type = @import("type.zig").Type; const TypedValue = @import("TypedValue.zig"); const assert = std.debug.assert; +const log = std.log; const BigIntConst = std.math.big.int.Const; const BigIntMutable = std.math.big.int.Mutable; const Target = std.Target; @@ -235,7 +236,7 @@ pub const Decl = struct { pub fn dump(self: *Decl) void { const loc = std.zig.findLineColumn(self.scope.source.bytes, self.src); - std.debug.warn("{}:{}:{} name={} status={}", .{ + std.debug.print("{}:{}:{} name={} status={}", .{ self.scope.sub_file_path, loc.line + 1, loc.column + 1, @@ -243,9 +244,9 @@ pub const Decl = struct { @tagName(self.analysis), }); if (self.typedValueManaged()) |tvm| { - std.debug.warn(" ty={} val={}", .{ tvm.typed_value.ty, tvm.typed_value.val }); + std.debug.print(" ty={} val={}", .{ tvm.typed_value.ty, tvm.typed_value.val }); } - std.debug.warn("\n", .{}); + std.debug.print("\n", .{}); } pub fn typedValueManaged(self: *Decl) ?*TypedValue.Managed { @@ -544,7 +545,7 @@ pub const Scope = struct { pub fn dumpSrc(self: *File, src: usize) void { const loc = std.zig.findLineColumn(self.source.bytes, src); - std.debug.warn("{}:{}:{}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 }); + std.debug.print("{}:{}:{}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 }); } pub fn getSource(self: *File, module: *Module) ![:0]const u8 { @@ -646,7 +647,7 @@ pub const Scope = struct { pub fn dumpSrc(self: *ZIRModule, src: usize) void { const loc = std.zig.findLineColumn(self.source.bytes, src); - std.debug.warn("{}:{}:{}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 }); + std.debug.print("{}:{}:{}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 }); } pub fn getSource(self: *ZIRModule, module: *Module) ![:0]const u8 { @@ -946,7 +947,6 @@ pub fn update(self: *Module) !void { } self.link_error_flags = self.bin_file.errorFlags(); - std.log.debug(.module, "link_error_flags: {}\n", .{self.link_error_flags}); // If there are any errors, we anticipate the source files being loaded // to report error messages. Otherwise we unload all source files to save memory. @@ -1109,7 +1109,7 @@ pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void { assert(decl.analysis == .complete); return; } - //std.debug.warn("re-analyzing {}\n", .{decl.name}); + log.debug(.module, "re-analyzing {}\n", .{decl.name}); // The exports this Decl performs will be re-discovered, so we remove them here // prior to re-analysis. @@ -1546,7 +1546,7 @@ fn analyzeRootSrcFile(self: *Module, root_scope: *Scope.File) !void { // Handle explicitly deleted decls from the source code. Not to be confused // with when we delete decls because they are no longer referenced. for (deleted_decls.items()) |entry| { - //std.debug.warn("noticed '{}' deleted from source\n", .{entry.key.name}); + log.debug(.module, "noticed '{}' deleted from source\n", .{entry.key.name}); try self.deleteDecl(entry.key); } } @@ -1575,7 +1575,6 @@ fn analyzeRootZIRModule(self: *Module, root_scope: *Scope.ZIRModule) !void { const name_hash = root_scope.fullyQualifiedNameHash(src_decl.name); if (self.decl_table.get(name_hash)) |decl| { deleted_decls.removeAssertDiscard(decl); - //std.debug.warn("'{}' contents: '{}'\n", .{ src_decl.name, src_decl.contents }); if (!srcHashEql(src_decl.contents_hash, decl.contents_hash)) { try self.markOutdatedDecl(decl); decl.contents_hash = src_decl.contents_hash; @@ -1600,7 +1599,7 @@ fn analyzeRootZIRModule(self: *Module, root_scope: *Scope.ZIRModule) !void { // Handle explicitly deleted decls from the source code. Not to be confused // with when we delete decls because they are no longer referenced. for (deleted_decls.items()) |entry| { - //std.debug.warn("noticed '{}' deleted from source\n", .{entry.key.name}); + log.debug(.module, "noticed '{}' deleted from source\n", .{entry.key.name}); try self.deleteDecl(entry.key); } } @@ -1612,7 +1611,7 @@ fn deleteDecl(self: *Module, decl: *Decl) !void { // not be present in the set, and this does nothing. decl.scope.removeDecl(decl); - //std.debug.warn("deleting decl '{}'\n", .{decl.name}); + log.debug(.module, "deleting decl '{}'\n", .{decl.name}); const name_hash = decl.fullyQualifiedNameHash(); self.decl_table.removeAssertDiscard(name_hash); // Remove itself from its dependencies, because we are about to destroy the decl pointer. @@ -1698,17 +1697,17 @@ fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void { const fn_zir = func.analysis.queued; defer fn_zir.arena.promote(self.gpa).deinit(); func.analysis = .{ .in_progress = {} }; - //std.debug.warn("set {} to in_progress\n", .{decl.name}); + log.debug(.module, "set {} to in_progress\n", .{decl.name}); try zir_sema.analyzeBody(self, &inner_block.base, fn_zir.body); const instructions = try arena.allocator.dupe(*Inst, inner_block.instructions.items); func.analysis = .{ .success = .{ .instructions = instructions } }; - //std.debug.warn("set {} to success\n", .{decl.name}); + log.debug(.module, "set {} to success\n", .{decl.name}); } fn markOutdatedDecl(self: *Module, decl: *Decl) !void { - //std.debug.warn("mark {} outdated\n", .{decl.name}); + log.debug(.module, "mark {} outdated\n", .{decl.name}); try self.work_queue.writeItem(.{ .analyze_decl = decl }); if (self.failed_decls.remove(decl)) |entry| { entry.value.destroy(self.gpa); @@ -2817,7 +2816,7 @@ pub fn dumpInst(self: *Module, scope: *Scope, inst: *Inst) void { const source = zir_module.getSource(self) catch @panic("dumpInst failed to get source"); const loc = std.zig.findLineColumn(source, inst.src); if (inst.tag == .constant) { - std.debug.warn("constant ty={} val={} src={}:{}:{}\n", .{ + std.debug.print("constant ty={} val={} src={}:{}:{}\n", .{ inst.ty, inst.castTag(.constant).?.val, zir_module.subFilePath(), @@ -2825,7 +2824,7 @@ pub fn dumpInst(self: *Module, scope: *Scope, inst: *Inst) void { loc.column + 1, }); } else if (inst.deaths == 0) { - std.debug.warn("{} ty={} src={}:{}:{}\n", .{ + std.debug.print("{} ty={} src={}:{}:{}\n", .{ @tagName(inst.tag), inst.ty, zir_module.subFilePath(), @@ -2833,7 +2832,7 @@ pub fn dumpInst(self: *Module, scope: *Scope, inst: *Inst) void { loc.column + 1, }); } else { - std.debug.warn("{} ty={} deaths={b} src={}:{}:{}\n", .{ + std.debug.print("{} ty={} deaths={b} src={}:{}:{}\n", .{ @tagName(inst.tag), inst.ty, inst.deaths, diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 4c5d433f05..8b545ab546 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -10,9 +10,7 @@ const Module = @import("Module.zig"); const link = @import("link.zig"); const Package = @import("Package.zig"); const zir = @import("zir.zig"); - -// TODO Improve async I/O enough that we feel comfortable doing this. -//pub const io_mode = .evented; +const build_options = @import("build_options"); pub const max_src_size = 2 * 1024 * 1024 * 1024; // 2 GiB @@ -47,18 +45,16 @@ pub fn log( if (@enumToInt(level) > @enumToInt(std.log.level)) return; - const scope_prefix = "(" ++ switch (scope) { - // Uncomment to hide logs - //.compiler, - .module, - .liveness, - .link, - => return, + const scope_name = @tagName(scope); + const ok = comptime for (build_options.log_scopes) |log_scope| { + if (mem.eql(u8, log_scope, scope_name)) + break true; + } else false; - else => @tagName(scope), - } ++ "): "; + if (!ok) + return; - const prefix = "[" ++ @tagName(level) ++ "] " ++ scope_prefix; + const prefix = "[" ++ @tagName(level) ++ "] " ++ "(" ++ @tagName(scope) ++ "): "; // Print the message to stderr, silently ignoring any errors std.debug.print(prefix ++ format, args); From 0c598100d8dbcf94fa8916a8aa5dcf5f55ef6d46 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Aug 2020 22:07:21 -0700 Subject: [PATCH 13/21] stage2: fix use-after-free of export symbol name --- src-self-hosted/Module.zig | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 0eb79e92ed..4b564da4eb 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -1673,6 +1673,7 @@ fn deleteDeclExports(self: *Module, decl: *Decl) void { entry.value.destroy(self.gpa); } _ = self.symbol_exports.remove(exp.options.name); + self.gpa.free(exp.options.name); self.gpa.destroy(exp); } self.gpa.free(kv.value); @@ -1773,7 +1774,7 @@ pub fn resolveDefinedValue(self: *Module, scope: *Scope, base: *Inst) !?Value { return null; } -pub fn analyzeExport(self: *Module, scope: *Scope, src: usize, symbol_name: []const u8, exported_decl: *Decl) !void { +pub fn analyzeExport(self: *Module, scope: *Scope, src: usize, borrowed_symbol_name: []const u8, exported_decl: *Decl) !void { try self.ensureDeclAnalyzed(exported_decl); const typed_value = exported_decl.typed_value.most_recent.typed_value; switch (typed_value.ty.zigTypeTag()) { @@ -1787,6 +1788,9 @@ pub fn analyzeExport(self: *Module, scope: *Scope, src: usize, symbol_name: []co const new_export = try self.gpa.create(Export); errdefer self.gpa.destroy(new_export); + const symbol_name = try self.gpa.dupe(u8, borrowed_symbol_name); + errdefer self.gpa.free(symbol_name); + const owner_decl = scope.decl().?; new_export.* = .{ @@ -1799,7 +1803,7 @@ pub fn analyzeExport(self: *Module, scope: *Scope, src: usize, symbol_name: []co }; // Add to export_owners table. - const eo_gop = self.export_owners.getOrPut(self.gpa, owner_decl) catch unreachable; + const eo_gop = self.export_owners.getOrPutAssumeCapacity(owner_decl); if (!eo_gop.found_existing) { eo_gop.entry.value = &[0]*Export{}; } @@ -1808,7 +1812,7 @@ pub fn analyzeExport(self: *Module, scope: *Scope, src: usize, symbol_name: []co errdefer eo_gop.entry.value = self.gpa.shrink(eo_gop.entry.value, eo_gop.entry.value.len - 1); // Add to exported_decl table. - const de_gop = self.decl_exports.getOrPut(self.gpa, exported_decl) catch unreachable; + const de_gop = self.decl_exports.getOrPutAssumeCapacity(exported_decl); if (!de_gop.found_existing) { de_gop.entry.value = &[0]*Export{}; } From 9b3a70c8aab74b804cac0d7c15e4e7518a88c868 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Aug 2020 22:22:47 -0700 Subject: [PATCH 14/21] stage2: move link.File.ELF.SrcFn field from Module.Fn to Module.Decl SrcFn represents the function in the linked output file, if the `Decl` is a function. This is stored here and not in `Fn` because `Decl` survives across updates but `Fn` does not. TODO Look into making `Fn` a longer lived structure and moving this field there to save on memory usage. --- src-self-hosted/Module.zig | 10 +++++++--- src-self-hosted/link.zig | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 4b564da4eb..87b69d6117 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -176,6 +176,13 @@ pub const Decl = struct { /// This is populated regardless of semantic analysis and code generation. link: link.File.Elf.TextBlock = link.File.Elf.TextBlock.empty, + /// Represents the function in the linked output file, if the `Decl` is a function. + /// This is stored here and not in `Fn` because `Decl` survives across updates but + /// `Fn` does not. + /// TODO Look into making `Fn` a longer lived structure and moving this field there + /// to save on memory usage. + fn_link: link.File.Elf.SrcFn = link.File.Elf.SrcFn.empty, + contents_hash: std.zig.SrcHash, /// The shallow set of other decls whose typed_value could possibly change if this Decl's @@ -280,9 +287,6 @@ pub const Fn = struct { }, owner_decl: *Decl, - /// Represents the function in the linked output file. - link: link.File.Elf.SrcFn = link.File.Elf.SrcFn.empty, - /// This memory is temporary and points to stack memory for the duration /// of Fn analysis. pub const Analysis = struct { diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index bd572019c0..012a44d757 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1922,7 +1922,7 @@ pub const File = struct { // Now we have the full contents and may allocate a region to store it. const debug_line_sect = &self.sections.items[self.debug_line_section_index.?]; - const src_fn = &typed_value.val.cast(Value.Payload.Function).?.func.link; + const src_fn = &decl.fn_link; if (self.dbg_line_fn_last) |last| { if (src_fn.prev == null and src_fn.next == null) { // Append new function. From cb25d8e4bc78d6fe21e0cb0b979aa8250a086c2e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Aug 2020 23:13:54 -0700 Subject: [PATCH 15/21] stage2 .debug_line: handle Decl deletes and updates --- src-self-hosted/link.zig | 66 ++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 012a44d757..0746a3f6fb 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1731,11 +1731,8 @@ pub const File = struct { pub fn allocateDeclIndexes(self: *Elf, decl: *Module.Decl) !void { if (decl.link.local_sym_index != 0) return; - // Here we also ensure capacity for the free lists so that they can be appended to without fail. try self.local_symbols.ensureCapacity(self.allocator, self.local_symbols.items.len + 1); - try self.local_symbol_free_list.ensureCapacity(self.allocator, self.local_symbols.items.len); try self.offset_table.ensureCapacity(self.allocator, self.offset_table.items.len + 1); - try self.offset_table_free_list.ensureCapacity(self.allocator, self.local_symbols.items.len); if (self.local_symbol_free_list.popOrNull()) |i| { log.debug(.link, "reusing symbol index {} for {}\n", .{ i, decl.name }); @@ -1768,15 +1765,37 @@ pub const File = struct { } pub fn freeDecl(self: *Elf, decl: *Module.Decl) void { + // Appending to free lists is allowed to fail because the free lists are heuristics based anyway. self.freeTextBlock(&decl.link); if (decl.link.local_sym_index != 0) { - self.local_symbol_free_list.appendAssumeCapacity(decl.link.local_sym_index); - self.offset_table_free_list.appendAssumeCapacity(decl.link.offset_table_index); + self.local_symbol_free_list.append(self.allocator, decl.link.local_sym_index) catch {}; + self.offset_table_free_list.append(self.allocator, decl.link.offset_table_index) catch {}; self.local_symbols.items[decl.link.local_sym_index].st_info = 0; decl.link.local_sym_index = 0; } + // TODO make this logic match freeTextBlock. Maybe abstract the logic out since the same thing + // is desired for both. + _ = self.dbg_line_fn_free_list.remove(&decl.fn_link); + if (decl.fn_link.prev) |prev| { + _ = self.dbg_line_fn_free_list.put(self.allocator, prev, {}) catch {}; + prev.next = decl.fn_link.next; + if (decl.fn_link.next) |next| { + next.prev = prev; + } else { + self.dbg_line_fn_last = prev; + } + } else if (decl.fn_link.next) |next| { + self.dbg_line_fn_first = next; + next.prev = null; + } + if (self.dbg_line_fn_first == &decl.fn_link) { + self.dbg_line_fn_first = null; + } + if (self.dbg_line_fn_last == &decl.fn_link) { + self.dbg_line_fn_last = null; + } } pub fn updateDecl(self: *Elf, module: *Module, decl: *Module.Decl) !void { @@ -1923,18 +1942,35 @@ pub const File = struct { const debug_line_sect = &self.sections.items[self.debug_line_section_index.?]; const src_fn = &decl.fn_link; + src_fn.len = @intCast(u32, dbg_line_buffer.items.len); if (self.dbg_line_fn_last) |last| { - if (src_fn.prev == null and src_fn.next == null) { + if (src_fn.next) |next| { + // Update existing function - non-last item. + if (src_fn.off + src_fn.len + min_nop_size > next.off) { + // It grew too big, so we move it to a new location. + if (src_fn.prev) |prev| { + _ = self.dbg_line_fn_free_list.put(self.allocator, prev, {}) catch {}; + prev.next = src_fn.next; + } + next.prev = src_fn.prev; + // Populate where it used to be with NOPs. + const file_pos = debug_line_sect.sh_offset + src_fn.off; + try self.pwriteWithNops(0, &[0]u8{}, src_fn.len, file_pos); + // TODO Look at the free list before appending at the end. + src_fn.prev = last; + last.next = src_fn; + self.dbg_line_fn_last = src_fn; + + src_fn.off = last.off + (last.len * alloc_num / alloc_den); + } + } else if (src_fn.prev == null) { // Append new function. + // TODO Look at the free list before appending at the end. src_fn.prev = last; last.next = src_fn; self.dbg_line_fn_last = src_fn; src_fn.off = last.off + (last.len * alloc_num / alloc_den); - src_fn.len = @intCast(u32, dbg_line_buffer.items.len); - } else { - // Update existing function. - @panic("TODO updateDecl for .debug_line: add new SrcFn: update"); } } else { // This is the first function of the Line Number Program. @@ -1942,7 +1978,6 @@ pub const File = struct { self.dbg_line_fn_last = src_fn; src_fn.off = self.dbgLineNeededHeaderBytes() * alloc_num / alloc_den; - src_fn.len = @intCast(u32, dbg_line_buffer.items.len); } const needed_size = src_fn.off + src_fn.len; @@ -1982,10 +2017,7 @@ pub const File = struct { const tracy = trace(@src()); defer tracy.end(); - // In addition to ensuring capacity for global_symbols, we also ensure capacity for freeing all of - // them, so that deleting exports is guaranteed to succeed. try self.global_symbols.ensureCapacity(self.allocator, self.global_symbols.items.len + exports.len); - try self.global_symbol_free_list.ensureCapacity(self.allocator, self.global_symbols.items.len); const typed_value = decl.typed_value.most_recent.typed_value; if (decl.link.local_sym_index == 0) return; const decl_sym = self.local_symbols.items[decl.link.local_sym_index]; @@ -2052,7 +2084,7 @@ pub const File = struct { pub fn deleteExport(self: *Elf, exp: Export) void { const sym_index = exp.sym_index orelse return; - self.global_symbol_free_list.appendAssumeCapacity(sym_index); + self.global_symbol_free_list.append(self.allocator, sym_index) catch {}; self.global_symbols.items[sym_index].st_info = 0; } @@ -2284,7 +2316,7 @@ pub const File = struct { } /// Writes to the file a buffer, prefixed and suffixed by the specified number of - /// bytes of NOPs. Asserts each padding size is at least two bytes and total padding bytes + /// bytes of NOPs. Asserts each padding size is at least `min_nop_size` and total padding bytes /// are less than 126,976 bytes (if this limit is ever reached, this function can be /// improved to make more than one pwritev call, or the limit can be raised by a fixed /// amount by increasing the length of `vecs`). @@ -2361,6 +2393,8 @@ pub const File = struct { try self.file.?.pwritevAll(vecs[0..vec_index], offset - prev_padding_size); } + const min_nop_size = 2; + }; }; From 30ee08dfc2236a9c25826dbde82f9865bae5cf30 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Aug 2020 23:47:54 -0700 Subject: [PATCH 16/21] stage2: stop needlessly re-analyzing unchanged functions --- src-self-hosted/Module.zig | 8 +++----- src-self-hosted/link.zig | 15 +++++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 87b69d6117..335bb06908 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -1108,11 +1108,9 @@ pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void { .codegen_failure_retryable, => return error.AnalysisFail, - .complete, .outdated => blk: { - if (decl.generation == self.generation) { - assert(decl.analysis == .complete); - return; - } + .complete => return, + + .outdated => blk: { log.debug(.module, "re-analyzing {}\n", .{decl.name}); // The exports this Decl performs will be re-discovered, so we remove them here diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 0746a3f6fb..b96c1ae3a0 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1324,7 +1324,7 @@ pub const File = struct { shstrtab_sect.sh_offset = self.findFreeSpace(needed_size, 1); } shstrtab_sect.sh_size = needed_size; - log.debug(.link, "shstrtab start=0x{x} end=0x{x}\n", .{ shstrtab_sect.sh_offset, shstrtab_sect.sh_offset + needed_size }); + log.debug(.link, "writing shstrtab start=0x{x} end=0x{x}\n", .{ shstrtab_sect.sh_offset, shstrtab_sect.sh_offset + needed_size }); try self.file.?.pwriteAll(self.shstrtab.items, shstrtab_sect.sh_offset); if (!self.shdr_table_dirty) { @@ -1980,11 +1980,17 @@ pub const File = struct { src_fn.off = self.dbgLineNeededHeaderBytes() * alloc_num / alloc_den; } - const needed_size = src_fn.off + src_fn.len; + const last_src_fn = self.dbg_line_fn_last.?; + const needed_size = last_src_fn.off + last_src_fn.len; if (needed_size != debug_line_sect.sh_size) { if (needed_size > self.allocatedSize(debug_line_sect.sh_offset)) { const new_offset = self.findFreeSpace(needed_size, 1); - const existing_size = src_fn.off; + const existing_size = last_src_fn.off; + log.debug(.link, "moving .debug_line section: {} bytes from 0x{x} to 0x{x}\n", .{ + existing_size, + debug_line_sect.sh_offset, + new_offset, + }); const amt = try self.file.?.copyRangeAll(debug_line_sect.sh_offset, self.file.?, new_offset, existing_size); if (amt != existing_size) return error.InputOutput; debug_line_sect.sh_offset = new_offset; @@ -2112,7 +2118,6 @@ pub const File = struct { fn writeSectHeader(self: *Elf, index: usize) !void { const foreign_endian = self.base.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian(); - const offset = self.sections.items[index].sh_offset; switch (self.base.options.target.cpu.arch.ptrBitWidth()) { 32 => { var shdr: [1]elf.Elf32_Shdr = undefined; @@ -2120,6 +2125,7 @@ pub const File = struct { if (foreign_endian) { bswapAllFields(elf.Elf32_Shdr, &shdr[0]); } + const offset = self.shdr_table_offset.? + index * @sizeOf(elf.Elf32_Shdr); return self.file.?.pwriteAll(mem.sliceAsBytes(&shdr), offset); }, 64 => { @@ -2127,6 +2133,7 @@ pub const File = struct { if (foreign_endian) { bswapAllFields(elf.Elf64_Shdr, &shdr[0]); } + const offset = self.shdr_table_offset.? + index * @sizeOf(elf.Elf64_Shdr); return self.file.?.pwriteAll(mem.sliceAsBytes(&shdr), offset); }, else => return error.UnsupportedArchitecture, From 0d696a48dadffbb3dcd3f7ada6867129da4d70c8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 4 Aug 2020 00:22:11 -0700 Subject: [PATCH 17/21] stage2 .debug_line: handle Decl line numbers changing --- src-self-hosted/Module.zig | 39 ++++++++++++++++++++++++-------------- src-self-hosted/link.zig | 33 ++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 335bb06908..5a260c4cd2 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -89,6 +89,9 @@ const WorkItem = union(enum) { /// It may have already be analyzed, or it may have been determined /// to be outdated; in this case perform semantic analysis again. analyze_decl: *Decl, + /// The source file containing the Decl has been updated, and so the + /// Decl may need its line number information updated in the debug info. + update_line_number: *Decl, }; pub const Export = struct { @@ -1064,22 +1067,14 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void { error.AnalysisFail => { decl.analysis = .dependency_failure; }, - error.CGenFailure => { - // Error is handled by CBE, don't try adding it again - }, else => { try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1); - const result = self.failed_decls.getOrPutAssumeCapacity(decl); - if (result.found_existing) { - std.debug.panic("Internal error: attempted to override error '{}' with 'unable to codegen: {}'", .{ result.entry.value.msg, @errorName(err) }); - } else { - result.entry.value = try ErrorMsg.create( - self.gpa, - decl.src(), - "unable to codegen: {}", - .{@errorName(err)}, - ); - } + self.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create( + self.gpa, + decl.src(), + "unable to codegen: {}", + .{@errorName(err)}, + )); decl.analysis = .codegen_failure_retryable; }, }; @@ -1091,6 +1086,18 @@ pub fn performAllTheWork(self: *Module) error{OutOfMemory}!void { error.AnalysisFail => continue, }; }, + .update_line_number => |decl| { + self.bin_file.updateDeclLineNumber(self, decl) catch |err| { + try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1); + self.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create( + self.gpa, + decl.src(), + "unable to update line number: {}", + .{@errorName(err)}, + )); + decl.analysis = .codegen_failure_retryable; + }; + }, }; } @@ -1530,6 +1537,10 @@ fn analyzeRootSrcFile(self: *Module, root_scope: *Scope.File) !void { if (!srcHashEql(decl.contents_hash, contents_hash)) { try self.markOutdatedDecl(decl); decl.contents_hash = contents_hash; + } else if (decl.fn_link.len != 0) { + // TODO Look into detecting when this would be unnecessary by storing enough state + // in `Decl` to notice that the line number did not change. + self.work_queue.writeItemAssumeCapacity(.{ .update_line_number = decl }); } } } else { diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index b96c1ae3a0..c3f5146113 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -85,6 +85,13 @@ pub const File = struct { } } + pub fn updateDeclLineNumber(base: *File, module: *Module, decl: *Module.Decl) !void { + switch (base.tag) { + .elf => return @fieldParentPtr(Elf, "base", base).updateDeclLineNumber(module, decl), + .c => {}, + } + } + pub fn allocateDeclIndexes(base: *File, decl: *Module.Decl) !void { switch (base.tag) { .elf => return @fieldParentPtr(Elf, "base", base).allocateDeclIndexes(decl), @@ -203,7 +210,7 @@ pub const File = struct { pub fn fail(self: *C, src: usize, comptime format: []const u8, args: anytype) !void { self.error_msg = try Module.ErrorMsg.create(self.allocator, src, format, args); - return error.CGenFailure; + return error.AnalysisFail; } pub fn deinit(self: *File.C) void { @@ -217,7 +224,7 @@ pub const File = struct { pub fn updateDecl(self: *File.C, module: *Module, decl: *Module.Decl) !void { c_codegen.generate(self, decl) catch |err| { - if (err == error.CGenFailure) { + if (err == error.AnalysisFail) { try module.failed_decls.put(module.gpa, decl, self.error_msg); } return err; @@ -2088,6 +2095,28 @@ pub const File = struct { } } + /// Must be called only after a successful call to `updateDecl`. + pub fn updateDeclLineNumber(self: *Elf, module: *Module, decl: *const Module.Decl) !void { + const tracy = trace(@src()); + defer tracy.end(); + + const scope_file = decl.scope.cast(Module.Scope.File).?; + const tree = scope_file.contents.tree; + const file_ast_decls = tree.root_node.decls(); + // TODO Look into improving the performance here by adding a token-index-to-line + // lookup table. Currently this involves scanning over the source code for newlines. + const fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?; + const block = fn_proto.body().?.castTag(.Block).?; + const line_delta = std.zig.lineDelta(tree.source, 0, tree.token_locs[block.lbrace].start); + const casted_line_off = @intCast(u28, line_delta); + + const shdr = &self.sections.items[self.debug_line_section_index.?]; + const file_pos = shdr.sh_offset + decl.fn_link.off + self.getRelocDbgLineOff(); + var data: [4]u8 = undefined; + leb128.writeUnsignedFixed(4, &data, casted_line_off); + try self.file.?.pwriteAll(&data, file_pos); + } + pub fn deleteExport(self: *Elf, exp: Export) void { const sym_index = exp.sym_index orelse return; self.global_symbol_free_list.append(self.allocator, sym_index) catch {}; From 7d70774fde694b343c90ce8830c269bda23afe84 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 4 Aug 2020 01:38:53 -0700 Subject: [PATCH 18/21] stage2: fix memory leak with exported decl name --- src-self-hosted/Module.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/src-self-hosted/Module.zig b/src-self-hosted/Module.zig index 5a260c4cd2..bda2b3205f 100644 --- a/src-self-hosted/Module.zig +++ b/src-self-hosted/Module.zig @@ -896,6 +896,7 @@ pub fn deinit(self: *Module) void { fn freeExportList(gpa: *Allocator, export_list: []*Export) void { for (export_list) |exp| { + gpa.free(exp.options.name); gpa.destroy(exp); } gpa.free(export_list); From ca19c42b74e94b24f82fca95c5411557b29e1809 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 4 Aug 2020 01:39:03 -0700 Subject: [PATCH 19/21] stage2: fix updating debug line info not resizing properly --- src-self-hosted/link.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index c3f5146113..db879e2732 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1960,6 +1960,7 @@ pub const File = struct { prev.next = src_fn.next; } next.prev = src_fn.prev; + src_fn.next = null; // Populate where it used to be with NOPs. const file_pos = debug_line_sect.sh_offset + src_fn.off; try self.pwriteWithNops(0, &[0]u8{}, src_fn.len, file_pos); From b7a883b7d1247e9342805a8d06e70fd4d02f11b3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 4 Aug 2020 01:41:22 -0700 Subject: [PATCH 20/21] stage2: link: fix not freeing debug line free list --- src-self-hosted/link.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index db879e2732..ee8322350d 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -552,6 +552,7 @@ pub const File = struct { self.local_symbol_free_list.deinit(self.allocator); self.offset_table_free_list.deinit(self.allocator); self.text_block_free_list.deinit(self.allocator); + self.dbg_line_fn_free_list.deinit(self.allocator); self.offset_table.deinit(self.allocator); if (self.owns_file_handle) { if (self.file) |f| f.close(); From 331f6a07a98206c3b5c096e73860ef1b7a3dfe85 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 4 Aug 2020 02:02:23 -0700 Subject: [PATCH 21/21] stage2: fix ZIR support and C back end --- src-self-hosted/codegen.zig | 27 ++++++++++++++++++--------- src-self-hosted/codegen/c.zig | 16 +++++++++++----- src-self-hosted/link.zig | 29 +++++++++++++++++++---------- 3 files changed, 48 insertions(+), 24 deletions(-) diff --git a/src-self-hosted/codegen.zig b/src-self-hosted/codegen.zig index 4bd126402b..7c2d8ca896 100644 --- a/src-self-hosted/codegen.zig +++ b/src-self-hosted/codegen.zig @@ -396,12 +396,21 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { const branch = try branch_stack.addOne(); branch.* = .{}; - const scope_file = module_fn.owner_decl.scope.cast(Module.Scope.File).?; - const tree = scope_file.contents.tree; - const fn_proto = tree.root_node.decls()[module_fn.owner_decl.src_index].castTag(.FnProto).?; - const block = fn_proto.body().?.castTag(.Block).?; - const lbrace_src = tree.token_locs[block.lbrace].start; - const rbrace_src = tree.token_locs[block.rbrace].start; + const src_data: struct {lbrace_src: usize, rbrace_src: usize, source: []const u8} = blk: { + if (module_fn.owner_decl.scope.cast(Module.Scope.File)) |scope_file| { + const tree = scope_file.contents.tree; + const fn_proto = tree.root_node.decls()[module_fn.owner_decl.src_index].castTag(.FnProto).?; + const block = fn_proto.body().?.castTag(.Block).?; + const lbrace_src = tree.token_locs[block.lbrace].start; + const rbrace_src = tree.token_locs[block.rbrace].start; + break :blk .{ .lbrace_src = lbrace_src, .rbrace_src = rbrace_src, .source = tree.source }; + } else if (module_fn.owner_decl.scope.cast(Module.Scope.ZIRModule)) |zir_module| { + const byte_off = zir_module.contents.module.decls[module_fn.owner_decl.src_index].inst.src; + break :blk .{ .lbrace_src = byte_off, .rbrace_src = byte_off, .source = zir_module.source.bytes }; + } else { + unreachable; + } + }; var function = Self{ .gpa = bin_file.allocator, @@ -419,9 +428,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .src = src, .stack_align = undefined, .prev_di_pc = 0, - .prev_di_src = lbrace_src, - .rbrace_src = rbrace_src, - .source = tree.source, + .prev_di_src = src_data.lbrace_src, + .rbrace_src = src_data.rbrace_src, + .source = src_data.source, }; defer function.exitlude_jump_relocs.deinit(bin_file.allocator); diff --git a/src-self-hosted/codegen/c.zig b/src-self-hosted/codegen/c.zig index 509491677a..e8fe435af1 100644 --- a/src-self-hosted/codegen/c.zig +++ b/src-self-hosted/codegen/c.zig @@ -89,17 +89,17 @@ fn genFn(file: *C, decl: *Decl) !void { const func: *Module.Fn = tv.val.cast(Value.Payload.Function).?.func; const instructions = func.analysis.success.instructions; if (instructions.len > 0) { + try writer.writeAll("\n"); for (instructions) |inst| { - try writer.writeAll("\n "); switch (inst.tag) { .assembly => try genAsm(file, inst.castTag(.assembly).?, decl), .call => try genCall(file, inst.castTag(.call).?, decl), .ret => try genRet(file, inst.castTag(.ret).?, decl, tv.ty.fnReturnType()), - .retvoid => try file.main.writer().print("return;", .{}), + .retvoid => try file.main.writer().print(" return;\n", .{}), + .dbg_stmt => try genDbgStmt(file, inst.castTag(.dbg_stmt).?, decl), else => |e| return file.fail(decl.src(), "TODO implement C codegen for {}", .{e}), } } - try writer.writeAll("\n"); } try writer.writeAll("}\n\n"); @@ -112,6 +112,7 @@ fn genRet(file: *C, inst: *Inst.UnOp, decl: *Decl, expected_return_type: Type) ! fn genCall(file: *C, inst: *Inst.Call, decl: *Decl) !void { const writer = file.main.writer(); const header = file.header.writer(); + try writer.writeAll(" "); if (inst.func.castTag(.constant)) |func_inst| { if (func_inst.val.cast(Value.Payload.Function)) |func_val| { const target = func_val.func.owner_decl; @@ -126,7 +127,7 @@ fn genCall(file: *C, inst: *Inst.Call, decl: *Decl) !void { try renderFunctionSignature(file, header, target); try header.writeAll(";\n"); } - try writer.print("{}();", .{tname}); + try writer.print("{}();\n", .{tname}); } else { return file.fail(decl.src(), "TODO non-function call target?", .{}); } @@ -138,8 +139,13 @@ fn genCall(file: *C, inst: *Inst.Call, decl: *Decl) !void { } } +fn genDbgStmt(file: *C, inst: *Inst.NoOp, decl: *Decl) !void { + // TODO emit #line directive here with line number and filename +} + fn genAsm(file: *C, as: *Inst.Assembly, decl: *Decl) !void { const writer = file.main.writer(); + try writer.writeAll(" "); for (as.inputs) |i, index| { if (i[0] == '{' and i[i.len - 1] == '}') { const reg = i[1 .. i.len - 1]; @@ -187,5 +193,5 @@ fn genAsm(file: *C, as: *Inst.Assembly, decl: *Decl) !void { } } } - try writer.writeAll(");"); + try writer.writeAll(");\n"); } diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index ee8322350d..7bf83d7576 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -1825,15 +1825,24 @@ pub const File = struct { // For functions we need to add a prologue to the debug line program. try dbg_line_buffer.ensureCapacity(26); - const scope_file = decl.scope.cast(Module.Scope.File).?; - const tree = scope_file.contents.tree; - const file_ast_decls = tree.root_node.decls(); - // TODO Look into improving the performance here by adding a token-index-to-line - // lookup table. Currently this involves scanning over the source code for newlines. - const fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?; - const block = fn_proto.body().?.castTag(.Block).?; - const line_delta = std.zig.lineDelta(tree.source, 0, tree.token_locs[block.lbrace].start); - const casted_line_off = @intCast(u28, line_delta); + const line_off: u28 = blk: { + if (decl.scope.cast(Module.Scope.File)) |scope_file| { + const tree = scope_file.contents.tree; + const file_ast_decls = tree.root_node.decls(); + // TODO Look into improving the performance here by adding a token-index-to-line + // lookup table. Currently this involves scanning over the source code for newlines. + const fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?; + const block = fn_proto.body().?.castTag(.Block).?; + const line_delta = std.zig.lineDelta(tree.source, 0, tree.token_locs[block.lbrace].start); + break :blk @intCast(u28, line_delta); + } else if (decl.scope.cast(Module.Scope.ZIRModule)) |zir_module| { + const byte_off = zir_module.contents.module.decls[decl.src_index].inst.src; + const line_delta = std.zig.lineDelta(zir_module.source.bytes, 0, byte_off); + break :blk @intCast(u28, line_delta); + } else { + unreachable; + } + }; const ptr_width_bytes = self.ptrWidthBytes(); dbg_line_buffer.appendSliceAssumeCapacity(&[_]u8{ @@ -1850,7 +1859,7 @@ pub const File = struct { // to this function's begin curly. assert(self.getRelocDbgLineOff() == dbg_line_buffer.items.len); // Here we use a ULEB128-fixed-4 to make sure this field can be overwritten later. - leb128.writeUnsignedFixed(4, dbg_line_buffer.addManyAsArrayAssumeCapacity(4), casted_line_off); + leb128.writeUnsignedFixed(4, dbg_line_buffer.addManyAsArrayAssumeCapacity(4), line_off); dbg_line_buffer.appendAssumeCapacity(DW.LNS_set_file); assert(self.getRelocDbgFileIndex() == dbg_line_buffer.items.len);