From 62c282ba4699bd00d0cb74b3253bac7a925eae68 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 24 Sep 2024 10:46:39 +0200 Subject: [PATCH 1/9] elf: do not re-create synthetic sections if already created --- src/link/Elf.zig | 189 ++++++++++++++++++--------------- src/link/Elf/LinkerDefined.zig | 1 + 2 files changed, 107 insertions(+), 83 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index ba3abcf330..8e09c7832d 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -147,6 +147,7 @@ thunks: std.ArrayListUnmanaged(Thunk) = .empty, /// List of output merge sections with deduped contents. merge_sections: std.ArrayListUnmanaged(MergeSection) = .empty, +comment_merge_section_index: ?MergeSection.Index = null, first_eflags: ?elf.Elf64_Word = null, @@ -2790,6 +2791,7 @@ fn checkDuplicates(self: *Elf) !void { pub fn addCommentString(self: *Elf) !void { const gpa = self.base.comp.gpa; + if (self.comment_merge_section_index != null) return; const msec_index = try self.getOrCreateMergeSection(".comment", elf.SHF_MERGE | elf.SHF_STRINGS, elf.SHT_PROGBITS); const msec = self.mergeSection(msec_index); const res = try msec.insertZ(gpa, "zig " ++ builtin.zig_version_string); @@ -2803,6 +2805,7 @@ pub fn addCommentString(self: *Elf) !void { msub.entsize = 1; msub.alive = true; res.sub.* = msub_index; + self.comment_merge_section_index = msec_index; } pub fn resolveMergeSections(self: *Elf) !void { @@ -2920,7 +2923,7 @@ fn initSyntheticSections(self: *Elf) !void { .offset = std.math.maxInt(u64), }); } - if (comp.link_eh_frame_hdr) { + if (comp.link_eh_frame_hdr and self.eh_frame_hdr_section_index == null) { self.eh_frame_hdr_section_index = try self.addSection(.{ .name = try self.insertShString(".eh_frame_hdr"), .type = elf.SHT_PROGBITS, @@ -2931,7 +2934,7 @@ fn initSyntheticSections(self: *Elf) !void { } } - if (self.got.entries.items.len > 0) { + if (self.got.entries.items.len > 0 and self.got_section_index == null) { self.got_section_index = try self.addSection(.{ .name = try self.insertShString(".got"), .type = elf.SHT_PROGBITS, @@ -2941,13 +2944,15 @@ fn initSyntheticSections(self: *Elf) !void { }); } - self.got_plt_section_index = try self.addSection(.{ - .name = try self.insertShString(".got.plt"), - .type = elf.SHT_PROGBITS, - .flags = elf.SHF_ALLOC | elf.SHF_WRITE, - .addralign = @alignOf(u64), - .offset = std.math.maxInt(u64), - }); + if (self.got_plt_section_index == null) { + self.got_plt_section_index = try self.addSection(.{ + .name = try self.insertShString(".got.plt"), + .type = elf.SHT_PROGBITS, + .flags = elf.SHF_ALLOC | elf.SHF_WRITE, + .addralign = @alignOf(u64), + .offset = std.math.maxInt(u64), + }); + } const needs_rela_dyn = blk: { if (self.got.flags.needs_rela or self.got.flags.needs_tlsld or self.copy_rel.symbols.items.len > 0) @@ -2960,7 +2965,7 @@ fn initSyntheticSections(self: *Elf) !void { } break :blk false; }; - if (needs_rela_dyn) { + if (needs_rela_dyn and self.rela_dyn_section_index == null) { self.rela_dyn_section_index = try self.addSection(.{ .name = try self.insertShString(".rela.dyn"), .type = elf.SHT_RELA, @@ -2972,24 +2977,28 @@ fn initSyntheticSections(self: *Elf) !void { } if (self.plt.symbols.items.len > 0) { - self.plt_section_index = try self.addSection(.{ - .name = try self.insertShString(".plt"), - .type = elf.SHT_PROGBITS, - .flags = elf.SHF_ALLOC | elf.SHF_EXECINSTR, - .addralign = 16, - .offset = std.math.maxInt(u64), - }); - self.rela_plt_section_index = try self.addSection(.{ - .name = try self.insertShString(".rela.plt"), - .type = elf.SHT_RELA, - .flags = elf.SHF_ALLOC, - .addralign = @alignOf(elf.Elf64_Rela), - .entsize = @sizeOf(elf.Elf64_Rela), - .offset = std.math.maxInt(u64), - }); + if (self.plt_section_index == null) { + self.plt_section_index = try self.addSection(.{ + .name = try self.insertShString(".plt"), + .type = elf.SHT_PROGBITS, + .flags = elf.SHF_ALLOC | elf.SHF_EXECINSTR, + .addralign = 16, + .offset = std.math.maxInt(u64), + }); + } + if (self.rela_plt_section_index == null) { + self.rela_plt_section_index = try self.addSection(.{ + .name = try self.insertShString(".rela.plt"), + .type = elf.SHT_RELA, + .flags = elf.SHF_ALLOC, + .addralign = @alignOf(elf.Elf64_Rela), + .entsize = @sizeOf(elf.Elf64_Rela), + .offset = std.math.maxInt(u64), + }); + } } - if (self.plt_got.symbols.items.len > 0) { + if (self.plt_got.symbols.items.len > 0 and self.plt_got_section_index == null) { self.plt_got_section_index = try self.addSection(.{ .name = try self.insertShString(".plt.got"), .type = elf.SHT_PROGBITS, @@ -2999,7 +3008,7 @@ fn initSyntheticSections(self: *Elf) !void { }); } - if (self.copy_rel.symbols.items.len > 0) { + if (self.copy_rel.symbols.items.len > 0 and self.copy_rel_section_index == null) { self.copy_rel_section_index = try self.addSection(.{ .name = try self.insertShString(".copyrel"), .type = elf.SHT_NOBITS, @@ -3017,7 +3026,7 @@ fn initSyntheticSections(self: *Elf) !void { if (self.base.isStatic() and !comp.config.pie) break :blk false; break :blk target.dynamic_linker.get() != null; }; - if (needs_interp) { + if (needs_interp and self.interp_section_index == null) { self.interp_section_index = try self.addSection(.{ .name = try self.insertShString(".interp"), .type = elf.SHT_PROGBITS, @@ -3028,67 +3037,81 @@ fn initSyntheticSections(self: *Elf) !void { } if (self.isEffectivelyDynLib() or self.shared_objects.items.len > 0 or comp.config.pie) { - self.dynstrtab_section_index = try self.addSection(.{ - .name = try self.insertShString(".dynstr"), - .flags = elf.SHF_ALLOC, - .type = elf.SHT_STRTAB, - .entsize = 1, - .addralign = 1, - .offset = std.math.maxInt(u64), - }); - self.dynamic_section_index = try self.addSection(.{ - .name = try self.insertShString(".dynamic"), - .flags = elf.SHF_ALLOC | elf.SHF_WRITE, - .type = elf.SHT_DYNAMIC, - .entsize = @sizeOf(elf.Elf64_Dyn), - .addralign = @alignOf(elf.Elf64_Dyn), - .offset = std.math.maxInt(u64), - }); - self.dynsymtab_section_index = try self.addSection(.{ - .name = try self.insertShString(".dynsym"), - .flags = elf.SHF_ALLOC, - .type = elf.SHT_DYNSYM, - .addralign = @alignOf(elf.Elf64_Sym), - .entsize = @sizeOf(elf.Elf64_Sym), - .info = 1, - .offset = std.math.maxInt(u64), - }); - self.hash_section_index = try self.addSection(.{ - .name = try self.insertShString(".hash"), - .flags = elf.SHF_ALLOC, - .type = elf.SHT_HASH, - .addralign = 4, - .entsize = 4, - .offset = std.math.maxInt(u64), - }); - self.gnu_hash_section_index = try self.addSection(.{ - .name = try self.insertShString(".gnu.hash"), - .flags = elf.SHF_ALLOC, - .type = elf.SHT_GNU_HASH, - .addralign = 8, - .offset = std.math.maxInt(u64), - }); + if (self.dynstrtab_section_index == null) { + self.dynstrtab_section_index = try self.addSection(.{ + .name = try self.insertShString(".dynstr"), + .flags = elf.SHF_ALLOC, + .type = elf.SHT_STRTAB, + .entsize = 1, + .addralign = 1, + .offset = std.math.maxInt(u64), + }); + } + if (self.dynamic_section_index == null) { + self.dynamic_section_index = try self.addSection(.{ + .name = try self.insertShString(".dynamic"), + .flags = elf.SHF_ALLOC | elf.SHF_WRITE, + .type = elf.SHT_DYNAMIC, + .entsize = @sizeOf(elf.Elf64_Dyn), + .addralign = @alignOf(elf.Elf64_Dyn), + .offset = std.math.maxInt(u64), + }); + } + if (self.dynsymtab_section_index == null) { + self.dynsymtab_section_index = try self.addSection(.{ + .name = try self.insertShString(".dynsym"), + .flags = elf.SHF_ALLOC, + .type = elf.SHT_DYNSYM, + .addralign = @alignOf(elf.Elf64_Sym), + .entsize = @sizeOf(elf.Elf64_Sym), + .info = 1, + .offset = std.math.maxInt(u64), + }); + } + if (self.hash_section_index == null) { + self.hash_section_index = try self.addSection(.{ + .name = try self.insertShString(".hash"), + .flags = elf.SHF_ALLOC, + .type = elf.SHT_HASH, + .addralign = 4, + .entsize = 4, + .offset = std.math.maxInt(u64), + }); + } + if (self.gnu_hash_section_index == null) { + self.gnu_hash_section_index = try self.addSection(.{ + .name = try self.insertShString(".gnu.hash"), + .flags = elf.SHF_ALLOC, + .type = elf.SHT_GNU_HASH, + .addralign = 8, + .offset = std.math.maxInt(u64), + }); + } const needs_versions = for (self.dynsym.entries.items) |entry| { const sym = self.symbol(entry.ref).?; if (sym.flags.import and sym.version_index & elf.VERSYM_VERSION > elf.VER_NDX_GLOBAL) break true; } else false; if (needs_versions) { - self.versym_section_index = try self.addSection(.{ - .name = try self.insertShString(".gnu.version"), - .flags = elf.SHF_ALLOC, - .type = elf.SHT_GNU_VERSYM, - .addralign = @alignOf(elf.Elf64_Versym), - .entsize = @sizeOf(elf.Elf64_Versym), - .offset = std.math.maxInt(u64), - }); - self.verneed_section_index = try self.addSection(.{ - .name = try self.insertShString(".gnu.version_r"), - .flags = elf.SHF_ALLOC, - .type = elf.SHT_GNU_VERNEED, - .addralign = @alignOf(elf.Elf64_Verneed), - .offset = std.math.maxInt(u64), - }); + if (self.versym_section_index == null) { + self.versym_section_index = try self.addSection(.{ + .name = try self.insertShString(".gnu.version"), + .flags = elf.SHF_ALLOC, + .type = elf.SHT_GNU_VERSYM, + .addralign = @alignOf(elf.Elf64_Versym), + .entsize = @sizeOf(elf.Elf64_Versym), + .offset = std.math.maxInt(u64), + }); + } + if (self.verneed_section_index == null) { + self.verneed_section_index = try self.addSection(.{ + .name = try self.insertShString(".gnu.version_r"), + .flags = elf.SHF_ALLOC, + .type = elf.SHT_GNU_VERNEED, + .addralign = @alignOf(elf.Elf64_Verneed), + .offset = std.math.maxInt(u64), + }); + } } } diff --git a/src/link/Elf/LinkerDefined.zig b/src/link/Elf/LinkerDefined.zig index 60f9482efb..9515dfcd43 100644 --- a/src/link/Elf/LinkerDefined.zig +++ b/src/link/Elf/LinkerDefined.zig @@ -145,6 +145,7 @@ pub fn initStartStopSymbols(self: *LinkerDefined, elf_file: *Elf) !void { try self.symbols_resolver.ensureUnusedCapacity(gpa, nsyms); for (slice.items(.shdr)) |shdr| { + // TODO use getOrPut for incremental so that we don't create duplicates if (elf_file.getStartStopBasename(shdr)) |name| { const start_name = try std.fmt.allocPrintZ(gpa, "__start_{s}", .{name}); defer gpa.free(start_name); From db3db1150e48cf506593489dff93adda9e1111fe Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 24 Sep 2024 10:56:48 +0200 Subject: [PATCH 2/9] elf: do not re-create special program headers if already created --- src/link/Elf.zig | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 8e09c7832d..1ffde07b7a 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -3159,36 +3159,38 @@ pub fn initShStrtab(self: *Elf) !void { fn initSpecialPhdrs(self: *Elf) !void { comptime assert(max_number_of_special_phdrs == 5); - if (self.interp_section_index != null) { + if (self.interp_section_index != null and self.phdr_interp_index == null) { self.phdr_interp_index = try self.addPhdr(.{ .type = elf.PT_INTERP, .flags = elf.PF_R, .@"align" = 1, }); } - if (self.dynamic_section_index != null) { + if (self.dynamic_section_index != null and self.phdr_dynamic_index == null) { self.phdr_dynamic_index = try self.addPhdr(.{ .type = elf.PT_DYNAMIC, .flags = elf.PF_R | elf.PF_W, }); } - if (self.eh_frame_hdr_section_index != null) { + if (self.eh_frame_hdr_section_index != null and self.phdr_gnu_eh_frame_index == null) { self.phdr_gnu_eh_frame_index = try self.addPhdr(.{ .type = elf.PT_GNU_EH_FRAME, .flags = elf.PF_R, }); } - self.phdr_gnu_stack_index = try self.addPhdr(.{ - .type = elf.PT_GNU_STACK, - .flags = elf.PF_W | elf.PF_R, - .memsz = self.base.stack_size, - .@"align" = 1, - }); + if (self.phdr_gnu_stack_index == null) { + self.phdr_gnu_stack_index = try self.addPhdr(.{ + .type = elf.PT_GNU_STACK, + .flags = elf.PF_W | elf.PF_R, + .memsz = self.base.stack_size, + .@"align" = 1, + }); + } const has_tls = for (self.sections.items(.shdr)) |shdr| { if (shdr.sh_flags & elf.SHF_TLS != 0) break true; } else false; - if (has_tls) { + if (has_tls and self.phdr_tls_index == null) { self.phdr_tls_index = try self.addPhdr(.{ .type = elf.PT_TLS, .flags = elf.PF_R, From 82cf762b0205f036677d234abb35bf1deafb5c7e Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 24 Sep 2024 11:09:17 +0200 Subject: [PATCH 3/9] elf: reset (merge) sections sizes before updating --- src/link/Elf/AtomList.zig | 3 +++ src/link/Elf/merge_section.zig | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/link/Elf/AtomList.zig b/src/link/Elf/AtomList.zig index 5fc1893376..dfabbe0ff7 100644 --- a/src/link/Elf/AtomList.zig +++ b/src/link/Elf/AtomList.zig @@ -19,6 +19,9 @@ pub fn offset(list: AtomList, elf_file: *Elf) u64 { } pub fn updateSize(list: *AtomList, elf_file: *Elf) void { + // TODO perhaps a 'stale' flag would be better here? + list.size = 0; + list.alignment = .@"1"; for (list.atoms.items) |ref| { const atom_ptr = elf_file.atom(ref).?; assert(atom_ptr.alive); diff --git a/src/link/Elf/merge_section.zig b/src/link/Elf/merge_section.zig index cf6506a9ea..dc6239d738 100644 --- a/src/link/Elf/merge_section.zig +++ b/src/link/Elf/merge_section.zig @@ -95,6 +95,10 @@ pub const MergeSection = struct { } pub fn updateSize(msec: *MergeSection) void { + // TODO a 'stale' flag would be better here perhaps? + msec.size = 0; + msec.alignment = .@"1"; + msec.entsize = 0; for (msec.finalized_subsections.items) |msub_index| { const msub = msec.mergeSubsection(msub_index); assert(msub.alive); From 992c6c3f68dba24c1ca92cd7e78716d2fd336495 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 24 Sep 2024 11:45:01 +0200 Subject: [PATCH 4/9] elf: reset output symtab contexts before re-updating --- src/link/Elf.zig | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 1ffde07b7a..62d6a521e6 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -4127,6 +4127,7 @@ pub fn updateSymtabSize(self: *Elf) !void { nlocals += @intCast(self.sections.slice().len); if (self.requiresThunks()) for (self.thunks.items) |*th| { + th.output_symtab_ctx.reset(); th.output_symtab_ctx.ilocal = nlocals; th.calcSymtabSize(self); nlocals += th.output_symtab_ctx.nlocals; @@ -4138,6 +4139,7 @@ pub fn updateSymtabSize(self: *Elf) !void { const ctx = switch (file_ptr) { inline else => |x| &x.output_symtab_ctx, }; + ctx.reset(); ctx.ilocal = nlocals; ctx.iglobal = nglobals; try file_ptr.updateSymtabSize(self); @@ -4147,6 +4149,7 @@ pub fn updateSymtabSize(self: *Elf) !void { } if (self.got_section_index) |_| { + self.got.output_symtab_ctx.reset(); self.got.output_symtab_ctx.ilocal = nlocals; self.got.updateSymtabSize(self); nlocals += self.got.output_symtab_ctx.nlocals; @@ -4154,6 +4157,7 @@ pub fn updateSymtabSize(self: *Elf) !void { } if (self.plt_section_index) |_| { + self.plt.output_symtab_ctx.reset(); self.plt.output_symtab_ctx.ilocal = nlocals; self.plt.updateSymtabSize(self); nlocals += self.plt.output_symtab_ctx.nlocals; @@ -4161,6 +4165,7 @@ pub fn updateSymtabSize(self: *Elf) !void { } if (self.plt_got_section_index) |_| { + self.plt_got.output_symtab_ctx.reset(); self.plt_got.output_symtab_ctx.ilocal = nlocals; self.plt_got.updateSymtabSize(self); nlocals += self.plt_got.output_symtab_ctx.nlocals; @@ -4359,6 +4364,9 @@ pub fn writeSymtab(self: *Elf) !void { try self.symtab.resize(gpa, nsyms); const needed_strtab_size = math.cast(usize, strtab_shdr.sh_size - 1) orelse return error.Overflow; + // TODO we could resize instead and in ZigObject/Object always access as slice + self.strtab.clearRetainingCapacity(); + self.strtab.appendAssumeCapacity(0); try self.strtab.ensureUnusedCapacity(gpa, needed_strtab_size); for (slice.items(.shdr), 0..) |shdr, shndx| { @@ -5330,7 +5338,7 @@ fn fmtDumpState( { try writer.writeAll("atom lists\n"); for (slice.items(.shdr), slice.items(.atom_list_2), 0..) |shdr, atom_list, shndx| { - try writer.print("shdr({d}) : {s} : {}", .{ shndx, self.getShString(shdr.sh_name), atom_list.fmt(self) }); + try writer.print("shdr({d}) : {s} : {}\n", .{ shndx, self.getShString(shdr.sh_name), atom_list.fmt(self) }); } } @@ -5462,6 +5470,14 @@ pub const SymtabCtx = struct { nlocals: u32 = 0, nglobals: u32 = 0, strsize: u32 = 0, + + pub fn reset(ctx: *SymtabCtx) void { + ctx.ilocal = 0; + ctx.iglobal = 0; + ctx.nlocals = 0; + ctx.nglobals = 0; + ctx.strsize = 0; + } }; pub const null_sym = elf.Elf64_Sym{ From 10c68d05dbc2ff586045722c48dbf6b892587673 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 24 Sep 2024 13:07:58 +0200 Subject: [PATCH 5/9] elf: change how we create new program headers We do not want to create additional ones per update that are duplicates of existing ones. --- src/link/Elf.zig | 51 ++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 62d6a521e6..14da54d0f0 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1048,6 +1048,7 @@ pub fn flushModule(self: *Elf, arena: Allocator, tid: Zcu.PerThread.Id, prog_nod try self.updateMergeSectionSizes(); try self.updateSectionSizes(); + try self.addLoadPhdrs(); try self.allocatePhdrTable(); try self.allocateAllocSections(); try self.sortPhdrs(); @@ -3729,27 +3730,19 @@ fn getMaxNumberOfPhdrs() u64 { return num; } -/// Calculates how many segments (PT_LOAD progam headers) are required -/// to cover the set of sections. -/// We permit a maximum of 3**2 number of segments. -fn calcNumberOfSegments(self: *Elf) usize { - var covers: [9]bool = [_]bool{false} ** 9; +fn addLoadPhdrs(self: *Elf) error{OutOfMemory}!void { for (self.sections.items(.shdr)) |shdr| { if (shdr.sh_type == elf.SHT_NULL) continue; if (shdr.sh_flags & elf.SHF_ALLOC == 0) continue; const flags = shdrToPhdrFlags(shdr.sh_flags); - covers[flags - 1] = true; + if (self.getPhdr(.{ .flags = flags, .type = elf.PT_LOAD }) == null) { + _ = try self.addPhdr(.{ .flags = flags, .type = elf.PT_LOAD }); + } } - var count: usize = 0; - for (covers) |cover| { - if (cover) count += 1; - } - return count; } /// Allocates PHDR table in virtual memory and in file. fn allocatePhdrTable(self: *Elf) error{OutOfMemory}!void { - const new_load_segments = self.calcNumberOfSegments(); const phdr_table = &self.phdrs.items[self.phdr_table_index.?]; const phdr_table_load = &self.phdrs.items[self.phdr_table_load_index.?]; @@ -3761,7 +3754,7 @@ fn allocatePhdrTable(self: *Elf) error{OutOfMemory}!void { .p32 => @sizeOf(elf.Elf32_Phdr), .p64 => @sizeOf(elf.Elf64_Phdr), }; - const needed_size = (self.phdrs.items.len + new_load_segments) * phsize; + const needed_size = self.phdrs.items.len * phsize; const available_space = self.allocatedSize(phdr_table.p_offset); if (needed_size > available_space) { @@ -3904,15 +3897,14 @@ pub fn allocateAllocSections(self: *Elf) !void { const first = slice.items(.shdr)[cover.items[0]]; var new_offset = try self.findFreeSpace(filesz, @"align"); - const phndx = try self.addPhdr(.{ - .type = elf.PT_LOAD, - .offset = new_offset, - .addr = first.sh_addr, - .memsz = memsz, - .filesz = filesz, - .@"align" = @"align", - .flags = shdrToPhdrFlags(first.sh_flags), - }); + const phndx = self.getPhdr(.{ .type = elf.PT_LOAD, .flags = shdrToPhdrFlags(first.sh_flags) }).?; + const phdr = &self.phdrs.items[phndx]; + phdr.p_offset = new_offset; + phdr.p_vaddr = first.sh_addr; + phdr.p_paddr = first.sh_addr; + phdr.p_memsz = memsz; + phdr.p_filesz = filesz; + phdr.p_align = @"align"; for (cover.items) |shndx| { const shdr = &slice.items(.shdr)[shndx]; @@ -4747,7 +4739,20 @@ pub fn isEffectivelyDynLib(self: Elf) bool { }; } -pub fn addPhdr(self: *Elf, opts: struct { +fn getPhdr(self: *Elf, opts: struct { + type: u32 = 0, + flags: u32 = 0, +}) ?u16 { + for (self.phdrs.items, 0..) |phdr, phndx| { + if (self.phdr_table_load_index) |index| { + if (phndx == index) continue; + } + if (phdr.p_type == opts.type and phdr.p_flags == opts.flags) return @intCast(phndx); + } + return null; +} + +fn addPhdr(self: *Elf, opts: struct { type: u32 = 0, flags: u32 = 0, @"align": u64 = 0, From 50074a835a60736eae7aad7680654096ef3e50af Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 25 Sep 2024 13:55:32 +0200 Subject: [PATCH 6/9] elf: remove unused code --- src/link/Elf.zig | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 14da54d0f0..f610f2519d 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -569,19 +569,6 @@ pub fn growAllocSection(self: *Elf, shdr_index: u32, needed_size: u64, min_align if (maybe_phdr) |phdr| phdr.p_filesz = needed_size; } shdr.sh_size = needed_size; - - if (maybe_phdr) |phdr| { - const mem_capacity = self.allocatedVirtualSize(phdr.p_vaddr); - if (needed_size > mem_capacity) { - var err = try self.base.addErrorWithNotes(2); - try err.addMsg("fatal linker error: cannot expand load segment phdr({d}) in virtual memory", .{phndx.?}); - try err.addNote("TODO: emit relocations to memory locations in self-hosted backends", .{}); - try err.addNote("as a workaround, try increasing pre-allocated virtual memory of each segment", .{}); - } - - phdr.p_memsz = needed_size; - } - self.markDirty(shdr_index); } From 1c2048ebcc810490cc5eb5168589edafe727fec0 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 25 Sep 2024 16:31:56 +0200 Subject: [PATCH 7/9] incr-check: add --debug-link flag for verbose linker logging --- tools/incr-check.zig | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/incr-check.zig b/tools/incr-check.zig index eafe6e8c04..3266163ca0 100644 --- a/tools/incr-check.zig +++ b/tools/incr-check.zig @@ -3,7 +3,7 @@ const fatal = std.process.fatal; const Allocator = std.mem.Allocator; const Cache = std.Build.Cache; -const usage = "usage: incr-check [--zig-lib-dir lib] [--debug-zcu] [--emit none|bin|c] [--zig-cc-binary /path/to/zig]"; +const usage = "usage: incr-check [--zig-lib-dir lib] [--debug-zcu] [--debug-link] [--emit none|bin|c] [--zig-cc-binary /path/to/zig]"; const EmitMode = enum { none, @@ -22,6 +22,7 @@ pub fn main() !void { var opt_cc_zig: ?[]const u8 = null; var emit: EmitMode = .bin; var debug_zcu = false; + var debug_link = false; var arg_it = try std.process.argsWithAllocator(arena); _ = arg_it.skip(); @@ -35,6 +36,8 @@ pub fn main() !void { opt_lib_dir = arg_it.next() orelse fatal("expected arg after '--zig-lib-dir'\n{s}", .{usage}); } else if (std.mem.eql(u8, arg, "--debug-zcu")) { debug_zcu = true; + } else if (std.mem.eql(u8, arg, "--debug-link")) { + debug_link = true; } else if (std.mem.eql(u8, arg, "--zig-cc-binary")) { opt_cc_zig = arg_it.next() orelse fatal("expect arg after '--zig-cc-binary'\n{s}", .{usage}); } else { @@ -98,6 +101,11 @@ pub fn main() !void { if (debug_zcu) { try child_args.appendSlice(arena, &.{ "--debug-log", "zcu" }); } + if (debug_link) { + try child_args.appendSlice(arena, &.{ "--debug-log", "link", "--debug-log", "link_state", "--debug-log", "link_relocs" }); + } + + const debug_log_verbose = debug_zcu or debug_link; var child = std.process.Child.init(child_args.items, arena); child.stdin_behavior = .Pipe; @@ -131,7 +139,7 @@ pub fn main() !void { .tmp_dir = tmp_dir, .tmp_dir_path = tmp_dir_path, .child = &child, - .allow_stderr = debug_zcu, + .allow_stderr = debug_log_verbose, .emit = emit, .cc_child_args = &cc_child_args, }; @@ -148,7 +156,7 @@ pub fn main() !void { var update_node = prog_node.start(update.name, 0); defer update_node.end(); - if (debug_zcu) { + if (debug_log_verbose) { std.log.info("=== START UPDATE '{s}' ===", .{update.name}); } From 1714b74edfdf1dbc52f8dd1555abb0ff43dfdff8 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 25 Sep 2024 16:32:23 +0200 Subject: [PATCH 8/9] incr-check: actually handle end message for a compiler update --- tools/incr-check.zig | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/incr-check.zig b/tools/incr-check.zig index 3266163ca0..3a833c416e 100644 --- a/tools/incr-check.zig +++ b/tools/incr-check.zig @@ -266,7 +266,6 @@ const Eval = struct { try eval.checkSuccessOutcome(update, null, prog_node); // This message indicates the end of the update. stdout.discard(body.len); - return; } const digest = body[@sizeOf(EbpHdr)..][0..Cache.bin_digest_len]; @@ -290,7 +289,6 @@ const Eval = struct { try eval.checkSuccessOutcome(update, bin_path, prog_node); // This message indicates the end of the update. stdout.discard(body.len); - return; }, else => { // Ignore other messages. From 054dbb6798bd81cbea809999b2fdffc1b59e5ca3 Mon Sep 17 00:00:00 2001 From: mlugg Date: Wed, 25 Sep 2024 20:21:27 +0100 Subject: [PATCH 9/9] incr-check: specify backend in manifest, allow multiple targets/backends --- test/incremental/add_decl | 3 +- test/incremental/add_decl_namespaced | 3 +- test/incremental/delete_comptime_decls | 3 +- test/incremental/hello | 3 +- test/incremental/modify_inline_fn | 3 +- test/incremental/move_src | 3 +- test/incremental/remove_enum_field | 3 +- test/incremental/type_becomes_comptime_only | 3 +- test/incremental/unreferenced_error | 3 +- tools/incr-check.zig | 257 +++++++++++--------- 10 files changed, 159 insertions(+), 125 deletions(-) diff --git a/test/incremental/add_decl b/test/incremental/add_decl index 376a725efc..1f9788f481 100644 --- a/test/incremental/add_decl +++ b/test/incremental/add_decl @@ -1,4 +1,5 @@ -#target=x86_64-linux +#target=x86_64-linux-selfhosted +#target=x86_64-linux-cbe #update=initial version #file=main.zig const std = @import("std"); diff --git a/test/incremental/add_decl_namespaced b/test/incremental/add_decl_namespaced index 43123e0d0c..7063e3fa74 100644 --- a/test/incremental/add_decl_namespaced +++ b/test/incremental/add_decl_namespaced @@ -1,4 +1,5 @@ -#target=x86_64-linux +#target=x86_64-linux-selfhosted +#target=x86_64-linux-cbe #update=initial version #file=main.zig const std = @import("std"); diff --git a/test/incremental/delete_comptime_decls b/test/incremental/delete_comptime_decls index d0da7110c3..54411797cf 100644 --- a/test/incremental/delete_comptime_decls +++ b/test/incremental/delete_comptime_decls @@ -1,4 +1,5 @@ -#target=x86_64-linux +#target=x86_64-linux-selfhosted +#target=x86_64-linux-cbe #update=initial version #file=main.zig pub fn main() void {} diff --git a/test/incremental/hello b/test/incremental/hello index cad14d6e3d..9419b3d186 100644 --- a/test/incremental/hello +++ b/test/incremental/hello @@ -1,4 +1,5 @@ -#target=x86_64-linux +#target=x86_64-linux-selfhosted +#target=x86_64-linux-cbe #update=initial version #file=main.zig const std = @import("std"); diff --git a/test/incremental/modify_inline_fn b/test/incremental/modify_inline_fn index 633e7a0728..69ab1f429a 100644 --- a/test/incremental/modify_inline_fn +++ b/test/incremental/modify_inline_fn @@ -1,4 +1,5 @@ -#target=x86_64-linux +#target=x86_64-linux-selfhosted +#target=x86_64-linux-cbe #update=initial version #file=main.zig const std = @import("std"); diff --git a/test/incremental/move_src b/test/incremental/move_src index 8313bdfac2..8a3a24a269 100644 --- a/test/incremental/move_src +++ b/test/incremental/move_src @@ -1,4 +1,5 @@ -#target=x86_64-linux +#target=x86_64-linux-selfhosted +#target=x86_64-linux-cbe #update=initial version #file=main.zig const std = @import("std"); diff --git a/test/incremental/remove_enum_field b/test/incremental/remove_enum_field index 64193bb25b..c163e5e168 100644 --- a/test/incremental/remove_enum_field +++ b/test/incremental/remove_enum_field @@ -1,4 +1,5 @@ -#target=x86_64-linux +#target=x86_64-linux-selfhosted +#target=x86_64-linux-cbe #update=initial version #file=main.zig const MyEnum = enum(u8) { diff --git a/test/incremental/type_becomes_comptime_only b/test/incremental/type_becomes_comptime_only index 2dfa556e87..3878e0c61b 100644 --- a/test/incremental/type_becomes_comptime_only +++ b/test/incremental/type_becomes_comptime_only @@ -1,4 +1,5 @@ -#target=x86_64-linux +#target=x86_64-linux-selfhosted +#target=x86_64-linux-cbe #update=initial version #file=main.zig const SomeType = u32; diff --git a/test/incremental/unreferenced_error b/test/incremental/unreferenced_error index e3d67d6ad8..2031975986 100644 --- a/test/incremental/unreferenced_error +++ b/test/incremental/unreferenced_error @@ -1,4 +1,5 @@ -#target=x86_64-linux +#target=x86_64-linux-selfhosted +#target=x86_64-linux-cbe #update=initial version #file=main.zig const std = @import("std"); diff --git a/tools/incr-check.zig b/tools/incr-check.zig index 3a833c416e..047b129276 100644 --- a/tools/incr-check.zig +++ b/tools/incr-check.zig @@ -3,13 +3,7 @@ const fatal = std.process.fatal; const Allocator = std.mem.Allocator; const Cache = std.Build.Cache; -const usage = "usage: incr-check [--zig-lib-dir lib] [--debug-zcu] [--debug-link] [--emit none|bin|c] [--zig-cc-binary /path/to/zig]"; - -const EmitMode = enum { - none, - bin, - c, -}; +const usage = "usage: incr-check [--zig-lib-dir lib] [--debug-zcu] [--debug-link] [--zig-cc-binary /path/to/zig]"; pub fn main() !void { var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator); @@ -20,7 +14,6 @@ pub fn main() !void { var opt_input_file_name: ?[]const u8 = null; var opt_lib_dir: ?[]const u8 = null; var opt_cc_zig: ?[]const u8 = null; - var emit: EmitMode = .bin; var debug_zcu = false; var debug_link = false; @@ -28,11 +21,7 @@ pub fn main() !void { _ = arg_it.skip(); while (arg_it.next()) |arg| { if (arg.len > 0 and arg[0] == '-') { - if (std.mem.eql(u8, arg, "--emit")) { - const emit_str = arg_it.next() orelse fatal("expected arg after '--emit'\n{s}", .{usage}); - emit = std.meta.stringToEnum(EmitMode, emit_str) orelse - fatal("invalid emit mode '{s}'\n{s}", .{ emit_str, usage }); - } else if (std.mem.eql(u8, arg, "--zig-lib-dir")) { + if (std.mem.eql(u8, arg, "--zig-lib-dir")) { opt_lib_dir = arg_it.next() orelse fatal("expected arg after '--zig-lib-dir'\n{s}", .{usage}); } else if (std.mem.eql(u8, arg, "--debug-zcu")) { debug_zcu = true; @@ -76,109 +65,114 @@ pub fn main() !void { else null; - var child_args: std.ArrayListUnmanaged([]const u8) = .empty; - try child_args.appendSlice(arena, &.{ - resolved_zig_exe, - "build-exe", - case.root_source_file, - "-fincremental", - "-target", - case.target_query, - "--cache-dir", - ".local-cache", - "--global-cache-dir", - ".global_cache", - "--listen=-", - }); - if (opt_resolved_lib_dir) |resolved_lib_dir| { - try child_args.appendSlice(arena, &.{ "--zig-lib-dir", resolved_lib_dir }); - } - switch (emit) { - .bin => try child_args.appendSlice(arena, &.{ "-fno-llvm", "-fno-lld" }), - .none => try child_args.append(arena, "-fno-emit-bin"), - .c => try child_args.appendSlice(arena, &.{ "-ofmt=c", "-lc" }), - } - if (debug_zcu) { - try child_args.appendSlice(arena, &.{ "--debug-log", "zcu" }); - } - if (debug_link) { - try child_args.appendSlice(arena, &.{ "--debug-log", "link", "--debug-log", "link_state", "--debug-log", "link_relocs" }); - } - const debug_log_verbose = debug_zcu or debug_link; - var child = std.process.Child.init(child_args.items, arena); - child.stdin_behavior = .Pipe; - child.stdout_behavior = .Pipe; - child.stderr_behavior = .Pipe; - child.progress_node = child_prog_node; - child.cwd_dir = tmp_dir; - child.cwd = tmp_dir_path; + for (case.targets) |target| { + std.log.scoped(.status).info("target: '{s}-{s}'", .{ target.query, @tagName(target.backend) }); - var cc_child_args: std.ArrayListUnmanaged([]const u8) = .empty; - if (emit == .c) { - const resolved_cc_zig_exe = if (opt_cc_zig) |cc_zig_exe| - try std.fs.path.relative(arena, tmp_dir_path, cc_zig_exe) - else - resolved_zig_exe; - - try cc_child_args.appendSlice(arena, &.{ - resolved_cc_zig_exe, - "cc", + var child_args: std.ArrayListUnmanaged([]const u8) = .empty; + try child_args.appendSlice(arena, &.{ + resolved_zig_exe, + "build-exe", + case.root_source_file, + "-fincremental", "-target", - case.target_query, - "-I", - opt_resolved_lib_dir orelse fatal("'--zig-lib-dir' required when using '--emit c'", .{}), - "-o", + target.query, + "--cache-dir", + ".local-cache", + "--global-cache-dir", + ".global_cache", + "--listen=-", }); - } - - var eval: Eval = .{ - .arena = arena, - .case = case, - .tmp_dir = tmp_dir, - .tmp_dir_path = tmp_dir_path, - .child = &child, - .allow_stderr = debug_log_verbose, - .emit = emit, - .cc_child_args = &cc_child_args, - }; - - try child.spawn(); - - var poller = std.io.poll(arena, Eval.StreamEnum, .{ - .stdout = child.stdout.?, - .stderr = child.stderr.?, - }); - defer poller.deinit(); - - for (case.updates) |update| { - var update_node = prog_node.start(update.name, 0); - defer update_node.end(); - - if (debug_log_verbose) { - std.log.info("=== START UPDATE '{s}' ===", .{update.name}); + if (opt_resolved_lib_dir) |resolved_lib_dir| { + try child_args.appendSlice(arena, &.{ "--zig-lib-dir", resolved_lib_dir }); + } + switch (target.backend) { + .sema => try child_args.append(arena, "-fno-emit-bin"), + .selfhosted => try child_args.appendSlice(arena, &.{ "-fno-llvm", "-fno-lld" }), + .llvm => try child_args.appendSlice(arena, &.{ "-fllvm", "-flld" }), + .cbe => try child_args.appendSlice(arena, &.{ "-ofmt=c", "-lc" }), + } + if (debug_zcu) { + try child_args.appendSlice(arena, &.{ "--debug-log", "zcu" }); + } + if (debug_link) { + try child_args.appendSlice(arena, &.{ "--debug-log", "link", "--debug-log", "link_state", "--debug-log", "link_relocs" }); } - eval.write(update); - try eval.requestUpdate(); - try eval.check(&poller, update, update_node); + var child = std.process.Child.init(child_args.items, arena); + child.stdin_behavior = .Pipe; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; + child.progress_node = child_prog_node; + child.cwd_dir = tmp_dir; + child.cwd = tmp_dir_path; + + var cc_child_args: std.ArrayListUnmanaged([]const u8) = .empty; + if (target.backend == .cbe) { + const resolved_cc_zig_exe = if (opt_cc_zig) |cc_zig_exe| + try std.fs.path.relative(arena, tmp_dir_path, cc_zig_exe) + else + resolved_zig_exe; + + try cc_child_args.appendSlice(arena, &.{ + resolved_cc_zig_exe, + "cc", + "-target", + target.query, + "-I", + opt_resolved_lib_dir orelse fatal("'--zig-lib-dir' required when using backend 'cbe'", .{}), + "-o", + }); + } + + var eval: Eval = .{ + .arena = arena, + .case = case, + .target = target, + .tmp_dir = tmp_dir, + .tmp_dir_path = tmp_dir_path, + .child = &child, + .allow_stderr = debug_log_verbose, + .cc_child_args = &cc_child_args, + }; + + try child.spawn(); + + var poller = std.io.poll(arena, Eval.StreamEnum, .{ + .stdout = child.stdout.?, + .stderr = child.stderr.?, + }); + defer poller.deinit(); + + for (case.updates) |update| { + var update_node = prog_node.start(update.name, 0); + defer update_node.end(); + + if (debug_log_verbose) { + std.log.scoped(.status).info("update: '{s}'", .{update.name}); + } + + eval.write(update); + try eval.requestUpdate(); + try eval.check(&poller, update, update_node); + } + + try eval.end(&poller); + + waitChild(&child); } - - try eval.end(&poller); - - waitChild(&child); } const Eval = struct { arena: Allocator, case: Case, + target: Case.Target, tmp_dir: std.fs.Dir, tmp_dir_path: []const u8, child: *std.process.Child, allow_stderr: bool, - emit: EmitMode, - /// When `emit == .c`, this contains the first few arguments to `zig cc` to build the generated binary. + /// When `target.backend == .cbe`, this contains the first few arguments to `zig cc` to build the generated binary. /// The arguments `out.c in.c` must be appended before spawning the subprocess. cc_child_args: *std.ArrayListUnmanaged([]const u8), @@ -262,7 +256,7 @@ const Eval = struct { } } - if (eval.emit == .none) { + if (eval.target.backend == .sema) { try eval.checkSuccessOutcome(update, null, prog_node); // This message indicates the end of the update. stdout.discard(body.len); @@ -275,11 +269,11 @@ const Eval = struct { const bin_name = try std.zig.binNameAlloc(arena, .{ .root_name = name, .target = try std.zig.system.resolveTargetQuery(try std.Build.parseTargetQuery(.{ - .arch_os_abi = eval.case.target_query, - .object_format = switch (eval.emit) { - .none => unreachable, - .bin => null, - .c => "c", + .arch_os_abi = eval.target.query, + .object_format = switch (eval.target.backend) { + .sema => unreachable, + .selfhosted, .llvm => null, + .cbe => "c", }, })), .output_mode = .Exe, @@ -335,14 +329,14 @@ const Eval = struct { .stdout, .exit_code => {}, } const emitted_path = opt_emitted_path orelse { - std.debug.assert(eval.emit == .none); + std.debug.assert(eval.target.backend == .sema); return; }; - const binary_path = switch (eval.emit) { - .none => unreachable, - .bin => emitted_path, - .c => bin: { + const binary_path = switch (eval.target.backend) { + .sema => unreachable, + .selfhosted, .llvm => emitted_path, + .cbe => bin: { const rand_int = std.crypto.random.int(u64); const out_bin_name = "./out_" ++ std.fmt.hex(rand_int); try eval.buildCOutput(update, emitted_path, out_bin_name, prog_node); @@ -468,7 +462,26 @@ const Eval = struct { const Case = struct { updates: []Update, root_source_file: []const u8, - target_query: []const u8, + targets: []const Target, + + const Target = struct { + query: []const u8, + backend: Backend, + const Backend = enum { + /// Run semantic analysis only. Runtime output will not be tested, but we still verify + /// that compilation succeeds. Corresponds to `-fno-emit-bin`. + sema, + /// Use the self-hosted code generation backend for this target. + /// Corresponds to `-fno-llvm -fno-lld`. + selfhosted, + /// Use the LLVM backend. + /// Corresponds to `-fllvm -flld`. + llvm, + /// Use the C backend. The output is compiled with `zig cc`. + /// Corresponds to `-ofmt=c`. + cbe, + }; + }; const Update = struct { name: []const u8, @@ -498,9 +511,9 @@ const Case = struct { }; fn parse(arena: Allocator, bytes: []const u8) !Case { + var targets: std.ArrayListUnmanaged(Target) = .empty; var updates: std.ArrayListUnmanaged(Update) = .empty; var changes: std.ArrayListUnmanaged(FullContents) = .empty; - var target_query: ?[]const u8 = null; var it = std.mem.splitScalar(u8, bytes, '\n'); var line_n: usize = 1; var root_source_file: ?[]const u8 = null; @@ -512,8 +525,16 @@ const Case = struct { if (val.len == 0) { fatal("line {d}: missing value", .{line_n}); } else if (std.mem.eql(u8, key, "target")) { - if (target_query != null) fatal("line {d}: duplicate target", .{line_n}); - target_query = val; + const split_idx = std.mem.lastIndexOfScalar(u8, val, '-') orelse + fatal("line {d}: target does not include backend", .{line_n}); + const query = val[0..split_idx]; + const backend_str = val[split_idx + 1 ..]; + const backend: Target.Backend = std.meta.stringToEnum(Target.Backend, backend_str) orelse + fatal("line {d}: invalid backend '{s}'", .{ line_n, backend_str }); + try targets.append(arena, .{ + .query = query, + .backend = backend, + }); } else if (std.mem.eql(u8, key, "update")) { if (updates.items.len > 0) { const last_update = &updates.items[updates.items.len - 1]; @@ -565,15 +586,19 @@ const Case = struct { } } + if (targets.items.len == 0) { + fatal("missing target", .{}); + } + if (changes.items.len > 0) { const last_update = &updates.items[updates.items.len - 1]; - last_update.changes = try changes.toOwnedSlice(arena); + last_update.changes = changes.items; // arena so no need for toOwnedSlice } return .{ .updates = updates.items, .root_source_file = root_source_file orelse fatal("missing root source file", .{}), - .target_query = target_query orelse fatal("missing target", .{}), + .targets = targets.items, // arena so no need for toOwnedSlice }; } };