From f5f0d95f0e882ce43146b269f994c6c242f9ed3b Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Sun, 2 Apr 2023 20:36:41 -0400
Subject: [PATCH 01/13] x86_64: implement a more generic assembler for inline
 assembly

---
 src/arch/x86_64/CodeGen.zig | 199 ++++++++++++++++++++++++------------
 1 file changed, 135 insertions(+), 64 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index c7dfa2c6a9..8309fe7335 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -28,10 +28,11 @@ const Type = @import("../../type.zig").Type;
 const TypedValue = @import("../../TypedValue.zig");
 const Value = @import("../../value.zig").Value;
 
-const bits = @import("bits.zig");
 const abi = @import("abi.zig");
-const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
+const bits = @import("bits.zig");
+const encoder = @import("encoder.zig");
 const errUnionErrorOffset = codegen.errUnionErrorOffset;
+const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
 
 const Condition = bits.Condition;
 const Immediate = bits.Immediate;
@@ -6570,7 +6571,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
                 return self.fail("unrecognized constraint: '{s}'", .{constraint});
             args.putAssumeCapacity(name, mcv);
             switch (mcv) {
-                .register => |reg| _ = self.register_manager.lockRegAssumeUnused(reg),
+                .register => |reg| _ = if (RegisterManager.indexOfRegIntoTracked(reg)) |_|
+                    self.register_manager.lockRegAssumeUnused(reg),
                 else => {},
             }
             if (output == .none) result = mcv;
@@ -6609,70 +6611,139 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
         }
 
         const asm_source = mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
-        var line_it = mem.tokenize(u8, asm_source, "\n\r");
+        var line_it = mem.tokenize(u8, asm_source, "\n\r;");
         while (line_it.next()) |line| {
             var mnem_it = mem.tokenize(u8, line, " \t");
-            const mnem = mnem_it.next() orelse continue;
-            if (mem.startsWith(u8, mnem, "#")) continue;
-            var arg_it = mem.tokenize(u8, mnem_it.rest(), ", ");
-            if (std.ascii.eqlIgnoreCase(mnem, "syscall")) {
-                if (arg_it.next()) |trailing| if (!mem.startsWith(u8, trailing, "#"))
-                    return self.fail("Too many operands: '{s}'", .{line});
-                try self.asmOpOnly(.syscall);
-            } else if (std.ascii.eqlIgnoreCase(mnem, "push")) {
-                const src = arg_it.next() orelse
-                    return self.fail("Not enough operands: '{s}'", .{line});
-                if (arg_it.next()) |trailing| if (!mem.startsWith(u8, trailing, "#"))
-                    return self.fail("Too many operands: '{s}'", .{line});
-                if (mem.startsWith(u8, src, "$")) {
-                    const imm = std.fmt.parseInt(u32, src["$".len..], 0) catch
-                        return self.fail("Invalid immediate: '{s}'", .{src});
-                    try self.asmImmediate(.push, Immediate.u(imm));
-                } else if (mem.startsWith(u8, src, "%%")) {
-                    const reg = parseRegName(src["%%".len..]) orelse
-                        return self.fail("Invalid register: '{s}'", .{src});
-                    try self.asmRegister(.push, reg);
-                } else return self.fail("Unsupported operand: '{s}'", .{src});
-            } else if (std.ascii.eqlIgnoreCase(mnem, "pop")) {
-                const dst = arg_it.next() orelse
-                    return self.fail("Not enough operands: '{s}'", .{line});
-                if (arg_it.next()) |trailing| if (!mem.startsWith(u8, trailing, "#"))
-                    return self.fail("Too many operands: '{s}'", .{line});
-                if (mem.startsWith(u8, dst, "%%")) {
-                    const reg = parseRegName(dst["%%".len..]) orelse
-                        return self.fail("Invalid register: '{s}'", .{dst});
-                    try self.asmRegister(.pop, reg);
-                } else return self.fail("Unsupported operand: '{s}'", .{dst});
-            } else if (std.ascii.eqlIgnoreCase(mnem, "movq")) {
-                const src = arg_it.next() orelse
-                    return self.fail("Not enough operands: '{s}'", .{line});
-                const dst = arg_it.next() orelse
-                    return self.fail("Not enough operands: '{s}'", .{line});
-                if (arg_it.next()) |trailing| if (!mem.startsWith(u8, trailing, "#"))
-                    return self.fail("Too many operands: '{s}'", .{line});
-                if (mem.startsWith(u8, src, "%%")) {
-                    const colon = mem.indexOfScalarPos(u8, src, "%%".len + 2, ':');
-                    const src_reg = parseRegName(src["%%".len .. colon orelse src.len]) orelse
-                        return self.fail("Invalid register: '{s}'", .{src});
+            const mnem_str = mnem_it.next() orelse continue;
+            if (mem.startsWith(u8, mnem_str, "#")) continue;
+
+            const mnem_size: ?Memory.PtrSize = if (mem.endsWith(u8, mnem_str, "b"))
+                .byte
+            else if (mem.endsWith(u8, mnem_str, "w"))
+                .word
+            else if (mem.endsWith(u8, mnem_str, "l"))
+                .dword
+            else if (mem.endsWith(u8, mnem_str, "q"))
+                .qword
+            else
+                null;
+            const mnem = std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse
+                (if (mnem_size) |_|
+                std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])
+            else
+                null) orelse return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});
+
+            var op_it = mem.tokenize(u8, mnem_it.rest(), ",");
+            var ops = [1]encoder.Instruction.Operand{.none} ** 4;
+            for (&ops) |*op| {
+                const op_str = mem.trim(u8, op_it.next() orelse break, " \t");
+                if (mem.startsWith(u8, op_str, "#")) break;
+                if (mem.startsWith(u8, op_str, "%%")) {
+                    const colon = mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');
+                    const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse
+                        return self.fail("Invalid register: '{s}'", .{op_str});
                     if (colon) |colon_pos| {
-                        const src_disp = std.fmt.parseInt(i32, src[colon_pos + 1 ..], 0) catch
-                            return self.fail("Invalid immediate: '{s}'", .{src});
-                        if (mem.startsWith(u8, dst, "%[") and mem.endsWith(u8, dst, "]")) {
-                            switch (args.get(dst["%[".len .. dst.len - "]".len]) orelse
-                                return self.fail("no matching constraint for: '{s}'", .{dst})) {
-                                .register => |dst_reg| try self.asmRegisterMemory(
-                                    .mov,
-                                    dst_reg,
-                                    Memory.sib(.qword, .{ .base = src_reg, .disp = src_disp }),
-                                ),
-                                else => return self.fail("Invalid constraint: '{s}'", .{dst}),
-                            }
-                        } else return self.fail("Unsupported operand: '{s}'", .{dst});
-                    } else return self.fail("Unsupported operand: '{s}'", .{src});
-                }
-            } else {
-                return self.fail("Unsupported instruction: '{s}'", .{mnem});
-            }
+                        const disp = std.fmt.parseInt(i32, op_str[colon_pos + 1 ..], 0) catch
+                            return self.fail("Invalid displacement: '{s}'", .{op_str});
+                        op.* = .{ .mem = Memory.sib(
+                            mnem_size orelse return self.fail("Unknown size: '{s}'", .{op_str}),
+                            .{ .base = reg, .disp = disp },
+                        ) };
+                    } else {
+                        if (mnem_size) |size| if (reg.bitSize() != size.bitSize())
+                            return self.fail("Invalid register size: '{s}'", .{op_str});
+                        op.* = .{ .reg = reg };
+                    }
+                } else if (mem.startsWith(u8, op_str, "%[") and mem.endsWith(u8, op_str, "]")) {
+                    switch (args.get(op_str["%[".len .. op_str.len - "]".len]) orelse
+                        return self.fail("No matching constraint: '{s}'", .{op_str})) {
+                        .register => |reg| op.* = .{ .reg = reg },
+                        else => return self.fail("Invalid constraint: '{s}'", .{op_str}),
+                    }
+                } else if (mem.startsWith(u8, op_str, "$")) {
+                    if (std.fmt.parseInt(i32, op_str["$".len..], 0)) |s| {
+                        if (mnem_size) |size| {
+                            const max = @as(u64, std.math.maxInt(u64)) >>
+                                @intCast(u6, 64 - (size.bitSize() - 1));
+                            if ((if (s < 0) ~s else s) > max)
+                                return self.fail("Invalid immediate size: '{s}'", .{op_str});
+                        }
+                        op.* = .{ .imm = Immediate.s(s) };
+                    } else |_| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) |u| {
+                        if (mnem_size) |size| {
+                            const max = @as(u64, std.math.maxInt(u64)) >>
+                                @intCast(u6, 64 - size.bitSize());
+                            if (u > max)
+                                return self.fail("Invalid immediate size: '{s}'", .{op_str});
+                        }
+                        op.* = .{ .imm = Immediate.u(u) };
+                    } else |_| return self.fail("Invalid immediate: '{s}'", .{op_str});
+                } else return self.fail("Invalid operand: '{s}'", .{op_str});
+            } else if (op_it.next()) |op_str| return self.fail("Extra operand: '{s}'", .{op_str});
+
+            (switch (ops[0]) {
+                .none => self.asmOpOnly(mnem),
+                .reg => |reg0| switch (ops[1]) {
+                    .none => self.asmRegister(mnem, reg0),
+                    .reg => |reg1| switch (ops[2]) {
+                        .none => self.asmRegisterRegister(mnem, reg1, reg0),
+                        .reg => |reg2| switch (ops[3]) {
+                            .none => self.asmRegisterRegisterRegister(mnem, reg2, reg1, reg0),
+                            else => error.InvalidInstruction,
+                        },
+                        .mem => |mem2| switch (ops[3]) {
+                            .none => self.asmMemoryRegisterRegister(mnem, mem2, reg1, reg0),
+                            else => error.InvalidInstruction,
+                        },
+                        else => error.InvalidInstruction,
+                    },
+                    .mem => |mem1| switch (ops[2]) {
+                        .none => self.asmMemoryRegister(mnem, mem1, reg0),
+                        else => error.InvalidInstruction,
+                    },
+                    else => error.InvalidInstruction,
+                },
+                .mem => |mem0| switch (ops[1]) {
+                    .none => self.asmMemory(mnem, mem0),
+                    .reg => |reg1| switch (ops[2]) {
+                        .none => self.asmRegisterMemory(mnem, reg1, mem0),
+                        else => error.InvalidInstruction,
+                    },
+                    else => error.InvalidInstruction,
+                },
+                .imm => |imm0| switch (ops[1]) {
+                    .none => self.asmImmediate(mnem, imm0),
+                    .reg => |reg1| switch (ops[2]) {
+                        .none => self.asmRegisterImmediate(mnem, reg1, imm0),
+                        .reg => |reg2| switch (ops[3]) {
+                            .none => self.asmRegisterRegisterImmediate(mnem, reg2, reg1, imm0),
+                            else => error.InvalidInstruction,
+                        },
+                        .mem => |mem2| switch (ops[3]) {
+                            .none => self.asmMemoryRegisterImmediate(mnem, mem2, reg1, imm0),
+                            else => error.InvalidInstruction,
+                        },
+                        else => error.InvalidInstruction,
+                    },
+                    .mem => |mem1| switch (ops[2]) {
+                        .none => self.asmMemoryImmediate(mnem, mem1, imm0),
+                        else => error.InvalidInstruction,
+                    },
+                    else => error.InvalidInstruction,
+                },
+            }) catch |err| switch (err) {
+                error.InvalidInstruction => return self.fail(
+                    "Invalid instruction: '{s} {s} {s} {s} {s}'",
+                    .{
+                        @tagName(mnem),
+                        @tagName(ops[0]),
+                        @tagName(ops[1]),
+                        @tagName(ops[2]),
+                        @tagName(ops[3]),
+                    },
+                ),
+                else => |e| return e,
+            };
         }
     }
 

From 272acb7ee5368a32d9f4bc559e93d96759b80173 Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Sun, 2 Apr 2023 21:39:20 -0400
Subject: [PATCH 02/13] x86_64: implement storing large immediates

---
 src/arch/x86_64/CodeGen.zig | 241 +++++++++++++++---------------------
 test/behavior/bugs/3007.zig |   1 -
 test/behavior/eval.zig      |   3 -
 3 files changed, 98 insertions(+), 147 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 8309fe7335..f3bd6a9e8a 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -3669,52 +3669,67 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
                 .none => unreachable,
                 .dead => unreachable,
                 .unreach => unreachable,
-                .eflags => |cc| {
-                    try self.asmSetccMemory(Memory.sib(
-                        Memory.PtrSize.fromSize(abi_size),
-                        .{ .base = reg.to64() },
-                    ), cc);
+                .eflags => |cc| try self.asmSetccMemory(
+                    Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
+                    cc,
+                ),
+                .undef => if (self.wantSafety()) switch (abi_size) {
+                    1 => try self.store(ptr, .{ .immediate = 0xaa }, ptr_ty, value_ty),
+                    2 => try self.store(ptr, .{ .immediate = 0xaaaa }, ptr_ty, value_ty),
+                    4 => try self.store(ptr, .{ .immediate = 0xaaaaaaaa }, ptr_ty, value_ty),
+                    8 => try self.store(ptr, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, ptr_ty, value_ty),
+                    else => try self.genInlineMemset(
+                        ptr,
+                        .{ .immediate = 0xaa },
+                        .{ .immediate = abi_size },
+                        .{},
+                    ),
                 },
-                .undef => {
-                    if (!self.wantSafety()) return; // The already existing value will do just fine.
-                    switch (abi_size) {
-                        1 => try self.store(ptr, .{ .immediate = 0xaa }, ptr_ty, value_ty),
-                        2 => try self.store(ptr, .{ .immediate = 0xaaaa }, ptr_ty, value_ty),
-                        4 => try self.store(ptr, .{ .immediate = 0xaaaaaaaa }, ptr_ty, value_ty),
-                        8 => try self.store(ptr, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, ptr_ty, value_ty),
-                        else => try self.genInlineMemset(ptr, .{ .immediate = 0xaa }, .{ .immediate = abi_size }, .{}),
-                    }
-                },
-                .immediate => |imm| {
-                    switch (abi_size) {
-                        1, 2, 4 => {
-                            const immediate = if (value_ty.isSignedInt())
-                                Immediate.s(@intCast(i32, @bitCast(i64, imm)))
-                            else
-                                Immediate.u(@truncate(u32, imm));
-                            try self.asmMemoryImmediate(.mov, Memory.sib(
-                                Memory.PtrSize.fromSize(abi_size),
-                                .{ .base = reg.to64() },
-                            ), immediate);
-                        },
-                        8 => {
-                            // TODO: optimization: if the imm is only using the lower
-                            // 4 bytes and can be sign extended we can use a normal mov
-                            // with indirect addressing (mov [reg64], imm32).
-
-                            // movabs does not support indirect register addressing
-                            // so we need an extra register and an extra mov.
-                            const tmp_reg = try self.copyToTmpRegister(value_ty, value);
-                            return self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
-                        },
-                        else => {
-                            return self.fail("TODO implement set pointee with immediate of ABI size {d}", .{abi_size});
-                        },
-                    }
-                },
-                .register => |src_reg| {
-                    try self.genInlineMemcpyRegisterRegister(value_ty, reg, src_reg, 0);
+                .immediate => |imm| switch (self.regBitSize(value_ty)) {
+                    8 => try self.asmMemoryImmediate(
+                        .mov,
+                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
+                        if (math.cast(i8, @bitCast(i64, imm))) |small|
+                            Immediate.s(small)
+                        else
+                            Immediate.u(@intCast(u8, imm)),
+                    ),
+                    16 => try self.asmMemoryImmediate(
+                        .mov,
+                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
+                        if (math.cast(i16, @bitCast(i64, imm))) |small|
+                            Immediate.s(small)
+                        else
+                            Immediate.u(@intCast(u16, imm)),
+                    ),
+                    32 => try self.asmMemoryImmediate(
+                        .mov,
+                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
+                        if (math.cast(i32, @bitCast(i64, imm))) |small|
+                            Immediate.s(small)
+                        else
+                            Immediate.u(@intCast(u32, imm)),
+                    ),
+                    64 => if (math.cast(i32, @bitCast(i64, imm))) |small|
+                        try self.asmMemoryImmediate(
+                            .mov,
+                            Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
+                            Immediate.s(small),
+                        )
+                    else
+                        try self.asmMemoryRegister(
+                            .mov,
+                            Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
+                            registerAlias(try self.copyToTmpRegister(value_ty, value), abi_size),
+                        ),
+                    else => unreachable,
                 },
+                .register => |src_reg| try self.genInlineMemcpyRegisterRegister(
+                    value_ty,
+                    reg,
+                    src_reg,
+                    0,
+                ),
                 .register_overflow => |ro| {
                     const ro_reg_lock = self.register_manager.lockReg(ro.reg);
                     defer if (ro_reg_lock) |lock| self.register_manager.unlockReg(lock);
@@ -3765,8 +3780,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
             defer self.register_manager.unlockReg(addr_reg_lock);
 
             try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr);
-
-            // To get the actual address of the value we want to modify we have to go through the GOT
+            // Load the pointer, which is stored in memory
             try self.asmRegisterMemory(
                 .mov,
                 addr_reg.to64(),
@@ -3774,62 +3788,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
             );
 
             const new_ptr = MCValue{ .register = addr_reg.to64() };
-
-            switch (value) {
-                .immediate => |imm| {
-                    if (abi_size > 8) {
-                        return self.fail("TODO saving imm to memory for abi_size {}", .{abi_size});
-                    }
-
-                    if (abi_size == 8) {
-                        // TODO
-                        const top_bits: u32 = @intCast(u32, imm >> 32);
-                        const can_extend = if (value_ty.isUnsignedInt())
-                            (top_bits == 0) and (imm & 0x8000_0000) == 0
-                        else
-                            top_bits == 0xffff_ffff;
-
-                        if (!can_extend) {
-                            return self.fail("TODO imm64 would get incorrectly sign extended", .{});
-                        }
-                    }
-                    try self.asmMemoryImmediate(
-                        .mov,
-                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg.to64() }),
-                        Immediate.u(@intCast(u32, imm)),
-                    );
-                },
-                .register => {
-                    return self.store(new_ptr, value, ptr_ty, value_ty);
-                },
-                .linker_load, .memory => {
-                    if (abi_size <= 8) {
-                        const tmp_reg = try self.register_manager.allocReg(null, gp);
-                        const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
-                        defer self.register_manager.unlockReg(tmp_reg_lock);
-
-                        try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);
-                        try self.asmRegisterMemory(
-                            .mov,
-                            tmp_reg,
-                            Memory.sib(.qword, .{ .base = tmp_reg }),
-                        );
-
-                        return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
-                    }
-
-                    try self.genInlineMemcpy(new_ptr, value, .{ .immediate = abi_size }, .{});
-                },
-                .stack_offset => {
-                    if (abi_size <= 8) {
-                        const tmp_reg = try self.copyToTmpRegister(value_ty, value);
-                        return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
-                    }
-
-                    try self.genInlineMemcpy(new_ptr, value, .{ .immediate = abi_size }, .{});
-                },
-                else => return self.fail("TODO implement storing {} to MCValue.memory", .{value}),
-            }
+            return self.store(new_ptr, value, ptr_ty, value_ty);
         },
     }
 }
@@ -4887,41 +4846,39 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
                         registerAlias(src_reg, abi_size),
                     ),
                 },
-                .immediate => |imm| {
-                    switch (self.regBitSize(ty)) {
-                        8 => try self.asmRegisterImmediate(
-                            mir_tag,
-                            dst_alias,
-                            if (math.cast(i8, @bitCast(i64, imm))) |small|
-                                Immediate.s(small)
-                            else
-                                Immediate.u(@intCast(u8, imm)),
-                        ),
-                        16 => try self.asmRegisterImmediate(
-                            mir_tag,
-                            dst_alias,
-                            if (math.cast(i16, @bitCast(i64, imm))) |small|
-                                Immediate.s(small)
-                            else
-                                Immediate.u(@intCast(u16, imm)),
-                        ),
-                        32 => try self.asmRegisterImmediate(
-                            mir_tag,
-                            dst_alias,
-                            if (math.cast(i32, @bitCast(i64, imm))) |small|
-                                Immediate.s(small)
-                            else
-                                Immediate.u(@intCast(u32, imm)),
-                        ),
-                        64 => if (math.cast(i32, @bitCast(i64, imm))) |small|
-                            try self.asmRegisterImmediate(mir_tag, dst_alias, Immediate.s(small))
+                .immediate => |imm| switch (self.regBitSize(ty)) {
+                    8 => try self.asmRegisterImmediate(
+                        mir_tag,
+                        dst_alias,
+                        if (math.cast(i8, @bitCast(i64, imm))) |small|
+                            Immediate.s(small)
                         else
-                            try self.asmRegisterRegister(mir_tag, dst_alias, registerAlias(
-                                try self.copyToTmpRegister(ty, src_mcv),
-                                abi_size,
-                            )),
-                        else => unreachable,
-                    }
+                            Immediate.u(@intCast(u8, imm)),
+                    ),
+                    16 => try self.asmRegisterImmediate(
+                        mir_tag,
+                        dst_alias,
+                        if (math.cast(i16, @bitCast(i64, imm))) |small|
+                            Immediate.s(small)
+                        else
+                            Immediate.u(@intCast(u16, imm)),
+                    ),
+                    32 => try self.asmRegisterImmediate(
+                        mir_tag,
+                        dst_alias,
+                        if (math.cast(i32, @bitCast(i64, imm))) |small|
+                            Immediate.s(small)
+                        else
+                            Immediate.u(@intCast(u32, imm)),
+                    ),
+                    64 => if (math.cast(i32, @bitCast(i64, imm))) |small|
+                        try self.asmRegisterImmediate(mir_tag, dst_alias, Immediate.s(small))
+                    else
+                        try self.asmRegisterRegister(mir_tag, dst_alias, registerAlias(
+                            try self.copyToTmpRegister(ty, src_mcv),
+                            abi_size,
+                        )),
+                    else => unreachable,
                 },
                 .memory, .linker_load, .eflags => {
                     assert(abi_size <= 8);
@@ -4931,13 +4888,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
                     const reg = try self.copyToTmpRegister(ty, src_mcv);
                     return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = reg });
                 },
-                .stack_offset => |off| {
-                    try self.asmRegisterMemory(
-                        mir_tag,
-                        registerAlias(dst_reg, abi_size),
-                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
-                    );
-                },
+                .stack_offset => |off| try self.asmRegisterMemory(
+                    mir_tag,
+                    registerAlias(dst_reg, abi_size),
+                    Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
+                ),
             }
         },
         .memory, .linker_load, .stack_offset => {
diff --git a/test/behavior/bugs/3007.zig b/test/behavior/bugs/3007.zig
index df6a63b640..3ae1562e9b 100644
--- a/test/behavior/bugs/3007.zig
+++ b/test/behavior/bugs/3007.zig
@@ -21,7 +21,6 @@ fn get_foo() Foo.FooError!*Foo {
 test "fixed" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     default_foo = get_foo() catch null; // This Line
diff --git a/test/behavior/eval.zig b/test/behavior/eval.zig
index 7fa8066505..214f2beb07 100644
--- a/test/behavior/eval.zig
+++ b/test/behavior/eval.zig
@@ -967,7 +967,6 @@ test "closure capture type of runtime-known parameter" {
 }
 
 test "comptime break passing through runtime condition converted to runtime break" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const S = struct {
@@ -999,7 +998,6 @@ test "comptime break passing through runtime condition converted to runtime brea
 }
 
 test "comptime break to outer loop passing through runtime condition converted to runtime break" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1218,7 +1216,6 @@ test "storing an array of type in a field" {
 }
 
 test "pass pointer to field of comptime-only type as a runtime parameter" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 

From 1980f5479b2139112ff67de5d5f2e6af09fb215e Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Sun, 2 Apr 2023 22:25:35 -0400
Subject: [PATCH 03/13] x86_64: implement store to immediate address

---
 src/arch/x86_64/CodeGen.zig | 32 ++++++++++++--------------------
 test/behavior/inttoptr.zig  |  1 -
 2 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index f3bd6a9e8a..44362f9bc0 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -3651,10 +3651,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
         .dead => unreachable,
         .eflags => unreachable,
         .register_overflow => unreachable,
-        .immediate => |imm| {
-            try self.setRegOrMem(value_ty, .{ .memory = imm }, value);
-        },
-        .stack_offset => {
+        .immediate, .stack_offset => {
             const reg = try self.copyToTmpRegister(ptr_ty, ptr);
             try self.store(.{ .register = reg }, value, ptr_ty, value_ty);
         },
@@ -3748,23 +3745,18 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
                         -@intCast(i32, overflow_bit_offset),
                     );
                 },
-                .linker_load,
-                .memory,
-                .stack_offset,
-                => {
-                    if (abi_size <= 8) {
-                        const tmp_reg = try self.copyToTmpRegister(value_ty, value);
-                        return self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
-                    }
-
-                    try self.genInlineMemcpy(.{ .stack_offset = 0 }, value, .{ .immediate = abi_size }, .{
-                        .source_stack_base = .rbp,
-                        .dest_stack_base = reg.to64(),
-                    });
-                },
+                .linker_load, .memory, .stack_offset => if (abi_size <= 8) {
+                    const tmp_reg = try self.copyToTmpRegister(value_ty, value);
+                    try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
+                } else try self.genInlineMemcpy(
+                    .{ .stack_offset = 0 },
+                    value,
+                    .{ .immediate = abi_size },
+                    .{ .source_stack_base = .rbp, .dest_stack_base = reg.to64() },
+                ),
                 .ptr_stack_offset => {
                     const tmp_reg = try self.copyToTmpRegister(value_ty, value);
-                    return self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
+                    try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
                 },
             }
         },
@@ -3788,7 +3780,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
             );
 
             const new_ptr = MCValue{ .register = addr_reg.to64() };
-            return self.store(new_ptr, value, ptr_ty, value_ty);
+            try self.store(new_ptr, value, ptr_ty, value_ty);
         },
     }
 }
diff --git a/test/behavior/inttoptr.zig b/test/behavior/inttoptr.zig
index e1281c406d..29a263b2ce 100644
--- a/test/behavior/inttoptr.zig
+++ b/test/behavior/inttoptr.zig
@@ -11,7 +11,6 @@ fn addressToFunction() void {
 }
 
 test "mutate through ptr initialized with constant intToPtr value" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From fde1ec5d0e57471b43bab226a78da2eb61e6bb66 Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Sun, 2 Apr 2023 22:25:53 -0400
Subject: [PATCH 04/13] x86_64: remove returns from naked functions

---
 lib/std/os/linux/x86.zig    | 16 ++++++++++++----
 lib/std/os/linux/x86_64.zig |  7 +++++--
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/lib/std/os/linux/x86.zig b/lib/std/os/linux/x86.zig
index 6ad3c9fa31..2e67fa6b5b 100644
--- a/lib/std/os/linux/x86.zig
+++ b/lib/std/os/linux/x86.zig
@@ -125,36 +125,44 @@ pub extern fn clone(func: CloneFn, stack: usize, flags: u32, arg: usize, ptid: *
 
 pub fn restore() callconv(.Naked) void {
     switch (@import("builtin").zig_backend) {
-        .stage2_c => return asm volatile (
+        .stage2_c => asm volatile (
             \\ movl %[number], %%eax
             \\ int $0x80
+            \\ ret
             :
             : [number] "i" (@enumToInt(SYS.sigreturn)),
             : "memory"
         ),
-        else => return asm volatile ("int $0x80"
+        else => asm volatile (
+            \\ int $0x80
+            \\ ret
             :
             : [number] "{eax}" (@enumToInt(SYS.sigreturn)),
             : "memory"
         ),
     }
+    unreachable;
 }
 
 pub fn restore_rt() callconv(.Naked) void {
     switch (@import("builtin").zig_backend) {
-        .stage2_c => return asm volatile (
+        .stage2_c => asm volatile (
             \\ movl %[number], %%eax
             \\ int $0x80
+            \\ ret
             :
             : [number] "i" (@enumToInt(SYS.rt_sigreturn)),
             : "memory"
         ),
-        else => return asm volatile ("int $0x80"
+        else => asm volatile (
+            \\ int $0x80
+            \\ ret
             :
             : [number] "{eax}" (@enumToInt(SYS.rt_sigreturn)),
             : "memory"
         ),
     }
+    unreachable;
 }
 
 pub const O = struct {
diff --git a/lib/std/os/linux/x86_64.zig b/lib/std/os/linux/x86_64.zig
index bc1bccf53f..09047bda83 100644
--- a/lib/std/os/linux/x86_64.zig
+++ b/lib/std/os/linux/x86_64.zig
@@ -109,7 +109,7 @@ pub const restore = restore_rt;
 
 pub fn restore_rt() callconv(.Naked) void {
     switch (@import("builtin").zig_backend) {
-        .stage2_c => return asm volatile (
+        .stage2_c => asm volatile (
             \\ movl %[number], %%eax
             \\ syscall
             \\ retq
@@ -117,12 +117,15 @@ pub fn restore_rt() callconv(.Naked) void {
             : [number] "i" (@enumToInt(SYS.rt_sigreturn)),
             : "rcx", "r11", "memory"
         ),
-        else => return asm volatile ("syscall"
+        else => asm volatile (
+            \\ syscall
+            \\ retq
             :
             : [number] "{rax}" (@enumToInt(SYS.rt_sigreturn)),
             : "rcx", "r11", "memory"
         ),
     }
+    unreachable;
 }
 
 pub const mode_t = usize;

From cd24ab7f6e56c8365896a9b2dfc506a787346ecb Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 01:34:05 -0400
Subject: [PATCH 05/13] x86_64: canonicalise loops

---
 src/arch/x86_64/CodeGen.zig | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 44362f9bc0..a2d0d4da41 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -6132,7 +6132,28 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
     const loop = self.air.extraData(Air.Block, ty_pl.payload);
     const body = self.air.extra[loop.end..][0..loop.data.body_len];
     const jmp_target = @intCast(u32, self.mir_instructions.len);
-    try self.genBody(body);
+
+    {
+        try self.branch_stack.append(.{});
+        errdefer _ = self.branch_stack.pop();
+
+        try self.genBody(body);
+    }
+
+    var branch = self.branch_stack.pop();
+    defer branch.deinit(self.gpa);
+
+    log.debug("airLoop: %{d}", .{inst});
+    log.debug("Upper branches:", .{});
+    for (self.branch_stack.items) |bs| {
+        log.debug("{}", .{bs.fmtDebug()});
+    }
+    log.debug("Loop branch: {}", .{branch.fmtDebug()});
+
+    var dummy_branch = Branch{};
+    defer dummy_branch.deinit(self.gpa);
+    try self.canonicaliseBranches(true, &dummy_branch, &branch, true, false);
+
     _ = try self.asmJmpReloc(jmp_target);
     return self.finishAirBookkeeping();
 }

From f0d13489f819ecd1c0d1c23914127c225f507241 Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 04:55:15 -0400
Subject: [PATCH 06/13] Elf: add program headers for the program header table

---
 lib/std/start.zig                             |   1 -
 src/link/Elf.zig                              | 297 +++++++++++-------
 .../hello_world_with_updates.0.zig            |   2 +-
 .../hello_world_with_updates.0.zig            |   2 +-
 .../hello_world_with_updates.0.zig            |   2 +-
 .../hello_world_with_updates.0.zig            |   2 +-
 6 files changed, 181 insertions(+), 125 deletions(-)

diff --git a/lib/std/start.zig b/lib/std/start.zig
index b3a064c7f0..e79dd1b9a7 100644
--- a/lib/std/start.zig
+++ b/lib/std/start.zig
@@ -19,7 +19,6 @@ const start_sym_name = if (native_arch.isMIPS()) "__start" else "_start";
 // self-hosted is capable enough to handle all of the real start.zig logic.
 pub const simplified_logic =
     builtin.zig_backend == .stage2_wasm or
-    builtin.zig_backend == .stage2_x86_64 or
     builtin.zig_backend == .stage2_x86 or
     builtin.zig_backend == .stage2_aarch64 or
     builtin.zig_backend == .stage2_arm or
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 3b7c2efa0e..8079b09166 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -106,7 +106,12 @@ shdr_table_offset: ?u64 = null,
 /// Stored in native-endian format, depending on target endianness needs to be bswapped on read/write.
 /// Same order as in the file.
 program_headers: std.ArrayListUnmanaged(elf.Elf64_Phdr) = .{},
-phdr_table_offset: ?u64 = null,
+/// The index into the program headers of the PT_PHDR program header
+phdr_table_index: ?u16 = null,
+/// The index into the program headers of the PT_LOAD program header containing the phdr
+/// Most linkers would merge this with phdr_load_ro_index,
+/// but hot swap means we can't ensure they are consecutive.
+phdr_table_load_index: ?u16 = null,
 /// The index into the program headers of a PT_LOAD program header with Read and Execute flags
 phdr_load_re_index: ?u16 = null,
 /// The index into the program headers of the global offset table.
@@ -386,9 +391,10 @@ fn detectAllocCollision(self: *Elf, start: u64, size: u64) ?u64 {
 
     const end = start + padToIdeal(size);
 
-    if (self.shdr_table_offset) |off| {
-        const shdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Shdr) else @sizeOf(elf.Elf64_Shdr);
-        const tight_size = self.sections.slice().len * shdr_size;
+    if (self.phdr_table_index) |index| {
+        const off = self.program_headers.items[index].p_offset;
+        const phdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Phdr) else @sizeOf(elf.Elf64_Phdr);
+        const tight_size = self.program_headers.items.len * phdr_size;
         const increased_size = padToIdeal(tight_size);
         const test_end = off + increased_size;
         if (end > off and start < test_end) {
@@ -396,9 +402,9 @@ fn detectAllocCollision(self: *Elf, start: u64, size: u64) ?u64 {
         }
     }
 
-    if (self.phdr_table_offset) |off| {
-        const phdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Phdr) else @sizeOf(elf.Elf64_Phdr);
-        const tight_size = self.sections.slice().len * phdr_size;
+    if (self.shdr_table_offset) |off| {
+        const shdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Shdr) else @sizeOf(elf.Elf64_Shdr);
+        const tight_size = self.sections.slice().len * shdr_size;
         const increased_size = padToIdeal(tight_size);
         const test_end = off + increased_size;
         if (end > off and start < test_end) {
@@ -427,10 +433,11 @@ pub fn allocatedSize(self: *Elf, start: u64) u64 {
     if (start == 0)
         return 0;
     var min_pos: u64 = std.math.maxInt(u64);
-    if (self.shdr_table_offset) |off| {
+    if (self.phdr_table_index) |index| {
+        const off = self.program_headers.items[index].p_offset;
         if (off > start and off < min_pos) min_pos = off;
     }
-    if (self.phdr_table_offset) |off| {
+    if (self.shdr_table_offset) |off| {
         if (off > start and off < min_pos) min_pos = off;
     }
     for (self.sections.items(.shdr)) |section| {
@@ -462,96 +469,146 @@ pub fn populateMissingMetadata(self: *Elf) !void {
     };
     const ptr_size: u8 = self.ptrWidthBytes();
 
-    if (self.phdr_load_re_index == null) {
-        self.phdr_load_re_index = @intCast(u16, self.program_headers.items.len);
-        const file_size = self.base.options.program_code_size_hint;
-        const p_align = self.page_size;
-        const off = self.findFreeSpace(file_size, p_align);
-        log.debug("found PT_LOAD RE free space 0x{x} to 0x{x}", .{ off, off + file_size });
-        const entry_addr: u64 = self.entry_addr orelse if (self.base.options.target.cpu.arch == .spu_2) @as(u64, 0) else default_entry_addr;
-        try self.program_headers.append(gpa, .{
-            .p_type = elf.PT_LOAD,
-            .p_offset = off,
-            .p_filesz = file_size,
-            .p_vaddr = entry_addr,
-            .p_paddr = entry_addr,
-            .p_memsz = file_size,
-            .p_align = p_align,
-            .p_flags = elf.PF_X | elf.PF_R | elf.PF_W,
-        });
-        self.entry_addr = null;
-        self.phdr_table_dirty = true;
-    }
+    { // Program Headers
+        var new_phdr_table_index: ?u16 = null;
+        if (self.phdr_table_index == null) {
+            new_phdr_table_index = @intCast(u16, self.program_headers.items.len);
+            const phalign: u16 = switch (self.ptr_width) {
+                .p32 => @alignOf(elf.Elf32_Phdr),
+                .p64 => @alignOf(elf.Elf64_Phdr),
+            };
+            try self.program_headers.append(gpa, .{
+                .p_type = elf.PT_PHDR,
+                .p_offset = undefined,
+                .p_filesz = undefined,
+                .p_vaddr = undefined,
+                .p_paddr = undefined,
+                .p_memsz = undefined,
+                .p_align = phalign,
+                .p_flags = elf.PF_R,
+            });
+            self.phdr_table_dirty = true;
+        }
 
-    if (self.phdr_got_index == null) {
-        self.phdr_got_index = @intCast(u16, self.program_headers.items.len);
-        const file_size = @as(u64, ptr_size) * self.base.options.symbol_count_hint;
-        // We really only need ptr alignment but since we are using PROGBITS, linux requires
-        // page align.
-        const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
-        const off = self.findFreeSpace(file_size, p_align);
-        log.debug("found PT_LOAD GOT free space 0x{x} to 0x{x}", .{ off, off + file_size });
-        // TODO instead of hard coding the vaddr, make a function to find a vaddr to put things at.
-        // we'll need to re-use that function anyway, in case the GOT grows and overlaps something
-        // else in virtual memory.
-        const got_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x4000000 else 0x8000;
-        try self.program_headers.append(gpa, .{
-            .p_type = elf.PT_LOAD,
-            .p_offset = off,
-            .p_filesz = file_size,
-            .p_vaddr = got_addr,
-            .p_paddr = got_addr,
-            .p_memsz = file_size,
-            .p_align = p_align,
-            .p_flags = elf.PF_R | elf.PF_W,
-        });
-        self.phdr_table_dirty = true;
-    }
+        if (self.phdr_table_load_index == null) {
+            self.phdr_table_load_index = @intCast(u16, self.program_headers.items.len);
+            // TODO Same as for GOT
+            const phdr_addr: u64 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x1000000 else 0x1000;
+            try self.program_headers.append(gpa, .{
+                .p_type = elf.PT_LOAD,
+                .p_offset = undefined,
+                .p_filesz = undefined,
+                .p_vaddr = phdr_addr,
+                .p_paddr = phdr_addr,
+                .p_memsz = undefined,
+                .p_align = self.page_size,
+                .p_flags = elf.PF_R,
+            });
+            self.phdr_table_dirty = true;
+        }
 
-    if (self.phdr_load_ro_index == null) {
-        self.phdr_load_ro_index = @intCast(u16, self.program_headers.items.len);
-        // TODO Find a hint about how much data need to be in rodata ?
-        const file_size = 1024;
-        // Same reason as for GOT
-        const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
-        const off = self.findFreeSpace(file_size, p_align);
-        log.debug("found PT_LOAD RO free space 0x{x} to 0x{x}", .{ off, off + file_size });
-        // TODO Same as for GOT
-        const rodata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0xc000000 else 0xa000;
-        try self.program_headers.append(gpa, .{
-            .p_type = elf.PT_LOAD,
-            .p_offset = off,
-            .p_filesz = file_size,
-            .p_vaddr = rodata_addr,
-            .p_paddr = rodata_addr,
-            .p_memsz = file_size,
-            .p_align = p_align,
-            .p_flags = elf.PF_R | elf.PF_W,
-        });
-        self.phdr_table_dirty = true;
-    }
+        if (self.phdr_load_re_index == null) {
+            self.phdr_load_re_index = @intCast(u16, self.program_headers.items.len);
+            const file_size = self.base.options.program_code_size_hint;
+            const p_align = self.page_size;
+            const off = self.findFreeSpace(file_size, p_align);
+            log.debug("found PT_LOAD RE free space 0x{x} to 0x{x}", .{ off, off + file_size });
+            const entry_addr: u64 = self.entry_addr orelse if (self.base.options.target.cpu.arch == .spu_2) @as(u64, 0) else default_entry_addr;
+            try self.program_headers.append(gpa, .{
+                .p_type = elf.PT_LOAD,
+                .p_offset = off,
+                .p_filesz = file_size,
+                .p_vaddr = entry_addr,
+                .p_paddr = entry_addr,
+                .p_memsz = file_size,
+                .p_align = p_align,
+                .p_flags = elf.PF_X | elf.PF_R | elf.PF_W,
+            });
+            self.entry_addr = null;
+            self.phdr_table_dirty = true;
+        }
 
-    if (self.phdr_load_rw_index == null) {
-        self.phdr_load_rw_index = @intCast(u16, self.program_headers.items.len);
-        // TODO Find a hint about how much data need to be in data ?
-        const file_size = 1024;
-        // Same reason as for GOT
-        const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
-        const off = self.findFreeSpace(file_size, p_align);
-        log.debug("found PT_LOAD RW free space 0x{x} to 0x{x}", .{ off, off + file_size });
-        // TODO Same as for GOT
-        const rwdata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x10000000 else 0xc000;
-        try self.program_headers.append(gpa, .{
-            .p_type = elf.PT_LOAD,
-            .p_offset = off,
-            .p_filesz = file_size,
-            .p_vaddr = rwdata_addr,
-            .p_paddr = rwdata_addr,
-            .p_memsz = file_size,
-            .p_align = p_align,
-            .p_flags = elf.PF_R | elf.PF_W,
-        });
-        self.phdr_table_dirty = true;
+        if (self.phdr_got_index == null) {
+            self.phdr_got_index = @intCast(u16, self.program_headers.items.len);
+            const file_size = @as(u64, ptr_size) * self.base.options.symbol_count_hint;
+            // We really only need ptr alignment but since we are using PROGBITS, linux requires
+            // page align.
+            const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
+            const off = self.findFreeSpace(file_size, p_align);
+            log.debug("found PT_LOAD GOT free space 0x{x} to 0x{x}", .{ off, off + file_size });
+            // TODO instead of hard coding the vaddr, make a function to find a vaddr to put things at.
+            // we'll need to re-use that function anyway, in case the GOT grows and overlaps something
+            // else in virtual memory.
+            const got_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x4000000 else 0x8000;
+            try self.program_headers.append(gpa, .{
+                .p_type = elf.PT_LOAD,
+                .p_offset = off,
+                .p_filesz = file_size,
+                .p_vaddr = got_addr,
+                .p_paddr = got_addr,
+                .p_memsz = file_size,
+                .p_align = p_align,
+                .p_flags = elf.PF_R | elf.PF_W,
+            });
+            self.phdr_table_dirty = true;
+        }
+
+        if (self.phdr_load_ro_index == null) {
+            self.phdr_load_ro_index = @intCast(u16, self.program_headers.items.len);
+            // TODO Find a hint about how much data need to be in rodata ?
+            const file_size = 1024;
+            // Same reason as for GOT
+            const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
+            const off = self.findFreeSpace(file_size, p_align);
+            log.debug("found PT_LOAD RO free space 0x{x} to 0x{x}", .{ off, off + file_size });
+            // TODO Same as for GOT
+            const rodata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0xc000000 else 0xa000;
+            try self.program_headers.append(gpa, .{
+                .p_type = elf.PT_LOAD,
+                .p_offset = off,
+                .p_filesz = file_size,
+                .p_vaddr = rodata_addr,
+                .p_paddr = rodata_addr,
+                .p_memsz = file_size,
+                .p_align = p_align,
+                .p_flags = elf.PF_R | elf.PF_W,
+            });
+            self.phdr_table_dirty = true;
+        }
+
+        if (self.phdr_load_rw_index == null) {
+            self.phdr_load_rw_index = @intCast(u16, self.program_headers.items.len);
+            // TODO Find a hint about how much data need to be in data ?
+            const file_size = 1024;
+            // Same reason as for GOT
+            const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
+            const off = self.findFreeSpace(file_size, p_align);
+            log.debug("found PT_LOAD RW free space 0x{x} to 0x{x}", .{ off, off + file_size });
+            // TODO Same as for GOT
+            const rwdata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x10000000 else 0xc000;
+            try self.program_headers.append(gpa, .{
+                .p_type = elf.PT_LOAD,
+                .p_offset = off,
+                .p_filesz = file_size,
+                .p_vaddr = rwdata_addr,
+                .p_paddr = rwdata_addr,
+                .p_memsz = file_size,
+                .p_align = p_align,
+                .p_flags = elf.PF_R | elf.PF_W,
+            });
+            self.phdr_table_dirty = true;
+        }
+
+        if (new_phdr_table_index) |index| {
+            const phsize: u64 = switch (self.ptr_width) {
+                .p32 => @sizeOf(elf.Elf32_Phdr),
+                .p64 => @sizeOf(elf.Elf64_Phdr),
+            };
+            const phdr_table = &self.program_headers.items[index];
+            phdr_table.p_offset = self.findFreeSpace(self.program_headers.items.len * phsize, @intCast(u32, phdr_table.p_align));
+            self.phdr_table_index = index;
+            self.phdr_table_dirty = true;
+        }
     }
 
     if (self.shstrtab_index == null) {
@@ -849,19 +906,6 @@ pub fn populateMissingMetadata(self: *Elf) !void {
         self.shdr_table_dirty = true;
     }
 
-    const phsize: u64 = switch (self.ptr_width) {
-        .p32 => @sizeOf(elf.Elf32_Phdr),
-        .p64 => @sizeOf(elf.Elf64_Phdr),
-    };
-    const phalign: u16 = switch (self.ptr_width) {
-        .p32 => @alignOf(elf.Elf32_Phdr),
-        .p64 => @alignOf(elf.Elf64_Phdr),
-    };
-    if (self.phdr_table_offset == null) {
-        self.phdr_table_offset = self.findFreeSpace(self.program_headers.items.len * phsize, phalign);
-        self.phdr_table_dirty = true;
-    }
-
     {
         // Iterate over symbols, populating free_list and last_text_block.
         if (self.local_symbols.items.len != 1) {
@@ -1132,18 +1176,30 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
             .p32 => @sizeOf(elf.Elf32_Phdr),
             .p64 => @sizeOf(elf.Elf64_Phdr),
         };
-        const phalign: u16 = switch (self.ptr_width) {
-            .p32 => @alignOf(elf.Elf32_Phdr),
-            .p64 => @alignOf(elf.Elf64_Phdr),
-        };
-        const allocated_size = self.allocatedSize(self.phdr_table_offset.?);
+
+        const phdr_table_index = self.phdr_table_index.?;
+        const phdr_table = &self.program_headers.items[phdr_table_index];
+        const phdr_table_load = &self.program_headers.items[self.phdr_table_load_index.?];
+
+        const allocated_size = self.allocatedSize(phdr_table.p_offset);
         const needed_size = self.program_headers.items.len * phsize;
 
         if (needed_size > allocated_size) {
-            self.phdr_table_offset = null; // free the space
-            self.phdr_table_offset = self.findFreeSpace(needed_size, phalign);
+            self.phdr_table_index = null; // free the space
+            defer self.phdr_table_index = phdr_table_index;
+            phdr_table.p_offset = self.findFreeSpace(needed_size, @intCast(u32, phdr_table.p_align));
         }
 
+        phdr_table_load.p_offset = mem.alignBackwardGeneric(u64, phdr_table.p_offset, phdr_table_load.p_align);
+        const load_align_offset = phdr_table.p_offset - phdr_table_load.p_offset;
+        phdr_table_load.p_filesz = load_align_offset + needed_size;
+        phdr_table_load.p_memsz = load_align_offset + needed_size;
+
+        phdr_table.p_filesz = needed_size;
+        phdr_table.p_vaddr = phdr_table_load.p_vaddr + load_align_offset;
+        phdr_table.p_paddr = phdr_table_load.p_paddr + load_align_offset;
+        phdr_table.p_memsz = needed_size;
+
         switch (self.ptr_width) {
             .p32 => {
                 const buf = try gpa.alloc(elf.Elf32_Phdr, self.program_headers.items.len);
@@ -1155,7 +1211,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
                         mem.byteSwapAllFields(elf.Elf32_Phdr, phdr);
                     }
                 }
-                try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), self.phdr_table_offset.?);
+                try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), phdr_table.p_offset);
             },
             .p64 => {
                 const buf = try gpa.alloc(elf.Elf64_Phdr, self.program_headers.items.len);
@@ -1167,7 +1223,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
                         mem.byteSwapAllFields(elf.Elf64_Phdr, phdr);
                     }
                 }
-                try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), self.phdr_table_offset.?);
+                try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), phdr_table.p_offset);
             },
         }
         self.phdr_table_dirty = false;
@@ -1992,13 +2048,14 @@ fn writeElfHeader(self: *Elf) !void {
 
     const e_entry = if (elf_type == .REL) 0 else self.entry_addr.?;
 
+    const phdr_table_offset = self.program_headers.items[self.phdr_table_index.?].p_offset;
     switch (self.ptr_width) {
         .p32 => {
             mem.writeInt(u32, hdr_buf[index..][0..4], @intCast(u32, e_entry), endian);
             index += 4;
 
             // e_phoff
-            mem.writeInt(u32, hdr_buf[index..][0..4], @intCast(u32, self.phdr_table_offset.?), endian);
+            mem.writeInt(u32, hdr_buf[index..][0..4], @intCast(u32, phdr_table_offset), endian);
             index += 4;
 
             // e_shoff
@@ -2011,7 +2068,7 @@ fn writeElfHeader(self: *Elf) !void {
             index += 8;
 
             // e_phoff
-            mem.writeInt(u64, hdr_buf[index..][0..8], self.phdr_table_offset.?, endian);
+            mem.writeInt(u64, hdr_buf[index..][0..8], phdr_table_offset, endian);
             index += 8;
 
             // e_shoff
diff --git a/test/cases/aarch64-macos/hello_world_with_updates.0.zig b/test/cases/aarch64-macos/hello_world_with_updates.0.zig
index 90aa14ab65..9f516ff139 100644
--- a/test/cases/aarch64-macos/hello_world_with_updates.0.zig
+++ b/test/cases/aarch64-macos/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=aarch64-macos
 //
-// :110:9: error: root struct of file 'tmp' has no member named 'main'
+// :109:9: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-linux/hello_world_with_updates.0.zig b/test/cases/x86_64-linux/hello_world_with_updates.0.zig
index 16c280774e..40abdd6c1f 100644
--- a/test/cases/x86_64-linux/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-linux/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=x86_64-linux
 //
-// :110:9: error: root struct of file 'tmp' has no member named 'main'
+// :109:9: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-macos/hello_world_with_updates.0.zig b/test/cases/x86_64-macos/hello_world_with_updates.0.zig
index d91bd9dc91..e0680c81d7 100644
--- a/test/cases/x86_64-macos/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-macos/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=x86_64-macos
 //
-// :110:9: error: root struct of file 'tmp' has no member named 'main'
+// :109:9: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-windows/hello_world_with_updates.0.zig b/test/cases/x86_64-windows/hello_world_with_updates.0.zig
index 0c4d2c2bc2..04e1d4cfad 100644
--- a/test/cases/x86_64-windows/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-windows/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=x86_64-windows
 //
-// :131:9: error: root struct of file 'tmp' has no member named 'main'
+// :130:9: error: root struct of file 'tmp' has no member named 'main'

From 562170681a3d0c8502892f8975f369c4b269fe1a Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 06:02:38 -0400
Subject: [PATCH 07/13] link: cleanup lazy symbols

We now only update one lazy symbol in flushModule.
Updating the rest from updateDecl is TBD.
---
 src/Module.zig     |   2 +-
 src/link.zig       |  26 ++++++------
 src/link/Coff.zig  |  99 +++++++++++++++++++++----------------------
 src/link/Elf.zig   |  92 +++++++++++++++++++++-------------------
 src/link/MachO.zig | 102 ++++++++++++++++++++++-----------------------
 5 files changed, 158 insertions(+), 163 deletions(-)

diff --git a/src/Module.zig b/src/Module.zig
index 9dab3edccd..70732369ca 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -555,7 +555,7 @@ pub const Decl = struct {
         _,
 
         pub fn init(oi: ?Index) OptionalIndex {
-            return oi orelse .none;
+            return @intToEnum(OptionalIndex, @enumToInt(oi orelse return .none));
         }
 
         pub fn unwrap(oi: OptionalIndex) ?Index {
diff --git a/src/link.zig b/src/link.zig
index 7ca94f4f6d..45873fec26 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -1106,23 +1106,21 @@ pub const File = struct {
     };
 
     pub const LazySymbol = struct {
-        kind: enum { code, const_data },
+        pub const Kind = enum { code, const_data };
+
+        kind: Kind,
         ty: Type,
 
-        pub const Context = struct {
-            mod: *Module,
+        pub fn initDecl(kind: Kind, decl: Module.Decl.OptionalIndex, mod: *Module) LazySymbol {
+            return .{ .kind = kind, .ty = if (decl.unwrap()) |decl_index|
+                mod.declPtr(decl_index).val.castTag(.ty).?.data
+            else
+                Type.anyerror };
+        }
 
-            pub fn hash(ctx: @This(), sym: LazySymbol) u32 {
-                var hasher = std.hash.Wyhash.init(0);
-                std.hash.autoHash(&hasher, sym.kind);
-                sym.ty.hashWithHasher(&hasher, ctx.mod);
-                return @truncate(u32, hasher.final());
-            }
-
-            pub fn eql(ctx: @This(), lhs: LazySymbol, rhs: LazySymbol, _: usize) bool {
-                return lhs.kind == rhs.kind and lhs.ty.eql(rhs.ty, ctx.mod);
-            }
-        };
+        pub fn getDecl(self: LazySymbol) Module.Decl.OptionalIndex {
+            return Module.Decl.OptionalIndex.init(self.ty.getOwnerDeclOrNull());
+        }
     };
 
     pub const C = @import("link/C.zig");
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 3a94feb841..825afff36d 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -145,16 +145,11 @@ const Section = struct {
     free_list: std.ArrayListUnmanaged(Atom.Index) = .{},
 };
 
-const LazySymbolTable = std.ArrayHashMapUnmanaged(
-    link.File.LazySymbol,
-    LazySymbolMetadata,
-    link.File.LazySymbol.Context,
-    true,
-);
+const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex, LazySymbolMetadata);
 
 const LazySymbolMetadata = struct {
-    atom: Atom.Index,
-    section: u16,
+    text_atom: ?Atom.Index = null,
+    rdata_atom: ?Atom.Index = null,
     alignment: u32,
 };
 
@@ -1176,10 +1171,28 @@ pub fn updateDecl(self: *Coff, module: *Module, decl_index: Module.Decl.Index) !
     return self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-fn updateLazySymbol(
+fn updateLazySymbol(self: *Coff, decl: Module.Decl.OptionalIndex, metadata: LazySymbolMetadata) !void {
+    const mod = self.base.options.module.?;
+    if (metadata.text_atom) |atom| try self.updateLazySymbolAtom(
+        link.File.LazySymbol.initDecl(.code, decl, mod),
+        atom,
+        self.text_section_index.?,
+        metadata.alignment,
+    );
+    if (metadata.rdata_atom) |atom| try self.updateLazySymbolAtom(
+        link.File.LazySymbol.initDecl(.const_data, decl, mod),
+        atom,
+        self.rdata_section_index.?,
+        metadata.alignment,
+    );
+}
+
+fn updateLazySymbolAtom(
     self: *Coff,
-    lazy_sym: link.File.LazySymbol,
-    lazy_metadata: LazySymbolMetadata,
+    sym: link.File.LazySymbol,
+    atom_index: Atom.Index,
+    section_index: u16,
+    required_alignment: u32,
 ) !void {
     const gpa = self.base.allocator;
     const mod = self.base.options.module.?;
@@ -1188,16 +1201,15 @@ fn updateLazySymbol(
     defer code_buffer.deinit();
 
     const name = try std.fmt.allocPrint(gpa, "__lazy_{s}_{}", .{
-        @tagName(lazy_sym.kind),
-        lazy_sym.ty.fmt(mod),
+        @tagName(sym.kind),
+        sym.ty.fmt(mod),
     });
     defer gpa.free(name);
 
-    const atom_index = lazy_metadata.atom;
     const atom = self.getAtomPtr(atom_index);
     const local_sym_index = atom.getSymbolIndex().?;
 
-    const src = if (lazy_sym.ty.getOwnerDeclOrNull()) |owner_decl|
+    const src = if (sym.ty.getOwnerDeclOrNull()) |owner_decl|
         mod.declPtr(owner_decl).srcLoc()
     else
         Module.SrcLoc{
@@ -1205,14 +1217,9 @@ fn updateLazySymbol(
             .parent_decl_node = undefined,
             .lazy = .unneeded,
         };
-    const res = try codegen.generateLazySymbol(
-        &self.base,
-        src,
-        lazy_sym,
-        &code_buffer,
-        .none,
-        .{ .parent_atom_index = local_sym_index },
-    );
+    const res = try codegen.generateLazySymbol(&self.base, src, sym, &code_buffer, .none, .{
+        .parent_atom_index = local_sym_index,
+    });
     const code = switch (res) {
         .ok => code_buffer.items,
         .fail => |em| {
@@ -1221,11 +1228,10 @@ fn updateLazySymbol(
         },
     };
 
-    const required_alignment = lazy_metadata.alignment;
     const code_len = @intCast(u32, code.len);
     const symbol = atom.getSymbolPtr(self);
     try self.setSymbolName(symbol, name);
-    symbol.section_number = @intToEnum(coff.SectionNumber, lazy_metadata.section + 1);
+    symbol.section_number = @intToEnum(coff.SectionNumber, section_index + 1);
     symbol.type = .{ .complex_type = .NULL, .base_type = .NULL };
 
     const vaddr = try self.allocateAtom(atom_index, code_len, required_alignment);
@@ -1250,24 +1256,18 @@ fn updateLazySymbol(
 
 pub fn getOrCreateAtomForLazySymbol(
     self: *Coff,
-    lazy_sym: link.File.LazySymbol,
+    sym: link.File.LazySymbol,
     alignment: u32,
 ) !Atom.Index {
-    const gop = try self.lazy_syms.getOrPutContext(self.base.allocator, lazy_sym, .{
-        .mod = self.base.options.module.?,
-    });
+    const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
     errdefer _ = self.lazy_syms.pop();
-    if (!gop.found_existing) {
-        gop.value_ptr.* = .{
-            .atom = try self.createAtom(),
-            .section = switch (lazy_sym.kind) {
-                .code => self.text_section_index.?,
-                .const_data => self.rdata_section_index.?,
-            },
-            .alignment = alignment,
-        };
-    }
-    return gop.value_ptr.atom;
+    if (!gop.found_existing) gop.value_ptr.* = .{ .alignment = alignment };
+    const atom = switch (sym.kind) {
+        .code => &gop.value_ptr.text_atom,
+        .const_data => &gop.value_ptr.rdata_atom,
+    };
+    if (atom.* == null) atom.* = try self.createAtom();
+    return atom.*.?;
 }
 
 pub fn getOrCreateAtomForDecl(self: *Coff, decl_index: Module.Decl.Index) !Atom.Index {
@@ -1600,17 +1600,13 @@ pub fn flushModule(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod
     sub_prog_node.activate();
     defer sub_prog_node.end();
 
-    {
-        var lazy_it = self.lazy_syms.iterator();
-        while (lazy_it.next()) |lazy_entry| {
-            self.updateLazySymbol(
-                lazy_entry.key_ptr.*,
-                lazy_entry.value_ptr.*,
-            ) catch |err| switch (err) {
-                error.CodegenFail => return error.FlushFailure,
-                else => |e| return e,
-            };
-        }
+    // Most lazy symbols can be updated when the corresponding decl is,
+    // so we only have to worry about the one without an associated decl.
+    if (self.lazy_syms.get(.none)) |metadata| {
+        self.updateLazySymbol(.none, metadata) catch |err| switch (err) {
+            error.CodegenFail => return error.FlushFailure,
+            else => |e| return e,
+        };
     }
 
     const gpa = self.base.allocator;
@@ -2489,6 +2485,7 @@ const Module = @import("../Module.zig");
 const Object = @import("Coff/Object.zig");
 const Relocation = @import("Coff/Relocation.zig");
 const StringTable = @import("strtab.zig").StringTable;
+const Type = @import("../type.zig").Type;
 const TypedValue = @import("../TypedValue.zig");
 
 pub const base_tag: link.File.Tag = .coff;
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 8079b09166..ec05f5895a 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -64,8 +64,8 @@ const Section = struct {
 };
 
 const LazySymbolMetadata = struct {
-    atom: Atom.Index,
-    shdr: u16,
+    text_atom: ?Atom.Index = null,
+    rodata_atom: ?Atom.Index = null,
     alignment: u32,
 };
 
@@ -208,7 +208,7 @@ relocs: RelocTable = .{},
 
 const RelocTable = std.AutoHashMapUnmanaged(Atom.Index, std.ArrayListUnmanaged(Atom.Reloc));
 const UnnamedConstTable = std.AutoHashMapUnmanaged(Module.Decl.Index, std.ArrayListUnmanaged(Atom.Index));
-const LazySymbolTable = std.ArrayHashMapUnmanaged(File.LazySymbol, LazySymbolMetadata, File.LazySymbol.Context, true);
+const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex, LazySymbolMetadata);
 
 /// When allocating, the ideal_capacity is calculated by
 /// actual_capacity + (actual_capacity / ideal_factor)
@@ -1065,17 +1065,13 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
     sub_prog_node.activate();
     defer sub_prog_node.end();
 
-    {
-        var lazy_it = self.lazy_syms.iterator();
-        while (lazy_it.next()) |lazy_entry| {
-            self.updateLazySymbol(
-                lazy_entry.key_ptr.*,
-                lazy_entry.value_ptr.*,
-            ) catch |err| switch (err) {
-                error.CodegenFail => return error.FlushFailure,
-                else => |e| return e,
-            };
-        }
+    // Most lazy symbols can be updated when the corresponding decl is,
+    // so we only have to worry about the one without an associated decl.
+    if (self.lazy_syms.get(.none)) |metadata| {
+        self.updateLazySymbol(.none, metadata) catch |err| switch (err) {
+            error.CodegenFail => return error.FlushFailure,
+            else => |e| return e,
+        };
     }
 
     // TODO This linker code currently assumes there is only 1 compilation unit and it
@@ -2424,22 +2420,16 @@ pub fn freeDecl(self: *Elf, decl_index: Module.Decl.Index) void {
     }
 }
 
-pub fn getOrCreateAtomForLazySymbol(self: *Elf, lazy_sym: File.LazySymbol, alignment: u32) !Atom.Index {
-    const gop = try self.lazy_syms.getOrPutContext(self.base.allocator, lazy_sym, .{
-        .mod = self.base.options.module.?,
-    });
+pub fn getOrCreateAtomForLazySymbol(self: *Elf, sym: File.LazySymbol, alignment: u32) !Atom.Index {
+    const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
     errdefer _ = self.lazy_syms.pop();
-    if (!gop.found_existing) {
-        gop.value_ptr.* = .{
-            .atom = try self.createAtom(),
-            .shdr = switch (lazy_sym.kind) {
-                .code => self.text_section_index.?,
-                .const_data => self.rodata_section_index.?,
-            },
-            .alignment = alignment,
-        };
-    }
-    return gop.value_ptr.atom;
+    if (!gop.found_existing) gop.value_ptr.* = .{ .alignment = alignment };
+    const atom = switch (sym.kind) {
+        .code => &gop.value_ptr.text_atom,
+        .const_data => &gop.value_ptr.rodata_atom,
+    };
+    if (atom.* == null) atom.* = try self.createAtom();
+    return atom.*.?;
 }
 
 pub fn getOrCreateAtomForDecl(self: *Elf, decl_index: Module.Decl.Index) !Atom.Index {
@@ -2708,7 +2698,29 @@ pub fn updateDecl(self: *Elf, module: *Module, decl_index: Module.Decl.Index) !v
     return self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-fn updateLazySymbol(self: *Elf, lazy_sym: File.LazySymbol, lazy_metadata: LazySymbolMetadata) !void {
+fn updateLazySymbol(self: *Elf, decl: Module.Decl.OptionalIndex, metadata: LazySymbolMetadata) !void {
+    const mod = self.base.options.module.?;
+    if (metadata.text_atom) |atom| try self.updateLazySymbolAtom(
+        File.LazySymbol.initDecl(.code, decl, mod),
+        atom,
+        self.text_section_index.?,
+        metadata.alignment,
+    );
+    if (metadata.rodata_atom) |atom| try self.updateLazySymbolAtom(
+        File.LazySymbol.initDecl(.const_data, decl, mod),
+        atom,
+        self.rodata_section_index.?,
+        metadata.alignment,
+    );
+}
+
+fn updateLazySymbolAtom(
+    self: *Elf,
+    sym: File.LazySymbol,
+    atom_index: Atom.Index,
+    shdr_index: u16,
+    required_alignment: u32,
+) !void {
     const gpa = self.base.allocator;
     const mod = self.base.options.module.?;
 
@@ -2717,19 +2729,18 @@ fn updateLazySymbol(self: *Elf, lazy_sym: File.LazySymbol, lazy_metadata: LazySy
 
     const name_str_index = blk: {
         const name = try std.fmt.allocPrint(gpa, "__lazy_{s}_{}", .{
-            @tagName(lazy_sym.kind),
-            lazy_sym.ty.fmt(mod),
+            @tagName(sym.kind),
+            sym.ty.fmt(mod),
         });
         defer gpa.free(name);
         break :blk try self.shstrtab.insert(gpa, name);
     };
     const name = self.shstrtab.get(name_str_index).?;
 
-    const atom_index = lazy_metadata.atom;
     const atom = self.getAtom(atom_index);
     const local_sym_index = atom.getSymbolIndex().?;
 
-    const src = if (lazy_sym.ty.getOwnerDeclOrNull()) |owner_decl|
+    const src = if (sym.ty.getOwnerDeclOrNull()) |owner_decl|
         mod.declPtr(owner_decl).srcLoc()
     else
         Module.SrcLoc{
@@ -2737,14 +2748,9 @@ fn updateLazySymbol(self: *Elf, lazy_sym: File.LazySymbol, lazy_metadata: LazySy
             .parent_decl_node = undefined,
             .lazy = .unneeded,
         };
-    const res = try codegen.generateLazySymbol(
-        &self.base,
-        src,
-        lazy_sym,
-        &code_buffer,
-        .none,
-        .{ .parent_atom_index = local_sym_index },
-    );
+    const res = try codegen.generateLazySymbol(&self.base, src, sym, &code_buffer, .none, .{
+        .parent_atom_index = local_sym_index,
+    });
     const code = switch (res) {
         .ok => code_buffer.items,
         .fail => |em| {
@@ -2753,7 +2759,6 @@ fn updateLazySymbol(self: *Elf, lazy_sym: File.LazySymbol, lazy_metadata: LazySy
         },
     };
 
-    const shdr_index = lazy_metadata.shdr;
     const phdr_index = self.sections.items(.phdr_index)[shdr_index];
     const local_sym = atom.getSymbolPtr(self);
     local_sym.* = .{
@@ -2764,7 +2769,6 @@ fn updateLazySymbol(self: *Elf, lazy_sym: File.LazySymbol, lazy_metadata: LazySy
         .st_value = 0,
         .st_size = 0,
     };
-    const required_alignment = lazy_metadata.alignment;
     const vaddr = try self.allocateAtom(atom_index, code.len, required_alignment);
     errdefer self.freeAtom(atom_index);
     log.debug("allocated text block for {s} at 0x{x}", .{ name, vaddr });
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 2e1076cdf6..b0a05e4050 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -232,16 +232,11 @@ const is_hot_update_compatible = switch (builtin.target.os.tag) {
     else => false,
 };
 
-const LazySymbolTable = std.ArrayHashMapUnmanaged(
-    link.File.LazySymbol,
-    LazySymbolMetadata,
-    link.File.LazySymbol.Context,
-    true,
-);
+const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex, LazySymbolMetadata);
 
 const LazySymbolMetadata = struct {
-    atom: Atom.Index,
-    section: u8,
+    text_atom: ?Atom.Index = null,
+    data_const_atom: ?Atom.Index = null,
     alignment: u32,
 };
 
@@ -513,17 +508,13 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
     sub_prog_node.activate();
     defer sub_prog_node.end();
 
-    {
-        var lazy_it = self.lazy_syms.iterator();
-        while (lazy_it.next()) |lazy_entry| {
-            self.updateLazySymbol(
-                lazy_entry.key_ptr.*,
-                lazy_entry.value_ptr.*,
-            ) catch |err| switch (err) {
-                error.CodegenFail => return error.FlushFailure,
-                else => |e| return e,
-            };
-        }
+    // Most lazy symbols can be updated when the corresponding decl is,
+    // so we only have to worry about the one without an associated decl.
+    if (self.lazy_syms.get(.none)) |metadata| {
+        self.updateLazySymbol(.none, metadata) catch |err| switch (err) {
+            error.CodegenFail => return error.FlushFailure,
+            else => |e| return e,
+        };
     }
 
     const module = self.base.options.module orelse return error.LinkingWithoutZigSourceUnimplemented;
@@ -2309,7 +2300,29 @@ pub fn updateDecl(self: *MachO, module: *Module, decl_index: Module.Decl.Index)
     try self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-fn updateLazySymbol(self: *MachO, lazy_sym: File.LazySymbol, lazy_metadata: LazySymbolMetadata) !void {
+fn updateLazySymbol(self: *MachO, decl: Module.Decl.OptionalIndex, metadata: LazySymbolMetadata) !void {
+    const mod = self.base.options.module.?;
+    if (metadata.text_atom) |atom| try self.updateLazySymbolAtom(
+        File.LazySymbol.initDecl(.code, decl, mod),
+        atom,
+        self.text_section_index.?,
+        metadata.alignment,
+    );
+    if (metadata.data_const_atom) |atom| try self.updateLazySymbolAtom(
+        File.LazySymbol.initDecl(.const_data, decl, mod),
+        atom,
+        self.data_const_section_index.?,
+        metadata.alignment,
+    );
+}
+
+fn updateLazySymbolAtom(
+    self: *MachO,
+    sym: File.LazySymbol,
+    atom_index: Atom.Index,
+    section_index: u8,
+    required_alignment: u32,
+) !void {
     const gpa = self.base.allocator;
     const mod = self.base.options.module.?;
 
@@ -2318,19 +2331,18 @@ fn updateLazySymbol(self: *MachO, lazy_sym: File.LazySymbol, lazy_metadata: Lazy
 
     const name_str_index = blk: {
         const name = try std.fmt.allocPrint(gpa, "___lazy_{s}_{}", .{
-            @tagName(lazy_sym.kind),
-            lazy_sym.ty.fmt(mod),
+            @tagName(sym.kind),
+            sym.ty.fmt(mod),
         });
         defer gpa.free(name);
         break :blk try self.strtab.insert(gpa, name);
     };
     const name = self.strtab.get(name_str_index).?;
 
-    const atom_index = lazy_metadata.atom;
     const atom = self.getAtomPtr(atom_index);
     const local_sym_index = atom.getSymbolIndex().?;
 
-    const src = if (lazy_sym.ty.getOwnerDeclOrNull()) |owner_decl|
+    const src = if (sym.ty.getOwnerDeclOrNull()) |owner_decl|
         mod.declPtr(owner_decl).srcLoc()
     else
         Module.SrcLoc{
@@ -2338,14 +2350,9 @@ fn updateLazySymbol(self: *MachO, lazy_sym: File.LazySymbol, lazy_metadata: Lazy
             .parent_decl_node = undefined,
             .lazy = .unneeded,
         };
-    const res = try codegen.generateLazySymbol(
-        &self.base,
-        src,
-        lazy_sym,
-        &code_buffer,
-        .none,
-        .{ .parent_atom_index = local_sym_index },
-    );
+    const res = try codegen.generateLazySymbol(&self.base, src, sym, &code_buffer, .none, .{
+        .parent_atom_index = local_sym_index,
+    });
     const code = switch (res) {
         .ok => code_buffer.items,
         .fail => |em| {
@@ -2354,11 +2361,10 @@ fn updateLazySymbol(self: *MachO, lazy_sym: File.LazySymbol, lazy_metadata: Lazy
         },
     };
 
-    const required_alignment = lazy_metadata.alignment;
     const symbol = atom.getSymbolPtr(self);
     symbol.n_strx = name_str_index;
     symbol.n_type = macho.N_SECT;
-    symbol.n_sect = lazy_metadata.section + 1;
+    symbol.n_sect = section_index + 1;
     symbol.n_desc = 0;
 
     const vaddr = try self.allocateAtom(atom_index, code.len, required_alignment);
@@ -2381,26 +2387,16 @@ fn updateLazySymbol(self: *MachO, lazy_sym: File.LazySymbol, lazy_metadata: Lazy
     try self.writeAtom(atom_index, code);
 }
 
-pub fn getOrCreateAtomForLazySymbol(
-    self: *MachO,
-    lazy_sym: File.LazySymbol,
-    alignment: u32,
-) !Atom.Index {
-    const gop = try self.lazy_syms.getOrPutContext(self.base.allocator, lazy_sym, .{
-        .mod = self.base.options.module.?,
-    });
+pub fn getOrCreateAtomForLazySymbol(self: *MachO, sym: File.LazySymbol, alignment: u32) !Atom.Index {
+    const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
     errdefer _ = self.lazy_syms.pop();
-    if (!gop.found_existing) {
-        gop.value_ptr.* = .{
-            .atom = try self.createAtom(),
-            .section = switch (lazy_sym.kind) {
-                .code => self.text_section_index.?,
-                .const_data => self.data_const_section_index.?,
-            },
-            .alignment = alignment,
-        };
-    }
-    return gop.value_ptr.atom;
+    if (!gop.found_existing) gop.value_ptr.* = .{ .alignment = alignment };
+    const atom = switch (sym.kind) {
+        .code => &gop.value_ptr.text_atom,
+        .const_data => &gop.value_ptr.data_const_atom,
+    };
+    if (atom.* == null) atom.* = try self.createAtom();
+    return atom.*.?;
 }
 
 pub fn getOrCreateAtomForDecl(self: *MachO, decl_index: Module.Decl.Index) !Atom.Index {

From 5900dc05804a4089eb7d1ccd8a36b4ade16c02e4 Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 04:55:27 -0400
Subject: [PATCH 08/13] x86_64: fix typos

---
 src/arch/x86_64/CodeGen.zig                  | 4 ++--
 src/print_air.zig                            | 1 +
 test/behavior/bugs/12776.zig                 | 1 -
 test/behavior/bugs/1442.zig                  | 1 -
 test/behavior/error.zig                      | 4 ----
 test/behavior/struct.zig                     | 3 ---
 test/behavior/switch_prong_err_enum.zig      | 1 -
 test/behavior/switch_prong_implicit_cast.zig | 1 -
 8 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index a2d0d4da41..748f996032 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -2528,7 +2528,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
         const dst_lock = self.register_manager.lockReg(dst_reg);
         defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
-        const pl_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
+        const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
         const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
         try self.asmRegisterMemory(
             .lea,
@@ -5602,7 +5602,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
 
         const rhs_mcv = try self.resolveInst(bin_op.rhs);
         const rhs_lock = switch (rhs_mcv) {
-            .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
+            .register => |reg| self.register_manager.lockReg(reg),
             else => null,
         };
         defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
diff --git a/src/print_air.zig b/src/print_air.zig
index 803a0f2886..2493d234eb 100644
--- a/src/print_air.zig
+++ b/src/print_air.zig
@@ -804,6 +804,7 @@ const Writer = struct {
         var case_i: u32 = 0;
 
         try w.writeOperand(s, inst, 0, pl_op.operand);
+        if (w.skip_body) return s.writeAll(", ...");
         const old_indent = w.indent;
         w.indent += 2;
 
diff --git a/test/behavior/bugs/12776.zig b/test/behavior/bugs/12776.zig
index f95709a2a4..1611fe27ee 100644
--- a/test/behavior/bugs/12776.zig
+++ b/test/behavior/bugs/12776.zig
@@ -31,7 +31,6 @@ const CPU = packed struct {
 test {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
 
     var ram = try RAM.new();
diff --git a/test/behavior/bugs/1442.zig b/test/behavior/bugs/1442.zig
index 5f788f6cef..f9bef6ec50 100644
--- a/test/behavior/bugs/1442.zig
+++ b/test/behavior/bugs/1442.zig
@@ -9,7 +9,6 @@ const Union = union(enum) {
 test "const error union field alignment" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     var union_or_err: anyerror!Union = Union{ .Color = 1234 };
     try std.testing.expect((union_or_err catch unreachable).Color == 1234);
diff --git a/test/behavior/error.zig b/test/behavior/error.zig
index b1f77f33a8..0cd9be05ca 100644
--- a/test/behavior/error.zig
+++ b/test/behavior/error.zig
@@ -370,7 +370,6 @@ fn intLiteral(str: []const u8) !?i64 {
 }
 
 test "nested error union function call in optional unwrap" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -499,7 +498,6 @@ test "function pointer with return type that is error union with payload which i
 }
 
 test "return result loc as peer result loc in inferred error set function" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -531,7 +529,6 @@ test "return result loc as peer result loc in inferred error set function" {
 }
 
 test "error payload type is correctly resolved" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -665,7 +662,6 @@ test "coerce error set to the current inferred error set" {
 }
 
 test "error union payload is properly aligned" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/struct.zig b/test/behavior/struct.zig
index 37b24a1ed5..1dcfce53dd 100644
--- a/test/behavior/struct.zig
+++ b/test/behavior/struct.zig
@@ -1116,7 +1116,6 @@ test "for loop over pointers to struct, getting field from struct pointer" {
 }
 
 test "anon init through error unions and optionals" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1162,7 +1161,6 @@ test "anon init through optional" {
 }
 
 test "anon init through error union" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1182,7 +1180,6 @@ test "anon init through error union" {
 }
 
 test "typed init through error unions and optionals" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/switch_prong_err_enum.zig b/test/behavior/switch_prong_err_enum.zig
index ddcc76634e..15d366d04f 100644
--- a/test/behavior/switch_prong_err_enum.zig
+++ b/test/behavior/switch_prong_err_enum.zig
@@ -23,7 +23,6 @@ fn doThing(form_id: u64) anyerror!FormValue {
 test "switch prong returns error enum" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     switch (doThing(17) catch unreachable) {
diff --git a/test/behavior/switch_prong_implicit_cast.zig b/test/behavior/switch_prong_implicit_cast.zig
index 26ec57676b..54107bb6bd 100644
--- a/test/behavior/switch_prong_implicit_cast.zig
+++ b/test/behavior/switch_prong_implicit_cast.zig
@@ -17,7 +17,6 @@ fn foo(id: u64) !FormValue {
 test "switch prong implicit cast" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const result = switch (foo(2) catch unreachable) {

From c0e9b849974cdbc93053919f54f0cf3830243572 Mon Sep 17 00:00:00 2001
From: Jakub Konka <kubkon@jakubkonka.com>
Date: Mon, 3 Apr 2023 18:30:48 +0200
Subject: [PATCH 09/13] elf: preallocate offsets for PT_PHDR and PT_LOAD
 (empty) segment

Otherwise, we will be using `undefined` as the offset to allocate
remaining phdrs and shdrs.
---
 src/link/Elf.zig | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index ec05f5895a..74a538b8cc 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -110,7 +110,7 @@ program_headers: std.ArrayListUnmanaged(elf.Elf64_Phdr) = .{},
 phdr_table_index: ?u16 = null,
 /// The index into the program headers of the PT_LOAD program header containing the phdr
 /// Most linkers would merge this with phdr_load_ro_index,
-/// but hot swap means we can't ensure they are consecutive.
+/// but incremental linking means we can't ensure they are consecutive.
 phdr_table_load_index: ?u16 = null,
 /// The index into the program headers of a PT_LOAD program header with Read and Execute flags
 phdr_load_re_index: ?u16 = null,
@@ -473,18 +473,19 @@ pub fn populateMissingMetadata(self: *Elf) !void {
         var new_phdr_table_index: ?u16 = null;
         if (self.phdr_table_index == null) {
             new_phdr_table_index = @intCast(u16, self.program_headers.items.len);
-            const phalign: u16 = switch (self.ptr_width) {
+            const p_align: u16 = switch (self.ptr_width) {
                 .p32 => @alignOf(elf.Elf32_Phdr),
                 .p64 => @alignOf(elf.Elf64_Phdr),
             };
+            const off = self.findFreeSpace(1, p_align);
             try self.program_headers.append(gpa, .{
                 .p_type = elf.PT_PHDR,
-                .p_offset = undefined,
-                .p_filesz = undefined,
-                .p_vaddr = undefined,
-                .p_paddr = undefined,
-                .p_memsz = undefined,
-                .p_align = phalign,
+                .p_offset = off,
+                .p_filesz = 1,
+                .p_vaddr = 0,
+                .p_paddr = 0,
+                .p_memsz = 0,
+                .p_align = p_align,
                 .p_flags = elf.PF_R,
             });
             self.phdr_table_dirty = true;
@@ -494,14 +495,18 @@ pub fn populateMissingMetadata(self: *Elf) !void {
             self.phdr_table_load_index = @intCast(u16, self.program_headers.items.len);
             // TODO Same as for GOT
             const phdr_addr: u64 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x1000000 else 0x1000;
+            const p_align = self.page_size;
+            const off = self.findFreeSpace(1, p_align);
+            const file_size: u32 = 1;
+            log.debug("found PT_LOAD free space 0x{x} to 0x{x}", .{ off, off + file_size });
             try self.program_headers.append(gpa, .{
                 .p_type = elf.PT_LOAD,
-                .p_offset = undefined,
-                .p_filesz = undefined,
+                .p_offset = off,
+                .p_filesz = file_size,
                 .p_vaddr = phdr_addr,
                 .p_paddr = phdr_addr,
-                .p_memsz = undefined,
-                .p_align = self.page_size,
+                .p_memsz = file_size,
+                .p_align = p_align,
                 .p_flags = elf.PF_R,
             });
             self.phdr_table_dirty = true;

From 9fd460821f992c51b35a54ba93562af93ac478f6 Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 13:55:20 -0400
Subject: [PATCH 10/13] elf: cleanup phdr tracking

Since one of the program header entries is now the program header table
itself, we can avoid tracking it explicitly and just track it as yet
another program segment.
---
 src/link/Elf.zig | 285 +++++++++++++++++++++--------------------------
 1 file changed, 128 insertions(+), 157 deletions(-)

diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 74a538b8cc..7a3a0b0fdd 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -391,17 +391,6 @@ fn detectAllocCollision(self: *Elf, start: u64, size: u64) ?u64 {
 
     const end = start + padToIdeal(size);
 
-    if (self.phdr_table_index) |index| {
-        const off = self.program_headers.items[index].p_offset;
-        const phdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Phdr) else @sizeOf(elf.Elf64_Phdr);
-        const tight_size = self.program_headers.items.len * phdr_size;
-        const increased_size = padToIdeal(tight_size);
-        const test_end = off + increased_size;
-        if (end > off and start < test_end) {
-            return test_end;
-        }
-    }
-
     if (self.shdr_table_offset) |off| {
         const shdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Shdr) else @sizeOf(elf.Elf64_Shdr);
         const tight_size = self.sections.slice().len * shdr_size;
@@ -433,10 +422,6 @@ pub fn allocatedSize(self: *Elf, start: u64) u64 {
     if (start == 0)
         return 0;
     var min_pos: u64 = std.math.maxInt(u64);
-    if (self.phdr_table_index) |index| {
-        const off = self.program_headers.items[index].p_offset;
-        if (off > start and off < min_pos) min_pos = off;
-    }
     if (self.shdr_table_offset) |off| {
         if (off > start and off < min_pos) min_pos = off;
     }
@@ -469,151 +454,133 @@ pub fn populateMissingMetadata(self: *Elf) !void {
     };
     const ptr_size: u8 = self.ptrWidthBytes();
 
-    { // Program Headers
-        var new_phdr_table_index: ?u16 = null;
-        if (self.phdr_table_index == null) {
-            new_phdr_table_index = @intCast(u16, self.program_headers.items.len);
-            const p_align: u16 = switch (self.ptr_width) {
-                .p32 => @alignOf(elf.Elf32_Phdr),
-                .p64 => @alignOf(elf.Elf64_Phdr),
-            };
-            const off = self.findFreeSpace(1, p_align);
-            try self.program_headers.append(gpa, .{
-                .p_type = elf.PT_PHDR,
-                .p_offset = off,
-                .p_filesz = 1,
-                .p_vaddr = 0,
-                .p_paddr = 0,
-                .p_memsz = 0,
-                .p_align = p_align,
-                .p_flags = elf.PF_R,
-            });
-            self.phdr_table_dirty = true;
-        }
+    if (self.phdr_table_index == null) {
+        self.phdr_table_index = @intCast(u16, self.program_headers.items.len);
+        const p_align: u16 = switch (self.ptr_width) {
+            .p32 => @alignOf(elf.Elf32_Phdr),
+            .p64 => @alignOf(elf.Elf64_Phdr),
+        };
+        try self.program_headers.append(gpa, .{
+            .p_type = elf.PT_PHDR,
+            .p_offset = 0,
+            .p_filesz = 0,
+            .p_vaddr = 0,
+            .p_paddr = 0,
+            .p_memsz = 0,
+            .p_align = p_align,
+            .p_flags = elf.PF_R,
+        });
+        self.phdr_table_dirty = true;
+    }
 
-        if (self.phdr_table_load_index == null) {
-            self.phdr_table_load_index = @intCast(u16, self.program_headers.items.len);
-            // TODO Same as for GOT
-            const phdr_addr: u64 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x1000000 else 0x1000;
-            const p_align = self.page_size;
-            const off = self.findFreeSpace(1, p_align);
-            const file_size: u32 = 1;
-            log.debug("found PT_LOAD free space 0x{x} to 0x{x}", .{ off, off + file_size });
-            try self.program_headers.append(gpa, .{
-                .p_type = elf.PT_LOAD,
-                .p_offset = off,
-                .p_filesz = file_size,
-                .p_vaddr = phdr_addr,
-                .p_paddr = phdr_addr,
-                .p_memsz = file_size,
-                .p_align = p_align,
-                .p_flags = elf.PF_R,
-            });
-            self.phdr_table_dirty = true;
-        }
+    if (self.phdr_table_load_index == null) {
+        self.phdr_table_load_index = @intCast(u16, self.program_headers.items.len);
+        // TODO Same as for GOT
+        const phdr_addr: u64 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x1000000 else 0x1000;
+        const p_align = self.page_size;
+        try self.program_headers.append(gpa, .{
+            .p_type = elf.PT_LOAD,
+            .p_offset = 0,
+            .p_filesz = 0,
+            .p_vaddr = phdr_addr,
+            .p_paddr = phdr_addr,
+            .p_memsz = 0,
+            .p_align = p_align,
+            .p_flags = elf.PF_R,
+        });
+        self.phdr_table_dirty = true;
+    }
 
-        if (self.phdr_load_re_index == null) {
-            self.phdr_load_re_index = @intCast(u16, self.program_headers.items.len);
-            const file_size = self.base.options.program_code_size_hint;
-            const p_align = self.page_size;
-            const off = self.findFreeSpace(file_size, p_align);
-            log.debug("found PT_LOAD RE free space 0x{x} to 0x{x}", .{ off, off + file_size });
-            const entry_addr: u64 = self.entry_addr orelse if (self.base.options.target.cpu.arch == .spu_2) @as(u64, 0) else default_entry_addr;
-            try self.program_headers.append(gpa, .{
-                .p_type = elf.PT_LOAD,
-                .p_offset = off,
-                .p_filesz = file_size,
-                .p_vaddr = entry_addr,
-                .p_paddr = entry_addr,
-                .p_memsz = file_size,
-                .p_align = p_align,
-                .p_flags = elf.PF_X | elf.PF_R | elf.PF_W,
-            });
-            self.entry_addr = null;
-            self.phdr_table_dirty = true;
-        }
+    if (self.phdr_load_re_index == null) {
+        self.phdr_load_re_index = @intCast(u16, self.program_headers.items.len);
+        const file_size = self.base.options.program_code_size_hint;
+        const p_align = self.page_size;
+        const off = self.findFreeSpace(file_size, p_align);
+        log.debug("found PT_LOAD RE free space 0x{x} to 0x{x}", .{ off, off + file_size });
+        const entry_addr: u64 = self.entry_addr orelse if (self.base.options.target.cpu.arch == .spu_2) @as(u64, 0) else default_entry_addr;
+        try self.program_headers.append(gpa, .{
+            .p_type = elf.PT_LOAD,
+            .p_offset = off,
+            .p_filesz = file_size,
+            .p_vaddr = entry_addr,
+            .p_paddr = entry_addr,
+            .p_memsz = file_size,
+            .p_align = p_align,
+            .p_flags = elf.PF_X | elf.PF_R | elf.PF_W,
+        });
+        self.entry_addr = null;
+        self.phdr_table_dirty = true;
+    }
 
-        if (self.phdr_got_index == null) {
-            self.phdr_got_index = @intCast(u16, self.program_headers.items.len);
-            const file_size = @as(u64, ptr_size) * self.base.options.symbol_count_hint;
-            // We really only need ptr alignment but since we are using PROGBITS, linux requires
-            // page align.
-            const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
-            const off = self.findFreeSpace(file_size, p_align);
-            log.debug("found PT_LOAD GOT free space 0x{x} to 0x{x}", .{ off, off + file_size });
-            // TODO instead of hard coding the vaddr, make a function to find a vaddr to put things at.
-            // we'll need to re-use that function anyway, in case the GOT grows and overlaps something
-            // else in virtual memory.
-            const got_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x4000000 else 0x8000;
-            try self.program_headers.append(gpa, .{
-                .p_type = elf.PT_LOAD,
-                .p_offset = off,
-                .p_filesz = file_size,
-                .p_vaddr = got_addr,
-                .p_paddr = got_addr,
-                .p_memsz = file_size,
-                .p_align = p_align,
-                .p_flags = elf.PF_R | elf.PF_W,
-            });
-            self.phdr_table_dirty = true;
-        }
+    if (self.phdr_got_index == null) {
+        self.phdr_got_index = @intCast(u16, self.program_headers.items.len);
+        const file_size = @as(u64, ptr_size) * self.base.options.symbol_count_hint;
+        // We really only need ptr alignment but since we are using PROGBITS, linux requires
+        // page align.
+        const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
+        const off = self.findFreeSpace(file_size, p_align);
+        log.debug("found PT_LOAD GOT free space 0x{x} to 0x{x}", .{ off, off + file_size });
+        // TODO instead of hard coding the vaddr, make a function to find a vaddr to put things at.
+        // we'll need to re-use that function anyway, in case the GOT grows and overlaps something
+        // else in virtual memory.
+        const got_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x4000000 else 0x8000;
+        try self.program_headers.append(gpa, .{
+            .p_type = elf.PT_LOAD,
+            .p_offset = off,
+            .p_filesz = file_size,
+            .p_vaddr = got_addr,
+            .p_paddr = got_addr,
+            .p_memsz = file_size,
+            .p_align = p_align,
+            .p_flags = elf.PF_R | elf.PF_W,
+        });
+        self.phdr_table_dirty = true;
+    }
 
-        if (self.phdr_load_ro_index == null) {
-            self.phdr_load_ro_index = @intCast(u16, self.program_headers.items.len);
-            // TODO Find a hint about how much data need to be in rodata ?
-            const file_size = 1024;
-            // Same reason as for GOT
-            const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
-            const off = self.findFreeSpace(file_size, p_align);
-            log.debug("found PT_LOAD RO free space 0x{x} to 0x{x}", .{ off, off + file_size });
-            // TODO Same as for GOT
-            const rodata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0xc000000 else 0xa000;
-            try self.program_headers.append(gpa, .{
-                .p_type = elf.PT_LOAD,
-                .p_offset = off,
-                .p_filesz = file_size,
-                .p_vaddr = rodata_addr,
-                .p_paddr = rodata_addr,
-                .p_memsz = file_size,
-                .p_align = p_align,
-                .p_flags = elf.PF_R | elf.PF_W,
-            });
-            self.phdr_table_dirty = true;
-        }
+    if (self.phdr_load_ro_index == null) {
+        self.phdr_load_ro_index = @intCast(u16, self.program_headers.items.len);
+        // TODO Find a hint about how much data need to be in rodata ?
+        const file_size = 1024;
+        // Same reason as for GOT
+        const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
+        const off = self.findFreeSpace(file_size, p_align);
+        log.debug("found PT_LOAD RO free space 0x{x} to 0x{x}", .{ off, off + file_size });
+        // TODO Same as for GOT
+        const rodata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0xc000000 else 0xa000;
+        try self.program_headers.append(gpa, .{
+            .p_type = elf.PT_LOAD,
+            .p_offset = off,
+            .p_filesz = file_size,
+            .p_vaddr = rodata_addr,
+            .p_paddr = rodata_addr,
+            .p_memsz = file_size,
+            .p_align = p_align,
+            .p_flags = elf.PF_R | elf.PF_W,
+        });
+        self.phdr_table_dirty = true;
+    }
 
-        if (self.phdr_load_rw_index == null) {
-            self.phdr_load_rw_index = @intCast(u16, self.program_headers.items.len);
-            // TODO Find a hint about how much data need to be in data ?
-            const file_size = 1024;
-            // Same reason as for GOT
-            const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
-            const off = self.findFreeSpace(file_size, p_align);
-            log.debug("found PT_LOAD RW free space 0x{x} to 0x{x}", .{ off, off + file_size });
-            // TODO Same as for GOT
-            const rwdata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x10000000 else 0xc000;
-            try self.program_headers.append(gpa, .{
-                .p_type = elf.PT_LOAD,
-                .p_offset = off,
-                .p_filesz = file_size,
-                .p_vaddr = rwdata_addr,
-                .p_paddr = rwdata_addr,
-                .p_memsz = file_size,
-                .p_align = p_align,
-                .p_flags = elf.PF_R | elf.PF_W,
-            });
-            self.phdr_table_dirty = true;
-        }
-
-        if (new_phdr_table_index) |index| {
-            const phsize: u64 = switch (self.ptr_width) {
-                .p32 => @sizeOf(elf.Elf32_Phdr),
-                .p64 => @sizeOf(elf.Elf64_Phdr),
-            };
-            const phdr_table = &self.program_headers.items[index];
-            phdr_table.p_offset = self.findFreeSpace(self.program_headers.items.len * phsize, @intCast(u32, phdr_table.p_align));
-            self.phdr_table_index = index;
-            self.phdr_table_dirty = true;
-        }
+    if (self.phdr_load_rw_index == null) {
+        self.phdr_load_rw_index = @intCast(u16, self.program_headers.items.len);
+        // TODO Find a hint about how much data need to be in data ?
+        const file_size = 1024;
+        // Same reason as for GOT
+        const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
+        const off = self.findFreeSpace(file_size, p_align);
+        log.debug("found PT_LOAD RW free space 0x{x} to 0x{x}", .{ off, off + file_size });
+        // TODO Same as for GOT
+        const rwdata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x10000000 else 0xc000;
+        try self.program_headers.append(gpa, .{
+            .p_type = elf.PT_LOAD,
+            .p_offset = off,
+            .p_filesz = file_size,
+            .p_vaddr = rwdata_addr,
+            .p_paddr = rwdata_addr,
+            .p_memsz = file_size,
+            .p_align = p_align,
+            .p_flags = elf.PF_R | elf.PF_W,
+        });
+        self.phdr_table_dirty = true;
     }
 
     if (self.shstrtab_index == null) {
@@ -1186,8 +1153,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
         const needed_size = self.program_headers.items.len * phsize;
 
         if (needed_size > allocated_size) {
-            self.phdr_table_index = null; // free the space
-            defer self.phdr_table_index = phdr_table_index;
+            phdr_table.p_offset = 0; // free the space
             phdr_table.p_offset = self.findFreeSpace(needed_size, @intCast(u32, phdr_table.p_align));
         }
 
@@ -1227,6 +1193,11 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
                 try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), phdr_table.p_offset);
             },
         }
+
+        // We don't actually care if the phdr load section overlaps, only the phdr section matters.
+        phdr_table_load.p_offset = 0;
+        phdr_table_load.p_filesz = 0;
+
         self.phdr_table_dirty = false;
     }
 

From 821eb595f4bd65d003a2cb499c7953f4f57a4c4a Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 14:14:55 -0400
Subject: [PATCH 11/13] x86_64: implement cmp_lt_errors_len

---
 src/arch/x86_64/CodeGen.zig | 63 ++++++++++++++++++++++++++++++++++---
 src/codegen.zig             | 10 ++++--
 test/behavior/cast.zig      |  1 -
 3 files changed, 65 insertions(+), 9 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 748f996032..8a3e0852a2 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -5650,9 +5650,62 @@ fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {
 
 fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
-    const operand = try self.resolveInst(un_op);
-    _ = operand;
-    const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCmpLtErrorsLen for {}", .{self.target.cpu.arch});
+    const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
+        const addr_reg = try self.register_manager.allocReg(null, gp);
+        const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
+        defer self.register_manager.unlockReg(addr_lock);
+
+        if (self.bin_file.cast(link.File.Elf)) |elf_file| {
+            const atom_index = try elf_file.getOrCreateAtomForLazySymbol(
+                .{ .kind = .const_data, .ty = Type.anyerror },
+                4, // dword alignment
+            );
+            const got_addr = elf_file.getAtom(atom_index).getOffsetTableAddress(elf_file);
+            try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
+                .base = .ds,
+                .disp = @intCast(i32, got_addr),
+            }));
+        } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
+            const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
+                .{ .kind = .const_data, .ty = Type.anyerror },
+                4, // dword alignment
+            );
+            const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
+            try self.genSetReg(Type.usize, addr_reg, .{ .linker_load = .{
+                .type = .got,
+                .sym_index = sym_index,
+            } });
+        } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
+            const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
+                .{ .kind = .const_data, .ty = Type.anyerror },
+                4, // dword alignment
+            );
+            const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
+            try self.genSetReg(Type.usize, addr_reg, .{ .linker_load = .{
+                .type = .got,
+                .sym_index = sym_index,
+            } });
+        } else {
+            return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
+        }
+
+        try self.spillEflagsIfOccupied();
+        self.eflags_inst = inst;
+
+        const op_ty = self.air.typeOf(un_op);
+        const op_abi_size = @intCast(u32, op_ty.abiSize(self.target.*));
+        const op_mcv = try self.resolveInst(un_op);
+        const dst_reg = switch (op_mcv) {
+            .register => |reg| reg,
+            else => try self.copyToTmpRegister(op_ty, op_mcv),
+        };
+        try self.asmRegisterMemory(
+            .cmp,
+            registerAlias(dst_reg, op_abi_size),
+            Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = addr_reg }),
+        );
+        break :result .{ .eflags = .b };
+    };
     return self.finishAir(inst, result, .{ un_op, .none, .none });
 }
 
@@ -8027,12 +8080,12 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
         try self.asmRegisterMemory(.mov, start_reg.to32(), Memory.sib(.dword, .{
             .base = addr_reg.to64(),
             .scale_index = .{ .scale = 4, .index = err_reg.to64() },
-            .disp = 0,
+            .disp = 4,
         }));
         try self.asmRegisterMemory(.mov, end_reg.to32(), Memory.sib(.dword, .{
             .base = addr_reg.to64(),
             .scale_index = .{ .scale = 4, .index = err_reg.to64() },
-            .disp = 4,
+            .disp = 8,
         }));
         try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());
         try self.asmRegisterMemory(.lea, start_reg.to64(), Memory.sib(.byte, .{
diff --git a/src/codegen.zig b/src/codegen.zig
index 67ada2fedd..57c33ad524 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -124,13 +124,17 @@ pub fn generateLazySymbol(
 
     if (lazy_sym.kind == .const_data and lazy_sym.ty.isAnyError()) {
         const err_names = mod.error_name_list.items;
-        try code.resize(err_names.len * 4);
-        for (err_names, 0..) |err_name, index| {
-            mem.writeInt(u32, code.items[index * 4 ..][0..4], @intCast(u32, code.items.len), endian);
+        mem.writeInt(u32, try code.addManyAsArray(4), @intCast(u32, err_names.len), endian);
+        var offset = code.items.len;
+        try code.resize((1 + err_names.len + 1) * 4);
+        for (err_names) |err_name| {
+            mem.writeInt(u32, code.items[offset..][0..4], @intCast(u32, code.items.len), endian);
+            offset += 4;
             try code.ensureUnusedCapacity(err_name.len + 1);
             code.appendSliceAssumeCapacity(err_name);
             code.appendAssumeCapacity(0);
         }
+        mem.writeInt(u32, code.items[offset..][0..4], @intCast(u32, code.items.len), endian);
         return Result.ok;
     } else return .{ .fail = try ErrorMsg.create(
         bin_file.allocator,
diff --git a/test/behavior/cast.zig b/test/behavior/cast.zig
index 470b17e0b6..8027515730 100644
--- a/test/behavior/cast.zig
+++ b/test/behavior/cast.zig
@@ -402,7 +402,6 @@ test "expected [*c]const u8, found [*:0]const u8" {
 
 test "explicit cast from integer to error type" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From c91929090db337810aa3b55f9bd26f5fd0af4fff Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 14:17:00 -0400
Subject: [PATCH 12/13] start: disable extra start logic on various x86_64
 subtargets

---
 lib/std/start.zig                                        | 1 +
 test/cases/aarch64-macos/hello_world_with_updates.0.zig  | 2 +-
 test/cases/x86_64-linux/hello_world_with_updates.0.zig   | 5 ++++-
 test/cases/x86_64-macos/hello_world_with_updates.0.zig   | 2 +-
 test/cases/x86_64-windows/hello_world_with_updates.0.zig | 4 +++-
 5 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/std/start.zig b/lib/std/start.zig
index e79dd1b9a7..1b686e43f5 100644
--- a/lib/std/start.zig
+++ b/lib/std/start.zig
@@ -19,6 +19,7 @@ const start_sym_name = if (native_arch.isMIPS()) "__start" else "_start";
 // self-hosted is capable enough to handle all of the real start.zig logic.
 pub const simplified_logic =
     builtin.zig_backend == .stage2_wasm or
+    (builtin.zig_backend == .stage2_x86_64 and (builtin.link_libc or builtin.os.tag == .plan9)) or
     builtin.zig_backend == .stage2_x86 or
     builtin.zig_backend == .stage2_aarch64 or
     builtin.zig_backend == .stage2_arm or
diff --git a/test/cases/aarch64-macos/hello_world_with_updates.0.zig b/test/cases/aarch64-macos/hello_world_with_updates.0.zig
index 9f516ff139..90aa14ab65 100644
--- a/test/cases/aarch64-macos/hello_world_with_updates.0.zig
+++ b/test/cases/aarch64-macos/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=aarch64-macos
 //
-// :109:9: error: root struct of file 'tmp' has no member named 'main'
+// :110:9: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-linux/hello_world_with_updates.0.zig b/test/cases/x86_64-linux/hello_world_with_updates.0.zig
index 40abdd6c1f..e498eacf93 100644
--- a/test/cases/x86_64-linux/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-linux/hello_world_with_updates.0.zig
@@ -2,4 +2,7 @@
 // output_mode=Exe
 // target=x86_64-linux
 //
-// :109:9: error: root struct of file 'tmp' has no member named 'main'
+// :604:45: error: root struct of file 'tmp' has no member named 'main'
+// :553:12: note: called from here
+// :503:36: note: called from here
+// :466:17: note: called from here
diff --git a/test/cases/x86_64-macos/hello_world_with_updates.0.zig b/test/cases/x86_64-macos/hello_world_with_updates.0.zig
index e0680c81d7..d91bd9dc91 100644
--- a/test/cases/x86_64-macos/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-macos/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=x86_64-macos
 //
-// :109:9: error: root struct of file 'tmp' has no member named 'main'
+// :110:9: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-windows/hello_world_with_updates.0.zig b/test/cases/x86_64-windows/hello_world_with_updates.0.zig
index 04e1d4cfad..320057b86e 100644
--- a/test/cases/x86_64-windows/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-windows/hello_world_with_updates.0.zig
@@ -2,4 +2,6 @@
 // output_mode=Exe
 // target=x86_64-windows
 //
-// :130:9: error: root struct of file 'tmp' has no member named 'main'
+// :604:45: error: root struct of file 'tmp' has no member named 'main'
+// :553:12: note: called from here
+// :389:65: note: called from here

From 83b7dbe52f75161a2ac6f5d8f39b275fd9473c15 Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Tue, 4 Apr 2023 11:39:02 -0400
Subject: [PATCH 13/13] cases: disable failing incremental tests

Enabling start.zig logic breaks incremental compilation
See #15174
---
 .../cases/aarch64-macos/hello_world_with_updates.0.zig |  2 +-
 test/cases/x86_64-linux/hello_world_with_updates.0.zig |  8 ++++----
 test/cases/x86_64-macos/hello_world_with_updates.0.zig |  2 +-
 .../x86_64-windows/hello_world_with_updates.0.zig      |  6 +++---
 test/src/Cases.zig                                     | 10 ++++++----
 5 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/test/cases/aarch64-macos/hello_world_with_updates.0.zig b/test/cases/aarch64-macos/hello_world_with_updates.0.zig
index 90aa14ab65..f939a7fa71 100644
--- a/test/cases/aarch64-macos/hello_world_with_updates.0.zig
+++ b/test/cases/aarch64-macos/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=aarch64-macos
 //
-// :110:9: error: root struct of file 'tmp' has no member named 'main'
+// :?:?: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-linux/hello_world_with_updates.0.zig b/test/cases/x86_64-linux/hello_world_with_updates.0.zig
index e498eacf93..c330234fdb 100644
--- a/test/cases/x86_64-linux/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-linux/hello_world_with_updates.0.zig
@@ -2,7 +2,7 @@
 // output_mode=Exe
 // target=x86_64-linux
 //
-// :604:45: error: root struct of file 'tmp' has no member named 'main'
-// :553:12: note: called from here
-// :503:36: note: called from here
-// :466:17: note: called from here
+// :?:?: error: root struct of file 'tmp' has no member named 'main'
+// :?:?: note: called from here
+// :?:?: note: called from here
+// :?:?: note: called from here
diff --git a/test/cases/x86_64-macos/hello_world_with_updates.0.zig b/test/cases/x86_64-macos/hello_world_with_updates.0.zig
index d91bd9dc91..7fc13a4fa5 100644
--- a/test/cases/x86_64-macos/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-macos/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=x86_64-macos
 //
-// :110:9: error: root struct of file 'tmp' has no member named 'main'
+// :?:?: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-windows/hello_world_with_updates.0.zig b/test/cases/x86_64-windows/hello_world_with_updates.0.zig
index 320057b86e..d322de2d1b 100644
--- a/test/cases/x86_64-windows/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-windows/hello_world_with_updates.0.zig
@@ -2,6 +2,6 @@
 // output_mode=Exe
 // target=x86_64-windows
 //
-// :604:45: error: root struct of file 'tmp' has no member named 'main'
-// :553:12: note: called from here
-// :389:65: note: called from here
+// :?:?: error: root struct of file 'tmp' has no member named 'main'
+// :?:?: note: called from here
+// :?:?: note: called from here
diff --git a/test/src/Cases.zig b/test/src/Cases.zig
index c3a4c1df47..890f4f6f89 100644
--- a/test/src/Cases.zig
+++ b/test/src/Cases.zig
@@ -1045,8 +1045,7 @@ pub fn main() !void {
     var ctx = Cases.init(gpa, arena);
 
     var test_it = TestIterator{ .filenames = filenames.items };
-    while (test_it.next()) |maybe_batch| {
-        const batch = maybe_batch orelse break;
+    while (try test_it.next()) |batch| {
         const strategy: TestStrategy = if (batch.len > 1) .incremental else .independent;
         var cases = std.ArrayList(usize).init(arena);
 
@@ -1084,6 +1083,11 @@ pub fn main() !void {
 
             for (cases.items) |case_index| {
                 const case = &ctx.cases.items[case_index];
+                if (strategy == .incremental and case.backend == .stage2 and case.target.getCpuArch() == .x86_64 and !case.link_libc and case.target.getOsTag() != .plan9) {
+                    // https://github.com/ziglang/zig/issues/15174
+                    continue;
+                }
+
                 switch (manifest.type) {
                     .compile => {
                         case.addCompile(src);
@@ -1115,8 +1119,6 @@ pub fn main() !void {
                 }
             }
         }
-    } else |err| {
-        return err;
     }
 
     return runCases(&ctx, zig_exe_path);