diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index 98d0fbc978..bad514bcd9 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -102,9 +102,12 @@ air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init, const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}; const MCValue = union(enum) { - /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. - /// TODO Look into deleting this tag and using `dead` instead, since every use - /// of MCValue.none should be instead looking at the type and noticing it is 0 bits. + /// No runtime bits. `void` types, empty structs, u0, enums with 1 + /// tag, etc. + /// + /// TODO Look into deleting this tag and using `dead` instead, + /// since every use of MCValue.none should be instead looking at + /// the type and noticing it is 0 bits. none, /// Control flow will not allow this value to be observed. unreach, @@ -113,28 +116,56 @@ const MCValue = union(enum) { /// The value is undefined. undef, /// A pointer-sized integer that fits in a register. - /// If the type is a pointer, this is the pointer address in virtual address space. + /// + /// If the type is a pointer, this is the pointer address in + /// virtual address space. immediate: u64, /// The value is in a target-specific register. register: Register, + /// The value is a tuple { wrapped: u32, overflow: u1 } where + /// wrapped is stored in the register and the overflow bit is + /// stored in the C flag of the CPSR. + /// + /// This MCValue is only generated by a add_with_overflow or + /// sub_with_overflow instruction operating on u32. + register_c_flag: Register, + /// The value is a tuple { wrapped: i32, overflow: u1 } where + /// wrapped is stored in the register and the overflow bit is + /// stored in the V flag of the CPSR. + /// + /// This MCValue is only generated by a add_with_overflow or + /// sub_with_overflow instruction operating on i32. + register_v_flag: Register, /// The value is in memory at a hard-coded address. - /// If the type is a pointer, it means the pointer address is at this memory location. + /// + /// If the type is a pointer, it means the pointer address is at + /// this memory location. memory: u64, - /// The value is in memory referenced indirectly via a GOT entry index. - /// If the type is a pointer, it means the pointer is referenced indirectly via GOT. - /// When lowered, linker will emit relocations of type ARM64_RELOC_GOT_LOAD_PAGE21 and ARM64_RELOC_GOT_LOAD_PAGEOFF12. + /// The value is in memory referenced indirectly via a GOT entry + /// index. + /// + /// If the type is a pointer, it means the pointer is referenced + /// indirectly via GOT. When lowered, linker will emit + /// relocations of type ARM64_RELOC_GOT_LOAD_PAGE21 and + /// ARM64_RELOC_GOT_LOAD_PAGEOFF12. got_load: u32, /// The value is in memory referenced directly via symbol index. - /// If the type is a pointer, it means the pointer is referenced directly via symbol index. - /// When lowered, linker will emit a relocation of type ARM64_RELOC_PAGE21 and ARM64_RELOC_PAGEOFF12. + /// + /// If the type is a pointer, it means the pointer is referenced + /// directly via symbol index. When lowered, linker will emit a + /// relocation of type ARM64_RELOC_PAGE21 and + /// ARM64_RELOC_PAGEOFF12. direct_load: u32, /// The value is one of the stack variables. - /// If the type is a pointer, it means the pointer address is in the stack at this offset. + /// + /// If the type is a pointer, it means the pointer address is in + /// the stack at this offset. stack_offset: u32, - /// The value is a pointer to one of the stack variables (payload is stack offset). + /// The value is a pointer to one of the stack variables (payload + /// is stack offset). ptr_stack_offset: u32, - /// The value is in the compare flags assuming an unsigned operation, - /// with this operator applied on top of it. + /// The value is in the compare flags assuming an unsigned + /// operation, with this operator applied on top of it. compare_flags_unsigned: math.CompareOperator, /// The value is in the compare flags assuming a signed operation, /// with this operator applied on top of it. @@ -716,8 +747,13 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void { branch.inst_table.putAssumeCapacity(inst, .dead); switch (prev_value) { .register => |reg| { - const canon_reg = toCanonicalReg(reg); - self.register_manager.freeReg(canon_reg); + self.register_manager.freeReg(reg); + }, + .register_c_flag, + .register_v_flag, + => |reg| { + self.register_manager.freeReg(reg); + self.compare_flags_inst = null; }, .compare_flags_signed, .compare_flags_unsigned => { self.compare_flags_inst = null; @@ -857,7 +893,13 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void const stack_mcv = try self.allocRegOrMem(inst, false); log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv }); const reg_mcv = self.getResolvedInstValue(inst); - assert(reg == toCanonicalReg(reg_mcv.register)); + switch (reg_mcv) { + .register, + .register_c_flag, + .register_v_flag, + => |r| assert(reg.id() == r.id()), + else => unreachable, // not a register + } const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; try branch.inst_table.put(self.gpa, inst, stack_mcv); try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv); @@ -868,7 +910,14 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void fn spillCompareFlagsIfOccupied(self: *Self) !void { if (self.compare_flags_inst) |inst_to_save| { const mcv = self.getResolvedInstValue(inst_to_save); - assert(mcv == .compare_flags_signed or mcv == .compare_flags_unsigned); + switch (mcv) { + .compare_flags_signed, + .compare_flags_unsigned, + .register_c_flag, + .register_v_flag, + => {}, + else => unreachable, // mcv doesn't occupy the compare flags + } const new_mcv = try self.allocRegOrMem(inst_to_save, true); try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv); @@ -1269,7 +1318,9 @@ fn binOpRegister( const mir_data: Mir.Inst.Data = switch (mir_tag) { .add_shifted_register, + .adds_shifted_register, .sub_shifted_register, + .subs_shifted_register, => .{ .rrr_imm6_shift = .{ .rd = dest_reg, .rn = lhs_reg, @@ -1384,7 +1435,9 @@ fn binOpImmediate( const mir_data: Mir.Inst.Data = switch (mir_tag) { .add_immediate, + .adds_immediate, .sub_immediate, + .subs_immediate, => .{ .rr_imm12_sh = .{ .rd = dest_reg, .rn = lhs_reg, @@ -1774,7 +1827,52 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void { break :result MCValue{ .stack_offset = stack_offset }; }, - 32, 64 => return self.fail("TODO overflow operations on integers u32/i32 and u64/i64", .{}), + 32, 64 => { + // Only say yes if the operation is + // commutative, i.e. we can swap both of the + // operands + const lhs_immediate_ok = switch (tag) { + .add_with_overflow => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), + .sub_with_overflow => false, + else => unreachable, + }; + const rhs_immediate_ok = switch (tag) { + .add_with_overflow, + .sub_with_overflow, + => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12), + else => unreachable, + }; + + const mir_tag_register: Mir.Inst.Tag = switch (tag) { + .add_with_overflow => .adds_shifted_register, + .sub_with_overflow => .subs_shifted_register, + else => unreachable, + }; + const mir_tag_immediate: Mir.Inst.Tag = switch (tag) { + .add_with_overflow => .adds_immediate, + .sub_with_overflow => .subs_immediate, + else => unreachable, + }; + + try self.spillCompareFlagsIfOccupied(); + self.compare_flags_inst = inst; + + const dest = blk: { + if (rhs_immediate_ok) { + break :blk try self.binOpImmediate(mir_tag_immediate, null, lhs, rhs, lhs_ty, false); + } else if (lhs_immediate_ok) { + // swap lhs and rhs + break :blk try self.binOpImmediate(mir_tag_immediate, null, rhs, lhs, rhs_ty, true); + } else { + break :blk try self.binOpRegister(mir_tag_register, null, lhs, rhs, lhs_ty, rhs_ty); + } + }; + + switch (int_info.signedness) { + .unsigned => break :result MCValue{ .register_c_flag = dest.register }, + .signed => break :result MCValue{ .register_v_flag = dest.register }, + } + }, else => return self.fail("TODO overflow operations on integers > u32/i32", .{}), } }, @@ -2148,8 +2246,11 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo .undef => unreachable, .unreach => unreachable, .dead => unreachable, - .compare_flags_unsigned => unreachable, - .compare_flags_signed => unreachable, + .compare_flags_unsigned, + .compare_flags_signed, + .register_c_flag, + .register_v_flag, + => unreachable, // cannot hold an address .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }), .ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }), .register => |addr_reg| { @@ -2366,8 +2467,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .undef => unreachable, .unreach => unreachable, .dead => unreachable, - .compare_flags_unsigned => unreachable, - .compare_flags_signed => unreachable, + .compare_flags_unsigned, + .compare_flags_signed, + .register_c_flag, + .register_v_flag, + => unreachable, // cannot hold an address .immediate => |imm| { try self.setRegOrMem(value_ty, .{ .memory = imm }, value); }, @@ -2487,6 +2591,40 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { .memory => |addr| { break :result MCValue{ .memory = addr + struct_field_offset }; }, + .register_c_flag, + .register_v_flag, + => |reg| { + switch (index) { + 0 => { + // get wrapped value: return register + break :result MCValue{ .register = reg }; + }, + 1 => { + // TODO return special MCValue condition flags + // get overflow bit: set register to C flag + // resp. V flag + const raw_dest_reg = try self.register_manager.allocReg(null); + const dest_reg = raw_dest_reg.to32(); + + // C flag: cset reg, cs + // V flag: cset reg, vs + _ = try self.addInst(.{ + .tag = .cset, + .data = .{ .r_cond = .{ + .rd = dest_reg, + .cond = switch (mcv) { + .register_c_flag => .cs, + .register_v_flag => .vs, + else => unreachable, + }, + } }, + }); + + break :result MCValue{ .register = dest_reg }; + }, + else => unreachable, + } + }, else => return self.fail("TODO implement codegen struct_field_val for {}", .{mcv}), } }; @@ -2531,7 +2669,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void { switch (mcv) { .register => |reg| { - self.register_manager.getRegAssumeFree(toCanonicalReg(reg), inst); + self.register_manager.getRegAssumeFree(reg, inst); }, else => {}, } @@ -2596,15 +2734,6 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. switch (mc_arg) { .none => continue, - .undef => unreachable, - .immediate => unreachable, - .unreach => unreachable, - .dead => unreachable, - .memory => unreachable, - .compare_flags_signed => unreachable, - .compare_flags_unsigned => unreachable, - .got_load => unreachable, - .direct_load => unreachable, .register => |reg| { try self.register_manager.getReg(reg, null); try self.genSetReg(arg_ty, reg, arg_mcv); @@ -2615,6 +2744,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. .ptr_stack_offset => { return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{}); }, + else => unreachable, } } @@ -3518,6 +3648,11 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro else => return self.fail("TODO implement storing other types abi_size={}", .{abi_size}), } }, + .register_c_flag, + .register_v_flag, + => { + return self.fail("TODO implement genSetStack {}", .{mcv}); + }, .got_load, .direct_load, .memory, @@ -3635,7 +3770,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void .tag = .cset, .data = .{ .r_cond = .{ .rd = reg, - .cond = condition.negate(), + .cond = condition, } }, }); }, @@ -3678,6 +3813,9 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void .data = .{ .rr = .{ .rd = reg, .rn = src_reg } }, }); }, + .register_c_flag, + .register_v_flag, + => unreachable, // doesn't fit into a register .got_load, .direct_load, => |sym_index| { @@ -4279,8 +4417,3 @@ fn registerAlias(reg: Register, size_bytes: u64) Register { unreachable; // TODO handle floating-point registers } } - -/// Resolves any aliased registers to the 64-bit wide ones. -fn toCanonicalReg(reg: Register) Register { - return reg.to64(); -} diff --git a/src/arch/aarch64/Emit.zig b/src/arch/aarch64/Emit.zig index d9dfbc6fac..85389f445e 100644 --- a/src/arch/aarch64/Emit.zig +++ b/src/arch/aarch64/Emit.zig @@ -77,8 +77,10 @@ pub fn emitMir( const inst = @intCast(u32, index); switch (tag) { .add_immediate => try emit.mirAddSubtractImmediate(inst), + .adds_immediate => try emit.mirAddSubtractImmediate(inst), .cmp_immediate => try emit.mirAddSubtractImmediate(inst), .sub_immediate => try emit.mirAddSubtractImmediate(inst), + .subs_immediate => try emit.mirAddSubtractImmediate(inst), .asr_register => try emit.mirShiftRegister(inst), .lsl_register => try emit.mirShiftRegister(inst), @@ -106,8 +108,10 @@ pub fn emitMir( .eor_immediate => try emit.mirLogicalImmediate(inst), .add_shifted_register => try emit.mirAddSubtractShiftedRegister(inst), + .adds_shifted_register => try emit.mirAddSubtractShiftedRegister(inst), .cmp_shifted_register => try emit.mirAddSubtractShiftedRegister(inst), .sub_shifted_register => try emit.mirAddSubtractShiftedRegister(inst), + .subs_shifted_register => try emit.mirAddSubtractShiftedRegister(inst), .cset => try emit.mirConditionalSelect(inst), @@ -454,7 +458,9 @@ fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void { const tag = emit.mir.instructions.items(.tag)[inst]; switch (tag) { .add_immediate, + .adds_immediate, .sub_immediate, + .subs_immediate, => { const rr_imm12_sh = emit.mir.instructions.items(.data)[inst].rr_imm12_sh; const rd = rr_imm12_sh.rd; @@ -464,7 +470,9 @@ fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void { switch (tag) { .add_immediate => try emit.writeInstruction(Instruction.add(rd, rn, imm12, sh)), + .adds_immediate => try emit.writeInstruction(Instruction.adds(rd, rn, imm12, sh)), .sub_immediate => try emit.writeInstruction(Instruction.sub(rd, rn, imm12, sh)), + .subs_immediate => try emit.writeInstruction(Instruction.subs(rd, rn, imm12, sh)), else => unreachable, } }, @@ -674,7 +682,9 @@ fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void { const tag = emit.mir.instructions.items(.tag)[inst]; switch (tag) { .add_shifted_register, + .adds_shifted_register, .sub_shifted_register, + .subs_shifted_register, => { const rrr_imm6_shift = emit.mir.instructions.items(.data)[inst].rrr_imm6_shift; const rd = rrr_imm6_shift.rd; @@ -685,7 +695,9 @@ fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void { switch (tag) { .add_shifted_register => try emit.writeInstruction(Instruction.addShiftedRegister(rd, rn, rm, shift, imm6)), + .adds_shifted_register => try emit.writeInstruction(Instruction.addsShiftedRegister(rd, rn, rm, shift, imm6)), .sub_shifted_register => try emit.writeInstruction(Instruction.subShiftedRegister(rd, rn, rm, shift, imm6)), + .subs_shifted_register => try emit.writeInstruction(Instruction.subsShiftedRegister(rd, rn, rm, shift, imm6)), else => unreachable, } }, @@ -717,7 +729,7 @@ fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void { 64 => .xzr, else => unreachable, }; - try emit.writeInstruction(Instruction.csinc(r_cond.rd, zr, zr, r_cond.cond)); + try emit.writeInstruction(Instruction.csinc(r_cond.rd, zr, zr, r_cond.cond.negate())); }, else => unreachable, } diff --git a/src/arch/aarch64/Mir.zig b/src/arch/aarch64/Mir.zig index 516ccca984..8c5b635649 100644 --- a/src/arch/aarch64/Mir.zig +++ b/src/arch/aarch64/Mir.zig @@ -26,8 +26,12 @@ pub const Inst = struct { pub const Tag = enum(u16) { /// Add (immediate) add_immediate, + /// Add, update condition flags (immediate) + adds_immediate, /// Add (shifted register) add_shifted_register, + /// Add, update condition flags (shifted register) + adds_shifted_register, /// Bitwise AND (shifted register) and_shifted_register, /// Arithmetic Shift Right (immediate) @@ -170,8 +174,12 @@ pub const Inst = struct { strh_register, /// Subtract (immediate) sub_immediate, + /// Subtract, update condition flags (immediate) + subs_immediate, /// Subtract (shifted register) sub_shifted_register, + /// Subtract, update condition flags (shifted register) + subs_shifted_register, /// Supervisor Call svc, /// Unsigned bitfield extract