Legalize: update for new Block API

This commit is contained in:
Jacob Young 2025-05-30 15:47:18 -04:00 committed by mlugg
parent d9b6d1ed33
commit 32a57bfeaa
No known key found for this signature in database
GPG Key ID: 3F5B7DCCBF4AF02E
2 changed files with 139 additions and 174 deletions

View File

@ -1241,10 +1241,10 @@ pub const CondBr = struct {
else_body_len: u32,
branch_hints: BranchHints,
pub const BranchHints = packed struct(u32) {
true: std.builtin.BranchHint,
false: std.builtin.BranchHint,
then_cov: CoveragePoint,
else_cov: CoveragePoint,
true: std.builtin.BranchHint = .none,
false: std.builtin.BranchHint = .none,
then_cov: CoveragePoint = .none,
else_cov: CoveragePoint = .none,
_: u24 = 0,
};
};

View File

@ -575,20 +575,20 @@ inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_tag:
fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_tag: ScalarizeDataTag) Error!Air.Inst.Data {
const pt = l.pt;
const zcu = pt.zcu;
const gpa = zcu.gpa;
const orig = l.air_instructions.get(@intFromEnum(orig_inst));
const res_ty = l.typeOfIndex(orig_inst);
const arity = switch (data_tag) {
.un_op, .ty_op => 1,
.bin_op, .ty_pl_vector_cmp => 2,
.pl_op_bin => 3,
};
const expected_instructions_len = l.air_instructions.len + (6 + arity + 8);
try l.air_instructions.ensureTotalCapacity(gpa, expected_instructions_len);
var res_block_buf: [4]Air.Inst.Index = undefined;
var res_block: Block = .init(&res_block_buf);
var inst_buf: [
5 + switch (data_tag) {
.un_op, .ty_op => 1,
.bin_op, .ty_pl_vector_cmp => 2,
.pl_op_bin => 3,
} + 9
]Air.Inst.Index = undefined;
try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
var res_block: Block = .init(&inst_buf);
{
const res_alloc_inst = res_block.add(l, .{
.tag = .alloc,
@ -606,39 +606,38 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_
} },
});
const loop_inst: Air.Inst.Index = @enumFromInt(l.air_instructions.len + (3 + arity + 7));
var loop_block_buf: [3 + arity + 2]Air.Inst.Index = undefined;
var loop_block: Block = .init(&loop_block_buf);
var loop: Loop = .init(l, &res_block);
loop.block = .init(res_block.stealRemainingCapacity());
{
const cur_index_inst = loop_block.add(l, .{
const cur_index_inst = loop.block.add(l, .{
.tag = .load,
.data = .{ .ty_op = .{
.ty = .usize_type,
.operand = index_alloc_inst.toRef(),
} },
});
_ = loop_block.add(l, .{
_ = loop.block.add(l, .{
.tag = .vector_store_elem,
.data = .{ .vector_store_elem = .{
.vector_ptr = res_alloc_inst.toRef(),
.payload = try l.addExtra(Air.Bin, .{
.lhs = cur_index_inst.toRef(),
.rhs = loop_block.add(l, res_elem: switch (data_tag) {
.un_op => .{
.rhs = res_elem: switch (data_tag) {
.un_op => loop.block.add(l, .{
.tag = orig.tag,
.data = .{ .un_op = loop_block.add(l, .{
.data = .{ .un_op = loop.block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = orig.data.un_op,
.rhs = cur_index_inst.toRef(),
} },
}).toRef() },
},
.ty_op => .{
}),
.ty_op => loop.block.add(l, .{
.tag = orig.tag,
.data = .{ .ty_op = .{
.ty = Air.internedToRef(orig.data.ty_op.ty.toType().scalarType(zcu).toIntern()),
.operand = loop_block.add(l, .{
.operand = loop.block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = orig.data.ty_op.operand,
@ -646,18 +645,18 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_
} },
}).toRef(),
} },
},
.bin_op => .{
}),
.bin_op => loop.block.add(l, .{
.tag = orig.tag,
.data = .{ .bin_op = .{
.lhs = loop_block.add(l, .{
.lhs = loop.block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = orig.data.bin_op.lhs,
.rhs = cur_index_inst.toRef(),
} },
}).toRef(),
.rhs = loop_block.add(l, .{
.rhs = loop.block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = orig.data.bin_op.rhs,
@ -665,61 +664,47 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_
} },
}).toRef(),
} },
},
}),
.ty_pl_vector_cmp => {
const extra = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;
break :res_elem .{
.tag = switch (orig.tag) {
break :res_elem try loop.block.addCmp(
l,
extra.compareOperator(),
loop.block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = extra.lhs,
.rhs = cur_index_inst.toRef(),
} },
}).toRef(),
loop.block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = extra.rhs,
.rhs = cur_index_inst.toRef(),
} },
}).toRef(),
.{ .optimized = switch (orig.tag) {
else => unreachable,
.cmp_vector => switch (extra.compareOperator()) {
.lt => .cmp_lt,
.lte => .cmp_lte,
.eq => .cmp_eq,
.gte => .cmp_gte,
.gt => .cmp_gt,
.neq => .cmp_neq,
},
.cmp_vector_optimized => switch (extra.compareOperator()) {
.lt => .cmp_lt_optimized,
.lte => .cmp_lte_optimized,
.eq => .cmp_eq_optimized,
.gte => .cmp_gte_optimized,
.gt => .cmp_gt_optimized,
.neq => .cmp_neq_optimized,
},
},
.data = .{ .bin_op = .{
.lhs = loop_block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = extra.lhs,
.rhs = cur_index_inst.toRef(),
} },
}).toRef(),
.rhs = loop_block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = extra.rhs,
.rhs = cur_index_inst.toRef(),
} },
}).toRef(),
.cmp_vector => false,
.cmp_vector_optimized => true,
} },
};
);
},
.pl_op_bin => {
const extra = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
break :res_elem .{
break :res_elem loop.block.add(l, .{
.tag = orig.tag,
.data = .{ .pl_op = .{
.payload = try l.addExtra(Air.Bin, .{
.lhs = loop_block.add(l, .{
.lhs = loop.block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = extra.lhs,
.rhs = cur_index_inst.toRef(),
} },
}).toRef(),
.rhs = loop_block.add(l, .{
.rhs = loop.block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = extra.rhs,
@ -727,7 +712,7 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_
} },
}).toRef(),
}),
.operand = loop_block.add(l, .{
.operand = loop.block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = orig.data.pl_op.operand,
@ -735,28 +720,32 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_
} },
}).toRef(),
} },
};
});
},
}).toRef(),
}.toRef(),
}),
} },
});
const not_done_inst = loop_block.add(l, .{
.tag = .cmp_lt,
.data = .{ .bin_op = .{
.lhs = cur_index_inst.toRef(),
.rhs = try pt.intRef(.usize, res_ty.vectorLen(zcu) - 1),
} },
});
var not_done_block_buf: [3]Air.Inst.Index = undefined;
var not_done_block: Block = .init(&not_done_block_buf);
var loop_cond_br: CondBr = .init(
l,
(try loop.block.addCmp(
l,
.lt,
cur_index_inst.toRef(),
try pt.intRef(.usize, res_ty.vectorLen(zcu) - 1),
.{},
)).toRef(),
&loop.block,
.{},
);
loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
{
_ = not_done_block.add(l, .{
_ = loop_cond_br.then_block.add(l, .{
.tag = .store,
.data = .{ .bin_op = .{
.lhs = index_alloc_inst.toRef(),
.rhs = not_done_block.add(l, .{
.rhs = loop_cond_br.then_block.add(l, .{
.tag = .add,
.data = .{ .bin_op = .{
.lhs = cur_index_inst.toRef(),
@ -765,19 +754,18 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_
}).toRef(),
} },
});
_ = not_done_block.add(l, .{
_ = loop_cond_br.then_block.add(l, .{
.tag = .repeat,
.data = .{ .repeat = .{ .loop_inst = loop_inst } },
.data = .{ .repeat = .{ .loop_inst = loop.inst } },
});
}
var done_block_buf: [2]Air.Inst.Index = undefined;
var done_block: Block = .init(&done_block_buf);
loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
{
_ = done_block.add(l, .{
_ = loop_cond_br.else_block.add(l, .{
.tag = .br,
.data = .{ .br = .{
.block_inst = orig_inst,
.operand = done_block.add(l, .{
.operand = loop_cond_br.else_block.add(l, .{
.tag = .load,
.data = .{ .ty_op = .{
.ty = Air.internedToRef(res_ty.toIntern()),
@ -787,32 +775,19 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_
} },
});
}
_ = loop_block.add(l, .{
.tag = .cond_br,
.data = .{ .pl_op = .{
.operand = not_done_inst.toRef(),
.payload = try l.addCondBrBodies(not_done_block.body(), done_block.body()),
} },
});
try loop_cond_br.finish(l);
}
assert(loop_inst == res_block.add(l, .{
.tag = .loop,
.data = .{ .ty_pl = .{
.ty = .noreturn_type,
.payload = try l.addBlockBody(loop_block.body()),
} },
}));
try loop.finish(l);
}
assert(l.air_instructions.len == expected_instructions_len);
return .{ .ty_pl = .{
.ty = Air.internedToRef(res_ty.toIntern()),
.payload = try l.addBlockBody(res_block.body()),
} };
}
fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
const pt = l.pt;
const zcu = pt.zcu;
const gpa = zcu.gpa;
const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
const operand_ref = ty_op.operand;
@ -882,12 +857,12 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
// })
// })
try l.air_instructions.ensureUnusedCapacity(gpa, 12);
var body_inst_buf: [12]Air.Inst.Index = undefined;
var inst_buf: [12]Air.Inst.Index = undefined;
try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
var condbr_buf: [2]CondBr = undefined;
var condbr_idx: usize = 0;
var main_block: Block = .init(&body_inst_buf);
var main_block: Block = .init(&inst_buf);
var cur_block: *Block = &main_block;
const panic_id: Zcu.SimplePanicId = if (dest_is_enum) .invalid_enum_value else .cast_truncated_data;
@ -898,11 +873,11 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
condbr_idx += 1;
const below_min_inst: Air.Inst.Index = if (have_min_check) inst: {
const min_val_ref = Air.internedToRef((try dest_int_ty.minInt(pt, operand_ty)).toIntern());
break :inst try cur_block.addCmp(l, is_vector, .lt, operand_ref, min_val_ref);
break :inst try cur_block.addCmp(l, .lt, operand_ref, min_val_ref, .{ .vector = is_vector });
} else undefined;
const above_max_inst: Air.Inst.Index = if (have_max_check) inst: {
const max_val_ref = Air.internedToRef((try dest_int_ty.maxInt(pt, operand_ty)).toIntern());
break :inst try cur_block.addCmp(l, is_vector, .gt, operand_ref, max_val_ref);
break :inst try cur_block.addCmp(l, .gt, operand_ref, max_val_ref, .{ .vector = is_vector });
} else undefined;
const out_of_range_inst: Air.Inst.Index = inst: {
if (have_min_check and have_max_check) break :inst cur_block.add(l, .{
@ -923,10 +898,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
.operation = .Or,
} },
}) else out_of_range_inst;
condbr.* = .init(l, scalar_out_of_range_inst.toRef(), cur_block, .{
.true = .cold,
.false = .none,
});
condbr.* = .init(l, scalar_out_of_range_inst.toRef(), cur_block, .{ .true = .cold });
condbr.then_block = .init(cur_block.stealRemainingCapacity());
try condbr.then_block.addPanic(l, panic_id);
condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
@ -957,10 +929,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
});
const condbr = &condbr_buf[condbr_idx];
condbr_idx += 1;
condbr.* = .init(l, is_named_inst.toRef(), cur_block, .{
.true = .none,
.false = .cold,
});
condbr.* = .init(l, is_named_inst.toRef(), cur_block, .{ .false = .cold });
condbr.else_block = .init(cur_block.stealRemainingCapacity());
try condbr.else_block.addPanic(l, panic_id);
condbr.then_block = .init(condbr.else_block.stealRemainingCapacity());
@ -986,7 +955,6 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_op_tag: Air.Inst.Tag) Error!Air.Inst.Data {
const pt = l.pt;
const zcu = pt.zcu;
const gpa = zcu.gpa;
const bin_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].bin_op;
const operand_ty = l.typeOf(bin_op.lhs);
@ -1009,10 +977,10 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_
// %8 = struct_field_val(%1, .@"0")
// %9 = br(%z, %8)
// })
try l.air_instructions.ensureUnusedCapacity(gpa, 9);
var body_inst_buf: [9]Air.Inst.Index = undefined;
var inst_buf: [9]Air.Inst.Index = undefined;
try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
var main_block: Block = .init(&body_inst_buf);
var main_block: Block = .init(&inst_buf);
const overflow_op_inst = main_block.add(l, .{
.tag = overflow_op_tag,
@ -1041,12 +1009,9 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_
.operation = .Or,
} },
}) else overflow_bits_inst;
const any_overflow_inst = try main_block.addCmp(l, false, .eq, any_overflow_bit_inst.toRef(), .one_u1);
const any_overflow_inst = try main_block.addCmp(l, .eq, any_overflow_bit_inst.toRef(), .one_u1, .{});
var condbr: CondBr = .init(l, any_overflow_inst.toRef(), &main_block, .{
.true = .cold,
.false = .none,
});
var condbr: CondBr = .init(l, any_overflow_inst.toRef(), &main_block, .{ .true = .cold });
condbr.then_block = .init(main_block.stealRemainingCapacity());
try condbr.then_block.addPanic(l, .integer_overflow);
condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
@ -1130,19 +1095,19 @@ const Block = struct {
fn addCmp(
b: *Block,
l: *Legalize,
is_vector: bool,
op: std.math.CompareOperator,
lhs: Air.Inst.Ref,
rhs: Air.Inst.Ref,
opts: struct { optimized: bool = false, vector: bool = false },
) Error!Air.Inst.Index {
const pt = l.pt;
if (is_vector) {
if (opts.vector) {
const bool_vec_ty = try pt.vectorType(.{
.child = .bool_type,
.len = l.typeOf(lhs).vectorLen(pt.zcu),
});
return b.add(l, .{
.tag = .cmp_vector,
.tag = if (opts.optimized) .cmp_vector_optimized else .cmp_vector,
.data = .{ .ty_pl = .{
.ty = Air.internedToRef(bool_vec_ty.toIntern()),
.payload = try l.addExtra(Air.VectorCmp, .{
@ -1155,12 +1120,12 @@ const Block = struct {
}
return b.add(l, .{
.tag = switch (op) {
.lt => .cmp_lt,
.lte => .cmp_lte,
.eq => .cmp_eq,
.gte => .cmp_gte,
.gt => .cmp_gt,
.neq => .cmp_neq,
.lt => if (opts.optimized) .cmp_lt_optimized else .cmp_lt,
.lte => if (opts.optimized) .cmp_lte_optimized else .cmp_lte,
.eq => if (opts.optimized) .cmp_eq_optimized else .cmp_eq,
.gte => if (opts.optimized) .cmp_gte_optimized else .cmp_gte,
.gt => if (opts.optimized) .cmp_gt_optimized else .cmp_gt,
.neq => if (opts.optimized) .cmp_neq_optimized else .cmp_neq,
},
.data = .{ .bin_op = .{
.lhs = lhs,
@ -1184,20 +1149,40 @@ const Block = struct {
}
};
const Loop = struct {
inst: Air.Inst.Index,
block: Block,
/// The return value has `block` initialized to `undefined`; it is the caller's reponsibility
/// to initialize it.
fn init(l: *Legalize, parent_block: *Block) Loop {
return .{
.inst = parent_block.add(l, .{
.tag = .loop,
.data = .{ .ty_pl = .{
.ty = .noreturn_type,
.payload = undefined,
} },
}),
.block = undefined,
};
}
fn finish(loop: Loop, l: *Legalize) Error!void {
const data = &l.air_instructions.items(.data)[@intFromEnum(loop.inst)];
data.ty_pl.payload = try l.addBlockBody(loop.block.body());
}
};
const CondBr = struct {
inst: Air.Inst.Index,
hints: BranchHints,
hints: Air.CondBr.BranchHints,
then_block: Block,
else_block: Block,
const BranchHints = struct {
true: std.builtin.BranchHint,
false: std.builtin.BranchHint,
};
/// The return value has `then_block` and `else_block` initialized to `undefined`; it is the
/// caller's reponsibility to initialize them.
fn init(l: *Legalize, operand: Air.Inst.Ref, parent_block: *Block, hints: BranchHints) CondBr {
fn init(l: *Legalize, operand: Air.Inst.Ref, parent_block: *Block, hints: Air.CondBr.BranchHints) CondBr {
return .{
.inst = parent_block.add(l, .{
.tag = .cond_br,
@ -1213,17 +1198,19 @@ const CondBr = struct {
}
fn finish(cond_br: CondBr, l: *Legalize) Error!void {
const then_body = cond_br.then_block.body();
const else_body = cond_br.else_block.body();
try l.air_extra.ensureUnusedCapacity(l.pt.zcu.gpa, 3 + then_body.len + else_body.len);
const data = &l.air_instructions.items(.data)[@intFromEnum(cond_br.inst)];
data.pl_op.payload = try l.addCondBrBodiesHints(
cond_br.then_block.body(),
cond_br.else_block.body(),
.{
.true = cond_br.hints.true,
.false = cond_br.hints.false,
.then_cov = .none,
.else_cov = .none,
},
);
data.pl_op.payload = @intCast(l.air_extra.items.len);
l.air_extra.appendSliceAssumeCapacity(&.{
@intCast(then_body.len),
@intCast(else_body.len),
@bitCast(cond_br.hints),
});
l.air_extra.appendSliceAssumeCapacity(@ptrCast(then_body));
l.air_extra.appendSliceAssumeCapacity(@ptrCast(else_body));
}
};
@ -1252,28 +1239,6 @@ fn addBlockBody(l: *Legalize, body: []const Air.Inst.Index) Error!u32 {
return @intCast(l.air_extra.items.len);
}
fn addCondBrBodies(l: *Legalize, then_body: []const Air.Inst.Index, else_body: []const Air.Inst.Index) Error!u32 {
return l.addCondBrBodiesHints(then_body, else_body, .{
.true = .none,
.false = .none,
.then_cov = .none,
.else_cov = .none,
});
}
fn addCondBrBodiesHints(l: *Legalize, then_body: []const Air.Inst.Index, else_body: []const Air.Inst.Index, hints: Air.CondBr.BranchHints) Error!u32 {
try l.air_extra.ensureUnusedCapacity(l.pt.zcu.gpa, 3 + then_body.len + else_body.len);
defer {
l.air_extra.appendSliceAssumeCapacity(&.{
@intCast(then_body.len),
@intCast(else_body.len),
@bitCast(hints),
});
l.air_extra.appendSliceAssumeCapacity(@ptrCast(then_body));
l.air_extra.appendSliceAssumeCapacity(@ptrCast(else_body));
}
return @intCast(l.air_extra.items.len);
}
// inline to propagate comptime `tag`s
inline fn replaceInst(l: *Legalize, inst: Air.Inst.Index, tag: Air.Inst.Tag, data: Air.Inst.Data) Air.Inst.Tag {
const orig_ty = if (std.debug.runtime_safety) l.typeOfIndex(inst) else {};