Merge pull request #23986 from mlugg/incremental-stuff

incremental: bugfix (and a debugging feature that helped me do that bugfix)
This commit is contained in:
Matthew Lugg 2025-05-25 18:02:16 +01:00 committed by GitHub
commit ef35c3d5fe
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 651 additions and 60 deletions

View File

@ -236,6 +236,8 @@ pub fn main() !void {
graph.debug_compiler_runtime_libs = true;
} else if (mem.eql(u8, arg, "--debug-compile-errors")) {
builder.debug_compile_errors = true;
} else if (mem.eql(u8, arg, "--debug-incremental")) {
builder.debug_incremental = true;
} else if (mem.eql(u8, arg, "--system")) {
// The usage text shows another argument after this parameter
// but it is handled by the parent process. The build runner

View File

@ -59,6 +59,7 @@ pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null,
args: ?[]const []const u8 = null,
debug_log_scopes: []const []const u8 = &.{},
debug_compile_errors: bool = false,
debug_incremental: bool = false,
debug_pkg_config: bool = false,
/// Number of stack frames captured when a `StackTrace` is recorded for debug purposes,
/// in particular at `Step` creation.
@ -385,6 +386,7 @@ fn createChildOnly(
.cache_root = parent.cache_root,
.debug_log_scopes = parent.debug_log_scopes,
.debug_compile_errors = parent.debug_compile_errors,
.debug_incremental = parent.debug_incremental,
.debug_pkg_config = parent.debug_pkg_config,
.enable_darling = parent.enable_darling,
.enable_qemu = parent.enable_qemu,

View File

@ -1447,6 +1447,10 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
try zig_args.append("--debug-compile-errors");
}
if (b.debug_incremental) {
try zig_args.append("--debug-incremental");
}
if (b.verbose_cimport) try zig_args.append("--verbose-cimport");
if (b.verbose_air) try zig_args.append("--verbose-air");
if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path}));

View File

@ -190,6 +190,8 @@ time_report: bool,
stack_report: bool,
debug_compiler_runtime_libs: bool,
debug_compile_errors: bool,
/// Do not check this field directly. Instead, use the `debugIncremental` wrapper function.
debug_incremental: bool,
incremental: bool,
alloc_failure_occurred: bool = false,
last_update_was_cache_hit: bool = false,
@ -768,6 +770,14 @@ pub const Directories = struct {
}
};
/// This small wrapper function just checks whether debug extensions are enabled before checking
/// `comp.debug_incremental`. It is inline so that comptime-known `false` propagates to the caller,
/// preventing debugging features from making it into release builds of the compiler.
pub inline fn debugIncremental(comp: *const Compilation) bool {
if (!build_options.enable_debug_extensions) return false;
return comp.debug_incremental;
}
pub const default_stack_protector_buffer_size = target_util.default_stack_protector_buffer_size;
pub const SemaError = Zcu.SemaError;
@ -1598,6 +1608,7 @@ pub const CreateOptions = struct {
verbose_llvm_cpu_features: bool = false,
debug_compiler_runtime_libs: bool = false,
debug_compile_errors: bool = false,
debug_incremental: bool = false,
incremental: bool = false,
/// Normally when you create a `Compilation`, Zig will automatically build
/// and link in required dependencies, such as compiler-rt and libc. When
@ -1968,6 +1979,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
.test_name_prefix = options.test_name_prefix,
.debug_compiler_runtime_libs = options.debug_compiler_runtime_libs,
.debug_compile_errors = options.debug_compile_errors,
.debug_incremental = options.debug_incremental,
.incremental = options.incremental,
.root_name = root_name,
.sysroot = sysroot,

View File

@ -0,0 +1,383 @@
//! This is a simple TCP server which exposes a REPL useful for debugging incremental compilation
//! issues. Eventually, this logic should move into `std.zig.Client`/`std.zig.Server` or something
//! similar, but for now, this works. The server is enabled by the '--debug-incremental' CLI flag.
//! The easiest way to interact with the REPL is to use `telnet`:
//! ```
//! telnet "::1" 7623
//! ```
//! 'help' will list available commands. When the debug server is enabled, the compiler tracks a lot
//! of extra state (see `Zcu.IncrementalDebugState`), so note that RSS will be higher than usual.
comptime {
// This file should only be referenced when debug extensions are enabled.
std.debug.assert(@import("build_options").enable_debug_extensions);
}
zcu: *Zcu,
thread: ?std.Thread,
running: std.atomic.Value(bool),
/// Held by our owner when an update is in-progress, and held by us when responding to a command.
/// So, essentially guards all access to `Compilation`, including `Zcu`.
mutex: std.Thread.Mutex,
pub fn init(zcu: *Zcu) IncrementalDebugServer {
return .{
.zcu = zcu,
.thread = null,
.running = .init(true),
.mutex = .{},
};
}
pub fn deinit(ids: *IncrementalDebugServer) void {
if (ids.thread) |t| {
ids.running.store(false, .monotonic);
t.join();
}
}
const port = 7623;
pub fn spawn(ids: *IncrementalDebugServer) void {
std.debug.print("spawning incremental debug server on port {d}\n", .{port});
ids.thread = std.Thread.spawn(.{ .allocator = ids.zcu.comp.arena }, runThread, .{ids}) catch |err|
std.process.fatal("failed to spawn incremental debug server: {s}", .{@errorName(err)});
}
fn runThread(ids: *IncrementalDebugServer) void {
const gpa = ids.zcu.gpa;
var cmd_buf: [1024]u8 = undefined;
var text_out: std.ArrayListUnmanaged(u8) = .empty;
defer text_out.deinit(gpa);
const addr = std.net.Address.parseIp6("::", port) catch unreachable;
var server = addr.listen(.{}) catch @panic("IncrementalDebugServer: failed to listen");
defer server.deinit();
const conn = server.accept() catch @panic("IncrementalDebugServer: failed to accept");
defer conn.stream.close();
while (ids.running.load(.monotonic)) {
conn.stream.writeAll("zig> ") catch @panic("IncrementalDebugServer: failed to write");
var fbs = std.io.fixedBufferStream(&cmd_buf);
conn.stream.reader().streamUntilDelimiter(fbs.writer(), '\n', cmd_buf.len) catch |err| switch (err) {
error.EndOfStream => break,
else => @panic("IncrementalDebugServer: failed to read command"),
};
const cmd_and_arg = std.mem.trim(u8, fbs.getWritten(), " \t\r\n");
const cmd: []const u8, const arg: []const u8 = if (std.mem.indexOfScalar(u8, cmd_and_arg, ' ')) |i|
.{ cmd_and_arg[0..i], cmd_and_arg[i + 1 ..] }
else
.{ cmd_and_arg, "" };
text_out.clearRetainingCapacity();
{
if (!ids.mutex.tryLock()) {
conn.stream.writeAll("waiting for in-progress update to finish...\n") catch @panic("IncrementalDebugServer: failed to write");
ids.mutex.lock();
}
defer ids.mutex.unlock();
handleCommand(ids.zcu, &text_out, cmd, arg) catch @panic("IncrementalDebugServer: out of memory");
}
text_out.append(gpa, '\n') catch @panic("IncrementalDebugServer: out of memory");
conn.stream.writeAll(text_out.items) catch @panic("IncrementalDebugServer: failed to write");
}
std.debug.print("closing incremental debug server\n", .{});
}
const help_str: []const u8 =
\\[str] arguments are any string.
\\[id] arguments are a numeric ID/index, like an InternPool index.
\\[unit] arguments are strings like 'func 1234' where '1234' is the relevant index (in this case an InternPool index).
\\
\\MISC
\\ summary
\\ Dump some information about the whole ZCU.
\\ nav_info [id]
\\ Dump basic info about a NAV.
\\
\\SEARCHING
\\ find_type [str]
\\ Find types (including dead ones) whose names contain the given substring.
\\ Starting with '^' or ending with '$' anchors to the start/end of the name.
\\ find_nav [str]
\\ Find NAVs (including dead ones) whose names contain the given substring.
\\ Starting with '^' or ending with '$' anchors to the start/end of the name.
\\
\\UNITS
\\ unit_info [unit]
\\ Dump basic info about an analysis unit.
\\ unit_dependencies [unit]
\\ List all units which an analysis unit depends on.
\\ unit_trace [unit]
\\ Dump the current reference trace of an analysis unit.
\\
\\TYPES
\\ type_info [id]
\\ Dump basic info about a type.
\\ type_namespace [id]
\\ List all declarations in the namespace of a type.
\\
;
fn handleCommand(zcu: *Zcu, output: *std.ArrayListUnmanaged(u8), cmd_str: []const u8, arg_str: []const u8) Allocator.Error!void {
const ip = &zcu.intern_pool;
const gpa = zcu.gpa;
const w = output.writer(gpa);
if (std.mem.eql(u8, cmd_str, "help")) {
try w.writeAll(help_str);
} else if (std.mem.eql(u8, cmd_str, "summary")) {
try w.print(
\\last generation: {d}
\\total container types: {d}
\\total NAVs: {d}
\\total units: {d}
\\
, .{
zcu.generation - 1,
zcu.incremental_debug_state.types.count(),
zcu.incremental_debug_state.navs.count(),
zcu.incremental_debug_state.units.count(),
});
} else if (std.mem.eql(u8, cmd_str, "nav_info")) {
const nav_index: InternPool.Nav.Index = @enumFromInt(parseIndex(arg_str) orelse return w.writeAll("malformed nav index"));
const create_gen = zcu.incremental_debug_state.navs.get(nav_index) orelse return w.writeAll("unknown nav index");
const nav = ip.getNav(nav_index);
try w.print(
\\name: '{}'
\\fqn: '{}'
\\status: {s}
\\created on generation: {d}
\\
, .{
nav.name.fmt(ip),
nav.fqn.fmt(ip),
@tagName(nav.status),
create_gen,
});
switch (nav.status) {
.unresolved => {},
.type_resolved, .fully_resolved => {
try w.writeAll("type: ");
try printType(.fromInterned(nav.typeOf(ip)), zcu, w);
try w.writeByte('\n');
},
}
} else if (std.mem.eql(u8, cmd_str, "find_type")) {
if (arg_str.len == 0) return w.writeAll("bad usage");
const anchor_start = arg_str[0] == '^';
const anchor_end = arg_str[arg_str.len - 1] == '$';
const query = arg_str[@intFromBool(anchor_start) .. arg_str.len - @intFromBool(anchor_end)];
var num_results: usize = 0;
for (zcu.incremental_debug_state.types.keys()) |type_ip_index| {
const ty: Type = .fromInterned(type_ip_index);
const ty_name = ty.containerTypeName(ip).toSlice(ip);
const success = switch (@as(u2, @intFromBool(anchor_start)) << 1 | @intFromBool(anchor_end)) {
0b00 => std.mem.indexOf(u8, ty_name, query) != null,
0b01 => std.mem.endsWith(u8, ty_name, query),
0b10 => std.mem.startsWith(u8, ty_name, query),
0b11 => std.mem.eql(u8, ty_name, query),
};
if (success) {
num_results += 1;
try w.print("* type {d} ('{s}')\n", .{ @intFromEnum(type_ip_index), ty_name });
}
}
try w.print("Found {d} results\n", .{num_results});
} else if (std.mem.eql(u8, cmd_str, "find_nav")) {
if (arg_str.len == 0) return w.writeAll("bad usage");
const anchor_start = arg_str[0] == '^';
const anchor_end = arg_str[arg_str.len - 1] == '$';
const query = arg_str[@intFromBool(anchor_start) .. arg_str.len - @intFromBool(anchor_end)];
var num_results: usize = 0;
for (zcu.incremental_debug_state.navs.keys()) |nav_index| {
const nav = ip.getNav(nav_index);
const nav_fqn = nav.fqn.toSlice(ip);
const success = switch (@as(u2, @intFromBool(anchor_start)) << 1 | @intFromBool(anchor_end)) {
0b00 => std.mem.indexOf(u8, nav_fqn, query) != null,
0b01 => std.mem.endsWith(u8, nav_fqn, query),
0b10 => std.mem.startsWith(u8, nav_fqn, query),
0b11 => std.mem.eql(u8, nav_fqn, query),
};
if (success) {
num_results += 1;
try w.print("* nav {d} ('{s}')\n", .{ @intFromEnum(nav_index), nav_fqn });
}
}
try w.print("Found {d} results\n", .{num_results});
} else if (std.mem.eql(u8, cmd_str, "unit_info")) {
const unit = parseAnalUnit(arg_str) orelse return w.writeAll("malformed anal unit");
const unit_info = zcu.incremental_debug_state.units.get(unit) orelse return w.writeAll("unknown anal unit");
var ref_str_buf: [32]u8 = undefined;
const ref_str: []const u8 = ref: {
const refs = try zcu.resolveReferences();
const ref = refs.get(unit) orelse break :ref "<unreferenced>";
const referencer = (ref orelse break :ref "<analysis root>").referencer;
break :ref printAnalUnit(referencer, &ref_str_buf);
};
const has_err: []const u8 = err: {
if (zcu.failed_analysis.contains(unit)) break :err "true";
if (zcu.transitive_failed_analysis.contains(unit)) break :err "true (transitive)";
break :err "false";
};
try w.print(
\\last update generation: {d}
\\current referencer: {s}
\\has error: {s}
\\
, .{
unit_info.last_update_gen,
ref_str,
has_err,
});
} else if (std.mem.eql(u8, cmd_str, "unit_dependencies")) {
const unit = parseAnalUnit(arg_str) orelse return w.writeAll("malformed anal unit");
const unit_info = zcu.incremental_debug_state.units.get(unit) orelse return w.writeAll("unknown anal unit");
for (unit_info.deps.items, 0..) |dependee, i| {
try w.print("[{d}] ", .{i});
switch (dependee) {
.src_hash, .namespace, .namespace_name, .zon_file, .embed_file => try w.print("{}", .{zcu.fmtDependee(dependee)}),
.nav_val, .nav_ty => |nav| try w.print("{s} {d}", .{ @tagName(dependee), @intFromEnum(nav) }),
.interned => |ip_index| switch (ip.indexToKey(ip_index)) {
.struct_type, .union_type, .enum_type => try w.print("type {d}", .{@intFromEnum(ip_index)}),
.func => try w.print("func {d}", .{@intFromEnum(ip_index)}),
else => unreachable,
},
.memoized_state => |stage| try w.print("memoized_state {s}", .{@tagName(stage)}),
}
try w.writeByte('\n');
}
} else if (std.mem.eql(u8, cmd_str, "unit_trace")) {
const unit = parseAnalUnit(arg_str) orelse return w.writeAll("malformed anal unit");
if (!zcu.incremental_debug_state.units.contains(unit)) return w.writeAll("unknown anal unit");
const refs = try zcu.resolveReferences();
if (!refs.contains(unit)) return w.writeAll("not referenced");
var opt_cur: ?AnalUnit = unit;
while (opt_cur) |cur| {
var buf: [32]u8 = undefined;
try w.print("* {s}\n", .{printAnalUnit(cur, &buf)});
opt_cur = if (refs.get(cur).?) |ref| ref.referencer else null;
}
} else if (std.mem.eql(u8, cmd_str, "type_info")) {
const ip_index: InternPool.Index = @enumFromInt(parseIndex(arg_str) orelse return w.writeAll("malformed ip index"));
const create_gen = zcu.incremental_debug_state.types.get(ip_index) orelse return w.writeAll("unknown type");
try w.print(
\\name: '{}'
\\created on generation: {d}
\\
, .{
Type.fromInterned(ip_index).containerTypeName(ip).fmt(ip),
create_gen,
});
} else if (std.mem.eql(u8, cmd_str, "type_namespace")) {
const ip_index: InternPool.Index = @enumFromInt(parseIndex(arg_str) orelse return w.writeAll("malformed ip index"));
if (!zcu.incremental_debug_state.types.contains(ip_index)) return w.writeAll("unknown type");
const ns = zcu.namespacePtr(Type.fromInterned(ip_index).getNamespaceIndex(zcu));
try w.print("{d} pub decls:\n", .{ns.pub_decls.count()});
for (ns.pub_decls.keys()) |nav| {
try w.print("* nav {d}\n", .{@intFromEnum(nav)});
}
try w.print("{d} non-pub decls:\n", .{ns.priv_decls.count()});
for (ns.priv_decls.keys()) |nav| {
try w.print("* nav {d}\n", .{@intFromEnum(nav)});
}
try w.print("{d} comptime decls:\n", .{ns.comptime_decls.items.len});
for (ns.comptime_decls.items) |id| {
try w.print("* comptime {d}\n", .{@intFromEnum(id)});
}
try w.print("{d} tests:\n", .{ns.test_decls.items.len});
for (ns.test_decls.items) |nav| {
try w.print("* nav {d}\n", .{@intFromEnum(nav)});
}
} else {
try w.writeAll("command not found; run 'help' for a command list");
}
}
fn parseIndex(str: []const u8) ?u32 {
return std.fmt.parseInt(u32, str, 10) catch null;
}
fn parseAnalUnit(str: []const u8) ?AnalUnit {
const split_idx = std.mem.indexOfScalar(u8, str, ' ') orelse return null;
const kind = str[0..split_idx];
const idx_str = str[split_idx + 1 ..];
if (std.mem.eql(u8, kind, "comptime")) {
return .wrap(.{ .@"comptime" = @enumFromInt(parseIndex(idx_str) orelse return null) });
} else if (std.mem.eql(u8, kind, "nav_val")) {
return .wrap(.{ .nav_val = @enumFromInt(parseIndex(idx_str) orelse return null) });
} else if (std.mem.eql(u8, kind, "nav_ty")) {
return .wrap(.{ .nav_ty = @enumFromInt(parseIndex(idx_str) orelse return null) });
} else if (std.mem.eql(u8, kind, "type")) {
return .wrap(.{ .type = @enumFromInt(parseIndex(idx_str) orelse return null) });
} else if (std.mem.eql(u8, kind, "func")) {
return .wrap(.{ .func = @enumFromInt(parseIndex(idx_str) orelse return null) });
} else if (std.mem.eql(u8, kind, "memoized_state")) {
return .wrap(.{ .memoized_state = std.meta.stringToEnum(
InternPool.MemoizedStateStage,
idx_str,
) orelse return null });
} else {
return null;
}
}
fn printAnalUnit(unit: AnalUnit, buf: *[32]u8) []const u8 {
const idx: u32 = switch (unit.unwrap()) {
.memoized_state => |stage| return std.fmt.bufPrint(buf, "memoized_state {s}", .{@tagName(stage)}) catch unreachable,
inline else => |i| @intFromEnum(i),
};
return std.fmt.bufPrint(buf, "{s} {d}", .{ @tagName(unit.unwrap()), idx }) catch unreachable;
}
fn printType(ty: Type, zcu: *const Zcu, w: anytype) !void {
const ip = &zcu.intern_pool;
switch (ip.indexToKey(ty.toIntern())) {
.int_type => |int| try w.print("{c}{d}", .{
@as(u8, if (int.signedness == .unsigned) 'u' else 'i'),
int.bits,
}),
.tuple_type => try w.writeAll("(tuple)"),
.error_set_type => try w.writeAll("(error set)"),
.inferred_error_set_type => try w.writeAll("(inferred error set)"),
.func_type => try w.writeAll("(function)"),
.anyframe_type => try w.writeAll("(anyframe)"),
.vector_type => {
try w.print("@Vector({d}, ", .{ty.vectorLen(zcu)});
try printType(ty.childType(zcu), zcu, w);
try w.writeByte(')');
},
.array_type => {
try w.print("[{d}]", .{ty.arrayLen(zcu)});
try printType(ty.childType(zcu), zcu, w);
},
.opt_type => {
try w.writeByte('?');
try printType(ty.optionalChild(zcu), zcu, w);
},
.error_union_type => {
try printType(ty.errorUnionSet(zcu), zcu, w);
try w.writeByte('!');
try printType(ty.errorUnionPayload(zcu), zcu, w);
},
.ptr_type => {
try w.writeAll("*(attrs) ");
try printType(ty.childType(zcu), zcu, w);
},
.simple_type => |simple| try w.writeAll(@tagName(simple)),
.struct_type,
.union_type,
.enum_type,
.opaque_type,
=> try w.print("{}[{d}]", .{ ty.containerTypeName(ip).fmt(ip), @intFromEnum(ty.toIntern()) }),
else => unreachable,
}
}
const std = @import("std");
const Allocator = std.mem.Allocator;
const Compilation = @import("Compilation.zig");
const Zcu = @import("Zcu.zig");
const InternPool = @import("InternPool.zig");
const Type = @import("Type.zig");
const AnalUnit = InternPool.AnalUnit;
const IncrementalDebugServer = @This();

View File

@ -2998,11 +2998,7 @@ fn zirStructDecl(
errdefer pt.destroyNamespace(new_namespace_index);
if (pt.zcu.comp.incremental) {
try ip.addDependency(
sema.gpa,
AnalUnit.wrap(.{ .type = wip_ty.index }),
.{ .src_hash = tracked_inst },
);
try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = tracked_inst });
}
const decls = sema.code.bodySlice(extra_index, decls_len);
@ -3017,6 +3013,7 @@ fn zirStructDecl(
}
try sema.declareDependency(.{ .interned = wip_ty.index });
try sema.addTypeReferenceEntry(src, wip_ty.index);
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
return Air.internedToRef(wip_ty.finish(ip, new_namespace_index));
}
@ -3247,6 +3244,7 @@ fn zirEnumDecl(
// We've finished the initial construction of this type, and are about to perform analysis.
// Set the namespace appropriately, and don't destroy anything on failure.
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
wip_ty.prepare(ip, new_namespace_index);
done = true;
@ -3377,11 +3375,7 @@ fn zirUnionDecl(
errdefer pt.destroyNamespace(new_namespace_index);
if (pt.zcu.comp.incremental) {
try zcu.intern_pool.addDependency(
gpa,
AnalUnit.wrap(.{ .type = wip_ty.index }),
.{ .src_hash = tracked_inst },
);
try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = tracked_inst });
}
const decls = sema.code.bodySlice(extra_index, decls_len);
@ -3396,6 +3390,7 @@ fn zirUnionDecl(
}
try sema.declareDependency(.{ .interned = wip_ty.index });
try sema.addTypeReferenceEntry(src, wip_ty.index);
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
return Air.internedToRef(wip_ty.finish(ip, new_namespace_index));
}
@ -3481,6 +3476,7 @@ fn zirOpaqueDecl(
try zcu.comp.queueJob(.{ .codegen_type = wip_ty.index });
}
try sema.addTypeReferenceEntry(src, wip_ty.index);
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
return Air.internedToRef(wip_ty.finish(ip, new_namespace_index));
}
@ -8026,6 +8022,11 @@ fn analyzeCall(
.generic_owner = func_val.?.toIntern(),
.comptime_args = comptime_args,
});
if (zcu.comp.debugIncremental()) {
const nav = ip.indexToKey(func_instance).func.owner_nav;
const gop = try zcu.incremental_debug_state.navs.getOrPut(gpa, nav);
if (!gop.found_existing) gop.value_ptr.* = zcu.generation;
}
// This call is problematic as it breaks guarantees about order-independency of semantic analysis.
// These guarantees are necessary for incremental compilation and parallel semantic analysis.
@ -20345,6 +20346,7 @@ fn structInitAnon(
if (block.ownerModule().strip) break :codegen_type;
try zcu.comp.queueJob(.{ .codegen_type = wip.index });
}
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip.index);
break :ty wip.finish(ip, new_namespace_index);
},
.existing => |ty| ty,
@ -21406,6 +21408,7 @@ fn zirReify(
});
try sema.addTypeReferenceEntry(src, wip_ty.index);
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
return Air.internedToRef(wip_ty.finish(ip, new_namespace_index));
},
.@"union" => {
@ -21611,6 +21614,7 @@ fn reifyEnum(
try sema.declareDependency(.{ .interned = wip_ty.index });
try sema.addTypeReferenceEntry(src, wip_ty.index);
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
wip_ty.prepare(ip, new_namespace_index);
wip_ty.setTagTy(ip, tag_ty.toIntern());
done = true;
@ -21920,6 +21924,7 @@ fn reifyUnion(
}
try sema.declareDependency(.{ .interned = wip_ty.index });
try sema.addTypeReferenceEntry(src, wip_ty.index);
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
return Air.internedToRef(wip_ty.finish(ip, new_namespace_index));
}
@ -22273,6 +22278,7 @@ fn reifyStruct(
}
try sema.declareDependency(.{ .interned = wip_ty.index });
try sema.addTypeReferenceEntry(src, wip_ty.index);
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
return Air.internedToRef(wip_ty.finish(ip, new_namespace_index));
}
@ -37485,8 +37491,8 @@ fn isKnownZigType(sema: *Sema, ref: Air.Inst.Ref, tag: std.builtin.TypeId) bool
}
pub fn declareDependency(sema: *Sema, dependee: InternPool.Dependee) !void {
const zcu = sema.pt.zcu;
if (!zcu.comp.incremental) return;
const pt = sema.pt;
if (!pt.zcu.comp.incremental) return;
const gop = try sema.dependencies.getOrPut(sema.gpa, dependee);
if (gop.found_existing) return;
@ -37508,7 +37514,7 @@ pub fn declareDependency(sema: *Sema, dependee: InternPool.Dependee) !void {
else => {},
}
try zcu.intern_pool.addDependency(sema.gpa, sema.owner, dependee);
try pt.addDependency(sema.owner, dependee);
}
fn isComptimeMutablePtr(sema: *Sema, val: Value) bool {
@ -37905,6 +37911,11 @@ pub fn resolveDeclaredEnum(
};
defer sema.deinit();
if (zcu.comp.debugIncremental()) {
const info = try zcu.incremental_debug_state.getUnitInfo(gpa, sema.owner);
info.last_update_gen = zcu.generation;
}
try sema.declareDependency(.{ .src_hash = tracked_inst });
var block: Block = .{

View File

@ -3797,6 +3797,11 @@ fn resolveStructInner(
return error.AnalysisFail;
}
if (zcu.comp.debugIncremental()) {
const info = try zcu.incremental_debug_state.getUnitInfo(gpa, owner);
info.last_update_gen = zcu.generation;
}
var analysis_arena = std.heap.ArenaAllocator.init(gpa);
defer analysis_arena.deinit();
@ -3851,6 +3856,11 @@ fn resolveUnionInner(
return error.AnalysisFail;
}
if (zcu.comp.debugIncremental()) {
const info = try zcu.incremental_debug_state.getUnitInfo(gpa, owner);
info.last_update_gen = zcu.generation;
}
var analysis_arena = std.heap.ArenaAllocator.init(gpa);
defer analysis_arena.deinit();

View File

@ -308,8 +308,56 @@ free_type_references: std.ArrayListUnmanaged(u32) = .empty,
/// Populated by analysis of `AnalUnit.wrap(.{ .memoized_state = s })`, where `s` depends on the element.
builtin_decl_values: BuiltinDecl.Memoized = .initFill(.none),
incremental_debug_state: if (build_options.enable_debug_extensions) IncrementalDebugState else void =
if (build_options.enable_debug_extensions) .init else {},
generation: u32 = 0,
pub const IncrementalDebugState = struct {
/// All container types in the ZCU, even dead ones.
/// Value is the generation the type was created on.
types: std.AutoArrayHashMapUnmanaged(InternPool.Index, u32),
/// All `Nav`s in the ZCU, even dead ones.
/// Value is the generation the `Nav` was created on.
navs: std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, u32),
/// All `AnalUnit`s in the ZCU, even dead ones.
units: std.AutoArrayHashMapUnmanaged(AnalUnit, UnitInfo),
pub const init: IncrementalDebugState = .{
.types = .empty,
.navs = .empty,
.units = .empty,
};
pub fn deinit(ids: *IncrementalDebugState, gpa: Allocator) void {
for (ids.units.values()) |*unit_info| {
unit_info.deps.deinit(gpa);
}
ids.types.deinit(gpa);
ids.navs.deinit(gpa);
ids.units.deinit(gpa);
}
pub const UnitInfo = struct {
last_update_gen: u32,
/// This information isn't easily recoverable from `InternPool`'s dependency storage format.
deps: std.ArrayListUnmanaged(InternPool.Dependee),
};
pub fn getUnitInfo(ids: *IncrementalDebugState, gpa: Allocator, unit: AnalUnit) Allocator.Error!*UnitInfo {
const gop = try ids.units.getOrPut(gpa, unit);
if (!gop.found_existing) gop.value_ptr.* = .{
.last_update_gen = std.math.maxInt(u32),
.deps = .empty,
};
return gop.value_ptr;
}
pub fn newType(ids: *IncrementalDebugState, zcu: *Zcu, ty: InternPool.Index) Allocator.Error!void {
try ids.types.putNoClobber(zcu.gpa, ty, zcu.generation);
}
pub fn newNav(ids: *IncrementalDebugState, zcu: *Zcu, nav: InternPool.Nav.Index) Allocator.Error!void {
try ids.navs.putNoClobber(zcu.gpa, nav, zcu.generation);
}
};
pub const PerThread = @import("Zcu/PerThread.zig");
pub const ImportTableAdapter = struct {
@ -2746,6 +2794,10 @@ pub fn deinit(zcu: *Zcu) void {
zcu.free_type_references.deinit(gpa);
if (zcu.resolved_references) |*r| r.deinit(gpa);
if (zcu.comp.debugIncremental()) {
zcu.incremental_debug_state.deinit(gpa);
}
}
zcu.intern_pool.deinit(gpa);
}

View File

@ -635,6 +635,12 @@ pub fn ensureMemoizedStateUpToDate(pt: Zcu.PerThread, stage: InternPool.Memoized
if (zcu.builtin_decl_values.get(to_check) != .none) return;
}
if (zcu.comp.debugIncremental()) {
const info = try zcu.incremental_debug_state.getUnitInfo(gpa, unit);
info.last_update_gen = zcu.generation;
info.deps.clearRetainingCapacity();
}
const any_changed: bool, const new_failed: bool = if (pt.analyzeMemoizedState(stage)) |any_changed|
.{ any_changed or prev_failed, false }
else |err| switch (err) {
@ -784,6 +790,12 @@ pub fn ensureComptimeUnitUpToDate(pt: Zcu.PerThread, cu_id: InternPool.ComptimeU
return;
}
if (zcu.comp.debugIncremental()) {
const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit);
info.last_update_gen = zcu.generation;
info.deps.clearRetainingCapacity();
}
const unit_prog_node = zcu.sema_prog_node.start("comptime", 0);
defer unit_prog_node.end();
@ -958,6 +970,12 @@ pub fn ensureNavValUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zcu
}
}
if (zcu.comp.debugIncremental()) {
const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit);
info.last_update_gen = zcu.generation;
info.deps.clearRetainingCapacity();
}
const unit_prog_node = zcu.sema_prog_node.start(nav.fqn.toSlice(ip), 0);
defer unit_prog_node.end();
@ -1004,6 +1022,35 @@ pub fn ensureNavValUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zcu
}
}
// If there isn't a type annotation, then we have also just resolved the type. That means the
// the type is up-to-date, so it won't have the chance to mark its own dependency on the value;
// we must do that ourselves.
type_deps_on_val: {
const inst_resolved = nav.analysis.?.zir_index.resolveFull(ip) orelse break :type_deps_on_val;
const file = zcu.fileByIndex(inst_resolved.file);
const zir_decl = file.zir.?.getDeclaration(inst_resolved.inst);
if (zir_decl.type_body != null) break :type_deps_on_val;
// The type does indeed depend on the value. We are responsible for populating all state of
// the `nav_ty`, including exports, references, errors, and dependencies.
const ty_unit: AnalUnit = .wrap(.{ .nav_ty = nav_id });
const ty_was_outdated = zcu.outdated.swapRemove(ty_unit) or
zcu.potentially_outdated.swapRemove(ty_unit);
if (ty_was_outdated) {
_ = zcu.outdated_ready.swapRemove(ty_unit);
zcu.deleteUnitExports(ty_unit);
zcu.deleteUnitReferences(ty_unit);
zcu.deleteUnitCompileLogs(ty_unit);
if (zcu.failed_analysis.fetchSwapRemove(ty_unit)) |kv| {
kv.value.destroy(gpa);
}
_ = zcu.transitive_failed_analysis.swapRemove(ty_unit);
ip.removeDependenciesForDepender(gpa, ty_unit);
}
try pt.addDependency(ty_unit, .{ .nav_val = nav_id });
if (new_failed) try zcu.transitive_failed_analysis.put(gpa, ty_unit, {});
if (ty_was_outdated) try zcu.markDependeeOutdated(.marked_po, .{ .nav_ty = nav_id });
}
if (new_failed) return error.AnalysisFail;
}
@ -1248,14 +1295,6 @@ fn analyzeNavVal(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zcu.CompileErr
// Mark the unit as completed before evaluating the export!
assert(zcu.analysis_in_progress.swapRemove(anal_unit));
if (zir_decl.type_body == null) {
// In this situation, it's possible that we were triggered by `analyzeNavType` up the stack. In that
// case, we must also signal that the *type* is now populated to make this export behave correctly.
// An alternative strategy would be to just put something on the job queue to perform the export, but
// this is a little more straightforward, if perhaps less elegant.
_ = zcu.analysis_in_progress.swapRemove(.wrap(.{ .nav_ty = nav_id }));
}
if (zir_decl.linkage == .@"export") {
const export_src = block.src(.{ .token_offset = @enumFromInt(@intFromBool(zir_decl.is_pub)) });
const name_slice = zir.nullTerminatedString(zir_decl.name);
@ -1296,6 +1335,18 @@ pub fn ensureNavTypeUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zc
log.debug("ensureNavTypeUpToDate {}", .{zcu.fmtAnalUnit(anal_unit)});
const type_resolved_by_value: bool = from_val: {
const analysis = nav.analysis orelse break :from_val false;
const inst_resolved = analysis.zir_index.resolveFull(ip) orelse break :from_val false;
const file = zcu.fileByIndex(inst_resolved.file);
const zir_decl = file.zir.?.getDeclaration(inst_resolved.inst);
break :from_val zir_decl.type_body == null;
};
if (type_resolved_by_value) {
// Logic at the end of `ensureNavValUpToDate` is directly responsible for populating our state.
return pt.ensureNavValUpToDate(nav_id);
}
// Determine whether or not this `Nav`'s type is outdated. This also includes checking if the
// status is `.unresolved`, which indicates that the value is outdated because it has *never*
// been analyzed so far.
@ -1331,6 +1382,12 @@ pub fn ensureNavTypeUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zc
}
}
if (zcu.comp.debugIncremental()) {
const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit);
info.last_update_gen = zcu.generation;
info.deps.clearRetainingCapacity();
}
const unit_prog_node = zcu.sema_prog_node.start(nav.fqn.toSlice(ip), 0);
defer unit_prog_node.end();
@ -1397,6 +1454,10 @@ fn analyzeNavType(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zcu.CompileEr
try zcu.analysis_in_progress.put(gpa, anal_unit, {});
defer _ = zcu.analysis_in_progress.swapRemove(anal_unit);
const zir_decl = zir.getDeclaration(inst_resolved.inst);
assert(old_nav.is_usingnamespace == (zir_decl.kind == .@"usingnamespace"));
const type_body = zir_decl.type_body.?;
var analysis_arena: std.heap.ArenaAllocator = .init(gpa);
defer analysis_arena.deinit();
@ -1436,9 +1497,6 @@ fn analyzeNavType(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zcu.CompileEr
};
defer block.instructions.deinit(gpa);
const zir_decl = zir.getDeclaration(inst_resolved.inst);
assert(old_nav.is_usingnamespace == (zir_decl.kind == .@"usingnamespace"));
const ty_src = block.src(.{ .node_offset_var_decl_ty = .zero });
block.comptime_reason = .{ .reason = .{
@ -1446,23 +1504,6 @@ fn analyzeNavType(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zcu.CompileEr
.r = .{ .simple = .type },
} };
const type_body = zir_decl.type_body orelse {
// The type of this `Nav` is inferred from the value.
// In other words, this `nav_ty` depends on the corresponding `nav_val`.
try sema.declareDependency(.{ .nav_val = nav_id });
try pt.ensureNavValUpToDate(nav_id);
// Note that the above call, if it did any work, has removed our `analysis_in_progress` entry for us.
// (Our `defer` will run anyway, but it does nothing in this case.)
// There's not a great way for us to know whether the type actually changed.
// For instance, perhaps the `nav_val` was already up-to-date, but this `nav_ty` is being
// analyzed because this declaration had a type annotation on the *previous* update.
// However, such cases are rare, and it's not unreasonable to re-analyze in them; and in
// other cases where we get here, it's because the `nav_val` was already re-analyzed and
// is outdated.
return .{ .type_changed = true };
};
const resolved_ty: Type = ty: {
const uncoerced_type_ref = try sema.resolveInlineBody(&block, type_body, inst_resolved.inst);
const type_ref = try sema.coerce(&block, .type, uncoerced_type_ref, ty_src);
@ -1564,6 +1605,12 @@ pub fn ensureFuncBodyUpToDate(pt: Zcu.PerThread, maybe_coerced_func_index: Inter
if (func.analysisUnordered(ip).is_analyzed) return;
}
if (zcu.comp.debugIncremental()) {
const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit);
info.last_update_gen = zcu.generation;
info.deps.clearRetainingCapacity();
}
const func_prog_node = zcu.sema_prog_node.start(ip.getNav(func.owner_nav).fqn.toSlice(ip), 0);
defer func_prog_node.end();
@ -1816,11 +1863,7 @@ fn createFileRootStruct(
ip.namespacePtr(namespace_index).owner_type = wip_ty.index;
if (zcu.comp.incremental) {
try ip.addDependency(
gpa,
.wrap(.{ .type = wip_ty.index }),
.{ .src_hash = tracked_inst },
);
try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = tracked_inst });
}
try pt.scanNamespace(namespace_index, decls);
@ -1832,6 +1875,7 @@ fn createFileRootStruct(
try zcu.comp.queueJob(.{ .codegen_type = wip_ty.index });
}
zcu.setFileRootType(file_index, wip_ty.index);
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
return wip_ty.finish(ip, namespace_index);
}
@ -2734,10 +2778,11 @@ const ScanDeclIter = struct {
else => unit: {
const name = maybe_name.unwrap().?;
const fqn = try namespace.internFullyQualifiedName(ip, gpa, pt.tid, name);
const nav = if (existing_unit) |eu|
eu.unwrap().nav_val
else
try ip.createDeclNav(gpa, pt.tid, name, fqn, tracked_inst, namespace_index, decl.kind == .@"usingnamespace");
const nav = if (existing_unit) |eu| eu.unwrap().nav_val else nav: {
const nav = try ip.createDeclNav(gpa, pt.tid, name, fqn, tracked_inst, namespace_index, decl.kind == .@"usingnamespace");
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newNav(zcu, nav);
break :nav nav;
};
const unit: AnalUnit = .wrap(.{ .nav_val = nav });
@ -3911,6 +3956,7 @@ pub fn getExtern(pt: Zcu.PerThread, key: InternPool.Key.Extern) Allocator.Error!
if (result.new_nav.unwrap()) |nav| {
// This job depends on any resolve_type_fully jobs queued up before it.
try pt.zcu.comp.queueJob(.{ .codegen_nav = nav });
if (pt.zcu.comp.debugIncremental()) try pt.zcu.incremental_debug_state.newNav(pt.zcu, nav);
}
return result.index;
}
@ -3979,6 +4025,12 @@ pub fn ensureTypeUpToDate(pt: Zcu.PerThread, ty: InternPool.Index) Zcu.SemaError
_ = zcu.transitive_failed_analysis.swapRemove(anal_unit);
zcu.intern_pool.removeDependenciesForDepender(gpa, anal_unit);
if (zcu.comp.debugIncremental()) {
const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit);
info.last_update_gen = zcu.generation;
info.deps.clearRetainingCapacity();
}
switch (ip.indexToKey(ty)) {
.struct_type => return pt.recreateStructType(ty, declared_ty_key),
.union_type => return pt.recreateUnionType(ty, declared_ty_key),
@ -4042,11 +4094,7 @@ fn recreateStructType(
errdefer wip_ty.cancel(ip, pt.tid);
wip_ty.setName(ip, struct_obj.name);
try ip.addDependency(
gpa,
.wrap(.{ .type = wip_ty.index }),
.{ .src_hash = key.zir_index },
);
try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = key.zir_index });
zcu.namespacePtr(struct_obj.namespace).owner_type = wip_ty.index;
// No need to re-scan the namespace -- `zirStructDecl` will ultimately do that if the type is still alive.
try zcu.comp.queueJob(.{ .resolve_type_fully = wip_ty.index });
@ -4058,6 +4106,7 @@ fn recreateStructType(
try zcu.comp.queueJob(.{ .codegen_type = wip_ty.index });
}
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
const new_ty = wip_ty.finish(ip, struct_obj.namespace);
if (inst_info.inst == .main_struct_inst) {
// This is the root type of a file! Update the reference.
@ -4138,11 +4187,7 @@ fn recreateUnionType(
errdefer wip_ty.cancel(ip, pt.tid);
wip_ty.setName(ip, union_obj.name);
try ip.addDependency(
gpa,
.wrap(.{ .type = wip_ty.index }),
.{ .src_hash = key.zir_index },
);
try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = key.zir_index });
zcu.namespacePtr(namespace_index).owner_type = wip_ty.index;
// No need to re-scan the namespace -- `zirUnionDecl` will ultimately do that if the type is still alive.
try zcu.comp.queueJob(.{ .resolve_type_fully = wip_ty.index });
@ -4154,6 +4199,7 @@ fn recreateUnionType(
try zcu.comp.queueJob(.{ .codegen_type = wip_ty.index });
}
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
return wip_ty.finish(ip, namespace_index);
}
@ -4255,6 +4301,7 @@ fn recreateEnumType(
zcu.namespacePtr(namespace_index).owner_type = wip_ty.index;
// No need to re-scan the namespace -- `zirEnumDecl` will ultimately do that if the type is still alive.
if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
wip_ty.prepare(ip, namespace_index);
done = true;
@ -4432,3 +4479,13 @@ pub fn refValue(pt: Zcu.PerThread, val: InternPool.Index) Zcu.SemaError!InternPo
.byte_offset = 0,
} });
}
pub fn addDependency(pt: Zcu.PerThread, unit: AnalUnit, dependee: InternPool.Dependee) Allocator.Error!void {
const zcu = pt.zcu;
const gpa = zcu.gpa;
try zcu.intern_pool.addDependency(gpa, unit, dependee);
if (zcu.comp.debugIncremental()) {
const info = try zcu.incremental_debug_state.getUnitInfo(gpa, unit);
try info.deps.append(gpa, dependee);
}
}

View File

@ -677,6 +677,7 @@ const usage_build_generic =
\\ --debug-compile-errors Crash with helpful diagnostics at the first compile error
\\ --debug-link-snapshot Enable dumping of the linker's state in JSON format
\\ --debug-rt Debug compiler runtime libraries
\\ --debug-incremental Enable incremental compilation debug features
\\
;
@ -832,6 +833,7 @@ fn buildOutputType(
var data_sections = false;
var listen: Listen = .none;
var debug_compile_errors = false;
var debug_incremental = false;
var verbose_link = (native_os != .wasi or builtin.link_libc) and
EnvVar.ZIG_VERBOSE_LINK.isSet();
var verbose_cc = (native_os != .wasi or builtin.link_libc) and
@ -1383,6 +1385,12 @@ fn buildOutputType(
}
} else if (mem.eql(u8, arg, "--debug-rt")) {
debug_compiler_runtime_libs = true;
} else if (mem.eql(u8, arg, "--debug-incremental")) {
if (build_options.enable_debug_extensions) {
debug_incremental = true;
} else {
warn("Zig was compiled without debug extensions. --debug-incremental has no effect.", .{});
}
} else if (mem.eql(u8, arg, "-fincremental")) {
dev.check(.incremental);
opt_incremental = true;
@ -3460,6 +3468,9 @@ fn buildOutputType(
};
const incremental = opt_incremental orelse false;
if (debug_incremental and !incremental) {
fatal("--debug-incremental requires -fincremental", .{});
}
const disable_lld_caching = !output_to_cache;
@ -3592,6 +3603,7 @@ fn buildOutputType(
.cache_mode = cache_mode,
.subsystem = subsystem,
.debug_compile_errors = debug_compile_errors,
.debug_incremental = debug_incremental,
.incremental = incremental,
.enable_link_snapshots = enable_link_snapshots,
.install_name = install_name,
@ -4195,9 +4207,25 @@ fn serve(
const main_progress_node = std.Progress.start(.{});
const file_system_inputs = comp.file_system_inputs.?;
const IncrementalDebugServer = if (build_options.enable_debug_extensions)
@import("IncrementalDebugServer.zig")
else
void;
var ids: IncrementalDebugServer = if (comp.debugIncremental()) ids: {
break :ids .init(comp.zcu orelse @panic("--debug-incremental requires a ZCU"));
} else undefined;
defer if (comp.debugIncremental()) ids.deinit();
if (comp.debugIncremental()) ids.spawn();
while (true) {
const hdr = try server.receiveMessage();
// Lock the debug server while hanling the message.
if (comp.debugIncremental()) ids.mutex.lock();
defer if (comp.debugIncremental()) ids.mutex.unlock();
switch (hdr.tag) {
.exit => return cleanExit(),
.update => {

View File

@ -0,0 +1,30 @@
#target=x86_64-linux-selfhosted
#target=x86_64-linux-cbe
#target=x86_64-windows-cbe
#target=wasm32-wasi-selfhosted
#update=initial version
#file=main.zig
const foo = @as(u8, 123);
comptime {
// depends on value of `foo`
if (foo != 123) unreachable;
}
comptime {
// depends on type of `foo`
if (@TypeOf(&foo) != *const u8) unreachable;
}
pub fn main() void {}
#expect_stdout=""
#update=change the type
#file=main.zig
const foo = @as(u16, 123);
comptime {
// depends on value of `foo`
if (foo != 123) unreachable;
}
comptime {
// depends on type of `foo`
if (@TypeOf(&foo) != *const u8) unreachable;
}
pub fn main() void {}
#expect_error=main.zig:8:37: error: reached unreachable code