diff --git a/src/InternPool.zig b/src/InternPool.zig index 1c501cb28e..6f7bb17b14 100644 --- a/src/InternPool.zig +++ b/src/InternPool.zig @@ -14,25 +14,6 @@ tid_shift_31: if (single_threaded) u0 else std.math.Log2Int(u32) = if (single_th /// Cached shift amount to put a `tid` in the top bits of a 32-bit value. tid_shift_32: if (single_threaded) u0 else std.math.Log2Int(u32) = if (single_threaded) 0 else 31, -/// Rather than allocating Decl objects with an Allocator, we instead allocate -/// them with this SegmentedList. This provides four advantages: -/// * Stable memory so that one thread can access a Decl object while another -/// thread allocates additional Decl objects from this list. -/// * It allows us to use u32 indexes to reference Decl objects rather than -/// pointers, saving memory in Type, Value, and dependency sets. -/// * Using integers to reference Decl objects rather than pointers makes -/// serialization trivial. -/// * It provides a unique integer to be used for anonymous symbol names, avoiding -/// multi-threaded contention on an atomic counter. -allocated_decls: std.SegmentedList(Module.Decl, 0) = .{}, -/// When a Decl object is freed from `allocated_decls`, it is pushed into this stack. -decls_free_list: std.ArrayListUnmanaged(DeclIndex) = .{}, - -/// Same pattern as with `allocated_decls`. -allocated_namespaces: std.SegmentedList(Module.Namespace, 0) = .{}, -/// Same pattern as with `decls_free_list`. -namespaces_free_list: std.ArrayListUnmanaged(NamespaceIndex) = .{}, - /// Some types such as enums, structs, and unions need to store mappings from field names /// to field index, or value to field index. In such cases, they will store the underlying /// field names and values directly, relying on one of these maps, stored separately, @@ -354,10 +335,14 @@ const Local = struct { /// atomic access. mutate: struct { arena: std.heap.ArenaAllocator.State, - items: Mutate, - extra: Mutate, - limbs: Mutate, - strings: Mutate, + + items: ListMutate, + extra: ListMutate, + limbs: ListMutate, + strings: ListMutate, + + decls: BucketListMutate, + namespaces: BucketListMutate, } align(std.atomic.cache_line), const Shared = struct { @@ -366,6 +351,9 @@ const Local = struct { limbs: Limbs, strings: Strings, + decls: Decls, + namespaces: Namespaces, + pub fn getLimbs(shared: *const Local.Shared) Limbs { return switch (@sizeOf(Limb)) { @sizeOf(u32) => shared.extra, @@ -383,14 +371,38 @@ const Local = struct { }; const Strings = List(struct { u8 }); - const Mutate = struct { + const decls_bucket_width = 8; + const decls_bucket_mask = (1 << decls_bucket_width) - 1; + const decl_next_free_field = "src_namespace"; + const Decls = List(struct { *[1 << decls_bucket_width]Module.Decl }); + + const namespaces_bucket_width = 8; + const namespaces_bucket_mask = (1 << namespaces_bucket_width) - 1; + const namespace_next_free_field = "decl_index"; + const Namespaces = List(struct { *[1 << namespaces_bucket_width]Module.Namespace }); + + const ListMutate = struct { len: u32, - const empty: Mutate = .{ + const empty: ListMutate = .{ .len = 0, }; }; + const BucketListMutate = struct { + last_bucket_len: u32, + buckets_list: ListMutate, + free_list: u32, + + const free_list_sentinel = std.math.maxInt(u32); + + const empty: BucketListMutate = .{ + .last_bucket_len = 0, + .buckets_list = ListMutate.empty, + .free_list = free_list_sentinel, + }; + }; + fn List(comptime Elem: type) type { assert(@typeInfo(Elem) == .Struct); return struct { @@ -400,7 +412,7 @@ const Local = struct { const Mutable = struct { gpa: std.mem.Allocator, arena: *std.heap.ArenaAllocator.State, - mutate: *Mutate, + mutate: *ListMutate, list: *ListSelf, const fields = std.enums.values(std.meta.FieldEnum(Elem)); @@ -664,6 +676,35 @@ const Local = struct { .list = &local.shared.strings, }; } + + /// Rather than allocating Decl objects with an Allocator, we instead allocate + /// them with this BucketList. This provides four advantages: + /// * Stable memory so that one thread can access a Decl object while another + /// thread allocates additional Decl objects from this list. + /// * It allows us to use u32 indexes to reference Decl objects rather than + /// pointers, saving memory in Type, Value, and dependency sets. + /// * Using integers to reference Decl objects rather than pointers makes + /// serialization trivial. + /// * It provides a unique integer to be used for anonymous symbol names, avoiding + /// multi-threaded contention on an atomic counter. + pub fn getMutableDecls(local: *Local, gpa: std.mem.Allocator) Decls.Mutable { + return .{ + .gpa = gpa, + .arena = &local.mutate.arena, + .mutate = &local.mutate.decls.buckets_list, + .list = &local.shared.decls, + }; + } + + /// Same pattern as with `getMutableDecls`. + pub fn getMutableNamespaces(local: *Local, gpa: std.mem.Allocator) Namespaces.Mutable { + return .{ + .gpa = gpa, + .arena = &local.mutate.arena, + .mutate = &local.mutate.namespaces.buckets_list, + .list = &local.shared.namespaces, + }; + } }; pub fn getLocal(ip: *InternPool, tid: Zcu.PerThread.Id) *Local { @@ -810,6 +851,29 @@ pub const ComptimeAllocIndex = enum(u32) { _ }; pub const DeclIndex = enum(u32) { _, + const Unwrapped = struct { + tid: Zcu.PerThread.Id, + bucket_index: u32, + index: u32, + + fn wrap(unwrapped: Unwrapped, ip: *const InternPool) DeclIndex { + assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); + assert(unwrapped.bucket_index <= ip.getIndexMask(u32) >> Local.decls_bucket_width); + assert(unwrapped.index <= Local.decls_bucket_mask); + return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 | + unwrapped.bucket_index << Local.decls_bucket_width | + unwrapped.index); + } + }; + fn unwrap(decl_index: DeclIndex, ip: *const InternPool) Unwrapped { + const index = @intFromEnum(decl_index) & ip.getIndexMask(u32); + return .{ + .tid = @enumFromInt(@intFromEnum(decl_index) >> ip.tid_shift_32 & ip.getTidMask()), + .bucket_index = index >> Local.decls_bucket_width, + .index = index & Local.decls_bucket_mask, + }; + } + pub fn toOptional(i: DeclIndex) OptionalDeclIndex { return @enumFromInt(@intFromEnum(i)); } @@ -832,6 +896,29 @@ pub const OptionalDeclIndex = enum(u32) { pub const NamespaceIndex = enum(u32) { _, + const Unwrapped = struct { + tid: Zcu.PerThread.Id, + bucket_index: u32, + index: u32, + + fn wrap(unwrapped: Unwrapped, ip: *const InternPool) NamespaceIndex { + assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); + assert(unwrapped.bucket_index <= ip.getIndexMask(u32) >> Local.namespaces_bucket_width); + assert(unwrapped.index <= Local.namespaces_bucket_mask); + return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 | + unwrapped.bucket_index << Local.namespaces_bucket_width | + unwrapped.index); + } + }; + fn unwrap(namespace_index: NamespaceIndex, ip: *const InternPool) Unwrapped { + const index = @intFromEnum(namespace_index) & ip.getIndexMask(u32); + return .{ + .tid = @enumFromInt(@intFromEnum(namespace_index) >> ip.tid_shift_32 & ip.getTidMask()), + .bucket_index = index >> Local.namespaces_bucket_width, + .index = index & Local.namespaces_bucket_mask, + }; + } + pub fn toOptional(i: NamespaceIndex) OptionalNamespaceIndex { return @enumFromInt(@intFromEnum(i)); } @@ -5114,13 +5201,20 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void { .extra = Local.Extra.empty, .limbs = Local.Limbs.empty, .strings = Local.Strings.empty, + + .decls = Local.Decls.empty, + .namespaces = Local.Namespaces.empty, }, .mutate = .{ .arena = .{}, - .items = Local.Mutate.empty, - .extra = Local.Mutate.empty, - .limbs = Local.Mutate.empty, - .strings = Local.Mutate.empty, + + .items = Local.ListMutate.empty, + .extra = Local.ListMutate.empty, + .limbs = Local.ListMutate.empty, + .strings = Local.ListMutate.empty, + + .decls = Local.BucketListMutate.empty, + .namespaces = Local.BucketListMutate.empty, }, }); @@ -5173,12 +5267,6 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void { } pub fn deinit(ip: *InternPool, gpa: Allocator) void { - ip.decls_free_list.deinit(gpa); - ip.allocated_decls.deinit(gpa); - - ip.namespaces_free_list.deinit(gpa); - ip.allocated_namespaces.deinit(gpa); - for (ip.maps.items) |*map| map.deinit(gpa); ip.maps.deinit(gpa); @@ -5198,7 +5286,23 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void { ip.files.deinit(gpa); gpa.free(ip.shards); - for (ip.locals) |*local| local.mutate.arena.promote(gpa).deinit(); + for (ip.locals) |*local| { + const buckets_len = local.mutate.namespaces.buckets_list.len; + if (buckets_len > 0) for ( + local.shared.namespaces.view().items(.@"0")[0..buckets_len], + 0.., + ) |namespace_bucket, buckets_index| { + for (namespace_bucket[0..if (buckets_index < buckets_len - 1) + namespace_bucket.len + else + local.mutate.namespaces.last_bucket_len]) |*namespace| + { + namespace.decls.deinit(gpa); + namespace.usingnamespace_set.deinit(gpa); + } + }; + local.mutate.arena.promote(gpa).deinit(); + } gpa.free(ip.locals); ip.* = undefined; @@ -7849,7 +7953,7 @@ fn finishFuncInstance( section: OptionalNullTerminatedString, ) Allocator.Error!void { const fn_owner_decl = ip.declPtr(ip.funcDeclOwner(generic_owner)); - const decl_index = try ip.createDecl(gpa, .{ + const decl_index = try ip.createDecl(gpa, tid, .{ .name = undefined, .src_namespace = fn_owner_decl.src_namespace, .has_tv = true, @@ -7864,7 +7968,7 @@ fn finishFuncInstance( .is_exported = fn_owner_decl.is_exported, .kind = .anon, }); - errdefer ip.destroyDecl(gpa, decl_index); + errdefer ip.destroyDecl(tid, decl_index); // Populate the owner_decl field which was left undefined until now. extra.view().items(.@"0")[ @@ -9078,15 +9182,17 @@ fn dumpStatsFallible(ip: *const InternPool, arena: Allocator) anyerror!void { var items_len: usize = 0; var extra_len: usize = 0; var limbs_len: usize = 0; + var decls_len: usize = 0; for (ip.locals) |*local| { items_len += local.mutate.items.len; extra_len += local.mutate.extra.len; limbs_len += local.mutate.limbs.len; + decls_len += local.mutate.decls.buckets_list.len; } const items_size = (1 + 4) * items_len; const extra_size = 4 * extra_len; const limbs_size = 8 * limbs_len; - const decls_size = ip.allocated_decls.len * @sizeOf(Module.Decl); + const decls_size = @sizeOf(Module.Decl) * decls_len; // TODO: map overhead size is not taken into account const total_size = @sizeOf(InternPool) + items_size + extra_size + limbs_size + decls_size; @@ -9106,7 +9212,7 @@ fn dumpStatsFallible(ip: *const InternPool, arena: Allocator) anyerror!void { extra_size, limbs_len, limbs_size, - ip.allocated_decls.len, + decls_len, decls_size, }); @@ -9513,64 +9619,120 @@ pub fn dumpGenericInstancesFallible(ip: *const InternPool, allocator: Allocator) try bw.flush(); } -pub fn declPtr(ip: *InternPool, index: DeclIndex) *Module.Decl { - return ip.allocated_decls.at(@intFromEnum(index)); +pub fn declPtr(ip: *InternPool, decl_index: DeclIndex) *Module.Decl { + return @constCast(ip.declPtrConst(decl_index)); } -pub fn declPtrConst(ip: *const InternPool, index: DeclIndex) *const Module.Decl { - return ip.allocated_decls.at(@intFromEnum(index)); +pub fn declPtrConst(ip: *const InternPool, decl_index: DeclIndex) *const Module.Decl { + const unwrapped_decl_index = decl_index.unwrap(ip); + const decls = ip.getLocalShared(unwrapped_decl_index.tid).decls.acquire(); + const decls_bucket = decls.view().items(.@"0")[unwrapped_decl_index.bucket_index]; + return &decls_bucket[unwrapped_decl_index.index]; } -pub fn namespacePtr(ip: *InternPool, index: NamespaceIndex) *Module.Namespace { - return ip.allocated_namespaces.at(@intFromEnum(index)); +pub fn namespacePtr(ip: *InternPool, namespace_index: NamespaceIndex) *Module.Namespace { + const unwrapped_namespace_index = namespace_index.unwrap(ip); + const namespaces = ip.getLocalShared(unwrapped_namespace_index.tid).namespaces.acquire(); + const namespaces_bucket = namespaces.view().items(.@"0")[unwrapped_namespace_index.bucket_index]; + return &namespaces_bucket[unwrapped_namespace_index.index]; } pub fn createDecl( ip: *InternPool, gpa: Allocator, + tid: Zcu.PerThread.Id, initialization: Module.Decl, ) Allocator.Error!DeclIndex { - if (ip.decls_free_list.popOrNull()) |index| { - ip.allocated_decls.at(@intFromEnum(index)).* = initialization; - return index; + const local = ip.getLocal(tid); + const free_list_next = local.mutate.decls.free_list; + if (free_list_next != Local.BucketListMutate.free_list_sentinel) { + const reused_decl_index: DeclIndex = @enumFromInt(free_list_next); + const reused_decl = ip.declPtr(reused_decl_index); + local.mutate.decls.free_list = @intFromEnum(@field(reused_decl, Local.decl_next_free_field)); + reused_decl.* = initialization; + return reused_decl_index; } - const ptr = try ip.allocated_decls.addOne(gpa); - ptr.* = initialization; - return @enumFromInt(ip.allocated_decls.len - 1); + const decls = local.getMutableDecls(gpa); + if (local.mutate.decls.last_bucket_len == 0) { + try decls.ensureUnusedCapacity(1); + var arena = decls.arena.promote(decls.gpa); + defer decls.arena.* = arena.state; + decls.appendAssumeCapacity(.{try arena.allocator().create( + [1 << Local.decls_bucket_width]Module.Decl, + )}); + } + const unwrapped_decl_index: DeclIndex.Unwrapped = .{ + .tid = tid, + .bucket_index = decls.mutate.len - 1, + .index = local.mutate.decls.last_bucket_len, + }; + local.mutate.decls.last_bucket_len = + (unwrapped_decl_index.index + 1) & Local.namespaces_bucket_mask; + const decl_index = unwrapped_decl_index.wrap(ip); + ip.declPtr(decl_index).* = initialization; + return decl_index; } -pub fn destroyDecl(ip: *InternPool, gpa: Allocator, index: DeclIndex) void { - ip.declPtr(index).* = undefined; - ip.decls_free_list.append(gpa, index) catch { - // In order to keep `destroyDecl` a non-fallible function, we ignore memory - // allocation failures here, instead leaking the Decl until garbage collection. - }; +pub fn destroyDecl(ip: *InternPool, tid: Zcu.PerThread.Id, decl_index: DeclIndex) void { + const local = ip.getLocal(tid); + const decl = ip.declPtr(decl_index); + decl.* = undefined; + @field(decl, Local.decl_next_free_field) = @enumFromInt(local.mutate.decls.free_list); + local.mutate.decls.free_list = @intFromEnum(decl_index); } pub fn createNamespace( ip: *InternPool, gpa: Allocator, + tid: Zcu.PerThread.Id, initialization: Module.Namespace, ) Allocator.Error!NamespaceIndex { - if (ip.namespaces_free_list.popOrNull()) |index| { - ip.allocated_namespaces.at(@intFromEnum(index)).* = initialization; - return index; + const local = ip.getLocal(tid); + const free_list_next = local.mutate.namespaces.free_list; + if (free_list_next != Local.BucketListMutate.free_list_sentinel) { + const reused_namespace_index: NamespaceIndex = @enumFromInt(free_list_next); + const reused_namespace = ip.namespacePtr(reused_namespace_index); + local.mutate.namespaces.free_list = + @intFromEnum(@field(reused_namespace, Local.namespace_next_free_field)); + reused_namespace.* = initialization; + return reused_namespace_index; } - const ptr = try ip.allocated_namespaces.addOne(gpa); - ptr.* = initialization; - return @enumFromInt(ip.allocated_namespaces.len - 1); + const namespaces = local.getMutableNamespaces(gpa); + if (local.mutate.namespaces.last_bucket_len == 0) { + try namespaces.ensureUnusedCapacity(1); + var arena = namespaces.arena.promote(namespaces.gpa); + defer namespaces.arena.* = arena.state; + namespaces.appendAssumeCapacity(.{try arena.allocator().create( + [1 << Local.namespaces_bucket_width]Module.Namespace, + )}); + } + const unwrapped_namespace_index: NamespaceIndex.Unwrapped = .{ + .tid = tid, + .bucket_index = namespaces.mutate.len - 1, + .index = local.mutate.namespaces.last_bucket_len, + }; + local.mutate.namespaces.last_bucket_len = + (unwrapped_namespace_index.index + 1) & Local.namespaces_bucket_mask; + const namespace_index = unwrapped_namespace_index.wrap(ip); + ip.namespacePtr(namespace_index).* = initialization; + return namespace_index; } -pub fn destroyNamespace(ip: *InternPool, gpa: Allocator, index: NamespaceIndex) void { - ip.namespacePtr(index).* = .{ +pub fn destroyNamespace( + ip: *InternPool, + tid: Zcu.PerThread.Id, + namespace_index: NamespaceIndex, +) void { + const local = ip.getLocal(tid); + const namespace = ip.namespacePtr(namespace_index); + namespace.* = .{ .parent = undefined, .file_scope = undefined, .decl_index = undefined, }; - ip.namespaces_free_list.append(gpa, index) catch { - // In order to keep `destroyNamespace` a non-fallible function, we ignore memory - // allocation failures here, instead leaking the Namespace until garbage collection. - }; + @field(namespace, Local.namespace_next_free_field) = + @enumFromInt(local.mutate.namespaces.free_list); + local.mutate.namespaces.free_list = @intFromEnum(namespace_index); } const EmbeddedNulls = enum { diff --git a/src/Sema.zig b/src/Sema.zig index f1c61fdd2a..b897228f58 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -2830,7 +2830,7 @@ fn zirStructDecl( inst, ); mod.declPtr(new_decl_index).owns_tv = true; - errdefer mod.abortAnonDecl(new_decl_index); + errdefer pt.abortAnonDecl(new_decl_index); if (pt.zcu.comp.debug_incremental) { try ip.addDependency( @@ -2841,12 +2841,12 @@ fn zirStructDecl( } // TODO: if AstGen tells us `@This` was not used in the fields, we can elide the namespace. - const new_namespace_index: InternPool.OptionalNamespaceIndex = if (true or decls_len > 0) (try mod.createNamespace(.{ + const new_namespace_index: InternPool.OptionalNamespaceIndex = if (true or decls_len > 0) (try pt.createNamespace(.{ .parent = block.namespace.toOptional(), .decl_index = new_decl_index, .file_scope = block.getFileScopeIndex(mod), })).toOptional() else .none; - errdefer if (new_namespace_index.unwrap()) |ns| mod.destroyNamespace(ns); + errdefer if (new_namespace_index.unwrap()) |ns| pt.destroyNamespace(ns); if (new_namespace_index.unwrap()) |ns| { const decls = sema.code.bodySlice(extra_index, decls_len); @@ -2872,8 +2872,8 @@ fn createAnonymousDeclTypeNamed( const ip = &zcu.intern_pool; const gpa = sema.gpa; const namespace = block.namespace; - const new_decl_index = try zcu.allocateNewDecl(namespace); - errdefer zcu.destroyDecl(new_decl_index); + const new_decl_index = try pt.allocateNewDecl(namespace); + errdefer pt.destroyDecl(new_decl_index); switch (name_strategy) { .anon => {}, // handled after switch @@ -3068,7 +3068,7 @@ fn zirEnumDecl( ); const new_decl = mod.declPtr(new_decl_index); new_decl.owns_tv = true; - errdefer if (!done) mod.abortAnonDecl(new_decl_index); + errdefer if (!done) pt.abortAnonDecl(new_decl_index); if (pt.zcu.comp.debug_incremental) { try mod.intern_pool.addDependency( @@ -3079,12 +3079,12 @@ fn zirEnumDecl( } // TODO: if AstGen tells us `@This` was not used in the fields, we can elide the namespace. - const new_namespace_index: InternPool.OptionalNamespaceIndex = if (true or decls_len > 0) (try mod.createNamespace(.{ + const new_namespace_index: InternPool.OptionalNamespaceIndex = if (true or decls_len > 0) (try pt.createNamespace(.{ .parent = block.namespace.toOptional(), .decl_index = new_decl_index, .file_scope = block.getFileScopeIndex(mod), })).toOptional() else .none; - errdefer if (!done) if (new_namespace_index.unwrap()) |ns| mod.destroyNamespace(ns); + errdefer if (!done) if (new_namespace_index.unwrap()) |ns| pt.destroyNamespace(ns); if (new_namespace_index.unwrap()) |ns| { try pt.scanNamespace(ns, decls, new_decl); @@ -3335,7 +3335,7 @@ fn zirUnionDecl( inst, ); mod.declPtr(new_decl_index).owns_tv = true; - errdefer mod.abortAnonDecl(new_decl_index); + errdefer pt.abortAnonDecl(new_decl_index); if (pt.zcu.comp.debug_incremental) { try mod.intern_pool.addDependency( @@ -3346,12 +3346,12 @@ fn zirUnionDecl( } // TODO: if AstGen tells us `@This` was not used in the fields, we can elide the namespace. - const new_namespace_index: InternPool.OptionalNamespaceIndex = if (true or decls_len > 0) (try mod.createNamespace(.{ + const new_namespace_index: InternPool.OptionalNamespaceIndex = if (true or decls_len > 0) (try pt.createNamespace(.{ .parent = block.namespace.toOptional(), .decl_index = new_decl_index, .file_scope = block.getFileScopeIndex(mod), })).toOptional() else .none; - errdefer if (new_namespace_index.unwrap()) |ns| mod.destroyNamespace(ns); + errdefer if (new_namespace_index.unwrap()) |ns| pt.destroyNamespace(ns); if (new_namespace_index.unwrap()) |ns| { const decls = sema.code.bodySlice(extra_index, decls_len); @@ -3425,7 +3425,7 @@ fn zirOpaqueDecl( inst, ); mod.declPtr(new_decl_index).owns_tv = true; - errdefer mod.abortAnonDecl(new_decl_index); + errdefer pt.abortAnonDecl(new_decl_index); if (pt.zcu.comp.debug_incremental) { try ip.addDependency( @@ -3435,12 +3435,12 @@ fn zirOpaqueDecl( ); } - const new_namespace_index: InternPool.OptionalNamespaceIndex = if (decls_len > 0) (try mod.createNamespace(.{ + const new_namespace_index: InternPool.OptionalNamespaceIndex = if (decls_len > 0) (try pt.createNamespace(.{ .parent = block.namespace.toOptional(), .decl_index = new_decl_index, .file_scope = block.getFileScopeIndex(mod), })).toOptional() else .none; - errdefer if (new_namespace_index.unwrap()) |ns| mod.destroyNamespace(ns); + errdefer if (new_namespace_index.unwrap()) |ns| pt.destroyNamespace(ns); if (new_namespace_index.unwrap()) |ns| { const decls = sema.code.bodySlice(extra_index, decls_len); @@ -21716,7 +21716,7 @@ fn zirReify( inst, ); mod.declPtr(new_decl_index).owns_tv = true; - errdefer mod.abortAnonDecl(new_decl_index); + errdefer pt.abortAnonDecl(new_decl_index); try pt.finalizeAnonDecl(new_decl_index); @@ -21916,7 +21916,7 @@ fn reifyEnum( inst, ); mod.declPtr(new_decl_index).owns_tv = true; - errdefer mod.abortAnonDecl(new_decl_index); + errdefer pt.abortAnonDecl(new_decl_index); wip_ty.prepare(ip, new_decl_index, .none); wip_ty.setTagTy(ip, tag_ty.toIntern()); @@ -22063,7 +22063,7 @@ fn reifyUnion( inst, ); mod.declPtr(new_decl_index).owns_tv = true; - errdefer mod.abortAnonDecl(new_decl_index); + errdefer pt.abortAnonDecl(new_decl_index); const field_types = try sema.arena.alloc(InternPool.Index, fields_len); const field_aligns = if (any_aligns) try sema.arena.alloc(InternPool.Alignment, fields_len) else undefined; @@ -22322,7 +22322,7 @@ fn reifyStruct( inst, ); mod.declPtr(new_decl_index).owns_tv = true; - errdefer mod.abortAnonDecl(new_decl_index); + errdefer pt.abortAnonDecl(new_decl_index); const struct_type = ip.loadStructType(wip_ty.index); @@ -26497,8 +26497,8 @@ fn zirBuiltinExtern( } const ptr_info = ty.ptrInfo(mod); - const new_decl_index = try mod.allocateNewDecl(sema.owner_decl.src_namespace); - errdefer mod.destroyDecl(new_decl_index); + const new_decl_index = try pt.allocateNewDecl(sema.owner_decl.src_namespace); + errdefer pt.destroyDecl(new_decl_index); const new_decl = mod.declPtr(new_decl_index); try mod.initNewAnonDecl( new_decl_index, @@ -36733,8 +36733,8 @@ fn generateUnionTagTypeNumbered( const gpa = sema.gpa; const ip = &mod.intern_pool; - const new_decl_index = try mod.allocateNewDecl(block.namespace); - errdefer mod.destroyDecl(new_decl_index); + const new_decl_index = try pt.allocateNewDecl(block.namespace); + errdefer pt.destroyDecl(new_decl_index); const fqn = try union_owner_decl.fullyQualifiedName(pt); const name = try ip.getOrPutStringFmt( gpa, @@ -36748,7 +36748,7 @@ fn generateUnionTagTypeNumbered( Value.@"unreachable", name, ); - errdefer mod.abortAnonDecl(new_decl_index); + errdefer pt.abortAnonDecl(new_decl_index); const new_decl = mod.declPtr(new_decl_index); new_decl.owns_tv = true; @@ -36785,8 +36785,8 @@ fn generateUnionTagTypeSimple( const new_decl_index = new_decl_index: { const fqn = try union_owner_decl.fullyQualifiedName(pt); - const new_decl_index = try mod.allocateNewDecl(block.namespace); - errdefer mod.destroyDecl(new_decl_index); + const new_decl_index = try pt.allocateNewDecl(block.namespace); + errdefer pt.destroyDecl(new_decl_index); const name = try ip.getOrPutStringFmt( gpa, pt.tid, @@ -36802,7 +36802,7 @@ fn generateUnionTagTypeSimple( mod.declPtr(new_decl_index).name_fully_qualified = true; break :new_decl_index new_decl_index; }; - errdefer mod.abortAnonDecl(new_decl_index); + errdefer pt.abortAnonDecl(new_decl_index); const enum_ty = try ip.getGeneratedTagEnumType(gpa, pt.tid, .{ .decl = new_decl_index, diff --git a/src/Zcu.zig b/src/Zcu.zig index 2f87bcca0f..92a0765ebb 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -2410,6 +2410,7 @@ pub fn init(mod: *Module, thread_count: usize) !void { } pub fn deinit(zcu: *Zcu) void { + const pt: Zcu.PerThread = .{ .tid = .main, .zcu = zcu }; const gpa = zcu.gpa; if (zcu.llvm_object) |llvm_object| { @@ -2422,7 +2423,7 @@ pub fn deinit(zcu: *Zcu) void { } for (0..zcu.import_table.entries.len) |file_index_usize| { const file_index: File.Index = @enumFromInt(file_index_usize); - zcu.destroyFile(file_index); + pt.destroyFile(file_index); } zcu.import_table.deinit(gpa); @@ -2497,68 +2498,9 @@ pub fn deinit(zcu: *Zcu) void { zcu.all_references.deinit(gpa); zcu.free_references.deinit(gpa); - { - var it = zcu.intern_pool.allocated_namespaces.iterator(0); - while (it.next()) |namespace| { - namespace.decls.deinit(gpa); - namespace.usingnamespace_set.deinit(gpa); - } - } - zcu.intern_pool.deinit(gpa); } -pub fn destroyDecl(mod: *Module, decl_index: Decl.Index) void { - const gpa = mod.gpa; - const ip = &mod.intern_pool; - - { - _ = mod.test_functions.swapRemove(decl_index); - if (mod.global_assembly.fetchSwapRemove(decl_index)) |kv| { - gpa.free(kv.value); - } - } - - ip.destroyDecl(gpa, decl_index); - - if (mod.emit_h) |mod_emit_h| { - const decl_emit_h = mod_emit_h.declPtr(decl_index); - decl_emit_h.fwd_decl.deinit(gpa); - decl_emit_h.* = undefined; - } -} - -fn deinitFile(zcu: *Zcu, file_index: File.Index) void { - const gpa = zcu.gpa; - const file = zcu.fileByIndex(file_index); - const is_builtin = file.mod.isBuiltin(); - log.debug("deinit File {s}", .{file.sub_file_path}); - if (is_builtin) { - file.unloadTree(gpa); - file.unloadZir(gpa); - } else { - gpa.free(file.sub_file_path); - file.unload(gpa); - } - file.references.deinit(gpa); - if (zcu.fileRootDecl(file_index).unwrap()) |root_decl| { - zcu.destroyDecl(root_decl); - } - if (file.prev_zir) |prev_zir| { - prev_zir.deinit(gpa); - gpa.destroy(prev_zir); - } - file.* = undefined; -} - -pub fn destroyFile(zcu: *Zcu, file_index: File.Index) void { - const gpa = zcu.gpa; - const file = zcu.fileByIndex(file_index); - const is_builtin = file.mod.isBuiltin(); - zcu.deinitFile(file_index); - if (!is_builtin) gpa.destroy(file); -} - pub fn declPtr(mod: *Module, index: Decl.Index) *Decl { return mod.intern_pool.declPtr(index); } @@ -3269,13 +3211,6 @@ fn computePathDigest(zcu: *Zcu, mod: *Package.Module, sub_file_path: []const u8) return bin; } -/// Cancel the creation of an anon decl and delete any references to it. -/// If other decls depend on this decl, they must be aborted first. -pub fn abortAnonDecl(mod: *Module, decl_index: Decl.Index) void { - assert(!mod.declIsRoot(decl_index)); - mod.destroyDecl(decl_index); -} - /// Delete all the Export objects that are caused by this `AnalUnit`. Re-analysis of /// this `AnalUnit` will cause them to be re-created (or not). pub fn deleteUnitExports(zcu: *Zcu, anal_unit: AnalUnit) void { @@ -3357,42 +3292,6 @@ pub fn addUnitReference(zcu: *Zcu, src_unit: AnalUnit, referenced_unit: AnalUnit gop.value_ptr.* = @intCast(ref_idx); } -pub fn createNamespace(mod: *Module, initialization: Namespace) !Namespace.Index { - return mod.intern_pool.createNamespace(mod.gpa, initialization); -} - -pub fn destroyNamespace(mod: *Module, index: Namespace.Index) void { - return mod.intern_pool.destroyNamespace(mod.gpa, index); -} - -pub fn allocateNewDecl(zcu: *Zcu, namespace: Namespace.Index) !Decl.Index { - const gpa = zcu.gpa; - const decl_index = try zcu.intern_pool.createDecl(gpa, .{ - .name = undefined, - .src_namespace = namespace, - .has_tv = false, - .owns_tv = false, - .val = undefined, - .alignment = undefined, - .@"linksection" = .none, - .@"addrspace" = .generic, - .analysis = .unreferenced, - .zir_decl_index = .none, - .is_pub = false, - .is_exported = false, - .kind = .anon, - }); - - if (zcu.emit_h) |zcu_emit_h| { - if (@intFromEnum(decl_index) >= zcu_emit_h.allocated_emit_h.len) { - try zcu_emit_h.allocated_emit_h.append(gpa, .{}); - assert(@intFromEnum(decl_index) == zcu_emit_h.allocated_emit_h.len); - } - } - - return decl_index; -} - pub fn getErrorValue( mod: *Module, name: InternPool.NullTerminatedString, diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index f8a3104dc0..a46f136a44 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -5,6 +5,58 @@ tid: Id, pub const Id = if (InternPool.single_threaded) enum { main } else enum(u8) { main, _ }; +pub fn destroyDecl(pt: Zcu.PerThread, decl_index: Zcu.Decl.Index) void { + const zcu = pt.zcu; + const gpa = zcu.gpa; + + { + _ = zcu.test_functions.swapRemove(decl_index); + if (zcu.global_assembly.fetchSwapRemove(decl_index)) |kv| { + gpa.free(kv.value); + } + } + + pt.zcu.intern_pool.destroyDecl(pt.tid, decl_index); + + if (zcu.emit_h) |zcu_emit_h| { + const decl_emit_h = zcu_emit_h.declPtr(decl_index); + decl_emit_h.fwd_decl.deinit(gpa); + decl_emit_h.* = undefined; + } +} + +fn deinitFile(pt: Zcu.PerThread, file_index: Zcu.File.Index) void { + const zcu = pt.zcu; + const gpa = zcu.gpa; + const file = zcu.fileByIndex(file_index); + const is_builtin = file.mod.isBuiltin(); + log.debug("deinit File {s}", .{file.sub_file_path}); + if (is_builtin) { + file.unloadTree(gpa); + file.unloadZir(gpa); + } else { + gpa.free(file.sub_file_path); + file.unload(gpa); + } + file.references.deinit(gpa); + if (zcu.fileRootDecl(file_index).unwrap()) |root_decl| { + pt.zcu.intern_pool.destroyDecl(pt.tid, root_decl); + } + if (file.prev_zir) |prev_zir| { + prev_zir.deinit(gpa); + gpa.destroy(prev_zir); + } + file.* = undefined; +} + +pub fn destroyFile(pt: Zcu.PerThread, file_index: Zcu.File.Index) void { + const gpa = pt.zcu.gpa; + const file = pt.zcu.fileByIndex(file_index); + const is_builtin = file.mod.isBuiltin(); + pt.deinitFile(file_index); + if (!is_builtin) gpa.destroy(file); +} + pub fn astGenFile( pt: Zcu.PerThread, file: *Zcu.File, @@ -930,14 +982,14 @@ fn semaFile(pt: Zcu.PerThread, file_index: Zcu.File.Index) Zcu.SemaError!void { // Because these three things each reference each other, `undefined` // placeholders are used before being set after the struct type gains an // InternPool index. - const new_namespace_index = try zcu.createNamespace(.{ + const new_namespace_index = try pt.createNamespace(.{ .parent = .none, .decl_index = undefined, .file_scope = file_index, }); - errdefer zcu.destroyNamespace(new_namespace_index); + errdefer pt.destroyNamespace(new_namespace_index); - const new_decl_index = try zcu.allocateNewDecl(new_namespace_index); + const new_decl_index = try pt.allocateNewDecl(new_namespace_index); const new_decl = zcu.declPtr(new_decl_index); errdefer @panic("TODO error handling"); @@ -1380,6 +1432,13 @@ pub fn embedFile( return pt.newEmbedFile(cur_file.mod, sub_file_path, resolved_path, gop.value_ptr, src_loc); } +/// Cancel the creation of an anon decl and delete any references to it. +/// If other decls depend on this decl, they must be aborted first. +pub fn abortAnonDecl(pt: Zcu.PerThread, decl_index: Zcu.Decl.Index) void { + assert(!pt.zcu.declIsRoot(decl_index)); + pt.destroyDecl(decl_index); +} + /// Finalize the creation of an anon decl. pub fn finalizeAnonDecl(pt: Zcu.PerThread, decl_index: Zcu.Decl.Index) Allocator.Error!void { if (pt.zcu.declPtr(decl_index).typeOf(pt.zcu).isFnOrHasRuntimeBits(pt)) { @@ -1674,7 +1733,7 @@ const ScanDeclIter = struct { break :decl_index .{ was_exported, decl_index }; } else decl_index: { // Create and set up a new Decl. - const new_decl_index = try zcu.allocateNewDecl(namespace_index); + const new_decl_index = try pt.allocateNewDecl(namespace_index); const new_decl = zcu.declPtr(new_decl_index); new_decl.kind = kind; new_decl.name = decl_name; @@ -1981,6 +2040,43 @@ pub fn analyzeFnBody(pt: Zcu.PerThread, func_index: InternPool.Index, arena: All }; } +pub fn createNamespace(pt: Zcu.PerThread, initialization: Zcu.Namespace) !Zcu.Namespace.Index { + return pt.zcu.intern_pool.createNamespace(pt.zcu.gpa, pt.tid, initialization); +} + +pub fn destroyNamespace(pt: Zcu.PerThread, namespace_index: Zcu.Namespace.Index) void { + return pt.zcu.intern_pool.destroyNamespace(pt.tid, namespace_index); +} + +pub fn allocateNewDecl(pt: Zcu.PerThread, namespace: Zcu.Namespace.Index) !Zcu.Decl.Index { + const zcu = pt.zcu; + const gpa = zcu.gpa; + const decl_index = try zcu.intern_pool.createDecl(gpa, pt.tid, .{ + .name = undefined, + .src_namespace = namespace, + .has_tv = false, + .owns_tv = false, + .val = undefined, + .alignment = undefined, + .@"linksection" = .none, + .@"addrspace" = .generic, + .analysis = .unreferenced, + .zir_decl_index = .none, + .is_pub = false, + .is_exported = false, + .kind = .anon, + }); + + if (zcu.emit_h) |zcu_emit_h| { + if (@intFromEnum(decl_index) >= zcu_emit_h.allocated_emit_h.len) { + try zcu_emit_h.allocated_emit_h.append(gpa, .{}); + assert(@intFromEnum(decl_index) == zcu_emit_h.allocated_emit_h.len); + } + } + + return decl_index; +} + fn lockAndClearFileCompileError(pt: Zcu.PerThread, file: *Zcu.File) void { switch (file.status) { .success_zir, .retryable_failure => {},