From ec0f76c5996e88f61d376640bf36ed7feb2b0ea6 Mon Sep 17 00:00:00 2001 From: Ryan Liptak Date: Tue, 3 Oct 2023 17:37:09 -0700 Subject: [PATCH] GeneralPurposeAllocator.searchBucket: check current bucket before searching the list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow up to #17383. This is a minor optimization that only matters when a small allocation is resized/free'd soon after it is allocated. The only real difference I was able to observe with this was via a synthetic benchmark that allocates a full bucket and then frees all but one of the slots, over and over in a loop: Debug build: Benchmark 1 (9 runs): gpa-degen-master.exe measurement mean ± σ min … max outliers delta wall_time 575ms ± 5.19ms 569ms … 583ms 0 ( 0%) 0% peak_rss 43.8MB ± 1.37KB 43.8MB … 43.8MB 1 (11%) 0% Benchmark 2 (10 runs): gpa-degen-search-cur.exe measurement mean ± σ min … max outliers delta wall_time 532ms ± 5.55ms 520ms … 539ms 0 ( 0%) ⚡- 7.5% ± 0.9% peak_rss 43.8MB ± 65.2KB 43.8MB … 44.0MB 1 (10%) + 0.0% ± 0.1% ReleaseFast build: Benchmark 1 (129 runs): gpa-degen-master-release.exe measurement mean ± σ min … max outliers delta wall_time 38.9ms ± 1.12ms 36.7ms … 42.4ms 8 ( 6%) 0% peak_rss 23.2MB ± 2.39KB 23.2MB … 23.2MB 0 ( 0%) 0% Benchmark 2 (151 runs): gpa-degen-search-cur-release.exe measurement mean ± σ min … max outliers delta wall_time 33.2ms ± 999us 31.9ms … 36.3ms 20 (13%) ⚡- 14.7% ± 0.6% peak_rss 23.2MB ± 2.26KB 23.2MB … 23.2MB 0 ( 0%) + 0.0% ± 0.0% --- lib/std/heap/general_purpose_allocator.zig | 24 +++++++++++++--------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index 2870dcee5a..6dc6df3998 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -541,10 +541,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { fn searchBucket( buckets: *Buckets, addr: usize, + current_bucket: ?*BucketHeader, ) ?*BucketHeader { - const search_page = mem.alignBackward(usize, addr, page_size); + const search_page: [*]align(page_size) u8 = @ptrFromInt(mem.alignBackward(usize, addr, page_size)); + if (current_bucket != null and current_bucket.?.page == search_page) { + return current_bucket; + } var search_header: BucketHeader = undefined; - search_header.page = @ptrFromInt(search_page); + search_header.page = search_page; const entry = buckets.getEntryFor(&search_header); return if (entry.node) |node| node.key else null; } @@ -712,7 +716,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { var bucket_index = math.log2(size_class_hint); var size_class: usize = size_class_hint; const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { - if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr))) |bucket| { + if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| { break bucket; } size_class *= 2; @@ -720,7 +724,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (config.retain_metadata) { if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) { // object not in active buckets or a large allocation, so search empty buckets - if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr))) |bucket| { + if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr), null)) |bucket| { // bucket is empty so is_used below will always be false and we exit there break :blk bucket; } else { @@ -830,7 +834,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { var bucket_index = math.log2(size_class_hint); var size_class: usize = size_class_hint; const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { - if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr))) |bucket| { + if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| { break bucket; } size_class *= 2; @@ -838,7 +842,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (config.retain_metadata) { if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) { // object not in active buckets or a large allocation, so search empty buckets - if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr))) |bucket| { + if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr), null)) |bucket| { // bucket is empty so is_used below will always be false and we exit there break :blk bucket; } else { @@ -1387,10 +1391,10 @@ test "double frees" { const index: usize = 6; const size_class: usize = @as(usize, 1) << 6; const small = try allocator.alloc(u8, size_class); - try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr)) != null); + try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr), gpa.cur_buckets[index]) != null); allocator.free(small); - try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr)) == null); - try std.testing.expect(GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr)) != null); + try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr), gpa.cur_buckets[index]) == null); + try std.testing.expect(GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr), null) != null); // detect a large allocation double free const large = try allocator.alloc(u8, 2 * page_size); @@ -1408,7 +1412,7 @@ test "double frees" { // check that flushing retained metadata doesn't disturb live allocations gpa.flushRetainedMetadata(); try std.testing.expect(gpa.empty_buckets.root == null); - try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(normal_small.ptr)) != null); + try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(normal_small.ptr), gpa.cur_buckets[index]) != null); try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(normal_large.ptr))); try std.testing.expect(!gpa.large_allocations.contains(@intFromPtr(large.ptr))); }