diff --git a/lib/std/heap.zig b/lib/std/heap.zig index e005101c6b..2d3a96676d 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -132,10 +132,6 @@ const CAllocator = struct { ) Allocator.Error!usize { _ = buf_align; _ = return_address; - if (new_len == 0) { - alignedFree(buf.ptr); - return 0; - } if (new_len <= buf.len) { return mem.alignAllocLen(buf.len, new_len, len_align); } @@ -147,6 +143,17 @@ const CAllocator = struct { } return error.OutOfMemory; } + + fn free( + _: *c_void, + buf: []u8, + buf_align: u29, + return_address: usize, + ) void { + _ = buf_align; + _ = return_address; + alignedFree(buf.ptr); + } }; /// Supports the full Allocator interface, including alignment, and exploiting @@ -159,6 +166,7 @@ pub const c_allocator = Allocator{ const c_allocator_vtable = Allocator.VTable{ .alloc = CAllocator.alloc, .resize = CAllocator.resize, + .free = CAllocator.free, }; /// Asserts allocations are within `@alignOf(std.c.max_align_t)` and directly calls @@ -173,6 +181,7 @@ pub const raw_c_allocator = Allocator{ const raw_c_allocator_vtable = Allocator.VTable{ .alloc = rawCAlloc, .resize = rawCResize, + .free = rawCFree, }; fn rawCAlloc( @@ -199,16 +208,23 @@ fn rawCResize( ) Allocator.Error!usize { _ = old_align; _ = ret_addr; - if (new_len == 0) { - c.free(buf.ptr); - return 0; - } if (new_len <= buf.len) { return mem.alignAllocLen(buf.len, new_len, len_align); } return error.OutOfMemory; } +fn rawCFree( + _: *c_void, + buf: []u8, + old_align: u29, + ret_addr: usize, +) void { + _ = old_align; + _ = ret_addr; + c.free(buf.ptr); +} + /// This allocator makes a syscall directly for every allocation and free. /// Thread-safe and lock-free. pub const page_allocator = if (builtin.target.isWasm()) @@ -238,6 +254,7 @@ const PageAllocator = struct { const vtable = Allocator.VTable{ .alloc = alloc, .resize = resize, + .free = free, }; fn alloc(_: *c_void, n: usize, alignment: u29, len_align: u29, ra: usize) error{OutOfMemory}![]u8 { @@ -351,16 +368,6 @@ const PageAllocator = struct { if (builtin.os.tag == .windows) { const w = os.windows; - if (new_size == 0) { - // From the docs: - // "If the dwFreeType parameter is MEM_RELEASE, this parameter - // must be 0 (zero). The function frees the entire region that - // is reserved in the initial allocation call to VirtualAlloc." - // So we can only use MEM_RELEASE when actually releasing the - // whole allocation. - w.VirtualFree(buf_unaligned.ptr, 0, w.MEM_RELEASE); - return 0; - } if (new_size <= buf_unaligned.len) { const base_addr = @ptrToInt(buf_unaligned.ptr); const old_addr_end = base_addr + buf_unaligned.len; @@ -391,8 +398,6 @@ const PageAllocator = struct { const ptr = @alignCast(mem.page_size, buf_unaligned.ptr + new_size_aligned); // TODO: if the next_mmap_addr_hint is within the unmapped range, update it os.munmap(ptr[0 .. buf_aligned_len - new_size_aligned]); - if (new_size_aligned == 0) - return 0; return alignPageAllocLen(new_size_aligned, new_size, len_align); } @@ -400,6 +405,19 @@ const PageAllocator = struct { // TODO: if the next_mmap_addr_hint is within the remapped range, update it return error.OutOfMemory; } + + fn free(_: *c_void, buf_unaligned: []u8, buf_align: u29, return_address: usize) void { + _ = buf_align; + _ = return_address; + + if (builtin.os.tag == .windows) { + os.windows.VirtualFree(buf_unaligned.ptr, 0, os.windows.MEM_RELEASE); + } else { + const buf_aligned_len = mem.alignForward(buf_unaligned.len, mem.page_size); + const ptr = @alignCast(mem.page_size, buf_unaligned.ptr); + os.munmap(ptr[0..buf_aligned_len]); + } + } }; const WasmPageAllocator = struct { @@ -412,6 +430,7 @@ const WasmPageAllocator = struct { const vtable = Allocator.VTable{ .alloc = alloc, .resize = resize, + .free = free, }; const PageStatus = enum(u1) { @@ -571,7 +590,21 @@ const WasmPageAllocator = struct { const base = nPages(@ptrToInt(buf.ptr)); freePages(base + new_n, base + current_n); } - return if (new_len == 0) 0 else alignPageAllocLen(new_n * mem.page_size, new_len, len_align); + return alignPageAllocLen(new_n * mem.page_size, new_len, len_align); + } + + fn free( + _: *c_void, + buf: []u8, + buf_align: u29, + return_address: usize, + ) void { + _ = buf_align; + _ = return_address; + const aligned_len = mem.alignForward(buf.len, mem.page_size); + const current_n = nPages(aligned_len); + const base = nPages(@ptrToInt(buf.ptr)); + freePages(base, base + current_n); } }; @@ -588,7 +621,7 @@ pub const HeapAllocator = switch (builtin.os.tag) { } pub fn allocator(self: *HeapAllocator) Allocator { - return Allocator.init(self, alloc, resize); + return Allocator.init(self, alloc, resize, free); } pub fn deinit(self: *HeapAllocator) void { @@ -644,10 +677,6 @@ pub const HeapAllocator = switch (builtin.os.tag) { ) error{OutOfMemory}!usize { _ = buf_align; _ = return_address; - if (new_size == 0) { - os.windows.HeapFree(self.heap_handle.?, 0, @intToPtr(*c_void, getRecordPtr(buf).*)); - return 0; - } const root_addr = getRecordPtr(buf).*; const align_offset = @ptrToInt(buf.ptr) - root_addr; @@ -669,6 +698,17 @@ pub const HeapAllocator = switch (builtin.os.tag) { getRecordPtr(buf.ptr[0..return_len]).* = root_addr; return return_len; } + + fn free( + self: *HeapAllocator, + buf: []u8, + buf_align: u29, + return_address: usize, + ) void { + _ = buf_align; + _ = return_address; + os.windows.HeapFree(self.heap_handle.?, 0, @intToPtr(*c_void, getRecordPtr(buf).*)); + } }, else => @compileError("Unsupported OS"), }; @@ -696,13 +736,18 @@ pub const FixedBufferAllocator = struct { /// *WARNING* using this at the same time as the interface returned by `threadSafeAllocator` is not thread safe pub fn allocator(self: *FixedBufferAllocator) Allocator { - return Allocator.init(self, alloc, resize); + return Allocator.init(self, alloc, resize, free); } /// Provides a lock free thread safe `Allocator` interface to the underlying `FixedBufferAllocator` /// *WARNING* using this at the same time as the interface returned by `getAllocator` is not thread safe pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator { - return Allocator.init(self, threadSafeAlloc, Allocator.NoResize(FixedBufferAllocator).noResize); + return Allocator.init( + self, + threadSafeAlloc, + Allocator.NoResize(FixedBufferAllocator).noResize, + Allocator.NoOpFree(FixedBufferAllocator).noOpFree, + ); } pub fn ownsPtr(self: *FixedBufferAllocator, ptr: [*]u8) bool { @@ -715,7 +760,7 @@ pub const FixedBufferAllocator = struct { /// NOTE: this will not work in all cases, if the last allocation had an adjusted_index /// then we won't be able to determine what the last allocation was. This is because - /// the alignForward operation done in alloc is not reverisible. + /// the alignForward operation done in alloc is not reversible. pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool { return buf.ptr + buf.len == self.buffer.ptr + self.end_index; } @@ -751,13 +796,13 @@ pub const FixedBufferAllocator = struct { if (!self.isLastAllocation(buf)) { if (new_size > buf.len) return error.OutOfMemory; - return if (new_size == 0) 0 else mem.alignAllocLen(buf.len, new_size, len_align); + return mem.alignAllocLen(buf.len, new_size, len_align); } if (new_size <= buf.len) { const sub = buf.len - new_size; self.end_index -= sub; - return if (new_size == 0) 0 else mem.alignAllocLen(buf.len - sub, new_size, len_align); + return mem.alignAllocLen(buf.len - sub, new_size, len_align); } const add = new_size - buf.len; @@ -768,6 +813,21 @@ pub const FixedBufferAllocator = struct { return new_size; } + fn free( + self: *FixedBufferAllocator, + buf: []u8, + buf_align: u29, + return_address: usize, + ) void { + _ = buf_align; + _ = return_address; + assert(self.ownsSlice(buf)); // sanity check + + if (self.isLastAllocation(buf)) { + self.end_index -= buf.len; + } + } + fn threadSafeAlloc(self: *FixedBufferAllocator, n: usize, ptr_align: u29, len_align: u29, ra: usize) ![]u8 { _ = len_align; _ = ra; @@ -810,7 +870,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type { /// WARNING: This functions both fetches a `std.mem.Allocator` interface to this allocator *and* resets the internal buffer allocator pub fn get(self: *Self) Allocator { self.fixed_buffer_allocator = FixedBufferAllocator.init(self.buffer[0..]); - return Allocator.init(self, alloc, resize); + return Allocator.init(self, alloc, resize, free); } fn alloc( @@ -821,7 +881,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type { return_address: usize, ) error{OutOfMemory}![]u8 { return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, ptr_align, len_align, return_address) catch - return self.fallback_allocator.vtable.alloc(self.fallback_allocator.ptr, len, ptr_align, len_align, return_address); + return self.fallback_allocator.rawAlloc(len, ptr_align, len_align, return_address); } fn resize( @@ -835,7 +895,20 @@ pub fn StackFallbackAllocator(comptime size: usize) type { if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) { return FixedBufferAllocator.resize(&self.fixed_buffer_allocator, buf, buf_align, new_len, len_align, return_address); } else { - return self.fallback_allocator.vtable.resize(self.fallback_allocator.ptr, buf, buf_align, new_len, len_align, return_address); + return self.fallback_allocator.rawResize(buf, buf_align, new_len, len_align, return_address); + } + } + + fn free( + self: *Self, + buf: []u8, + buf_align: u29, + return_address: usize, + ) void { + if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) { + return FixedBufferAllocator.free(&self.fixed_buffer_allocator, buf, buf_align, return_address); + } else { + return self.fallback_allocator.rawFree(buf, buf_align, return_address); } } }; diff --git a/lib/std/heap/arena_allocator.zig b/lib/std/heap/arena_allocator.zig index 35e3f0ada2..c5a8d6bc7e 100644 --- a/lib/std/heap/arena_allocator.zig +++ b/lib/std/heap/arena_allocator.zig @@ -24,7 +24,7 @@ pub const ArenaAllocator = struct { }; pub fn allocator(self: *ArenaAllocator) Allocator { - return Allocator.init(self, alloc, resize); + return Allocator.init(self, alloc, resize, free); } const BufNode = std.SinglyLinkedList([]u8).Node; @@ -47,7 +47,7 @@ pub const ArenaAllocator = struct { const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16); const big_enough_len = prev_len + actual_min_size; const len = big_enough_len + big_enough_len / 2; - const buf = try self.child_allocator.vtable.alloc(self.child_allocator.ptr, len, @alignOf(BufNode), 1, @returnAddress()); + const buf = try self.child_allocator.rawAlloc(len, @alignOf(BufNode), 1, @returnAddress()); const buf_node = @ptrCast(*BufNode, @alignCast(@alignOf(BufNode), buf.ptr)); buf_node.* = BufNode{ .data = buf, @@ -111,4 +111,16 @@ pub const ArenaAllocator = struct { return error.OutOfMemory; } } + + fn free(self: *ArenaAllocator, buf: []u8, buf_align: u29, ret_addr: usize) void { + _ = buf_align; + _ = ret_addr; + + const cur_node = self.state.buffer_list.first orelse return; + const cur_buf = cur_node.data[@sizeOf(BufNode)..]; + + if (@ptrToInt(cur_buf.ptr) + self.state.end_index == @ptrToInt(buf.ptr) + buf.len) { + self.state.end_index -= buf.len; + } + } }; diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index be7651980c..5687b1efb2 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -281,7 +281,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { }; pub fn allocator(self: *Self) Allocator { - return Allocator.init(self, alloc, resize); + return Allocator.init(self, alloc, resize, free); } fn bucketStackTrace( @@ -388,7 +388,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { var it = self.large_allocations.iterator(); while (it.next()) |large| { if (large.value_ptr.freed) { - _ = self.backing_allocator.vtable.resize(self.backing_allocator.ptr, large.value_ptr.bytes, large.value_ptr.ptr_align, 0, 0, @returnAddress()) catch unreachable; + self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.ptr_align, @returnAddress()); } } } @@ -529,9 +529,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (config.retain_metadata and entry.value_ptr.freed) { if (config.safety) { reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free)); - // Recoverable if this is a free. - if (new_size == 0) - return @as(usize, 0); @panic("Unrecoverable double free"); } else { unreachable; @@ -555,7 +552,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { // Do memory limit accounting with requested sizes rather than what backing_allocator returns // because if we want to return error.OutOfMemory, we have to leave allocation untouched, and - // that is impossible to guarantee after calling backing_allocator.vtable.resize. + // that is impossible to guarantee after calling backing_allocator.rawResize. const prev_req_bytes = self.total_requested_bytes; if (config.enable_memory_limit) { const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size; @@ -568,29 +565,12 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { self.total_requested_bytes = prev_req_bytes; }; - const result_len = if (config.never_unmap and new_size == 0) - 0 - else - try self.backing_allocator.vtable.resize(self.backing_allocator.ptr, old_mem, old_align, new_size, len_align, ret_addr); + const result_len = try self.backing_allocator.rawResize(old_mem, old_align, new_size, len_align, ret_addr); if (config.enable_memory_limit) { entry.value_ptr.requested_size = new_size; } - if (result_len == 0) { - if (config.verbose_log) { - log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); - } - - if (!config.retain_metadata) { - assert(self.large_allocations.remove(@ptrToInt(old_mem.ptr))); - } else { - entry.value_ptr.freed = true; - entry.value_ptr.captureStackTrace(ret_addr, .free); - } - return 0; - } - if (config.verbose_log) { log.info("large resize {d} bytes at {*} to {d}", .{ old_mem.len, old_mem.ptr, new_size, @@ -601,6 +581,64 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { return result_len; } + /// This function assumes the object is in the large object storage regardless + /// of the parameters. + fn freeLarge( + self: *Self, + old_mem: []u8, + old_align: u29, + ret_addr: usize, + ) void { + _ = old_align; + + const entry = self.large_allocations.getEntry(@ptrToInt(old_mem.ptr)) orelse { + if (config.safety) { + @panic("Invalid free"); + } else { + unreachable; + } + }; + + if (config.retain_metadata and entry.value_ptr.freed) { + if (config.safety) { + reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free)); + return; + } else { + unreachable; + } + } + + if (config.safety and old_mem.len != entry.value_ptr.bytes.len) { + var addresses: [stack_n]usize = [1]usize{0} ** stack_n; + var free_stack_trace = StackTrace{ + .instruction_addresses = &addresses, + .index = 0, + }; + std.debug.captureStackTrace(ret_addr, &free_stack_trace); + log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {s} Free: {s}", .{ + entry.value_ptr.bytes.len, + old_mem.len, + entry.value_ptr.getStackTrace(.alloc), + free_stack_trace, + }); + } + + if (config.enable_memory_limit) { + self.total_requested_bytes -= entry.value_ptr.requested_size; + } + + if (config.verbose_log) { + log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); + } + + if (!config.retain_metadata) { + assert(self.large_allocations.remove(@ptrToInt(old_mem.ptr))); + } else { + entry.value_ptr.freed = true; + entry.value_ptr.captureStackTrace(ret_addr, .free); + } + } + pub fn setRequestedMemoryLimit(self: *Self, limit: usize) void { self.requested_memory_limit = limit; } @@ -656,9 +694,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { if (!is_used) { if (config.safety) { reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free)); - // Recoverable if this is a free. - if (new_size == 0) - return @as(usize, 0); @panic("Unrecoverable double free"); } else { unreachable; @@ -678,52 +713,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { self.total_requested_bytes = prev_req_bytes; }; - if (new_size == 0) { - // Capture stack trace to be the "first free", in case a double free happens. - bucket.captureStackTrace(ret_addr, size_class, slot_index, .free); - - used_byte.* &= ~(@as(u8, 1) << used_bit_index); - bucket.used_count -= 1; - if (bucket.used_count == 0) { - if (bucket.next == bucket) { - // it's the only bucket and therefore the current one - self.buckets[bucket_index] = null; - } else { - bucket.next.prev = bucket.prev; - bucket.prev.next = bucket.next; - self.buckets[bucket_index] = bucket.prev; - } - if (!config.never_unmap) { - self.backing_allocator.free(bucket.page[0..page_size]); - } - if (!config.retain_metadata) { - self.freeBucket(bucket, size_class); - } else { - // move alloc_cursor to end so we can tell size_class later - const slot_count = @divExact(page_size, size_class); - bucket.alloc_cursor = @truncate(SlotIndex, slot_count); - if (self.empty_buckets) |prev_bucket| { - // empty_buckets is ordered newest to oldest through prev so that if - // config.never_unmap is false and backing_allocator reuses freed memory - // then searchBuckets will always return the newer, relevant bucket - bucket.prev = prev_bucket; - bucket.next = prev_bucket.next; - prev_bucket.next = bucket; - bucket.next.prev = bucket; - } else { - bucket.prev = bucket; - bucket.next = bucket; - } - self.empty_buckets = bucket; - } - } else { - @memset(old_mem.ptr, undefined, old_mem.len); - } - if (config.verbose_log) { - log.info("small free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); - } - return @as(usize, 0); - } const new_aligned_size = math.max(new_size, old_align); const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size); if (new_size_class <= size_class) { @@ -740,6 +729,114 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { return error.OutOfMemory; } + fn free( + self: *Self, + old_mem: []u8, + old_align: u29, + ret_addr: usize, + ) void { + const held = self.mutex.acquire(); + defer held.release(); + + assert(old_mem.len != 0); + + const aligned_size = math.max(old_mem.len, old_align); + if (aligned_size > largest_bucket_object_size) { + self.freeLarge(old_mem, old_align, ret_addr); + return; + } + const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size); + + var bucket_index = math.log2(size_class_hint); + var size_class: usize = size_class_hint; + const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { + if (searchBucket(self.buckets[bucket_index], @ptrToInt(old_mem.ptr))) |bucket| { + // move bucket to head of list to optimize search for nearby allocations + self.buckets[bucket_index] = bucket; + break bucket; + } + size_class *= 2; + } else blk: { + if (config.retain_metadata) { + if (!self.large_allocations.contains(@ptrToInt(old_mem.ptr))) { + // object not in active buckets or a large allocation, so search empty buckets + if (searchBucket(self.empty_buckets, @ptrToInt(old_mem.ptr))) |bucket| { + // bucket is empty so is_used below will always be false and we exit there + break :blk bucket; + } else { + @panic("Invalid free"); + } + } + } + self.freeLarge(old_mem, old_align, ret_addr); + return; + }; + const byte_offset = @ptrToInt(old_mem.ptr) - @ptrToInt(bucket.page); + const slot_index = @intCast(SlotIndex, byte_offset / size_class); + const used_byte_index = slot_index / 8; + const used_bit_index = @intCast(u3, slot_index % 8); + const used_byte = bucket.usedBits(used_byte_index); + const is_used = @truncate(u1, used_byte.* >> used_bit_index) != 0; + if (!is_used) { + if (config.safety) { + reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free)); + // Recoverable if this is a free. + return; + } else { + unreachable; + } + } + + // Definitely an in-use small alloc now. + if (config.enable_memory_limit) { + self.total_requested_bytes -= old_mem.len; + } + + // Capture stack trace to be the "first free", in case a double free happens. + bucket.captureStackTrace(ret_addr, size_class, slot_index, .free); + + used_byte.* &= ~(@as(u8, 1) << used_bit_index); + bucket.used_count -= 1; + if (bucket.used_count == 0) { + if (bucket.next == bucket) { + // it's the only bucket and therefore the current one + self.buckets[bucket_index] = null; + } else { + bucket.next.prev = bucket.prev; + bucket.prev.next = bucket.next; + self.buckets[bucket_index] = bucket.prev; + } + if (!config.never_unmap) { + self.backing_allocator.free(bucket.page[0..page_size]); + } + if (!config.retain_metadata) { + self.freeBucket(bucket, size_class); + } else { + // move alloc_cursor to end so we can tell size_class later + const slot_count = @divExact(page_size, size_class); + bucket.alloc_cursor = @truncate(SlotIndex, slot_count); + if (self.empty_buckets) |prev_bucket| { + // empty_buckets is ordered newest to oldest through prev so that if + // config.never_unmap is false and backing_allocator reuses freed memory + // then searchBuckets will always return the newer, relevant bucket + bucket.prev = prev_bucket; + bucket.next = prev_bucket.next; + prev_bucket.next = bucket; + bucket.next.prev = bucket; + } else { + bucket.prev = bucket; + bucket.next = bucket; + } + self.empty_buckets = bucket; + } + } else { + @memset(old_mem.ptr, undefined, old_mem.len); + } + if (config.verbose_log) { + log.info("small free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); + } + } + // Returns true if an allocation of `size` bytes is within the specified // limits if enable_memory_limit is true fn isAllocationAllowed(self: *Self, size: usize) bool { @@ -764,7 +861,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { const new_aligned_size = math.max(len, ptr_align); if (new_aligned_size > largest_bucket_object_size) { try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1); - const slice = try self.backing_allocator.vtable.alloc(self.backing_allocator.ptr, len, ptr_align, len_align, ret_addr); + const slice = try self.backing_allocator.rawAlloc(len, ptr_align, len_align, ret_addr); const gop = self.large_allocations.getOrPutAssumeCapacity(@ptrToInt(slice.ptr)); if (config.retain_metadata and !config.never_unmap) { diff --git a/lib/std/heap/log_to_writer_allocator.zig b/lib/std/heap/log_to_writer_allocator.zig index 5019a015bc..fa8c19e0a0 100644 --- a/lib/std/heap/log_to_writer_allocator.zig +++ b/lib/std/heap/log_to_writer_allocator.zig @@ -18,7 +18,7 @@ pub fn LogToWriterAllocator(comptime Writer: type) type { } pub fn allocator(self: *Self) Allocator { - return Allocator.init(self, alloc, resize); + return Allocator.init(self, alloc, resize, free); } fn alloc( @@ -29,7 +29,7 @@ pub fn LogToWriterAllocator(comptime Writer: type) type { ra: usize, ) error{OutOfMemory}![]u8 { self.writer.print("alloc : {}", .{len}) catch {}; - const result = self.parent_allocator.vtable.alloc(self.parent_allocator.ptr, len, ptr_align, len_align, ra); + const result = self.parent_allocator.rawAlloc(len, ptr_align, len_align, ra); if (result) |_| { self.writer.print(" success!\n", .{}) catch {}; } else |_| { @@ -46,14 +46,12 @@ pub fn LogToWriterAllocator(comptime Writer: type) type { len_align: u29, ra: usize, ) error{OutOfMemory}!usize { - if (new_len == 0) { - self.writer.print("free : {}\n", .{buf.len}) catch {}; - } else if (new_len <= buf.len) { + if (new_len <= buf.len) { self.writer.print("shrink: {} to {}\n", .{ buf.len, new_len }) catch {}; } else { self.writer.print("expand: {} to {}", .{ buf.len, new_len }) catch {}; } - if (self.parent_allocator.vtable.resize(self.parent_allocator.ptr, buf, buf_align, new_len, len_align, ra)) |resized_len| { + if (self.parent_allocator.rawResize(buf, buf_align, new_len, len_align, ra)) |resized_len| { if (new_len > buf.len) { self.writer.print(" success!\n", .{}) catch {}; } @@ -64,6 +62,16 @@ pub fn LogToWriterAllocator(comptime Writer: type) type { return e; } } + + fn free( + self: *Self, + buf: []u8, + buf_align: u29, + ra: usize, + ) void { + self.writer.print("free : {}\n", .{buf.len}) catch {}; + self.parent_allocator.rawFree(buf, buf_align, ra); + } }; } diff --git a/lib/std/heap/logging_allocator.zig b/lib/std/heap/logging_allocator.zig index 21c3546b11..b631cd0b1b 100644 --- a/lib/std/heap/logging_allocator.zig +++ b/lib/std/heap/logging_allocator.zig @@ -33,7 +33,7 @@ pub fn ScopedLoggingAllocator( } pub fn allocator(self: *Self) Allocator { - return Allocator.init(self, alloc, resize); + return Allocator.init(self, alloc, resize, free); } // This function is required as the `std.log.log` function is not public @@ -53,7 +53,7 @@ pub fn ScopedLoggingAllocator( len_align: u29, ra: usize, ) error{OutOfMemory}![]u8 { - const result = self.parent_allocator.vtable.alloc(self.parent_allocator.ptr, len, ptr_align, len_align, ra); + const result = self.parent_allocator.rawAlloc(len, ptr_align, len_align, ra); if (result) |_| { logHelper( success_log_level, @@ -78,10 +78,8 @@ pub fn ScopedLoggingAllocator( len_align: u29, ra: usize, ) error{OutOfMemory}!usize { - if (self.parent_allocator.vtable.resize(self.parent_allocator.ptr, buf, buf_align, new_len, len_align, ra)) |resized_len| { - if (new_len == 0) { - logHelper(success_log_level, "free - success - len: {}", .{buf.len}); - } else if (new_len <= buf.len) { + if (self.parent_allocator.rawResize(buf, buf_align, new_len, len_align, ra)) |resized_len| { + if (new_len <= buf.len) { logHelper( success_log_level, "shrink - success - {} to {}, len_align: {}, buf_align: {}", @@ -106,6 +104,16 @@ pub fn ScopedLoggingAllocator( return err; } } + + fn free( + self: *Self, + buf: []u8, + buf_align: u29, + ra: usize, + ) void { + self.parent_allocator.rawFree(buf, buf_align, ra); + logHelper(success_log_level, "free - len: {}", .{buf.len}); + } }; } diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 0594873514..9c7ce3867b 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -47,7 +47,7 @@ pub fn ValidationAllocator(comptime T: type) type { } pub fn allocator(self: *Self) Allocator { - return Allocator.init(self, alloc, resize); + return Allocator.init(self, alloc, resize, free); } fn getUnderlyingAllocatorPtr(self: *Self) Allocator { @@ -70,7 +70,7 @@ pub fn ValidationAllocator(comptime T: type) type { } const underlying = self.getUnderlyingAllocatorPtr(); - const result = try underlying.vtable.alloc(underlying.ptr, n, ptr_align, len_align, ret_addr); + const result = try underlying.rawAlloc(n, ptr_align, len_align, ret_addr); assert(mem.isAligned(@ptrToInt(result.ptr), ptr_align)); if (len_align == 0) { assert(result.len == n); @@ -95,7 +95,7 @@ pub fn ValidationAllocator(comptime T: type) type { assert(new_len >= len_align); } const underlying = self.getUnderlyingAllocatorPtr(); - const result = try underlying.vtable.resize(underlying.ptr, buf, buf_align, new_len, len_align, ret_addr); + const result = try underlying.rawResize(buf, buf_align, new_len, len_align, ret_addr); if (len_align == 0) { assert(result == new_len); } else { @@ -104,6 +104,19 @@ pub fn ValidationAllocator(comptime T: type) type { } return result; } + + pub fn free( + self: *Self, + buf: []u8, + buf_align: u29, + ret_addr: usize, + ) void { + _ = self; + _ = buf_align; + _ = ret_addr; + assert(buf.len > 0); + } + pub usingnamespace if (T == Allocator or !@hasDecl(T, "reset")) struct {} else struct { pub fn reset(self: *Self) void { self.underlying_allocator.reset(); @@ -139,6 +152,7 @@ const fail_allocator = Allocator{ const failAllocator_vtable = Allocator.VTable{ .alloc = failAllocatorAlloc, .resize = Allocator.NoResize(c_void).noResize, + .free = Allocator.NoOpFree(c_void).noOpFree, }; fn failAllocatorAlloc(_: *c_void, n: usize, alignment: u29, len_align: u29, ra: usize) Allocator.Error![]u8 { diff --git a/lib/std/mem/Allocator.zig b/lib/std/mem/Allocator.zig index df3974f795..6edad7e05b 100644 --- a/lib/std/mem/Allocator.zig +++ b/lib/std/mem/Allocator.zig @@ -5,6 +5,7 @@ const assert = std.debug.assert; const math = std.math; const mem = std.mem; const Allocator = @This(); +const builtin = @import("builtin"); pub const Error = error{OutOfMemory}; @@ -28,9 +29,6 @@ pub const VTable = struct { /// length returned by `alloc` or `resize`. `buf_align` must equal the same value /// that was passed as the `ptr_align` parameter to the original `alloc` call. /// - /// Passing a `new_len` of 0 frees and invalidates the buffer such that it can no - /// longer be passed to `resize`. - /// /// error.OutOfMemory can only be returned if `new_len` is greater than `buf.len`. /// If `buf` cannot be expanded to accomodate `new_len`, then the allocation MUST be /// unmodified and error.OutOfMemory MUST be returned. @@ -40,36 +38,54 @@ pub const VTable = struct { /// provide a way to modify the alignment of a pointer. Rather it provides an API for /// accepting more bytes of memory from the allocator than requested. /// - /// `new_len` must be greater than or equal to `len_align` and must be aligned by `len_align`. + /// `new_len` must be greater than zero, greater than or equal to `len_align` and must be aligned by `len_align`. /// /// `ret_addr` is optionally provided as the first return address of the allocation call stack. /// If the value is `0` it means no return address has been provided. resize: fn (ptr: *c_void, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize, + + /// Free and invalidate a buffer. `buf.len` must equal the most recent length returned by `alloc` or `resize`. + /// `buf_align` must equal the same value that was passed as the `ptr_align` parameter to the original `alloc` call. + /// + /// `ret_addr` is optionally provided as the first return address of the allocation call stack. + /// If the value is `0` it means no return address has been provided. + free: fn (ptr: *c_void, buf: []u8, buf_align: u29, ret_addr: usize) void, }; pub fn init( pointer: anytype, comptime allocFn: fn (ptr: @TypeOf(pointer), len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8, comptime resizeFn: fn (ptr: @TypeOf(pointer), buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize, + comptime freeFn: fn (ptr: @TypeOf(pointer), buf: []u8, buf_align: u29, ret_addr: usize) void, ) Allocator { const Ptr = @TypeOf(pointer); - assert(@typeInfo(Ptr) == .Pointer); // Must be a pointer - assert(@typeInfo(Ptr).Pointer.size == .One); // Must be a single-item pointer + const ptr_info = @typeInfo(Ptr); + + assert(ptr_info == .Pointer); // Must be a pointer + assert(ptr_info.Pointer.size == .One); // Must be a single-item pointer + + const alignment = ptr_info.Pointer.alignment; + const gen = struct { fn alloc(ptr: *c_void, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8 { - const alignment = @typeInfo(Ptr).Pointer.alignment; const self = @ptrCast(Ptr, @alignCast(alignment, ptr)); - return allocFn(self, len, ptr_align, len_align, ret_addr); + return @call(.{ .modifier = .always_inline }, allocFn, .{ self, len, ptr_align, len_align, ret_addr }); } fn resize(ptr: *c_void, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize { - const alignment = @typeInfo(Ptr).Pointer.alignment; + assert(new_len != 0); const self = @ptrCast(Ptr, @alignCast(alignment, ptr)); - return resizeFn(self, buf, buf_align, new_len, len_align, ret_addr); + return @call(.{ .modifier = .always_inline }, resizeFn, .{ self, buf, buf_align, new_len, len_align, ret_addr }); + } + fn free(ptr: *c_void, buf: []u8, buf_align: u29, ret_addr: usize) void { + const self = @ptrCast(Ptr, @alignCast(alignment, ptr)); + @call(.{ .modifier = .always_inline }, freeFn, .{ self, buf, buf_align, ret_addr }); } }; + const vtable = VTable{ .alloc = gen.alloc, .resize = gen.resize, + .free = gen.free, }; return .{ @@ -100,6 +116,56 @@ pub fn NoResize(comptime AllocatorType: type) type { }; } +/// Set freeFn to `NoOpFree(AllocatorType).noOpFree` if free is a no-op. +pub fn NoOpFree(comptime AllocatorType: type) type { + return struct { + pub fn noOpFree( + self: *AllocatorType, + buf: []u8, + buf_align: u29, + ret_addr: usize, + ) void { + _ = self; + _ = buf; + _ = buf_align; + _ = ret_addr; + } + }; +} + +/// Set freeFn to `PanicFree(AllocatorType).noOpFree` if free is not a supported operation. +pub fn PanicFree(comptime AllocatorType: type) type { + return struct { + pub fn noOpFree( + self: *AllocatorType, + buf: []u8, + buf_align: u29, + ret_addr: usize, + ) void { + _ = self; + _ = buf; + _ = buf_align; + _ = ret_addr; + @panic("free is not a supported operation for the allocator: " ++ @typeName(AllocatorType)); + } + }; +} + +/// This function is not intended to be called except from within the implementation of an Allocator +pub inline fn rawAlloc(self: Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8 { + return self.vtable.alloc(self.ptr, len, ptr_align, len_align, ret_addr); +} + +/// This function is not intended to be called except from within the implementation of an Allocator +pub inline fn rawResize(self: Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize { + return self.vtable.resize(self.ptr, buf, buf_align, new_len, len_align, ret_addr); +} + +/// This function is not intended to be called except from within the implementation of an Allocator +pub inline fn rawFree(self: Allocator, buf: []u8, buf_align: u29, ret_addr: usize) void { + return self.vtable.free(self.ptr, buf, buf_align, ret_addr); +} + /// Realloc is used to modify the size or alignment of an existing allocation, /// as well as to provide the allocator with an opportunity to move an allocation /// to a better location. @@ -133,8 +199,7 @@ fn reallocBytes( /// Guaranteed to be >= 1. /// Guaranteed to be a power of 2. old_alignment: u29, - /// If `new_byte_count` is 0 then this is a free and it is guaranteed that - /// `old_mem.len != 0`. + /// `new_byte_count` must be greater than zero new_byte_count: usize, /// Guaranteed to be >= 1. /// Guaranteed to be a power of 2. @@ -147,18 +212,20 @@ fn reallocBytes( return_address: usize, ) Error![]u8 { if (old_mem.len == 0) { - const new_mem = try self.vtable.alloc(self.ptr, new_byte_count, new_alignment, len_align, return_address); + const new_mem = try self.rawAlloc(new_byte_count, new_alignment, len_align, return_address); // TODO: https://github.com/ziglang/zig/issues/4298 @memset(new_mem.ptr, undefined, new_byte_count); return new_mem; } + assert(new_byte_count > 0); // `new_byte_count` must greater than zero, this is a resize not a free + if (mem.isAligned(@ptrToInt(old_mem.ptr), new_alignment)) { if (new_byte_count <= old_mem.len) { const shrunk_len = self.shrinkBytes(old_mem, old_alignment, new_byte_count, len_align, return_address); return old_mem.ptr[0..shrunk_len]; } - if (self.vtable.resize(self.ptr, old_mem, old_alignment, new_byte_count, len_align, return_address)) |resized_len| { + if (self.rawResize(old_mem, old_alignment, new_byte_count, len_align, return_address)) |resized_len| { assert(resized_len >= new_byte_count); // TODO: https://github.com/ziglang/zig/issues/4298 @memset(old_mem.ptr + new_byte_count, undefined, resized_len - new_byte_count); @@ -184,11 +251,11 @@ fn moveBytes( ) Error![]u8 { assert(old_mem.len > 0); assert(new_len > 0); - const new_mem = try self.vtable.alloc(self.ptr, new_len, new_alignment, len_align, return_address); + const new_mem = try self.rawAlloc(new_len, new_alignment, len_align, return_address); @memcpy(new_mem.ptr, old_mem.ptr, math.min(new_len, old_mem.len)); // TODO https://github.com/ziglang/zig/issues/4298 @memset(old_mem.ptr, undefined, old_mem.len); - _ = self.shrinkBytes(old_mem, old_align, 0, 0, return_address); + self.rawFree(old_mem, old_align, return_address); return new_mem; } @@ -207,7 +274,7 @@ pub fn destroy(self: Allocator, ptr: anytype) void { const T = info.child; if (@sizeOf(T) == 0) return; const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr)); - _ = self.shrinkBytes(non_const_ptr[0..@sizeOf(T)], info.alignment, 0, 0, @returnAddress()); + self.rawFree(non_const_ptr[0..@sizeOf(T)], info.alignment, @returnAddress()); } /// Allocates an array of `n` items of type `T` and sets all the @@ -326,7 +393,7 @@ pub fn allocAdvancedWithRetAddr( .exact => 0, .at_least => size_of_T, }; - const byte_slice = try self.vtable.alloc(self.ptr, byte_count, a, len_align, return_address); + const byte_slice = try self.rawAlloc(byte_count, a, len_align, return_address); switch (exact) { .exact => assert(byte_slice.len == byte_count), .at_least => assert(byte_slice.len >= byte_count), @@ -351,7 +418,7 @@ pub fn resize(self: Allocator, old_mem: anytype, new_n: usize) Error!@TypeOf(old } const old_byte_slice = mem.sliceAsBytes(old_mem); const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory; - const rc = try self.vtable.resize(self.ptr, old_byte_slice, Slice.alignment, new_byte_count, 0, @returnAddress()); + const rc = try self.rawResize(old_byte_slice, Slice.alignment, new_byte_count, 0, @returnAddress()); assert(rc == new_byte_count); const new_byte_slice = old_byte_slice.ptr[0..new_byte_count]; return mem.bytesAsSlice(T, new_byte_slice); @@ -465,6 +532,11 @@ pub fn alignedShrinkWithRetAddr( if (new_n == old_mem.len) return old_mem; + if (new_n == 0) { + self.free(old_mem); + return @as([*]align(new_alignment) T, undefined)[0..0]; + } + assert(new_n < old_mem.len); assert(new_alignment <= Slice.alignment); @@ -489,7 +561,7 @@ pub fn free(self: Allocator, memory: anytype) void { const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr)); // TODO: https://github.com/ziglang/zig/issues/4298 @memset(non_const_ptr, undefined, bytes_len); - _ = self.shrinkBytes(non_const_ptr[0..bytes_len], Slice.alignment, 0, 0, @returnAddress()); + self.rawFree(non_const_ptr[0..bytes_len], Slice.alignment, @returnAddress()); } /// Copies `m` to newly allocated memory. Caller owns the memory. @@ -520,5 +592,5 @@ pub fn shrinkBytes( return_address: usize, ) usize { assert(new_len <= buf.len); - return self.vtable.resize(self.ptr, buf, buf_align, new_len, len_align, return_address) catch unreachable; + return self.rawResize(buf, buf_align, new_len, len_align, return_address) catch unreachable; } diff --git a/lib/std/testing/failing_allocator.zig b/lib/std/testing/failing_allocator.zig index c26163a6f3..5e1084037e 100644 --- a/lib/std/testing/failing_allocator.zig +++ b/lib/std/testing/failing_allocator.zig @@ -41,7 +41,7 @@ pub const FailingAllocator = struct { } pub fn allocator(self: *FailingAllocator) mem.Allocator { - return mem.Allocator.init(self, alloc, resize); + return mem.Allocator.init(self, alloc, resize, free); } fn alloc( @@ -54,7 +54,7 @@ pub const FailingAllocator = struct { if (self.index == self.fail_index) { return error.OutOfMemory; } - const result = try self.internal_allocator.vtable.alloc(self.internal_allocator.ptr, len, ptr_align, len_align, return_address); + const result = try self.internal_allocator.rawAlloc(len, ptr_align, len_align, return_address); self.allocated_bytes += result.len; self.allocations += 1; self.index += 1; @@ -69,18 +69,26 @@ pub const FailingAllocator = struct { len_align: u29, ra: usize, ) error{OutOfMemory}!usize { - const r = self.internal_allocator.vtable.resize(self.internal_allocator.ptr, old_mem, old_align, new_len, len_align, ra) catch |e| { + const r = self.internal_allocator.rawResize(old_mem, old_align, new_len, len_align, ra) catch |e| { std.debug.assert(new_len > old_mem.len); return e; }; - if (new_len == 0) { - self.deallocations += 1; - self.freed_bytes += old_mem.len; - } else if (r < old_mem.len) { + if (r < old_mem.len) { self.freed_bytes += old_mem.len - r; } else { self.allocated_bytes += r - old_mem.len; } return r; } + + fn free( + self: *FailingAllocator, + old_mem: []u8, + old_align: u29, + ra: usize, + ) void { + self.internal_allocator.rawFree(old_mem, old_align, ra); + self.deallocations += 1; + self.freed_bytes += old_mem.len; + } }; diff --git a/src/tracy.zig b/src/tracy.zig index 83e31e5764..064374030f 100644 --- a/src/tracy.zig +++ b/src/tracy.zig @@ -155,13 +155,10 @@ pub fn TracyAllocator(comptime name: ?[:0]const u8) type { } } - if (resized_len != 0) { - // this was a shrink or a resize - if (name) |n| { - allocNamed(buf.ptr, resized_len, n); - } else { - alloc(buf.ptr, resized_len); - } + if (name) |n| { + allocNamed(buf.ptr, resized_len, n); + } else { + alloc(buf.ptr, resized_len); } return resized_len; @@ -172,6 +169,15 @@ pub fn TracyAllocator(comptime name: ?[:0]const u8) type { return err; } } + + fn freeFn(self: *Self, buf: []u8, buf_align: u29, ret_addr: usize) void { + self.parent_allocator.rawFree(buf, buf_align, ret_addr); + if (name) |n| { + freeNamed(buf.ptr, n); + } else { + free(buf.ptr); + } + } }; }