From 284de7d957037c8a7032bd6e2a95bd5f55b73666 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 29 Jan 2025 14:16:25 -0800 Subject: [PATCH] adjust runtime page size APIs * fix merge conflicts * rename the declarations * reword documentation * extract FixedBufferAllocator to separate file * take advantage of locals * remove the assertion about max alignment in Allocator API, leaving it Allocator implementation defined * fix non-inline function call in start logic The GeneralPurposeAllocator implementation is totally broken because it uses global state but I didn't address that in this commit. --- lib/fuzzer.zig | 2 +- lib/std/Build/Fuzz/WebServer.zig | 2 +- lib/std/Thread.zig | 2 +- lib/std/c.zig | 20 +- lib/std/crypto/tlcsprng.zig | 11 +- lib/std/debug.zig | 15 +- lib/std/debug/Dwarf.zig | 10 +- lib/std/debug/MemoryAccessor.zig | 4 +- lib/std/debug/SelfInfo.zig | 6 +- lib/std/dynamic_library.zig | 13 +- lib/std/heap.zig | 993 ++++++++------------- lib/std/heap/FixedBufferAllocator.zig | 218 +++++ lib/std/heap/PageAllocator.zig | 25 +- lib/std/heap/general_purpose_allocator.zig | 16 +- lib/std/mem.zig | 22 +- lib/std/mem/Allocator.zig | 11 +- lib/std/os/linux/IoUring.zig | 16 +- lib/std/os/linux/tls.zig | 20 +- lib/std/posix.zig | 20 +- lib/std/process.zig | 2 +- lib/std/start.zig | 2 +- lib/std/std.zig | 9 +- src/Package/Fetch.zig | 2 +- 23 files changed, 723 insertions(+), 718 deletions(-) create mode 100644 lib/std/heap/FixedBufferAllocator.zig diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index eb37e5e2fd..026b7cbc15 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -480,7 +480,7 @@ pub const MemoryMappedList = struct { /// of this ArrayList in accordance with the respective documentation. In /// all cases, "invalidated" means that the memory has been passed to this /// allocator's resize or free function. - items: []align(std.heap.min_page_size) volatile u8, + items: []align(std.heap.page_size_min) volatile u8, /// How many bytes this list can hold without allocating additional memory. capacity: usize, diff --git a/lib/std/Build/Fuzz/WebServer.zig b/lib/std/Build/Fuzz/WebServer.zig index 0563d6782a..87cd7a1a1d 100644 --- a/lib/std/Build/Fuzz/WebServer.zig +++ b/lib/std/Build/Fuzz/WebServer.zig @@ -41,7 +41,7 @@ const fuzzer_arch_os_abi = "wasm32-freestanding"; const fuzzer_cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext"; const CoverageMap = struct { - mapped_memory: []align(std.heap.min_page_size) const u8, + mapped_memory: []align(std.heap.page_size_min) const u8, coverage: Coverage, source_locations: []Coverage.SourceLocation, /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested. diff --git a/lib/std/Thread.zig b/lib/std/Thread.zig index 3be80c1641..6dcb956184 100644 --- a/lib/std/Thread.zig +++ b/lib/std/Thread.zig @@ -1155,7 +1155,7 @@ const LinuxThreadImpl = struct { completion: Completion = Completion.init(.running), child_tid: std.atomic.Value(i32) = std.atomic.Value(i32).init(1), parent_tid: i32 = undefined, - mapped: []align(std.heap.min_page_size) u8, + mapped: []align(std.heap.page_size_min) u8, /// Calls `munmap(mapped.ptr, mapped.len)` then `exit(1)` without touching the stack (which lives in `mapped.ptr`). /// Ported over from musl libc's pthread detached implementation: diff --git a/lib/std/c.zig b/lib/std/c.zig index df03f81673..6e1e9beb9f 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -3,7 +3,7 @@ const builtin = @import("builtin"); const c = @This(); const maxInt = std.math.maxInt; const assert = std.debug.assert; -const min_page_size = std.heap.min_page_size; +const page_size = std.heap.page_size_min; const native_abi = builtin.abi; const native_arch = builtin.cpu.arch; const native_os = builtin.os.tag; @@ -2229,7 +2229,7 @@ pub const SC = switch (native_os) { }; pub const _SC = switch (native_os) { - .bridgeos, .driverkit, .ios, .macos, .tvos, .visionos, .watchos => enum(c_int) { + .driverkit, .ios, .macos, .tvos, .visionos, .watchos => enum(c_int) { PAGESIZE = 29, }, .dragonfly => enum(c_int) { @@ -9265,7 +9265,7 @@ pub extern "c" fn getpwnam(name: [*:0]const u8) ?*passwd; pub extern "c" fn getpwuid(uid: uid_t) ?*passwd; pub extern "c" fn getrlimit64(resource: rlimit_resource, rlim: *rlimit) c_int; pub extern "c" fn lseek64(fd: fd_t, offset: i64, whence: c_int) i64; -pub extern "c" fn mmap64(addr: ?*align(min_page_size) anyopaque, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: i64) *anyopaque; +pub extern "c" fn mmap64(addr: ?*align(page_size) anyopaque, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: i64) *anyopaque; pub extern "c" fn open64(path: [*:0]const u8, oflag: O, ...) c_int; pub extern "c" fn openat64(fd: c_int, path: [*:0]const u8, oflag: O, ...) c_int; pub extern "c" fn pread64(fd: fd_t, buf: [*]u8, nbyte: usize, offset: i64) isize; @@ -9357,13 +9357,13 @@ pub extern "c" fn signalfd(fd: fd_t, mask: *const sigset_t, flags: u32) c_int; pub extern "c" fn prlimit(pid: pid_t, resource: rlimit_resource, new_limit: *const rlimit, old_limit: *rlimit) c_int; pub extern "c" fn mincore( - addr: *align(min_page_size) anyopaque, + addr: *align(page_size) anyopaque, length: usize, vec: [*]u8, ) c_int; pub extern "c" fn madvise( - addr: *align(min_page_size) anyopaque, + addr: *align(page_size) anyopaque, length: usize, advice: u32, ) c_int; @@ -9506,9 +9506,9 @@ pub extern "c" fn writev(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint) i pub extern "c" fn pwritev(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint, offset: off_t) isize; pub extern "c" fn write(fd: fd_t, buf: [*]const u8, nbyte: usize) isize; pub extern "c" fn pwrite(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: off_t) isize; -pub extern "c" fn mmap(addr: ?*align(min_page_size) anyopaque, len: usize, prot: c_uint, flags: MAP, fd: fd_t, offset: off_t) *anyopaque; -pub extern "c" fn munmap(addr: *align(min_page_size) const anyopaque, len: usize) c_int; -pub extern "c" fn mprotect(addr: *align(min_page_size) anyopaque, len: usize, prot: c_uint) c_int; +pub extern "c" fn mmap(addr: ?*align(page_size) anyopaque, len: usize, prot: c_uint, flags: MAP, fd: fd_t, offset: off_t) *anyopaque; +pub extern "c" fn munmap(addr: *align(page_size) const anyopaque, len: usize) c_int; +pub extern "c" fn mprotect(addr: *align(page_size) anyopaque, len: usize, prot: c_uint) c_int; pub extern "c" fn link(oldpath: [*:0]const u8, newpath: [*:0]const u8) c_int; pub extern "c" fn linkat(oldfd: fd_t, oldpath: [*:0]const u8, newfd: fd_t, newpath: [*:0]const u8, flags: c_int) c_int; pub extern "c" fn unlink(path: [*:0]const u8) c_int; @@ -10191,7 +10191,7 @@ const private = struct { }; extern "c" fn getrusage(who: c_int, usage: *rusage) c_int; extern "c" fn gettimeofday(noalias tv: ?*timeval, noalias tz: ?*timezone) c_int; - extern "c" fn msync(addr: *align(min_page_size) const anyopaque, len: usize, flags: c_int) c_int; + extern "c" fn msync(addr: *align(page_size) const anyopaque, len: usize, flags: c_int) c_int; extern "c" fn nanosleep(rqtp: *const timespec, rmtp: ?*timespec) c_int; extern "c" fn pipe2(fds: *[2]fd_t, flags: O) c_int; extern "c" fn readdir(dir: *DIR) ?*dirent; @@ -10239,7 +10239,7 @@ const private = struct { extern "c" fn __getrusage50(who: c_int, usage: *rusage) c_int; extern "c" fn __gettimeofday50(noalias tv: ?*timeval, noalias tz: ?*timezone) c_int; extern "c" fn __libc_thr_yield() c_int; - extern "c" fn __msync13(addr: *align(min_page_size) const anyopaque, len: usize, flags: c_int) c_int; + extern "c" fn __msync13(addr: *align(page_size) const anyopaque, len: usize, flags: c_int) c_int; extern "c" fn __nanosleep50(rqtp: *const timespec, rmtp: ?*timespec) c_int; extern "c" fn __sigaction14(sig: c_int, noalias act: ?*const Sigaction, noalias oact: ?*Sigaction) c_int; extern "c" fn __sigfillset14(set: ?*sigset_t) void; diff --git a/lib/std/crypto/tlcsprng.zig b/lib/std/crypto/tlcsprng.zig index dbe56a390f..a1d9beb9b5 100644 --- a/lib/std/crypto/tlcsprng.zig +++ b/lib/std/crypto/tlcsprng.zig @@ -6,7 +6,6 @@ const std = @import("std"); const builtin = @import("builtin"); const mem = std.mem; -const heap = std.heap; const native_os = builtin.os.tag; const posix = std.posix; @@ -43,7 +42,7 @@ var install_atfork_handler = std.once(struct { } }.do); -threadlocal var wipe_mem: []align(heap.min_page_size) u8 = &[_]u8{}; +threadlocal var wipe_mem: []align(std.heap.page_size_min) u8 = &[_]u8{}; fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void { if (os_has_arc4random) { @@ -78,7 +77,7 @@ fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void { } else { // Use a static thread-local buffer. const S = struct { - threadlocal var buf: Context align(heap.min_page_size) = .{ + threadlocal var buf: Context align(std.heap.page_size_min) = .{ .init_state = .uninitialized, .rng = undefined, }; @@ -86,7 +85,7 @@ fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void { wipe_mem = mem.asBytes(&S.buf); } } - const ctx = @as(*Context, @ptrCast(wipe_mem.ptr)); + const ctx: *Context = @ptrCast(wipe_mem.ptr); switch (ctx.init_state) { .uninitialized => { @@ -142,7 +141,7 @@ fn childAtForkHandler() callconv(.c) void { } fn fillWithCsprng(buffer: []u8) void { - const ctx = @as(*Context, @ptrCast(wipe_mem.ptr)); + const ctx: *Context = @ptrCast(wipe_mem.ptr); return ctx.rng.fill(buffer); } @@ -158,7 +157,7 @@ fn initAndFill(buffer: []u8) void { // the `std.options.cryptoRandomSeed` function is provided. std.options.cryptoRandomSeed(&seed); - const ctx = @as(*Context, @ptrCast(wipe_mem.ptr)); + const ctx: *Context = @ptrCast(wipe_mem.ptr); ctx.rng = Rng.init(seed); std.crypto.secureZero(u8, &seed); diff --git a/lib/std/debug.zig b/lib/std/debug.zig index a3aacf769e..9deca6de49 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -2,7 +2,6 @@ const builtin = @import("builtin"); const std = @import("std.zig"); const math = std.math; const mem = std.mem; -const heap = std.heap; const io = std.io; const posix = std.posix; const fs = std.fs; @@ -1238,7 +1237,7 @@ test printLineFromFileAnyOs { const overlap = 10; var writer = file.writer(); - try writer.writeByteNTimes('a', heap.min_page_size - overlap); + try writer.writeByteNTimes('a', std.heap.page_size_min - overlap); try writer.writeByte('\n'); try writer.writeByteNTimes('a', overlap); @@ -1253,10 +1252,10 @@ test printLineFromFileAnyOs { defer allocator.free(path); var writer = file.writer(); - try writer.writeByteNTimes('a', heap.max_page_size); + try writer.writeByteNTimes('a', std.heap.page_size_max); try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 }); - try expectEqualStrings(("a" ** heap.max_page_size) ++ "\n", output.items); + try expectEqualStrings(("a" ** std.heap.page_size_max) ++ "\n", output.items); output.clearRetainingCapacity(); } { @@ -1266,18 +1265,18 @@ test printLineFromFileAnyOs { defer allocator.free(path); var writer = file.writer(); - try writer.writeByteNTimes('a', 3 * heap.max_page_size); + try writer.writeByteNTimes('a', 3 * std.heap.page_size_max); try expectError(error.EndOfFile, printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 2, .column = 0 })); try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 }); - try expectEqualStrings(("a" ** (3 * heap.max_page_size)) ++ "\n", output.items); + try expectEqualStrings(("a" ** (3 * std.heap.page_size_max)) ++ "\n", output.items); output.clearRetainingCapacity(); try writer.writeAll("a\na"); try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 }); - try expectEqualStrings(("a" ** (3 * heap.max_page_size)) ++ "a\n", output.items); + try expectEqualStrings(("a" ** (3 * std.heap.page_size_max)) ++ "a\n", output.items); output.clearRetainingCapacity(); try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 2, .column = 0 }); @@ -1291,7 +1290,7 @@ test printLineFromFileAnyOs { defer allocator.free(path); var writer = file.writer(); - const real_file_start = 3 * heap.min_page_size; + const real_file_start = 3 * std.heap.page_size_min; try writer.writeByteNTimes('\n', real_file_start); try writer.writeAll("abc\ndef"); diff --git a/lib/std/debug/Dwarf.zig b/lib/std/debug/Dwarf.zig index d4cd674898..b72ddcac47 100644 --- a/lib/std/debug/Dwarf.zig +++ b/lib/std/debug/Dwarf.zig @@ -2120,8 +2120,8 @@ fn pcRelBase(field_ptr: usize, pc_rel_offset: i64) !usize { pub const ElfModule = struct { base_address: usize, dwarf: Dwarf, - mapped_memory: []align(std.heap.min_page_size) const u8, - external_mapped_memory: ?[]align(std.heap.min_page_size) const u8, + mapped_memory: []align(std.heap.page_size_min) const u8, + external_mapped_memory: ?[]align(std.heap.page_size_min) const u8, pub fn deinit(self: *@This(), allocator: Allocator) void { self.dwarf.deinit(allocator); @@ -2167,11 +2167,11 @@ pub const ElfModule = struct { /// sections from an external file. pub fn load( gpa: Allocator, - mapped_mem: []align(std.heap.min_page_size) const u8, + mapped_mem: []align(std.heap.page_size_min) const u8, build_id: ?[]const u8, expected_crc: ?u32, parent_sections: *Dwarf.SectionArray, - parent_mapped_mem: ?[]align(std.heap.min_page_size) const u8, + parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8, elf_filename: ?[]const u8, ) LoadError!Dwarf.ElfModule { if (expected_crc) |crc| if (crc != std.hash.crc.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo; @@ -2423,7 +2423,7 @@ pub const ElfModule = struct { build_id: ?[]const u8, expected_crc: ?u32, parent_sections: *Dwarf.SectionArray, - parent_mapped_mem: ?[]align(std.heap.min_page_size) const u8, + parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8, ) LoadError!Dwarf.ElfModule { const elf_file = elf_file_path.root_dir.handle.openFile(elf_file_path.sub_path, .{}) catch |err| switch (err) { error.FileNotFound => return missing(), diff --git a/lib/std/debug/MemoryAccessor.zig b/lib/std/debug/MemoryAccessor.zig index 8c9c93b52e..7857656554 100644 --- a/lib/std/debug/MemoryAccessor.zig +++ b/lib/std/debug/MemoryAccessor.zig @@ -7,7 +7,7 @@ const native_os = builtin.os.tag; const std = @import("../std.zig"); const posix = std.posix; const File = std.fs.File; -const min_page_size = std.heap.min_page_size; +const page_size_min = std.heap.page_size_min; const MemoryAccessor = @This(); @@ -96,7 +96,7 @@ pub fn isValidMemory(address: usize) bool { const page_size = std.heap.pageSize(); const aligned_address = address & ~(page_size - 1); if (aligned_address == 0) return false; - const aligned_memory = @as([*]align(min_page_size) u8, @ptrFromInt(aligned_address))[0..page_size]; + const aligned_memory = @as([*]align(page_size_min) u8, @ptrFromInt(aligned_address))[0..page_size]; if (native_os == .windows) { const windows = std.os.windows; diff --git a/lib/std/debug/SelfInfo.zig b/lib/std/debug/SelfInfo.zig index ee676ecf3e..b51a8f18d2 100644 --- a/lib/std/debug/SelfInfo.zig +++ b/lib/std/debug/SelfInfo.zig @@ -504,7 +504,7 @@ pub const Module = switch (native_os) { .macos, .ios, .watchos, .tvos, .visionos => struct { base_address: usize, vmaddr_slide: usize, - mapped_memory: []align(std.heap.min_page_size) const u8, + mapped_memory: []align(std.heap.page_size_min) const u8, symbols: []const MachoSymbol, strings: [:0]const u8, ofiles: OFileTable, @@ -1046,7 +1046,7 @@ pub fn readElfDebugInfo( build_id: ?[]const u8, expected_crc: ?u32, parent_sections: *Dwarf.SectionArray, - parent_mapped_mem: ?[]align(std.heap.min_page_size) const u8, + parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8, ) !Dwarf.ElfModule { nosuspend { const elf_file = (if (elf_filename) |filename| blk: { @@ -1088,7 +1088,7 @@ const MachoSymbol = struct { /// Takes ownership of file, even on error. /// TODO it's weird to take ownership even on error, rework this code. -fn mapWholeFile(file: File) ![]align(std.heap.min_page_size) const u8 { +fn mapWholeFile(file: File) ![]align(std.heap.page_size_min) const u8 { nosuspend { defer file.close(); diff --git a/lib/std/dynamic_library.zig b/lib/std/dynamic_library.zig index 38511f7f29..b31fa5ea4d 100644 --- a/lib/std/dynamic_library.zig +++ b/lib/std/dynamic_library.zig @@ -1,7 +1,6 @@ const std = @import("std.zig"); const builtin = @import("builtin"); const mem = std.mem; -const heap = std.heap; const testing = std.testing; const elf = std.elf; const windows = std.os.windows; @@ -144,7 +143,7 @@ pub const ElfDynLib = struct { hashtab: [*]posix.Elf_Symndx, versym: ?[*]elf.Versym, verdef: ?*elf.Verdef, - memory: []align(heap.min_page_size) u8, + memory: []align(std.heap.page_size_min) u8, pub const Error = ElfDynLibError; @@ -220,11 +219,13 @@ pub const ElfDynLib = struct { const stat = try file.stat(); const size = std.math.cast(usize, stat.size) orelse return error.FileTooBig; + const page_size = std.heap.pageSize(); + // This one is to read the ELF info. We do more mmapping later // corresponding to the actual LOAD sections. const file_bytes = try posix.mmap( null, - mem.alignForward(usize, size, heap.pageSize()), + mem.alignForward(usize, size, page_size), posix.PROT.READ, .{ .TYPE = .PRIVATE }, fd, @@ -285,10 +286,10 @@ pub const ElfDynLib = struct { elf.PT_LOAD => { // The VirtAddr may not be page-aligned; in such case there will be // extra nonsense mapped before/after the VirtAddr,MemSiz - const aligned_addr = (base + ph.p_vaddr) & ~(@as(usize, heap.pageSize()) - 1); + const aligned_addr = (base + ph.p_vaddr) & ~(@as(usize, page_size) - 1); const extra_bytes = (base + ph.p_vaddr) - aligned_addr; - const extended_memsz = mem.alignForward(usize, ph.p_memsz + extra_bytes, heap.pageSize()); - const ptr = @as([*]align(heap.min_page_size) u8, @ptrFromInt(aligned_addr)); + const extended_memsz = mem.alignForward(usize, ph.p_memsz + extra_bytes, page_size); + const ptr = @as([*]align(std.heap.page_size_min) u8, @ptrFromInt(aligned_addr)); const prot = elfToMmapProt(ph.p_flags); if ((ph.p_flags & elf.PF_W) == 0) { // If it does not need write access, it can be mapped from the fd. diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 7351c6fa09..746fc74bd0 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -8,376 +8,6 @@ const c = std.c; const Allocator = std.mem.Allocator; const windows = std.os.windows; -const default_min_page_size: ?usize = switch (builtin.os.tag) { - .bridgeos, .driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) { - .x86_64 => 4 << 10, - .aarch64 => 16 << 10, - else => null, - }, - .windows => switch (builtin.cpu.arch) { - // -- - .x86, .x86_64 => 4 << 10, - // SuperH => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, - // DEC Alpha => 8 << 10, - // Itanium => 8 << 10, - .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, - else => null, - }, - .wasi => switch (builtin.cpu.arch) { - .wasm32, .wasm64 => 64 << 10, - else => null, - }, - // https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187 - .uefi => 4 << 10, - .freebsd => switch (builtin.cpu.arch) { - // FreeBSD/sys/* - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv32, .riscv64 => 4 << 10, - else => null, - }, - .netbsd => switch (builtin.cpu.arch) { - // NetBSD/sys/arch/* - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .sparc => 4 << 10, - .sparc64 => 8 << 10, - .riscv32, .riscv64 => 4 << 10, - // Sun-2 - .m68k => 2 << 10, - else => null, - }, - .dragonfly => switch (builtin.cpu.arch) { - .x86, .x86_64 => 4 << 10, - else => null, - }, - .openbsd => switch (builtin.cpu.arch) { - // OpenBSD/sys/arch/* - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, - .mips64, .mips64el => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv64 => 4 << 10, - .sparc64 => 8 << 10, - else => null, - }, - .solaris, .illumos => switch (builtin.cpu.arch) { - // src/uts/*/sys/machparam.h - .x86, .x86_64 => 4 << 10, - .sparc, .sparc64 => 8 << 10, - else => null, - }, - .fuchsia => switch (builtin.cpu.arch) { - // fuchsia/kernel/arch/*/include/arch/defines.h - .x86_64 => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .riscv64 => 4 << 10, - else => null, - }, - // https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13 - .serenity => 4 << 10, - .haiku => switch (builtin.cpu.arch) { - // haiku/headers/posix/arch/*/limits.h - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .m68k => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv64 => 4 << 10, - .sparc64 => 8 << 10, - .x86, .x86_64 => 4 << 10, - else => null, - }, - .hurd => switch (builtin.cpu.arch) { - // gnumach/*/include/mach/*/vm_param.h - .x86, .x86_64 => 4 << 10, - .aarch64 => null, - else => null, - }, - .plan9 => switch (builtin.cpu.arch) { - // 9front/sys/src/9/*/mem.h - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, - .sparc => 4 << 10, - else => null, - }, - .ps3 => switch (builtin.cpu.arch) { - // cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html - .powerpc64 => 1 << 20, // 1 MiB - else => null, - }, - .ps4 => switch (builtin.cpu.arch) { - // https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95 - .x86, .x86_64 => 4 << 10, - else => null, - }, - .ps5 => switch (builtin.cpu.arch) { - // https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95 - .x86, .x86_64 => 16 << 10, - else => null, - }, - // system/lib/libc/musl/arch/emscripten/bits/limits.h - .emscripten => 64 << 10, - .linux => switch (builtin.cpu.arch) { - // Linux/arch/*/Kconfig - .arc => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .csky => 4 << 10, - .hexagon => 4 << 10, - .loongarch32, .loongarch64 => 4 << 10, - .m68k => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv32, .riscv64 => 4 << 10, - .s390x => 4 << 10, - .sparc => 4 << 10, - .sparc64 => 8 << 10, - .x86, .x86_64 => 4 << 10, - .xtensa => 4 << 10, - else => null, - }, - .freestanding => switch (builtin.cpu.arch) { - .wasm32, .wasm64 => 64 << 10, - else => null, - }, - else => null, -}; - -const default_max_page_size: ?usize = switch (builtin.os.tag) { - .bridgeos, .driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) { - .x86_64 => 4 << 10, - .aarch64 => 16 << 10, - else => null, - }, - .windows => switch (builtin.cpu.arch) { - // -- - .x86, .x86_64 => 4 << 10, - // SuperH => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, - // DEC Alpha => 8 << 10, - // Itanium => 8 << 10, - .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, - else => null, - }, - .wasi => switch (builtin.cpu.arch) { - .wasm32, .wasm64 => 64 << 10, - else => null, - }, - // https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187 - .uefi => 4 << 10, - .freebsd => switch (builtin.cpu.arch) { - // FreeBSD/sys/* - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv32, .riscv64 => 4 << 10, - else => null, - }, - .netbsd => switch (builtin.cpu.arch) { - // NetBSD/sys/arch/* - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 64 << 10, - .mips, .mipsel, .mips64, .mips64el => 16 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 16 << 10, - .sparc => 8 << 10, - .sparc64 => 8 << 10, - .riscv32, .riscv64 => 4 << 10, - .m68k => 8 << 10, - else => null, - }, - .dragonfly => switch (builtin.cpu.arch) { - .x86, .x86_64 => 4 << 10, - else => null, - }, - .openbsd => switch (builtin.cpu.arch) { - // OpenBSD/sys/arch/* - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, - .mips64, .mips64el => 16 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv64 => 4 << 10, - .sparc64 => 8 << 10, - else => null, - }, - .solaris, .illumos => switch (builtin.cpu.arch) { - // src/uts/*/sys/machparam.h - .x86, .x86_64 => 4 << 10, - .sparc, .sparc64 => 8 << 10, - else => null, - }, - .fuchsia => switch (builtin.cpu.arch) { - // fuchsia/kernel/arch/*/include/arch/defines.h - .x86_64 => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .riscv64 => 4 << 10, - else => null, - }, - // https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13 - .serenity => 4 << 10, - .haiku => switch (builtin.cpu.arch) { - // haiku/headers/posix/arch/*/limits.h - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 4 << 10, - .m68k => 4 << 10, - .mips, .mipsel, .mips64, .mips64el => 4 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, - .riscv64 => 4 << 10, - .sparc64 => 8 << 10, - .x86, .x86_64 => 4 << 10, - else => null, - }, - .hurd => switch (builtin.cpu.arch) { - // gnumach/*/include/mach/*/vm_param.h - .x86, .x86_64 => 4 << 10, - .aarch64 => null, - else => null, - }, - .plan9 => switch (builtin.cpu.arch) { - // 9front/sys/src/9/*/mem.h - .x86, .x86_64 => 4 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 64 << 10, - .mips, .mipsel, .mips64, .mips64el => 16 << 10, - .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, - .sparc => 4 << 10, - else => null, - }, - .ps3 => switch (builtin.cpu.arch) { - // cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html - .powerpc64 => 1 << 20, // 1 MiB - else => null, - }, - .ps4 => switch (builtin.cpu.arch) { - // https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95 - .x86, .x86_64 => 4 << 10, - else => null, - }, - .ps5 => switch (builtin.cpu.arch) { - // https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95 - .x86, .x86_64 => 16 << 10, - else => null, - }, - // system/lib/libc/musl/arch/emscripten/bits/limits.h - .emscripten => 64 << 10, - .linux => switch (builtin.cpu.arch) { - // Linux/arch/*/Kconfig - .arc => 16 << 10, - .thumb, .thumbeb, .arm, .armeb => 4 << 10, - .aarch64, .aarch64_be => 64 << 10, - .csky => 4 << 10, - .hexagon => 256 << 10, - .loongarch32, .loongarch64 => 64 << 10, - .m68k => 8 << 10, - .mips, .mipsel, .mips64, .mips64el => 64 << 10, - .powerpc, .powerpc64, .powerpc64le, .powerpcle => 256 << 10, - .riscv32, .riscv64 => 4 << 10, - .s390x => 4 << 10, - .sparc => 4 << 10, - .sparc64 => 8 << 10, - .x86, .x86_64 => 4 << 10, - .xtensa => 4 << 10, - else => null, - }, - .freestanding => switch (builtin.cpu.arch) { - .wasm32, .wasm64 => 64 << 10, - else => null, - }, - else => null, -}; - -/// The compile-time minimum page size that the target might have. -/// All pointers from `mmap` or `VirtualAlloc` are aligned to at least `min_page_size`, but their -/// actual alignment may be much bigger. -/// This value can be overridden via `std.options.min_page_size`. -/// On many systems, the actual page size can only be determined at runtime with `pageSize()`. -pub const min_page_size: usize = std.options.min_page_size orelse (default_min_page_size orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other) - @compileError("freestanding/other explicitly has no min_page_size. One can be provided with std.options.min_page_size") -else - @compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has no min_page_size. One can be provided with std.options.min_page_size")); - -/// The compile-time maximum page size that the target might have. -/// Targeting a system with a larger page size may require overriding `std.options.max_page_size`, -/// as well as using the linker arugment `-z max-page-size=`. -/// The actual page size can only be determined at runtime with `pageSize()`. -pub const max_page_size: usize = std.options.max_page_size orelse (default_max_page_size orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other) - @compileError("freestanding/other explicitly has no max_page_size. One can be provided with std.options.max_page_size") -else - @compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has no max_page_size. One can be provided with std.options.max_page_size")); - -/// Returns the system page size. -/// If the page size is comptime-known, `pageSize()` returns it directly. -/// Otherwise, `pageSize()` defers to `std.options.queryPageSizeFn()`. -pub fn pageSize() usize { - if (min_page_size == max_page_size) { - return min_page_size; - } - return std.options.queryPageSizeFn(); -} - -// A cache used by `defaultQueryPageSize()` to avoid repeating syscalls. -var page_size_cache = std.atomic.Value(usize).init(0); - -// The default implementation in `std.options.queryPageSizeFn`. -// The first time it is called, it asserts that the page size is within the comptime bounds. -pub fn defaultQueryPageSize() usize { - var size = page_size_cache.load(.unordered); - if (size > 0) return size; - size = switch (builtin.os.tag) { - .linux => if (builtin.link_libc) @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE))) else std.os.linux.getauxval(std.elf.AT_PAGESZ), - .bridgeos, .driverkit, .ios, .macos, .tvos, .visionos, .watchos => blk: { - const task_port = std.c.mach_task_self(); - // mach_task_self may fail "if there are any resource failures or other errors". - if (task_port == std.c.TASK_NULL) - break :blk 0; - var info_count = std.c.TASK_VM_INFO_COUNT; - var vm_info: std.c.task_vm_info_data_t = undefined; - vm_info.page_size = 0; - _ = std.c.task_info( - task_port, - std.c.TASK_VM_INFO, - @as(std.c.task_info_t, @ptrCast(&vm_info)), - &info_count, - ); - assert(vm_info.page_size != 0); - break :blk @as(usize, @intCast(vm_info.page_size)); - }, - .windows => blk: { - var info: std.os.windows.SYSTEM_INFO = undefined; - std.os.windows.kernel32.GetSystemInfo(&info); - break :blk info.dwPageSize; - }, - else => if (builtin.link_libc) - if (std.c._SC != void and @hasDecl(std.c._SC, "PAGESIZE")) - @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE))) - else - @compileError("missing _SC.PAGESIZE declaration for " ++ @tagName(builtin.os.tag) ++ "-" ++ @tagName(builtin.os.tag)) - else if (builtin.os.tag == .freestanding or builtin.os.tag == .other) - @compileError("pageSize on freestanding/other is not supported with the default std.options.queryPageSizeFn") - else - @compileError("pageSize on " ++ @tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " is not supported without linking libc, using the default implementation"), - }; - - assert(size >= min_page_size); - assert(size <= max_page_size); - page_size_cache.store(size, .unordered); - - return size; -} - pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator; pub const loggingAllocator = @import("heap/logging_allocator.zig").loggingAllocator; pub const ScopedLoggingAllocator = @import("heap/logging_allocator.zig").ScopedLoggingAllocator; @@ -391,6 +21,7 @@ pub const WasmAllocator = @import("heap/WasmAllocator.zig"); pub const PageAllocator = @import("heap/PageAllocator.zig"); pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig"); pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator; +pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig"); const memory_pool = @import("heap/memory_pool.zig"); pub const MemoryPool = memory_pool.MemoryPool; @@ -399,7 +30,97 @@ pub const MemoryPoolExtra = memory_pool.MemoryPoolExtra; pub const MemoryPoolOptions = memory_pool.Options; /// TODO Utilize this on Windows. -pub var next_mmap_addr_hint: ?[*]align(min_page_size) u8 = null; +pub var next_mmap_addr_hint: ?[*]align(page_size_min) u8 = null; + +/// comptime-known minimum page size of the target. +/// +/// All pointers from `mmap` or `VirtualAlloc` are aligned to at least +/// `page_size_min`, but their actual alignment may be bigger. +/// +/// This value can be overridden via `std.options.page_size_min`. +/// +/// On many systems, the actual page size can only be determined at runtime +/// with `pageSize`. +pub const page_size_min: usize = std.options.page_size_min orelse (page_size_min_default orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other) + @compileError("freestanding/other page_size_min must provided with std.options.page_size_min") +else + @compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has unknown page_size_min; populate std.options.page_size_min")); + +/// comptime-known maximum page size of the target. +/// +/// Targeting a system with a larger page size may require overriding +/// `std.options.page_size_max`, as well as providing a corresponding linker +/// option. +/// +/// The actual page size can only be determined at runtime with `pageSize`. +pub const page_size_max: usize = std.options.page_size_max orelse (page_size_max_default orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other) + @compileError("freestanding/other page_size_max must provided with std.options.page_size_max") +else + @compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has unknown page_size_max; populate std.options.page_size_max")); + +/// If the page size is comptime-known, return value is comptime. +/// Otherwise, calls `std.options.queryPageSize` which by default queries the +/// host operating system at runtime. +pub inline fn pageSize() usize { + if (page_size_min == page_size_max) return page_size_min; + return std.options.queryPageSize(); +} + +test pageSize { + assert(std.math.isPowerOfTwo(pageSize())); +} + +/// The default implementation of `std.options.queryPageSize`. +/// Asserts that the page size is within `page_size_min` and `page_size_max` +pub fn defaultQueryPageSize() usize { + const global = struct { + var cached_result: std.atomic.Value(usize) = .init(0); + }; + var size = global.cached_result.load(.unordered); + if (size > 0) return size; + size = switch (builtin.os.tag) { + .linux => if (builtin.link_libc) @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE))) else std.os.linux.getauxval(std.elf.AT_PAGESZ), + .driverkit, .ios, .macos, .tvos, .visionos, .watchos => blk: { + const task_port = std.c.mach_task_self(); + // mach_task_self may fail "if there are any resource failures or other errors". + if (task_port == std.c.TASK_NULL) + break :blk 0; + var info_count = std.c.TASK_VM_INFO_COUNT; + var vm_info: std.c.task_vm_info_data_t = undefined; + vm_info.page_size = 0; + _ = std.c.task_info( + task_port, + std.c.TASK_VM_INFO, + @as(std.c.task_info_t, @ptrCast(&vm_info)), + &info_count, + ); + assert(vm_info.page_size != 0); + break :blk @intCast(vm_info.page_size); + }, + .windows => blk: { + var info: std.os.windows.SYSTEM_INFO = undefined; + std.os.windows.kernel32.GetSystemInfo(&info); + break :blk info.dwPageSize; + }, + else => if (builtin.link_libc) + @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE))) + else if (builtin.os.tag == .freestanding or builtin.os.tag == .other) + @compileError("unsupported target: freestanding/other") + else + @compileError("pageSize on " ++ @tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " is not supported without linking libc, using the default implementation"), + }; + + assert(size >= page_size_min); + assert(size <= page_size_max); + global.cached_result.store(size, .unordered); + + return size; +} + +test defaultQueryPageSize { + if (builtin.cpu.arch.isWasm()) return error.SkipZigTest; + assert(std.math.isPowerOfTwo(defaultQueryPageSize())); +} const CAllocator = struct { comptime { @@ -623,13 +344,6 @@ pub const wasm_allocator: Allocator = .{ .vtable = &WasmAllocator.vtable, }; -/// Verifies that the adjusted length will still map to the full length -pub fn alignPageAllocLen(full_len: usize, len: usize) usize { - const aligned_len = mem.alignAllocLen(full_len, len); - assert(mem.alignForward(usize, aligned_len, pageSize()) == full_len); - return aligned_len; -} - pub const HeapAllocator = switch (builtin.os.tag) { .windows => struct { heap_handle: ?HeapHandle, @@ -730,145 +444,6 @@ pub const HeapAllocator = switch (builtin.os.tag) { else => @compileError("Unsupported OS"), }; -fn sliceContainsPtr(container: []u8, ptr: [*]u8) bool { - return @intFromPtr(ptr) >= @intFromPtr(container.ptr) and - @intFromPtr(ptr) < (@intFromPtr(container.ptr) + container.len); -} - -fn sliceContainsSlice(container: []u8, slice: []u8) bool { - return @intFromPtr(slice.ptr) >= @intFromPtr(container.ptr) and - (@intFromPtr(slice.ptr) + slice.len) <= (@intFromPtr(container.ptr) + container.len); -} - -pub const FixedBufferAllocator = struct { - end_index: usize, - buffer: []u8, - - pub fn init(buffer: []u8) FixedBufferAllocator { - return FixedBufferAllocator{ - .buffer = buffer, - .end_index = 0, - }; - } - - /// *WARNING* using this at the same time as the interface returned by `threadSafeAllocator` is not thread safe - pub fn allocator(self: *FixedBufferAllocator) Allocator { - return .{ - .ptr = self, - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .free = free, - }, - }; - } - - /// Provides a lock free thread safe `Allocator` interface to the underlying `FixedBufferAllocator` - /// *WARNING* using this at the same time as the interface returned by `allocator` is not thread safe - pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator { - return .{ - .ptr = self, - .vtable = &.{ - .alloc = threadSafeAlloc, - .resize = Allocator.noResize, - .free = Allocator.noFree, - }, - }; - } - - pub fn ownsPtr(self: *FixedBufferAllocator, ptr: [*]u8) bool { - return sliceContainsPtr(self.buffer, ptr); - } - - pub fn ownsSlice(self: *FixedBufferAllocator, slice: []u8) bool { - return sliceContainsSlice(self.buffer, slice); - } - - /// NOTE: this will not work in all cases, if the last allocation had an adjusted_index - /// then we won't be able to determine what the last allocation was. This is because - /// the alignForward operation done in alloc is not reversible. - pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool { - return buf.ptr + buf.len == self.buffer.ptr + self.end_index; - } - - fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { - const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); - _ = ra; - const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); - const adjust_off = mem.alignPointerOffset(self.buffer.ptr + self.end_index, ptr_align) orelse return null; - const adjusted_index = self.end_index + adjust_off; - const new_end_index = adjusted_index + n; - if (new_end_index > self.buffer.len) return null; - self.end_index = new_end_index; - return self.buffer.ptr + adjusted_index; - } - - fn resize( - ctx: *anyopaque, - buf: []u8, - log2_buf_align: u8, - new_size: usize, - return_address: usize, - ) bool { - const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); - _ = log2_buf_align; - _ = return_address; - assert(@inComptime() or self.ownsSlice(buf)); - - if (!self.isLastAllocation(buf)) { - if (new_size > buf.len) return false; - return true; - } - - if (new_size <= buf.len) { - const sub = buf.len - new_size; - self.end_index -= sub; - return true; - } - - const add = new_size - buf.len; - if (add + self.end_index > self.buffer.len) return false; - - self.end_index += add; - return true; - } - - fn free( - ctx: *anyopaque, - buf: []u8, - log2_buf_align: u8, - return_address: usize, - ) void { - const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); - _ = log2_buf_align; - _ = return_address; - assert(@inComptime() or self.ownsSlice(buf)); - - if (self.isLastAllocation(buf)) { - self.end_index -= buf.len; - } - } - - fn threadSafeAlloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { - const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); - _ = ra; - const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); - var end_index = @atomicLoad(usize, &self.end_index, .seq_cst); - while (true) { - const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null; - const adjusted_index = end_index + adjust_off; - const new_end_index = adjusted_index + n; - if (new_end_index > self.buffer.len) return null; - end_index = @cmpxchgWeak(usize, &self.end_index, end_index, new_end_index, .seq_cst, .seq_cst) orelse - return self.buffer[adjusted_index..new_end_index].ptr; - } - } - - pub fn reset(self: *FixedBufferAllocator) void { - self.end_index = 0; - } -}; - /// Returns a `StackFallbackAllocator` allocating using either a /// `FixedBufferAllocator` on an array of size `size` and falling back to /// `fallback_allocator` if that fails. @@ -975,7 +550,7 @@ test "raw_c_allocator" { } } -test "PageAllocator" { +test PageAllocator { const allocator = page_allocator; try testAllocator(allocator); try testAllocatorAligned(allocator); @@ -985,7 +560,7 @@ test "PageAllocator" { } if (builtin.os.tag == .windows) { - const slice = try allocator.alignedAlloc(u8, min_page_size, 128); + const slice = try allocator.alignedAlloc(u8, page_size_min, 128); slice[0] = 0x12; slice[127] = 0x34; allocator.free(slice); @@ -997,7 +572,7 @@ test "PageAllocator" { } } -test "HeapAllocator" { +test HeapAllocator { if (builtin.os.tag == .windows) { // https://github.com/ziglang/zig/issues/13702 if (builtin.cpu.arch == .aarch64) return error.SkipZigTest; @@ -1013,7 +588,7 @@ test "HeapAllocator" { } } -test "ArenaAllocator" { +test ArenaAllocator { var arena_allocator = ArenaAllocator.init(page_allocator); defer arena_allocator.deinit(); const allocator = arena_allocator.allocator(); @@ -1024,38 +599,6 @@ test "ArenaAllocator" { try testAllocatorAlignedShrink(allocator); } -var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined; -test "FixedBufferAllocator" { - var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..])); - const allocator = fixed_buffer_allocator.allocator(); - - try testAllocator(allocator); - try testAllocatorAligned(allocator); - try testAllocatorLargeAlignment(allocator); - try testAllocatorAlignedShrink(allocator); -} - -test "FixedBufferAllocator.reset" { - var buf: [8]u8 align(@alignOf(u64)) = undefined; - var fba = FixedBufferAllocator.init(buf[0..]); - const allocator = fba.allocator(); - - const X = 0xeeeeeeeeeeeeeeee; - const Y = 0xffffffffffffffff; - - const x = try allocator.create(u64); - x.* = X; - try testing.expectError(error.OutOfMemory, allocator.create(u64)); - - fba.reset(); - const y = try allocator.create(u64); - y.* = Y; - - // we expect Y to have overwritten X. - try testing.expect(x.* == y.*); - try testing.expect(y.* == Y); -} - test "StackFallbackAllocator" { { var stack_allocator = stackFallback(4096, std.testing.allocator); @@ -1075,46 +618,6 @@ test "StackFallbackAllocator" { } } -test "FixedBufferAllocator Reuse memory on realloc" { - var small_fixed_buffer: [10]u8 = undefined; - // check if we re-use the memory - { - var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]); - const allocator = fixed_buffer_allocator.allocator(); - - const slice0 = try allocator.alloc(u8, 5); - try testing.expect(slice0.len == 5); - const slice1 = try allocator.realloc(slice0, 10); - try testing.expect(slice1.ptr == slice0.ptr); - try testing.expect(slice1.len == 10); - try testing.expectError(error.OutOfMemory, allocator.realloc(slice1, 11)); - } - // check that we don't re-use the memory if it's not the most recent block - { - var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]); - const allocator = fixed_buffer_allocator.allocator(); - - var slice0 = try allocator.alloc(u8, 2); - slice0[0] = 1; - slice0[1] = 2; - const slice1 = try allocator.alloc(u8, 2); - const slice2 = try allocator.realloc(slice0, 4); - try testing.expect(slice0.ptr != slice2.ptr); - try testing.expect(slice1.ptr != slice2.ptr); - try testing.expect(slice2[0] == 1); - try testing.expect(slice2[1] == 2); - } -} - -test "Thread safe FixedBufferAllocator" { - var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]); - - try testAllocator(fixed_buffer_allocator.threadSafeAllocator()); - try testAllocatorAligned(fixed_buffer_allocator.threadSafeAllocator()); - try testAllocatorLargeAlignment(fixed_buffer_allocator.threadSafeAllocator()); - try testAllocatorAlignedShrink(fixed_buffer_allocator.threadSafeAllocator()); -} - /// This one should not try alignments that exceed what C malloc can handle. pub fn testAllocator(base_allocator: mem.Allocator) !void { var validationAllocator = mem.validationWrap(base_allocator); @@ -1194,7 +697,7 @@ pub fn testAllocatorLargeAlignment(base_allocator: mem.Allocator) !void { var validationAllocator = mem.validationWrap(base_allocator); const allocator = validationAllocator.allocator(); - const large_align: usize = min_page_size / 2; + const large_align: usize = page_size_min / 2; var align_mask: usize = undefined; align_mask = @shlWithOverflow(~@as(usize, 0), @as(Allocator.Log2Align, @ctz(large_align)))[0]; @@ -1251,19 +754,296 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void { try testing.expect(slice[60] == 0x34); } -test "pageSize() smoke test" { - const size = std.heap.pageSize(); - // Check that pageSize is a power of 2. - std.debug.assert(size & (size - 1) == 0); -} +const page_size_min_default: ?usize = switch (builtin.os.tag) { + .driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) { + .x86_64 => 4 << 10, + .aarch64 => 16 << 10, + else => null, + }, + .windows => switch (builtin.cpu.arch) { + // -- + .x86, .x86_64 => 4 << 10, + // SuperH => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, + // DEC Alpha => 8 << 10, + // Itanium => 8 << 10, + .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, + else => null, + }, + .wasi => switch (builtin.cpu.arch) { + .wasm32, .wasm64 => 64 << 10, + else => null, + }, + // https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187 + .uefi => 4 << 10, + .freebsd => switch (builtin.cpu.arch) { + // FreeBSD/sys/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv32, .riscv64 => 4 << 10, + else => null, + }, + .netbsd => switch (builtin.cpu.arch) { + // NetBSD/sys/arch/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .sparc => 4 << 10, + .sparc64 => 8 << 10, + .riscv32, .riscv64 => 4 << 10, + // Sun-2 + .m68k => 2 << 10, + else => null, + }, + .dragonfly => switch (builtin.cpu.arch) { + .x86, .x86_64 => 4 << 10, + else => null, + }, + .openbsd => switch (builtin.cpu.arch) { + // OpenBSD/sys/arch/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, + .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv64 => 4 << 10, + .sparc64 => 8 << 10, + else => null, + }, + .solaris, .illumos => switch (builtin.cpu.arch) { + // src/uts/*/sys/machparam.h + .x86, .x86_64 => 4 << 10, + .sparc, .sparc64 => 8 << 10, + else => null, + }, + .fuchsia => switch (builtin.cpu.arch) { + // fuchsia/kernel/arch/*/include/arch/defines.h + .x86_64 => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .riscv64 => 4 << 10, + else => null, + }, + // https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13 + .serenity => 4 << 10, + .haiku => switch (builtin.cpu.arch) { + // haiku/headers/posix/arch/*/limits.h + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .m68k => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv64 => 4 << 10, + .sparc64 => 8 << 10, + .x86, .x86_64 => 4 << 10, + else => null, + }, + .hurd => switch (builtin.cpu.arch) { + // gnumach/*/include/mach/*/vm_param.h + .x86, .x86_64 => 4 << 10, + .aarch64 => null, + else => null, + }, + .plan9 => switch (builtin.cpu.arch) { + // 9front/sys/src/9/*/mem.h + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, + .sparc => 4 << 10, + else => null, + }, + .ps3 => switch (builtin.cpu.arch) { + // cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html + .powerpc64 => 1 << 20, // 1 MiB + else => null, + }, + .ps4 => switch (builtin.cpu.arch) { + // https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95 + .x86, .x86_64 => 4 << 10, + else => null, + }, + .ps5 => switch (builtin.cpu.arch) { + // https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95 + .x86, .x86_64 => 16 << 10, + else => null, + }, + // system/lib/libc/musl/arch/emscripten/bits/limits.h + .emscripten => 64 << 10, + .linux => switch (builtin.cpu.arch) { + // Linux/arch/*/Kconfig + .arc => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .csky => 4 << 10, + .hexagon => 4 << 10, + .loongarch32, .loongarch64 => 4 << 10, + .m68k => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv32, .riscv64 => 4 << 10, + .s390x => 4 << 10, + .sparc => 4 << 10, + .sparc64 => 8 << 10, + .x86, .x86_64 => 4 << 10, + .xtensa => 4 << 10, + else => null, + }, + .freestanding => switch (builtin.cpu.arch) { + .wasm32, .wasm64 => 64 << 10, + else => null, + }, + else => null, +}; -test "defaultQueryPageSize() smoke test" { - // queryPageSize() does not always get called by pageSize() - if (builtin.cpu.arch.isWasm()) return error.SkipZigTest; - const size = defaultQueryPageSize(); - // Check that pageSize is a power of 2. - std.debug.assert(size & (size - 1) == 0); -} +const page_size_max_default: ?usize = switch (builtin.os.tag) { + .driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) { + .x86_64 => 4 << 10, + .aarch64 => 16 << 10, + else => null, + }, + .windows => switch (builtin.cpu.arch) { + // -- + .x86, .x86_64 => 4 << 10, + // SuperH => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, + // DEC Alpha => 8 << 10, + // Itanium => 8 << 10, + .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, + else => null, + }, + .wasi => switch (builtin.cpu.arch) { + .wasm32, .wasm64 => 64 << 10, + else => null, + }, + // https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187 + .uefi => 4 << 10, + .freebsd => switch (builtin.cpu.arch) { + // FreeBSD/sys/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv32, .riscv64 => 4 << 10, + else => null, + }, + .netbsd => switch (builtin.cpu.arch) { + // NetBSD/sys/arch/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 64 << 10, + .mips, .mipsel, .mips64, .mips64el => 16 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 16 << 10, + .sparc => 8 << 10, + .sparc64 => 8 << 10, + .riscv32, .riscv64 => 4 << 10, + .m68k => 8 << 10, + else => null, + }, + .dragonfly => switch (builtin.cpu.arch) { + .x86, .x86_64 => 4 << 10, + else => null, + }, + .openbsd => switch (builtin.cpu.arch) { + // OpenBSD/sys/arch/* + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10, + .mips64, .mips64el => 16 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv64 => 4 << 10, + .sparc64 => 8 << 10, + else => null, + }, + .solaris, .illumos => switch (builtin.cpu.arch) { + // src/uts/*/sys/machparam.h + .x86, .x86_64 => 4 << 10, + .sparc, .sparc64 => 8 << 10, + else => null, + }, + .fuchsia => switch (builtin.cpu.arch) { + // fuchsia/kernel/arch/*/include/arch/defines.h + .x86_64 => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .riscv64 => 4 << 10, + else => null, + }, + // https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13 + .serenity => 4 << 10, + .haiku => switch (builtin.cpu.arch) { + // haiku/headers/posix/arch/*/limits.h + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 4 << 10, + .m68k => 4 << 10, + .mips, .mipsel, .mips64, .mips64el => 4 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10, + .riscv64 => 4 << 10, + .sparc64 => 8 << 10, + .x86, .x86_64 => 4 << 10, + else => null, + }, + .hurd => switch (builtin.cpu.arch) { + // gnumach/*/include/mach/*/vm_param.h + .x86, .x86_64 => 4 << 10, + .aarch64 => null, + else => null, + }, + .plan9 => switch (builtin.cpu.arch) { + // 9front/sys/src/9/*/mem.h + .x86, .x86_64 => 4 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 64 << 10, + .mips, .mipsel, .mips64, .mips64el => 16 << 10, + .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10, + .sparc => 4 << 10, + else => null, + }, + .ps3 => switch (builtin.cpu.arch) { + // cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html + .powerpc64 => 1 << 20, // 1 MiB + else => null, + }, + .ps4 => switch (builtin.cpu.arch) { + // https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95 + .x86, .x86_64 => 4 << 10, + else => null, + }, + .ps5 => switch (builtin.cpu.arch) { + // https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95 + .x86, .x86_64 => 16 << 10, + else => null, + }, + // system/lib/libc/musl/arch/emscripten/bits/limits.h + .emscripten => 64 << 10, + .linux => switch (builtin.cpu.arch) { + // Linux/arch/*/Kconfig + .arc => 16 << 10, + .thumb, .thumbeb, .arm, .armeb => 4 << 10, + .aarch64, .aarch64_be => 64 << 10, + .csky => 4 << 10, + .hexagon => 256 << 10, + .loongarch32, .loongarch64 => 64 << 10, + .m68k => 8 << 10, + .mips, .mipsel, .mips64, .mips64el => 64 << 10, + .powerpc, .powerpc64, .powerpc64le, .powerpcle => 256 << 10, + .riscv32, .riscv64 => 4 << 10, + .s390x => 4 << 10, + .sparc => 4 << 10, + .sparc64 => 8 << 10, + .x86, .x86_64 => 4 << 10, + .xtensa => 4 << 10, + else => null, + }, + .freestanding => switch (builtin.cpu.arch) { + .wasm32, .wasm64 => 64 << 10, + else => null, + }, + else => null, +}; test { _ = LoggingAllocator; @@ -1272,6 +1052,7 @@ test { _ = @import("heap/memory_pool.zig"); _ = ArenaAllocator; _ = GeneralPurposeAllocator; + _ = FixedBufferAllocator; if (builtin.target.isWasm()) { _ = WasmAllocator; } diff --git a/lib/std/heap/FixedBufferAllocator.zig b/lib/std/heap/FixedBufferAllocator.zig new file mode 100644 index 0000000000..5995dfe154 --- /dev/null +++ b/lib/std/heap/FixedBufferAllocator.zig @@ -0,0 +1,218 @@ +const std = @import("../std.zig"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const mem = std.mem; + +const FixedBufferAllocator = @This(); + +end_index: usize, +buffer: []u8, + +pub fn init(buffer: []u8) FixedBufferAllocator { + return FixedBufferAllocator{ + .buffer = buffer, + .end_index = 0, + }; +} + +/// Using this at the same time as the interface returned by `threadSafeAllocator` is not thread safe. +pub fn allocator(self: *FixedBufferAllocator) Allocator { + return .{ + .ptr = self, + .vtable = &.{ + .alloc = alloc, + .resize = resize, + .free = free, + }, + }; +} + +/// Provides a lock free thread safe `Allocator` interface to the underlying `FixedBufferAllocator` +/// +/// Using this at the same time as the interface returned by `allocator` is not thread safe. +pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator { + return .{ + .ptr = self, + .vtable = &.{ + .alloc = threadSafeAlloc, + .resize = Allocator.noResize, + .free = Allocator.noFree, + }, + }; +} + +pub fn ownsPtr(self: *FixedBufferAllocator, ptr: [*]u8) bool { + return sliceContainsPtr(self.buffer, ptr); +} + +pub fn ownsSlice(self: *FixedBufferAllocator, slice: []u8) bool { + return sliceContainsSlice(self.buffer, slice); +} + +/// This has false negatives when the last allocation had an +/// adjusted_index. In such case we won't be able to determine what the +/// last allocation was because the alignForward operation done in alloc is +/// not reversible. +pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool { + return buf.ptr + buf.len == self.buffer.ptr + self.end_index; +} + +pub fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { + const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); + _ = ra; + const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); + const adjust_off = mem.alignPointerOffset(self.buffer.ptr + self.end_index, ptr_align) orelse return null; + const adjusted_index = self.end_index + adjust_off; + const new_end_index = adjusted_index + n; + if (new_end_index > self.buffer.len) return null; + self.end_index = new_end_index; + return self.buffer.ptr + adjusted_index; +} + +pub fn resize( + ctx: *anyopaque, + buf: []u8, + log2_buf_align: u8, + new_size: usize, + return_address: usize, +) bool { + const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); + _ = log2_buf_align; + _ = return_address; + assert(@inComptime() or self.ownsSlice(buf)); + + if (!self.isLastAllocation(buf)) { + if (new_size > buf.len) return false; + return true; + } + + if (new_size <= buf.len) { + const sub = buf.len - new_size; + self.end_index -= sub; + return true; + } + + const add = new_size - buf.len; + if (add + self.end_index > self.buffer.len) return false; + + self.end_index += add; + return true; +} + +pub fn free( + ctx: *anyopaque, + buf: []u8, + log2_buf_align: u8, + return_address: usize, +) void { + const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); + _ = log2_buf_align; + _ = return_address; + assert(@inComptime() or self.ownsSlice(buf)); + + if (self.isLastAllocation(buf)) { + self.end_index -= buf.len; + } +} + +fn threadSafeAlloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { + const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); + _ = ra; + const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); + var end_index = @atomicLoad(usize, &self.end_index, .seq_cst); + while (true) { + const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null; + const adjusted_index = end_index + adjust_off; + const new_end_index = adjusted_index + n; + if (new_end_index > self.buffer.len) return null; + end_index = @cmpxchgWeak(usize, &self.end_index, end_index, new_end_index, .seq_cst, .seq_cst) orelse + return self.buffer[adjusted_index..new_end_index].ptr; + } +} + +pub fn reset(self: *FixedBufferAllocator) void { + self.end_index = 0; +} + +fn sliceContainsPtr(container: []u8, ptr: [*]u8) bool { + return @intFromPtr(ptr) >= @intFromPtr(container.ptr) and + @intFromPtr(ptr) < (@intFromPtr(container.ptr) + container.len); +} + +fn sliceContainsSlice(container: []u8, slice: []u8) bool { + return @intFromPtr(slice.ptr) >= @intFromPtr(container.ptr) and + (@intFromPtr(slice.ptr) + slice.len) <= (@intFromPtr(container.ptr) + container.len); +} + +var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined; + +test FixedBufferAllocator { + var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..])); + const a = fixed_buffer_allocator.allocator(); + + try std.heap.testAllocator(a); + try std.heap.testAllocatorAligned(a); + try std.heap.testAllocatorLargeAlignment(a); + try std.heap.testAllocatorAlignedShrink(a); +} + +test reset { + var buf: [8]u8 align(@alignOf(u64)) = undefined; + var fba = FixedBufferAllocator.init(buf[0..]); + const a = fba.allocator(); + + const X = 0xeeeeeeeeeeeeeeee; + const Y = 0xffffffffffffffff; + + const x = try a.create(u64); + x.* = X; + try std.testing.expectError(error.OutOfMemory, a.create(u64)); + + fba.reset(); + const y = try a.create(u64); + y.* = Y; + + // we expect Y to have overwritten X. + try std.testing.expect(x.* == y.*); + try std.testing.expect(y.* == Y); +} + +test "reuse memory on realloc" { + var small_fixed_buffer: [10]u8 = undefined; + // check if we re-use the memory + { + var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]); + const a = fixed_buffer_allocator.allocator(); + + const slice0 = try a.alloc(u8, 5); + try std.testing.expect(slice0.len == 5); + const slice1 = try a.realloc(slice0, 10); + try std.testing.expect(slice1.ptr == slice0.ptr); + try std.testing.expect(slice1.len == 10); + try std.testing.expectError(error.OutOfMemory, a.realloc(slice1, 11)); + } + // check that we don't re-use the memory if it's not the most recent block + { + var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]); + const a = fixed_buffer_allocator.allocator(); + + var slice0 = try a.alloc(u8, 2); + slice0[0] = 1; + slice0[1] = 2; + const slice1 = try a.alloc(u8, 2); + const slice2 = try a.realloc(slice0, 4); + try std.testing.expect(slice0.ptr != slice2.ptr); + try std.testing.expect(slice1.ptr != slice2.ptr); + try std.testing.expect(slice2[0] == 1); + try std.testing.expect(slice2[1] == 2); + } +} + +test "thread safe version" { + var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]); + + try std.heap.testAllocator(fixed_buffer_allocator.threadSafeAllocator()); + try std.heap.testAllocatorAligned(fixed_buffer_allocator.threadSafeAllocator()); + try std.heap.testAllocatorLargeAlignment(fixed_buffer_allocator.threadSafeAllocator()); + try std.heap.testAllocatorAlignedShrink(fixed_buffer_allocator.threadSafeAllocator()); +} diff --git a/lib/std/heap/PageAllocator.zig b/lib/std/heap/PageAllocator.zig index 1e9058717e..c261dcec43 100644 --- a/lib/std/heap/PageAllocator.zig +++ b/lib/std/heap/PageAllocator.zig @@ -2,14 +2,14 @@ const std = @import("../std.zig"); const builtin = @import("builtin"); const Allocator = std.mem.Allocator; const mem = std.mem; -const heap = std.heap; const maxInt = std.math.maxInt; const assert = std.debug.assert; const native_os = builtin.os.tag; const windows = std.os.windows; const posix = std.posix; +const page_size_min = std.heap.page_size_min; -pub const vtable = Allocator.VTable{ +pub const vtable: Allocator.VTable = .{ .alloc = alloc, .resize = resize, .free = free, @@ -19,7 +19,6 @@ fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { _ = ra; _ = log2_align; assert(n > 0); - if (n > maxInt(usize) - (heap.pageSize() - 1)) return null; if (native_os == .windows) { const addr = windows.VirtualAlloc( @@ -35,7 +34,10 @@ fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { return @ptrCast(addr); } - const aligned_len = mem.alignForward(usize, n, heap.pageSize()); + const page_size = std.heap.pageSize(); + if (n >= maxInt(usize) - page_size) return null; + + const aligned_len = mem.alignForward(usize, n, page_size); const hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered); const slice = posix.mmap( hint, @@ -45,8 +47,8 @@ fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { -1, 0, ) catch return null; - assert(mem.isAligned(@intFromPtr(slice.ptr), heap.pageSize())); - const new_hint: [*]align(heap.min_page_size) u8 = @alignCast(slice.ptr + aligned_len); + assert(mem.isAligned(@intFromPtr(slice.ptr), page_size_min)); + const new_hint: [*]align(std.heap.page_size_min) u8 = @alignCast(slice.ptr + aligned_len); _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, hint, new_hint, .monotonic, .monotonic); return slice.ptr; } @@ -60,13 +62,14 @@ fn resize( ) bool { _ = log2_buf_align; _ = return_address; - const new_size_aligned = mem.alignForward(usize, new_size, heap.pageSize()); + const page_size = std.heap.pageSize(); + const new_size_aligned = mem.alignForward(usize, new_size, page_size); if (native_os == .windows) { if (new_size <= buf_unaligned.len) { const base_addr = @intFromPtr(buf_unaligned.ptr); const old_addr_end = base_addr + buf_unaligned.len; - const new_addr_end = mem.alignForward(usize, base_addr + new_size, heap.pageSize()); + const new_addr_end = mem.alignForward(usize, base_addr + new_size, page_size); if (old_addr_end > new_addr_end) { // For shrinking that is not releasing, we will only // decommit the pages not needed anymore. @@ -78,14 +81,14 @@ fn resize( } return true; } - const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, heap.pageSize()); + const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, page_size); if (new_size_aligned <= old_size_aligned) { return true; } return false; } - const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, heap.pageSize()); + const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, page_size); if (new_size_aligned == buf_aligned_len) return true; @@ -108,7 +111,7 @@ fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) v if (native_os == .windows) { windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE); } else { - const buf_aligned_len = mem.alignForward(usize, slice.len, heap.pageSize()); + const buf_aligned_len = mem.alignForward(usize, slice.len, std.heap.pageSize()); posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len])); } } diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index bfb824d4aa..4a03dfecb8 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -75,7 +75,7 @@ //! BucketHeader, followed by "used bits", and two stack traces for each slot //! (allocation trace and free trace). //! -//! The buckets array contains buckets for every size class below `max_page_size`. +//! The buckets array contains buckets for every size class below `page_size_max`. //! At runtime, only size classes below `pageSize()` will actually be used for allocations. //! //! The "used bits" are 1 bit per slot representing whether the slot is used. @@ -102,13 +102,13 @@ const math = std.math; const assert = std.debug.assert; const mem = std.mem; const Allocator = std.mem.Allocator; -const min_page_size = std.heap.min_page_size; -const max_page_size = std.heap.max_page_size; +const page_size_min = std.heap.page_size_min; +const page_size_max = std.heap.page_size_max; const pageSize = std.heap.pageSize; const StackTrace = std.builtin.StackTrace; /// Integer type for pointing to slots in a small allocation -const SlotIndex = std.meta.Int(.unsigned, math.log2(max_page_size) + 1); +const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size_max) + 1); const default_test_stack_trace_frames: usize = if (builtin.is_test) 10 else 6; const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) default_test_stack_trace_frames else 0; @@ -214,7 +214,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { pub const Error = mem.Allocator.Error; - const small_bucket_count = math.log2(max_page_size); + const small_bucket_count = math.log2(page_size_max); const largest_bucket_object_size = 1 << (small_bucket_count - 1); const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size); fn used_small_bucket_count() usize { @@ -287,7 +287,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { // * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation const BucketHeader = struct { - page: [*]align(min_page_size) u8, + page: [*]align(page_size_min) u8, alloc_cursor: SlotIndex, used_count: SlotIndex, @@ -591,7 +591,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { addr: usize, current_bucket: ?*BucketHeader, ) ?*BucketHeader { - const search_page: [*]align(min_page_size) u8 = @ptrFromInt(mem.alignBackward(usize, addr, pageSize())); + const search_page: [*]align(page_size_min) u8 = @ptrFromInt(mem.alignBackward(usize, addr, pageSize())); if (current_bucket != null and current_bucket.?.page == search_page) { return current_bucket; } @@ -1062,7 +1062,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } fn createBucket(self: *Self, size_class: usize) Error!*BucketHeader { - const page = try self.backing_allocator.alignedAlloc(u8, min_page_size, pageSize()); + const page = try self.backing_allocator.alignedAlloc(u8, page_size_min, pageSize()); errdefer self.backing_allocator.free(page); const bucket_size = bucketSize(size_class); diff --git a/lib/std/mem.zig b/lib/std/mem.zig index a0ed952e7b..f0531bab1e 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -1048,17 +1048,18 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co // as we don't read into a new page. This should be the case for most architectures // which use paged memory, however should be confirmed before adding a new arch below. .aarch64, .x86, .x86_64 => if (std.simd.suggestVectorLength(T)) |block_len| { + const page_size = std.heap.pageSize(); const block_size = @sizeOf(T) * block_len; const Block = @Vector(block_len, T); const mask: Block = @splat(sentinel); - comptime std.debug.assert(std.heap.max_page_size % @sizeOf(Block) == 0); - std.debug.assert(std.heap.pageSize() % @sizeOf(Block) == 0); + comptime assert(std.heap.page_size_max % @sizeOf(Block) == 0); + assert(page_size % @sizeOf(Block) == 0); // First block may be unaligned const start_addr = @intFromPtr(&p[i]); - const offset_in_page = start_addr & (std.heap.pageSize() - 1); - if (offset_in_page <= std.heap.pageSize() - @sizeOf(Block)) { + const offset_in_page = start_addr & (page_size - 1); + if (offset_in_page <= page_size - @sizeOf(Block)) { // Will not read past the end of a page, full block. const block: Block = p[i..][0..block_len].*; const matches = block == mask; @@ -1078,7 +1079,7 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co } } - std.debug.assert(std.mem.isAligned(@intFromPtr(&p[i]), block_size)); + assert(std.mem.isAligned(@intFromPtr(&p[i]), block_size)); while (true) { const block: *const Block = @ptrCast(@alignCast(p[i..][0..block_len])); const matches = block.* == mask; @@ -1101,23 +1102,24 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co test "indexOfSentinel vector paths" { const Types = [_]type{ u8, u16, u32, u64 }; const allocator = std.testing.allocator; + const page_size = std.heap.pageSize(); inline for (Types) |T| { const block_len = std.simd.suggestVectorLength(T) orelse continue; // Allocate three pages so we guarantee a page-crossing address with a full page after - const memory = try allocator.alloc(T, 3 * std.heap.pageSize() / @sizeOf(T)); + const memory = try allocator.alloc(T, 3 * page_size / @sizeOf(T)); defer allocator.free(memory); @memset(memory, 0xaa); // Find starting page-alignment = 0 var start: usize = 0; const start_addr = @intFromPtr(&memory); - start += (std.mem.alignForward(usize, start_addr, std.heap.pageSize()) - start_addr) / @sizeOf(T); - try testing.expect(start < std.heap.pageSize() / @sizeOf(T)); + start += (std.mem.alignForward(usize, start_addr, page_size) - start_addr) / @sizeOf(T); + try testing.expect(start < page_size / @sizeOf(T)); // Validate all sub-block alignments - const search_len = std.heap.pageSize() / @sizeOf(T); + const search_len = page_size / @sizeOf(T); memory[start + search_len] = 0; for (0..block_len) |offset| { try testing.expectEqual(search_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start + offset]))); @@ -1125,7 +1127,7 @@ test "indexOfSentinel vector paths" { memory[start + search_len] = 0xaa; // Validate page boundary crossing - const start_page_boundary = start + (std.heap.pageSize() / @sizeOf(T)); + const start_page_boundary = start + (page_size / @sizeOf(T)); memory[start_page_boundary + block_len] = 0; for (0..block_len) |offset| { try testing.expectEqual(2 * block_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start_page_boundary - block_len + offset]))); diff --git a/lib/std/mem/Allocator.zig b/lib/std/mem/Allocator.zig index 38fb9a5b96..7f3334f1d2 100644 --- a/lib/std/mem/Allocator.zig +++ b/lib/std/mem/Allocator.zig @@ -18,11 +18,15 @@ ptr: *anyopaque, vtable: *const VTable, pub const VTable = struct { - /// Attempt to allocate exactly `len` bytes aligned to `1 << ptr_align`. + /// Allocate exactly `len` bytes aligned to `1 << ptr_align`, or return `null` + /// indicating the allocation failed. /// /// `ret_addr` is optionally provided as the first return address of the /// allocation call stack. If the value is `0` it means no return address /// has been provided. + /// + /// The returned slice of memory must have been `@memset` to `undefined` + /// by the allocator implementation. alloc: *const fn (ctx: *anyopaque, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8, /// Attempt to expand or shrink memory in place. `buf.len` must equal the @@ -215,11 +219,6 @@ fn allocWithSizeAndAlignment(self: Allocator, comptime size: usize, comptime ali } fn allocBytesWithAlignment(self: Allocator, comptime alignment: u29, byte_count: usize, return_address: usize) Error![*]align(alignment) u8 { - // The Zig Allocator interface is not intended to solve alignments beyond - // the minimum OS page size. For these use cases, the caller must use OS - // APIs directly. - if (!@inComptime() and alignment > std.heap.pageSize()) @panic("Alignment must be smaller than page size."); - if (byte_count == 0) { const ptr = comptime std.mem.alignBackward(usize, math.maxInt(usize), alignment); return @as([*]align(alignment) u8, @ptrFromInt(ptr)); diff --git a/lib/std/os/linux/IoUring.zig b/lib/std/os/linux/IoUring.zig index d01fc6e0ec..2e43709075 100644 --- a/lib/std/os/linux/IoUring.zig +++ b/lib/std/os/linux/IoUring.zig @@ -3,12 +3,12 @@ const std = @import("std"); const builtin = @import("builtin"); const assert = std.debug.assert; const mem = std.mem; -const heap = std.heap; const net = std.net; const posix = std.posix; const linux = std.os.linux; const testing = std.testing; const is_linux = builtin.os.tag == .linux; +const page_size_min = std.heap.page_size_min; fd: posix.fd_t = -1, sq: SubmissionQueue, @@ -1342,8 +1342,8 @@ pub const SubmissionQueue = struct { dropped: *u32, array: []u32, sqes: []linux.io_uring_sqe, - mmap: []align(heap.min_page_size) u8, - mmap_sqes: []align(heap.min_page_size) u8, + mmap: []align(page_size_min) u8, + mmap_sqes: []align(page_size_min) u8, // We use `sqe_head` and `sqe_tail` in the same way as liburing: // We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`. @@ -1461,7 +1461,7 @@ pub const BufferGroup = struct { /// Pointer to the memory shared by the kernel. /// `buffers_count` of `io_uring_buf` structures are shared by the kernel. /// First `io_uring_buf` is overlaid by `io_uring_buf_ring` struct. - br: *align(heap.min_page_size) linux.io_uring_buf_ring, + br: *align(page_size_min) linux.io_uring_buf_ring, /// Contiguous block of memory of size (buffers_count * buffer_size). buffers: []u8, /// Size of each buffer in buffers. @@ -1556,7 +1556,7 @@ pub const BufferGroup = struct { /// `fd` is IO_Uring.fd for which the provided buffer ring is being registered. /// `entries` is the number of entries requested in the buffer ring, must be power of 2. /// `group_id` is the chosen buffer group ID, unique in IO_Uring. -pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(heap.min_page_size) linux.io_uring_buf_ring { +pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(page_size_min) linux.io_uring_buf_ring { if (entries == 0 or entries > 1 << 15) return error.EntriesNotInRange; if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo; @@ -1572,7 +1572,7 @@ pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(heap. errdefer posix.munmap(mmap); assert(mmap.len == mmap_size); - const br: *align(heap.min_page_size) linux.io_uring_buf_ring = @ptrCast(mmap.ptr); + const br: *align(page_size_min) linux.io_uring_buf_ring = @ptrCast(mmap.ptr); try register_buf_ring(fd, @intFromPtr(br), entries, group_id); return br; } @@ -1614,9 +1614,9 @@ fn handle_register_buf_ring_result(res: usize) !void { } // Unregisters a previously registered shared buffer ring, returned from io_uring_setup_buf_ring. -pub fn free_buf_ring(fd: posix.fd_t, br: *align(heap.min_page_size) linux.io_uring_buf_ring, entries: u32, group_id: u16) void { +pub fn free_buf_ring(fd: posix.fd_t, br: *align(page_size_min) linux.io_uring_buf_ring, entries: u32, group_id: u16) void { unregister_buf_ring(fd, group_id) catch {}; - var mmap: []align(heap.min_page_size) u8 = undefined; + var mmap: []align(page_size_min) u8 = undefined; mmap.ptr = @ptrCast(br); mmap.len = entries * @sizeOf(linux.io_uring_buf); posix.munmap(mmap); diff --git a/lib/std/os/linux/tls.zig b/lib/std/os/linux/tls.zig index 7917fe9d1b..3180b04d2b 100644 --- a/lib/std/os/linux/tls.zig +++ b/lib/std/os/linux/tls.zig @@ -11,13 +11,13 @@ const std = @import("std"); const mem = std.mem; -const heap = std.heap; const elf = std.elf; const math = std.math; const assert = std.debug.assert; const native_arch = @import("builtin").cpu.arch; const linux = std.os.linux; const posix = std.posix; +const page_size_min = std.heap.page_size_min; /// Represents an ELF TLS variant. /// @@ -485,13 +485,13 @@ pub fn prepareArea(area: []u8) usize { }; } -// The main motivation for the size chosen here is that this is how much ends up being requested for -// the thread-local variables of the `std.crypto.random` implementation. I'm not sure why it ends up -// being so much; the struct itself is only 64 bytes. I think it has to do with being page-aligned -// and LLVM or LLD is not smart enough to lay out the TLS data in a space-conserving way. Anyway, I -// think it's fine because it's less than 3 pages of memory, and putting it in the ELF like this is -// equivalent to moving the `mmap` call below into the kernel, avoiding syscall overhead. -var main_thread_area_buffer: [0x2100]u8 align(heap.min_page_size) = undefined; +/// The main motivation for the size chosen here is that this is how much ends up being requested for +/// the thread-local variables of the `std.crypto.random` implementation. I'm not sure why it ends up +/// being so much; the struct itself is only 64 bytes. I think it has to do with being page-aligned +/// and LLVM or LLD is not smart enough to lay out the TLS data in a space-conserving way. Anyway, I +/// think it's fine because it's less than 3 pages of memory, and putting it in the ELF like this is +/// equivalent to moving the `mmap` call below into the kernel, avoiding syscall overhead. +var main_thread_area_buffer: [0x2100]u8 align(page_size_min) = undefined; /// Computes the layout of the static TLS area, allocates the area, initializes all of its fields, /// and assigns the architecture-specific value to the TP register. @@ -504,7 +504,7 @@ pub fn initStatic(phdrs: []elf.Phdr) void { const area = blk: { // Fast path for the common case where the TLS data is really small, avoid an allocation and // use our local buffer. - if (area_desc.alignment <= heap.min_page_size and area_desc.size <= main_thread_area_buffer.len) { + if (area_desc.alignment <= page_size_min and area_desc.size <= main_thread_area_buffer.len) { break :blk main_thread_area_buffer[0..area_desc.size]; } @@ -518,7 +518,7 @@ pub fn initStatic(phdrs: []elf.Phdr) void { ); if (@as(isize, @bitCast(begin_addr)) < 0) @trap(); - const area_ptr: [*]align(heap.min_page_size) u8 = @ptrFromInt(begin_addr); + const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr); // Make sure the slice is correctly aligned. const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment); diff --git a/lib/std/posix.zig b/lib/std/posix.zig index 46cab28e78..5b36c9b139 100644 --- a/lib/std/posix.zig +++ b/lib/std/posix.zig @@ -18,13 +18,13 @@ const builtin = @import("builtin"); const root = @import("root"); const std = @import("std.zig"); const mem = std.mem; -const heap = std.heap; const fs = std.fs; const max_path_bytes = fs.max_path_bytes; const maxInt = std.math.maxInt; const cast = std.math.cast; const assert = std.debug.assert; const native_os = builtin.os.tag; +const page_size_min = std.heap.page_size_min; test { _ = @import("posix/test.zig"); @@ -4695,7 +4695,7 @@ pub const MProtectError = error{ OutOfMemory, } || UnexpectedError; -pub fn mprotect(memory: []align(heap.min_page_size) u8, protection: u32) MProtectError!void { +pub fn mprotect(memory: []align(page_size_min) u8, protection: u32) MProtectError!void { if (native_os == .windows) { const win_prot: windows.DWORD = switch (@as(u3, @truncate(protection))) { 0b000 => windows.PAGE_NOACCESS, @@ -4760,21 +4760,21 @@ pub const MMapError = error{ /// * SIGSEGV - Attempted write into a region mapped as read-only. /// * SIGBUS - Attempted access to a portion of the buffer that does not correspond to the file pub fn mmap( - ptr: ?[*]align(heap.min_page_size) u8, + ptr: ?[*]align(page_size_min) u8, length: usize, prot: u32, flags: system.MAP, fd: fd_t, offset: u64, -) MMapError![]align(heap.min_page_size) u8 { +) MMapError![]align(page_size_min) u8 { const mmap_sym = if (lfs64_abi) system.mmap64 else system.mmap; const rc = mmap_sym(ptr, length, prot, @bitCast(flags), fd, @bitCast(offset)); const err: E = if (builtin.link_libc) blk: { - if (rc != std.c.MAP_FAILED) return @as([*]align(heap.min_page_size) u8, @ptrCast(@alignCast(rc)))[0..length]; + if (rc != std.c.MAP_FAILED) return @as([*]align(page_size_min) u8, @ptrCast(@alignCast(rc)))[0..length]; break :blk @enumFromInt(system._errno().*); } else blk: { const err = errno(rc); - if (err == .SUCCESS) return @as([*]align(heap.min_page_size) u8, @ptrFromInt(rc))[0..length]; + if (err == .SUCCESS) return @as([*]align(page_size_min) u8, @ptrFromInt(rc))[0..length]; break :blk err; }; switch (err) { @@ -4800,7 +4800,7 @@ pub fn mmap( /// Zig's munmap function does not, for two reasons: /// * It violates the Zig principle that resource deallocation must succeed. /// * The Windows function, VirtualFree, has this restriction. -pub fn munmap(memory: []align(heap.min_page_size) const u8) void { +pub fn munmap(memory: []align(page_size_min) const u8) void { switch (errno(system.munmap(memory.ptr, memory.len))) { .SUCCESS => return, .INVAL => unreachable, // Invalid parameters. @@ -4814,7 +4814,7 @@ pub const MSyncError = error{ PermissionDenied, } || UnexpectedError; -pub fn msync(memory: []align(heap.min_page_size) u8, flags: i32) MSyncError!void { +pub fn msync(memory: []align(page_size_min) u8, flags: i32) MSyncError!void { switch (errno(system.msync(memory.ptr, memory.len, flags))) { .SUCCESS => return, .PERM => return error.PermissionDenied, @@ -7136,7 +7136,7 @@ pub const MincoreError = error{ } || UnexpectedError; /// Determine whether pages are resident in memory. -pub fn mincore(ptr: [*]align(heap.min_page_size) u8, length: usize, vec: [*]u8) MincoreError!void { +pub fn mincore(ptr: [*]align(page_size_min) u8, length: usize, vec: [*]u8) MincoreError!void { return switch (errno(system.mincore(ptr, length, vec))) { .SUCCESS => {}, .AGAIN => error.SystemResources, @@ -7182,7 +7182,7 @@ pub const MadviseError = error{ /// Give advice about use of memory. /// This syscall is optional and is sometimes configured to be disabled. -pub fn madvise(ptr: [*]align(heap.min_page_size) u8, length: usize, advice: u32) MadviseError!void { +pub fn madvise(ptr: [*]align(page_size_min) u8, length: usize, advice: u32) MadviseError!void { switch (errno(system.madvise(ptr, length, advice))) { .SUCCESS => return, .PERM => return error.PermissionDenied, diff --git a/lib/std/process.zig b/lib/std/process.zig index 0e10128e0e..dd08e88af2 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -1560,7 +1560,7 @@ pub fn posixGetUserInfo(name: []const u8) !UserInfo { ReadGroupId, }; - var buf: [std.heap.min_page_size]u8 = undefined; + var buf: [std.heap.page_size_min]u8 = undefined; var name_index: usize = 0; var state = State.Start; var uid: posix.uid_t = 0; diff --git a/lib/std/start.zig b/lib/std/start.zig index 0163f0054e..a91df35700 100644 --- a/lib/std/start.zig +++ b/lib/std/start.zig @@ -576,7 +576,7 @@ fn expandStackSize(phdrs: []elf.Phdr) void { switch (phdr.p_type) { elf.PT_GNU_STACK => { if (phdr.p_memsz == 0) break; - assert(phdr.p_memsz % std.heap.pageSize() == 0); + assert(phdr.p_memsz % std.heap.page_size_min == 0); // Silently fail if we are unable to get limits. const limits = std.posix.getrlimit(.STACK) catch break; diff --git a/lib/std/std.zig b/lib/std/std.zig index 43b163bb2d..558710015c 100644 --- a/lib/std/std.zig +++ b/lib/std/std.zig @@ -119,9 +119,12 @@ pub const Options = struct { args: anytype, ) void = log.defaultLog, - min_page_size: ?usize = null, - max_page_size: ?usize = null, - queryPageSizeFn: fn () usize = heap.defaultQueryPageSize, + /// Overrides `std.heap.page_size_min`. + page_size_min: ?usize = null, + /// Overrides `std.heap.page_size_max`. + page_size_max: ?usize = null, + /// Overrides default implementation for determining OS page size at runtime. + queryPageSize: fn () usize = heap.defaultQueryPageSize, fmt_max_depth: usize = fmt.default_max_depth, diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index e8e0840f09..9b119161de 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1249,7 +1249,7 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { .{@errorName(err)}, )); defer zip_file.close(); - var buf: [std.heap.min_page_size]u8 = undefined; + var buf: [4096]u8 = undefined; while (true) { const len = reader.readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString( "read zip stream failed: {s}",