Zig source code is encoded in UTF-8. An invalid UTF-8 byte sequence results in a compile error.
diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 9a39bc0425..4ecdc3803b 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -883,24 +883,39 @@ pub const Dir = struct { /// [WTF-16](https://simonsapin.github.io/wtf-8/#potentially-ill-formed-utf-16) encoded. pub fn openFileW(self: Dir, sub_path_w: []const u16, flags: File.OpenFlags) File.OpenError!File { const w = os.windows; - return @as(File, .{ - .handle = try os.windows.OpenFile(sub_path_w, .{ + const file: File = .{ + .handle = try w.OpenFile(sub_path_w, .{ .dir = self.fd, .access_mask = w.SYNCHRONIZE | (if (flags.read) @as(u32, w.GENERIC_READ) else 0) | (if (flags.write) @as(u32, w.GENERIC_WRITE) else 0), - .share_access = switch (flags.lock) { - .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE, - .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE, - .Exclusive => w.FILE_SHARE_DELETE, - }, - .share_access_nonblocking = flags.lock_nonblocking, .creation = w.FILE_OPEN, .io_mode = flags.intended_io_mode, }), .capable_io_mode = std.io.default_mode, .intended_io_mode = flags.intended_io_mode, - }); + }; + var io: w.IO_STATUS_BLOCK = undefined; + const range_off: w.LARGE_INTEGER = 0; + const range_len: w.LARGE_INTEGER = 1; + const exclusive = switch (flags.lock) { + .None => return file, + .Shared => false, + .Exclusive => true, + }; + try w.LockFile( + file.handle, + null, + null, + null, + &io, + &range_off, + &range_len, + null, + @boolToInt(flags.lock_nonblocking), + @boolToInt(exclusive), + ); + return file; } /// Creates, opens, or overwrites a file with write access. @@ -1019,16 +1034,10 @@ pub const Dir = struct { pub fn createFileW(self: Dir, sub_path_w: []const u16, flags: File.CreateFlags) File.OpenError!File { const w = os.windows; const read_flag = if (flags.read) @as(u32, w.GENERIC_READ) else 0; - return @as(File, .{ + const file: File = .{ .handle = try os.windows.OpenFile(sub_path_w, .{ .dir = self.fd, .access_mask = w.SYNCHRONIZE | w.GENERIC_WRITE | read_flag, - .share_access = switch (flags.lock) { - .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE, - .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE, - .Exclusive => w.FILE_SHARE_DELETE, - }, - .share_access_nonblocking = flags.lock_nonblocking, .creation = if (flags.exclusive) @as(u32, w.FILE_CREATE) else if (flags.truncate) @@ -1039,7 +1048,28 @@ pub const Dir = struct { }), .capable_io_mode = std.io.default_mode, .intended_io_mode = flags.intended_io_mode, - }); + }; + var io: w.IO_STATUS_BLOCK = undefined; + const range_off: w.LARGE_INTEGER = 0; + const range_len: w.LARGE_INTEGER = 1; + const exclusive = switch (flags.lock) { + .None => return file, + .Shared => false, + .Exclusive => true, + }; + try w.LockFile( + file.handle, + null, + null, + null, + &io, + &range_off, + &range_len, + null, + @boolToInt(flags.lock_nonblocking), + @boolToInt(exclusive), + ); + return file; } pub const openRead = @compileError("deprecated in favor of openFile"); diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index fecf2310dd..a4300e7376 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -74,17 +74,28 @@ pub const File = struct { read: bool = true, write: bool = false, - /// Open the file with a lock to prevent other processes from accessing it at the - /// same time. An exclusive lock will prevent other processes from acquiring a lock. - /// A shared lock will prevent other processes from acquiring a exclusive lock, but - /// doesn't prevent other process from getting their own shared locks. + /// Open the file with an advisory lock to coordinate with other processes + /// accessing it at the same time. An exclusive lock will prevent other + /// processes from acquiring a lock. A shared lock will prevent other + /// processes from acquiring a exclusive lock, but does not prevent + /// other process from getting their own shared locks. /// - /// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1]. + /// The lock is advisory, except on Linux in very specific cirsumstances[1]. /// This means that a process that does not respect the locking API can still get access /// to the file, despite the lock. /// - /// Windows' file locks are mandatory, and any process attempting to access the file will - /// receive an error. + /// On these operating systems, the lock is acquired atomically with + /// opening the file: + /// * Darwin + /// * DragonFlyBSD + /// * FreeBSD + /// * Haiku + /// * NetBSD + /// * OpenBSD + /// On these operating systems, the lock is acquired via a separate syscall + /// after opening the file: + /// * Linux + /// * Windows /// /// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt lock: Lock = .None, @@ -120,17 +131,28 @@ pub const File = struct { /// `error.PathAlreadyExists` to be returned. exclusive: bool = false, - /// Open the file with a lock to prevent other processes from accessing it at the - /// same time. An exclusive lock will prevent other processes from acquiring a lock. - /// A shared lock will prevent other processes from acquiring a exclusive lock, but - /// doesn't prevent other process from getting their own shared locks. + /// Open the file with an advisory lock to coordinate with other processes + /// accessing it at the same time. An exclusive lock will prevent other + /// processes from acquiring a lock. A shared lock will prevent other + /// processes from acquiring a exclusive lock, but does not prevent + /// other process from getting their own shared locks. /// - /// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1]. + /// The lock is advisory, except on Linux in very specific cirsumstances[1]. /// This means that a process that does not respect the locking API can still get access /// to the file, despite the lock. /// - /// Windows's file locks are mandatory, and any process attempting to access the file will - /// receive an error. + /// On these operating systems, the lock is acquired atomically with + /// opening the file: + /// * Darwin + /// * DragonFlyBSD + /// * FreeBSD + /// * Haiku + /// * NetBSD + /// * OpenBSD + /// On these operating systems, the lock is acquired via a separate syscall + /// after opening the file: + /// * Linux + /// * Windows /// /// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt lock: Lock = .None, @@ -829,4 +851,165 @@ pub const File = struct { pub fn seekableStream(file: File) SeekableStream { return .{ .context = file }; } + + const range_off: windows.LARGE_INTEGER = 0; + const range_len: windows.LARGE_INTEGER = 1; + + pub const LockError = error{ + SystemResources, + } || os.UnexpectedError; + + /// Blocks when an incompatible lock is held by another process. + /// A process may hold only one type of lock (shared or exclusive) on + /// a file. When a process terminates in any way, the lock is released. + /// + /// Assumes the file is unlocked. + /// + /// TODO: integrate with async I/O + pub fn lock(file: File, l: Lock) LockError!void { + if (is_windows) { + var io_status_block: windows.IO_STATUS_BLOCK = undefined; + const exclusive = switch (l) { + .None => return, + .Shared => false, + .Exclusive => true, + }; + return windows.LockFile( + file.handle, + null, + null, + null, + &io_status_block, + &range_off, + &range_len, + null, + windows.FALSE, // non-blocking=false + @boolToInt(exclusive), + ) catch |err| switch (err) { + error.WouldBlock => unreachable, // non-blocking=false + else => |e| return e, + }; + } else { + return os.flock(file.handle, switch (l) { + .None => os.LOCK_UN, + .Shared => os.LOCK_SH, + .Exclusive => os.LOCK_EX, + }) catch |err| switch (err) { + error.WouldBlock => unreachable, // non-blocking=false + else => |e| return e, + }; + } + } + + /// Assumes the file is locked. + pub fn unlock(file: File) void { + if (is_windows) { + var io_status_block: windows.IO_STATUS_BLOCK = undefined; + return windows.UnlockFile( + file.handle, + &io_status_block, + &range_off, + &range_len, + null, + ) catch |err| switch (err) { + error.RangeNotLocked => unreachable, // Function assumes unlocked. + error.Unexpected => unreachable, // Resource deallocation must succeed. + }; + } else { + return os.flock(file.handle, os.LOCK_UN) catch |err| switch (err) { + error.WouldBlock => unreachable, // unlocking can't block + error.SystemResources => unreachable, // We are deallocating resources. + error.Unexpected => unreachable, // Resource deallocation must succeed. + }; + } + } + + /// Attempts to obtain a lock, returning `true` if the lock is + /// obtained, and `false` if there was an existing incompatible lock held. + /// A process may hold only one type of lock (shared or exclusive) on + /// a file. When a process terminates in any way, the lock is released. + /// + /// Assumes the file is unlocked. + /// + /// TODO: integrate with async I/O + pub fn tryLock(file: File, l: Lock) LockError!bool { + if (is_windows) { + var io_status_block: windows.IO_STATUS_BLOCK = undefined; + const exclusive = switch (l) { + .None => return, + .Shared => false, + .Exclusive => true, + }; + windows.LockFile( + file.handle, + null, + null, + null, + &io_status_block, + &range_off, + &range_len, + null, + windows.TRUE, // non-blocking=true + @boolToInt(exclusive), + ) catch |err| switch (err) { + error.WouldBlock => return false, + else => |e| return e, + }; + } else { + os.flock(file.handle, switch (l) { + .None => os.LOCK_UN, + .Shared => os.LOCK_SH | os.LOCK_NB, + .Exclusive => os.LOCK_EX | os.LOCK_NB, + }) catch |err| switch (err) { + error.WouldBlock => return false, + else => |e| return e, + }; + } + return true; + } + + /// Assumes the file is already locked in exclusive mode. + /// Atomically modifies the lock to be in shared mode, without releasing it. + /// + /// TODO: integrate with async I/O + pub fn downgradeLock(file: File) LockError!void { + if (is_windows) { + // On Windows it works like a semaphore + exclusivity flag. To implement this + // function, we first obtain another lock in shared mode. This changes the + // exclusivity flag, but increments the semaphore to 2. So we follow up with + // an NtUnlockFile which decrements the semaphore but does not modify the + // exclusivity flag. + var io_status_block: windows.IO_STATUS_BLOCK = undefined; + windows.LockFile( + file.handle, + null, + null, + null, + &io_status_block, + &range_off, + &range_len, + null, + windows.TRUE, // non-blocking=true + windows.FALSE, // exclusive=false + ) catch |err| switch (err) { + error.WouldBlock => unreachable, // File was not locked in exclusive mode. + else => |e| return e, + }; + return windows.UnlockFile( + file.handle, + &io_status_block, + &range_off, + &range_len, + null, + ) catch |err| switch (err) { + error.RangeNotLocked => unreachable, // File was not locked. + error.Unexpected => unreachable, // Resource deallocation must succeed. + }; + } else { + return os.flock(file.handle, os.LOCK_SH | os.LOCK_NB) catch |err| switch (err) { + error.WouldBlock => unreachable, // File was not locked in exclusive mode. + else => |e| return e, + }; + } + } }; diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 1fade2a462..cbd4ce065b 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -49,7 +49,6 @@ pub const OpenFileOptions = struct { dir: ?HANDLE = null, sa: ?*SECURITY_ATTRIBUTES = null, share_access: ULONG = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE, - share_access_nonblocking: bool = false, creation: ULONG, io_mode: std.io.ModeOverride, /// If true, tries to open path as a directory. @@ -60,8 +59,6 @@ pub const OpenFileOptions = struct { follow_symlinks: bool = true, }; -/// TODO when share_access_nonblocking is false, this implementation uses -/// untinterruptible sleep() to block. This is not the final iteration of the API. pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE { if (mem.eql(u16, sub_path_w, &[_]u16{'.'}) and !options.open_dir) { return error.IsDir; @@ -94,53 +91,39 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN // If we're not following symlinks, we need to ensure we don't pass in any synchronization flags such as FILE_SYNCHRONOUS_IO_NONALERT. const flags: ULONG = if (options.follow_symlinks) file_or_dir_flag | blocking_flag else file_or_dir_flag | FILE_OPEN_REPARSE_POINT; - var delay: usize = 1; - while (true) { - const rc = ntdll.NtCreateFile( - &result, - options.access_mask, - &attr, - &io, - null, - FILE_ATTRIBUTE_NORMAL, - options.share_access, - options.creation, - flags, - null, - 0, - ); - switch (rc) { - .SUCCESS => { - if (std.io.is_async and options.io_mode == .evented) { - _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined; - } - return result; - }, - .OBJECT_NAME_INVALID => unreachable, - .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, - .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, - .NO_MEDIA_IN_DEVICE => return error.NoDevice, - .INVALID_PARAMETER => unreachable, - .SHARING_VIOLATION => { - if (options.share_access_nonblocking) { - return error.WouldBlock; - } - // TODO sleep in a way that is interruptable - // TODO integrate with async I/O - std.time.sleep(delay); - if (delay < 1 * std.time.ns_per_s) { - delay *= 2; - } - continue; - }, - .ACCESS_DENIED => return error.AccessDenied, - .PIPE_BUSY => return error.PipeBusy, - .OBJECT_PATH_SYNTAX_BAD => unreachable, - .OBJECT_NAME_COLLISION => return error.PathAlreadyExists, - .FILE_IS_A_DIRECTORY => return error.IsDir, - .NOT_A_DIRECTORY => return error.NotDir, - else => return unexpectedStatus(rc), - } + const rc = ntdll.NtCreateFile( + &result, + options.access_mask, + &attr, + &io, + null, + FILE_ATTRIBUTE_NORMAL, + options.share_access, + options.creation, + flags, + null, + 0, + ); + switch (rc) { + .SUCCESS => { + if (std.io.is_async and options.io_mode == .evented) { + _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined; + } + return result; + }, + .OBJECT_NAME_INVALID => unreachable, + .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, + .NO_MEDIA_IN_DEVICE => return error.NoDevice, + .INVALID_PARAMETER => unreachable, + .SHARING_VIOLATION => return error.AccessDenied, + .ACCESS_DENIED => return error.AccessDenied, + .PIPE_BUSY => return error.PipeBusy, + .OBJECT_PATH_SYNTAX_BAD => unreachable, + .OBJECT_NAME_COLLISION => return error.PathAlreadyExists, + .FILE_IS_A_DIRECTORY => return error.IsDir, + .NOT_A_DIRECTORY => return error.NotDir, + else => return unexpectedStatus(rc), } } @@ -1679,6 +1662,64 @@ pub fn SetFileTime( } } +pub const LockFileError = error{ + SystemResources, + WouldBlock, +} || std.os.UnexpectedError; + +pub fn LockFile( + FileHandle: HANDLE, + Event: ?HANDLE, + ApcRoutine: ?*IO_APC_ROUTINE, + ApcContext: ?*c_void, + IoStatusBlock: *IO_STATUS_BLOCK, + ByteOffset: *const LARGE_INTEGER, + Length: *const LARGE_INTEGER, + Key: ?*ULONG, + FailImmediately: BOOLEAN, + ExclusiveLock: BOOLEAN, +) !void { + const rc = ntdll.NtLockFile( + FileHandle, + Event, + ApcRoutine, + ApcContext, + IoStatusBlock, + ByteOffset, + Length, + Key, + FailImmediately, + ExclusiveLock, + ); + switch (rc) { + .SUCCESS => return, + .INSUFFICIENT_RESOURCES => return error.SystemResources, + .LOCK_NOT_GRANTED => return error.WouldBlock, + .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer + else => return unexpectedStatus(rc), + } +} + +pub const UnlockFileError = error{ + RangeNotLocked, +} || std.os.UnexpectedError; + +pub fn UnlockFile( + FileHandle: HANDLE, + IoStatusBlock: *IO_STATUS_BLOCK, + ByteOffset: *const LARGE_INTEGER, + Length: *const LARGE_INTEGER, + Key: ?*ULONG, +) !void { + const rc = ntdll.NtUnlockFile(FileHandle, IoStatusBlock, ByteOffset, Length, Key); + switch (rc) { + .SUCCESS => return, + .RANGE_NOT_LOCKED => return error.RangeNotLocked, + .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer + else => return unexpectedStatus(rc), + } +} + pub fn teb() *TEB { return switch (builtin.target.cpu.arch) { .i386 => asm volatile ( diff --git a/lib/std/os/windows/ntdll.zig b/lib/std/os/windows/ntdll.zig index aa7e382ee0..ddc08e4bb2 100644 --- a/lib/std/os/windows/ntdll.zig +++ b/lib/std/os/windows/ntdll.zig @@ -139,3 +139,24 @@ pub extern "NtDll" fn RtlWaitOnAddress( AddressSize: SIZE_T, Timeout: ?*const LARGE_INTEGER, ) callconv(WINAPI) NTSTATUS; + +pub extern "NtDll" fn NtLockFile( + FileHandle: HANDLE, + Event: ?HANDLE, + ApcRoutine: ?*IO_APC_ROUTINE, + ApcContext: ?*c_void, + IoStatusBlock: *IO_STATUS_BLOCK, + ByteOffset: *const LARGE_INTEGER, + Length: *const LARGE_INTEGER, + Key: ?*ULONG, + FailImmediately: BOOLEAN, + ExclusiveLock: BOOLEAN, +) callconv(WINAPI) NTSTATUS; + +pub extern "NtDll" fn NtUnlockFile( + FileHandle: HANDLE, + IoStatusBlock: *IO_STATUS_BLOCK, + ByteOffset: *const LARGE_INTEGER, + Length: *const LARGE_INTEGER, + Key: ?*ULONG, +) callconv(WINAPI) NTSTATUS; diff --git a/src/Cache.zig b/src/Cache.zig index 6d20ca6367..94c1b41c61 100644 --- a/src/Cache.zig +++ b/src/Cache.zig @@ -181,6 +181,12 @@ pub const Manifest = struct { hash: HashHelper, manifest_file: ?fs.File, manifest_dirty: bool, + /// Set this flag to true before calling hit() in order to indicate that + /// upon a cache hit, the code using the cache will not modify the files + /// within the cache directory. This allows multiple processes to utilize + /// the same cache directory at the same time. + want_shared_lock: bool = true, + have_exclusive_lock: bool = false, files: std.ArrayListUnmanaged(File) = .{}, hex_digest: [hex_digest_len]u8, /// Populated when hit() returns an error because of one @@ -257,7 +263,9 @@ pub const Manifest = struct { /// /// This function will also acquire an exclusive lock to the manifest file. This means /// that a process holding a Manifest will block any other process attempting to - /// acquire the lock. + /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the + /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest + /// file to be locked in exclusive mode. /// /// The lock on the manifest file is released when `deinit` is called. As another /// option, one may call `toOwnedLock` to obtain a smaller object which can represent @@ -285,31 +293,62 @@ pub const Manifest = struct { mem.copy(u8, &manifest_file_path, &self.hex_digest); manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*; - if (self.files.items.len != 0) { - self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ + if (self.files.items.len == 0) { + // If there are no file inputs, we check if the manifest file exists instead of + // comparing the hashes on the files used for the cached item + while (true) { + if (self.cache.manifest_dir.openFile(&manifest_file_path, .{ + .read = true, + .write = true, + .lock = .Exclusive, + .lock_nonblocking = self.want_shared_lock, + })) |manifest_file| { + self.manifest_file = manifest_file; + self.have_exclusive_lock = true; + break; + } else |open_err| switch (open_err) { + error.WouldBlock => { + self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{ + .lock = .Shared, + }); + break; + }, + error.FileNotFound => { + if (self.cache.manifest_dir.createFile(&manifest_file_path, .{ + .read = true, + .truncate = false, + .lock = .Exclusive, + .lock_nonblocking = self.want_shared_lock, + })) |manifest_file| { + self.manifest_file = manifest_file; + self.manifest_dirty = true; + self.have_exclusive_lock = true; + return false; // cache miss; exclusive lock already held + } else |err| switch (err) { + error.WouldBlock => continue, + else => |e| return e, + } + }, + else => |e| return e, + } + } + } else { + if (self.cache.manifest_dir.createFile(&manifest_file_path, .{ .read = true, .truncate = false, .lock = .Exclusive, - }); - } else { - // If there are no file inputs, we check if the manifest file exists instead of - // comparing the hashes on the files used for the cached item - self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{ - .read = true, - .write = true, - .lock = .Exclusive, - }) catch |err| switch (err) { - error.FileNotFound => { - self.manifest_dirty = true; - self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ - .read = true, - .truncate = false, - .lock = .Exclusive, + .lock_nonblocking = self.want_shared_lock, + })) |manifest_file| { + self.manifest_file = manifest_file; + self.have_exclusive_lock = true; + } else |err| switch (err) { + error.WouldBlock => { + self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{ + .lock = .Shared, }); - return false; }, else => |e| return e, - }; + } } const file_contents = try self.manifest_file.?.reader().readAllAlloc(self.cache.gpa, manifest_file_size_max); @@ -360,7 +399,10 @@ pub const Manifest = struct { } const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch |err| switch (err) { - error.FileNotFound => return false, + error.FileNotFound => { + try self.upgradeToExclusiveLock(); + return false; + }, else => return error.CacheUnavailable, }; defer this_file.close(); @@ -405,6 +447,7 @@ pub const Manifest = struct { // cache miss // keep the manifest file open self.unhit(bin_digest, input_file_count); + try self.upgradeToExclusiveLock(); return false; } @@ -417,9 +460,11 @@ pub const Manifest = struct { return err; }; } + try self.upgradeToExclusiveLock(); return false; } + try self.downgradeToSharedLock(); return true; } @@ -585,34 +630,58 @@ pub const Manifest = struct { return out_digest; } + /// If `want_shared_lock` is true, this function automatically downgrades the + /// lock from exclusive to shared. pub fn writeManifest(self: *Manifest) !void { const manifest_file = self.manifest_file.?; - if (!self.manifest_dirty) return; + if (self.manifest_dirty) { + self.manifest_dirty = false; - var contents = std.ArrayList(u8).init(self.cache.gpa); - defer contents.deinit(); + var contents = std.ArrayList(u8).init(self.cache.gpa); + defer contents.deinit(); - const writer = contents.writer(); - var encoded_digest: [hex_digest_len]u8 = undefined; + const writer = contents.writer(); + var encoded_digest: [hex_digest_len]u8 = undefined; - for (self.files.items) |file| { - _ = std.fmt.bufPrint( - &encoded_digest, - "{s}", - .{std.fmt.fmtSliceHexLower(&file.bin_digest)}, - ) catch unreachable; - try writer.print("{d} {d} {d} {s} {s}\n", .{ - file.stat.size, - file.stat.inode, - file.stat.mtime, - &encoded_digest, - file.path, - }); + for (self.files.items) |file| { + _ = std.fmt.bufPrint( + &encoded_digest, + "{s}", + .{std.fmt.fmtSliceHexLower(&file.bin_digest)}, + ) catch unreachable; + try writer.print("{d} {d} {d} {s} {s}\n", .{ + file.stat.size, + file.stat.inode, + file.stat.mtime, + &encoded_digest, + file.path, + }); + } + + try manifest_file.setEndPos(contents.items.len); + try manifest_file.pwriteAll(contents.items, 0); } - try manifest_file.setEndPos(contents.items.len); - try manifest_file.pwriteAll(contents.items, 0); - self.manifest_dirty = false; + if (self.want_shared_lock) { + try self.downgradeToSharedLock(); + } + } + + fn downgradeToSharedLock(self: *Manifest) !void { + if (!self.have_exclusive_lock) return; + const manifest_file = self.manifest_file.?; + try manifest_file.downgradeLock(); + self.have_exclusive_lock = false; + } + + fn upgradeToExclusiveLock(self: *Manifest) !void { + if (self.have_exclusive_lock) return; + const manifest_file = self.manifest_file.?; + // Here we intentionally have a period where the lock is released, in case there are + // other processes holding a shared lock. + manifest_file.unlock(); + try manifest_file.lock(.Exclusive); + self.have_exclusive_lock = true; } /// Obtain only the data needed to maintain a lock on the manifest file. @@ -881,27 +950,27 @@ test "no file inputs" { defer cache.manifest_dir.close(); { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.addBytes("1234"); + man.hash.addBytes("1234"); // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit()); + try testing.expectEqual(false, try man.hit()); - digest1 = ch.final(); + digest1 = man.final(); - try ch.writeManifest(); + try man.writeManifest(); } { - var ch = cache.obtain(); - defer ch.deinit(); + var man = cache.obtain(); + defer man.deinit(); - ch.hash.addBytes("1234"); + man.hash.addBytes("1234"); - try testing.expect(try ch.hit()); - digest2 = ch.final(); - try ch.writeManifest(); + try testing.expect(try man.hit()); + digest2 = man.final(); + try man.writeManifest(); } try testing.expectEqual(digest1, digest2); diff --git a/src/Compilation.zig b/src/Compilation.zig index f202034242..10b2a9af11 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -39,7 +39,6 @@ gpa: *Allocator, arena_state: std.heap.ArenaAllocator.State, bin_file: *link.File, c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{}, -c_object_cache_digest_set: std.AutoHashMapUnmanaged(Cache.BinDigest, void) = .{}, stage1_lock: ?Cache.Lock = null, stage1_cache_manifest: *Cache.Manifest = undefined, @@ -1590,7 +1589,6 @@ pub fn destroy(self: *Compilation) void { key.destroy(gpa); } self.c_object_table.deinit(gpa); - self.c_object_cache_digest_set.deinit(gpa); for (self.failed_c_objects.values()) |value| { value.destroy(gpa); @@ -1627,7 +1625,6 @@ pub fn update(self: *Compilation) !void { defer tracy.end(); self.clearMiscFailures(); - self.c_object_cache_digest_set.clearRetainingCapacity(); // For compiling C objects, we rely on the cache hash system to avoid duplicating work. // Add a Job for each C object. @@ -2615,25 +2612,6 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P try man.hashCSource(c_object.src); - { - const is_collision = blk: { - const bin_digest = man.hash.peekBin(); - - const lock = comp.mutex.acquire(); - defer lock.release(); - - const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, bin_digest); - break :blk gop.found_existing; - }; - if (is_collision) { - return comp.failCObj( - c_object, - "the same source file was already added to the same compilation with the same flags", - .{}, - ); - } - } - var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); defer arena_allocator.deinit(); const arena = &arena_allocator.allocator;