mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2026-02-09 19:10:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9258 from ziglang/shared-cache-locking

Browse Source 
Shared Cache Locking
This commit is contained in:
Andrew Kelley 2021-06-29 17:29:59 -04:00 committed by GitHub
commit 37fbf5b0d3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 487 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
doc/langref.html.in
View File

@ -10713,6 +10713,14 @@ fn readU32Be() u32 {}
See the Zig Standard Library for more examples.
</p>
{#header_close#}
{#header_open|Doc Comment Guidance#}
<ul>
<li>Omit any information that is redundant based on the name of the thing being documented.</li>
<li>Duplicating information onto multiple similar functions is encouraged because it helps IDEs and other tools provide better help text.</li>
<li>Use the word <strong>assume</strong> to indicate invariants that cause {#link|Undefined Behavior#} when violated.</li>
<li>Use the word <strong>assert</strong> to indicate invariants that cause <em>safety-checked</em> {#link|Undefined Behavior#} when violated.</li>
</ul>
{#header_close#}
{#header_close#}
{#header_open|Source Encoding#}
<p>Zig source code is encoded in UTF-8. An invalid UTF-8 byte sequence results in a compile error.</p>

64
lib/std/fs.zig
View File

@ -883,24 +883,39 @@ pub const Dir = struct {
/// [WTF-16](https://simonsapin.github.io/wtf-8/#potentially-ill-formed-utf-16) encoded.
pub fn openFileW(self: Dir, sub_path_w: []const u16, flags: File.OpenFlags) File.OpenError!File {
const w = os.windows;
return @as(File, .{
.handle = try os.windows.OpenFile(sub_path_w, .{
const file: File = .{
.handle = try w.OpenFile(sub_path_w, .{
.dir = self.fd,
.access_mask = w.SYNCHRONIZE |
(if (flags.read) @as(u32, w.GENERIC_READ) else 0) |
(if (flags.write) @as(u32, w.GENERIC_WRITE) else 0),
.share_access = switch (flags.lock) {
.None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
.Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
.Exclusive => w.FILE_SHARE_DELETE,
},
.share_access_nonblocking = flags.lock_nonblocking,
.creation = w.FILE_OPEN,
.io_mode = flags.intended_io_mode,
}),
.capable_io_mode = std.io.default_mode,
.intended_io_mode = flags.intended_io_mode,
});
};
var io: w.IO_STATUS_BLOCK = undefined;
const range_off: w.LARGE_INTEGER = 0;
const range_len: w.LARGE_INTEGER = 1;
const exclusive = switch (flags.lock) {
.None => return file,
.Shared => false,
.Exclusive => true,
};
try w.LockFile(
file.handle,
null,
null,
null,
&io,
&range_off,
&range_len,
null,
@boolToInt(flags.lock_nonblocking),
@boolToInt(exclusive),
);
return file;
}
/// Creates, opens, or overwrites a file with write access.
@ -1019,16 +1034,10 @@ pub const Dir = struct {
pub fn createFileW(self: Dir, sub_path_w: []const u16, flags: File.CreateFlags) File.OpenError!File {
const w = os.windows;
const read_flag = if (flags.read) @as(u32, w.GENERIC_READ) else 0;
return @as(File, .{
const file: File = .{
.handle = try os.windows.OpenFile(sub_path_w, .{
.dir = self.fd,
.access_mask = w.SYNCHRONIZE | w.GENERIC_WRITE | read_flag,
.share_access = switch (flags.lock) {
.None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
.Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
.Exclusive => w.FILE_SHARE_DELETE,
},
.share_access_nonblocking = flags.lock_nonblocking,
.creation = if (flags.exclusive)
@as(u32, w.FILE_CREATE)
else if (flags.truncate)
@ -1039,7 +1048,28 @@ pub const Dir = struct {
}),
.capable_io_mode = std.io.default_mode,
.intended_io_mode = flags.intended_io_mode,
});
};
var io: w.IO_STATUS_BLOCK = undefined;
const range_off: w.LARGE_INTEGER = 0;
const range_len: w.LARGE_INTEGER = 1;
const exclusive = switch (flags.lock) {
.None => return file,
.Shared => false,
.Exclusive => true,
};
try w.LockFile(
file.handle,
null,
null,
null,
&io,
&range_off,
&range_len,
null,
@boolToInt(flags.lock_nonblocking),
@boolToInt(exclusive),
);
return file;
}
pub const openRead = @compileError("deprecated in favor of openFile");

211
lib/std/fs/file.zig
View File

@ -74,17 +74,28 @@ pub const File = struct {
read: bool = true,
write: bool = false,
/// Open the file with a lock to prevent other processes from accessing it at the
/// same time. An exclusive lock will prevent other processes from acquiring a lock.
/// A shared lock will prevent other processes from acquiring a exclusive lock, but
/// doesn't prevent other process from getting their own shared locks.
/// Open the file with an advisory lock to coordinate with other processes
/// accessing it at the same time. An exclusive lock will prevent other
/// processes from acquiring a lock. A shared lock will prevent other
/// processes from acquiring a exclusive lock, but does not prevent
/// other process from getting their own shared locks.
///
/// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1].
/// The lock is advisory, except on Linux in very specific cirsumstances[1].
/// This means that a process that does not respect the locking API can still get access
/// to the file, despite the lock.
///
/// Windows' file locks are mandatory, and any process attempting to access the file will
/// receive an error.
/// On these operating systems, the lock is acquired atomically with
/// opening the file:
/// * Darwin
/// * DragonFlyBSD
/// * FreeBSD
/// * Haiku
/// * NetBSD
/// * OpenBSD
/// On these operating systems, the lock is acquired via a separate syscall
/// after opening the file:
/// * Linux
/// * Windows
///
/// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt
lock: Lock = .None,
@ -120,17 +131,28 @@ pub const File = struct {
/// `error.PathAlreadyExists` to be returned.
exclusive: bool = false,
/// Open the file with a lock to prevent other processes from accessing it at the
/// same time. An exclusive lock will prevent other processes from acquiring a lock.
/// A shared lock will prevent other processes from acquiring a exclusive lock, but
/// doesn't prevent other process from getting their own shared locks.
/// Open the file with an advisory lock to coordinate with other processes
/// accessing it at the same time. An exclusive lock will prevent other
/// processes from acquiring a lock. A shared lock will prevent other
/// processes from acquiring a exclusive lock, but does not prevent
/// other process from getting their own shared locks.
///
/// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1].
/// The lock is advisory, except on Linux in very specific cirsumstances[1].
/// This means that a process that does not respect the locking API can still get access
/// to the file, despite the lock.
///
/// Windows's file locks are mandatory, and any process attempting to access the file will
/// receive an error.
/// On these operating systems, the lock is acquired atomically with
/// opening the file:
/// * Darwin
/// * DragonFlyBSD
/// * FreeBSD
/// * Haiku
/// * NetBSD
/// * OpenBSD
/// On these operating systems, the lock is acquired via a separate syscall
/// after opening the file:
/// * Linux
/// * Windows
///
/// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt
lock: Lock = .None,
@ -829,4 +851,165 @@ pub const File = struct {
pub fn seekableStream(file: File) SeekableStream {
return .{ .context = file };
}
const range_off: windows.LARGE_INTEGER = 0;
const range_len: windows.LARGE_INTEGER = 1;
pub const LockError = error{
SystemResources,
} || os.UnexpectedError;
/// Blocks when an incompatible lock is held by another process.
/// A process may hold only one type of lock (shared or exclusive) on
/// a file. When a process terminates in any way, the lock is released.
///
/// Assumes the file is unlocked.
///
/// TODO: integrate with async I/O
pub fn lock(file: File, l: Lock) LockError!void {
if (is_windows) {
var io_status_block: windows.IO_STATUS_BLOCK = undefined;
const exclusive = switch (l) {
.None => return,
.Shared => false,
.Exclusive => true,
};
return windows.LockFile(
file.handle,
null,
null,
null,
&io_status_block,
&range_off,
&range_len,
null,
windows.FALSE, // non-blocking=false
@boolToInt(exclusive),
) catch |err| switch (err) {
error.WouldBlock => unreachable, // non-blocking=false
else => |e| return e,
};
} else {
return os.flock(file.handle, switch (l) {
.None => os.LOCK_UN,
.Shared => os.LOCK_SH,
.Exclusive => os.LOCK_EX,
}) catch |err| switch (err) {
error.WouldBlock => unreachable, // non-blocking=false
else => |e| return e,
};
}
}
/// Assumes the file is locked.
pub fn unlock(file: File) void {
if (is_windows) {
var io_status_block: windows.IO_STATUS_BLOCK = undefined;
return windows.UnlockFile(
file.handle,
&io_status_block,
&range_off,
&range_len,
null,
) catch |err| switch (err) {
error.RangeNotLocked => unreachable, // Function assumes unlocked.
error.Unexpected => unreachable, // Resource deallocation must succeed.
};
} else {
return os.flock(file.handle, os.LOCK_UN) catch |err| switch (err) {
error.WouldBlock => unreachable, // unlocking can't block
error.SystemResources => unreachable, // We are deallocating resources.
error.Unexpected => unreachable, // Resource deallocation must succeed.
};
}
}
/// Attempts to obtain a lock, returning `true` if the lock is
/// obtained, and `false` if there was an existing incompatible lock held.
/// A process may hold only one type of lock (shared or exclusive) on
/// a file. When a process terminates in any way, the lock is released.
///
/// Assumes the file is unlocked.
///
/// TODO: integrate with async I/O
pub fn tryLock(file: File, l: Lock) LockError!bool {
if (is_windows) {
var io_status_block: windows.IO_STATUS_BLOCK = undefined;
const exclusive = switch (l) {
.None => return,
.Shared => false,
.Exclusive => true,
};
windows.LockFile(
file.handle,
null,
null,
null,
&io_status_block,
&range_off,
&range_len,
null,
windows.TRUE, // non-blocking=true
@boolToInt(exclusive),
) catch |err| switch (err) {
error.WouldBlock => return false,
else => |e| return e,
};
} else {
os.flock(file.handle, switch (l) {
.None => os.LOCK_UN,
.Shared => os.LOCK_SH | os.LOCK_NB,
.Exclusive => os.LOCK_EX | os.LOCK_NB,
}) catch |err| switch (err) {
error.WouldBlock => return false,
else => |e| return e,
};
}
return true;
}
/// Assumes the file is already locked in exclusive mode.
/// Atomically modifies the lock to be in shared mode, without releasing it.
///
/// TODO: integrate with async I/O
pub fn downgradeLock(file: File) LockError!void {
if (is_windows) {
// On Windows it works like a semaphore + exclusivity flag. To implement this
// function, we first obtain another lock in shared mode. This changes the
// exclusivity flag, but increments the semaphore to 2. So we follow up with
// an NtUnlockFile which decrements the semaphore but does not modify the
// exclusivity flag.
var io_status_block: windows.IO_STATUS_BLOCK = undefined;
windows.LockFile(
file.handle,
null,
null,
null,
&io_status_block,
&range_off,
&range_len,
null,
windows.TRUE, // non-blocking=true
windows.FALSE, // exclusive=false
) catch |err| switch (err) {
error.WouldBlock => unreachable, // File was not locked in exclusive mode.
else => |e| return e,
};
return windows.UnlockFile(
file.handle,
&io_status_block,
&range_off,
&range_len,
null,
) catch |err| switch (err) {
error.RangeNotLocked => unreachable, // File was not locked.
error.Unexpected => unreachable, // Resource deallocation must succeed.
};
} else {
return os.flock(file.handle, os.LOCK_SH | os.LOCK_NB) catch |err| switch (err) {
error.WouldBlock => unreachable, // File was not locked in exclusive mode.
else => |e| return e,
};
}
}
};

141
lib/std/os/windows.zig
View File

@ -49,7 +49,6 @@ pub const OpenFileOptions = struct {
dir: ?HANDLE = null,
sa: ?*SECURITY_ATTRIBUTES = null,
share_access: ULONG = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE,
share_access_nonblocking: bool = false,
creation: ULONG,
io_mode: std.io.ModeOverride,
/// If true, tries to open path as a directory.
@ -60,8 +59,6 @@ pub const OpenFileOptions = struct {
follow_symlinks: bool = true,
};
/// TODO when share_access_nonblocking is false, this implementation uses
/// untinterruptible sleep() to block. This is not the final iteration of the API.
pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE {
if (mem.eql(u16, sub_path_w, &[_]u16{'.'}) and !options.open_dir) {
return error.IsDir;
@ -94,53 +91,39 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN
// If we're not following symlinks, we need to ensure we don't pass in any synchronization flags such as FILE_SYNCHRONOUS_IO_NONALERT.
const flags: ULONG = if (options.follow_symlinks) file_or_dir_flag | blocking_flag else file_or_dir_flag | FILE_OPEN_REPARSE_POINT;
var delay: usize = 1;
while (true) {
const rc = ntdll.NtCreateFile(
&result,
options.access_mask,
&attr,
&io,
null,
FILE_ATTRIBUTE_NORMAL,
options.share_access,
options.creation,
flags,
null,
0,
);
switch (rc) {
.SUCCESS => {
if (std.io.is_async and options.io_mode == .evented) {
_ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
}
return result;
},
.OBJECT_NAME_INVALID => unreachable,
.OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
.OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
.NO_MEDIA_IN_DEVICE => return error.NoDevice,
.INVALID_PARAMETER => unreachable,
.SHARING_VIOLATION => {
if (options.share_access_nonblocking) {
return error.WouldBlock;
}
// TODO sleep in a way that is interruptable
// TODO integrate with async I/O
std.time.sleep(delay);
if (delay < 1 * std.time.ns_per_s) {
delay *= 2;
}
continue;
},
.ACCESS_DENIED => return error.AccessDenied,
.PIPE_BUSY => return error.PipeBusy,
.OBJECT_PATH_SYNTAX_BAD => unreachable,
.OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
.FILE_IS_A_DIRECTORY => return error.IsDir,
.NOT_A_DIRECTORY => return error.NotDir,
else => return unexpectedStatus(rc),
}
const rc = ntdll.NtCreateFile(
&result,
options.access_mask,
&attr,
&io,
null,
FILE_ATTRIBUTE_NORMAL,
options.share_access,
options.creation,
flags,
null,
0,
);
switch (rc) {
.SUCCESS => {
if (std.io.is_async and options.io_mode == .evented) {
_ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
}
return result;
},
.OBJECT_NAME_INVALID => unreachable,
.OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
.OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
.NO_MEDIA_IN_DEVICE => return error.NoDevice,
.INVALID_PARAMETER => unreachable,
.SHARING_VIOLATION => return error.AccessDenied,
.ACCESS_DENIED => return error.AccessDenied,
.PIPE_BUSY => return error.PipeBusy,
.OBJECT_PATH_SYNTAX_BAD => unreachable,
.OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
.FILE_IS_A_DIRECTORY => return error.IsDir,
.NOT_A_DIRECTORY => return error.NotDir,
else => return unexpectedStatus(rc),
}
}
@ -1679,6 +1662,64 @@ pub fn SetFileTime(
}
}
pub const LockFileError = error{
SystemResources,
WouldBlock,
} || std.os.UnexpectedError;
pub fn LockFile(
FileHandle: HANDLE,
Event: ?HANDLE,
ApcRoutine: ?*IO_APC_ROUTINE,
ApcContext: ?*c_void,
IoStatusBlock: *IO_STATUS_BLOCK,
ByteOffset: *const LARGE_INTEGER,
Length: *const LARGE_INTEGER,
Key: ?*ULONG,
FailImmediately: BOOLEAN,
ExclusiveLock: BOOLEAN,
) !void {
const rc = ntdll.NtLockFile(
FileHandle,
Event,
ApcRoutine,
ApcContext,
IoStatusBlock,
ByteOffset,
Length,
Key,
FailImmediately,
ExclusiveLock,
);
switch (rc) {
.SUCCESS => return,
.INSUFFICIENT_RESOURCES => return error.SystemResources,
.LOCK_NOT_GRANTED => return error.WouldBlock,
.ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
else => return unexpectedStatus(rc),
}
}
pub const UnlockFileError = error{
RangeNotLocked,
} || std.os.UnexpectedError;
pub fn UnlockFile(
FileHandle: HANDLE,
IoStatusBlock: *IO_STATUS_BLOCK,
ByteOffset: *const LARGE_INTEGER,
Length: *const LARGE_INTEGER,
Key: ?*ULONG,
) !void {
const rc = ntdll.NtUnlockFile(FileHandle, IoStatusBlock, ByteOffset, Length, Key);
switch (rc) {
.SUCCESS => return,
.RANGE_NOT_LOCKED => return error.RangeNotLocked,
.ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
else => return unexpectedStatus(rc),
}
}
pub fn teb() *TEB {
return switch (builtin.target.cpu.arch) {
.i386 => asm volatile (

21
lib/std/os/windows/ntdll.zig
View File

@ -139,3 +139,24 @@ pub extern "NtDll" fn RtlWaitOnAddress(
AddressSize: SIZE_T,
Timeout: ?*const LARGE_INTEGER,
) callconv(WINAPI) NTSTATUS;
pub extern "NtDll" fn NtLockFile(
FileHandle: HANDLE,
Event: ?HANDLE,
ApcRoutine: ?*IO_APC_ROUTINE,
ApcContext: ?*c_void,
IoStatusBlock: *IO_STATUS_BLOCK,
ByteOffset: *const LARGE_INTEGER,
Length: *const LARGE_INTEGER,
Key: ?*ULONG,
FailImmediately: BOOLEAN,
ExclusiveLock: BOOLEAN,
) callconv(WINAPI) NTSTATUS;
pub extern "NtDll" fn NtUnlockFile(
FileHandle: HANDLE,
IoStatusBlock: *IO_STATUS_BLOCK,
ByteOffset: *const LARGE_INTEGER,
Length: *const LARGE_INTEGER,
Key: ?*ULONG,
) callconv(WINAPI) NTSTATUS;

177
src/Cache.zig
View File

@ -181,6 +181,12 @@ pub const Manifest = struct {
hash: HashHelper,
manifest_file: ?fs.File,
manifest_dirty: bool,
/// Set this flag to true before calling hit() in order to indicate that
/// upon a cache hit, the code using the cache will not modify the files
/// within the cache directory. This allows multiple processes to utilize
/// the same cache directory at the same time.
want_shared_lock: bool = true,
have_exclusive_lock: bool = false,
files: std.ArrayListUnmanaged(File) = .{},
hex_digest: [hex_digest_len]u8,
/// Populated when hit() returns an error because of one
@ -257,7 +263,9 @@ pub const Manifest = struct {
///
/// This function will also acquire an exclusive lock to the manifest file. This means
/// that a process holding a Manifest will block any other process attempting to
/// acquire the lock.
/// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
/// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
/// file to be locked in exclusive mode.
///
/// The lock on the manifest file is released when `deinit` is called. As another
/// option, one may call `toOwnedLock` to obtain a smaller object which can represent
@ -285,31 +293,62 @@ pub const Manifest = struct {
mem.copy(u8, &manifest_file_path, &self.hex_digest);
manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
if (self.files.items.len != 0) {
self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
if (self.files.items.len == 0) {
// If there are no file inputs, we check if the manifest file exists instead of
// comparing the hashes on the files used for the cached item
while (true) {
if (self.cache.manifest_dir.openFile(&manifest_file_path, .{
.read = true,
.write = true,
.lock = .Exclusive,
.lock_nonblocking = self.want_shared_lock,
})) |manifest_file| {
self.manifest_file = manifest_file;
self.have_exclusive_lock = true;
break;
} else |open_err| switch (open_err) {
error.WouldBlock => {
self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
.lock = .Shared,
});
break;
},
error.FileNotFound => {
if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
.read = true,
.truncate = false,
.lock = .Exclusive,
.lock_nonblocking = self.want_shared_lock,
})) |manifest_file| {
self.manifest_file = manifest_file;
self.manifest_dirty = true;
self.have_exclusive_lock = true;
return false; // cache miss; exclusive lock already held
} else |err| switch (err) {
error.WouldBlock => continue,
else => |e| return e,
}
},
else => |e| return e,
}
}
} else {
if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
.read = true,
.truncate = false,
.lock = .Exclusive,
});
} else {
// If there are no file inputs, we check if the manifest file exists instead of
// comparing the hashes on the files used for the cached item
self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{
.read = true,
.write = true,
.lock = .Exclusive,
}) catch |err| switch (err) {
error.FileNotFound => {
self.manifest_dirty = true;
self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
.read = true,
.truncate = false,
.lock = .Exclusive,
.lock_nonblocking = self.want_shared_lock,
})) |manifest_file| {
self.manifest_file = manifest_file;
self.have_exclusive_lock = true;
} else |err| switch (err) {
error.WouldBlock => {
self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
.lock = .Shared,
});
return false;
},
else => |e| return e,
};
}
}
const file_contents = try self.manifest_file.?.reader().readAllAlloc(self.cache.gpa, manifest_file_size_max);
@ -360,7 +399,10 @@ pub const Manifest = struct {
}
const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch |err| switch (err) {
error.FileNotFound => return false,
error.FileNotFound => {
try self.upgradeToExclusiveLock();
return false;
},
else => return error.CacheUnavailable,
};
defer this_file.close();
@ -405,6 +447,7 @@ pub const Manifest = struct {
// cache miss
// keep the manifest file open
self.unhit(bin_digest, input_file_count);
try self.upgradeToExclusiveLock();
return false;
}
@ -417,9 +460,11 @@ pub const Manifest = struct {
return err;
};
}
try self.upgradeToExclusiveLock();
return false;
}
try self.downgradeToSharedLock();
return true;
}
@ -585,34 +630,58 @@ pub const Manifest = struct {
return out_digest;
}
/// If `want_shared_lock` is true, this function automatically downgrades the
/// lock from exclusive to shared.
pub fn writeManifest(self: *Manifest) !void {
const manifest_file = self.manifest_file.?;
if (!self.manifest_dirty) return;
if (self.manifest_dirty) {
self.manifest_dirty = false;
var contents = std.ArrayList(u8).init(self.cache.gpa);
defer contents.deinit();
var contents = std.ArrayList(u8).init(self.cache.gpa);
defer contents.deinit();
const writer = contents.writer();
var encoded_digest: [hex_digest_len]u8 = undefined;
const writer = contents.writer();
var encoded_digest: [hex_digest_len]u8 = undefined;
for (self.files.items) |file| {
_ = std.fmt.bufPrint(
&encoded_digest,
"{s}",
.{std.fmt.fmtSliceHexLower(&file.bin_digest)},
) catch unreachable;
try writer.print("{d} {d} {d} {s} {s}\n", .{
file.stat.size,
file.stat.inode,
file.stat.mtime,
&encoded_digest,
file.path,
});
for (self.files.items) |file| {
_ = std.fmt.bufPrint(
&encoded_digest,
"{s}",
.{std.fmt.fmtSliceHexLower(&file.bin_digest)},
) catch unreachable;
try writer.print("{d} {d} {d} {s} {s}\n", .{
file.stat.size,
file.stat.inode,
file.stat.mtime,
&encoded_digest,
file.path,
});
}
try manifest_file.setEndPos(contents.items.len);
try manifest_file.pwriteAll(contents.items, 0);
}
try manifest_file.setEndPos(contents.items.len);
try manifest_file.pwriteAll(contents.items, 0);
self.manifest_dirty = false;
if (self.want_shared_lock) {
try self.downgradeToSharedLock();
}
}
fn downgradeToSharedLock(self: *Manifest) !void {
if (!self.have_exclusive_lock) return;
const manifest_file = self.manifest_file.?;
try manifest_file.downgradeLock();
self.have_exclusive_lock = false;
}
fn upgradeToExclusiveLock(self: *Manifest) !void {
if (self.have_exclusive_lock) return;
const manifest_file = self.manifest_file.?;
// Here we intentionally have a period where the lock is released, in case there are
// other processes holding a shared lock.
manifest_file.unlock();
try manifest_file.lock(.Exclusive);
self.have_exclusive_lock = true;
}
/// Obtain only the data needed to maintain a lock on the manifest file.
@ -881,27 +950,27 @@ test "no file inputs" {
defer cache.manifest_dir.close();
{
var ch = cache.obtain();
defer ch.deinit();
var man = cache.obtain();
defer man.deinit();
ch.hash.addBytes("1234");
man.hash.addBytes("1234");
// There should be nothing in the cache
try testing.expectEqual(false, try ch.hit());
try testing.expectEqual(false, try man.hit());
digest1 = ch.final();
digest1 = man.final();
try ch.writeManifest();
try man.writeManifest();
}
{
var ch = cache.obtain();
defer ch.deinit();
var man = cache.obtain();
defer man.deinit();
ch.hash.addBytes("1234");
man.hash.addBytes("1234");
try testing.expect(try ch.hit());
digest2 = ch.final();
try ch.writeManifest();
try testing.expect(try man.hit());
digest2 = man.final();
try man.writeManifest();
}
try testing.expectEqual(digest1, digest2);

22
src/Compilation.zig
View File

@ -39,7 +39,6 @@ gpa: *Allocator,
arena_state: std.heap.ArenaAllocator.State,
bin_file: *link.File,
c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
c_object_cache_digest_set: std.AutoHashMapUnmanaged(Cache.BinDigest, void) = .{},
stage1_lock: ?Cache.Lock = null,
stage1_cache_manifest: *Cache.Manifest = undefined,
@ -1590,7 +1589,6 @@ pub fn destroy(self: *Compilation) void {
key.destroy(gpa);
}
self.c_object_table.deinit(gpa);
self.c_object_cache_digest_set.deinit(gpa);
for (self.failed_c_objects.values()) |value| {
value.destroy(gpa);
@ -1627,7 +1625,6 @@ pub fn update(self: *Compilation) !void {
defer tracy.end();
self.clearMiscFailures();
self.c_object_cache_digest_set.clearRetainingCapacity();
// For compiling C objects, we rely on the cache hash system to avoid duplicating work.
// Add a Job for each C object.
@ -2615,25 +2612,6 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P
try man.hashCSource(c_object.src);
{
const is_collision = blk: {
const bin_digest = man.hash.peekBin();
const lock = comp.mutex.acquire();
defer lock.release();
const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, bin_digest);
break :blk gop.found_existing;
};
if (is_collision) {
return comp.failCObj(
c_object,
"the same source file was already added to the same compilation with the same flags",
.{},
);
}
}
var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
defer arena_allocator.deinit();
const arena = &arena_allocator.allocator;