adrien/ZipponDB
1
0
Fork 0
Code Actions Releases Activity

Check if additional data count is reached at each iteration

Browse Source 
Before I was checking after the filter.evaluate is true. So I can end up
with the limit reach, but the thread will continue parsing until it find
one that is true, add it and then check if the limit is reach.

Which mean I endup with more than what I want and the thread could stop
before, making things faster.
This commit is contained in:
Adrien Bouvais 2024-11-17 20:45:58 +01:00
parent 65ba0263cc
commit 7b2a754d81
3 changed files with 22 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/fileEngine.zig
View File

@ -279,7 +279,7 @@ pub const FileEngine = struct {
}
/// Use a struct name and filter to populate a map with all UUID bytes as key and void as value
/// This map is use as value for the link array, so I can do a `contains` on it.
/// This map is use as value for the ConditionValue of links, so I can do a `contains` on it.
pub fn populateVoidUUIDMap(
self: *FileEngine,
struct_name: []const u8,
@ -330,6 +330,16 @@ pub const FileEngine = struct {
for (thread_writer_list) |list| {
for (list.items) |uuid| _ = map.getOrPut(uuid) catch return ZipponError.MemoryError;
}
if (additional_data.entity_count_to_find == 0) return;
if (map.count() > additional_data.entity_count_to_find) {
log.err("Found {d} entity in populateVoidUUIDMap but max is: {d}", .{ map.count(), additional_data.entity_count_to_find });
var iter = map.iterator();
while (iter.next()) |entry| {
log.debug("{s}", .{UUID.format_bytes(entry.key_ptr.bytes)});
}
}
}
fn populateVoidUUIDMapOneFile(
@ -358,6 +368,7 @@ pub const FileEngine = struct {
defer iter.deinit();
while (iter.next() catch return) |row| {
if (sync_context.checkStructLimit()) break;
if (filter == null or filter.?.evaluate(row)) {
list.*.append(UUID{ .bytes = row[0].UUID }) catch |err| {
sync_context.logError("Error initializing DataIterator", err);
@ -474,6 +485,7 @@ pub const FileEngine = struct {
sync_context.logError("Error in iter next", err);
return;
}) |row| {
if (sync_context.checkStructLimit()) break;
if (filter) |f| if (!f.evaluate(row)) continue;
writeEntity(
@ -719,6 +731,7 @@ pub const FileEngine = struct {
sync_context.logError("Parsing files", err);
return;
}) |row| {
if (sync_context.checkStructLimit()) break;
if (filter == null or filter.?.evaluate(row)) {
// Add the unchanged Data in the new_data_buff
new_data_buff[0] = row[0];
@ -872,6 +885,7 @@ pub const FileEngine = struct {
sync_context.logError("Error during iter", err);
return;
}) |row| {
if (sync_context.checkStructLimit()) break;
if (filter == null or filter.?.evaluate(row)) {
writer.print("{{\"{s}\"}},", .{UUID.format_bytes(row[0].UUID)}) catch |err| {
sync_context.logError("Error writting", err);

1
src/schemaEngine.zig
View File

@ -125,7 +125,6 @@ pub const SchemaEngine = struct {
/// Chech if the name of a struct is in the current schema
pub fn isStructNameExists(self: *SchemaEngine, struct_name: []const u8) bool {
var i: u16 = 0;
log.debug("\n\n{any}\n\n", .{self.struct_array});
while (i < self.struct_array.len) : (i += 1) if (std.mem.eql(u8, self.struct_array[i].name, struct_name)) return true;
return false;
}

8
src/threadEngine.zig
View File

@ -34,7 +34,13 @@ pub const ThreadSyncContext = struct {
pub fn incrementAndCheckStructLimit(self: *ThreadSyncContext) bool {
if (self.max_struct == 0) return false;
const new_count = self.processed_struct.fetchAdd(1, .monotonic);
return new_count >= self.max_struct;
return (new_count + 1) >= self.max_struct;
}
pub fn checkStructLimit(self: *ThreadSyncContext) bool {
if (self.max_struct == 0) return false;
const count = self.processed_struct.load(.monotonic);
return (count) >= self.max_struct;
}
pub fn logError(self: *ThreadSyncContext, message: []const u8, err: anyerror) void {