Check if additional data count is reached at each iteration

Before I was checking after the filter.evaluate is true. So I can end up
with the limit reach, but the thread will continue parsing until it find
one that is true, add it and then check if the limit is reach.

Which mean I endup with more than what I want and the thread could stop
before, making things faster.
This commit is contained in:
Adrien Bouvais 2024-11-17 20:45:58 +01:00
parent 65ba0263cc
commit 7b2a754d81
3 changed files with 22 additions and 3 deletions

View File

@ -279,7 +279,7 @@ pub const FileEngine = struct {
} }
/// Use a struct name and filter to populate a map with all UUID bytes as key and void as value /// Use a struct name and filter to populate a map with all UUID bytes as key and void as value
/// This map is use as value for the link array, so I can do a `contains` on it. /// This map is use as value for the ConditionValue of links, so I can do a `contains` on it.
pub fn populateVoidUUIDMap( pub fn populateVoidUUIDMap(
self: *FileEngine, self: *FileEngine,
struct_name: []const u8, struct_name: []const u8,
@ -330,6 +330,16 @@ pub const FileEngine = struct {
for (thread_writer_list) |list| { for (thread_writer_list) |list| {
for (list.items) |uuid| _ = map.getOrPut(uuid) catch return ZipponError.MemoryError; for (list.items) |uuid| _ = map.getOrPut(uuid) catch return ZipponError.MemoryError;
} }
if (additional_data.entity_count_to_find == 0) return;
if (map.count() > additional_data.entity_count_to_find) {
log.err("Found {d} entity in populateVoidUUIDMap but max is: {d}", .{ map.count(), additional_data.entity_count_to_find });
var iter = map.iterator();
while (iter.next()) |entry| {
log.debug("{s}", .{UUID.format_bytes(entry.key_ptr.bytes)});
}
}
} }
fn populateVoidUUIDMapOneFile( fn populateVoidUUIDMapOneFile(
@ -358,6 +368,7 @@ pub const FileEngine = struct {
defer iter.deinit(); defer iter.deinit();
while (iter.next() catch return) |row| { while (iter.next() catch return) |row| {
if (sync_context.checkStructLimit()) break;
if (filter == null or filter.?.evaluate(row)) { if (filter == null or filter.?.evaluate(row)) {
list.*.append(UUID{ .bytes = row[0].UUID }) catch |err| { list.*.append(UUID{ .bytes = row[0].UUID }) catch |err| {
sync_context.logError("Error initializing DataIterator", err); sync_context.logError("Error initializing DataIterator", err);
@ -474,6 +485,7 @@ pub const FileEngine = struct {
sync_context.logError("Error in iter next", err); sync_context.logError("Error in iter next", err);
return; return;
}) |row| { }) |row| {
if (sync_context.checkStructLimit()) break;
if (filter) |f| if (!f.evaluate(row)) continue; if (filter) |f| if (!f.evaluate(row)) continue;
writeEntity( writeEntity(
@ -719,6 +731,7 @@ pub const FileEngine = struct {
sync_context.logError("Parsing files", err); sync_context.logError("Parsing files", err);
return; return;
}) |row| { }) |row| {
if (sync_context.checkStructLimit()) break;
if (filter == null or filter.?.evaluate(row)) { if (filter == null or filter.?.evaluate(row)) {
// Add the unchanged Data in the new_data_buff // Add the unchanged Data in the new_data_buff
new_data_buff[0] = row[0]; new_data_buff[0] = row[0];
@ -872,6 +885,7 @@ pub const FileEngine = struct {
sync_context.logError("Error during iter", err); sync_context.logError("Error during iter", err);
return; return;
}) |row| { }) |row| {
if (sync_context.checkStructLimit()) break;
if (filter == null or filter.?.evaluate(row)) { if (filter == null or filter.?.evaluate(row)) {
writer.print("{{\"{s}\"}},", .{UUID.format_bytes(row[0].UUID)}) catch |err| { writer.print("{{\"{s}\"}},", .{UUID.format_bytes(row[0].UUID)}) catch |err| {
sync_context.logError("Error writting", err); sync_context.logError("Error writting", err);

View File

@ -125,7 +125,6 @@ pub const SchemaEngine = struct {
/// Chech if the name of a struct is in the current schema /// Chech if the name of a struct is in the current schema
pub fn isStructNameExists(self: *SchemaEngine, struct_name: []const u8) bool { pub fn isStructNameExists(self: *SchemaEngine, struct_name: []const u8) bool {
var i: u16 = 0; var i: u16 = 0;
log.debug("\n\n{any}\n\n", .{self.struct_array});
while (i < self.struct_array.len) : (i += 1) if (std.mem.eql(u8, self.struct_array[i].name, struct_name)) return true; while (i < self.struct_array.len) : (i += 1) if (std.mem.eql(u8, self.struct_array[i].name, struct_name)) return true;
return false; return false;
} }

View File

@ -34,7 +34,13 @@ pub const ThreadSyncContext = struct {
pub fn incrementAndCheckStructLimit(self: *ThreadSyncContext) bool { pub fn incrementAndCheckStructLimit(self: *ThreadSyncContext) bool {
if (self.max_struct == 0) return false; if (self.max_struct == 0) return false;
const new_count = self.processed_struct.fetchAdd(1, .monotonic); const new_count = self.processed_struct.fetchAdd(1, .monotonic);
return new_count >= self.max_struct; return (new_count + 1) >= self.max_struct;
}
pub fn checkStructLimit(self: *ThreadSyncContext) bool {
if (self.max_struct == 0) return false;
const count = self.processed_struct.load(.monotonic);
return (count) >= self.max_struct;
} }
pub fn logError(self: *ThreadSyncContext, message: []const u8, err: anyerror) void { pub fn logError(self: *ThreadSyncContext, message: []const u8, err: anyerror) void {