Check if additional data count is reached at each iteration
Before I was checking after the filter.evaluate is true. So I can end up with the limit reach, but the thread will continue parsing until it find one that is true, add it and then check if the limit is reach. Which mean I endup with more than what I want and the thread could stop before, making things faster.
This commit is contained in:
parent
65ba0263cc
commit
7b2a754d81
@ -279,7 +279,7 @@ pub const FileEngine = struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Use a struct name and filter to populate a map with all UUID bytes as key and void as value
|
/// Use a struct name and filter to populate a map with all UUID bytes as key and void as value
|
||||||
/// This map is use as value for the link array, so I can do a `contains` on it.
|
/// This map is use as value for the ConditionValue of links, so I can do a `contains` on it.
|
||||||
pub fn populateVoidUUIDMap(
|
pub fn populateVoidUUIDMap(
|
||||||
self: *FileEngine,
|
self: *FileEngine,
|
||||||
struct_name: []const u8,
|
struct_name: []const u8,
|
||||||
@ -330,6 +330,16 @@ pub const FileEngine = struct {
|
|||||||
for (thread_writer_list) |list| {
|
for (thread_writer_list) |list| {
|
||||||
for (list.items) |uuid| _ = map.getOrPut(uuid) catch return ZipponError.MemoryError;
|
for (list.items) |uuid| _ = map.getOrPut(uuid) catch return ZipponError.MemoryError;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (additional_data.entity_count_to_find == 0) return;
|
||||||
|
|
||||||
|
if (map.count() > additional_data.entity_count_to_find) {
|
||||||
|
log.err("Found {d} entity in populateVoidUUIDMap but max is: {d}", .{ map.count(), additional_data.entity_count_to_find });
|
||||||
|
var iter = map.iterator();
|
||||||
|
while (iter.next()) |entry| {
|
||||||
|
log.debug("{s}", .{UUID.format_bytes(entry.key_ptr.bytes)});
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn populateVoidUUIDMapOneFile(
|
fn populateVoidUUIDMapOneFile(
|
||||||
@ -358,6 +368,7 @@ pub const FileEngine = struct {
|
|||||||
defer iter.deinit();
|
defer iter.deinit();
|
||||||
|
|
||||||
while (iter.next() catch return) |row| {
|
while (iter.next() catch return) |row| {
|
||||||
|
if (sync_context.checkStructLimit()) break;
|
||||||
if (filter == null or filter.?.evaluate(row)) {
|
if (filter == null or filter.?.evaluate(row)) {
|
||||||
list.*.append(UUID{ .bytes = row[0].UUID }) catch |err| {
|
list.*.append(UUID{ .bytes = row[0].UUID }) catch |err| {
|
||||||
sync_context.logError("Error initializing DataIterator", err);
|
sync_context.logError("Error initializing DataIterator", err);
|
||||||
@ -474,6 +485,7 @@ pub const FileEngine = struct {
|
|||||||
sync_context.logError("Error in iter next", err);
|
sync_context.logError("Error in iter next", err);
|
||||||
return;
|
return;
|
||||||
}) |row| {
|
}) |row| {
|
||||||
|
if (sync_context.checkStructLimit()) break;
|
||||||
if (filter) |f| if (!f.evaluate(row)) continue;
|
if (filter) |f| if (!f.evaluate(row)) continue;
|
||||||
|
|
||||||
writeEntity(
|
writeEntity(
|
||||||
@ -719,6 +731,7 @@ pub const FileEngine = struct {
|
|||||||
sync_context.logError("Parsing files", err);
|
sync_context.logError("Parsing files", err);
|
||||||
return;
|
return;
|
||||||
}) |row| {
|
}) |row| {
|
||||||
|
if (sync_context.checkStructLimit()) break;
|
||||||
if (filter == null or filter.?.evaluate(row)) {
|
if (filter == null or filter.?.evaluate(row)) {
|
||||||
// Add the unchanged Data in the new_data_buff
|
// Add the unchanged Data in the new_data_buff
|
||||||
new_data_buff[0] = row[0];
|
new_data_buff[0] = row[0];
|
||||||
@ -872,6 +885,7 @@ pub const FileEngine = struct {
|
|||||||
sync_context.logError("Error during iter", err);
|
sync_context.logError("Error during iter", err);
|
||||||
return;
|
return;
|
||||||
}) |row| {
|
}) |row| {
|
||||||
|
if (sync_context.checkStructLimit()) break;
|
||||||
if (filter == null or filter.?.evaluate(row)) {
|
if (filter == null or filter.?.evaluate(row)) {
|
||||||
writer.print("{{\"{s}\"}},", .{UUID.format_bytes(row[0].UUID)}) catch |err| {
|
writer.print("{{\"{s}\"}},", .{UUID.format_bytes(row[0].UUID)}) catch |err| {
|
||||||
sync_context.logError("Error writting", err);
|
sync_context.logError("Error writting", err);
|
||||||
|
@ -125,7 +125,6 @@ pub const SchemaEngine = struct {
|
|||||||
/// Chech if the name of a struct is in the current schema
|
/// Chech if the name of a struct is in the current schema
|
||||||
pub fn isStructNameExists(self: *SchemaEngine, struct_name: []const u8) bool {
|
pub fn isStructNameExists(self: *SchemaEngine, struct_name: []const u8) bool {
|
||||||
var i: u16 = 0;
|
var i: u16 = 0;
|
||||||
log.debug("\n\n{any}\n\n", .{self.struct_array});
|
|
||||||
while (i < self.struct_array.len) : (i += 1) if (std.mem.eql(u8, self.struct_array[i].name, struct_name)) return true;
|
while (i < self.struct_array.len) : (i += 1) if (std.mem.eql(u8, self.struct_array[i].name, struct_name)) return true;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -34,7 +34,13 @@ pub const ThreadSyncContext = struct {
|
|||||||
pub fn incrementAndCheckStructLimit(self: *ThreadSyncContext) bool {
|
pub fn incrementAndCheckStructLimit(self: *ThreadSyncContext) bool {
|
||||||
if (self.max_struct == 0) return false;
|
if (self.max_struct == 0) return false;
|
||||||
const new_count = self.processed_struct.fetchAdd(1, .monotonic);
|
const new_count = self.processed_struct.fetchAdd(1, .monotonic);
|
||||||
return new_count >= self.max_struct;
|
return (new_count + 1) >= self.max_struct;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn checkStructLimit(self: *ThreadSyncContext) bool {
|
||||||
|
if (self.max_struct == 0) return false;
|
||||||
|
const count = self.processed_struct.load(.monotonic);
|
||||||
|
return (count) >= self.max_struct;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn logError(self: *ThreadSyncContext, message: []const u8, err: anyerror) void {
|
pub fn logError(self: *ThreadSyncContext, message: []const u8, err: anyerror) void {
|
||||||
|
Loading…
x
Reference in New Issue
Block a user