Error found!!

Got an error that I could find for a while. It was when parsing a lot of
files.

Turn out it was the Thread Pool or something because if I run on 1 core,
it's ok.

Different thing in this commit, I just want to freeze a state that
benchmark work.

src/cli/core.zig

@@ -27,15 +27,15 @@ file_engine: FileEngine = undefined,
 schema_engine: SchemaEngine = undefined,
 thread_engine: ThreadEngine = undefined,
 
-pub fn init(parent_allocator: Allocator, potential_main_path: ?[]const u8, potential_schema_path: ?[]const u8) Self {
+pub fn init(allocator: Allocator, potential_main_path: ?[]const u8, potential_schema_path: ?[]const u8) Self {
     var self = Self{};
 
-    const arena = parent_allocator.create(std.heap.ArenaAllocator) catch {
+    const arena = allocator.create(std.heap.ArenaAllocator) catch {
         log.err("Error when init Engine DB allocator", .{});
         self.state = .MissingAllocator;
         return self;
     };
-    arena.* = std.heap.ArenaAllocator.init(parent_allocator);
+    arena.* = std.heap.ArenaAllocator.init(allocator);
     self.arena = arena;
     self.allocator = arena.allocator();
 
@@ -152,7 +152,7 @@ pub fn getEnvVariable(allocator: Allocator, variable: []const u8) ?[]const u8 {
 
 pub fn runQuery(self: *Self, null_term_query_str: [:0]const u8) void {
     var parser = ziqlParser.init(&self.file_engine, &self.schema_engine);
-    parser.parse(null_term_query_str) catch |err| log.err("Error parsing: {any}", .{err});
+    parser.parse(self.allocator, null_term_query_str) catch |err| log.err("Error parsing: {any}", .{err});
 }
 
 pub fn deinit(self: *Self) void {

src/file/entityWriter.zig

@@ -8,6 +8,8 @@ const DataType = dtype.DataType;
 const DateTime = dtype.DateTime;
 const UUID = dtype.UUID;
 
+// TODO: Move this from FileEngine
+
 const ZipponError = @import("error").ZipponError;
 
 pub fn writeEntityTable(

src/file/read.zig

@@ -240,7 +240,6 @@ pub fn parseEntities(
     const allocator = arena.allocator();
 
     var buff = std.ArrayList(u8).init(entry_allocator);
-    defer buff.deinit();
     const writer = buff.writer();
 
     const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name);
@@ -268,30 +267,35 @@ pub fn parseEntities(
     // But at the end, only the number of use CPU/Thread will use list simultanously
     // So I could pass list from a thread to another technicly
     var thread_writer_list = allocator.alloc(std.ArrayList(u8), to_parse.len) catch return ZipponError.MemoryError;
+    const data_types = try self.schema_engine.structName2DataType(struct_name);
 
     // Start parsing all file in multiple thread
+    var wg: std.Thread.WaitGroup = .{};
     for (to_parse, 0..) |file_index, i| {
         thread_writer_list[file_index] = std.ArrayList(u8).init(allocator);
 
-        self.thread_pool.spawn(parseEntitiesOneFile, .{
-            thread_writer_list[i].writer(),
-            file_index,
-            dir,
-            sstruct.zid_schema,
-            filter,
-            additional_data.*,
-            try self.schema_engine.structName2DataType(struct_name),
-            &sync_context,
-        }) catch return ZipponError.ThreadError;
+        self.thread_pool.spawnWg(
+            &wg,
+            parseEntitiesOneFile,
+            .{
+                thread_writer_list[i].writer(),
+                file_index,
+                dir,
+                sstruct.zid_schema,
+                filter,
+                additional_data.*,
+                data_types,
+                &sync_context,
+            },
+        );
     }
-
-    // Wait for all thread to either finish or return an error
-    while (!sync_context.isComplete()) std.time.sleep(100_000); // Check every 0.1ms
+    wg.wait();
 
     // Append all writer to each other
     writer.writeByte('[') catch return ZipponError.WriteError;
     for (thread_writer_list) |list| writer.writeAll(list.items) catch return ZipponError.WriteError;
     writer.writeByte(']') catch return ZipponError.WriteError;
+    for (thread_writer_list) |list| list.deinit();
 
     // Now I need to do the relation stuff, meaning parsing new files to get the relationship value
     // Without relationship to return, this function is basically finish here
@@ -321,21 +325,12 @@ fn parseEntitiesOneFile(
     defer fa.reset();
     const allocator = fa.allocator();
 
-    const path = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{file_index}) catch |err| {
-        sync_context.logError("Error creating file path", err);
-        return;
-    };
+    const path = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{file_index}) catch return;
+    var iter = zid.DataIterator.init(allocator, path, dir, zid_schema) catch return;
 
-    var iter = zid.DataIterator.init(allocator, path, dir, zid_schema) catch |err| {
-        sync_context.logError("Error initializing DataIterator", err);
-        return;
-    };
-
-    while (iter.next() catch |err| {
-        sync_context.logError("Error in iter next", err);
-        return;
-    }) |row| {
-        if (sync_context.checkStructLimit()) break;
+    while (iter.next() catch return) |row| {
+        fa.reset();
+        if (sync_context.checkStructLimit()) return;
         if (filter) |f| if (!f.evaluate(row)) continue;
 
         EntityWriter.writeEntityJSON(
@@ -343,14 +338,10 @@ fn parseEntitiesOneFile(
             row,
             additional_data,
             data_types,
-        ) catch |err| {
-            sync_context.logError("Error writing entity", err);
-            return;
-        };
-        if (sync_context.incrementAndCheckStructLimit()) break;
-    }
+        ) catch return;
 
-    _ = sync_context.completeThread();
+        if (sync_context.incrementAndCheckStructLimit()) return;
+    }
 }
 
 // Receive a map of UUID -> empty JsonString
@@ -410,22 +401,25 @@ pub fn parseEntitiesRelationMap(
     ) catch return ZipponError.MemoryError;
 
     // Start parsing all file in multiple thread
+    var wg: std.Thread.WaitGroup = .{};
     for (to_parse, 0..) |file_index, i| {
         thread_map_list[i] = relation_map.map.cloneWithAllocator(allocator) catch return ZipponError.MemoryError;
 
-        self.thread_pool.spawn(parseEntitiesRelationMapOneFile, .{
-            &thread_map_list[i],
-            file_index,
-            dir,
-            sstruct.zid_schema,
-            relation_map.additional_data,
-            try self.schema_engine.structName2DataType(struct_name),
-            &sync_context,
-        }) catch return ZipponError.ThreadError;
+        self.thread_pool.spawnWg(
+            &wg,
+            parseEntitiesRelationMapOneFile,
+            .{
+                &thread_map_list[i],
+                file_index,
+                dir,
+                sstruct.zid_schema,
+                relation_map.additional_data,
+                try self.schema_engine.structName2DataType(struct_name),
+                &sync_context,
+            },
+        );
     }
-
-    // Wait for all thread to either finish or return an error
-    while (!sync_context.isComplete()) std.time.sleep(100_000); // Check every 0.1ms
+    wg.wait();
 
     // Now here I should have a list of copy of the map with all UUID a bit everywhere
 
@@ -470,21 +464,11 @@ fn parseEntitiesRelationMapOneFile(
     var string_list = std.ArrayList(u8).init(allocator);
     const writer = string_list.writer();
 
-    const path = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{file_index}) catch |err| {
-        sync_context.logError("Error creating file path", err);
-        return;
-    };
+    const path = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{file_index}) catch return;
+    var iter = zid.DataIterator.init(allocator, path, dir, zid_schema) catch return;
 
-    var iter = zid.DataIterator.init(allocator, path, dir, zid_schema) catch |err| {
-        sync_context.logError("Error initializing DataIterator", err);
-        return;
-    };
-
-    while (iter.next() catch |err| {
-        sync_context.logError("Error in iter next", err);
-        return;
-    }) |row| {
-        if (sync_context.checkStructLimit()) break;
+    while (iter.next() catch return) |row| {
+        if (sync_context.checkStructLimit()) return;
         if (!map.contains(row[0].UUID)) continue;
         defer string_list.clearRetainingCapacity();
 
@@ -493,23 +477,12 @@ fn parseEntitiesRelationMapOneFile(
             row,
             additional_data,
             data_types,
-        ) catch |err| {
-            sync_context.logError("Error writing entity", err);
-            return;
-        };
+        ) catch return;
+
         map.put(row[0].UUID, JsonString{
-            .slice = parent_alloc.dupe(u8, string_list.items) catch |err| {
-                sync_context.logError("Error duping data", err);
-                return;
-            },
+            .slice = parent_alloc.dupe(u8, string_list.items) catch return,
             .init = true,
-        }) catch |err| {
-            sync_context.logError("Error writing entity", err);
-            return;
-        };
-
-        if (sync_context.incrementAndCheckStructLimit()) break;
+        }) catch return;
+        if (sync_context.incrementAndCheckStructLimit()) return;
     }
-
-    _ = sync_context.completeThread();
 }

src/main.zig

@@ -11,13 +11,14 @@ var date_buffer: [64]u8 = undefined;
 var date_fa = std.heap.FixedBufferAllocator.init(&date_buffer);
 const date_allocator = date_fa.allocator();
 
-pub const std_options = .{
+pub const std_options = std.Options{
+    .log_level = .info,
     .logFn = myLog,
 };
 
 pub fn myLog(
     comptime message_level: std.log.Level,
-    comptime scope: @Type(.EnumLiteral),
+    comptime scope: @Type(.enum_literal),
     comptime format: []const u8,
     args: anytype,
 ) void {
@@ -48,10 +49,17 @@ pub fn setLogPath(path: []const u8) void {
 }
 
 pub fn main() !void {
-    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
-    defer arena.deinit();
+    var gpa = std.heap.GeneralPurposeAllocator(.{ .safety = true }){};
+    const allocator = gpa.allocator();
+    defer {
+        const deinit_status = gpa.deinit();
+        switch (deinit_status) {
+            .leak => std.debug.print("Leak...\n", .{}),
+            .ok => {},
+        }
+    }
 
-    var cli = Cli.init(arena.allocator(), null, null);
+    var cli = Cli.init(allocator, null, null);
     defer cli.deinit();
 
     try cli.start();

src/thread/context.zig

@@ -2,6 +2,8 @@ const std = @import("std");
 const log = std.log.scoped(.thread);
 const U64 = std.atomic.Value(u64);
 
+// Remove the use waitgroup instead
+
 pub const Self = @This();
 
 processed_struct: U64 = U64.init(0),
@@ -27,13 +29,13 @@ pub fn completeThread(self: *Self) void {
 
 pub fn incrementAndCheckStructLimit(self: *Self) bool {
     if (self.max_struct == 0) return false;
-    const new_count = self.processed_struct.fetchAdd(1, .monotonic);
+    const new_count = self.processed_struct.fetchAdd(1, .acquire);
     return (new_count + 1) >= self.max_struct;
 }
 
 pub fn checkStructLimit(self: *Self) bool {
     if (self.max_struct == 0) return false;
-    const count = self.processed_struct.load(.monotonic);
+    const count = self.processed_struct.load(.acquire);
     return (count) >= self.max_struct;
 }
 

src/ziql/parser.zig

@@ -80,8 +80,8 @@ pub fn init(file_engine: *FileEngine, schema_engine: *SchemaEngine) Self {
     };
 }
 
-pub fn parse(self: *Self, buffer: [:0]const u8) ZipponError!void {
-    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+pub fn parse(self: *Self, parent_allocator: Allocator, buffer: [:0]const u8) ZipponError!void {
+    var arena = std.heap.ArenaAllocator.init(parent_allocator);
     defer arena.deinit();
     const allocator = arena.allocator();