Moved thread stuff into a directory

src/dataStructure/UUIDFileIndex.zig

@@ -28,14 +28,7 @@ pub fn deinit(self: *UUIDIndexMap) void {
 }
 
 pub fn put(self: *UUIDIndexMap, uuid: UUID, file_index: usize) !void {
-    const allocator = self.arena.allocator();
-    const new_uuid = try allocator.create(UUID);
-    new_uuid.* = uuid;
-
-    const new_file_index = try allocator.create(usize);
-    new_file_index.* = file_index;
-
-    try self.map.*.put(new_uuid.*, new_file_index.*);
+    try self.map.*.put(uuid, file_index);
 }
 
 pub fn contains(self: UUIDIndexMap, uuid: UUID) bool {

src/entityWriter.zig

@@ -10,7 +10,6 @@ const UUID = dtype.UUID;
 
 const ZipponError = @import("errors.zig").ZipponError;
 
-pub const EntityWriter = struct {
 pub fn writeEntityTable(
     writer: anytype,
     row: []zid.Data,
@@ -166,4 +165,3 @@ pub const EntityWriter = struct {
     }
     return start;
 }
-};

src/fileEngine.zig

@@ -4,10 +4,10 @@ const zid = @import("ZipponData");
 const U64 = std.atomic.Value(u64);
 const Pool = std.Thread.Pool;
 const Allocator = std.mem.Allocator;
-const SchemaEngine = @import("schemaEngine.zig").SchemaEngine;
+const SchemaEngine = @import("schemaEngine.zig");
 const SchemaStruct = @import("schemaEngine.zig").SchemaStruct;
-const ThreadSyncContext = @import("threadEngine.zig").ThreadSyncContext;
-const EntityWriter = @import("entityWriter.zig").EntityWriter;
+const ThreadSyncContext = @import("thread/context.zig");
+const EntityWriter = @import("entityWriter.zig");
 
 const dtype = @import("dtype");
 const s2t = dtype.s2t;
@@ -15,9 +15,9 @@ const UUID = dtype.UUID;
 const DateTime = dtype.DateTime;
 const DataType = dtype.DataType;
 
-const AdditionalData = @import("dataStructure/additionalData.zig").AdditionalData;
+const AdditionalData = @import("dataStructure/additionalData.zig");
 const Filter = @import("dataStructure/filter.zig").Filter;
-const RelationMap = @import("dataStructure/relationMap.zig").RelationMap;
+const RelationMap = @import("dataStructure/relationMap.zig");
 const JsonString = @import("dataStructure/relationMap.zig").JsonString;
 const ConditionValue = @import("dataStructure/filter.zig").ConditionValue;
 

src/main.zig

@@ -4,16 +4,16 @@ const send = utils.send;
 const Allocator = std.mem.Allocator;
 const Pool = std.Thread.Pool;
 
-const FileEngine = @import("fileEngine.zig").FileEngine;
-const SchemaEngine = @import("schemaEngine.zig").SchemaEngine;
-const ThreadEngine = @import("threadEngine.zig").ThreadEngine;
+const FileEngine = @import("fileEngine.zig");
+const SchemaEngine = @import("schemaEngine.zig");
+const ThreadEngine = @import("thread/engine.zig");
 
 const cliTokenizer = @import("tokenizers/cli.zig").Tokenizer;
 const cliToken = @import("tokenizers/cli.zig").Token;
 
 const ziqlTokenizer = @import("tokenizers/ziql.zig").Tokenizer;
 const ziqlToken = @import("tokenizers/ziql.zig").Token;
-const ziqlParser = @import("ziqlParser.zig").Parser;
+const ziqlParser = @import("ziqlParser.zig");
 
 const ZipponError = @import("errors.zig").ZipponError;
 
@@ -86,7 +86,7 @@ pub const DBEngine = struct {
     pub fn init(potential_main_path: ?[]const u8, potential_schema_path: ?[]const u8) DBEngine {
         var self = DBEngine{};
 
-        self.thread_engine = ThreadEngine.init();
+        self.thread_engine = ThreadEngine.init() catch @panic("TODO");
 
         const potential_main_path_or_environment_variable = potential_main_path orelse utils.getEnvVariable("ZIPPONDB_PATH");
         if (potential_main_path_or_environment_variable) |main_path| {

src/schemaEngine.zig

@@ -1,18 +1,18 @@
 const std = @import("std");
 const zid = @import("ZipponData");
 const Allocator = std.mem.Allocator;
-const Parser = @import("schemaParser.zig").Parser;
+const Parser = @import("schemaParser.zig");
 const Tokenizer = @import("tokenizers/schema.zig").Tokenizer;
 const ZipponError = @import("errors.zig").ZipponError;
 const dtype = @import("dtype");
 const DataType = dtype.DataType;
-const AdditionalData = @import("dataStructure/additionalData.zig").AdditionalData;
-const RelationMap = @import("dataStructure/relationMap.zig").RelationMap;
+const AdditionalData = @import("dataStructure/additionalData.zig");
+const RelationMap = @import("dataStructure/relationMap.zig");
 const JsonString = @import("dataStructure/relationMap.zig").JsonString;
 const ConditionValue = @import("dataStructure/filter.zig").ConditionValue;
 const UUID = dtype.UUID;
-const UUIDFileIndex = @import("dataStructure/UUIDFileIndex.zig").UUIDIndexMap;
-const FileEngine = @import("fileEngine.zig").FileEngine;
+const UUIDFileIndex = @import("dataStructure/UUIDFileIndex.zig");
+const FileEngine = @import("fileEngine.zig");
 
 // TODO: Create a schemaEngine directory and add this as core and the parser with it
 

src/thread/context.zig

@@ -0,0 +1,43 @@
+const std = @import("std");
+const log = std.log.scoped(.thread);
+const U64 = std.atomic.Value(u64);
+
+pub const Self = @This();
+
+processed_struct: U64 = U64.init(0),
+error_file: U64 = U64.init(0),
+completed_file: U64 = U64.init(0),
+max_struct: u64,
+max_file: u64,
+
+pub fn init(max_struct: u64, max_file: u64) Self {
+    return Self{
+        .max_struct = max_struct,
+        .max_file = max_file,
+    };
+}
+
+pub fn isComplete(self: *Self) bool {
+    return (self.completed_file.load(.acquire) + self.error_file.load(.acquire)) >= self.max_file;
+}
+
+pub fn completeThread(self: *Self) void {
+    _ = self.completed_file.fetchAdd(1, .release);
+}
+
+pub fn incrementAndCheckStructLimit(self: *Self) bool {
+    if (self.max_struct == 0) return false;
+    const new_count = self.processed_struct.fetchAdd(1, .monotonic);
+    return (new_count + 1) >= self.max_struct;
+}
+
+pub fn checkStructLimit(self: *Self) bool {
+    if (self.max_struct == 0) return false;
+    const count = self.processed_struct.load(.monotonic);
+    return (count) >= self.max_struct;
+}
+
+pub fn logError(self: *Self, message: []const u8, err: anyerror) void {
+    log.err("{s}: {any}", .{ message, err });
+    _ = self.error_file.fetchAdd(1, .acquire);
+}

src/thread/engine.zig

@@ -0,0 +1,38 @@
+const std = @import("std");
+const Pool = std.Thread.Pool;
+const Allocator = std.mem.Allocator;
+
+const CPU_CORE = @import("config").CPU_CORE;
+const log = std.log.scoped(.thread);
+const ZipponError = @import("../errors.zig").ZipponError;
+
+pub const Self = @This();
+
+var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+const allocator = arena.allocator();
+
+thread_arena: *std.heap.ThreadSafeAllocator,
+thread_pool: *Pool,
+
+pub fn init() ZipponError!Self {
+    const thread_arena = allocator.create(std.heap.ThreadSafeAllocator) catch return ZipponError.MemoryError;
+    thread_arena.* = std.heap.ThreadSafeAllocator{
+        .child_allocator = allocator,
+    };
+
+    const thread_pool = allocator.create(Pool) catch return ZipponError.MemoryError;
+    thread_pool.init(Pool.Options{
+        .allocator = thread_arena.allocator(),
+        .n_jobs = CPU_CORE,
+    }) catch return ZipponError.ThreadError;
+
+    return Self{
+        .thread_pool = thread_pool,
+        .thread_arena = thread_arena,
+    };
+}
+
+pub fn deinit(self: *Self) void {
+    self.thread_pool.deinit();
+    arena.deinit();
+}

src/threadEngine.zig

@@ -1,81 +0,0 @@
-// TODO: Put the ThreadSynx stuff and create a ThreadEngine with the arena, pool, and some methods
-
-const std = @import("std");
-const U64 = std.atomic.Value(u64);
-const Pool = std.Thread.Pool;
-const Allocator = std.mem.Allocator;
-
-const ZipponError = @import("errors.zig").ZipponError;
-const CPU_CORE = @import("config").CPU_CORE;
-const OUT_BUFFER_SIZE = @import("config").OUT_BUFFER_SIZE;
-const log = std.log.scoped(.thread);
-
-const allocator = std.heap.page_allocator;
-
-var thread_arena: std.heap.ThreadSafeAllocator = undefined;
-var thread_pool: Pool = undefined;
-
-pub const ThreadSyncContext = struct {
-    processed_struct: std.atomic.Value(u64) = std.atomic.Value(u64).init(0),
-    error_file: std.atomic.Value(u64) = std.atomic.Value(u64).init(0),
-    completed_file: std.atomic.Value(u64) = std.atomic.Value(u64).init(0),
-    max_struct: u64,
-    max_file: u64,
-
-    pub fn init(max_struct: u64, max_file: u64) ThreadSyncContext {
-        return ThreadSyncContext{
-            .max_struct = max_struct,
-            .max_file = max_file,
-        };
-    }
-
-    pub fn isComplete(self: *ThreadSyncContext) bool {
-        return (self.completed_file.load(.acquire) + self.error_file.load(.acquire)) >= self.max_file;
-    }
-
-    pub fn completeThread(self: *ThreadSyncContext) void {
-        _ = self.completed_file.fetchAdd(1, .release);
-    }
-
-    pub fn incrementAndCheckStructLimit(self: *ThreadSyncContext) bool {
-        if (self.max_struct == 0) return false;
-        const new_count = self.processed_struct.fetchAdd(1, .monotonic);
-        return (new_count + 1) >= self.max_struct;
-    }
-
-    pub fn checkStructLimit(self: *ThreadSyncContext) bool {
-        if (self.max_struct == 0) return false;
-        const count = self.processed_struct.load(.monotonic);
-        return (count) >= self.max_struct;
-    }
-
-    pub fn logError(self: *ThreadSyncContext, message: []const u8, err: anyerror) void {
-        log.err("{s}: {any}", .{ message, err });
-        _ = self.error_file.fetchAdd(1, .acquire);
-    }
-};
-
-pub const ThreadEngine = @This();
-
-thread_arena: *std.heap.ThreadSafeAllocator,
-thread_pool: *Pool,
-
-pub fn init() ThreadEngine {
-    thread_arena = std.heap.ThreadSafeAllocator{
-        .child_allocator = allocator,
-    };
-
-    thread_pool.init(std.Thread.Pool.Options{
-        .allocator = thread_arena.allocator(),
-        .n_jobs = CPU_CORE,
-    }) catch @panic("=(");
-
-    return ThreadEngine{
-        .thread_pool = &thread_pool,
-        .thread_arena = &thread_arena,
-    };
-}
-
-pub fn deinit(_: ThreadEngine) void {
-    thread_pool.deinit();
-}

src/ziqlParser.zig

@@ -66,6 +66,7 @@ const State = enum {
 };
 
 pub const Parser = @This();
+
 toker: *Tokenizer,
 file_engine: *FileEngine,
 schema_engine: *SchemaEngine,