Push before leaving for christmas

src/entityWriter.zig

@@ -77,23 +77,43 @@ pub const EntityWriter = struct {
             .IntArray => while (iter.next()) |v| writer.print("{d}, ", .{v.Int}) catch return ZipponError.WriteError,
             .FloatArray => while (iter.next()) |v| writer.print("{d}", .{v.Float}) catch return ZipponError.WriteError,
             .StrArray => while (iter.next()) |v| writer.print("\"{s}\"", .{v.Str}) catch return ZipponError.WriteError,
-            .UUIDArray => while (iter.next()) |v| writer.print("\"{s}\"", .{UUID.format_bytes(v.UUID)}) catch return ZipponError.WriteError,
+            .UUIDArray => while (iter.next()) |v| writer.print("\"{{|<{s}>|}}\"", .{UUID.format_bytes(v.UUID)}) catch return ZipponError.WriteError,
             .BoolArray => while (iter.next()) |v| writer.print("{any}", .{v.Bool}) catch return ZipponError.WriteError,
-            .UnixArray => {
-                while (iter.next()) |v| {
-                    const datetime = DateTime.initUnix(v.Unix);
-                    writer.writeByte('"') catch return ZipponError.WriteError;
-                    switch (data_type) {
-                        .date => datetime.format("YYYY/MM/DD", writer) catch return ZipponError.WriteError,
-                        .time => datetime.format("HH:mm:ss.SSSS", writer) catch return ZipponError.WriteError,
-                        .datetime => datetime.format("YYYY/MM/DD-HH:mm:ss.SSSS", writer) catch return ZipponError.WriteError,
-                        else => unreachable,
-                    }
-                    writer.writeAll("\", ") catch return ZipponError.WriteError;
+            .UnixArray => while (iter.next()) |v| {
+                const datetime = DateTime.initUnix(v.Unix);
+                writer.writeByte('"') catch return ZipponError.WriteError;
+                switch (data_type) {
+                    .date => datetime.format("YYYY/MM/DD", writer) catch return ZipponError.WriteError,
+                    .time => datetime.format("HH:mm:ss.SSSS", writer) catch return ZipponError.WriteError,
+                    .datetime => datetime.format("YYYY/MM/DD-HH:mm:ss.SSSS", writer) catch return ZipponError.WriteError,
+                    else => unreachable,
                 }
+                writer.writeAll("\", ") catch return ZipponError.WriteError;
             },
             else => unreachable,
         }
         writer.writeByte(']') catch return ZipponError.WriteError;
     }
+
+    /// TODO:
+    /// Take a string in the JSON format and look for {|<[16]u8>|}, then will look into the map and check if it can find this UUID
+    /// If it find it, it ill replace the {|<[16]u8>|} will the value
+    pub fn updateWithRelation(writer: anytype, input: []const u8, to_add: std.AutoHashMap([16]u8, []const u8)) ZipponError!void {
+        var start: usize = 0;
+        while (std.mem.indexOf(u8, input[start..], "{|<[")) |pos| {
+            const pattern_start = start + pos;
+            const pattern_end = std.mem.indexOf(u8, input[pattern_start..], "]>|}") orelse break;
+            const full_pattern_end = pattern_start + pattern_end + 4;
+
+            // Write the text before the pattern
+            try writer.writeAll(input[start..pattern_start]);
+
+            const uuid_bytes = input[pattern_start + 3 .. full_pattern_end - 3];
+            writer.writeAll(to_add.get(uuid_bytes).?);
+            start = full_pattern_end;
+        }
+
+        // Write any remaining text
+        try writer.writeAll(input[start..]);
+    }
 };

src/fileEngine.zig

@@ -17,6 +17,8 @@ const DataType = dtype.DataType;
 
 const AdditionalData = @import("stuffs/additionalData.zig").AdditionalData;
 const Filter = @import("stuffs/filter.zig").Filter;
+const RelationMap = @import("stuffs/relationMap.zig").RelationMap;
+const JsonString = @import("stuffs/relationMap.zig").JsonString;
 const ConditionValue = @import("stuffs/filter.zig").ConditionValue;
 
 const ZipponError = @import("stuffs/errors.zig").ZipponError;
@@ -382,29 +384,29 @@ pub const FileEngine = struct {
 
     /// Take a filter, parse all file and if one struct if validate by the filter, write it in a JSON format to the writer
     /// filter can be null. This will return all of them
-    /// TODO: For relationship, if they are in additional_data and I need to return it with the other members, I will need to parse the file
-    /// This is difficult, because that mean I need to parse file while parsing files ? I dont like that because it may be the same struct
-    /// And because of multi thread, I can read the same file at the same time...
     pub fn parseEntities(
         self: *FileEngine,
         struct_name: []const u8,
         filter: ?Filter,
         additional_data: *AdditionalData,
-        writer: anytype,
-    ) ZipponError!void {
+        entry_allocator: Allocator,
+    ) ZipponError![]const u8 {
         var fa = std.heap.FixedBufferAllocator.init(&parsing_buffer);
         fa.reset();
         const allocator = fa.allocator();
 
+        var buff = std.ArrayList(u8).init(entry_allocator);
+        defer buff.deinit();
+        const writer = buff.writer();
+
         const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name);
         const max_file_index = try self.maxFileIndex(sstruct.name);
 
         log.debug("Max file index {d}", .{max_file_index});
 
         // If there is no member to find, that mean we need to return all members, so let's populate additional data with all of them
-        if (additional_data.childrens.items.len == 0) {
+        if (additional_data.childrens.items.len == 0)
             additional_data.populateWithEverythingExceptLink(sstruct.members, sstruct.types) catch return FileEngineError.MemoryError;
-        }
 
         // Open the dir that contain all files
         const dir = try utils.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{ .access_sub_paths = false });
@@ -441,9 +443,14 @@ pub const FileEngine = struct {
         }
 
         // Append all writer to each other
-        //writer.writeByte('[') catch return FileEngineError.WriteError;
+        writer.writeByte('[') catch return FileEngineError.WriteError;
         for (thread_writer_list) |list| writer.writeAll(list.items) catch return FileEngineError.WriteError;
-        //writer.writeByte(']') catch return FileEngineError.WriteError;
+        writer.writeByte(']') catch return FileEngineError.WriteError;
+
+        // Here now I need to already have a populated list of RelationMap
+        // I will then call parseEntitiesRelationMap on each
+
+        return buff.toOwnedSlice();
     }
 
     fn parseEntitiesOneFile(
@@ -493,6 +500,141 @@ pub const FileEngine = struct {
         _ = sync_context.completeThread();
     }
 
+    // Receive a map of UUID -> null
+    // Will parse the files and update the value to the JSON string of the entity that represent the key
+    // Will then write the input with the JSON in the map looking for {|<>|}
+    // Once the new input received, call parseEntitiesRelationMap again the string still contain {|<>|} because of sub relationship
+    // The buffer contain the string with {|<>|} and need to be updated at the end
+    // TODO: Filter file that need to be parse to prevent parsing everything all the time
+    pub fn parseEntitiesRelationMap(
+        self: *FileEngine,
+        relation_map: *RelationMap,
+        buff: *std.ArrayList(u8),
+    ) ZipponError!void {
+        var fa = std.heap.FixedBufferAllocator.init(&parsing_buffer);
+        fa.reset();
+        const allocator = fa.allocator();
+
+        var new_buff = std.ArrayList(u8).init(allocator);
+        defer new_buff.deinit();
+        const writer = new_buff.writer();
+
+        const sstruct = try self.schema_engine.structName2SchemaStruct(relation_map.struct_name);
+        const max_file_index = try self.maxFileIndex(sstruct.name);
+
+        log.debug("Max file index {d}", .{max_file_index});
+
+        // If there is no member to find, that mean we need to return all members, so let's populate additional data with all of them
+        if (relation_map.additional_data.childrens.items.len == 0) {
+            relation_map.additional_data.populateWithEverythingExceptLink(sstruct.members, sstruct.types) catch return FileEngineError.MemoryError;
+        }
+
+        // Open the dir that contain all files
+        const dir = try utils.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{ .access_sub_paths = false });
+
+        // Multi thread stuffs
+        var sync_context = ThreadSyncContext.init(
+            relation_map.additional_data.limit,
+            max_file_index + 1,
+        );
+
+        // Do one writer for each thread otherwise it create error by writing at the same time
+        var thread_map_list = allocator.alloc(std.AutoHashMap([16]u8, JsonString), max_file_index + 1) catch return FileEngineError.MemoryError;
+
+        // Start parsing all file in multiple thread
+        for (0..(max_file_index + 1)) |file_index| {
+            thread_map_list[file_index] = relation_map.map.cloneWithAllocator(allocator);
+
+            self.thread_pool.spawn(parseEntitiesRelationMapOneFile, .{
+                &thread_map_list[file_index],
+                file_index,
+                dir,
+                sstruct.zid_schema,
+                relation_map.additional_data,
+                try self.schema_engine.structName2DataType(relation_map.struct_name),
+                &sync_context,
+            }) catch return FileEngineError.ThreadError;
+        }
+
+        // Wait for all thread to either finish or return an error
+        while (!sync_context.isComplete()) {
+            std.time.sleep(10_000_000); // Check every 10ms
+        }
+
+        // Now here I should have a list of copy of the map with all UUID a bit everywhere
+
+        // Put all in the same map
+        for (thread_map_list) |map| {
+            var iter = map.iterator();
+            while (iter.next()) |entry| {
+                if (entry.value_ptr.*) |json_string| relation_map.map.put(entry.key_ptr.*, json_string);
+            }
+        }
+
+        // Here I write the new string and update the buff to have the new version
+        EntityWriter.updateWithRelation(writer, buff.items, relation_map.map);
+        buff.clearRetainingCapacity();
+        buff.writer().writeAll(new_buff.items);
+
+        // Now here I need to iterate if buff.items still have {|<>|}
+    }
+
+    fn parseEntitiesRelationMapOneFile(
+        map: *std.AutoHashMap([16]u8, []const u8),
+        file_index: u64,
+        dir: std.fs.Dir,
+        zid_schema: []zid.DType,
+        additional_data: *AdditionalData,
+        data_types: []const DataType,
+        sync_context: *ThreadSyncContext,
+    ) void {
+        var data_buffer: [BUFFER_SIZE]u8 = undefined;
+        var fa = std.heap.FixedBufferAllocator.init(&data_buffer);
+        defer fa.reset();
+        const allocator = fa.allocator();
+
+        const parent_alloc = map.allocator;
+        var string_list = std.ArrayList(u8).init(allocator);
+        const writer = string_list.writer();
+
+        const path = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{file_index}) catch |err| {
+            sync_context.logError("Error creating file path", err);
+            return;
+        };
+
+        var iter = zid.DataIterator.init(allocator, path, dir, zid_schema) catch |err| {
+            sync_context.logError("Error initializing DataIterator", err);
+            return;
+        };
+
+        while (iter.next() catch |err| {
+            sync_context.logError("Error in iter next", err);
+            return;
+        }) |row| {
+            if (sync_context.checkStructLimit()) break;
+            if (!map.contains(row[0].UUID)) continue;
+            defer string_list.clearRetainingCapacity();
+
+            EntityWriter.writeEntityJSON(
+                writer,
+                row,
+                additional_data,
+                data_types,
+            ) catch |err| {
+                sync_context.logError("Error writing entity", err);
+                return;
+            };
+            map.put(row[0].UUID, parent_alloc.dupe(u8, string_list.items)) catch |err| {
+                sync_context.logError("Error writing entity", err);
+                return;
+            };
+
+            if (sync_context.incrementAndCheckStructLimit()) break;
+        }
+
+        _ = sync_context.completeThread();
+    }
+
     // --------------------Change existing files--------------------
 
     // TODO: Make it in batch too

src/stuffs/additionalData.zig

@@ -1,5 +1,6 @@
 const std = @import("std");
 const Allocator = std.mem.Allocator;
+const RelationMap = @import("relationMap.zig").RelationMap;
 const dtype = @import("dtype");
 const DataType = dtype.DataType;
 
@@ -33,6 +34,15 @@ pub const AdditionalData = struct {
         self.childrens.items[self.childrens.items.len - 1].additional_data = AdditionalData.init(self.allocator);
         return &self.childrens.items[self.childrens.items.len - 1].additional_data.?;
     }
+
+    /// Create an array of empty RelationMap based on the additionalData
+    pub fn relationMapArrayInit(self: AdditionalData, allocator: Allocator) ZipponError!?[]RelationMap {
+    // So here I should have relationship if children are relations
+    var array = std.ArrayList(RelationMap).init(allocator);
+    for (self.childrens.items) |child| {
+        child.
+    }
+}
 };
 
 // This is name in: [name]

src/stuffs/relationMap.zig

@@ -0,0 +1,32 @@
+// A relation map is all data needed to add relationship at the during parsing
+// How it work is that, the first time I parse the struct files, like User, I populate a map of UUID empty string
+// And in the JSON string I write {|<[16]u8>|} inside. Then I can use this struct to parse the file again
+// And if the UUID is in the map, I write the JSON if in its value in the map
+// It need to be recurcive as additional data can do stuff like [name, friends [name, best_friend]]
+// I could use parseEntities in a recursive way. But that mean ready the file at each loop =/
+//
+// No no no, But on the other hands that would solve the issue of getting the UUID of the best_friend
+// Fuck thats true, I didnt think about that. I can one populate the UUID that I want from the current depth of the additional data
+// So I need to parse multiple time. But that sove when using with multiple parse.
+// Because GRAB User [comments [post]]. How do I get the UUID of the Post if I only parse User ?
+//
+// Ok so I need to go recursive on parseEntities
+// So I parse one time, if additional data has relationship, I create a list of RelationMap
+// When I parse, I populate RelationMap with UUID I want
+// Then for each RelationMap, I parse the files again this time to update the first JSON that now have {|<>|}
+// With a sub additionalData. If there is an additional data relation, I recurcive.
+// So I need an option in parseEntity to either write the first JSON or update the existing one
+
+const std = @import("std");
+const AdditionalData = @import("stuffs/additionalData.zig").AdditionalData;
+
+pub const JsonString = struct {
+    slice: []const u8 = "",
+    init: bool = false,
+};
+
+pub const RelationMap = struct {
+    struct_name: []const u8,
+    additional_data: AdditionalData,
+    map: *std.AutoHashMap([16]u8, JsonString),
+};

src/ziqlParser.zig

@@ -184,19 +184,13 @@ pub const Parser = struct {
                     var filter = try self.parseFilter(allocator, struct_name, false);
                     defer filter.deinit();
 
-                    var buff = std.ArrayList(u8).init(allocator);
-                    defer buff.deinit();
-
-                    try self.file_engine.parseEntities(struct_name, filter, &additional_data, &buff.writer());
-                    send("{s}", .{buff.items});
+                    const json_string = try self.file_engine.parseEntities(struct_name, filter, &additional_data, allocator);
+                    send("{s}", .{json_string});
                     state = .end;
                 },
                 .eof => {
-                    var buff = std.ArrayList(u8).init(allocator);
-                    defer buff.deinit();
-
-                    try self.file_engine.parseEntities(struct_name, null, &additional_data, &buff.writer());
-                    send("{s}", .{buff.items});
+                    const json_string = try self.file_engine.parseEntities(struct_name, null, &additional_data, allocator);
+                    send("{s}", .{json_string});
                     state = .end;
                 },
                 else => return printError(