diff --git a/lib/types/stringToType.zig b/lib/types/stringToType.zig index 5ca05c9..8e6e7e9 100644 --- a/lib/types/stringToType.zig +++ b/lib/types/stringToType.zig @@ -2,9 +2,6 @@ const std = @import("std"); const UUID = @import("uuid.zig").UUID; const DateTime = @import("date.zig").DateTime; -// FIXME: Stop returning arrayList and use toOwnedSlice instead - -// TODO: Put those functions somewhere else pub fn parseInt(value_str: []const u8) i32 { return std.fmt.parseInt(i32, value_str, 10) catch return 0; } diff --git a/src/dataStructure/relationMap.zig b/src/dataStructure/relationMap.zig index c43bf80..46dc2e2 100644 --- a/src/dataStructure/relationMap.zig +++ b/src/dataStructure/relationMap.zig @@ -1,6 +1,6 @@ // A relation map is all data needed to add relationship at the during parsing // How it work is that, the first time I parse the struct files, like User, I populate a map of UUID empty string -// And in the JSON string I write {|<[16]u8>|} inside. Then I can use this struct to parse the file again +// And in the JSON string I write {<|[16]u8|>} inside. Then I can use this struct to parse the file again // And if the UUID is in the map, I write the JSON if in its value in the map // It need to be recurcive as additional data can do stuff like [name, friends [name, best_friend]] // I could use parseEntities in a recursive way. But that mean ready the file at each loop =/ @@ -13,7 +13,7 @@ // Ok so I need to go recursive on parseEntities // So I parse one time, if additional data has relationship, I create a list of RelationMap // When I parse, I populate RelationMap with UUID I want -// Then for each RelationMap, I parse the files again this time to update the first JSON that now have {|<>|} +// Then for each RelationMap, I parse the files again this time to update the first JSON that now have {<||>} // With a sub additionalData. If there is an additional data relation, I recurcive. // So I need an option in parseEntity to either write the first JSON or update the existing one // @@ -34,16 +34,16 @@ member_name: []const u8, additional_data: AdditionalData, map: *std.AutoHashMap([16]u8, JsonString), -/// Will use a string in the JSON format and look for {|<[16]u8>|} +/// Will use a string in the JSON format and look for {<|[16]u8|>} /// It will then check if it is for the right member name and if so, add an empty JSON string at the key pub fn populate(self: *RelationMap, input: []const u8) ZipponError!void { var uuid_bytes: [16]u8 = undefined; var start: usize = 0; - while (std.mem.indexOf(u8, input[start..], "{|<")) |pos| { + while (std.mem.indexOf(u8, input[start..], "{<|")) |pos| { const pattern_start = start + pos + 3; const pattern_end = pattern_start + 16; - const member_end = if (input[pattern_start - 4] == '[') pattern_start - 6 else pattern_start - 5; // This should be ": {|<" + const member_end = if (input[pattern_start - 4] == '[') pattern_start - 6 else pattern_start - 5; // This should be ": {<|" var member_start = member_end - 1; while (input[member_start] != ' ') : (member_start -= 1) {} member_start += 1; @@ -62,11 +62,11 @@ pub fn populate(self: *RelationMap, input: []const u8) ZipponError!void { } } -// Array are pack in format {|<[16]u8>|},{|<[16]u8>|},{|<[16]u8>|},{|<[16]u8>|}, +// Array are pack in format {<|[16]u8|>},{<|[16]u8|>},{<|[16]u8|>},{<|[16]u8|>}, fn populateArray(self: *RelationMap, input: []const u8, origin: usize) ZipponError!usize { var uuid_bytes: [16]u8 = undefined; var start = origin; - while (input.len > start + 23 and std.mem.eql(u8, input[start .. start + 3], "{|<") and std.mem.eql(u8, input[start + 19 .. start + 23], ">|},")) : (start += 23) { + while (input.len > start + 23 and std.mem.eql(u8, input[start .. start + 3], "{<|") and std.mem.eql(u8, input[start + 19 .. start + 23], "|>},")) : (start += 23) { for (start + 3..start + 19, 0..) |i, j| uuid_bytes[j] = input[i]; self.map.put(uuid_bytes, JsonString{}) catch return ZipponError.MemoryError; } diff --git a/src/file/entityWriter.zig b/src/file/entityWriter.zig index 2dec972..4dfc218 100644 --- a/src/file/entityWriter.zig +++ b/src/file/entityWriter.zig @@ -76,7 +76,7 @@ fn writeValue(writer: anytype, value: zid.Data, data_type: DataType) !void { } const uuid = try UUID.parse("00000000-0000-0000-0000-000000000000"); // Maybe pass that comptime to prevent parsing it everytime if (!std.meta.eql(v, uuid.bytes)) { - try writer.print("{{|<{s}>|}}", .{v}); + try writer.print("{{<|{s}|>}}", .{v}); } else { try writer.print("{{}}", .{}); } @@ -104,7 +104,7 @@ fn writeArray(writer: anytype, data: zid.Data, data_type: DataType) ZipponError! .IntArray => while (iter.next()) |v| writer.print("{d}, ", .{v.Int}) catch return ZipponError.WriteError, .FloatArray => while (iter.next()) |v| writer.print("{d}", .{v.Float}) catch return ZipponError.WriteError, .StrArray => while (iter.next()) |v| writer.print("\"{s}\"", .{v.Str}) catch return ZipponError.WriteError, - .UUIDArray => while (iter.next()) |v| writer.print("{{|<{s}>|}},", .{v.UUID}) catch return ZipponError.WriteError, + .UUIDArray => while (iter.next()) |v| writer.print("{{<|{s}|>}},", .{v.UUID}) catch return ZipponError.WriteError, .BoolArray => while (iter.next()) |v| writer.print("{any}", .{v.Bool}) catch return ZipponError.WriteError, .UnixArray => while (iter.next()) |v| { const datetime = DateTime.initUnix(v.Unix); @@ -122,12 +122,12 @@ fn writeArray(writer: anytype, data: zid.Data, data_type: DataType) ZipponError! writer.writeByte(']') catch return ZipponError.WriteError; } -/// Take a string in the JSON format and look for {|<[16]u8>|}, then will look into the map and check if it can find this UUID -/// If it find it, it ill replace the {|<[16]u8>|} will the value +/// Take a string in the JSON format and look for {<|[16]u8|>}, then will look into the map and check if it can find this UUID +/// If it find it, it ill replace the {<|[16]u8|>} will the value pub fn updateWithRelation(writer: anytype, input: []const u8, map: std.AutoHashMap([16]u8, JsonString)) ZipponError!void { var uuid_bytes: [16]u8 = undefined; var start: usize = 0; - while (std.mem.indexOf(u8, input[start..], "{|<")) |pos| { + while (std.mem.indexOf(u8, input[start..], "{<|")) |pos| { const pattern_start = start + pos + 3; const pattern_end = pattern_start + 16; @@ -155,7 +155,7 @@ pub fn updateWithRelation(writer: anytype, input: []const u8, map: std.AutoHashM fn updateArray(writer: anytype, input: []const u8, map: std.AutoHashMap([16]u8, JsonString), origin: usize) ZipponError!usize { var uuid_bytes: [16]u8 = undefined; var start = origin; - while (input.len > start + 23 and std.mem.eql(u8, input[start .. start + 3], "{|<") and std.mem.eql(u8, input[start + 19 .. start + 23], ">|},")) : (start += 23) { + while (input.len > start + 23 and std.mem.eql(u8, input[start .. start + 3], "{<|") and std.mem.eql(u8, input[start + 19 .. start + 23], "|>},")) : (start += 23) { @memcpy(uuid_bytes[0..], input[start + 3 .. start + 19]); if (map.get(uuid_bytes)) |json_string| { writer.writeAll(json_string.slice) catch return ZipponError.WriteError; diff --git a/src/file/read.zig b/src/file/read.zig index f101447..803993b 100644 --- a/src/file/read.zig +++ b/src/file/read.zig @@ -50,18 +50,17 @@ pub fn populateFileIndexUUIDMap( defer arena.deinit(); const allocator = arena.allocator(); - const max_file_index = try self.maxFileIndex(sstruct.name); - const dir = try self.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{}); + const to_parse = try self.allFileIndex(allocator, sstruct.name); // Multi-threading setup var sync_context = ThreadSyncContext.init( 0, - max_file_index + 1, + to_parse.len, ); // Create a thread-safe writer for each file - var thread_writer_list = allocator.alloc(std.ArrayList(UUID), max_file_index + 1) catch return ZipponError.MemoryError; + var thread_writer_list = allocator.alloc(std.ArrayList(UUID), to_parse.len) catch return ZipponError.MemoryError; defer { for (thread_writer_list) |list| list.deinit(); allocator.free(thread_writer_list); @@ -72,10 +71,10 @@ pub fn populateFileIndexUUIDMap( } // Spawn threads for each file - for (0..(max_file_index + 1)) |file_index| { + for (to_parse, 0..) |file_index, i| { self.thread_pool.spawn(populateFileIndexUUIDMapOneFile, .{ sstruct, - &thread_writer_list[file_index], + &thread_writer_list[i], file_index, dir, &sync_context, @@ -141,29 +140,29 @@ pub fn populateVoidUUIDMap( const allocator = arena.allocator(); const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name); - const max_file_index = try self.maxFileIndex(sstruct.name); const dir = try self.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{}); + const to_parse = try self.allFileIndex(allocator, sstruct.name); // Multi-threading setup var sync_context = ThreadSyncContext.init( additional_data.limit, - max_file_index + 1, + to_parse.len, ); // Create a thread-safe writer for each file - var thread_writer_list = allocator.alloc(std.ArrayList(UUID), max_file_index + 1) catch return ZipponError.MemoryError; + var thread_writer_list = allocator.alloc(std.ArrayList(UUID), to_parse.len + 1) catch return ZipponError.MemoryError; for (thread_writer_list) |*list| { list.* = std.ArrayList(UUID).init(allocator); } // Spawn threads for each file - for (0..(max_file_index + 1)) |file_index| { + for (to_parse, 0..) |file_index, i| { self.thread_pool.spawn(populateVoidUUIDMapOneFile, .{ sstruct, filter, - &thread_writer_list[file_index], + &thread_writer_list[i], file_index, dir, &sync_context, @@ -249,9 +248,7 @@ pub fn parseEntities( const writer = buff.writer(); const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name); - const max_file_index = try self.maxFileIndex(sstruct.name); - - log.debug("Max file index {d}", .{max_file_index}); + const to_parse = try self.allFileIndex(allocator, struct_name); // If there is no member to find, that mean we need to return all members, so let's populate additional data with all of them if (additional_data.childrens.items.len == 0) @@ -267,21 +264,21 @@ pub fn parseEntities( // Multi thread stuffs var sync_context = ThreadSyncContext.init( additional_data.limit, - max_file_index + 1, + to_parse.len, ); // Do an array of writer for each thread // Could I create just the number of max cpu ? Because if I have 1000 files, I do 1000 list // But at the end, only the number of use CPU/Thread will use list simultanously // So I could pass list from a thread to another technicly - var thread_writer_list = allocator.alloc(std.ArrayList(u8), max_file_index + 1) catch return ZipponError.MemoryError; + var thread_writer_list = allocator.alloc(std.ArrayList(u8), to_parse.len) catch return ZipponError.MemoryError; // Start parsing all file in multiple thread - for (0..(max_file_index + 1)) |file_index| { + for (to_parse, 0..) |file_index, i| { thread_writer_list[file_index] = std.ArrayList(u8).init(allocator); self.thread_pool.spawn(parseEntitiesOneFile, .{ - thread_writer_list[file_index].writer(), + thread_writer_list[i].writer(), file_index, dir, sstruct.zid_schema, @@ -303,11 +300,11 @@ pub fn parseEntities( // Now I need to do the relation stuff, meaning parsing new files to get the relationship value // Without relationship to return, this function is basically finish here - // Here I take the JSON string and I parse it to find all {|<>|} and add them to the relation map with an empty JsonString + // Here I take the JSON string and I parse it to find all {<||>} and add them to the relation map with an empty JsonString for (relation_maps) |*relation_map| try relation_map.populate(buff.items); // I then call parseEntitiesRelationMap on each - // This will update the buff items to be the same Json but with {|<[16]u8>|} replaced with the right Json + // This will update the buff items to be the same Json but with {<|[16]u8|>} replaced with the right Json for (relation_maps) |*relation_map| try self.parseEntitiesRelationMap(allocator, relation_map.struct_name, relation_map, &buff); return buff.toOwnedSlice() catch return ZipponError.MemoryError; @@ -362,9 +359,9 @@ fn parseEntitiesOneFile( // Receive a map of UUID -> empty JsonString // Will parse the files and update the value to the JSON string of the entity that represent the key -// Will then write the input with the JSON in the map looking for {|<>|} -// Once the new input received, call parseEntitiesRelationMap again the string still contain {|<>|} because of sub relationship -// The buffer contain the string with {|<>|} and need to be updated at the end +// Will then write the input with the JSON in the map looking for {<||>} +// Once the new input received, call parseEntitiesRelationMap again the string still contain {<||>} because of sub relationship +// The buffer contain the string with {<||>} and need to be updated at the end pub fn parseEntitiesRelationMap( self: *Self, parent_allocator: Allocator, @@ -449,13 +446,13 @@ pub fn parseEntitiesRelationMap( buff.clearRetainingCapacity(); buff.writer().writeAll(new_buff.items) catch return ZipponError.WriteError; - // Now here I need to iterate if buff.items still have {|<>|} + // Now here I need to iterate if buff.items still have {<||>} - // Here I take the JSON string and I parse it to find all {|<>|} and add them to the relation map with an empty JsonString + // Here I take the JSON string and I parse it to find all {<||>} and add them to the relation map with an empty JsonString for (relation_maps) |*sub_relation_map| try sub_relation_map.populate(buff.items); // I then call parseEntitiesRelationMap on each - // This will update the buff items to be the same Json but with {|<[16]u8>|} replaced with the right Json + // This will update the buff items to be the same Json but with {<|[16]u8|>} replaced with the right Json for (relation_maps) |*sub_relation_map| try parseEntitiesRelationMap(self, allocator, sub_relation_map.struct_name, sub_relation_map, buff); } diff --git a/src/file/utils.zig b/src/file/utils.zig index 9bebdc8..5a15db0 100644 --- a/src/file/utils.zig +++ b/src/file/utils.zig @@ -111,7 +111,6 @@ pub fn string2Data(allocator: Allocator, value: ConditionValue) ZipponError!zid. } /// Take a map from the parseNewData and return an ordered array of Data to be use in a DataWriter -/// TODO: Optimize and maybe put it somewhere else than fileEngine pub fn orderedNewData( self: *Self, allocator: Allocator, diff --git a/src/file/write.zig b/src/file/write.zig index 7461a7b..4d91ada 100644 --- a/src/file/write.zig +++ b/src/file/write.zig @@ -76,18 +76,18 @@ pub fn updateEntities( const allocator = arena.allocator(); const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name); - const max_file_index = try self.maxFileIndex(sstruct.name); const dir = try self.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{}); + const to_parse = try self.allFileIndex(allocator, struct_name); // Multi-threading setup var sync_context = ThreadSyncContext.init( additional_data.limit, - max_file_index + 1, + to_parse.len, ); // Create a thread-safe writer for each file - var thread_writer_list = allocator.alloc(std.ArrayList(u8), max_file_index + 1) catch return ZipponError.MemoryError; + var thread_writer_list = allocator.alloc(std.ArrayList(u8), to_parse.len) catch return ZipponError.MemoryError; for (thread_writer_list) |*list| { list.* = std.ArrayList(u8).init(allocator); } @@ -101,13 +101,13 @@ pub fn updateEntities( } // Spawn threads for each file - for (0..(max_file_index + 1)) |file_index| { + for (to_parse, 0..) |file_index, i| { self.thread_pool.spawn(updateEntitiesOneFile, .{ new_data_buff, sstruct, filter, &map, - thread_writer_list[file_index].writer(), + thread_writer_list[i].writer(), file_index, dir, &sync_context, @@ -239,28 +239,28 @@ pub fn deleteEntities( const allocator = arena.allocator(); const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name); - const max_file_index = try self.maxFileIndex(sstruct.name); const dir = try self.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{}); + const to_parse = try self.allFileIndex(allocator, struct_name); // Multi-threading setup var sync_context = ThreadSyncContext.init( additional_data.limit, - max_file_index + 1, + to_parse.len, ); // Create a thread-safe writer for each file - var thread_writer_list = allocator.alloc(std.ArrayList(u8), max_file_index + 1) catch return ZipponError.MemoryError; + var thread_writer_list = allocator.alloc(std.ArrayList(u8), to_parse.len) catch return ZipponError.MemoryError; for (thread_writer_list) |*list| { list.* = std.ArrayList(u8).init(allocator); } // Spawn threads for each file - for (0..(max_file_index + 1)) |file_index| { + for (to_parse, 0..) |file_index, i| { self.thread_pool.spawn(deleteEntitiesOneFile, .{ sstruct, filter, - thread_writer_list[file_index].writer(), + thread_writer_list[i].writer(), file_index, dir, &sync_context, @@ -277,7 +277,8 @@ pub fn deleteEntities( } writer.writeByte(']') catch return ZipponError.WriteError; - // Update UUID file index map FIXME: Stop doing that and just remove UUID from the map itself instead of reparsing everything at the end + // FIXME: Stop doing that and just remove UUID from the map itself instead of reparsing everything at the end + // It's just that I can't do it in deleteEntitiesOneFile itself sstruct.uuid_file_index.map.clearRetainingCapacity(); _ = sstruct.uuid_file_index.arena.reset(.free_all); try self.populateFileIndexUUIDMap(sstruct, sstruct.uuid_file_index); diff --git a/src/ziql/tokenizer.zig b/src/ziql/tokenizer.zig index a47ec48..2041921 100644 --- a/src/ziql/tokenizer.zig +++ b/src/ziql/tokenizer.zig @@ -19,6 +19,7 @@ pub const Token = struct { .{ "NOW", .keyword_now }, .{ "APPEND", .keyword_append }, .{ "POP", .keyword_pop }, + .{ "CLEAR", .keyword_clear }, .{ "REMOVE", .keyword_remove }, .{ "REMOVEAT", .keyword_remove_at }, .{ "grab", .keyword_grab }, @@ -56,6 +57,7 @@ pub const Token = struct { keyword_now, keyword_append, keyword_pop, + keyword_clear, keyword_remove, keyword_remove_at,