diff --git a/TECHNICAL_DOCS.md b/TECHNICAL_DOCS.md index 3ecd57c..a51b13d 100644 --- a/TECHNICAL_DOCS.md +++ b/TECHNICAL_DOCS.md @@ -1,5 +1,3 @@ -# How the file system work - # All class of ZipponDB ## Tokenizer diff --git a/build.zig b/build.zig index 2dbf1d9..1acb250 100644 --- a/build.zig +++ b/build.zig @@ -51,13 +51,13 @@ pub fn build(b: *std.Build) void { }); const run_tests4 = b.addRunArtifact(tests4); - //const tests5 = b.addTest(.{ - // .root_source_file = b.path("src/ADD.zig"), - // .target = target, - // .optimize = optimize, - // .name = "ADD", - //}); - //const run_tests5 = b.addRunArtifact(tests5); + const tests5 = b.addTest(.{ + .root_source_file = b.path("src/uuid.zig"), + .target = target, + .optimize = optimize, + .name = "UUID", + }); + const run_tests5 = b.addRunArtifact(tests5); const tests6 = b.addTest(.{ .root_source_file = b.path("src/GRAB.zig"), @@ -72,6 +72,6 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_tests2.step); test_step.dependOn(&run_tests3.step); test_step.dependOn(&run_tests4.step); - //test_step.dependOn(&run_tests5.step); + test_step.dependOn(&run_tests5.step); test_step.dependOn(&run_tests6.step); } diff --git a/src/ADD.zig b/src/ADD.zig index 0b4b0d8..ab96c23 100644 --- a/src/ADD.zig +++ b/src/ADD.zig @@ -1,14 +1,13 @@ const std = @import("std"); const dtypes = @import("dtypes.zig"); const UUID = @import("uuid.zig").UUID; -const ziqlTokenizer = @import("ziqlTokenizer.zig").Tokenizer; -const ziqlToken = @import("ziqlTokenizer.zig").Token; +const Tokenizer = @import("ziqlTokenizer.zig").Tokenizer; +const Token = @import("ziqlTokenizer.zig").Token; +const DataEngine = @import("dataEngine.zig").DataEngine; const Allocator = std.mem.Allocator; const stdout = std.io.getStdOut().writer(); -// TODO: Use a Parser struct like in GRAB - // Query that need to work now // ADD User (name='Adrien', email='adrien.bouvais@gmail.com') OK // ADD User (name='Adrien', email='adrien.bouvais@gmail.com', age = 26) OK @@ -24,330 +23,231 @@ const stdout = std.io.getStdOut().writer(); /// It will parse the reste of the query and create a map of member name / value. /// Then add those value to the appropriete file. The proper file is the first one with a size < to the limit. /// If no file is found, a new one is created. -pub fn parseDataAndAddToFile(allocator: Allocator, struct_name: []const u8, toker: *ziqlTokenizer) !void { - const token = toker.next(); - switch (token.tag) { - .l_paren => {}, - else => { - try stdout.print("Error: Expected ( after the struct name of an ADD command.\nE.g. ADD User (name = 'bob')\n", .{}); - return; - }, - } - - const buffer = try allocator.alloc(u8, 1024 * 100); - defer allocator.free(buffer); - - var member_map = getMapOfMember(allocator, toker) catch return; - defer member_map.deinit(); - - if (!checkIfAllMemberInMap(struct_name, &member_map)) return; - - const entity = try dtypes.createEntityFromMap(allocator, struct_name, member_map); - const uuid_str = entity.User.*.id.format_uuid(); - defer stdout.print("Added new {s} successfully using UUID: {s}\n", .{ - struct_name, - uuid_str, - }) catch {}; - - const member_names = dtypes.structName2structMembers(struct_name); - for (member_names) |member_name| { - var file_map = getFilesStat(allocator, struct_name, member_name) catch { - try stdout.print("Error: File stat error", .{}); - return; - }; - const potential_file_name_to_use = getFirstUsableFile(file_map); - if (potential_file_name_to_use) |file_name| { - const file_index = fileName2Index(file_name) catch @panic("Error in fileName2Index"); - try stdout.print("Using file: {s} with a size of {d}\n", .{ file_name, file_map.get(file_name).?.size }); - - const path = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/{s}", .{ - struct_name, - member_name, - file_name, - }); - - var file = std.fs.cwd().openFile(path, .{ - .mode = .read_write, - }) catch { - try stdout.print("Error opening data file.", .{}); - return; - }; - defer file.close(); - - try file.seekFromEnd(0); - try file.writer().print("{s} {s}\n", .{ uuid_str, member_map.get(member_name).? }); - - const path_to_main = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/main.zippondata", .{ - struct_name, - member_name, - }); - - var file_main = std.fs.cwd().openFile(path_to_main, .{ - .mode = .read_write, - }) catch { - try stdout.print("Error opening data file.", .{}); - return; - }; - defer file_main.close(); - - try appendToLineAtIndex(allocator, file_main, file_index, &uuid_str); - } else { - const max_index = maxFileIndex(file_map); - - const new_file_path = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/{d}.zippondata", .{ - struct_name, - member_name, - max_index + 1, - }); - - try stdout.print("new file path: {s}\n", .{new_file_path}); - - // TODO: Create new file and save the data inside - const new_file = std.fs.cwd().createFile(new_file_path, .{}) catch @panic("Error creating new data file"); - defer new_file.close(); - - try new_file.writer().print("{s} {s}\n", .{ &uuid_str, member_map.get(member_name).? }); - - const path_to_main = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/main.zippondata", .{ - struct_name, - member_name, - }); - - var file_main = std.fs.cwd().openFile(path_to_main, .{ - .mode = .read_write, - }) catch { - try stdout.print("Error opening data file.", .{}); - @panic(""); - }; - defer file_main.close(); - - try file_main.seekFromEnd(0); - try file_main.writeAll("\n "); - try file_main.seekTo(0); - try appendToLineAtIndex(allocator, file_main, max_index + 1, &uuid_str); - } - } -} - /// Take the main.zippondata file, the index of the file where the data is saved and the string to add at the end of the line -fn appendToLineAtIndex(allocator: std.mem.Allocator, file: std.fs.File, index: usize, str: []const u8) !void { - const buffer = try allocator.alloc(u8, 1024 * 100); - defer allocator.free(buffer); +pub const Parser = struct { + arena: std.heap.ArenaAllocator, + allocator: Allocator, - var reader = file.reader(); + toker: *Tokenizer, + data_engine: *DataEngine, - var line_num: usize = 1; - while (try reader.readUntilDelimiterOrEof(buffer, '\n')) |_| { - if (line_num == index) { - try file.seekBy(-1); - try file.writer().print("{s} ", .{str}); - return; - } - line_num += 1; - } -} - -/// Return a map of file path => Stat for one struct and member name -fn getFilesStat(allocator: Allocator, struct_name: []const u8, member_name: []const u8) !*std.StringHashMap(std.fs.File.Stat) { - const cwd = std.fs.cwd(); - - const buffer = try allocator.alloc(u8, 1024); // Adjust the size as needed - defer allocator.free(buffer); - - const path = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}", .{ struct_name, member_name }); - - var file_map = std.StringHashMap(std.fs.File.Stat).init(allocator); - - const member_dir = cwd.openDir(path, .{ .iterate = true }) catch { - try stdout.print("Error opening struct directory", .{}); - @panic(""); - }; - - var iter = member_dir.iterate(); - while (try iter.next()) |entry| { - if (entry.kind != std.fs.Dir.Entry.Kind.file) continue; - - const file_stat = member_dir.statFile(entry.name) catch - { - try stdout.print("Error getting stat of a file", .{}); - @panic(""); + pub fn init(allocator: Allocator, toker: *Tokenizer, data_engine: *DataEngine) Parser { + var arena = std.heap.ArenaAllocator.init(allocator); + return Parser{ + .arena = arena, + .allocator = arena.allocator(), + .toker = toker, + .data_engine = data_engine, }; - - file_map.put(entry.name, file_stat) catch @panic("Error adding stat to map"); } - return &file_map; -} - -/// Use the map of file stat to find the first file with under the bytes limit. -/// return the name of the file. If none is found, return null. -fn getFirstUsableFile(map: *std.StringHashMap(std.fs.File.Stat)) ?[]const u8 { - var iter = map.keyIterator(); - while (iter.next()) |key| { - if (std.mem.eql(u8, key.*, "main.zippondata")) continue; - if (map.get(key.*).?.size < dtypes.parameter_max_file_size_in_bytes) return key.*; + pub fn deinit(self: *Parser) void { + self.arena.deinit(); } - return null; -} -fn fileName2Index(file_name: []const u8) !usize { - try stdout.print("Got file name: {s}\n", .{file_name}); - var iter_file_name = std.mem.tokenize(u8, file_name, "."); - const num_str = iter_file_name.next().?; - const num: usize = try std.fmt.parseInt(usize, num_str, 10); - return num; -} - -/// Iter over all file and get the max name and return the value of it as i32 -/// So for example if there is 1.zippondata and 2.zippondata it return 2. -fn maxFileIndex(map: *std.StringHashMap(std.fs.File.Stat)) usize { - var iter = map.keyIterator(); - var index_max: usize = 0; - while (iter.next()) |key| { - if (std.mem.eql(u8, key.*, "main.zippondata")) continue; - var iter_file_name = std.mem.tokenize(u8, key.*, "."); - const num_str = iter_file_name.next().?; - const num: usize = std.fmt.parseInt(usize, num_str, 10) catch @panic("Error parsing file name into usize"); - if (num > index_max) index_max = num; - } - return index_max; -} - -const MemberMapError = error{ - NotMemberName, - NotEqualSign, - NotStringOrNumber, - NotComma, - PuttingNull, -}; - -/// Take the tokenizer and return a map of the query for the ADD command. -/// Keys are the member name and value are the string of the value in the query. E.g. 'Adrien' or '10' -pub fn getMapOfMember(allocator: Allocator, toker: *ziqlTokenizer) !std.StringHashMap([]const u8) { - std.debug.print("Started\n\n", .{}); - var token = toker.next(); - std.debug.print("{any}\n\n", .{token}); - - var member_map = std.StringHashMap([]const u8).init( - allocator, - ); - - std.debug.print("OK \n\n", .{}); - - while (token.tag != ziqlToken.Tag.eof) : ({ - token = toker.next(); - std.debug.print("{any}", .{token}); - }) { - std.debug.print("{any}\n\n", .{token}); + pub fn parse(self: *Parser, struct_name: []const u8) !void { + var token = self.toker.next(); switch (token.tag) { - .r_paren => continue, - .invalid => stdout.print("Error: Invalid token: {s}", .{toker.getTokenSlice(token)}) catch {}, - .identifier => { - const member_name_str = toker.getTokenSlice(token); - token = toker.next(); - switch (token.tag) { - .equal => { - token = toker.next(); - switch (token.tag) { - .string_literal, .number_literal => { - const value_str = toker.getTokenSlice(token); - member_map.put(member_name_str, value_str) catch @panic("Could not add member name and value to map in getMapOfMember"); - token = toker.next(); - switch (token.tag) { - .comma, .r_paren => continue, - else => { - stdout.print("Error: Expected , after string or number got: {s}. E.g. ADD User (name='bob', age=10)", .{toker.getTokenSlice(token)}) catch {}; - return MemberMapError.NotComma; - }, - } - }, - .keyword_null => { - try stdout.print("Found null value\n", .{}); - const value_str = "null"; - member_map.put(member_name_str, value_str) catch { - try stdout.print("Error putting null value into the map\n", .{}); - return MemberMapError.PuttingNull; - }; - token = toker.next(); - switch (token.tag) { - .comma, .r_paren => continue, - else => { - stdout.print("Error: Expected , after string or number got: {s}. E.g. ADD User (name='bob', age=10)", .{toker.getTokenSlice(token)}) catch {}; - return MemberMapError.NotComma; - }, - } - }, - .l_bracket => { - var array_values = std.ArrayList([]const u8).init(allocator); - token = toker.next(); - while (token.tag != ziqlToken.Tag.r_bracket) : (token = toker.next()) { - switch (token.tag) { - .string_literal, .number_literal => { - const value_str = toker.getTokenSlice(token); - array_values.append(value_str) catch @panic("Could not add value to array in getMapOfMember"); - }, - .invalid => stdout.print("Error: Invalid token: {s}", .{toker.getTokenSlice(token)}) catch {}, - else => { - stdout.print("Error: Expected string or number in array got: {s}. E.g. ADD User (scores=[10 20 30])", .{toker.getTokenSlice(token)}) catch {}; - return MemberMapError.NotStringOrNumber; - }, - } - } - const array_str = try std.mem.join(allocator, " ", array_values.items); - member_map.put(member_name_str, array_str) catch @panic("Could not add member name and value to map in getMapOfMember"); - }, // TODO - else => { - stdout.print("Error: Expected string or number after a = got: {s}. E.g. ADD User (name='bob')", .{toker.getTokenSlice(token)}) catch {}; - return MemberMapError.NotStringOrNumber; - }, - } - }, - else => { - stdout.print("Error: Expected = after a member declaration get {s}. E.g. ADD User (name='bob')", .{toker.getTokenSlice(token)}) catch {}; - return MemberMapError.NotEqualSign; - }, - } - }, + .l_paren => {}, else => { - stdout.print("Error: Unknow token: {s}. This should be the name of a member. E.g. name in ADD User (name='bob')", .{toker.getTokenSlice(token)}) catch {}; - return MemberMapError.NotMemberName; + try self.print_error("Error: Expected (", &token); }, } + + const buffer = try self.allocator.alloc(u8, 1024 * 100); + defer self.allocator.free(buffer); + + var data = self.parseData(); // data is a map with key as member name and value as str of the value inserted in the query. So age = 12 is the string 12 here + defer data.deinit(); + + if (!self.checkIfAllMemberInMap(struct_name, &data)) return; + + const entity = try dtypes.createEntityFromMap(self.allocator, struct_name, data); + const uuid_str = entity.User.*.id.format_uuid(); + defer stdout.print("Added new {s} successfully using UUID: {s}\n", .{ + struct_name, + uuid_str, + }) catch {}; + + const member_names = dtypes.structName2structMembers(struct_name); + for (member_names) |member_name| { + var file_map = self.data_engine.getFilesStat(struct_name, member_name) catch { + try stdout.print("Error: File stat error", .{}); + return; + }; + const potential_file_name_to_use = self.data_engine.getFirstUsableFile(file_map); + if (potential_file_name_to_use) |file_name| { + const file_index = self.data_engine.fileName2Index(file_name); + try stdout.print("Using file: {s} with a size of {d}\n", .{ file_name, file_map.get(file_name).?.size }); + + const path = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/{s}", .{ + struct_name, + member_name, + file_name, + }); + + var file = std.fs.cwd().openFile(path, .{ + .mode = .read_write, + }) catch { + try stdout.print("Error opening data file.", .{}); + return; + }; + defer file.close(); + + try file.seekFromEnd(0); + try file.writer().print("{s} {s}\n", .{ uuid_str, data.get(member_name).? }); + + const path_to_main = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/main.zippondata", .{ + struct_name, + member_name, + }); + + var file_main = std.fs.cwd().openFile(path_to_main, .{ + .mode = .read_write, + }) catch { + try stdout.print("Error opening data file.", .{}); + return; + }; + defer file_main.close(); + + try self.data_engine.appendToLineAtIndex(file_main, file_index, &uuid_str); + } else { + const max_index = self.data_engine.maxFileIndex(file_map); + + const new_file_path = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/{d}.zippondata", .{ + struct_name, + member_name, + max_index + 1, + }); + + try stdout.print("new file path: {s}\n", .{new_file_path}); + + // TODO: Create new file and save the data inside + const new_file = std.fs.cwd().createFile(new_file_path, .{}) catch @panic("Error creating new data file"); + defer new_file.close(); + + try new_file.writer().print("{s} {s}\n", .{ &uuid_str, data.get(member_name).? }); + + const path_to_main = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/main.zippondata", .{ + struct_name, + member_name, + }); + + var file_main = std.fs.cwd().openFile(path_to_main, .{ + .mode = .read_write, + }) catch { + try stdout.print("Error opening data file.", .{}); + @panic(""); + }; + defer file_main.close(); + + try file_main.seekFromEnd(0); + try file_main.writeAll("\n "); + try file_main.seekTo(0); + try self.data_engine.appendToLineAtIndex(file_main, max_index + 1, &uuid_str); + } + } } - return member_map; -} + /// Take the tokenizer and return a map of the query for the ADD command. + /// Keys are the member name and value are the string of the value in the query. E.g. 'Adrien' or '10' + pub fn parseData(self: *Parser) std.StringHashMap([]const u8) { + var token = self.toker.next(); -/// Using the name of a struct from dtypes and the map of member name => value string from the query. -/// Check if the map keys are exactly the same as the name of the member of the struct. -/// Basically checking if the query contain all value that a struct need to be init. -fn checkIfAllMemberInMap(struct_name: []const u8, map: *std.StringHashMap([]const u8)) bool { - const all_struct_member = dtypes.structName2structMembers(struct_name); - var count: u16 = 0; + var member_map = std.StringHashMap([]const u8).init( + self.allocator, + ); - for (all_struct_member) |key| { - if (map.contains(key)) count += 1 else stdout.print("Error: ADD query of struct: {s}; missing member: {s}\n", .{ - struct_name, - key, - }) catch {}; + while (token.tag != Token.Tag.eof) : (token = self.toker.next()) { + switch (token.tag) { + .r_paren => continue, + .identifier => { + const member_name_str = self.toker.getTokenSlice(token); + token = self.toker.next(); + switch (token.tag) { + .equal => { + token = self.toker.next(); + switch (token.tag) { + .string_literal, .number_literal => { + const value_str = self.toker.getTokenSlice(token); + member_map.put(member_name_str, value_str) catch @panic("Could not add member name and value to map in getMapOfMember"); + token = self.toker.next(); + switch (token.tag) { + .comma, .r_paren => continue, + else => self.print_error("Error: Expected , after string or number. E.g. ADD User (name='bob', age=10)", &token) catch {}, + } + }, + .keyword_null => { + const value_str = "null"; + member_map.put(member_name_str, value_str) catch self.print_error("Error: 001", &token) catch {}; + token = self.toker.next(); + switch (token.tag) { + .comma, .r_paren => continue, + else => self.print_error("Error: Expected , after string or number. E.g. ADD User (name='bob', age=10)", &token) catch {}, + } + }, + .l_bracket => { + var array_values = std.ArrayList([]const u8).init(self.allocator); + token = self.toker.next(); + while (token.tag != Token.Tag.r_bracket) : (token = self.toker.next()) { + switch (token.tag) { + .string_literal, .number_literal => { + const value_str = self.toker.getTokenSlice(token); + array_values.append(value_str) catch @panic("Could not add value to array in getMapOfMember"); + }, + else => self.print_error("Error: Expected string or number in array. E.g. ADD User (scores=[10 20 30])", &token) catch {}, + } + } + // Maybe change that as it just recreate a string that is already in the buffer + const array_str = std.mem.join(self.allocator, " ", array_values.items) catch @panic("Couln't join the value of array"); + member_map.put(member_name_str, array_str) catch @panic("Could not add member name and value to map in getMapOfMember"); + }, + else => self.print_error("Error: Expected string or number after =. E.g. ADD User (name='bob')", &token) catch {}, + } + }, + else => self.print_error("Error: Expected = after a member declaration. E.g. ADD User (name='bob')", &token) catch {}, + } + }, + else => self.print_error("Error: Unknow token. This should be the name of a member. E.g. name in ADD User (name='bob')", &token) catch {}, + } + } + + return member_map; } - return ((count == all_struct_member.len) and (count == map.count())); -} + fn checkIfAllMemberInMap(_: *Parser, struct_name: []const u8, map: *std.StringHashMap([]const u8)) bool { + const all_struct_member = dtypes.structName2structMembers(struct_name); + var count: u16 = 0; -test "Get map of members" { - // std.testing.refAllDecls(@This()); - // _ = @import("query_functions/ADD.zig").getMapOfMember; + for (all_struct_member) |key| { + if (map.contains(key)) count += 1 else stdout.print("Error: ADD query of struct: {s}; missing member: {s}\n", .{ + struct_name, + key, + }) catch {}; + } - const allocator = std.testing.allocator; + return ((count == all_struct_member.len) and (count == map.count())); + } - const in = "(name='Adrien', email='adrien@gmail.com', age=26, scores=[42 100 5])"; - const null_term_in = try allocator.dupeZ(u8, in); + fn print_error(self: *Parser, message: []const u8, token: *Token) !void { + try stdout.print("\n", .{}); + try stdout.print("{s}\n", .{self.toker.buffer}); - var toker = ziqlTokenizer.init(null_term_in); + // Calculate the number of spaces needed to reach the start position. + var spaces: usize = 0; + while (spaces < token.loc.start) : (spaces += 1) { + try stdout.print(" ", .{}); + } - const member_map = try getMapOfMember(allocator, &toker); - std.debug.print("{s}", .{member_map.get("name").?}); + // Print the '^' characters for the error span. + var i: usize = token.loc.start; + while (i < token.loc.end) : (i += 1) { + try stdout.print("^", .{}); + } + try stdout.print(" \n", .{}); // Align with the message - allocator.free(null_term_in); -} + try stdout.print("{s}\n", .{message}); + + @panic(""); + } +}; diff --git a/src/GRAB.zig b/src/GRAB.zig index f889c71..9add188 100644 --- a/src/GRAB.zig +++ b/src/GRAB.zig @@ -2,6 +2,8 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const Tokenizer = @import("ziqlTokenizer.zig").Tokenizer; const Token = @import("ziqlTokenizer.zig").Token; +const DataEngine = @import("dataEngine.zig").DataEngine; +const UUID = @import("uuid.zig").UUID; // To work now // GRAB User {} @@ -15,12 +17,31 @@ const Token = @import("ziqlTokenizer.zig").Token; const stdout = std.io.getStdOut().writer(); pub const Parser = struct { + arena: std.heap.ArenaAllocator, allocator: Allocator, toker: *Tokenizer, + data_engine: *DataEngine, state: State, additional_data: AdditionalData, + pub fn init(allocator: Allocator, toker: *Tokenizer, data_engine: *DataEngine) Parser { + var arena = std.heap.ArenaAllocator.init(allocator); + return Parser{ + .arena = arena, + .allocator = arena.allocator(), + .toker = toker, + .data_engine = data_engine, + .state = State.start, + .additional_data = AdditionalData.init(allocator), + }; + } + + pub fn deinit(self: *Parser) void { + self.additional_data.deinit(); + self.arena.deinit(); + } + // This is the [] part pub const AdditionalData = struct { entity_count_to_find: usize = 0, @@ -31,7 +52,10 @@ pub const Parser = struct { } pub fn deinit(self: *AdditionalData) void { - // Get all additional data that are in the list to also deinit them + for (self.member_to_find.items) |elem| { + elem.additional_data.deinit(); + } + self.member_to_find.deinit(); } }; @@ -63,20 +87,6 @@ pub const Parser = struct { expect_filter, }; - pub fn init(allocator: Allocator, toker: *Tokenizer) Parser { - return Parser{ - .allocator = allocator, - .toker = toker, - .state = State.start, - .additional_data = AdditionalData.init(allocator), - }; - } - - pub fn deinit(self: *Parser) void { - // FIXME: I think additionalData inside additionalData are not deinit - self.additional_data.deinit(); - } - pub fn parse(self: *Parser) !void { var token = self.toker.next(); while (self.state != State.end) : (token = self.toker.next()) { @@ -243,6 +253,37 @@ pub const Parser = struct { } }; +// TODO: Optimize. Maybe just do a new list and return it instead +fn OR(arr1: *std.ArrayList(UUID), arr2: *std.ArrayList(UUID)) std.ArrayList(UUID) { + defer arr1.deinit(); + defer arr2.deinit(); + + var arr = try arr1.clone(); + + for (0..arr2.items.len) |i| { + if (!arr.contains(arr2[i])) { + arr.append(arr2[i]); + } + } + + return arr; +} + +fn AND(arr1: *std.ArrayList(UUID), arr2: *std.ArrayList(UUID)) std.ArrayList(UUID) { + defer arr1.deinit(); + defer arr2.deinit(); + + var arr = try arr1.clone(); + + for (0..arr1.items.len) |i| { + if (arr2.contains(arr1[i])) { + arr.append(arr1[i]); + } + } + + return arr; +} + test "Test AdditionalData" { const allocator = std.testing.allocator; @@ -251,7 +292,7 @@ test "Test AdditionalData" { testAdditionalData("[1]", additional_data1); var additional_data2 = Parser.AdditionalData.init(allocator); - defer additional_data2.deinit(); + defer additional_data2.member_to_find.deinit(); try additional_data2.member_to_find.append( Parser.AdditionalDataMember.init( allocator, @@ -260,25 +301,64 @@ test "Test AdditionalData" { ); testAdditionalData("[name]", additional_data2); - std.debug.print("AdditionalData Parsing OK \n", .{}); + var additional_data3 = Parser.AdditionalData.init(allocator); + additional_data3.entity_count_to_find = 1; + defer additional_data3.member_to_find.deinit(); + try additional_data3.member_to_find.append( + Parser.AdditionalDataMember.init( + allocator, + "name", + ), + ); + testAdditionalData("[1; name]", additional_data3); + + var additional_data4 = Parser.AdditionalData.init(allocator); + additional_data4.entity_count_to_find = 100; + defer additional_data4.member_to_find.deinit(); + try additional_data4.member_to_find.append( + Parser.AdditionalDataMember.init( + allocator, + "friend", + ), + ); + testAdditionalData("[100; friend [name]]", additional_data4); } fn testAdditionalData(source: [:0]const u8, expected_AdditionalData: Parser.AdditionalData) void { const allocator = std.testing.allocator; var tokenizer = Tokenizer.init(source); - var additional_data = Parser.AdditionalData.init(allocator); + var data_engine = DataEngine.init(allocator); + defer data_engine.deinit(); + var parser = Parser.init(allocator, &tokenizer, &data_engine); + + defer parser.deinit(); _ = tokenizer.next(); - var parser = Parser.init(allocator, &tokenizer); - parser.parse_additional_data(&additional_data) catch |err| { + parser.parse_additional_data(&parser.additional_data) catch |err| { std.debug.print("Error parsing additional data: {any}\n", .{err}); }; - std.debug.print("{any}\n\n", .{additional_data}); + compareAdditionalData(expected_AdditionalData, parser.additional_data); +} - std.testing.expectEqual(expected_AdditionalData, additional_data) catch { - std.debug.print("Additional data are not equal for: {s}\n", .{source}); +// TODO: Check AdditionalData inside AdditionalData +fn compareAdditionalData(ad1: Parser.AdditionalData, ad2: Parser.AdditionalData) void { + std.testing.expectEqual(ad1.entity_count_to_find, ad2.entity_count_to_find) catch { + std.debug.print("Additional data entity_count_to_find are not equal.\n", .{}); }; - parser.deinit(); + var founded = false; + + for (ad1.member_to_find.items) |elem1| { + founded = false; + for (ad2.member_to_find.items) |elem2| { + if (std.mem.eql(u8, elem1.name, elem2.name)) { + compareAdditionalData(elem1.additional_data, elem2.additional_data); + founded = true; + break; + } + } + + if (!founded) @panic("Member not found"); + } } diff --git a/src/cliTokenizer.zig b/src/cliTokenizer.zig index 4816f5b..b23af91 100644 --- a/src/cliTokenizer.zig +++ b/src/cliTokenizer.zig @@ -159,7 +159,6 @@ pub const Tokenizer = struct { test "Basics" { try testTokenize("help", &.{.keyword_help}); try testTokenize("run \"Hello world\"", &.{ .keyword_run, .string_literal }); - std.debug.print("CLI tokenizer OK\n", .{}); } fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void { diff --git a/src/data-parsing.zig b/src/data-parsing.zig index 47bb77c..8357bf3 100644 --- a/src/data-parsing.zig +++ b/src/data-parsing.zig @@ -3,6 +3,7 @@ const std = @import("std"); // Series of functions to use just before creating an entity. // Will transform the string of data into data of the right type. +// Maybe return a null or something else pub fn parseInt(value_str: []const u8) i64 { return std.fmt.parseInt(i64, value_str, 10) catch return 0; } @@ -27,7 +28,6 @@ test "Data parsing" { for (in1, 0..) |value, i| { try std.testing.expect(parseInt(value) == expected_out1[i]); } - std.debug.print("OK\tData parsing: Int\n", .{}); // Int array const in2 = "[1 14 44 42 hello]"; @@ -35,5 +35,4 @@ test "Data parsing" { defer out2.deinit(); const expected_out2: [5]i64 = .{ 1, 14, 44, 42, 0 }; try std.testing.expect(std.mem.eql(i64, out2.items, &expected_out2)); - std.debug.print("OK\tData parsing: Int array\n", .{}); } diff --git a/src/dataEngine.zig b/src/dataEngine.zig new file mode 100644 index 0000000..8dd3c64 --- /dev/null +++ b/src/dataEngine.zig @@ -0,0 +1,103 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Tokenizer = @import("ziqlTokenizer.zig").Tokenizer; + +/// Manage everything that is relate to read or write in files +/// Or even get stats, whatever. If it touch files, it's here +pub const DataEngine = struct { + arena: std.heap.ArenaAllocator, + allocator: Allocator, + dir: std.fs.Dir, // The path to the DATA folder + max_file_size: usize = 1e+8, // 100mb + + pub fn init(allocator: Allocator) DataEngine { + var arena = std.heap.ArenaAllocator.init(allocator); + const dir = std.fs.cwd().openDir("ZipponDB/DATA", .{}) catch @panic("Error opening ZipponDB/DATA"); + return DataEngine{ + .arena = arena, + .allocator = arena.allocator(), + .dir = dir, + }; + } + + pub fn deinit(self: *DataEngine) void { + self.arena.deinit(); + } + + /// Iter over all file and get the max name and return the value of it as usize + /// So for example if there is 1.zippondata and 2.zippondata it return 2. + fn maxFileIndex(_: *DataEngine, map: *std.StringHashMap(std.fs.File.Stat)) usize { + var iter = map.keyIterator(); + var index_max: usize = 0; + while (iter.next()) |key| { + if (std.mem.eql(u8, key.*, "main.zippondata")) continue; + var iter_file_name = std.mem.tokenize(u8, key.*, "."); + const num_str = iter_file_name.next().?; + const num: usize = std.fmt.parseInt(usize, num_str, 10) catch @panic("Error parsing file name into usize"); + if (num > index_max) index_max = num; + } + return index_max; + } + + /// Use a filename in the format 1.zippondata and return the 1 + fn fileName2Index(_: *DataEngine, file_name: []const u8) usize { + var iter_file_name = std.mem.tokenize(u8, file_name, "."); + const num_str = iter_file_name.next().?; + const num: usize = std.fmt.parseInt(usize, num_str, 10) catch @panic("Couln't parse the int of a zippondata file."); + return num; + } + + /// Add an UUID at a specific index of a file + /// Used when some data are deleted from previous zippondata files and are now bellow the file size limit + fn appendToLineAtIndex(self: *DataEngine, file: std.fs.File, index: usize, str: []const u8) !void { + const buffer = try self.allocator.alloc(u8, 1024 * 100); + defer self.allocator.free(buffer); + + var reader = file.reader(); + + var line_num: usize = 1; + while (try reader.readUntilDelimiterOrEof(buffer, '\n')) |_| { + if (line_num == index) { + try file.seekBy(-1); + try file.writer().print("{s} ", .{str}); + return; + } + line_num += 1; + } + } + + /// Return a map of file path => Stat; for one struct and member name + /// E.g. for User & name + fn getFilesStat(self: *DataEngine, struct_name: []const u8, member_name: []const u8) !*std.StringHashMap(std.fs.File.Stat) { + const buffer = try self.allocator.alloc(u8, 1024); // Adjust the size as needed + defer self.allocator.free(buffer); + + const path = try std.fmt.bufPrint(buffer, "{s}{s}/{s}", .{ self.path.basename(), struct_name, member_name }); + + var file_map = std.StringHashMap(std.fs.File.Stat).init(self.allocator); + + const member_dir = self.path.openDir(path, .{ .iterate = true }) catch @panic("Error opening struct directory"); + + var iter = member_dir.iterate(); + while (try iter.next()) |entry| { + if (entry.kind != std.fs.Dir.Entry.Kind.file) continue; + + const file_stat = member_dir.statFile(entry.name) catch @panic("Error getting stat of a file"); + + file_map.put(entry.name, file_stat) catch @panic("Error adding stat to map"); + } + + return &file_map; + } + + /// Use the map of file stat to find the first file with under the bytes limit. + /// return the name of the file. If none is found, return null. + fn getFirstUsableFile(self: *DataEngine, map: *std.StringHashMap(std.fs.File.Stat)) ?[]const u8 { + var iter = map.keyIterator(); + while (iter.next()) |key| { + if (std.mem.eql(u8, key.*, "main.zippondata")) continue; + if (map.get(key.*).?.size < self.max_file_size) return key.*; + } + return null; + } +}; diff --git a/src/dbengine.zig b/src/dbengine.zig index 87eef59..2784b0d 100644 --- a/src/dbengine.zig +++ b/src/dbengine.zig @@ -1,11 +1,12 @@ const std = @import("std"); const dtypes = @import("dtypes.zig"); const UUID = @import("uuid.zig").UUID; -const ziqlTokenizer = @import("ziqlTokenizer.zig").Tokenizer; -const ziqlToken = @import("ziqlTokenizer.zig").Token; +const Tokenizer = @import("ziqlTokenizer.zig").Tokenizer; +const Token = @import("ziqlTokenizer.zig").Token; const grabParser = @import("GRAB.zig").Parser; +const addParser = @import("ADD.zig").Parser; +const DataEngine = @import("dataEngine.zig").DataEngine; const Allocator = std.mem.Allocator; -const parseDataAndAddToFile = @import("ADD.zig").parseDataAndAddToFile; pub const Error = error{UUIDNotFound}; const stdout = std.io.getStdOut().writer(); @@ -17,22 +18,6 @@ pub fn main() !void { const buffer = try allocator.alloc(u8, 1024); defer allocator.free(buffer); - // Init the map storage string map that track all array of struct - var storage = std.StringHashMap(*std.ArrayList(dtypes.Types)).init(allocator); - defer storage.deinit(); - - // Create all array and put them in the main map - // Use MultiArrayList in the future to save memory maybe ? - for (dtypes.struct_name_list) |struct_name| { - var array = std.ArrayList(dtypes.Types).init(allocator); - try storage.put(struct_name, &array); - } - - // Add user - //const adrien = dtypes.User.init("Adrien", "adrien@gmail.com"); - //try storage.get("User").?.append(dtypes.Types{ .User = &adrien }); - //const adrien_get = storage.get("User").?.items[0].User; - var args = try std.process.argsWithAllocator(allocator); defer args.deinit(); @@ -40,21 +25,28 @@ pub fn main() !void { _ = args.next(); const null_term_query_str = args.next().?; - var ziqlToker = ziqlTokenizer.init(null_term_query_str); - const first_token = ziqlToker.next(); - const struct_name_token = ziqlToker.next(); + var toker = Tokenizer.init(null_term_query_str); + const first_token = toker.next(); + const struct_name_token = toker.next(); + + var data_engine = DataEngine.init(allocator); switch (first_token.tag) { .keyword_grab => { - var parser = grabParser.init(allocator, &ziqlToker); + if (!isStructInSchema(toker.getTokenSlice(struct_name_token))) { + try stdout.print("Error: No struct named '{s}' in current schema.", .{toker.getTokenSlice(struct_name_token)}); + return; + } + var parser = grabParser.init(allocator, &toker, &data_engine); try parser.parse(); }, .keyword_add => { - if (!isStructInSchema(ziqlToker.getTokenSlice(struct_name_token))) { - try stdout.print("Error: No struct named '{s}' in current schema.", .{ziqlToker.getTokenSlice(struct_name_token)}); + if (!isStructInSchema(toker.getTokenSlice(struct_name_token))) { + try stdout.print("Error: No struct named '{s}' in current schema.", .{toker.getTokenSlice(struct_name_token)}); return; } - try parseDataAndAddToFile(allocator, ziqlToker.getTokenSlice(struct_name_token), &ziqlToker); + var parser = addParser.init(allocator, &toker, &data_engine); + try parser.parse(toker.getTokenSlice(struct_name_token)); }, .keyword_update => { try stdout.print("Not yet implemented.\n", .{}); diff --git a/src/dtypes.zig b/src/dtypes.zig index f9809b5..0b47a7e 100644 --- a/src/dtypes.zig +++ b/src/dtypes.zig @@ -2,8 +2,6 @@ const std = @import("std"); const UUID = @import("uuid.zig").UUID; const dataParsing = @import("data-parsing.zig"); -pub const parameter_max_file_size_in_bytes = 500; // THe number of bytes than each file can be before splitting - pub const User = struct { id: UUID, name: []const u8, diff --git a/src/schemaParser.zig b/src/schemaParser.zig index 0a1bf78..58c3dd4 100644 --- a/src/schemaParser.zig +++ b/src/schemaParser.zig @@ -1,7 +1,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; -const Toker = @import("tokenizers/schemaTokenizer.zig").Tokenizer; -const Token = @import("tokenizers/schemaTokenizer.zig").Token; +const Toker = @import("schemaTokenizer.zig").Tokenizer; +const Token = @import("schemaTokenizer.zig").Token; pub const Parser = struct { file: std.fs.File, diff --git a/src/schemaTokenizer.zig b/src/schemaTokenizer.zig index c0a2020..8a7b246 100644 --- a/src/schemaTokenizer.zig +++ b/src/schemaTokenizer.zig @@ -157,7 +157,6 @@ pub const Tokenizer = struct { test "keywords" { try testTokenize("int float str date", &.{ .type_int, .type_float, .type_str, .type_date }); - std.debug.print("Schema type OK\n", .{}); } test "basic query" { @@ -182,7 +181,6 @@ test "basic query" { .identifier, .r_paren, }); - std.debug.print("Schema OK\n", .{}); } fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void { diff --git a/src/uuid.zig b/src/uuid.zig index 0b2c110..b1905c5 100644 --- a/src/uuid.zig +++ b/src/uuid.zig @@ -165,9 +165,6 @@ test "check to_string works" { uuid1.to_string(&string1); uuid1.to_string(&string2); - std.debug.print("\nUUID {s} \n", .{uuid1}); - std.debug.print("\nFirst call to_string {s} \n", .{string1}); - std.debug.print("Second call to_string {s} \n", .{string2}); try testing.expectEqual(string1, string2); } diff --git a/src/ziqlTokenizer.zig b/src/ziqlTokenizer.zig index 0b5d708..0e3f10d 100644 --- a/src/ziqlTokenizer.zig +++ b/src/ziqlTokenizer.zig @@ -345,7 +345,6 @@ pub const Tokenizer = struct { test "keywords" { try testTokenize("GRAB UPDATE ADD DELETE IN", &.{ .keyword_grab, .keyword_update, .keyword_add, .keyword_delete, .keyword_in }); - std.debug.print("ZiQL keywords OK\n", .{}); } test "basic query" { @@ -354,7 +353,6 @@ test "basic query" { try testTokenize("GRAB User [1; name] {}", &.{ .keyword_grab, .identifier, .l_bracket, .number_literal, .semicolon, .identifier, .r_bracket, .l_brace, .r_brace }); try testTokenize("GRAB User{}|ASCENDING name|", &.{ .keyword_grab, .identifier, .l_brace, .r_brace, .pipe, .identifier, .identifier, .pipe }); try testTokenize("DELETE User[1]{name='Adrien'}|ASCENDING name, age|", &.{ .keyword_delete, .identifier, .l_bracket, .number_literal, .r_bracket, .l_brace, .identifier, .equal, .string_literal, .r_brace, .pipe, .identifier, .identifier, .comma, .identifier, .pipe }); - std.debug.print("ZiQL query OK\n", .{}); } fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {