From 475e00b13e9dbc2cf313ec19a5e938fd81ef9e1f Mon Sep 17 00:00:00 2001 From: MrBounty Date: Sun, 22 Sep 2024 22:58:13 +0200 Subject: [PATCH] Moved temporally everything to the root for testing test. Also small changes --- TODO.md | 12 - build.zig | 27 ++- src/{query_functions => }/ADD.zig | 34 ++- src/GRAB.zig | 284 +++++++++++++++++++++++ src/{tokenizers => }/cliTokenizer.zig | 0 src/{parsers => }/data-parsing.zig | 0 src/dbconsole.zig | 15 +- src/dbengine.zig | 12 +- src/dtypes.zig | 2 +- src/query_functions/DELETE.zig | 0 src/query_functions/GRAB.zig | 100 -------- src/{parsers => }/schemaParser.zig | 4 +- src/{tokenizers => }/schemaTokenizer.zig | 0 src/test.zig | 22 -- src/{tokenizers => }/ziqlTokenizer.zig | 0 15 files changed, 345 insertions(+), 167 deletions(-) delete mode 100644 TODO.md rename src/{query_functions => }/ADD.zig (94%) create mode 100644 src/GRAB.zig rename src/{tokenizers => }/cliTokenizer.zig (100%) rename src/{parsers => }/data-parsing.zig (100%) delete mode 100644 src/query_functions/DELETE.zig delete mode 100644 src/query_functions/GRAB.zig rename src/{parsers => }/schemaParser.zig (97%) rename src/{tokenizers => }/schemaTokenizer.zig (100%) delete mode 100644 src/test.zig rename src/{tokenizers => }/ziqlTokenizer.zig (100%) diff --git a/TODO.md b/TODO.md deleted file mode 100644 index c60e581..0000000 --- a/TODO.md +++ /dev/null @@ -1,12 +0,0 @@ -How to create a new entity ? - -## Engine roadmap -[ ] Generate empty data file -[ ] Parse data file -[ ] Add one User in the file -[ ] Can get all User with GRAB User -[ ] Filter User on name -[ ] Filter User on name and age -[ ] Get all Message of one User -[ ] Delete User -[ ] Update User diff --git a/build.zig b/build.zig index 815a529..2dbf1d9 100644 --- a/build.zig +++ b/build.zig @@ -20,7 +20,7 @@ pub fn build(b: *std.Build) void { // Test step const tests1 = b.addTest(.{ - .root_source_file = b.path("src/parsers/data-parsing.zig"), + .root_source_file = b.path("src/data-parsing.zig"), .target = target, .optimize = optimize, .name = "Data parsing", @@ -28,7 +28,7 @@ pub fn build(b: *std.Build) void { const run_tests1 = b.addRunArtifact(tests1); const tests2 = b.addTest(.{ - .root_source_file = b.path("src/tokenizers/cliTokenizer.zig"), + .root_source_file = b.path("src/cliTokenizer.zig"), .target = target, .optimize = optimize, .name = "CLI tokenizer", @@ -36,7 +36,7 @@ pub fn build(b: *std.Build) void { const run_tests2 = b.addRunArtifact(tests2); const tests3 = b.addTest(.{ - .root_source_file = b.path("src/tokenizers/ziqlTokenizer.zig"), + .root_source_file = b.path("src/ziqlTokenizer.zig"), .target = target, .optimize = optimize, .name = "ZiQL tokenizer", @@ -44,25 +44,34 @@ pub fn build(b: *std.Build) void { const run_tests3 = b.addRunArtifact(tests3); const tests4 = b.addTest(.{ - .root_source_file = b.path("src/tokenizers/schemaTokenizer.zig"), + .root_source_file = b.path("src/schemaTokenizer.zig"), .target = target, .optimize = optimize, .name = "Schema tokenizer", }); const run_tests4 = b.addRunArtifact(tests4); - const tests5 = b.addTest(.{ - .root_source_file = b.path("src/test.zig"), + //const tests5 = b.addTest(.{ + // .root_source_file = b.path("src/ADD.zig"), + // .target = target, + // .optimize = optimize, + // .name = "ADD", + //}); + //const run_tests5 = b.addRunArtifact(tests5); + + const tests6 = b.addTest(.{ + .root_source_file = b.path("src/GRAB.zig"), .target = target, .optimize = optimize, - .name = "ADD functions", + .name = "GRAB", }); - const run_tests5 = b.addRunArtifact(tests5); + const run_tests6 = b.addRunArtifact(tests6); const test_step = b.step("test", "Run unit tests"); test_step.dependOn(&run_tests1.step); test_step.dependOn(&run_tests2.step); test_step.dependOn(&run_tests3.step); test_step.dependOn(&run_tests4.step); - test_step.dependOn(&run_tests5.step); + //test_step.dependOn(&run_tests5.step); + test_step.dependOn(&run_tests6.step); } diff --git a/src/query_functions/ADD.zig b/src/ADD.zig similarity index 94% rename from src/query_functions/ADD.zig rename to src/ADD.zig index 2a1828f..0b4b0d8 100644 --- a/src/query_functions/ADD.zig +++ b/src/ADD.zig @@ -1,15 +1,13 @@ const std = @import("std"); -const dtypes = @import("../dtypes.zig"); -const UUID = @import("../uuid.zig").UUID; -const ziqlTokenizer = @import("../tokenizers/ziqlTokenizer.zig").Tokenizer; -const ziqlToken = @import("../tokenizers/ziqlTokenizer.zig").Token; +const dtypes = @import("dtypes.zig"); +const UUID = @import("uuid.zig").UUID; +const ziqlTokenizer = @import("ziqlTokenizer.zig").Tokenizer; +const ziqlToken = @import("ziqlTokenizer.zig").Token; const Allocator = std.mem.Allocator; const stdout = std.io.getStdOut().writer(); -// TODO to improve this part of the code: -// 1. Use logging -// 2. Create a struct that manage files with member: stdout, folder (e.g. the User folder), +// TODO: Use a Parser struct like in GRAB // Query that need to work now // ADD User (name='Adrien', email='adrien.bouvais@gmail.com') OK @@ -236,7 +234,10 @@ pub fn getMapOfMember(allocator: Allocator, toker: *ziqlTokenizer) !std.StringHa std.debug.print("OK \n\n", .{}); - while (token.tag != ziqlToken.Tag.eof) : (token = toker.next()) { + while (token.tag != ziqlToken.Tag.eof) : ({ + token = toker.next(); + std.debug.print("{any}", .{token}); + }) { std.debug.print("{any}\n\n", .{token}); switch (token.tag) { .r_paren => continue, @@ -333,3 +334,20 @@ fn checkIfAllMemberInMap(struct_name: []const u8, map: *std.StringHashMap([]cons return ((count == all_struct_member.len) and (count == map.count())); } + +test "Get map of members" { + // std.testing.refAllDecls(@This()); + // _ = @import("query_functions/ADD.zig").getMapOfMember; + + const allocator = std.testing.allocator; + + const in = "(name='Adrien', email='adrien@gmail.com', age=26, scores=[42 100 5])"; + const null_term_in = try allocator.dupeZ(u8, in); + + var toker = ziqlTokenizer.init(null_term_in); + + const member_map = try getMapOfMember(allocator, &toker); + std.debug.print("{s}", .{member_map.get("name").?}); + + allocator.free(null_term_in); +} diff --git a/src/GRAB.zig b/src/GRAB.zig new file mode 100644 index 0000000..f889c71 --- /dev/null +++ b/src/GRAB.zig @@ -0,0 +1,284 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Tokenizer = @import("ziqlTokenizer.zig").Tokenizer; +const Token = @import("ziqlTokenizer.zig").Token; + +// To work now +// GRAB User {} +// GRAB User {name = 'Adrien'} +// GRAB User {name='Adrien' AND age < 30} +// GRAB User [1] {} +// GRAB User [10; name] {age < 30} +// +// For later + +const stdout = std.io.getStdOut().writer(); + +pub const Parser = struct { + allocator: Allocator, + toker: *Tokenizer, + state: State, + + additional_data: AdditionalData, + + // This is the [] part + pub const AdditionalData = struct { + entity_count_to_find: usize = 0, + member_to_find: std.ArrayList(AdditionalDataMember), + + pub fn init(allocator: Allocator) AdditionalData { + return AdditionalData{ .member_to_find = std.ArrayList(AdditionalDataMember).init(allocator) }; + } + + pub fn deinit(self: *AdditionalData) void { + // Get all additional data that are in the list to also deinit them + self.member_to_find.deinit(); + } + }; + + // This is name in: [name] + // There is an additional data because it can be [friend [1; name]] + const AdditionalDataMember = struct { + name: []const u8, + additional_data: AdditionalData, + + pub fn init(allocator: Allocator, name: []const u8) AdditionalDataMember { + const additional_data = AdditionalData.init(allocator); + return AdditionalDataMember{ .name = name, .additional_data = additional_data }; + } + }; + + const State = enum { + start, + invalid, + end, + + // For the additional data + expect_count_of_entity_to_find, + expect_semicolon_OR_right_bracket, + expect_member, + next_member_OR_end_OR_new_additional_data, + next_member_OR_end, + + expect_filter, + }; + + pub fn init(allocator: Allocator, toker: *Tokenizer) Parser { + return Parser{ + .allocator = allocator, + .toker = toker, + .state = State.start, + .additional_data = AdditionalData.init(allocator), + }; + } + + pub fn deinit(self: *Parser) void { + // FIXME: I think additionalData inside additionalData are not deinit + self.additional_data.deinit(); + } + + pub fn parse(self: *Parser) !void { + var token = self.toker.next(); + while (self.state != State.end) : (token = self.toker.next()) { + switch (self.state) { + .start => { + switch (token.tag) { + .l_bracket => { + try self.parse_additional_data(&self.additional_data); + }, + else => { + try stdout.print("Found {any}\n", .{token.tag}); + + return; + }, + } + }, + else => return, + } + } + } + + /// When this function is call, the tokenizer last token retrieved should be [. + /// Check if an int is here -> check if ; is here -> check if member is here -> check if [ is here -> loop + pub fn parse_additional_data(self: *Parser, additional_data: *AdditionalData) !void { + var token = self.toker.next(); + var skip_next = false; + self.state = State.expect_count_of_entity_to_find; + + while (self.state != State.end) : ({ + token = if (!skip_next) self.toker.next() else token; + skip_next = false; + }) { + switch (self.state) { + .expect_count_of_entity_to_find => { + switch (token.tag) { + .number_literal => { + const count = std.fmt.parseInt(usize, self.toker.getTokenSlice(token), 10) catch { + try stdout.print( + "Error parsing query: {s} need to be a number.", + .{self.toker.getTokenSlice(token)}, + ); + self.state = .invalid; + continue; + }; + additional_data.entity_count_to_find = count; + self.state = .expect_semicolon_OR_right_bracket; + }, + else => { + self.state = .expect_member; + skip_next = true; + }, + } + }, + .expect_semicolon_OR_right_bracket => { + switch (token.tag) { + .semicolon => { + self.state = .expect_member; + }, + .r_bracket => { + return; + }, + else => { + try self.print_error( + "Error: Expect ';' or ']'.", + &token, + ); + self.state = .invalid; + }, + } + }, + .expect_member => { + switch (token.tag) { + .identifier => { + // TODO: Check if the member name exist + try additional_data.member_to_find.append( + AdditionalDataMember.init( + self.allocator, + self.toker.getTokenSlice(token), + ), + ); + + self.state = .next_member_OR_end_OR_new_additional_data; + }, + else => { + try self.print_error( + "Error: A member name should be here.", + &token, + ); + }, + } + }, + .next_member_OR_end_OR_new_additional_data => { + switch (token.tag) { + .comma => { + self.state = .expect_member; + }, + .r_bracket => { + return; + }, + .l_bracket => { + try self.parse_additional_data( + &additional_data.member_to_find.items[additional_data.member_to_find.items.len - 1].additional_data, + ); + self.state = .next_member_OR_end; + }, + else => { + try self.print_error( + "Error: Expected a comma ',' or the end or a new list of member to return.", + &token, + ); + }, + } + }, + .next_member_OR_end => { + switch (token.tag) { + .comma => { + try stdout.print("Expected new member\n", .{}); + self.state = .expect_member; + }, + .r_bracket => { + return; + }, + else => { + try self.print_error( + "Error: Expected a new member name or the end of the list of member name to return.", + &token, + ); + }, + } + }, + .invalid => { + @panic("=)"); + }, + else => { + try self.print_error( + "Error: Unknow state.", + &token, + ); + }, + } + } + } + + fn print_error(self: *Parser, message: []const u8, token: *Token) !void { + try stdout.print("\n", .{}); + try stdout.print("{s}\n", .{self.toker.buffer}); + + // Calculate the number of spaces needed to reach the start position. + var spaces: usize = 0; + while (spaces < token.loc.start) : (spaces += 1) { + try stdout.print(" ", .{}); + } + + // Print the '^' characters for the error span. + var i: usize = token.loc.start; + while (i < token.loc.end) : (i += 1) { + try stdout.print("^", .{}); + } + try stdout.print(" \n", .{}); // Align with the message + + try stdout.print("{s}\n", .{message}); + + @panic(""); + } +}; + +test "Test AdditionalData" { + const allocator = std.testing.allocator; + + var additional_data1 = Parser.AdditionalData.init(allocator); + additional_data1.entity_count_to_find = 1; + testAdditionalData("[1]", additional_data1); + + var additional_data2 = Parser.AdditionalData.init(allocator); + defer additional_data2.deinit(); + try additional_data2.member_to_find.append( + Parser.AdditionalDataMember.init( + allocator, + "name", + ), + ); + testAdditionalData("[name]", additional_data2); + + std.debug.print("AdditionalData Parsing OK \n", .{}); +} + +fn testAdditionalData(source: [:0]const u8, expected_AdditionalData: Parser.AdditionalData) void { + const allocator = std.testing.allocator; + var tokenizer = Tokenizer.init(source); + var additional_data = Parser.AdditionalData.init(allocator); + + _ = tokenizer.next(); + var parser = Parser.init(allocator, &tokenizer); + parser.parse_additional_data(&additional_data) catch |err| { + std.debug.print("Error parsing additional data: {any}\n", .{err}); + }; + + std.debug.print("{any}\n\n", .{additional_data}); + + std.testing.expectEqual(expected_AdditionalData, additional_data) catch { + std.debug.print("Additional data are not equal for: {s}\n", .{source}); + }; + + parser.deinit(); +} diff --git a/src/tokenizers/cliTokenizer.zig b/src/cliTokenizer.zig similarity index 100% rename from src/tokenizers/cliTokenizer.zig rename to src/cliTokenizer.zig diff --git a/src/parsers/data-parsing.zig b/src/data-parsing.zig similarity index 100% rename from src/parsers/data-parsing.zig rename to src/data-parsing.zig diff --git a/src/dbconsole.zig b/src/dbconsole.zig index eaec210..dde6342 100644 --- a/src/dbconsole.zig +++ b/src/dbconsole.zig @@ -1,10 +1,10 @@ const std = @import("std"); const Allocator = std.mem.Allocator; -const cliTokenizer = @import("tokenizers/cliTokenizer.zig").Tokenizer; -const cliToken = @import("tokenizers/cliTokenizer.zig").Token; -const schemaTokenizer = @import("tokenizers/schemaTokenizer.zig").Tokenizer; -const schemaToken = @import("tokenizers/schemaTokenizer.zig").Token; -const schemaParser = @import("parsers/schemaParser.zig").Parser; +const cliTokenizer = @import("cliTokenizer.zig").Tokenizer; +const cliToken = @import("cliTokenizer.zig").Token; +const schemaTokenizer = @import("schemaTokenizer.zig").Tokenizer; +const schemaToken = @import("schemaTokenizer.zig").Token; +const schemaParser = @import("schemaParser.zig").Parser; pub fn main() !void { checkAndCreateDirectories(); @@ -189,9 +189,10 @@ fn runCommand(null_term_query_str: [:0]const u8) void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; const allocator = gpa.allocator(); - const argv = &[_][]const u8{ "./engine", null_term_query_str }; + // TODO: Use the folder ENGINE + const args = &[_][]const u8{ "./engine", null_term_query_str }; - const result = std.process.Child.run(.{ .allocator = allocator, .argv = argv }) catch |err| switch (err) { + const result = std.process.Child.run(.{ .allocator = allocator, .argv = args, .max_output_bytes = 4084 }) catch |err| switch (err) { error.FileNotFound => { std.debug.print("No engine found, please use `schema build` to make one.\n", .{}); return; diff --git a/src/dbengine.zig b/src/dbengine.zig index 6f52b05..87eef59 100644 --- a/src/dbengine.zig +++ b/src/dbengine.zig @@ -1,11 +1,11 @@ const std = @import("std"); const dtypes = @import("dtypes.zig"); const UUID = @import("uuid.zig").UUID; -const ziqlTokenizer = @import("tokenizers/ziqlTokenizer.zig").Tokenizer; -const ziqlToken = @import("tokenizers/ziqlTokenizer.zig").Token; -const grabParser = @import("query_functions/GRAB.zig").Parser; +const ziqlTokenizer = @import("ziqlTokenizer.zig").Tokenizer; +const ziqlToken = @import("ziqlTokenizer.zig").Token; +const grabParser = @import("GRAB.zig").Parser; const Allocator = std.mem.Allocator; -const parseDataAndAddToFile = @import("query_functions/ADD.zig").parseDataAndAddToFile; +const parseDataAndAddToFile = @import("ADD.zig").parseDataAndAddToFile; pub const Error = error{UUIDNotFound}; const stdout = std.io.getStdOut().writer(); @@ -46,8 +46,8 @@ pub fn main() !void { switch (first_token.tag) { .keyword_grab => { - var parser = grabParser.init(&ziqlToker); - try parser.parse_additional_data(); + var parser = grabParser.init(allocator, &ziqlToker); + try parser.parse(); }, .keyword_add => { if (!isStructInSchema(ziqlToker.getTokenSlice(struct_name_token))) { diff --git a/src/dtypes.zig b/src/dtypes.zig index 96438dd..f9809b5 100644 --- a/src/dtypes.zig +++ b/src/dtypes.zig @@ -1,6 +1,6 @@ const std = @import("std"); const UUID = @import("uuid.zig").UUID; -const dataParsing = @import("parsers/data-parsing.zig"); +const dataParsing = @import("data-parsing.zig"); pub const parameter_max_file_size_in_bytes = 500; // THe number of bytes than each file can be before splitting diff --git a/src/query_functions/DELETE.zig b/src/query_functions/DELETE.zig deleted file mode 100644 index e69de29..0000000 diff --git a/src/query_functions/GRAB.zig b/src/query_functions/GRAB.zig deleted file mode 100644 index c5a28f4..0000000 --- a/src/query_functions/GRAB.zig +++ /dev/null @@ -1,100 +0,0 @@ -const std = @import("std"); -const Allocator = std.mem.Allocator; -const ziqlTokenizer = @import("../tokenizers/ziqlTokenizer.zig").Tokenizer; - -// To work now -// GRAB User {} -// GRAB User {name = 'Adrien'} -// GRAB User {name='Adrien' AND age < 30} -// GRAB User [1] {} -// GRAB User [10; name] {age < 30} -// -// For later - -const stdout = std.io.getStdOut().writer(); - -const AdditionalData = struct { - entity_to_find: usize = 0, - member_to_find: std.ArrayList([]const u8), - - pub fn init(allocator: Allocator) AdditionalData { - return AdditionalData{ .member_to_find = std.ArrayList(u8).init(allocator) }; - } -}; - -pub const Parser = struct { - allocator: Allocator, - additional_data: AdditionalData, - toker: *ziqlTokenizer, - state: State, - - const State = enum { - start, - invalid, - end, - - expect_additional_data, - expect_count_of_entity_to_find, - expect_semicolon, - - expect_filter, - }; - - pub fn init(allocator: Allocator, toker: *ziqlTokenizer) Parser { - return Parser{ .allocator = allocator, .toker = toker, .state = State.expect_additional_data, .additional_data = AdditionalData.init(allocator) }; - } - - pub fn deinit(self: *Parser) void { - self.additional_data.member_to_find.deinit(); - } - - pub fn parse_additional_data(self: *Parser) !void { - var token = self.toker.next(); - while (self.state != State.end) : (token = self.toker.next()) { - switch (self.state) { - .expect_additional_data => { - switch (token.tag) { - .l_bracket => { - try stdout.print("Additional data found.\n", .{}); - self.state = State.expect_count_of_entity_to_find; - }, - else => { - try stdout.print("No additional data found.\n", .{}); - self.state = State.expect_filter; - }, - } - }, - .expect_count_of_entity_to_find => { - switch (token.tag) { - .number_literal => { - try stdout.print("Count of entity found.\n", .{}); - self.state = State.expect_semicolon; - }, - else => { - try stdout.print("No count of entity found.\n", .{}); - self.state = State.expect_filter; - }, - } - }, - .expect_semicolon => { - switch (token.tag) { - .semicolon => { - try stdout.print("Found semiconlon.\n", .{}); - self.state = State.expect_semicolon; - }, - else => { - try stdout.print("Expected semicon here: {s}.\n", .{self.toker.buffer[token.loc.start - 5 .. token.loc.end + 5]}); - self.state = State.invalid; - }, - } - }, - .invalid => { - return; - }, - else => { - try stdout.print("End\n", .{}); - }, - } - } - } -}; diff --git a/src/parsers/schemaParser.zig b/src/schemaParser.zig similarity index 97% rename from src/parsers/schemaParser.zig rename to src/schemaParser.zig index 9951b3a..0a1bf78 100644 --- a/src/parsers/schemaParser.zig +++ b/src/schemaParser.zig @@ -1,7 +1,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; -const Toker = @import("../tokenizers/schemaTokenizer.zig").Tokenizer; -const Token = @import("../tokenizers/schemaTokenizer.zig").Token; +const Toker = @import("tokenizers/schemaTokenizer.zig").Tokenizer; +const Token = @import("tokenizers/schemaTokenizer.zig").Token; pub const Parser = struct { file: std.fs.File, diff --git a/src/tokenizers/schemaTokenizer.zig b/src/schemaTokenizer.zig similarity index 100% rename from src/tokenizers/schemaTokenizer.zig rename to src/schemaTokenizer.zig diff --git a/src/test.zig b/src/test.zig deleted file mode 100644 index 35f736a..0000000 --- a/src/test.zig +++ /dev/null @@ -1,22 +0,0 @@ -const std = @import("std"); -const dtypes = @import("dtypes.zig"); -const UUID = @import("uuid.zig").UUID; -const ziqlTokenizer = @import("tokenizers/ziqlTokenizer.zig").Tokenizer; -const ziqlToken = @import("tokenizers/ziqlTokenizer.zig").Token; - -// Test for functions in for_add.zig -const getMapOfMember = @import("query_functions/ADD.zig").getMapOfMember; - -test "Get map of members" { - const allocator = std.testing.allocator; - - const in = "(name='Adrien', email='adrien@gmail.com', age=26, scores=[42 100 5])"; - const null_term_in = try allocator.dupeZ(u8, in); - - var toker = ziqlTokenizer.init(null_term_in); - - const member_map = try getMapOfMember(allocator, &toker); - std.debug.print("{s}", .{member_map.get("name").?}); - - allocator.free(null_term_in); -} diff --git a/src/tokenizers/ziqlTokenizer.zig b/src/ziqlTokenizer.zig similarity index 100% rename from src/tokenizers/ziqlTokenizer.zig rename to src/ziqlTokenizer.zig