diff --git a/README.md b/README.md index 6e2454b..89995df 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,14 @@ Then you do what you want with it, including: import zippondb as zdb client = zdb.newClient('path/to/binary') -print(client.run('describe')) +print(client.exe('describe')) -users = client.run('GRAB User {}') +# Return named tuple +users = client.exe('GRAB User {}') for user in users: print(user.name) -client.run('save') +client.exe('save') ``` # Benchmark diff --git a/TODO.md b/TODO.md index 2873be2..c60e581 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,12 @@ -# UUID -[ ] Create new random UUID -[ ] Find one UUID in an array +How to create a new entity ? -Possible way to look for data +## Engine roadmap +[ ] Generate empty data file +[ ] Parse data file +[ ] Add one User in the file +[ ] Can get all User with GRAB User +[ ] Filter User on name +[ ] Filter User on name and age +[ ] Get all Message of one User +[ ] Delete User +[ ] Update User diff --git a/ZipponDB/DATA/Message/1.zippondata b/ZipponDB/DATA/Message/1.zippondata new file mode 100644 index 0000000..e69de29 diff --git a/ZipponDB/DATA/Message/main.zippondata b/ZipponDB/DATA/Message/main.zippondata new file mode 100644 index 0000000..e69de29 diff --git a/ZipponDB/DATA/User/1.zippondata b/ZipponDB/DATA/User/1.zippondata new file mode 100644 index 0000000..e69de29 diff --git a/ZipponDB/DATA/User/main.zippondata b/ZipponDB/DATA/User/main.zippondata new file mode 100644 index 0000000..e69de29 diff --git a/build.zig b/build.zig index 8f17023..f142c8b 100644 --- a/build.zig +++ b/build.zig @@ -1,66 +1,39 @@ const std = @import("std"); -// Although this function looks imperative, note that its job is to -// declaratively construct a build graph that will be executed by an external -// runner. pub fn build(b: *std.Build) void { - // Standard target options allows the person running `zig build` to choose - // what target to build for. Here we do not override the defaults, which - // means any target is allowed, and the default is native. Other options - // for restricting supported target set are available. + // Build part const target = b.standardTargetOptions(.{}); - - // Standard optimization options allow the person running `zig build` to select - // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall. Here we do not - // set a preferred release mode, allowing the user to decide how to optimize. const optimize = b.standardOptimizeOption(.{}); - const exe = b.addExecutable(.{ .name = "zippon", - .root_source_file = b.path("src/main.zig"), + .root_source_file = b.path("src/dbconsole.zig"), .target = target, .optimize = optimize, }); - - // This declares intent for the executable to be installed into the - // standard location when the user invokes the "install" step (the default - // step when running `zig build`). b.installArtifact(exe); - - // This *creates* a Run step in the build graph, to be executed when another - // step is evaluated that depends on it. The next line below will establish - // such a dependency. const run_cmd = b.addRunArtifact(exe); - - // By making the run step depend on the install step, it will be run from the - // installation directory rather than directly from within the cache directory. - // This is not necessary, however, if the application depends on other installed - // files, this ensures they will be present and in the expected location. run_cmd.step.dependOn(b.getInstallStep()); - // This allows the user to pass arguments to the application in the build - // command itself, like this: `zig build run -- arg1 arg2 etc` - if (b.args) |args| { - run_cmd.addArgs(args); - } - - // This creates a build step. It will be visible in the `zig build --help` menu, - // and can be selected like this: `zig build run` - // This will evaluate the `run` step rather than the default, which is "install". + // Run step const run_step = b.step("run", "Run the app"); run_step.dependOn(&run_cmd.step); - const exe_unit_tests = b.addTest(.{ - .root_source_file = b.path("src/main.zig"), + // Test step + const cliTokenizer_tests = b.addTest(.{ + .root_source_file = b.path("src/tokenizers/cliTokenizer.zig"), .target = target, .optimize = optimize, + .name = "CLID Tokenizer test", }); - - const run_exe_unit_tests = b.addRunArtifact(exe_unit_tests); - - // Similar to creating the run step earlier, this exposes a `test` step to - // the `zig build --help` menu, providing a way for the user to request - // running the unit tests. + const ziqlTokenizer_tests = b.addTest(.{ + .root_source_file = b.path("src/tokenizers/ziqlTokenizer.zig"), + .target = target, + .optimize = optimize, + .name = "ZiQL Tokenizer test", + }); + const run_cliTokenizer_tests = b.addRunArtifact(cliTokenizer_tests); + const run_ziqlTokenizer_tests = b.addRunArtifact(ziqlTokenizer_tests); const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&run_exe_unit_tests.step); + test_step.dependOn(&run_cliTokenizer_tests.step); + test_step.dependOn(&run_ziqlTokenizer_tests.step); } diff --git a/engine b/engine new file mode 100755 index 0000000..bc35e25 Binary files /dev/null and b/engine differ diff --git a/engine.o b/engine.o new file mode 100644 index 0000000..6ddb03a Binary files /dev/null and b/engine.o differ diff --git a/logo.jpeg b/logo.jpeg new file mode 100644 index 0000000..b8eef0c Binary files /dev/null and b/logo.jpeg differ diff --git a/logo.jpeg:Zone.Identifier b/logo.jpeg:Zone.Identifier new file mode 100644 index 0000000..048b14d --- /dev/null +++ b/logo.jpeg:Zone.Identifier @@ -0,0 +1,4 @@ +[ZoneTransfer] +ZoneId=3 +ReferrerUrl=https://mail.google.com/ +HostUrl=https://mail-attachment.googleusercontent.com/attachment/u/0/?ui=2&ik=b279f47372&attid=0.1&permmsgid=msg-a:r8703073128856945292&th=191d2610e6ac0507&view=att&disp=safe&realattid=191d260f98767218d6b1&saddbat=ANGjdJ-PHqY7IKd_rjkeXoDIC6965LqdvgX38a9gJ6IQbDXX7B87E1As8ppcdRNY8De4kzGBLCEKZAzdNCdBkizisq-GFnmfKRFQZNw_FnKqMJ7K64Wq1UrRKP9FfeB2quAuXebgOWjZQ58oyXR6W_pvCNjRqccl558diPyzIQdUfA49qmDBQSwLzK5NPZe2kPWpY70pYS_Rn4DTKZdAcG7c3zLqW-uRYHoSGhNKQ-r9R5r2ML78F3gVPFVLvOlLIASM8gcDS4eDBWjhExrpxi8D-pTMoJbr8NFA7oF9cfUbUFk_eHRnF_a8O1KfgfzzoOdBlkcEIT4T8omx_gLgA8GkgFT37smp8jjmSd-RnzyOr5Gx4GJnjv6sWx1Q1kNktMR79LKcvlNCCoPoYaQebu4iJQzqZ6tJ1w00zHnGzoZowkEqZGaL40U9fq8LXowWA9I7mpPvKliqYn2hInMU51Zb70qpV3a_2i2dv7BuafUcGaI9RPbf4wN3FMNkTwvqRpx_NBaO8vPlX3g3ore283EWBQwBJphMjbRpGiwDUmBAqBN3YF7Vy4Y_EDh_yLHV4uDslcXbNRflCl4NDQL8RNvyu7KSPWH8dOD5HsXHETgaDHCS1zIXmbgtzRaDtDp_dRTMEvF7ibxEF5O8IHZTfWCZA0L4tCG3YODGMu8edU6Re2Ptw4jFGUQlsODpjMaOkSjKAmxukqq4yw2qZE9RR5CYMY8m_ZNWIyg2DXBvWp2as6wYcyc2an7zNyWCANetu0xYCYN2Rbsrh7XnhfnAho7SjfI74k2wp1MZM5Z07ZIbc5fRApMkAreFVlnevA9AAKNyP-USLSxHsZ-C5Q-JZl-MEoqaevwuezhXlktl-2RH1xAvwIAvchi-a55LDMIogcBhgo_hLBT2KrTKfqMRayFpemk_u9cl8UYdiNUjqjQWufbFx9b6MIlIO6eIcJFb1XkzXeIMhcThNL63hYgJSdh1G4k_57zF5V9HB1pNeoa4Ve5jNu25qkuutlK6c3A diff --git a/schema.zipponschema b/schema.zipponschema new file mode 100644 index 0000000..cc18a46 --- /dev/null +++ b/schema.zipponschema @@ -0,0 +1,8 @@ +User ( + name: str, + email: str, +) + +Message ( + content: str, +) diff --git a/src/dbconsole.zig b/src/dbconsole.zig new file mode 100644 index 0000000..3891b53 --- /dev/null +++ b/src/dbconsole.zig @@ -0,0 +1,226 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const cliTokenizer = @import("tokenizers/cliTokenizer.zig").Tokenizer; +const cliToken = @import("tokenizers/cliTokenizer.zig").Token; +const schemaTokenizer = @import("tokenizers/schemaTokenizer.zig").Tokenizer; +const schemaToken = @import("tokenizers/schemaTokenizer.zig").Token; +const schemaParser = @import("parsers/schemaParser.zig").Parser; + +pub fn main() !void { + checkAndCreateDirectories(); + + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + const allocator = gpa.allocator(); + + while (true) { + std.debug.print("> ", .{}); + var line_buf: [1024]u8 = undefined; + const line = try std.io.getStdIn().reader().readUntilDelimiterOrEof(&line_buf, '\n'); + + if (line) |line_str| { + const null_term_line_str = try allocator.dupeZ(u8, line_str[0..line_str.len]); + + var cliToker = cliTokenizer.init(null_term_line_str); + const command_token = cliToker.next(); + switch (command_token.tag) { + .keyword_run => { + const query_token = cliToker.next(); + switch (query_token.tag) { + .string_literal => { + const null_term_query_str = try allocator.dupeZ(u8, line_str[query_token.loc.start + 1 .. query_token.loc.end - 1]); + runCommand(null_term_query_str); + }, + else => { + std.debug.print("After command run, need a string of a query, eg: \"GRAB User\"\n", .{}); + continue; + }, + } + }, + .keyword_schema => { + const second_token = cliToker.next(); + + switch (second_token.tag) { + .keyword_describe => { + runCommand("__DESCRIBE__"); + }, + .keyword_build => { + const file_name_token = cliToker.next(); + var file_name = try allocator.alloc(u8, 1024); + var len: usize = 0; + + switch (file_name_token.tag) { + .eof => { + std.mem.copyForwards(u8, file_name, "schema.zipponschema"); + len = 19; + }, + else => file_name = line_str[file_name_token.loc.start..file_name_token.loc.end], + } + + std.debug.print("{s}", .{file_name[0..len]}); + + //blk: { + //createDtypeFile(file_name[0..len]) catch |err| switch (err) { + // error.FileNotFound => { + // std.debug.print("Error: Can't find file: {s}\n", .{file_name[0..len]}); + // break :blk; + // }, + // else => { + // std.debug.print("Error: Unknow error when creating Dtype file: {any}\n", .{err}); + // break :blk; + // }, + //}; + try buildEngine(); + //} + allocator.free(file_name); + }, + .keyword_help => { + std.debug.print("{s}", .{ + \\Here are all available options to use with the schema command: + \\ + \\describe Print the schema use by the current engine. + \\build Build a new engine using a schema file. Args => filename: str, path of schema file to use. Default 'schema.zipponschema'. + \\ + }); + }, + else => { + std.debug.print("schema available options: describe, build & help\n", .{}); + }, + } + }, + .keyword_help => { + std.debug.print("{s}", .{ + \\Welcome to ZipponDB! + \\ + \\run To run a query. Args => query: str, the query to execute. + \\schema Build a new engine and print current schema. + \\kill To stop the process without saving + \\save Save the database to the normal files. + \\dump Create a new folder with all data as copy. Args => foldername: str, the name of the folder. + \\bump Replace current data with a previous dump. Args => foldername: str, the name of the folder. + \\ + }); + }, + .keyword_quit => { + break; + }, + else => { + std.debug.print("Command need to start with a keyword, including: run, schema, help and quit\n", .{}); + }, + } + } + } +} + +fn createDtypeFile(file_path: []const u8) !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + const allocator = gpa.allocator(); + + const cwd = std.fs.cwd(); + var file = try cwd.openFile(file_path, .{ + .mode = .read_only, + }); + defer file.close(); + + const buffer = try file.readToEndAlloc(allocator, 1024); + const file_contents = try allocator.dupeZ(u8, buffer[0..]); // Duplicate to a null terminated string + + var schemaToker = schemaTokenizer.init(file_contents); + var parser = schemaParser.init(); + parser.parse(&schemaToker, buffer); + + // Free memory + allocator.free(buffer); + allocator.free(file_contents); + const check = gpa.deinit(); + switch (check) { + .ok => return, + .leak => std.debug.print("Error: Leak in createDtypeFile!\n", .{}), + } +} + +fn buildEngine() !void { + const argv = &[_][]const u8{ + "zig", + "build-exe", + "src/dbengine.zig", + "--name", + "engine", + }; + + var child = std.process.Child.init(argv, std.heap.page_allocator); + try child.spawn(); + _ = try child.wait(); + + const dtypes = @import("dtypes.zig"); + + const data_dir = try std.fs.cwd().openDir("ZipponDB/DATA", .{}); + for (dtypes.struct_name_list) |struct_name| { + data_dir.makeDir(struct_name) catch |err| switch (err) { + error.PathAlreadyExists => {}, + else => @panic("Error other than path already exists when trying to create the a member directory.\n"), + }; + const dir = try data_dir.openDir(struct_name, .{}); + _ = dir.createFile("main.zippondata", .{}) catch |err| switch (err) { + error.PathAlreadyExists => {}, + else => @panic("Error: can't create main.zippondata"), + }; + _ = dir.createFile("1.zippondata", .{}) catch |err| switch (err) { + error.PathAlreadyExists => {}, + else => @panic("Error: can't create 1.zippondata"), + }; + } +} + +fn runCommand(null_term_query_str: [:0]const u8) void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + const allocator = gpa.allocator(); + + const argv = &[_][]const u8{ "./engine", null_term_query_str }; + + const result = std.process.Child.run(.{ .allocator = allocator, .argv = argv }) catch |err| switch (err) { + error.FileNotFound => { + std.debug.print("No engine found, please use `schema build` to make one.\n", .{}); + return; + }, + else => { + std.debug.print("Error: Unknow error when trying to run the engine: {any}\n", .{err}); + return; + }, + }; + switch (result.term) { + .Exited => {}, + .Signal => std.debug.print("Error: term signal in runCommand\n", .{}), + .Stopped => std.debug.print("Error: term stopped in runCommand\n", .{}), + .Unknown => std.debug.print("Error: term unknow in runCommand\n", .{}), + } + + std.debug.print("{s}\n", .{result.stdout}); + + allocator.free(result.stdout); + allocator.free(result.stderr); + + const check = gpa.deinit(); + switch (check) { + .ok => return, + .leak => std.debug.print("Error: Leak in runCommand!\n", .{}), + } +} + +fn checkAndCreateDirectories() void { + const cwd = std.fs.cwd(); + + cwd.makeDir("ZipponDB") catch |err| switch (err) { + error.PathAlreadyExists => {}, + else => @panic("Error other than path already exists when trying to create the ZipponDB directory.\n"), + }; + + cwd.makeDir("ZipponDB/DATA") catch |err| switch (err) { + error.PathAlreadyExists => {}, + else => @panic("Error other than path already exists when trying to create the DATA directory.\n"), + }; + + cwd.makeDir("ZipponDB/ENGINE") catch |err| switch (err) { + error.PathAlreadyExists => {}, + else => @panic("Error other than path already exists when trying to create the ENGINE directory.\n"), + }; +} diff --git a/src/dbengine.zig b/src/dbengine.zig new file mode 100644 index 0000000..58ea1aa --- /dev/null +++ b/src/dbengine.zig @@ -0,0 +1,92 @@ +const std = @import("std"); +const dtypes = @import("dtypes.zig"); +const UUID = @import("uuid.zig").UUID; +const ziqlTokenizer = @import("tokenizers/ziqlTokenizer.zig").Tokenizer; +const ziqlToken = @import("tokenizers/ziqlTokenizer.zig").Token; +const Allocator = std.mem.Allocator; + +pub const Error = error{UUIDNotFound}; +const stdout = std.io.getStdOut().writer(); + +pub fn main() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + const allocator = gpa.allocator(); + + // Init the map storage string map that track all array of struct + var storage = std.StringHashMap(*std.ArrayList(dtypes.Types)).init(allocator); + defer storage.deinit(); + + // Create all array and put them in the main map + // Use MultiArrayList in the future to save memory maybe ? + for (dtypes.struct_name_list) |struct_name| { + var array = std.ArrayList(dtypes.Types).init(allocator); + try storage.put(struct_name, &array); + } + + // Add user + const adrien = dtypes.User.init("Adrien", "adrien@gmail.com"); + try storage.get("User").?.append(dtypes.Types{ .User = &adrien }); + const adrien_get = storage.get("User").?.items[0].User; + + if (std.meta.eql(adrien_get, &adrien)) { + try stdout.print("adrien == adrien_get\n\n", .{}); + } + + // Add a new user + // const newUser = try dtypes.User.new(allocator, "Adrien", "adrien@gmail.com"); + // try storage.get("User").?.append(dtypes.Types{ .user = newUser }); + + var args = try std.process.argsWithAllocator(allocator); + defer args.deinit(); + + // Remove the first argument + _ = args.next(); + const null_term_query_str = args.next(); + + var ziqlToker = ziqlTokenizer.init(null_term_query_str.?); + const firstToken = ziqlToker.next(); + switch (firstToken.tag) { + .keyword_grab => { + try stdout.print("Hello from engine\n", .{}); + }, + .keyword_add => { + try stdout.print("Not yet implemented.\n", .{}); + }, + .keyword_update => { + try stdout.print("Not yet implemented.\n", .{}); + }, + .keyword_delete => { + try stdout.print("Not yet implemented.\n", .{}); + }, + .keyword__describe__ => { + try stdout.print("{s}", .{dtypes.describe_str}); + }, + else => { + try stdout.print("Query need to start with a keyword, including: GRAB ADD UPDATE DELETE\n", .{}); + }, + } +} + +fn getById(array: anytype, id: UUID) !*dtypes.User { + for (array.items) |data| { + if (data.id.compare(id)) { + return data; + } + } + return error.UUIDNotFound; +} + +// Function to add and test: +// - Create one entity +// - Search one entity filtering a list of key/value. Eg: User with name = 'Adrien' and age > 10 + +test "getById" { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + const allocator = gpa.allocator(); + var users = std.ArrayList(*dtypes.User).init(allocator); + try users.append(try dtypes.User.new(allocator, "Adrien", "adrien@gmail.com")); + + const adrien = try getById(users, users.items[0].id); + + try std.testing.expect(UUID.compare(users.items[0].id, adrien.id)); +} diff --git a/src/dtypes.zig b/src/dtypes.zig index 53c1959..4a67679 100644 --- a/src/dtypes.zig +++ b/src/dtypes.zig @@ -1,42 +1,40 @@ const std = @import("std"); const UUID = @import("uuid.zig").UUID; -pub const Types = union { - user: *User, - message: *Message, -}; +pub const parameter_max_file_size = 1e+7; // THe number of bytes than each file can be before splitting pub const User = struct { id: UUID, name: []const u8, email: []const u8, - messages: std.ArrayList(*Message), - pub fn new(allocator: std.mem.Allocator, name: []const u8, email: []const u8) !*User { - const user = try allocator.create(User); - user.* = .{ - .id = UUID.init(), - .name = name, - .email = email, - .messages = std.ArrayList(*Message).init(allocator), - }; - return user; + pub fn init(name: []const u8, email: []const u8) User { + return User{ .id = UUID.init(), .name = name, .email = email }; } }; pub const Message = struct { id: UUID, content: []const u8, - user: *User, - pub fn new(allocator: std.mem.Allocator, content: []const u8, user: *User) !*Message { - const message = try allocator.create(Message); - message.* = .{ - .id = UUID.init(), - .content = content, - .user = user, - }; - try user.*.messages.append(message); - return message; + pub fn init(content: []const u8) Message { + return Message{ .id = UUID.init(), .content = content }; } }; + +pub const Types = union { + User: *const User, + Message: *const Message, +}; + +pub const struct_name_list: [2][]const u8 = .{ + "User", + "Message", +}; + +pub const struct_member_list: [2][][]const u8 = .{ + .{ "name", "email" }, + .{"content"}, +}; + +pub const describe_str = "User (\n name: str,\n email: str,\n)\n\nMessage (\n content: str,\n)\n"; diff --git a/src/dtypes_example.zig b/src/dtypes_example.zig new file mode 100644 index 0000000..cf3b47f --- /dev/null +++ b/src/dtypes_example.zig @@ -0,0 +1,33 @@ +const std = @import("std"); +const UUID = @import("uuid.zig").UUID; + +pub const User = struct { + id: UUID, + name: []u8, + email: []u8, + + pub fn init(id: UUID, name: []const u8, email: []const u8) User { + return User{ .id = id, .name = name, .email = email }; + } +}; + +pub const Message = struct { + id: UUID, + content: []u8, + + pub fn init(id: UUID, content: []const u8) Message { + return Message{ .id = id, .content = content }; + } +}; + +pub const Types = union { + User: *User, + Message: *Message, +}; + +pub const struct_name_list: [2][]const u8 = .{ + "User", + "Message", +}; + +pub const describe_str = "User (\n name: str,\n email: str,\n)\n\nMessage (\n content: str,\n)\n"; diff --git a/src/main.zig b/src/main.zig deleted file mode 100644 index 80682c3..0000000 --- a/src/main.zig +++ /dev/null @@ -1,106 +0,0 @@ -const std = @import("std"); -const UUID = @import("uuid.zig").UUID; -const dtypes = @import("dtypes.zig"); -const ziqlTokenizer = @import("ziqlTokenizer.zig").Tokenizer; -const ziqlToken = @import("ziqlTokenizer.zig").Token; -const cliTokenizer = @import("cliTokenizer.zig").Tokenizer; -const cliToken = @import("cliTokenizer.zig").Token; -const Allocator = std.mem.Allocator; -const print = std.debug.print; - -pub const Error = error{UUIDNotFound}; - -const Commands = enum { run, describe, help, unknow, @"run describe help unknow" }; - -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - const allocator = gpa.allocator(); - - // Init the map storage string map that track all array of struct - var storage = std.StringHashMap(*std.ArrayList(dtypes.Types)).init(allocator); - defer storage.deinit(); - - // Create all array and put them in the main map - var userArray = std.ArrayList(dtypes.Types).init(allocator); - try storage.put("User", &userArray); - - var postArray = std.ArrayList(dtypes.Types).init(allocator); - try storage.put("Post", &postArray); - - var commentArray = std.ArrayList(dtypes.Types).init(allocator); - try storage.put("Comment", &commentArray); - - // Add a new user - const newUser = try dtypes.User.new(allocator, "Adrien", "adrien@gmail.com"); - try storage.get("User").?.append(dtypes.Types{ .user = newUser }); - - std.debug.print("{s}\n", .{storage.get("User").?.items[0].user.email}); - - while (true) { - std.debug.print("> ", .{}); - var line_buf: [1024]u8 = undefined; - const line = try std.io.getStdIn().reader().readUntilDelimiterOrEof(&line_buf, '\n'); - if (line) |line_str| { - const null_term_line_str = try allocator.dupeZ(u8, line_str[0..line_str.len]); - - var cliToker = cliTokenizer.init(null_term_line_str); - const commandToken = cliToker.next(); - switch (commandToken.tag) { - .keyword_run => { - const query_token = cliToker.next(); - switch (query_token.tag) { - .string_literal => { - std.debug.print("Running query: {s}\n", .{line_str[query_token.loc.start + 1 .. query_token.loc.end - 1]}); - }, - else => { - std.debug.print("After command run, need a string of a query, eg: \"GRAB User\"\n", .{}); - continue; - }, - } - }, - .keyword_describe => { - std.debug.print("Current schema: \n\nUser (\n\tid: UUID,\n\tname; str,\n\temail: str,\n\tmessages: []Message\n)\n\nMessage (\n\tid: UUID,\n\tcontent; str,\n\tfrom: User,\n)\n", .{}); - }, - .keyword_help => { - std.debug.print("Welcome to ZipponDB.\n\nrun\t\tTo run a query. Args: query: str, the query to execute.\ndescribe\tTo print the current schema.\nkill\t\tTo stop the process without saving\nsave\t\tSave the database to the normal files.\ndump\t\tCreate a new folder with all data as copy. Args: foldername: str, the name of the folder.\nbump\t\tReplace current data with a previous dump; Note: Save the current state with the dump command. Args: foldername: str, the name of the folder to use.\n", .{}); - }, - .keyword_quit => { - break; - }, - else => { - std.debug.print("Command need to start with a keyword, including: run, describe, help and quit\n", .{}); - }, - } - } - } -} - -fn getById(array: anytype, id: UUID) !*dtypes.User { - for (array.items) |data| { - if (data.id.compare(id)) { - return data; - } - } - return error.UUIDNotFound; -} - -fn startsWithDoubleQuote(s: []const u8) bool { - if (s.len < 2) return false; - return s[0] == '"' and s[s.len - 1] == '"'; -} - -fn endsWithDoubleQuote(s: []const u8) bool { - if (s.len < 2) return false; - return s[s.len - 1] == '"'; -} - -test "getById" { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - const allocator = gpa.allocator(); - var users = std.ArrayList(*dtypes.User).init(allocator); - try users.append(try dtypes.User.new(allocator, "Adrien", "adrien@gmail.com")); - - const adrien = try getById(users, users.items[0].id); - - try std.testing.expect(UUID.compare(users.items[0].id, adrien.id)); -} diff --git a/src/parsers/schemaParser.zig b/src/parsers/schemaParser.zig new file mode 100644 index 0000000..22cf235 --- /dev/null +++ b/src/parsers/schemaParser.zig @@ -0,0 +1,180 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Toker = @import("../tokenizers/schemaTokenizer.zig").Tokenizer; +const Token = @import("../tokenizers/schemaTokenizer.zig").Token; + +pub const Parser = struct { + file: std.fs.File, + + const State = enum { + start, + invalid, + + expect_l_paren, + expect_r_paren, + expect_member_name, + expect_two_dot, + expect_value_type, + expect_comma, + }; + + pub fn init() Parser { + return .{ + .file = undefined, + }; + } + + fn writeToFile(self: *const Parser, text: []const u8) void { + const bytes_written = self.file.write(text) catch |err| { + std.debug.print("Error when writing dtypes.zig: {}", .{err}); + return; + }; + _ = bytes_written; + } + + pub fn parse(self: *Parser, toker: *Toker, buffer: []u8) void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + const allocator = gpa.allocator(); + var struct_array = std.ArrayList([]u8).init(allocator); + + var state: State = .start; + + std.fs.cwd().deleteFile("src/dtypes.zig") catch {}; + + self.file = std.fs.cwd().createFile("src/dtypes.zig", .{}) catch |err| { + std.debug.print("Error when writing dtypes.zig: {}", .{err}); + return; + }; + defer self.file.close(); + + self.writeToFile("const std = @import(\"std\");\nconst UUID = @import(\"uuid.zig\").UUID;\n\n"); + + var token = toker.next(); + while (token.tag != Token.Tag.eof) : (token = toker.next()) { + switch (state) { + .start => switch (token.tag) { + .identifier => { + state = .expect_l_paren; + self.writeToFile("pub const "); + self.writeToFile(buffer[token.loc.start..token.loc.end]); + self.writeToFile(" = struct {\n"); + self.writeToFile(" id: UUID,\n"); + + // TODO: Check if struct name is already use + struct_array.append(buffer[token.loc.start..token.loc.end]) catch @panic("Error appending a struct name."); + }, + else => { + state = .invalid; + }, + }, + .expect_l_paren => switch (token.tag) { + .l_paren => { + state = .expect_member_name; + }, + else => { + state = .invalid; + }, + }, + .expect_member_name => switch (token.tag) { + .identifier => { + state = .expect_two_dot; + self.writeToFile(" "); + self.writeToFile(buffer[token.loc.start..token.loc.end]); + }, + .r_paren => { + state = .start; + self.writeToFile("};\n\n"); + }, + else => { + state = .invalid; + }, + }, + .expect_two_dot => switch (token.tag) { + .two_dot => { + state = .expect_value_type; + self.writeToFile(": "); + }, + else => { + state = .invalid; + }, + }, + .expect_value_type => switch (token.tag) { + .type_int => { + state = .expect_comma; + self.writeToFile("i64"); + }, + .type_str => { + state = .expect_comma; + self.writeToFile("[] u8"); + }, + .type_float => { + state = .expect_comma; + self.writeToFile("f64"); + }, + .type_date => { + @panic("Date not yet implemented"); + }, + .identifier => { + @panic("Link not yet implemented"); + }, + .lr_bracket => { + @panic("Array not yet implemented"); + }, + else => { + state = .invalid; + }, + }, + .expect_comma => switch (token.tag) { + .comma => { + state = .expect_member_name; + self.writeToFile(",\n"); + }, + else => { + state = .invalid; + }, + }, + .invalid => { + // TODO: Better errors + @panic("Error: Schema need to start with an Identifier."); + }, + else => { + @panic(""); + }, + } + } + + // Make the union `Type` with all different struct + self.writeToFile("pub const Types = union {\n"); + for (struct_array.items) |struct_name| { + self.writeToFile(" "); + self.writeToFile(struct_name); + self.writeToFile(": *"); + self.writeToFile(struct_name); + self.writeToFile(",\n"); + } + self.writeToFile("};\n\n"); + + // Make an array of struct name + self.writeToFile("pub const struct_name_list: ["); + var int_buffer: [20]u8 = undefined; + const len = std.fmt.formatIntBuf(&int_buffer, @as(usize, struct_array.items.len), 10, .lower, .{}); + self.writeToFile(int_buffer[0..len]); + self.writeToFile("][]const u8 = .{ "); + for (struct_array.items) |struct_name| { + self.writeToFile(" \""); + self.writeToFile(struct_name); + self.writeToFile("\", "); + } + self.writeToFile("};\n\n"); + + // Create the var that contain the description of the current schema to be printed when running: + // The query "__DESCRIBE__" on the engine + // Or the command `schema describe` on the console + self.writeToFile("pub const describe_str = \""); + var escaped_text: [1024]u8 = undefined; + const replacement_count = std.mem.replace(u8, buffer, "\n", "\\n", &escaped_text); + const escaped_text_len = replacement_count + buffer.len; + self.writeToFile(escaped_text[0..escaped_text_len]); + self.writeToFile("\";"); + } +}; diff --git a/src/cliTokenizer.zig b/src/tokenizers/cliTokenizer.zig similarity index 92% rename from src/cliTokenizer.zig rename to src/tokenizers/cliTokenizer.zig index 991d366..4816f5b 100644 --- a/src/cliTokenizer.zig +++ b/src/tokenizers/cliTokenizer.zig @@ -14,16 +14,11 @@ pub const Token = struct { .{ "run", .keyword_run }, .{ "help", .keyword_help }, .{ "describe", .keyword_describe }, + .{ "build", .keyword_build }, + .{ "schema", .keyword_schema }, .{ "quit", .keyword_quit }, }); - pub fn isKeyword(self: Token) bool { - switch (self.tag) { - .keyword_run, .keyword_describe, .keyword_help, .keyword_quit => return true, - else => return false, - } - } - pub fn getKeyword(bytes: []const u8) ?Tag { return keywords.get(bytes); } @@ -35,6 +30,8 @@ pub const Token = struct { keyword_run, keyword_help, keyword_describe, + keyword_schema, + keyword_build, keyword_quit, string_literal, @@ -46,11 +43,6 @@ pub const Tokenizer = struct { buffer: [:0]const u8, index: usize, - /// For debugging purposes. - pub fn dump(self: *Tokenizer, token: *const Token) void { - std.debug.print("{s} \"{s}\"\n", .{ @tagName(token.tag), self.buffer[token.loc.start..token.loc.end] }); - } - pub fn init(buffer: [:0]const u8) Tokenizer { // Skip the UTF-8 BOM if present. return .{ diff --git a/src/tokenizers/schemaTokenizer.zig b/src/tokenizers/schemaTokenizer.zig new file mode 100644 index 0000000..c0a2020 --- /dev/null +++ b/src/tokenizers/schemaTokenizer.zig @@ -0,0 +1,201 @@ +// From https://github.com/ziglang/zig/blob/master/lib/std/zig/tokenizer.zig +const std = @import("std"); + +pub const Token = struct { + tag: Tag, + loc: Loc, + + pub const Loc = struct { + start: usize, + end: usize, + }; + + pub const types = std.StaticStringMap(Tag).initComptime(.{ + .{ "int", .type_int }, + .{ "float", .type_float }, + .{ "str", .type_str }, + .{ "date", .type_date }, + }); + + pub fn getType(bytes: []const u8) ?Tag { + return types.get(bytes); + } + + pub const Tag = enum { + eof, + invalid, + + type_int, + type_float, + type_str, + type_date, + + identifier, + l_paren, + r_paren, + lr_bracket, + comma, + period, + two_dot, + }; +}; + +pub const Tokenizer = struct { + buffer: [:0]const u8, + index: usize, + + pub fn init(buffer: [:0]const u8) Tokenizer { + // Skip the UTF-8 BOM if present. + return .{ + .buffer = buffer, + .index = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0, + }; + } + + const State = enum { + start, + invalid, + identifier, + l_bracket, + }; + + pub fn next(self: *Tokenizer) Token { + var state: State = .start; + var result: Token = .{ + .tag = undefined, + .loc = .{ + .start = self.index, + .end = undefined, + }, + }; + while (true) : (self.index += 1) { + const c = self.buffer[self.index]; + switch (state) { + .start => switch (c) { + 0 => { + if (self.index == self.buffer.len) return .{ + .tag = .eof, + .loc = .{ + .start = self.index, + .end = self.index, + }, + }; + state = .invalid; + }, + ' ', '\n', '\t', '\r' => { + result.loc.start = self.index + 1; + }, + 'a'...'z', 'A'...'Z', '_' => { + state = .identifier; + result.tag = .identifier; + }, + '(' => { + result.tag = .l_paren; + self.index += 1; + break; + }, + ')' => { + result.tag = .r_paren; + self.index += 1; + break; + }, + '[' => { + state = .l_bracket; + }, + ',' => { + result.tag = .comma; + self.index += 1; + break; + }, + '.' => { + result.tag = .period; + self.index += 1; + break; + }, + ':' => { + result.tag = .two_dot; + self.index += 1; + break; + }, + else => { + state = .invalid; + }, + }, + + .invalid => { + // TODO make a better invalid handler + @panic("Unknow char!!!"); + }, + + .identifier => switch (c) { + 'a'...'z', 'A'...'Z', '_', '0'...'9' => continue, + else => { + if (Token.getType(self.buffer[result.loc.start..self.index])) |tag| { + result.tag = tag; + } + break; + }, + }, + + .l_bracket => switch (c) { + ']' => { + result.tag = .lr_bracket; + self.index += 1; + break; + }, + else => { + state = .invalid; + }, + }, + } + } + + result.loc.end = self.index; + return result; + } +}; + +test "keywords" { + try testTokenize("int float str date", &.{ .type_int, .type_float, .type_str, .type_date }); + std.debug.print("Schema type OK\n", .{}); +} + +test "basic query" { + try testTokenize("User ()", &.{ .identifier, .l_paren, .r_paren }); + try testTokenize("User (name:str)", &.{ .identifier, .l_paren, .identifier, .two_dot, .type_str, .r_paren }); + try testTokenize("User (name:str, email: str, messages:[]Message.from)", &.{ + .identifier, + .l_paren, + .identifier, + .two_dot, + .type_str, + .comma, + .identifier, + .two_dot, + .type_str, + .comma, + .identifier, + .two_dot, + .lr_bracket, + .identifier, + .period, + .identifier, + .r_paren, + }); + std.debug.print("Schema OK\n", .{}); +} + +fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void { + var tokenizer = Tokenizer.init(source); + for (expected_token_tags) |expected_token_tag| { + const token = tokenizer.next(); + try std.testing.expectEqual(expected_token_tag, token.tag); + } + // Last token should always be eof, even when the last token was invalid, + // in which case the tokenizer is in an invalid state, which can only be + // recovered by opinionated means outside the scope of this implementation. + const last_token = tokenizer.next(); + try std.testing.expectEqual(Token.Tag.eof, last_token.tag); + try std.testing.expectEqual(source.len, last_token.loc.start); + try std.testing.expectEqual(source.len, last_token.loc.end); +} diff --git a/src/ziqlTokenizer.zig b/src/tokenizers/ziqlTokenizer.zig similarity index 99% rename from src/ziqlTokenizer.zig rename to src/tokenizers/ziqlTokenizer.zig index 0dc1953..867ed77 100644 --- a/src/ziqlTokenizer.zig +++ b/src/tokenizers/ziqlTokenizer.zig @@ -16,6 +16,7 @@ pub const Token = struct { .{ "DELETE", .keyword_delete }, .{ "ADD", .keyword_add }, .{ "IN", .keyword_in }, + .{ "__DESCRIBE__", .keyword__describe__ }, }); pub fn getKeyword(bytes: []const u8) ?Tag { @@ -31,6 +32,7 @@ pub const Token = struct { keyword_delete, keyword_add, keyword_in, + keyword__describe__, string_literal, number_literal, diff --git a/zig-out/bin/zippon b/zig-out/bin/zippon index 9c3ab19..9278fbb 100755 Binary files a/zig-out/bin/zippon and b/zig-out/bin/zippon differ