adrien/ZipponDB
1
0
Fork 0
Code Actions Releases Activity

Big rework - Now use global ziql parser - still buggy, need to debug the tests

Browse Source
This commit is contained in:
Adrien Bouvais 2024-10-07 00:40:24 +02:00
parent ed1d879aef
commit 44e48a5276
17 changed files with 1615 additions and 1027 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
build.zig
View File

@ -6,7 +6,7 @@ pub fn build(b: *std.Build) void {
const optimize = b.standardOptimizeOption(.{});
const exe = b.addExecutable(.{
.name = "zippon",
.root_source_file = b.path("src/dbconsole.zig"),
.root_source_file = b.path("src/cliParser.zig"),
.target = target,
.optimize = optimize,
});
@ -20,7 +20,7 @@ pub fn build(b: *std.Build) void {
// Test step
const tests1 = b.addTest(.{
.root_source_file = b.path("src/data-parsing.zig"),
.root_source_file = b.path("src/dataParser.zig"),
.target = target,
.optimize = optimize,
.name = "Data parsing",
@ -28,7 +28,7 @@ pub fn build(b: *std.Build) void {
const run_tests1 = b.addRunArtifact(tests1);
const tests2 = b.addTest(.{
.root_source_file = b.path("src/cliTokenizer.zig"),
.root_source_file = b.path("src/tokenizers/cli.zig"),
.target = target,
.optimize = optimize,
.name = "CLI tokenizer",
@ -36,7 +36,7 @@ pub fn build(b: *std.Build) void {
const run_tests2 = b.addRunArtifact(tests2);
const tests3 = b.addTest(.{
.root_source_file = b.path("src/ziqlTokenizer.zig"),
.root_source_file = b.path("src/tokenizers/ziql.zig"),
.target = target,
.optimize = optimize,
.name = "ZiQL tokenizer",
@ -44,7 +44,7 @@ pub fn build(b: *std.Build) void {
const run_tests3 = b.addRunArtifact(tests3);
const tests4 = b.addTest(.{
.root_source_file = b.path("src/schemaTokenizer.zig"),
.root_source_file = b.path("src/tokenizers/schema.zig"),
.target = target,
.optimize = optimize,
.name = "Schema tokenizer",
@ -52,7 +52,7 @@ pub fn build(b: *std.Build) void {
const run_tests4 = b.addRunArtifact(tests4);
const tests5 = b.addTest(.{
.root_source_file = b.path("src/uuid.zig"),
.root_source_file = b.path("src/types/uuid.zig"),
.target = target,
.optimize = optimize,
.name = "UUID",
@ -60,13 +60,22 @@ pub fn build(b: *std.Build) void {
const run_tests5 = b.addRunArtifact(tests5);
const tests6 = b.addTest(.{
.root_source_file = b.path("src/GRAB.zig"),
.root_source_file = b.path("src/fileEngine.zig"),
.target = target,
.optimize = optimize,
.name = "GRAB",
.name = "File Engine",
});
const run_tests6 = b.addRunArtifact(tests6);
const tests7 = b.addTest(.{
.root_source_file = b.path("src/ziqlParser.zig"),
.target = target,
.optimize = optimize,
.name = "ZiQL parser",
//.test_runner = b.path("test_runner.zig"),
});
const run_tests7 = b.addRunArtifact(tests7);
const test_step = b.step("test", "Run unit tests");
test_step.dependOn(&run_tests1.step);
test_step.dependOn(&run_tests2.step);
@ -74,4 +83,5 @@ pub fn build(b: *std.Build) void {
test_step.dependOn(&run_tests4.step);
test_step.dependOn(&run_tests5.step);
test_step.dependOn(&run_tests6.step);
test_step.dependOn(&run_tests7.step);
}

184
src/ADD.zig
View File

@ -1,184 +0,0 @@
const std = @import("std");
const metadata = @import("metadata.zig");
const UUID = @import("uuid.zig").UUID;
const Tokenizer = @import("ziqlTokenizer.zig").Tokenizer;
const Token = @import("ziqlTokenizer.zig").Token;
const DataEngine = @import("dataEngine.zig").DataEngine;
const Allocator = std.mem.Allocator;
const stdout = std.io.getStdOut().writer();
// Query that need to work now
// ADD User (name='Adrien', email='adrien.bouvais@gmail.com') OK
// ADD User (name='Adrien', email='adrien.bouvais@gmail.com', age = 26) OK
// ADD User (name='Adrien', email='adrien.bouvais@gmail.com', books = ['book1', 'book2']) OK
// ADD User (name='Adrien', email=null) OK
//
// For later: links
// ADD User (name = 'Adrien', best_friend = {name='bob'}, friends = {name != 'bob'}) NOT OK
// ADD User (name = 'Adrien', friends = {(name = 'bob' AND age > 16) OR (id = '0000-0000')} ) NOT OK
// TODO: make real test
/// Function for the ADD query command.
/// It will parse the reste of the query and create a map of member name / value.
/// Then add those value to the appropriete file. The proper file is the first one with a size < to the limit.
/// If no file is found, a new one is created.
/// Take the main.zippondata file, the index of the file where the data is saved and the string to add at the end of the line
pub const Parser = struct {
allocator: Allocator,
toker: *Tokenizer,
pub fn init(allocator: Allocator, toker: *Tokenizer) Parser {
return Parser{
.allocator = allocator,
.toker = toker,
};
}
pub fn parse(self: *Parser) !void {
var data_engine = DataEngine.init(self.allocator, null);
defer data_engine.deinit();
var struct_name_token = self.toker.next();
const struct_name = self.toker.getTokenSlice(struct_name_token);
if (!metadata.isStructNameExists(struct_name)) self.print_error("Struct not found in current schema", &struct_name_token);
var token = self.toker.next();
switch (token.tag) {
.l_paren => {},
else => {
self.print_error("Error: Expected (", &token);
},
}
var data_map = self.parseData(struct_name);
defer data_map.deinit();
if (self.checkIfAllMemberInMap(struct_name, &data_map)) {
try data_engine.writeEntity(struct_name, data_map);
} else |_| {}
}
/// Take the tokenizer and return a map of the query for the ADD command.
/// Keys are the member name and value are the string of the value in the query. E.g. 'Adrien' or '10'
/// TODO: Make it clean using a State like other parser
pub fn parseData(self: *Parser, struct_name: []const u8) std.StringHashMap([]const u8) {
var token = self.toker.next();
var member_map = std.StringHashMap([]const u8).init(self.allocator);
while (token.tag != Token.Tag.eof) : (token = self.toker.next()) {
switch (token.tag) {
.r_paren => continue,
.identifier => {
const member_name_str = self.toker.getTokenSlice(token);
if (!metadata.isMemberNameInStruct(struct_name, member_name_str)) self.print_error("Member not found in struct.", &token);
token = self.toker.next();
switch (token.tag) {
.equal => {
token = self.toker.next();
switch (token.tag) {
.string_literal, .number_literal => {
const value_str = self.toker.getTokenSlice(token);
member_map.put(member_name_str, value_str) catch self.print_error("Could not add member name and value to map in getMapOfMember", &token);
token = self.toker.next();
switch (token.tag) {
.comma, .r_paren => continue,
else => self.print_error("Error: Expected , after string or number. E.g. ADD User (name='bob', age=10)", &token),
}
},
.keyword_null => {
const value_str = "null";
member_map.put(member_name_str, value_str) catch self.print_error("Error: 001", &token);
token = self.toker.next();
switch (token.tag) {
.comma, .r_paren => continue,
else => self.print_error("Error: Expected , after string or number. E.g. ADD User (name='bob', age=10)", &token),
}
},
// Create a tag to prevent creating an array then join them. Instead just read the buffer from [ to ] in the tekenizer itself
.l_bracket => {
var array_values = std.ArrayList([]const u8).init(self.allocator);
token = self.toker.next();
while (token.tag != Token.Tag.r_bracket) : (token = self.toker.next()) {
switch (token.tag) {
.string_literal, .number_literal => {
const value_str = self.toker.getTokenSlice(token);
array_values.append(value_str) catch self.print_error("Could not add value to array in getMapOfMember", &token);
},
else => self.print_error("Error: Expected string or number in array. E.g. ADD User (scores=[10 20 30])", &token),
}
}
// Maybe change that as it just recreate a string that is already in the buffer
const array_str = std.mem.join(self.allocator, " ", array_values.items) catch {
self.print_error("Couln't join the value of array", &token);
@panic("=)");
};
member_map.put(member_name_str, array_str) catch self.print_error("Could not add member name and value to map in getMapOfMember", &token);
token = self.toker.next();
switch (token.tag) {
.comma, .r_paren => continue,
else => self.print_error("Error: Expected , after string or number. E.g. ADD User (name='bob', age=10)", &token),
}
},
else => self.print_error("Error: Expected string or number after =. E.g. ADD User (name='bob')", &token),
}
},
else => self.print_error("Error: Expected = after a member declaration. E.g. ADD User (name='bob')", &token),
}
},
else => self.print_error("Error: Unknow token. This should be the name of a member. E.g. name in ADD User (name='bob')", &token),
}
}
return member_map;
}
const AddError = error{NotAllMemberInMap};
fn checkIfAllMemberInMap(_: *Parser, struct_name: []const u8, map: *std.StringHashMap([]const u8)) !void {
const all_struct_member = metadata.structName2structMembers(struct_name);
var count: u16 = 0;
var started_printing = false;
for (all_struct_member) |key| {
if (map.contains(key)) count += 1 else {
if (!started_printing) {
try stdout.print("Error: ADD query of struct: {s}; missing member: {s}", .{ struct_name, key });
started_printing = true;
} else {
try stdout.print(" {s}", .{key});
}
}
}
if (started_printing) try stdout.print("\n", .{});
if (!((count == all_struct_member.len) and (count == map.count()))) return error.NotAllMemberInMap;
}
fn print_error(self: *Parser, message: []const u8, token: *Token) void {
stdout.print("\n", .{}) catch {};
stdout.print("{s}\n", .{self.toker.buffer}) catch {};
// Calculate the number of spaces needed to reach the start position.
var spaces: usize = 0;
while (spaces < token.loc.start) : (spaces += 1) {
stdout.print(" ", .{}) catch {};
}
// Print the '^' characters for the error span.
var i: usize = token.loc.start;
while (i < token.loc.end) : (i += 1) {
stdout.print("^", .{}) catch {};
}
stdout.print(" \n", .{}) catch {}; // Align with the message
stdout.print("{s}\n", .{message}) catch {};
@panic("");
}
};

433
src/GRAB.zig
View File

@ -1,433 +0,0 @@
const std = @import("std");
const metadata = @import("metadata.zig");
const Allocator = std.mem.Allocator;
const Tokenizer = @import("ziqlTokenizer.zig").Tokenizer;
const Token = @import("ziqlTokenizer.zig").Token;
const DataEngine = @import("dataEngine.zig").DataEngine;
const UUID = @import("uuid.zig").UUID;
// To work now
// GRAB User {}
// GRAB User {name = 'Adrien'}
// GRAB User {name='Adrien' AND age < 30}
// GRAB User [1] {}
// GRAB User [10; name] {age < 30}
//
// For later
const stdout = std.io.getStdOut().writer();
pub const Parser = struct {
arena: std.heap.ArenaAllocator,
allocator: Allocator,
toker: *Tokenizer,
state: State,
additional_data: AdditionalData,
pub fn init(allocator: Allocator, toker: *Tokenizer) Parser {
var arena = std.heap.ArenaAllocator.init(allocator);
return Parser{
.arena = arena,
.allocator = arena.allocator(),
.toker = toker,
.state = State.start,
.additional_data = AdditionalData.init(allocator),
};
}
pub fn deinit(self: *Parser) void {
self.additional_data.deinit();
self.arena.deinit();
}
// This is the [] part
pub const AdditionalData = struct {
entity_count_to_find: usize = 0,
member_to_find: std.ArrayList(AdditionalDataMember),
pub fn init(allocator: Allocator) AdditionalData {
return AdditionalData{ .member_to_find = std.ArrayList(AdditionalDataMember).init(allocator) };
}
pub fn deinit(self: *AdditionalData) void {
for (0..self.member_to_find.items.len) |i| {
std.debug.print("{d}\n", .{i});
self.member_to_find.items[i].additional_data.deinit();
}
self.member_to_find.deinit();
}
};
// This is name in: [name]
// There is an additional data because it can be [friend [1; name]]
const AdditionalDataMember = struct {
name: []const u8,
additional_data: AdditionalData,
pub fn init(allocator: Allocator, name: []const u8) AdditionalDataMember {
const additional_data = AdditionalData.init(allocator);
return AdditionalDataMember{ .name = name, .additional_data = additional_data };
}
};
const State = enum {
start,
invalid,
end,
// For the main parse function
expect_filter,
// For the additional data parser
expect_count_of_entity_to_find,
expect_semicolon_OR_right_bracket,
expect_member,
next_member_OR_end_OR_new_additional_data,
next_member_OR_end,
// For the filter parser
expect_condition,
};
pub fn parse(self: *Parser) !void {
var data_engine = DataEngine.init(self.allocator, null);
defer data_engine.deinit();
var struct_name_token = self.toker.next();
if (!self.isStructInSchema(self.toker.getTokenSlice(struct_name_token))) {
try self.printError("Error: Struct name not in current shema.", &struct_name_token);
return;
}
var token = self.toker.next();
var keep_next = false;
while (self.state != State.end) : ({
token = if (!keep_next) self.toker.next() else token;
keep_next = false;
}) {
switch (self.state) {
.start => {
switch (token.tag) {
.l_bracket => {
try self.parseAdditionalData(&self.additional_data);
self.state = State.expect_filter;
},
.l_brace => {
self.state = State.expect_filter;
keep_next = true;
},
else => {
try self.printError("Error: Expected filter starting with {} or what to return starting with []", &token);
return;
},
}
},
.expect_filter => {
var array = std.ArrayList(UUID).init(self.allocator);
try self.parseFilter(&array, struct_name_token);
self.state = State.end;
},
else => return,
}
}
}
fn parseFilter(self: *Parser, left_array: *std.ArrayList(UUID), struct_name_token: Token) !void {
const right_array = std.ArrayList(UUID).init(self.allocator);
var token = self.toker.next();
var keep_next = false;
self.state = State.expect_member;
_ = right_array;
_ = left_array;
while (self.state != State.end) : ({
token = if (!keep_next) self.toker.next() else token;
keep_next = false;
}) {
switch (self.state) {
.expect_member => {
if (!self.isMemberPartOfStruct(self.toker.getTokenSlice(struct_name_token), self.toker.getTokenSlice(token))) {
try self.printError("Error: Member not part of struct.", &token);
}
self.state = State.expect_condition;
},
else => return,
}
}
}
/// When this function is call, the tokenizer last token retrieved should be [.
/// Check if an int is here -> check if ; is here -> check if member is here -> check if [ is here -> loop
pub fn parseAdditionalData(self: *Parser, additional_data: *AdditionalData) !void {
var token = self.toker.next();
var keep_next = false;
self.state = State.expect_count_of_entity_to_find;
while (self.state != State.end) : ({
token = if (!keep_next) self.toker.next() else token;
keep_next = false;
}) {
switch (self.state) {
.expect_count_of_entity_to_find => {
switch (token.tag) {
.number_literal => {
const count = std.fmt.parseInt(usize, self.toker.getTokenSlice(token), 10) catch {
try self.printError("Error while transforming this into a integer.", &token);
self.state = .invalid;
continue;
};
additional_data.entity_count_to_find = count;
self.state = .expect_semicolon_OR_right_bracket;
},
else => {
self.state = .expect_member;
keep_next = true;
},
}
},
.expect_semicolon_OR_right_bracket => {
switch (token.tag) {
.semicolon => {
self.state = .expect_member;
},
.r_bracket => {
return;
},
else => {
try self.printError(
"Error: Expect ';' or ']'.",
&token,
);
self.state = .invalid;
},
}
},
.expect_member => {
switch (token.tag) {
.identifier => {
// TODO: Check if the member name exist
try additional_data.member_to_find.append(
AdditionalDataMember.init(
self.allocator,
self.toker.getTokenSlice(token),
),
);
self.state = .next_member_OR_end_OR_new_additional_data;
},
else => {
try self.printError(
"Error: A member name should be here.",
&token,
);
},
}
},
.next_member_OR_end_OR_new_additional_data => {
switch (token.tag) {
.comma => {
self.state = .expect_member;
},
.r_bracket => {
return;
},
.l_bracket => {
try self.parseAdditionalData(
&additional_data.member_to_find.items[additional_data.member_to_find.items.len - 1].additional_data,
);
self.state = .next_member_OR_end;
},
else => {
try self.printError(
"Error: Expected a comma ',' or the end or a new list of member to return.",
&token,
);
},
}
},
.next_member_OR_end => {
switch (token.tag) {
.comma => {
self.state = .expect_member;
},
.r_bracket => {
return;
},
else => {
try self.printError(
"Error: Expected a comma or the end of the list of member name to return.",
&token,
);
},
}
},
.invalid => {
@panic("=)");
},
else => {
try self.printError(
"Error: Unknow state.",
&token,
);
},
}
}
}
fn printError(self: *Parser, message: []const u8, token: *Token) !void {
try stdout.print("\n", .{});
try stdout.print("{s}\n", .{self.toker.buffer});
// Calculate the number of spaces needed to reach the start position.
var spaces: usize = 0;
while (spaces < token.loc.start) : (spaces += 1) {
try stdout.print(" ", .{});
}
// Print the '^' characters for the error span.
var i: usize = token.loc.start;
while (i < token.loc.end) : (i += 1) {
try stdout.print("^", .{});
}
try stdout.print(" \n", .{}); // Align with the message
try stdout.print("{s}\n", .{message});
@panic("");
}
/// Take a struct name and a member name and return true if the member name is part of the struct
fn isMemberPartOfStruct(_: *Parser, struct_name: []const u8, member_name: []const u8) bool {
const all_struct_member = metadata.structName2structMembers(struct_name);
for (all_struct_member) |key| {
if (std.mem.eql(u8, key, member_name)) return true;
}
return false;
}
/// Check if a string is a name of a struct in the currently use engine
fn isStructInSchema(_: *Parser, struct_name_to_check: []const u8) bool {
for (metadata.struct_name_list) |struct_name| {
if (std.mem.eql(u8, struct_name_to_check, struct_name)) {
return true;
}
}
return false;
}
};
// TODO: Optimize. Maybe just do a new list and return it instead
fn OR(arr1: *std.ArrayList(UUID), arr2: *std.ArrayList(UUID)) std.ArrayList(UUID) {
defer arr1.deinit();
defer arr2.deinit();
var arr = try arr1.clone();
for (0..arr2.items.len) |i| {
if (!arr.contains(arr2[i])) {
arr.append(arr2[i]);
}
}
return arr;
}
fn AND(arr1: *std.ArrayList(UUID), arr2: *std.ArrayList(UUID)) std.ArrayList(UUID) {
defer arr1.deinit();
defer arr2.deinit();
var arr = try arr1.clone();
for (0..arr1.items.len) |i| {
if (arr2.contains(arr1[i])) {
arr.append(arr1[i]);
}
}
return arr;
}
test "Test AdditionalData" {
const allocator = std.testing.allocator;
var additional_data1 = Parser.AdditionalData.init(allocator);
additional_data1.entity_count_to_find = 1;
testAdditionalData("[1]", additional_data1);
var additional_data2 = Parser.AdditionalData.init(allocator);
defer additional_data2.deinit();
try additional_data2.member_to_find.append(
Parser.AdditionalDataMember.init(
allocator,
"name",
),
);
testAdditionalData("[name]", additional_data2);
var additional_data3 = Parser.AdditionalData.init(allocator);
additional_data3.entity_count_to_find = 1;
defer additional_data3.deinit();
try additional_data3.member_to_find.append(
Parser.AdditionalDataMember.init(
allocator,
"name",
),
);
testAdditionalData("[1; name]", additional_data3);
var additional_data4 = Parser.AdditionalData.init(allocator);
additional_data4.entity_count_to_find = 100;
defer additional_data4.deinit();
try additional_data4.member_to_find.append(
Parser.AdditionalDataMember.init(
allocator,
"friend",
),
);
testAdditionalData("[100; friend [name]]", additional_data4);
}
fn testAdditionalData(source: [:0]const u8, expected_AdditionalData: Parser.AdditionalData) void {
const allocator = std.testing.allocator;
var tokenizer = Tokenizer.init(source);
var data_engine = DataEngine.init(allocator);
defer data_engine.deinit();
var parser = Parser.init(allocator, &tokenizer, &data_engine);
defer parser.deinit();
_ = tokenizer.next();
parser.parse_additional_data(&parser.additional_data) catch |err| {
std.debug.print("Error parsing additional data: {any}\n", .{err});
};
compareAdditionalData(expected_AdditionalData, parser.additional_data);
}
// TODO: Check AdditionalData inside AdditionalData
fn compareAdditionalData(ad1: Parser.AdditionalData, ad2: Parser.AdditionalData) void {
std.testing.expectEqual(ad1.entity_count_to_find, ad2.entity_count_to_find) catch {
std.debug.print("Additional data entity_count_to_find are not equal.\n", .{});
};
var founded = false;
for (ad1.member_to_find.items) |elem1| {
founded = false;
for (ad2.member_to_find.items) |elem2| {
if (std.mem.eql(u8, elem1.name, elem2.name)) {
compareAdditionalData(elem1.additional_data, elem2.additional_data);
founded = true;
break;
}
}
if (!founded) @panic("Member not found");
}
}

122
src/cliParser.zig Normal file
View File

@ -0,0 +1,122 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
const cliTokenizer = @import("tokenizers/cli.zig").Tokenizer;
const cliToken = @import("tokenizers/cli.zig").Token;
const ziqlTokenizer = @import("tokenizers/ziql.zig").Tokenizer;
const ziqlToken = @import("tokenizers/ziql.zig").Token;
const ziqlParser = @import("ziqlParser.zig").Parser;
const stdout = std.io.getStdOut().writer();
pub fn main() !void {
// TODO: Use an environment variable for the path of the DB
checkAndCreateDirectories();
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
const allocator = gpa.allocator();
defer {
switch (gpa.deinit()) {
.ok => std.debug.print("No memory leak baby !\n", .{}),
.leak => {
std.debug.print("We fucked it up bro...\n", .{});
@panic("=(");
},
}
}
const line_buf = try allocator.alloc(u8, 1024 * 50);
defer allocator.free(line_buf);
while (true) {
std.debug.print("> ", .{});
const line = try std.io.getStdIn().reader().readUntilDelimiterOrEof(line_buf, '\n');
if (line) |line_str| {
const null_term_line_str = try allocator.dupeZ(u8, line_str[0..line_str.len]);
defer allocator.free(null_term_line_str);
var cliToker = cliTokenizer.init(null_term_line_str);
const command_token = cliToker.next();
switch (command_token.tag) {
.keyword_run => {
const query_token = cliToker.next();
switch (query_token.tag) {
.string_literal => {
const null_term_query_str = try allocator.dupeZ(u8, line_str[query_token.loc.start + 1 .. query_token.loc.end - 1]);
defer allocator.free(null_term_query_str);
try runCommand(null_term_query_str);
},
.keyword_help => std.debug.print("The run command will take a ZiQL query between \" and run it. eg: run \"GRAB User\"\n"),
else => std.debug.print("After command run, need a string of a query, eg: \"GRAB User\"\n", .{}),
}
},
.keyword_schema => {
const second_token = cliToker.next();
switch (second_token.tag) {
.keyword_describe => try runCommand("__DESCRIBE__"),
.keyword_build => std.debug.print("Need to do the SchemaEngine tu update and migrate the schema"),
.keyword_help => {
std.debug.print("{s}", .{
\\Here are all available options to use with the schema command:
\\
\\describe Print the schema use by the current engine.
\\build Build a new engine using a schema file. Args => filename: str, path of schema file to use. Default 'schema.zipponschema'.
\\
});
},
else => std.debug.print("schema available options: describe, build & help\n", .{}),
}
},
.keyword_help => {
std.debug.print("{s}", .{
\\Welcome to ZipponDB!
\\
\\run To run a query. Args => query: str, the query to execute.
\\schema Build a new engine and print current schema.
\\kill To stop the process without saving
\\save Save the database to the normal files.
\\dump Create a new folder with all data as copy. Args => foldername: str, the name of the folder.
\\bump Replace current data with a previous dump. Args => foldername: str, the name of the folder.
\\
});
},
.keyword_quit => break,
.eof => {},
else => std.debug.print("Command need to start with a keyword, including: run, schema, help and quit\n", .{}),
}
}
}
}
pub fn runCommand(null_term_query_str: [:0]const u8) !void {
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
const allocator = gpa.allocator();
var toker = ziqlTokenizer.init(null_term_query_str);
var parser = ziqlParser.init(allocator, &toker);
defer parser.deinit();
try parser.parse();
}
fn checkAndCreateDirectories() void {
const cwd = std.fs.cwd();
cwd.makeDir("ZipponDB") catch |err| switch (err) {
error.PathAlreadyExists => {},
else => @panic("Error other than path already exists when trying to create the ZipponDB directory.\n"),
};
cwd.makeDir("ZipponDB/DATA") catch |err| switch (err) {
error.PathAlreadyExists => {},
else => @panic("Error other than path already exists when trying to create the DATA directory.\n"),
};
cwd.makeDir("ZipponDB/ENGINE") catch |err| switch (err) {
error.PathAlreadyExists => {},
else => @panic("Error other than path already exists when trying to create the ENGINE directory.\n"),
};
}

0
src/data-parsing.zig → src/dataParser.zig
View File

211
src/dbconsole.zig
View File

@ -1,211 +0,0 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
const cliTokenizer = @import("cliTokenizer.zig").Tokenizer;
const cliToken = @import("cliTokenizer.zig").Token;
const schemaTokenizer = @import("schemaTokenizer.zig").Tokenizer;
const schemaToken = @import("schemaTokenizer.zig").Token;
const schemaParser = @import("schemaParser.zig").Parser;
pub fn main() !void {
checkAndCreateDirectories();
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
const allocator = gpa.allocator();
while (true) {
std.debug.print("> ", .{});
var line_buf: [1024]u8 = undefined;
const line = try std.io.getStdIn().reader().readUntilDelimiterOrEof(&line_buf, '\n');
if (line) |line_str| {
const null_term_line_str = try allocator.dupeZ(u8, line_str[0..line_str.len]);
var cliToker = cliTokenizer.init(null_term_line_str);
const command_token = cliToker.next();
switch (command_token.tag) {
.keyword_run => {
const query_token = cliToker.next();
switch (query_token.tag) {
.string_literal => {
const null_term_query_str = try allocator.dupeZ(u8, line_str[query_token.loc.start + 1 .. query_token.loc.end - 1]);
runCommand(null_term_query_str);
},
else => {
std.debug.print("After command run, need a string of a query, eg: \"GRAB User\"\n", .{});
continue;
},
}
},
.keyword_schema => {
const second_token = cliToker.next();
switch (second_token.tag) {
.keyword_describe => {
runCommand("__DESCRIBE__");
},
.keyword_build => {
const file_name_token = cliToker.next();
var file_name = try allocator.alloc(u8, 1024);
var len: usize = 0;
switch (file_name_token.tag) {
.eof => {
std.mem.copyForwards(u8, file_name, "schema.zipponschema");
len = 19;
},
else => file_name = line_str[file_name_token.loc.start..file_name_token.loc.end],
}
std.debug.print("{s}", .{file_name[0..len]});
//blk: {
//createDtypeFile(file_name[0..len]) catch |err| switch (err) {
// error.FileNotFound => {
// std.debug.print("Error: Can't find file: {s}\n", .{file_name[0..len]});
// break :blk;
// },
// else => {
// std.debug.print("Error: Unknow error when creating Dtype file: {any}\n", .{err});
// break :blk;
// },
//};
try buildEngine();
//}
allocator.free(file_name);
},
.keyword_help => {
std.debug.print("{s}", .{
\\Here are all available options to use with the schema command:
\\
\\describe Print the schema use by the current engine.
\\build Build a new engine using a schema file. Args => filename: str, path of schema file to use. Default 'schema.zipponschema'.
\\
});
},
else => {
std.debug.print("schema available options: describe, build & help\n", .{});
},
}
},
.keyword_help => {
std.debug.print("{s}", .{
\\Welcome to ZipponDB!
\\
\\run To run a query. Args => query: str, the query to execute.
\\schema Build a new engine and print current schema.
\\kill To stop the process without saving
\\save Save the database to the normal files.
\\dump Create a new folder with all data as copy. Args => foldername: str, the name of the folder.
\\bump Replace current data with a previous dump. Args => foldername: str, the name of the folder.
\\
});
},
.keyword_quit => {
break;
},
.eof => {},
else => {
std.debug.print("Command need to start with a keyword, including: run, schema, help and quit\n", .{});
},
}
}
}
}
fn createDtypeFile(file_path: []const u8) !void {
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
const allocator = gpa.allocator();
const cwd = std.fs.cwd();
var file = try cwd.openFile(file_path, .{
.mode = .read_only,
});
defer file.close();
const buffer = try file.readToEndAlloc(allocator, 1024);
const file_contents = try allocator.dupeZ(u8, buffer[0..]); // Duplicate to a null terminated string
var schemaToker = schemaTokenizer.init(file_contents);
var parser = schemaParser.init();
parser.parse(&schemaToker, buffer);
// Free memory
allocator.free(buffer);
allocator.free(file_contents);
const check = gpa.deinit();
switch (check) {
.ok => return,
.leak => std.debug.print("Error: Leak in createDtypeFile!\n", .{}),
}
}
fn buildEngine() !void {
const argv = &[_][]const u8{
"zig",
"build-exe",
"src/dbengine.zig",
"--name",
"engine",
};
var child = std.process.Child.init(argv, std.heap.page_allocator);
try child.spawn();
_ = try child.wait();
runCommand("__INIT__");
}
fn runCommand(null_term_query_str: [:0]const u8) void {
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
const allocator = gpa.allocator();
// TODO: Use the folder ENGINE
const args = &[_][]const u8{ "./engine", null_term_query_str };
const result = std.process.Child.run(.{ .allocator = allocator, .argv = args }) catch |err| switch (err) {
error.FileNotFound => {
std.debug.print("No engine found, please use `schema build` to make one.\n", .{});
return;
},
else => {
std.debug.print("Error: Unknow error when trying to run the engine: {any}\n", .{err});
return;
},
};
switch (result.term) {
.Exited => {},
.Signal => std.debug.print("Error: term signal in runCommand\n", .{}),
.Stopped => std.debug.print("Error: term stopped in runCommand\n", .{}),
.Unknown => std.debug.print("Error: term unknow in runCommand\n", .{}),
}
std.debug.print("{s}\n", .{result.stdout});
allocator.free(result.stdout);
allocator.free(result.stderr);
const check = gpa.deinit();
switch (check) {
.ok => return,
.leak => std.debug.print("Error: Leak in runCommand!\n", .{}),
}
}
fn checkAndCreateDirectories() void {
const cwd = std.fs.cwd();
cwd.makeDir("ZipponDB") catch |err| switch (err) {
error.PathAlreadyExists => {},
else => @panic("Error other than path already exists when trying to create the ZipponDB directory.\n"),
};
cwd.makeDir("ZipponDB/DATA") catch |err| switch (err) {
error.PathAlreadyExists => {},
else => @panic("Error other than path already exists when trying to create the DATA directory.\n"),
};
cwd.makeDir("ZipponDB/ENGINE") catch |err| switch (err) {
error.PathAlreadyExists => {},
else => @panic("Error other than path already exists when trying to create the ENGINE directory.\n"),
};
}

55
src/dbengine.zig
View File

@ -1,55 +0,0 @@
const std = @import("std");
const DataEngine = @import("dataEngine.zig").DataEngine;
const Tokenizer = @import("ziqlTokenizer.zig").Tokenizer;
const grabParser = @import("GRAB.zig").Parser;
const addParser = @import("ADD.zig").Parser;
pub const Error = error{UUIDNotFound};
const stdout = std.io.getStdOut().writer();
pub fn main() !void {
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
const allocator = gpa.allocator();
const buffer = try allocator.alloc(u8, 1024);
defer allocator.free(buffer);
var args = try std.process.argsWithAllocator(allocator);
defer args.deinit();
// Remove the first argument
_ = args.next();
const null_term_query_str = args.next().?;
var toker = Tokenizer.init(null_term_query_str);
const first_token = toker.next();
switch (first_token.tag) {
.keyword_grab => {
var parser = grabParser.init(allocator, &toker);
try parser.parse();
},
.keyword_add => {
var parser = addParser.init(allocator, &toker);
parser.parse() catch |err| {
try stdout.print("Error: {any} while parsin ADD.\n", .{err});
};
},
.keyword_update => {
try stdout.print("Not yet implemented.\n", .{});
},
.keyword_delete => {
try stdout.print("Not yet implemented.\n", .{});
},
.keyword__describe__ => {
try stdout.print("{s}", .{@embedFile("schema.zipponschema")});
},
.keyword__init__ => {
var data_engine = DataEngine.init(allocator, null);
try data_engine.initDataFolder();
},
else => {
try stdout.print("Query need to start with a keyword, including: GRAB ADD UPDATE DELETE\n", .{});
},
}
}

122
src/dataEngine.zig → src/fileEngine.zig
View File

@ -1,13 +1,16 @@
const std = @import("std");
const dataParsing = @import("data-parsing.zig");
const metadata = @import("metadata.zig");
const dataParsing = @import("dataParser.zig");
const schemaEngine = @import("schemaEngine.zig");
const Allocator = std.mem.Allocator;
const UUID = @import("uuid.zig").UUID;
const UUID = @import("types/uuid.zig").UUID;
const DataType = @import("types/dataType.zig").DataType;
const stdout = std.io.getStdOut().writer();
//TODO: Create a union class and chose between file and memory
/// Manage everything that is relate to read or write in files
/// Or even get stats, whatever. If it touch files, it's here
pub const DataEngine = struct {
pub const FileEngine = struct {
allocator: Allocator,
dir: std.fs.Dir, // The path to the DATA folder
max_file_size: usize = 1e+8, // 100mb
@ -31,18 +34,6 @@ pub const DataEngine = struct {
inferior_or_equal,
};
/// Suported dataType for the DB
const DataType = enum {
int,
float,
str,
bool_,
int_array,
float_array,
str_array,
bool_array,
};
const ComparisonValue = union {
int: i64,
float: f64,
@ -58,31 +49,41 @@ pub const DataEngine = struct {
/// An Operation from equal, different, superior, superior_or_equal, ...
/// The DataType from int, float and str
/// TODO: Change the value to be the right type and not just a string all the time
const Condition = struct {
pub const Condition = struct {
struct_name: []const u8,
member_name: []const u8,
value: []const u8,
operation: Operation,
data_type: DataType,
member_name: []const u8 = undefined,
value: []const u8 = undefined,
operation: Operation = undefined,
data_type: DataType = undefined,
pub fn init(struct_name: []const u8) Condition {
return Condition{ .struct_name = struct_name };
}
};
pub fn init(allocator: Allocator, DATA_path: ?[]const u8) DataEngine {
pub fn init(allocator: Allocator, DATA_path: ?[]const u8) FileEngine {
const path = DATA_path orelse "ZipponDB/DATA";
const dir = std.fs.cwd().openDir(path, .{}) catch @panic("Error opening ZipponDB/DATA");
return DataEngine{
return FileEngine{
.allocator = allocator,
.dir = dir,
};
}
pub fn deinit(self: *DataEngine) void {
pub fn deinit(self: *FileEngine) void {
self.dir.close();
}
/// Take a condition and an array of UUID and fill the array with all UUID that match the condition
pub fn getUUIDListUsingCondition(self: *DataEngine, condition: Condition, uuid_array: *std.ArrayList(UUID)) !void {
const file_names = self.getFilesNames(condition.struct_name, condition.member_name) catch @panic("Can't get list of files");
defer self.deinitFilesNames(&file_names);
pub fn getUUIDListUsingCondition(self: *FileEngine, condition: Condition, uuid_array: *std.ArrayList(UUID)) !void {
var file_names = std.ArrayList([]const u8).init(self.allocator);
self.getFilesNames(condition.struct_name, condition.member_name, &file_names) catch @panic("Can't get list of files");
defer {
for (file_names.items) |elem| {
self.allocator.free(elem);
}
file_names.deinit();
}
const sub_path = std.fmt.allocPrint(
self.allocator,
@ -92,7 +93,6 @@ pub const DataEngine = struct {
defer self.allocator.free(sub_path);
var file = self.dir.openFile(sub_path, .{}) catch @panic("Can't open first file to init a data iterator");
// defer self.allocator.free(sub_path);
var output: [1024 * 50]u8 = undefined; // Maybe need to increase that as it limit the size of a line in files
var output_fbs = std.io.fixedBufferStream(&output);
@ -108,12 +108,21 @@ pub const DataEngine = struct {
.int => compare_value = ComparisonValue{ .int = dataParsing.parseInt(condition.value) },
.str => compare_value = ComparisonValue{ .str = condition.value },
.float => compare_value = ComparisonValue{ .float = dataParsing.parseFloat(condition.value) },
.bool_ => compare_value = ComparisonValue{ .bool_ = dataParsing.parseBool(condition.value) },
.bool => compare_value = ComparisonValue{ .bool_ = dataParsing.parseBool(condition.value) },
.int_array => compare_value = ComparisonValue{ .int_array = dataParsing.parseArrayInt(self.allocator, condition.value) },
.str_array => compare_value = ComparisonValue{ .str_array = dataParsing.parseArrayStr(self.allocator, condition.value) },
.float_array => compare_value = ComparisonValue{ .float_array = dataParsing.parseArrayFloat(self.allocator, condition.value) },
.bool_array => compare_value = ComparisonValue{ .bool_array = dataParsing.parseArrayBool(self.allocator, condition.value) },
}
defer {
switch (condition.data_type) {
.int_array => compare_value.int_array.deinit(),
.str_array => compare_value.str_array.deinit(),
.float_array => compare_value.float_array.deinit(),
.bool_array => compare_value.bool_array.deinit(),
else => {},
}
}
while (true) {
output_fbs.reset();
@ -143,7 +152,7 @@ pub const DataEngine = struct {
.int => if (compare_value.int == dataParsing.parseInt(output_fbs.getWritten()[37..])) try uuid_array.append(try UUID.parse(output_fbs.getWritten()[0..36])),
.float => if (compare_value.float == dataParsing.parseFloat(output_fbs.getWritten()[37..])) try uuid_array.append(try UUID.parse(output_fbs.getWritten()[0..36])),
.str => if (std.mem.eql(u8, compare_value.str, output_fbs.getWritten()[38 .. output_fbs.getWritten().len - 1])) try uuid_array.append(try UUID.parse(output_fbs.getWritten()[0..36])),
.bool_ => if (compare_value.bool_ == dataParsing.parseBool(output_fbs.getWritten()[37..])) try uuid_array.append(try UUID.parse(output_fbs.getWritten()[0..36])),
.bool => if (compare_value.bool_ == dataParsing.parseBool(output_fbs.getWritten()[37..])) try uuid_array.append(try UUID.parse(output_fbs.getWritten()[0..36])),
// TODO: Implement for array too
else => {},
}
@ -153,7 +162,7 @@ pub const DataEngine = struct {
.int => if (compare_value.int != dataParsing.parseInt(output_fbs.getWritten()[37..])) try uuid_array.append(try UUID.parse(output_fbs.getWritten()[0..36])),
.float => if (compare_value.float != dataParsing.parseFloat(output_fbs.getWritten()[37..])) try uuid_array.append(try UUID.parse(output_fbs.getWritten()[0..36])),
.str => if (!std.mem.eql(u8, compare_value.str, output_fbs.getWritten()[38 .. output_fbs.getWritten().len - 1])) try uuid_array.append(try UUID.parse(output_fbs.getWritten()[0..36])),
.bool_ => if (compare_value.bool_ != dataParsing.parseBool(output_fbs.getWritten()[37..])) try uuid_array.append(try UUID.parse(output_fbs.getWritten()[0..36])),
.bool => if (compare_value.bool_ != dataParsing.parseBool(output_fbs.getWritten()[37..])) try uuid_array.append(try UUID.parse(output_fbs.getWritten()[0..36])),
// TODO: Implement for array too
else => {},
}
@ -195,14 +204,14 @@ pub const DataEngine = struct {
}
// TODO: Test leak on that
pub fn writeEntity(self: *DataEngine, struct_name: []const u8, data_map: std.StringHashMap([]const u8)) !void {
pub fn writeEntity(self: *FileEngine, struct_name: []const u8, data_map: std.StringHashMap([]const u8)) !void {
const uuid_str = UUID.init().format_uuid();
defer stdout.print("Added new {s} successfully using UUID: {s}\n", .{
struct_name,
uuid_str,
}) catch {};
const member_names = metadata.structName2structMembers(struct_name);
const member_names = schemaEngine.structName2structMembers(struct_name);
for (member_names) |member_name| {
const potential_file_name_to_use = try self.getFirstUsableFile(struct_name, member_name);
@ -252,10 +261,10 @@ pub const DataEngine = struct {
member_name,
max_index + 1,
});
defer self.allocator.free(new_file_path);
try stdout.print("new file path: {s}\n", .{new_file_path});
// TODO: Create new file and save the data inside
const new_file = self.dir.createFile(new_file_path, .{}) catch @panic("Error creating new data file");
defer new_file.close();
@ -284,7 +293,7 @@ pub const DataEngine = struct {
}
/// Use a filename in the format 1.zippondata and return the 1
fn fileName2Index(_: *DataEngine, file_name: []const u8) usize {
fn fileName2Index(_: *FileEngine, file_name: []const u8) usize {
var iter_file_name = std.mem.tokenize(u8, file_name, ".");
const num_str = iter_file_name.next().?;
const num: usize = std.fmt.parseInt(usize, num_str, 10) catch @panic("Couln't parse the int of a zippondata file.");
@ -293,7 +302,7 @@ pub const DataEngine = struct {
/// Add an UUID at a specific index of a file
/// Used when some data are deleted from previous zippondata files and are now bellow the file size limit
fn addUUIDToMainFile(_: *DataEngine, file: std.fs.File, index: usize, uuid_str: []const u8) !void {
fn addUUIDToMainFile(_: *FileEngine, file: std.fs.File, index: usize, uuid_str: []const u8) !void {
var output: [1024 * 50]u8 = undefined; // Maybe need to increase that as it limit the size of a line in files
var output_fbs = std.io.fixedBufferStream(&output);
const writer = output_fbs.writer();
@ -320,35 +329,26 @@ pub const DataEngine = struct {
}
}
fn getFilesNames(self: *DataEngine, struct_name: []const u8, member_name: []const u8) !std.ArrayList([]const u8) {
fn getFilesNames(self: *FileEngine, struct_name: []const u8, member_name: []const u8, file_names: *std.ArrayList([]const u8)) !void {
const sub_path = try std.fmt.allocPrint(self.allocator, "{s}/{s}", .{ struct_name, member_name });
var file_names = std.ArrayList([]const u8).init(self.allocator);
const member_dir = self.dir.openDir(sub_path, .{ .iterate = true }) catch @panic("Error opening member directory");
defer self.allocator.free(sub_path);
var member_dir = try self.dir.openDir(sub_path, .{ .iterate = true });
defer member_dir.close();
var iter = member_dir.iterate();
while (try iter.next()) |entry| {
if ((entry.kind != std.fs.Dir.Entry.Kind.file) or (std.mem.eql(u8, "main.zippondata", entry.name))) continue;
try file_names.append(try self.allocator.dupe(u8, entry.name));
try file_names.*.append(try self.allocator.dupe(u8, entry.name));
}
return file_names;
}
fn deinitFilesNames(self: *DataEngine, array: *const std.ArrayList([]const u8)) void {
for (array.items) |elem| {
self.allocator.free(elem);
}
array.deinit();
}
/// Use the map of file stat to find the first file with under the bytes limit.
/// return the name of the file. If none is found, return null.
fn getFirstUsableFile(self: *DataEngine, struct_name: []const u8, member_name: []const u8) !?[]const u8 {
fn getFirstUsableFile(self: *FileEngine, struct_name: []const u8, member_name: []const u8) !?[]const u8 {
const sub_path = try std.fmt.allocPrint(self.allocator, "{s}/{s}", .{ struct_name, member_name });
defer self.allocator.free(sub_path);
var member_dir = try self.dir.openDir(sub_path, .{ .iterate = true });
defer member_dir.close();
@ -364,7 +364,7 @@ pub const DataEngine = struct {
/// Iter over all file and get the max name and return the value of it as usize
/// So for example if there is 1.zippondata and 2.zippondata it return 2.
fn maxFileIndex(self: *DataEngine, struct_name: []const u8, member_name: []const u8) !usize {
fn maxFileIndex(self: *FileEngine, struct_name: []const u8, member_name: []const u8) !usize {
const buffer = try self.allocator.alloc(u8, 1024); // Adjust the size as needed
defer self.allocator.free(buffer);
@ -381,21 +381,23 @@ pub const DataEngine = struct {
}
// TODO: Give the option to keep , dump or erase the data
pub fn initDataFolder(self: *DataEngine) !void {
for (metadata.struct_name_list) |struct_name| {
pub fn initDataFolder(self: *FileEngine) !void {
for (schemaEngine.struct_name_list) |struct_name| {
self.dir.makeDir(struct_name) catch |err| switch (err) {
error.PathAlreadyExists => {},
else => return DataEngineError.ErrorCreateStructFolder,
};
const struct_dir = try self.dir.openDir(struct_name, .{});
defer struct_dir.close();
const member_names = metadata.structName2structMembers(struct_name);
const member_names = schemaEngine.structName2structMembers(struct_name);
for (member_names) |member_name| {
struct_dir.makeDir(member_name) catch |err| switch (err) {
error.PathAlreadyExists => continue,
else => return DataEngineError.ErrorCreateMemberFolder,
};
const member_dir = try struct_dir.openDir(member_name, .{});
defer member_dir.close();
blk: {
const file = member_dir.createFile("main.zippondata", .{}) catch |err| switch (err) {
@ -413,15 +415,13 @@ pub const DataEngine = struct {
}
};
test "File iterator" {
test "Get list of UUID using condition" {
const allocator = std.testing.allocator;
var data_engine = DataEngine.init(allocator, null);
var data_engine = FileEngine.init(allocator, null);
var uuid_array = std.ArrayList(UUID).init(allocator);
defer uuid_array.deinit();
const condition = DataEngine.Condition{ .struct_name = "User", .member_name = "email", .value = "adrien@mail.com", .operation = .equal, .data_type = .str };
const condition = FileEngine.Condition{ .struct_name = "User", .member_name = "email", .value = "adrien@mail.com", .operation = .equal, .data_type = .str };
try data_engine.getUUIDListUsingCondition(condition, &uuid_array);
std.debug.print("Found {d} uuid with first as {any}\n\n", .{ uuid_array.items.len, uuid_array.items[0] });
}

31
src/metadata.zig
View File

@ -1,31 +0,0 @@
const std = @import("std");
// Maybe create a struct like StructMetadata for the string list of member and name, ect
pub const struct_name_list: [2][]const u8 = .{
"User",
"Message",
};
pub const struct_member_list: [2][]const []const u8 = .{
&[_][]const u8{ "name", "email", "age", "scores" },
&[_][]const u8{"content"},
};
/// Get the list of all member name for a struct name
pub fn structName2structMembers(struct_name: []const u8) []const []const u8 {
var i: u16 = 0;
while (i < struct_name_list.len) : (i += 1) if (std.mem.eql(u8, struct_name_list[i], struct_name)) break;
return struct_member_list[i];
}
pub fn isStructNameExists(struct_name: []const u8) bool {
for (struct_name_list) |sn| if (std.mem.eql(u8, sn, struct_name)) return true;
return false;
}
pub fn isMemberNameInStruct(struct_name: []const u8, member_name: []const u8) bool {
for (structName2structMembers(struct_name)) |mn| if (std.mem.eql(u8, mn, member_name)) return true;
return false;
}

100
src/schemaEngine.zig Normal file
View File

@ -0,0 +1,100 @@
// This file is named and use as a struct but is in fact just a series of utils functions to get and check the schema
// TODO: create a struct like SchemaEngine so I can do propre testing and it make update it easier
// Also can put the migration stuff in here
const std = @import("std");
const DataType = @import("types/dataType.zig").DataType;
const struct_name_list: [2][]const u8 = .{
"User",
"Message",
};
const struct_member_list: [2][]const []const u8 = .{
&[_][]const u8{ "name", "email", "age", "scores", "friends" },
&[_][]const u8{"content"},
};
const struct_type_list: [2][]const DataType = .{
&[_]DataType{ .str, .str, .int, .int_array, .bool_array },
&[_]DataType{.str},
};
/// Get the type of the member
pub fn memberName2DataType(struct_name: []const u8, member_name: []const u8) ?DataType {
var i: u16 = 0;
for (structName2structMembers(struct_name)) |mn| {
if (std.mem.eql(u8, mn, member_name)) return structName2DataType(struct_name)[i];
i += 1;
}
return null;
}
/// Get the list of all member name for a struct name
pub fn structName2structMembers(struct_name: []const u8) []const []const u8 {
var i: u16 = 0;
while (i < struct_name_list.len) : (i += 1) if (std.mem.eql(u8, struct_name_list[i], struct_name)) break;
if (i == struct_name_list.len) {
std.debug.print("{s} \n", .{struct_name});
@panic("Struct name not found!");
}
return struct_member_list[i];
}
pub fn structName2DataType(struct_name: []const u8) []const DataType {
var i: u16 = 0;
while (i < struct_name_list.len) : (i += 1) if (std.mem.eql(u8, struct_name_list[i], struct_name)) break;
return struct_type_list[i];
}
/// Chech if the name of a struct is in the current schema
pub fn isStructNameExists(struct_name: []const u8) bool {
for (struct_name_list) |sn| if (std.mem.eql(u8, sn, struct_name)) return true;
return false;
}
/// Check if a struct have the member name
pub fn isMemberNameInStruct(struct_name: []const u8, member_name: []const u8) bool {
for (structName2structMembers(struct_name)) |mn| if (std.mem.eql(u8, mn, member_name)) return true;
return false;
}
/// Take a struct name and a member name and return true if the member name is part of the struct
pub fn isMemberPartOfStruct(struct_name: []const u8, member_name: []const u8) bool {
const all_struct_member = structName2structMembers(struct_name);
for (all_struct_member) |key| {
if (std.mem.eql(u8, key, member_name)) return true;
}
return false;
}
/// Check if a string is a name of a struct in the currently use engine
pub fn isStructInSchema(struct_name_to_check: []const u8) bool {
for (struct_name_list) |struct_name| {
if (std.mem.eql(u8, struct_name_to_check, struct_name)) {
return true;
}
}
return false;
}
// Return true if the map have all the member name as key and not more
pub fn checkIfAllMemberInMap(struct_name: []const u8, map: *std.StringHashMap([]const u8)) bool {
const all_struct_member = structName2structMembers(struct_name);
var count: u16 = 0;
for (all_struct_member) |key| {
if (map.contains(key)) count += 1;
}
return ((count == all_struct_member.len) and (count == map.count()));
}

0
src/cliTokenizer.zig → src/tokenizers/cli.zig
View File

0
src/schemaTokenizer.zig → src/tokenizers/schema.zig
View File

103
src/ziqlTokenizer.zig → src/tokenizers/ziql.zig
View File

@ -19,6 +19,10 @@ pub const Token = struct {
.{ "null", .keyword_null },
.{ "__DESCRIBE__", .keyword__describe__ },
.{ "__INIT__", .keyword__init__ },
.{ "true", .bool_literal_true },
.{ "false", .bool_literal_false },
.{ "AND", .keyword_and },
.{ "OR", .keyword_or },
});
pub fn getKeyword(bytes: []const u8) ?Tag {
@ -35,30 +39,35 @@ pub const Token = struct {
keyword_add,
keyword_in,
keyword_null,
keyword_and,
keyword_or,
keyword__describe__,
keyword__init__,
string_literal,
number_literal,
int_literal,
float_literal,
bool_literal_true,
bool_literal_false,
identifier,
equal,
bang,
pipe,
l_paren,
r_paren,
l_bracket,
r_bracket,
l_brace,
r_brace,
semicolon,
comma,
angle_bracket_left,
angle_bracket_right,
angle_bracket_left_equal,
angle_bracket_right_equal,
equal_angle_bracket_right,
period,
bang_equal,
bang, // !
pipe, // |
l_paren, // (
r_paren, // )
l_bracket, // [
r_bracket, // ]
l_brace, // {
r_brace, // }
semicolon, // ;
comma, // ,
angle_bracket_left, // <
angle_bracket_right, // >
angle_bracket_left_equal, // <=
angle_bracket_right_equal, // >=
equal_angle_bracket_right, // =>
period, // .
bang_equal, // !=
};
};
@ -89,7 +98,6 @@ pub const Tokenizer = struct {
angle_bracket_right,
string_literal_backslash,
int_exponent,
int_period,
float,
float_exponent,
int,
@ -187,13 +195,12 @@ pub const Tokenizer = struct {
break;
},
'.' => {
result.tag = .period;
self.index += 1;
break;
state = .float;
result.tag = .float_literal;
},
'0'...'9' => {
state = .int;
result.tag = .number_literal;
result.tag = .int_literal;
},
else => {
state = .invalid;
@ -210,6 +217,8 @@ pub const Tokenizer = struct {
else => {
if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |tag| {
result.tag = tag;
} else {
result.tag = .identifier;
}
break;
},
@ -301,40 +310,44 @@ pub const Tokenizer = struct {
},
.int => switch (c) {
'.' => state = .int_period,
'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => continue,
'e', 'E', 'p', 'P' => state = .int_exponent,
'.' => {
state = .float;
result.tag = .float_literal;
},
'e', 'E' => {
state = .int_exponent;
result.tag = .float_literal;
},
'_', '0'...'9' => continue,
else => break,
},
.int_exponent => switch (c) {
'-', '+' => {
'+', '-', '0'...'9' => {
state = .float;
},
else => {
self.index -= 1;
state = .int;
},
},
.int_period => switch (c) {
'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {
state = .float;
},
'e', 'E', 'p', 'P' => state = .float_exponent,
else => {
self.index -= 1;
break;
},
},
.float => switch (c) {
'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => continue,
'e', 'E', 'p', 'P' => state = .float_exponent,
else => break,
'e', 'E' => {
state = .float_exponent;
},
'_', '0'...'9' => {
continue;
},
else => {
break;
},
},
.float_exponent => switch (c) {
'-', '+' => state = .float,
'+', '-', '0'...'9' => {
continue;
},
else => {
self.index -= 1;
state = .float;
break;
},
},
}
@ -352,9 +365,11 @@ test "keywords" {
test "basic query" {
try testTokenize("GRAB User {}", &.{ .keyword_grab, .identifier, .l_brace, .r_brace });
try testTokenize("GRAB User { name = 'Adrien'}", &.{ .keyword_grab, .identifier, .l_brace, .identifier, .equal, .string_literal, .r_brace });
try testTokenize("GRAB User [1; name] {}", &.{ .keyword_grab, .identifier, .l_bracket, .number_literal, .semicolon, .identifier, .r_bracket, .l_brace, .r_brace });
try testTokenize("GRAB User { age = 1.5}", &.{ .keyword_grab, .identifier, .l_brace, .identifier, .equal, .float_literal, .r_brace });
try testTokenize("GRAB User { admin = true}", &.{ .keyword_grab, .identifier, .l_brace, .identifier, .equal, .bool_literal_true, .r_brace });
try testTokenize("GRAB User [1; name] {}", &.{ .keyword_grab, .identifier, .l_bracket, .int_literal, .semicolon, .identifier, .r_bracket, .l_brace, .r_brace });
try testTokenize("GRAB User{}|ASCENDING name|", &.{ .keyword_grab, .identifier, .l_brace, .r_brace, .pipe, .identifier, .identifier, .pipe });
try testTokenize("DELETE User[1]{name='Adrien'}|ASCENDING name, age|", &.{ .keyword_delete, .identifier, .l_bracket, .number_literal, .r_bracket, .l_brace, .identifier, .equal, .string_literal, .r_brace, .pipe, .identifier, .identifier, .comma, .identifier, .pipe });
try testTokenize("DELETE User[1]{name='Adrien'}|ASCENDING name, age|", &.{ .keyword_delete, .identifier, .l_bracket, .int_literal, .r_bracket, .l_brace, .identifier, .equal, .string_literal, .r_brace, .pipe, .identifier, .identifier, .comma, .identifier, .pipe });
}
fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {

11
src/types/dataType.zig Normal file
View File

@ -0,0 +1,11 @@
/// Suported dataType for the DB
pub const DataType = enum {
int,
float,
str,
bool,
int_array,
float_array,
str_array,
bool_array,
};

0
src/uuid.zig → src/types/uuid.zig
View File

946
src/ziqlParser.zig Normal file
View File

@ -0,0 +1,946 @@
const std = @import("std");
const schemaEngine = @import("schemaEngine.zig");
const DataEngine = @import("fileEngine.zig").FileEngine;
const Condition = @import("fileEngine.zig").FileEngine.Condition;
const Tokenizer = @import("tokenizers/ziql.zig").Tokenizer;
const Token = @import("tokenizers/ziql.zig").Token;
const UUID = @import("types/uuid.zig").UUID;
const Allocator = std.mem.Allocator;
pub const Parser = struct {
allocator: Allocator,
toker: *Tokenizer,
state: State,
data_engine: *DataEngine,
additional_data: AdditionalData,
struct_name: []const u8 = undefined,
action: Action = undefined,
pub fn init(allocator: Allocator, toker: *Tokenizer) Parser {
var data_engine = DataEngine.init(allocator, null);
return Parser{
.allocator = allocator,
.toker = toker,
.state = State.start,
.data_engine = &data_engine,
.additional_data = AdditionalData.init(allocator),
};
}
pub fn deinit(self: *Parser) void {
self.additional_data.deinit();
//self.allocator.free(self.struct_name);
//self.data_engine.deinit();
}
const Action = enum {
GRAB,
ADD,
UPDATE,
DELETE,
};
const State = enum {
start,
invalid,
end,
// Endpoint
parse_new_data_and_add_data,
filter_and_send,
// For the main parse function
expect_struct_name,
expect_filter,
expect_additional_data,
expect_filter_or_additional_data,
expect_new_data,
expect_right_arrow,
// For the additional data parser
expect_count_of_entity_to_find,
expect_semicolon_OR_right_bracket,
expect_member,
expect_comma_OR_r_bracket_OR_l_bracket,
expect_comma_OR_r_bracket,
// For the filter parser
expect_left_condition, // Condition is a struct in DataEngine, it's all info necessary to get a list of UUID usinf DataEngine.getUUIDListUsingCondition
expect_operation, // Operations are = != < <= > >=
expect_value,
expect_ANDOR_OR_end,
expect_right_uuid_array,
// For the new data
expect_equal,
expect_new_value,
expect_comma_OR_end,
add_member_to_map,
add_array_to_map,
};
/// This is the [] part
/// IDK if saving it into the Parser struct is a good idea
pub const AdditionalData = struct {
entity_count_to_find: usize = 0,
member_to_find: std.ArrayList(AdditionalDataMember),
pub fn init(allocator: Allocator) AdditionalData {
return AdditionalData{ .member_to_find = std.ArrayList(AdditionalDataMember).init(allocator) };
}
pub fn deinit(self: *AdditionalData) void {
for (0..self.member_to_find.items.len) |i| {
self.member_to_find.items[i].additional_data.deinit();
}
self.member_to_find.deinit();
}
};
// This is name in: [name]
// There is an additional data because it can be [friend [1; name]]
const AdditionalDataMember = struct {
name: []const u8,
additional_data: AdditionalData,
pub fn init(allocator: Allocator, name: []const u8) AdditionalDataMember {
const additional_data = AdditionalData.init(allocator);
return AdditionalDataMember{ .name = name, .additional_data = additional_data };
}
};
pub fn parse(self: *Parser) !void {
var token = self.toker.next();
var keep_next = false; // Use in the loop to prevent to get the next token when continue. Just need to make it true and it is reset at every loop
while (self.state != State.end) : ({
token = if (!keep_next) self.toker.next() else token;
keep_next = false;
}) {
switch (self.state) {
.start => {
switch (token.tag) {
.keyword_grab => {
self.action = Action.GRAB;
self.state = State.expect_struct_name;
},
.keyword_add => {
self.action = Action.ADD;
self.state = State.expect_struct_name;
},
.keyword_update => {
self.action = Action.UPDATE;
self.state = State.expect_struct_name;
},
.keyword_delete => {
self.action = Action.DELETE;
self.state = State.expect_struct_name;
},
.keyword__describe__ => {
std.debug.print("{s}", .{@embedFile("schema.zipponschema")});
self.state = State.end;
},
.keyword__init__ => {
try self.data_engine.initDataFolder();
self.state = State.end;
},
else => {
self.printError("Error: Expected action keyword. Available: GRAB ADD DELETE UPDATE", &token);
self.state = State.end;
},
}
},
.expect_struct_name => {
// Check if the struct name is in the schema
self.struct_name = try self.allocator.dupe(u8, self.toker.getTokenSlice(token));
if (!schemaEngine.isStructNameExists(self.struct_name)) self.printError("Error: struct name not found in schema.", &token);
switch (self.action) {
.ADD => self.state = State.expect_new_data,
else => self.state = State.expect_filter_or_additional_data,
}
},
.expect_filter_or_additional_data => {
keep_next = true;
switch (token.tag) {
.l_bracket => self.state = State.expect_additional_data,
.l_brace => self.state = State.filter_and_send,
else => self.printError("Error: Expect [ for additional data or { for a filter", &token),
}
},
.expect_additional_data => {
try self.parseAdditionalData(&self.additional_data);
self.state = State.filter_and_send;
},
.filter_and_send => {
var array = std.ArrayList(UUID).init(self.allocator);
defer array.deinit();
try self.parseFilter(&array);
self.sendEntity(array.items);
self.state = State.end;
},
.expect_new_data => {
switch (token.tag) {
.l_paren => {
keep_next = true;
self.state = State.parse_new_data_and_add_data;
},
else => self.printError("Error: Expecting new data starting with (", &token),
}
},
.parse_new_data_and_add_data => {
switch (self.action) {
.ADD => {
const data_map = std.StringHashMap([]const u8).init(self.allocator);
defer data_map.deinit();
self.parseNewData(&data_map);
if (!schemaEngine.checkIfAllMemberInMap(self.struct_name, data_map)) {}
try self.data_engine.writeEntity(self.struct_name, data_map);
self.state = State.end;
},
.UPDATE => {}, // TODO:
else => unreachable,
}
},
else => unreachable,
}
}
}
// TODO: Use that when I want to return data to the use, need to understand how it's work.
// I think for now put the ordering using additional data here
// Maybe to a struct Communicator to handle all communication between use and cli
fn sendEntity(self: *Parser, uuid_array: []UUID) void {
_ = self;
std.debug.print("Number of uuid to send: {d}", .{uuid_array.len});
}
/// Take an array of UUID and populate it to be the array that represent filter between {}
/// Main is to know if between {} or (), main is true if between {} or the first to be call
/// TODO: Create a parseCondition
fn parseFilter(self: *Parser, left_array: *std.ArrayList(UUID), struct_name: []const u8, main: bool) !void {
var token = self.toker.next();
var keep_next = false;
self.state = State.expect_left_condition;
var left_condition = Condition.init(struct_name);
var curent_operation: enum { and_, or_ } = undefined;
while (self.state != State.end) : ({
token = if (!keep_next) self.toker.next() else token;
keep_next = false;
}) {
switch (self.state) {
.expect_left_condition => {
self.parseCondition(&left_condition, &token);
try self.data_engine.getUUIDListUsingCondition(left_condition, left_array);
self.state = State.expect_ANDOR_OR_end;
},
.expect_ANDOR_OR_end => {
switch (token.tag) {
.r_brace => {
if (main) {
self.state = State.end;
} else {
self.printError("Error: Expected } to end main condition or AND/OR to continue it", &token);
}
},
.r_paren => {
if (!main) {
self.state = State.end;
} else {
self.printError("Error: Expected ) to end inside condition or AND/OR to continue it", &token);
}
},
.keyword_and => {
curent_operation = .and_;
self.state = State.expect_right_uuid_array;
},
.keyword_or => {
curent_operation = .or_;
self.state = State.expect_right_uuid_array;
},
else => self.printError("Error: Expected a condition including AND or OR or } or )", &token),
}
},
.expect_right_uuid_array => {
var right_array = std.ArrayList(UUID).init(self.allocator);
defer right_array.deinit();
switch (token.tag) {
.l_paren => try self.parseFilter(&right_array, struct_name, false), // run parserFilter to get the right array
.identifier => {
var right_condition = Condition.init(struct_name);
self.parseCondition(&right_condition, &token);
try self.data_engine.getUUIDListUsingCondition(right_condition, &right_array);
}, // Create a new condition and compare it
else => self.printError("Error: Expecting ( or member name.", &token),
}
switch (curent_operation) {
.and_ => {
try AND(left_array, &right_array);
},
.or_ => {
try OR(left_array, &right_array);
},
}
self.state = .expect_ANDOR_OR_end;
},
else => unreachable,
}
}
}
fn parseCondition(self: *Parser, condition: *Condition, token_ptr: *Token) void {
var keep_next = false;
self.state = State.expect_member;
var token = token_ptr.*;
while (self.state != State.end) : ({
token = if (!keep_next) self.toker.next() else token;
keep_next = false;
}) {
switch (self.state) {
.expect_member => {
switch (token.tag) {
.identifier => {
if (!schemaEngine.isMemberPartOfStruct(condition.struct_name, self.toker.getTokenSlice(token))) {
self.printError("Error: Member not part of struct.", &token);
}
condition.data_type = schemaEngine.memberName2DataType(condition.struct_name, self.toker.getTokenSlice(token)) orelse @panic("Couldn't find the struct and member");
condition.member_name = self.toker.getTokenSlice(token);
self.state = State.expect_operation;
},
else => self.printError("Error: Expected member name.", &token),
}
},
.expect_operation => {
switch (token.tag) {
.equal => condition.operation = .equal, // =
.angle_bracket_left => condition.operation = .inferior, // <
.angle_bracket_right => condition.operation = .superior, // >
.angle_bracket_left_equal => condition.operation = .inferior_or_equal, // <=
.angle_bracket_right_equal => condition.operation = .superior_or_equal, // >=
.bang_equal => condition.operation = .different, // !=
else => self.printError("Error: Expected condition. Including < > <= >= = !=", &token),
}
self.state = State.expect_value;
},
.expect_value => {
switch (condition.data_type) {
.int => {
switch (token.tag) {
.int_literal => condition.value = self.toker.getTokenSlice(token),
else => self.printError("Error: Expected int", &token),
}
},
.float => {
switch (token.tag) {
.float_literal => condition.value = self.toker.getTokenSlice(token),
else => self.printError("Error: Expected float", &token),
}
},
.str => {
switch (token.tag) {
.string_literal => condition.value = self.toker.getTokenSlice(token),
else => self.printError("Error: Expected string", &token),
}
},
.bool => {
switch (token.tag) {
.bool_literal_true, .bool_literal_false => condition.value = self.toker.getTokenSlice(token),
else => self.printError("Error: Expected bool", &token),
}
},
.int_array => {
const start_index = token.loc.start;
token = self.toker.next();
while (token.tag != Token.Tag.r_bracket) : (token = self.toker.next()) {
switch (token.tag) {
.int_literal => continue,
else => self.printError("Error: Expected int or ].", &token),
}
}
condition.value = self.toker.buffer[start_index..token.loc.end];
},
.float_array => {
const start_index = token.loc.start;
token = self.toker.next();
while (token.tag != Token.Tag.r_bracket) : (token = self.toker.next()) {
switch (token.tag) {
.float_literal => continue,
else => self.printError("Error: Expected float or ].", &token),
}
}
condition.value = self.toker.buffer[start_index..token.loc.end];
},
.str_array => {
const start_index = token.loc.start;
token = self.toker.next();
while (token.tag != Token.Tag.r_bracket) : (token = self.toker.next()) {
switch (token.tag) {
.string_literal => continue,
else => self.printError("Error: Expected string or ].", &token),
}
}
condition.value = self.toker.buffer[start_index..token.loc.end];
},
.bool_array => {
const start_index = token.loc.start;
token = self.toker.next();
while (token.tag != Token.Tag.r_bracket) : (token = self.toker.next()) {
switch (token.tag) {
.bool_literal_false, .bool_literal_true => continue,
else => self.printError("Error: Expected bool or ].", &token),
}
}
condition.value = self.toker.buffer[start_index..token.loc.end];
},
}
self.state = .end;
},
else => unreachable,
}
}
}
/// When this function is call, the tokenizer last token retrieved should be [.
/// Check if an int is here -> check if ; is here -> check if member is here -> check if [ is here -> loop
fn parseAdditionalData(self: *Parser, additional_data: *AdditionalData) !void {
var token = self.toker.next();
var keep_next = false;
self.state = State.expect_count_of_entity_to_find;
while (self.state != State.end) : ({
token = if (!keep_next) self.toker.next() else token;
keep_next = false;
}) {
switch (self.state) {
.expect_count_of_entity_to_find => {
switch (token.tag) {
.int_literal => {
const count = std.fmt.parseInt(usize, self.toker.getTokenSlice(token), 10) catch {
self.printError("Error while transforming this into a integer.", &token);
self.state = .invalid;
continue;
};
additional_data.entity_count_to_find = count;
self.state = .expect_semicolon_OR_right_bracket;
},
else => {
self.state = .expect_member;
keep_next = true;
},
}
},
.expect_semicolon_OR_right_bracket => {
switch (token.tag) {
.semicolon => self.state = .expect_member,
.r_bracket => self.state = State.end,
else => self.printError("Error: Expect ';' or ']'.", &token),
}
},
.expect_member => {
switch (token.tag) {
.identifier => {
if (!schemaEngine.isMemberNameInStruct(self.struct_name, self.toker.getTokenSlice(token))) self.printError("Member not found in struct.", &token);
try additional_data.member_to_find.append(
AdditionalDataMember.init(
self.allocator,
self.toker.getTokenSlice(token),
),
);
self.state = .expect_comma_OR_r_bracket_OR_l_bracket;
},
else => self.printError("Error: Expected a member name.", &token),
}
},
.expect_comma_OR_r_bracket_OR_l_bracket => {
switch (token.tag) {
.comma => self.state = .expect_member,
.r_bracket => {
self.state = State.end;
keep_next = true;
},
.l_bracket => {
try self.parseAdditionalData(
&additional_data.member_to_find.items[additional_data.member_to_find.items.len - 1].additional_data,
);
self.state = .expect_comma_OR_r_bracket;
},
else => self.printError("Error: Expected , or ] or [", &token),
}
},
.expect_comma_OR_r_bracket => {
switch (token.tag) {
.comma => self.state = .expect_member,
.r_bracket => {
self.state = State.end;
keep_next = true;
},
else => self.printError("Error: Expected , or ]", &token),
}
},
else => unreachable,
}
}
}
/// Take the tokenizer and return a map of the query for the ADD command.
/// Keys are the member name and value are the string of the value in the query. E.g. 'Adrien' or '10'
/// Entry token need to be (
fn parseNewData(self: *Parser, member_map: *std.StringHashMap([]const u8)) void {
var token = self.toker.next();
var keep_next = false;
var member_name: []const u8 = undefined; // Maybe use allocator.alloc
self.state = State.expect_member;
while (self.state != State.end) : ({
token = if (!keep_next) self.toker.next() else token;
keep_next = false;
}) {
switch (self.state) {
.expect_member => {
switch (token.tag) {
.identifier => {
member_name = self.toker.getTokenSlice(token);
if (!schemaEngine.isMemberNameInStruct(self.struct_name, member_name)) self.printError("Member not found in struct.", &token);
self.state = State.expect_equal;
},
else => self.printError("Error: Expected member name.", &token),
}
},
.expect_equal => {
switch (token.tag) {
// TODO: Add more comparison like IN or other stuff
.equal => self.state = State.expect_new_value,
else => self.printError("Error: Expected =", &token),
}
},
.expect_new_value => {
const data_type = schemaEngine.memberName2DataType(self.struct_name, member_name);
switch (data_type.?) {
.int => {
switch (token.tag) {
.int_literal, .keyword_null => {
keep_next = true;
self.state = State.add_member_to_map;
},
else => self.printError("Error: Expected int", &token),
}
},
.float => {
switch (token.tag) {
.float_literal, .keyword_null => {
keep_next = true;
self.state = State.add_member_to_map;
},
else => self.printError("Error: Expected float", &token),
}
},
.bool => {
switch (token.tag) {
.bool_literal_true, .bool_literal_false, .keyword_null => {
keep_next = true;
self.state = State.add_member_to_map;
},
else => self.printError("Error: Expected bool: true false", &token),
}
},
.str => {
switch (token.tag) {
.string_literal, .keyword_null => {
keep_next = true;
self.state = State.add_member_to_map;
},
else => self.printError("Error: Expected string between ''", &token),
}
},
// TODO: Maybe upgrade that to use multiple state
.int_array => {
switch (token.tag) {
.l_bracket => {
const start_index = token.loc.start;
token = self.toker.next();
while (token.tag != Token.Tag.r_bracket) : (token = self.toker.next()) {
switch (token.tag) {
.int_literal => continue,
else => self.printError("Error: Expected int or ].", &token),
}
}
// Maybe change that as it just recreate a string that is already in the buffer
member_map.put(member_name, self.toker.buffer[start_index..token.loc.end]) catch @panic("Couln't add string of array in data map");
self.state = State.expect_comma_OR_end;
},
else => self.printError("Error: Expected [ to start an array", &token),
}
},
.float_array => {
switch (token.tag) {
.l_bracket => {
const start_index = token.loc.start;
token = self.toker.next();
while (token.tag != Token.Tag.r_bracket) : (token = self.toker.next()) {
switch (token.tag) {
.float_literal => continue,
else => self.printError("Error: Expected float or ].", &token),
}
}
// Maybe change that as it just recreate a string that is already in the buffer
member_map.put(member_name, self.toker.buffer[start_index..token.loc.end]) catch @panic("Couln't add string of array in data map");
self.state = State.expect_comma_OR_end;
},
else => self.printError("Error: Expected [ to start an array", &token),
}
},
.bool_array => {
switch (token.tag) {
.l_bracket => {
const start_index = token.loc.start;
token = self.toker.next();
while (token.tag != Token.Tag.r_bracket) : (token = self.toker.next()) {
switch (token.tag) {
.bool_literal_false, .bool_literal_true => continue,
else => self.printError("Error: Expected bool or ].", &token),
}
}
// Maybe change that as it just recreate a string that is already in the buffer
member_map.put(member_name, self.toker.buffer[start_index..token.loc.end]) catch @panic("Couln't add string of array in data map");
self.state = State.expect_comma_OR_end;
},
else => self.printError("Error: Expected [ to start an array", &token),
}
},
.str_array => {
switch (token.tag) {
.l_bracket => {
const start_index = token.loc.start;
token = self.toker.next();
while (token.tag != Token.Tag.r_bracket) : (token = self.toker.next()) {
switch (token.tag) {
.string_literal => continue,
else => self.printError("Error: Expected str or ].", &token),
}
}
// Maybe change that as it just recreate a string that is already in the buffer
member_map.put(member_name, self.toker.buffer[start_index..token.loc.end]) catch @panic("Couln't add string of array in data map");
self.state = State.expect_comma_OR_end;
},
else => self.printError("Error: Expected [ to start an array", &token),
}
},
}
},
.add_member_to_map => {
member_map.put(member_name, self.toker.getTokenSlice(token)) catch @panic("Could not add member name and value to map in getMapOfMember");
self.state = State.expect_comma_OR_end;
},
.add_array_to_map => {},
.expect_comma_OR_end => {
switch (token.tag) {
.r_paren => self.state = State.end,
.comma => self.state = State.expect_member,
else => self.printError("Error: Expect , or )", &token),
}
},
else => unreachable,
}
}
}
fn printError(self: *Parser, message: []const u8, token: *Token) void {
std.debug.print("\n", .{});
std.debug.print("{s}\n", .{self.toker.buffer});
// Calculate the number of spaces needed to reach the start position.
var spaces: usize = 0;
while (spaces < token.loc.start) : (spaces += 1) {
std.debug.print(" ", .{});
}
// Print the '^' characters for the error span.
var i: usize = token.loc.start;
while (i < token.loc.end) : (i += 1) {
std.debug.print("^", .{});
}
std.debug.print(" \n", .{}); // Align with the message
std.debug.print("{s}\n", .{message});
std.debug.print("{any}\n{any}\n", .{ token.tag, token.loc });
@panic("");
}
};
// TODO: Optimize both
fn OR(arr1: *std.ArrayList(UUID), arr2: *std.ArrayList(UUID)) !void {
for (0..arr2.items.len) |i| {
if (!containUUID(arr1.*, arr2.items[i])) {
try arr1.append(arr2.items[i]);
}
}
}
fn AND(arr1: *std.ArrayList(UUID), arr2: *std.ArrayList(UUID)) !void {
var i: usize = 0;
for (0..arr1.items.len) |_| {
if (!containUUID(arr2.*, arr1.items[i])) {
_ = arr1.orderedRemove(i);
} else {
i += 1;
}
}
}
test "OR & AND" {
const allocator = std.testing.allocator;
var right_arr = std.ArrayList(UUID).init(allocator);
defer right_arr.deinit();
try right_arr.append(try UUID.parse("00000000-0000-0000-0000-000000000000"));
try right_arr.append(try UUID.parse("00000000-0000-0000-0000-000000000001"));
try right_arr.append(try UUID.parse("00000000-0000-0000-0000-000000000005"));
try right_arr.append(try UUID.parse("00000000-0000-0000-0000-000000000006"));
try right_arr.append(try UUID.parse("00000000-0000-0000-0000-000000000007"));
var left_arr1 = std.ArrayList(UUID).init(allocator);
defer left_arr1.deinit();
try left_arr1.append(try UUID.parse("00000000-0000-0000-0000-000000000000"));
try left_arr1.append(try UUID.parse("00000000-0000-0000-0000-000000000001"));
try left_arr1.append(try UUID.parse("00000000-0000-0000-0000-000000000002"));
try left_arr1.append(try UUID.parse("00000000-0000-0000-0000-000000000003"));
try left_arr1.append(try UUID.parse("00000000-0000-0000-0000-000000000004"));
var expected_arr1 = std.ArrayList(UUID).init(allocator);
defer expected_arr1.deinit();
try expected_arr1.append(try UUID.parse("00000000-0000-0000-0000-000000000000"));
try expected_arr1.append(try UUID.parse("00000000-0000-0000-0000-000000000001"));
try AND(&left_arr1, &right_arr);
try std.testing.expect(compareUUIDArray(left_arr1, expected_arr1));
var left_arr2 = std.ArrayList(UUID).init(allocator);
defer left_arr2.deinit();
try left_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000000"));
try left_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000001"));
try left_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000002"));
try left_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000003"));
try left_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000004"));
var expected_arr2 = std.ArrayList(UUID).init(allocator);
defer expected_arr2.deinit();
try expected_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000000"));
try expected_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000001"));
try expected_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000002"));
try expected_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000003"));
try expected_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000004"));
try expected_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000005"));
try expected_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000006"));
try expected_arr2.append(try UUID.parse("00000000-0000-0000-0000-000000000007"));
try OR(&left_arr2, &right_arr);
try std.testing.expect(compareUUIDArray(left_arr2, expected_arr2));
}
fn containUUID(arr: std.ArrayList(UUID), value: UUID) bool {
return for (arr.items) |elem| {
if (value.compare(elem)) break true;
} else false;
}
fn compareUUIDArray(arr1: std.ArrayList(UUID), arr2: std.ArrayList(UUID)) bool {
if (arr1.items.len != arr2.items.len) {
std.debug.print("Not same array len when comparing UUID. arr1: {d} arr2: {d}\n", .{ arr1.items.len, arr2.items.len });
return false;
}
for (0..arr1.items.len) |i| {
if (!containUUID(arr2, arr1.items[i])) return false;
}
return true;
}
test "Parse filter" {
const allocator = std.testing.allocator;
var tokenizer = Tokenizer.init("{name = 'Adrien'}");
var parser = Parser.init(allocator, &tokenizer);
_ = tokenizer.next();
var uuid_array = std.ArrayList(UUID).init(allocator);
defer uuid_array.deinit();
try parser.parseFilter(&uuid_array, "User", true);
}
test "Parse condition" {
const condition1 = Condition{ .data_type = .int, .member_name = "age", .operation = .superior_or_equal, .struct_name = "User", .value = "26" };
try testConditionParsing("age >= 26", condition1);
const condition2 = Condition{ .data_type = .int_array, .member_name = "scores", .operation = .equal, .struct_name = "User", .value = "[1 2 42]" };
try testConditionParsing("scores = [1 2 42]", condition2);
const condition3 = Condition{ .data_type = .str, .member_name = "email", .operation = .equal, .struct_name = "User", .value = "'adrien@email.com'" };
try testConditionParsing("email = 'adrien@email.com'", condition3);
}
fn testConditionParsing(source: [:0]const u8, expected_condition: Condition) !void {
const allocator = std.testing.allocator;
var tokenizer = Tokenizer.init(source);
var parser = Parser.init(allocator, &tokenizer);
var token = tokenizer.next();
var condition = Condition.init("User");
parser.parseCondition(&condition, &token);
try std.testing.expect(compareCondition(expected_condition, condition));
}
fn compareCondition(c1: Condition, c2: Condition) bool {
return ((std.mem.eql(u8, c1.value, c2.value)) and (std.mem.eql(u8, c1.struct_name, c2.struct_name)) and (std.mem.eql(u8, c1.member_name, c2.member_name)) and (c1.operation == c2.operation) and (c1.data_type == c2.data_type));
}
// TODO: Test Filter parser
test "Parse new data" {
const allocator = std.testing.allocator;
var map1 = std.StringHashMap([]const u8).init(allocator);
defer map1.deinit();
try map1.put("name", "'Adrien'");
testNewDataParsing("(name = 'Adrien')", map1);
var map2 = std.StringHashMap([]const u8).init(allocator);
defer map2.deinit();
try map2.put("name", "'Adrien'");
try map2.put("email", "'adrien@email.com'");
try map2.put("scores", "[1 4 19]");
try map2.put("age", "26");
testNewDataParsing("(name = 'Adrien', scores = [1 4 19], age = 26, email = 'adrien@email.com')", map2);
}
fn testNewDataParsing(source: [:0]const u8, expected_member_map: std.StringHashMap([]const u8)) void {
const allocator = std.testing.allocator;
var tokenizer = Tokenizer.init(source);
var parser = Parser.init(allocator, &tokenizer);
parser.struct_name = allocator.dupe(u8, "User") catch @panic("Cant alloc struct name");
defer parser.deinit();
defer allocator.free(parser.struct_name);
var data_map = std.StringHashMap([]const u8).init(allocator);
defer data_map.deinit();
_ = tokenizer.next();
parser.parseNewData(&data_map);
var iterator = expected_member_map.iterator();
var expected_total_count: usize = 0;
var found_count: usize = 0;
var error_found = false;
while (iterator.next()) |entry| {
expected_total_count += 1;
if (!data_map.contains(entry.key_ptr.*)) {
std.debug.print("Error new data parsing: Missing {s} in parsed map.\n", .{entry.key_ptr.*});
error_found = true;
continue;
}
if (!std.mem.eql(u8, entry.value_ptr.*, data_map.get(entry.key_ptr.*).?)) {
std.debug.print("Error new data parsing: Wrong data for {s} in parsed map.\n Expected: {s}\n Got: {s}", .{ entry.key_ptr.*, entry.value_ptr.*, data_map.get(entry.key_ptr.*).? });
error_found = true;
continue;
}
found_count += 1;
}
if ((error_found) or (expected_total_count != found_count)) @panic("=(");
}
test "Parse additional data" {
const allocator = std.testing.allocator;
var additional_data1 = Parser.AdditionalData.init(allocator);
additional_data1.entity_count_to_find = 1;
testAdditionalData("[1]", additional_data1);
var additional_data2 = Parser.AdditionalData.init(allocator);
defer additional_data2.deinit();
try additional_data2.member_to_find.append(
Parser.AdditionalDataMember.init(
allocator,
"name",
),
);
testAdditionalData("[name]", additional_data2);
var additional_data3 = Parser.AdditionalData.init(allocator);
additional_data3.entity_count_to_find = 1;
defer additional_data3.deinit();
try additional_data3.member_to_find.append(
Parser.AdditionalDataMember.init(
allocator,
"name",
),
);
testAdditionalData("[1; name]", additional_data3);
var additional_data4 = Parser.AdditionalData.init(allocator);
additional_data4.entity_count_to_find = 100;
defer additional_data4.deinit();
try additional_data4.member_to_find.append(
Parser.AdditionalDataMember.init(
allocator,
"friends",
),
);
testAdditionalData("[100; friends [name]]", additional_data4);
}
fn testAdditionalData(source: [:0]const u8, expected_AdditionalData: Parser.AdditionalData) void {
const allocator = std.testing.allocator;
var tokenizer = Tokenizer.init(source);
var parser = Parser.init(allocator, &tokenizer);
parser.struct_name = allocator.dupe(u8, "User") catch @panic("Cant alloc struct name");
defer parser.deinit();
defer allocator.free(parser.struct_name);
_ = tokenizer.next();
parser.parseAdditionalData(&parser.additional_data) catch |err| {
std.debug.print("Error parsing additional data: {any}\n", .{err});
};
compareAdditionalData(expected_AdditionalData, parser.additional_data);
}
fn compareAdditionalData(ad1: Parser.AdditionalData, ad2: Parser.AdditionalData) void {
std.testing.expectEqual(ad1.entity_count_to_find, ad2.entity_count_to_find) catch {
std.debug.print("Additional data entity_count_to_find are not equal.\n", .{});
};
var founded = false;
for (ad1.member_to_find.items) |elem1| {
founded = false;
for (ad2.member_to_find.items) |elem2| {
if (std.mem.eql(u8, elem1.name, elem2.name)) {
compareAdditionalData(elem1.additional_data, elem2.additional_data);
founded = true;
break;
}
}
std.testing.expect(founded) catch {
std.debug.print("{s} not found\n", .{elem1.name});
@panic("=(");
};
}
}

298
test_runner.zig Normal file
View File

@ -0,0 +1,298 @@
const std = @import("std");
const builtin = @import("builtin");
const Allocator = std.mem.Allocator;
const BORDER = "=" ** 80;
// use in custom panic handler
var current_test: ?[]const u8 = null;
pub fn main() !void {
var mem: [8192]u8 = undefined;
var fba = std.heap.FixedBufferAllocator.init(&mem);
const allocator = fba.allocator();
const env = Env.init(allocator);
defer env.deinit(allocator);
var slowest = SlowTracker.init(allocator, 5);
defer slowest.deinit();
var pass: usize = 0;
var fail: usize = 0;
var skip: usize = 0;
var leak: usize = 0;
const printer = Printer.init();
printer.fmt("\r\x1b[0K", .{}); // beginning of line and clear to end of line
for (builtin.test_functions) |t| {
if (isSetup(t)) {
t.func() catch |err| {
printer.status(.fail, "\nsetup \"{s}\" failed: {}\n", .{ t.name, err });
return err;
};
}
}
for (builtin.test_functions) |t| {
if (isSetup(t) or isTeardown(t)) {
continue;
}
var status = Status.pass;
slowest.startTiming();
const is_unnamed_test = isUnnamed(t);
if (env.filter) |f| {
if (!is_unnamed_test and std.mem.indexOf(u8, t.name, f) == null) {
continue;
}
}
const friendly_name = blk: {
const name = t.name;
var it = std.mem.splitScalar(u8, name, '.');
while (it.next()) |value| {
if (std.mem.eql(u8, value, "test")) {
const rest = it.rest();
break :blk if (rest.len > 0) rest else name;
}
}
break :blk name;
};
current_test = friendly_name;
std.testing.allocator_instance = .{};
const result = t.func();
current_test = null;
const ns_taken = slowest.endTiming(friendly_name);
if (std.testing.allocator_instance.deinit() == .leak) {
leak += 1;
printer.status(.fail, "\n{s}\n\"{s}\" - Memory Leak\n{s}\n", .{ BORDER, friendly_name, BORDER });
}
if (result) |_| {
pass += 1;
} else |err| switch (err) {
error.SkipZigTest => {
skip += 1;
status = .skip;
},
else => {
status = .fail;
fail += 1;
printer.status(.fail, "\n{s}\n\"{s}\" - {s}\n{s}\n", .{ BORDER, friendly_name, @errorName(err), BORDER });
if (@errorReturnTrace()) |trace| {
std.debug.dumpStackTrace(trace.*);
}
if (env.fail_first) {
break;
}
},
}
if (env.verbose) {
const ms = @as(f64, @floatFromInt(ns_taken)) / 1_000_000.0;
printer.status(status, "{s} ({d:.2}ms)\n", .{ friendly_name, ms });
} else {
printer.status(status, ".", .{});
}
}
for (builtin.test_functions) |t| {
if (isTeardown(t)) {
t.func() catch |err| {
printer.status(.fail, "\nteardown \"{s}\" failed: {}\n", .{ t.name, err });
return err;
};
}
}
const total_tests = pass + fail;
const status = if (fail == 0) Status.pass else Status.fail;
printer.status(status, "\n{d} of {d} test{s} passed\n", .{ pass, total_tests, if (total_tests != 1) "s" else "" });
if (skip > 0) {
printer.status(.skip, "{d} test{s} skipped\n", .{ skip, if (skip != 1) "s" else "" });
}
if (leak > 0) {
printer.status(.fail, "{d} test{s} leaked\n", .{ leak, if (leak != 1) "s" else "" });
}
printer.fmt("\n", .{});
try slowest.display(printer);
printer.fmt("\n", .{});
std.posix.exit(if (fail == 0) 0 else 1);
}
const Printer = struct {
out: std.fs.File.Writer,
fn init() Printer {
return .{
.out = std.io.getStdErr().writer(),
};
}
fn fmt(self: Printer, comptime format: []const u8, args: anytype) void {
std.fmt.format(self.out, format, args) catch unreachable;
}
fn status(self: Printer, s: Status, comptime format: []const u8, args: anytype) void {
const color = switch (s) {
.pass => "\x1b[32m",
.fail => "\x1b[31m",
.skip => "\x1b[33m",
else => "",
};
const out = self.out;
out.writeAll(color) catch @panic("writeAll failed?!");
std.fmt.format(out, format, args) catch @panic("std.fmt.format failed?!");
self.fmt("\x1b[0m", .{});
}
};
const Status = enum {
pass,
fail,
skip,
text,
};
const SlowTracker = struct {
const SlowestQueue = std.PriorityDequeue(TestInfo, void, compareTiming);
max: usize,
slowest: SlowestQueue,
timer: std.time.Timer,
fn init(allocator: Allocator, count: u32) SlowTracker {
const timer = std.time.Timer.start() catch @panic("failed to start timer");
var slowest = SlowestQueue.init(allocator, {});
slowest.ensureTotalCapacity(count) catch @panic("OOM");
return .{
.max = count,
.timer = timer,
.slowest = slowest,
};
}
const TestInfo = struct {
ns: u64,
name: []const u8,
};
fn deinit(self: SlowTracker) void {
self.slowest.deinit();
}
fn startTiming(self: *SlowTracker) void {
self.timer.reset();
}
fn endTiming(self: *SlowTracker, test_name: []const u8) u64 {
var timer = self.timer;
const ns = timer.lap();
var slowest = &self.slowest;
if (slowest.count() < self.max) {
// Capacity is fixed to the # of slow tests we want to track
// If we've tracked fewer tests than this capacity, than always add
slowest.add(TestInfo{ .ns = ns, .name = test_name }) catch @panic("failed to track test timing");
return ns;
}
{
// Optimization to avoid shifting the dequeue for the common case
// where the test isn't one of our slowest.
const fastest_of_the_slow = slowest.peekMin() orelse unreachable;
if (fastest_of_the_slow.ns > ns) {
// the test was faster than our fastest slow test, don't add
return ns;
}
}
// the previous fastest of our slow tests, has been pushed off.
_ = slowest.removeMin();
slowest.add(TestInfo{ .ns = ns, .name = test_name }) catch @panic("failed to track test timing");
return ns;
}
fn display(self: *SlowTracker, printer: Printer) !void {
var slowest = self.slowest;
const count = slowest.count();
printer.fmt("Slowest {d} test{s}: \n", .{ count, if (count != 1) "s" else "" });
while (slowest.removeMinOrNull()) |info| {
const ms = @as(f64, @floatFromInt(info.ns)) / 1_000_000.0;
printer.fmt(" {d:.2}ms\t{s}\n", .{ ms, info.name });
}
}
fn compareTiming(context: void, a: TestInfo, b: TestInfo) std.math.Order {
_ = context;
return std.math.order(a.ns, b.ns);
}
};
const Env = struct {
verbose: bool,
fail_first: bool,
filter: ?[]const u8,
fn init(allocator: Allocator) Env {
return .{
.verbose = readEnvBool(allocator, "TEST_VERBOSE", true),
.fail_first = readEnvBool(allocator, "TEST_FAIL_FIRST", false),
.filter = readEnv(allocator, "TEST_FILTER"),
};
}
fn deinit(self: Env, allocator: Allocator) void {
if (self.filter) |f| {
allocator.free(f);
}
}
fn readEnv(allocator: Allocator, key: []const u8) ?[]const u8 {
const v = std.process.getEnvVarOwned(allocator, key) catch |err| {
if (err == error.EnvironmentVariableNotFound) {
return null;
}
std.log.warn("failed to get env var {s} due to err {}", .{ key, err });
return null;
};
return v;
}
fn readEnvBool(allocator: Allocator, key: []const u8, deflt: bool) bool {
const value = readEnv(allocator, key) orelse return deflt;
defer allocator.free(value);
return std.ascii.eqlIgnoreCase(value, "true");
}
};
pub fn panic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, ret_addr: ?usize) noreturn {
if (current_test) |ct| {
std.debug.print("\x1b[31m{s}\npanic running \"{s}\"\n{s}\x1b[0m\n", .{ BORDER, ct, BORDER });
}
std.builtin.panic(msg, error_return_trace, ret_addr);
}
fn isUnnamed(t: std.builtin.TestFn) bool {
const marker = ".test_";
const test_name = t.name;
const index = std.mem.indexOf(u8, test_name, marker) orelse return false;
_ = std.fmt.parseInt(u32, test_name[index + marker.len ..], 10) catch return false;
return true;
}
fn isSetup(t: std.builtin.TestFn) bool {
return std.mem.endsWith(u8, t.name, "tests:beforeAll");
}
fn isTeardown(t: std.builtin.TestFn) bool {
return std.mem.endsWith(u8, t.name, "tests:afterAll");
}