adrien/ZipponDB
1
0
Fork 0
Code Actions Releases Activity

Commit to continue working in the car to europa park

Browse Source
This commit is contained in:
Adrien Bouvais 2024-09-29 06:30:45 +02:00
parent a40a57df3d
commit 87f070231e
13 changed files with 443 additions and 381 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
TECHNICAL_DOCS.md
View File

@ -1,5 +1,3 @@
# How the file system work
# All class of ZipponDB
## Tokenizer

16
build.zig
View File

@ -51,13 +51,13 @@ pub fn build(b: *std.Build) void {
});
const run_tests4 = b.addRunArtifact(tests4);
//const tests5 = b.addTest(.{
// .root_source_file = b.path("src/ADD.zig"),
// .target = target,
// .optimize = optimize,
// .name = "ADD",
//});
//const run_tests5 = b.addRunArtifact(tests5);
const tests5 = b.addTest(.{
.root_source_file = b.path("src/uuid.zig"),
.target = target,
.optimize = optimize,
.name = "UUID",
});
const run_tests5 = b.addRunArtifact(tests5);
const tests6 = b.addTest(.{
.root_source_file = b.path("src/GRAB.zig"),
@ -72,6 +72,6 @@ pub fn build(b: *std.Build) void {
test_step.dependOn(&run_tests2.step);
test_step.dependOn(&run_tests3.step);
test_step.dependOn(&run_tests4.step);
//test_step.dependOn(&run_tests5.step);
test_step.dependOn(&run_tests5.step);
test_step.dependOn(&run_tests6.step);
}

514
src/ADD.zig
View File

@ -1,14 +1,13 @@
const std = @import("std");
const dtypes = @import("dtypes.zig");
const UUID = @import("uuid.zig").UUID;
const ziqlTokenizer = @import("ziqlTokenizer.zig").Tokenizer;
const ziqlToken = @import("ziqlTokenizer.zig").Token;
const Tokenizer = @import("ziqlTokenizer.zig").Tokenizer;
const Token = @import("ziqlTokenizer.zig").Token;
const DataEngine = @import("dataEngine.zig").DataEngine;
const Allocator = std.mem.Allocator;
const stdout = std.io.getStdOut().writer();
// TODO: Use a Parser struct like in GRAB
// Query that need to work now
// ADD User (name='Adrien', email='adrien.bouvais@gmail.com') OK
// ADD User (name='Adrien', email='adrien.bouvais@gmail.com', age = 26) OK
@ -24,330 +23,231 @@ const stdout = std.io.getStdOut().writer();
/// It will parse the reste of the query and create a map of member name / value.
/// Then add those value to the appropriete file. The proper file is the first one with a size < to the limit.
/// If no file is found, a new one is created.
pub fn parseDataAndAddToFile(allocator: Allocator, struct_name: []const u8, toker: *ziqlTokenizer) !void {
const token = toker.next();
switch (token.tag) {
.l_paren => {},
else => {
try stdout.print("Error: Expected ( after the struct name of an ADD command.\nE.g. ADD User (name = 'bob')\n", .{});
return;
},
}
const buffer = try allocator.alloc(u8, 1024 * 100);
defer allocator.free(buffer);
var member_map = getMapOfMember(allocator, toker) catch return;
defer member_map.deinit();
if (!checkIfAllMemberInMap(struct_name, &member_map)) return;
const entity = try dtypes.createEntityFromMap(allocator, struct_name, member_map);
const uuid_str = entity.User.*.id.format_uuid();
defer stdout.print("Added new {s} successfully using UUID: {s}\n", .{
struct_name,
uuid_str,
}) catch {};
const member_names = dtypes.structName2structMembers(struct_name);
for (member_names) |member_name| {
var file_map = getFilesStat(allocator, struct_name, member_name) catch {
try stdout.print("Error: File stat error", .{});
return;
};
const potential_file_name_to_use = getFirstUsableFile(file_map);
if (potential_file_name_to_use) |file_name| {
const file_index = fileName2Index(file_name) catch @panic("Error in fileName2Index");
try stdout.print("Using file: {s} with a size of {d}\n", .{ file_name, file_map.get(file_name).?.size });
const path = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/{s}", .{
struct_name,
member_name,
file_name,
});
var file = std.fs.cwd().openFile(path, .{
.mode = .read_write,
}) catch {
try stdout.print("Error opening data file.", .{});
return;
};
defer file.close();
try file.seekFromEnd(0);
try file.writer().print("{s} {s}\n", .{ uuid_str, member_map.get(member_name).? });
const path_to_main = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/main.zippondata", .{
struct_name,
member_name,
});
var file_main = std.fs.cwd().openFile(path_to_main, .{
.mode = .read_write,
}) catch {
try stdout.print("Error opening data file.", .{});
return;
};
defer file_main.close();
try appendToLineAtIndex(allocator, file_main, file_index, &uuid_str);
} else {
const max_index = maxFileIndex(file_map);
const new_file_path = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/{d}.zippondata", .{
struct_name,
member_name,
max_index + 1,
});
try stdout.print("new file path: {s}\n", .{new_file_path});
// TODO: Create new file and save the data inside
const new_file = std.fs.cwd().createFile(new_file_path, .{}) catch @panic("Error creating new data file");
defer new_file.close();
try new_file.writer().print("{s} {s}\n", .{ &uuid_str, member_map.get(member_name).? });
const path_to_main = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/main.zippondata", .{
struct_name,
member_name,
});
var file_main = std.fs.cwd().openFile(path_to_main, .{
.mode = .read_write,
}) catch {
try stdout.print("Error opening data file.", .{});
@panic("");
};
defer file_main.close();
try file_main.seekFromEnd(0);
try file_main.writeAll("\n ");
try file_main.seekTo(0);
try appendToLineAtIndex(allocator, file_main, max_index + 1, &uuid_str);
}
}
}
/// Take the main.zippondata file, the index of the file where the data is saved and the string to add at the end of the line
fn appendToLineAtIndex(allocator: std.mem.Allocator, file: std.fs.File, index: usize, str: []const u8) !void {
const buffer = try allocator.alloc(u8, 1024 * 100);
defer allocator.free(buffer);
pub const Parser = struct {
arena: std.heap.ArenaAllocator,
allocator: Allocator,
var reader = file.reader();
toker: *Tokenizer,
data_engine: *DataEngine,
var line_num: usize = 1;
while (try reader.readUntilDelimiterOrEof(buffer, '\n')) |_| {
if (line_num == index) {
try file.seekBy(-1);
try file.writer().print("{s} ", .{str});
return;
}
line_num += 1;
}
}
/// Return a map of file path => Stat for one struct and member name
fn getFilesStat(allocator: Allocator, struct_name: []const u8, member_name: []const u8) !*std.StringHashMap(std.fs.File.Stat) {
const cwd = std.fs.cwd();
const buffer = try allocator.alloc(u8, 1024); // Adjust the size as needed
defer allocator.free(buffer);
const path = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}", .{ struct_name, member_name });
var file_map = std.StringHashMap(std.fs.File.Stat).init(allocator);
const member_dir = cwd.openDir(path, .{ .iterate = true }) catch {
try stdout.print("Error opening struct directory", .{});
@panic("");
};
var iter = member_dir.iterate();
while (try iter.next()) |entry| {
if (entry.kind != std.fs.Dir.Entry.Kind.file) continue;
const file_stat = member_dir.statFile(entry.name) catch
{
try stdout.print("Error getting stat of a file", .{});
@panic("");
pub fn init(allocator: Allocator, toker: *Tokenizer, data_engine: *DataEngine) Parser {
var arena = std.heap.ArenaAllocator.init(allocator);
return Parser{
.arena = arena,
.allocator = arena.allocator(),
.toker = toker,
.data_engine = data_engine,
};
file_map.put(entry.name, file_stat) catch @panic("Error adding stat to map");
}
return &file_map;
}
/// Use the map of file stat to find the first file with under the bytes limit.
/// return the name of the file. If none is found, return null.
fn getFirstUsableFile(map: *std.StringHashMap(std.fs.File.Stat)) ?[]const u8 {
var iter = map.keyIterator();
while (iter.next()) |key| {
if (std.mem.eql(u8, key.*, "main.zippondata")) continue;
if (map.get(key.*).?.size < dtypes.parameter_max_file_size_in_bytes) return key.*;
pub fn deinit(self: *Parser) void {
self.arena.deinit();
}
return null;
}
fn fileName2Index(file_name: []const u8) !usize {
try stdout.print("Got file name: {s}\n", .{file_name});
var iter_file_name = std.mem.tokenize(u8, file_name, ".");
const num_str = iter_file_name.next().?;
const num: usize = try std.fmt.parseInt(usize, num_str, 10);
return num;
}
/// Iter over all file and get the max name and return the value of it as i32
/// So for example if there is 1.zippondata and 2.zippondata it return 2.
fn maxFileIndex(map: *std.StringHashMap(std.fs.File.Stat)) usize {
var iter = map.keyIterator();
var index_max: usize = 0;
while (iter.next()) |key| {
if (std.mem.eql(u8, key.*, "main.zippondata")) continue;
var iter_file_name = std.mem.tokenize(u8, key.*, ".");
const num_str = iter_file_name.next().?;
const num: usize = std.fmt.parseInt(usize, num_str, 10) catch @panic("Error parsing file name into usize");
if (num > index_max) index_max = num;
}
return index_max;
}
const MemberMapError = error{
NotMemberName,
NotEqualSign,
NotStringOrNumber,
NotComma,
PuttingNull,
};
/// Take the tokenizer and return a map of the query for the ADD command.
/// Keys are the member name and value are the string of the value in the query. E.g. 'Adrien' or '10'
pub fn getMapOfMember(allocator: Allocator, toker: *ziqlTokenizer) !std.StringHashMap([]const u8) {
std.debug.print("Started\n\n", .{});
var token = toker.next();
std.debug.print("{any}\n\n", .{token});
var member_map = std.StringHashMap([]const u8).init(
allocator,
);
std.debug.print("OK \n\n", .{});
while (token.tag != ziqlToken.Tag.eof) : ({
token = toker.next();
std.debug.print("{any}", .{token});
}) {
std.debug.print("{any}\n\n", .{token});
pub fn parse(self: *Parser, struct_name: []const u8) !void {
var token = self.toker.next();
switch (token.tag) {
.r_paren => continue,
.invalid => stdout.print("Error: Invalid token: {s}", .{toker.getTokenSlice(token)}) catch {},
.identifier => {
const member_name_str = toker.getTokenSlice(token);
token = toker.next();
switch (token.tag) {
.equal => {
token = toker.next();
switch (token.tag) {
.string_literal, .number_literal => {
const value_str = toker.getTokenSlice(token);
member_map.put(member_name_str, value_str) catch @panic("Could not add member name and value to map in getMapOfMember");
token = toker.next();
switch (token.tag) {
.comma, .r_paren => continue,
else => {
stdout.print("Error: Expected , after string or number got: {s}. E.g. ADD User (name='bob', age=10)", .{toker.getTokenSlice(token)}) catch {};
return MemberMapError.NotComma;
},
}
},
.keyword_null => {
try stdout.print("Found null value\n", .{});
const value_str = "null";
member_map.put(member_name_str, value_str) catch {
try stdout.print("Error putting null value into the map\n", .{});
return MemberMapError.PuttingNull;
};
token = toker.next();
switch (token.tag) {
.comma, .r_paren => continue,
else => {
stdout.print("Error: Expected , after string or number got: {s}. E.g. ADD User (name='bob', age=10)", .{toker.getTokenSlice(token)}) catch {};
return MemberMapError.NotComma;
},
}
},
.l_bracket => {
var array_values = std.ArrayList([]const u8).init(allocator);
token = toker.next();
while (token.tag != ziqlToken.Tag.r_bracket) : (token = toker.next()) {
switch (token.tag) {
.string_literal, .number_literal => {
const value_str = toker.getTokenSlice(token);
array_values.append(value_str) catch @panic("Could not add value to array in getMapOfMember");
},
.invalid => stdout.print("Error: Invalid token: {s}", .{toker.getTokenSlice(token)}) catch {},
else => {
stdout.print("Error: Expected string or number in array got: {s}. E.g. ADD User (scores=[10 20 30])", .{toker.getTokenSlice(token)}) catch {};
return MemberMapError.NotStringOrNumber;
},
}
}
const array_str = try std.mem.join(allocator, " ", array_values.items);
member_map.put(member_name_str, array_str) catch @panic("Could not add member name and value to map in getMapOfMember");
}, // TODO
else => {
stdout.print("Error: Expected string or number after a = got: {s}. E.g. ADD User (name='bob')", .{toker.getTokenSlice(token)}) catch {};
return MemberMapError.NotStringOrNumber;
},
}
},
else => {
stdout.print("Error: Expected = after a member declaration get {s}. E.g. ADD User (name='bob')", .{toker.getTokenSlice(token)}) catch {};
return MemberMapError.NotEqualSign;
},
}
},
.l_paren => {},
else => {
stdout.print("Error: Unknow token: {s}. This should be the name of a member. E.g. name in ADD User (name='bob')", .{toker.getTokenSlice(token)}) catch {};
return MemberMapError.NotMemberName;
try self.print_error("Error: Expected (", &token);
},
}
const buffer = try self.allocator.alloc(u8, 1024 * 100);
defer self.allocator.free(buffer);
var data = self.parseData(); // data is a map with key as member name and value as str of the value inserted in the query. So age = 12 is the string 12 here
defer data.deinit();
if (!self.checkIfAllMemberInMap(struct_name, &data)) return;
const entity = try dtypes.createEntityFromMap(self.allocator, struct_name, data);
const uuid_str = entity.User.*.id.format_uuid();
defer stdout.print("Added new {s} successfully using UUID: {s}\n", .{
struct_name,
uuid_str,
}) catch {};
const member_names = dtypes.structName2structMembers(struct_name);
for (member_names) |member_name| {
var file_map = self.data_engine.getFilesStat(struct_name, member_name) catch {
try stdout.print("Error: File stat error", .{});
return;
};
const potential_file_name_to_use = self.data_engine.getFirstUsableFile(file_map);
if (potential_file_name_to_use) |file_name| {
const file_index = self.data_engine.fileName2Index(file_name);
try stdout.print("Using file: {s} with a size of {d}\n", .{ file_name, file_map.get(file_name).?.size });
const path = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/{s}", .{
struct_name,
member_name,
file_name,
});
var file = std.fs.cwd().openFile(path, .{
.mode = .read_write,
}) catch {
try stdout.print("Error opening data file.", .{});
return;
};
defer file.close();
try file.seekFromEnd(0);
try file.writer().print("{s} {s}\n", .{ uuid_str, data.get(member_name).? });
const path_to_main = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/main.zippondata", .{
struct_name,
member_name,
});
var file_main = std.fs.cwd().openFile(path_to_main, .{
.mode = .read_write,
}) catch {
try stdout.print("Error opening data file.", .{});
return;
};
defer file_main.close();
try self.data_engine.appendToLineAtIndex(file_main, file_index, &uuid_str);
} else {
const max_index = self.data_engine.maxFileIndex(file_map);
const new_file_path = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/{d}.zippondata", .{
struct_name,
member_name,
max_index + 1,
});
try stdout.print("new file path: {s}\n", .{new_file_path});
// TODO: Create new file and save the data inside
const new_file = std.fs.cwd().createFile(new_file_path, .{}) catch @panic("Error creating new data file");
defer new_file.close();
try new_file.writer().print("{s} {s}\n", .{ &uuid_str, data.get(member_name).? });
const path_to_main = try std.fmt.bufPrint(buffer, "ZipponDB/DATA/{s}/{s}/main.zippondata", .{
struct_name,
member_name,
});
var file_main = std.fs.cwd().openFile(path_to_main, .{
.mode = .read_write,
}) catch {
try stdout.print("Error opening data file.", .{});
@panic("");
};
defer file_main.close();
try file_main.seekFromEnd(0);
try file_main.writeAll("\n ");
try file_main.seekTo(0);
try self.data_engine.appendToLineAtIndex(file_main, max_index + 1, &uuid_str);
}
}
}
return member_map;
}
/// Take the tokenizer and return a map of the query for the ADD command.
/// Keys are the member name and value are the string of the value in the query. E.g. 'Adrien' or '10'
pub fn parseData(self: *Parser) std.StringHashMap([]const u8) {
var token = self.toker.next();
/// Using the name of a struct from dtypes and the map of member name => value string from the query.
/// Check if the map keys are exactly the same as the name of the member of the struct.
/// Basically checking if the query contain all value that a struct need to be init.
fn checkIfAllMemberInMap(struct_name: []const u8, map: *std.StringHashMap([]const u8)) bool {
const all_struct_member = dtypes.structName2structMembers(struct_name);
var count: u16 = 0;
var member_map = std.StringHashMap([]const u8).init(
self.allocator,
);
for (all_struct_member) |key| {
if (map.contains(key)) count += 1 else stdout.print("Error: ADD query of struct: {s}; missing member: {s}\n", .{
struct_name,
key,
}) catch {};
while (token.tag != Token.Tag.eof) : (token = self.toker.next()) {
switch (token.tag) {
.r_paren => continue,
.identifier => {
const member_name_str = self.toker.getTokenSlice(token);
token = self.toker.next();
switch (token.tag) {
.equal => {
token = self.toker.next();
switch (token.tag) {
.string_literal, .number_literal => {
const value_str = self.toker.getTokenSlice(token);
member_map.put(member_name_str, value_str) catch @panic("Could not add member name and value to map in getMapOfMember");
token = self.toker.next();
switch (token.tag) {
.comma, .r_paren => continue,
else => self.print_error("Error: Expected , after string or number. E.g. ADD User (name='bob', age=10)", &token) catch {},
}
},
.keyword_null => {
const value_str = "null";
member_map.put(member_name_str, value_str) catch self.print_error("Error: 001", &token) catch {};
token = self.toker.next();
switch (token.tag) {
.comma, .r_paren => continue,
else => self.print_error("Error: Expected , after string or number. E.g. ADD User (name='bob', age=10)", &token) catch {},
}
},
.l_bracket => {
var array_values = std.ArrayList([]const u8).init(self.allocator);
token = self.toker.next();
while (token.tag != Token.Tag.r_bracket) : (token = self.toker.next()) {
switch (token.tag) {
.string_literal, .number_literal => {
const value_str = self.toker.getTokenSlice(token);
array_values.append(value_str) catch @panic("Could not add value to array in getMapOfMember");
},
else => self.print_error("Error: Expected string or number in array. E.g. ADD User (scores=[10 20 30])", &token) catch {},
}
}
// Maybe change that as it just recreate a string that is already in the buffer
const array_str = std.mem.join(self.allocator, " ", array_values.items) catch @panic("Couln't join the value of array");
member_map.put(member_name_str, array_str) catch @panic("Could not add member name and value to map in getMapOfMember");
},
else => self.print_error("Error: Expected string or number after =. E.g. ADD User (name='bob')", &token) catch {},
}
},
else => self.print_error("Error: Expected = after a member declaration. E.g. ADD User (name='bob')", &token) catch {},
}
},
else => self.print_error("Error: Unknow token. This should be the name of a member. E.g. name in ADD User (name='bob')", &token) catch {},
}
}
return member_map;
}
return ((count == all_struct_member.len) and (count == map.count()));
}
fn checkIfAllMemberInMap(_: *Parser, struct_name: []const u8, map: *std.StringHashMap([]const u8)) bool {
const all_struct_member = dtypes.structName2structMembers(struct_name);
var count: u16 = 0;
test "Get map of members" {
// std.testing.refAllDecls(@This());
// _ = @import("query_functions/ADD.zig").getMapOfMember;
for (all_struct_member) |key| {
if (map.contains(key)) count += 1 else stdout.print("Error: ADD query of struct: {s}; missing member: {s}\n", .{
struct_name,
key,
}) catch {};
}
const allocator = std.testing.allocator;
return ((count == all_struct_member.len) and (count == map.count()));
}
const in = "(name='Adrien', email='adrien@gmail.com', age=26, scores=[42 100 5])";
const null_term_in = try allocator.dupeZ(u8, in);
fn print_error(self: *Parser, message: []const u8, token: *Token) !void {
try stdout.print("\n", .{});
try stdout.print("{s}\n", .{self.toker.buffer});
var toker = ziqlTokenizer.init(null_term_in);
// Calculate the number of spaces needed to reach the start position.
var spaces: usize = 0;
while (spaces < token.loc.start) : (spaces += 1) {
try stdout.print(" ", .{});
}
const member_map = try getMapOfMember(allocator, &toker);
std.debug.print("{s}", .{member_map.get("name").?});
// Print the '^' characters for the error span.
var i: usize = token.loc.start;
while (i < token.loc.end) : (i += 1) {
try stdout.print("^", .{});
}
try stdout.print(" \n", .{}); // Align with the message
allocator.free(null_term_in);
}
try stdout.print("{s}\n", .{message});
@panic("");
}
};

128
src/GRAB.zig
View File

@ -2,6 +2,8 @@ const std = @import("std");
const Allocator = std.mem.Allocator;
const Tokenizer = @import("ziqlTokenizer.zig").Tokenizer;
const Token = @import("ziqlTokenizer.zig").Token;
const DataEngine = @import("dataEngine.zig").DataEngine;
const UUID = @import("uuid.zig").UUID;
// To work now
// GRAB User {}
@ -15,12 +17,31 @@ const Token = @import("ziqlTokenizer.zig").Token;
const stdout = std.io.getStdOut().writer();
pub const Parser = struct {
arena: std.heap.ArenaAllocator,
allocator: Allocator,
toker: *Tokenizer,
data_engine: *DataEngine,
state: State,
additional_data: AdditionalData,
pub fn init(allocator: Allocator, toker: *Tokenizer, data_engine: *DataEngine) Parser {
var arena = std.heap.ArenaAllocator.init(allocator);
return Parser{
.arena = arena,
.allocator = arena.allocator(),
.toker = toker,
.data_engine = data_engine,
.state = State.start,
.additional_data = AdditionalData.init(allocator),
};
}
pub fn deinit(self: *Parser) void {
self.additional_data.deinit();
self.arena.deinit();
}
// This is the [] part
pub const AdditionalData = struct {
entity_count_to_find: usize = 0,
@ -31,7 +52,10 @@ pub const Parser = struct {
}
pub fn deinit(self: *AdditionalData) void {
// Get all additional data that are in the list to also deinit them
for (self.member_to_find.items) |elem| {
elem.additional_data.deinit();
}
self.member_to_find.deinit();
}
};
@ -63,20 +87,6 @@ pub const Parser = struct {
expect_filter,
};
pub fn init(allocator: Allocator, toker: *Tokenizer) Parser {
return Parser{
.allocator = allocator,
.toker = toker,
.state = State.start,
.additional_data = AdditionalData.init(allocator),
};
}
pub fn deinit(self: *Parser) void {
// FIXME: I think additionalData inside additionalData are not deinit
self.additional_data.deinit();
}
pub fn parse(self: *Parser) !void {
var token = self.toker.next();
while (self.state != State.end) : (token = self.toker.next()) {
@ -243,6 +253,37 @@ pub const Parser = struct {
}
};
// TODO: Optimize. Maybe just do a new list and return it instead
fn OR(arr1: *std.ArrayList(UUID), arr2: *std.ArrayList(UUID)) std.ArrayList(UUID) {
defer arr1.deinit();
defer arr2.deinit();
var arr = try arr1.clone();
for (0..arr2.items.len) |i| {
if (!arr.contains(arr2[i])) {
arr.append(arr2[i]);
}
}
return arr;
}
fn AND(arr1: *std.ArrayList(UUID), arr2: *std.ArrayList(UUID)) std.ArrayList(UUID) {
defer arr1.deinit();
defer arr2.deinit();
var arr = try arr1.clone();
for (0..arr1.items.len) |i| {
if (arr2.contains(arr1[i])) {
arr.append(arr1[i]);
}
}
return arr;
}
test "Test AdditionalData" {
const allocator = std.testing.allocator;
@ -251,7 +292,7 @@ test "Test AdditionalData" {
testAdditionalData("[1]", additional_data1);
var additional_data2 = Parser.AdditionalData.init(allocator);
defer additional_data2.deinit();
defer additional_data2.member_to_find.deinit();
try additional_data2.member_to_find.append(
Parser.AdditionalDataMember.init(
allocator,
@ -260,25 +301,64 @@ test "Test AdditionalData" {
);
testAdditionalData("[name]", additional_data2);
std.debug.print("AdditionalData Parsing OK \n", .{});
var additional_data3 = Parser.AdditionalData.init(allocator);
additional_data3.entity_count_to_find = 1;
defer additional_data3.member_to_find.deinit();
try additional_data3.member_to_find.append(
Parser.AdditionalDataMember.init(
allocator,
"name",
),
);
testAdditionalData("[1; name]", additional_data3);
var additional_data4 = Parser.AdditionalData.init(allocator);
additional_data4.entity_count_to_find = 100;
defer additional_data4.member_to_find.deinit();
try additional_data4.member_to_find.append(
Parser.AdditionalDataMember.init(
allocator,
"friend",
),
);
testAdditionalData("[100; friend [name]]", additional_data4);
}
fn testAdditionalData(source: [:0]const u8, expected_AdditionalData: Parser.AdditionalData) void {
const allocator = std.testing.allocator;
var tokenizer = Tokenizer.init(source);
var additional_data = Parser.AdditionalData.init(allocator);
var data_engine = DataEngine.init(allocator);
defer data_engine.deinit();
var parser = Parser.init(allocator, &tokenizer, &data_engine);
defer parser.deinit();
_ = tokenizer.next();
var parser = Parser.init(allocator, &tokenizer);
parser.parse_additional_data(&additional_data) catch |err| {
parser.parse_additional_data(&parser.additional_data) catch |err| {
std.debug.print("Error parsing additional data: {any}\n", .{err});
};
std.debug.print("{any}\n\n", .{additional_data});
compareAdditionalData(expected_AdditionalData, parser.additional_data);
}
std.testing.expectEqual(expected_AdditionalData, additional_data) catch {
std.debug.print("Additional data are not equal for: {s}\n", .{source});
// TODO: Check AdditionalData inside AdditionalData
fn compareAdditionalData(ad1: Parser.AdditionalData, ad2: Parser.AdditionalData) void {
std.testing.expectEqual(ad1.entity_count_to_find, ad2.entity_count_to_find) catch {
std.debug.print("Additional data entity_count_to_find are not equal.\n", .{});
};
parser.deinit();
var founded = false;
for (ad1.member_to_find.items) |elem1| {
founded = false;
for (ad2.member_to_find.items) |elem2| {
if (std.mem.eql(u8, elem1.name, elem2.name)) {
compareAdditionalData(elem1.additional_data, elem2.additional_data);
founded = true;
break;
}
}
if (!founded) @panic("Member not found");
}
}

1
src/cliTokenizer.zig
View File

@ -159,7 +159,6 @@ pub const Tokenizer = struct {
test "Basics" {
try testTokenize("help", &.{.keyword_help});
try testTokenize("run \"Hello world\"", &.{ .keyword_run, .string_literal });
std.debug.print("CLI tokenizer OK\n", .{});
}
fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {

3
src/data-parsing.zig
View File

@ -3,6 +3,7 @@ const std = @import("std");
// Series of functions to use just before creating an entity.
// Will transform the string of data into data of the right type.
// Maybe return a null or something else
pub fn parseInt(value_str: []const u8) i64 {
return std.fmt.parseInt(i64, value_str, 10) catch return 0;
}
@ -27,7 +28,6 @@ test "Data parsing" {
for (in1, 0..) |value, i| {
try std.testing.expect(parseInt(value) == expected_out1[i]);
}
std.debug.print("OK\tData parsing: Int\n", .{});
// Int array
const in2 = "[1 14 44 42 hello]";
@ -35,5 +35,4 @@ test "Data parsing" {
defer out2.deinit();
const expected_out2: [5]i64 = .{ 1, 14, 44, 42, 0 };
try std.testing.expect(std.mem.eql(i64, out2.items, &expected_out2));
std.debug.print("OK\tData parsing: Int array\n", .{});
}

103
src/dataEngine.zig Normal file
View File

@ -0,0 +1,103 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
const Tokenizer = @import("ziqlTokenizer.zig").Tokenizer;
/// Manage everything that is relate to read or write in files
/// Or even get stats, whatever. If it touch files, it's here
pub const DataEngine = struct {
arena: std.heap.ArenaAllocator,
allocator: Allocator,
dir: std.fs.Dir, // The path to the DATA folder
max_file_size: usize = 1e+8, // 100mb
pub fn init(allocator: Allocator) DataEngine {
var arena = std.heap.ArenaAllocator.init(allocator);
const dir = std.fs.cwd().openDir("ZipponDB/DATA", .{}) catch @panic("Error opening ZipponDB/DATA");
return DataEngine{
.arena = arena,
.allocator = arena.allocator(),
.dir = dir,
};
}
pub fn deinit(self: *DataEngine) void {
self.arena.deinit();
}
/// Iter over all file and get the max name and return the value of it as usize
/// So for example if there is 1.zippondata and 2.zippondata it return 2.
fn maxFileIndex(_: *DataEngine, map: *std.StringHashMap(std.fs.File.Stat)) usize {
var iter = map.keyIterator();
var index_max: usize = 0;
while (iter.next()) |key| {
if (std.mem.eql(u8, key.*, "main.zippondata")) continue;
var iter_file_name = std.mem.tokenize(u8, key.*, ".");
const num_str = iter_file_name.next().?;
const num: usize = std.fmt.parseInt(usize, num_str, 10) catch @panic("Error parsing file name into usize");
if (num > index_max) index_max = num;
}
return index_max;
}
/// Use a filename in the format 1.zippondata and return the 1
fn fileName2Index(_: *DataEngine, file_name: []const u8) usize {
var iter_file_name = std.mem.tokenize(u8, file_name, ".");
const num_str = iter_file_name.next().?;
const num: usize = std.fmt.parseInt(usize, num_str, 10) catch @panic("Couln't parse the int of a zippondata file.");
return num;
}
/// Add an UUID at a specific index of a file
/// Used when some data are deleted from previous zippondata files and are now bellow the file size limit
fn appendToLineAtIndex(self: *DataEngine, file: std.fs.File, index: usize, str: []const u8) !void {
const buffer = try self.allocator.alloc(u8, 1024 * 100);
defer self.allocator.free(buffer);
var reader = file.reader();
var line_num: usize = 1;
while (try reader.readUntilDelimiterOrEof(buffer, '\n')) |_| {
if (line_num == index) {
try file.seekBy(-1);
try file.writer().print("{s} ", .{str});
return;
}
line_num += 1;
}
}
/// Return a map of file path => Stat; for one struct and member name
/// E.g. for User & name
fn getFilesStat(self: *DataEngine, struct_name: []const u8, member_name: []const u8) !*std.StringHashMap(std.fs.File.Stat) {
const buffer = try self.allocator.alloc(u8, 1024); // Adjust the size as needed
defer self.allocator.free(buffer);
const path = try std.fmt.bufPrint(buffer, "{s}{s}/{s}", .{ self.path.basename(), struct_name, member_name });
var file_map = std.StringHashMap(std.fs.File.Stat).init(self.allocator);
const member_dir = self.path.openDir(path, .{ .iterate = true }) catch @panic("Error opening struct directory");
var iter = member_dir.iterate();
while (try iter.next()) |entry| {
if (entry.kind != std.fs.Dir.Entry.Kind.file) continue;
const file_stat = member_dir.statFile(entry.name) catch @panic("Error getting stat of a file");
file_map.put(entry.name, file_stat) catch @panic("Error adding stat to map");
}
return &file_map;
}
/// Use the map of file stat to find the first file with under the bytes limit.
/// return the name of the file. If none is found, return null.
fn getFirstUsableFile(self: *DataEngine, map: *std.StringHashMap(std.fs.File.Stat)) ?[]const u8 {
var iter = map.keyIterator();
while (iter.next()) |key| {
if (std.mem.eql(u8, key.*, "main.zippondata")) continue;
if (map.get(key.*).?.size < self.max_file_size) return key.*;
}
return null;
}
};

44
src/dbengine.zig
View File

@ -1,11 +1,12 @@
const std = @import("std");
const dtypes = @import("dtypes.zig");
const UUID = @import("uuid.zig").UUID;
const ziqlTokenizer = @import("ziqlTokenizer.zig").Tokenizer;
const ziqlToken = @import("ziqlTokenizer.zig").Token;
const Tokenizer = @import("ziqlTokenizer.zig").Tokenizer;
const Token = @import("ziqlTokenizer.zig").Token;
const grabParser = @import("GRAB.zig").Parser;
const addParser = @import("ADD.zig").Parser;
const DataEngine = @import("dataEngine.zig").DataEngine;
const Allocator = std.mem.Allocator;
const parseDataAndAddToFile = @import("ADD.zig").parseDataAndAddToFile;
pub const Error = error{UUIDNotFound};
const stdout = std.io.getStdOut().writer();
@ -17,22 +18,6 @@ pub fn main() !void {
const buffer = try allocator.alloc(u8, 1024);
defer allocator.free(buffer);
// Init the map storage string map that track all array of struct
var storage = std.StringHashMap(*std.ArrayList(dtypes.Types)).init(allocator);
defer storage.deinit();
// Create all array and put them in the main map
// Use MultiArrayList in the future to save memory maybe ?
for (dtypes.struct_name_list) |struct_name| {
var array = std.ArrayList(dtypes.Types).init(allocator);
try storage.put(struct_name, &array);
}
// Add user
//const adrien = dtypes.User.init("Adrien", "adrien@gmail.com");
//try storage.get("User").?.append(dtypes.Types{ .User = &adrien });
//const adrien_get = storage.get("User").?.items[0].User;
var args = try std.process.argsWithAllocator(allocator);
defer args.deinit();
@ -40,21 +25,28 @@ pub fn main() !void {
_ = args.next();
const null_term_query_str = args.next().?;
var ziqlToker = ziqlTokenizer.init(null_term_query_str);
const first_token = ziqlToker.next();
const struct_name_token = ziqlToker.next();
var toker = Tokenizer.init(null_term_query_str);
const first_token = toker.next();
const struct_name_token = toker.next();
var data_engine = DataEngine.init(allocator);
switch (first_token.tag) {
.keyword_grab => {
var parser = grabParser.init(allocator, &ziqlToker);
if (!isStructInSchema(toker.getTokenSlice(struct_name_token))) {
try stdout.print("Error: No struct named '{s}' in current schema.", .{toker.getTokenSlice(struct_name_token)});
return;
}
var parser = grabParser.init(allocator, &toker, &data_engine);
try parser.parse();
},
.keyword_add => {
if (!isStructInSchema(ziqlToker.getTokenSlice(struct_name_token))) {
try stdout.print("Error: No struct named '{s}' in current schema.", .{ziqlToker.getTokenSlice(struct_name_token)});
if (!isStructInSchema(toker.getTokenSlice(struct_name_token))) {
try stdout.print("Error: No struct named '{s}' in current schema.", .{toker.getTokenSlice(struct_name_token)});
return;
}
try parseDataAndAddToFile(allocator, ziqlToker.getTokenSlice(struct_name_token), &ziqlToker);
var parser = addParser.init(allocator, &toker, &data_engine);
try parser.parse(toker.getTokenSlice(struct_name_token));
},
.keyword_update => {
try stdout.print("Not yet implemented.\n", .{});

2
src/dtypes.zig
View File

@ -2,8 +2,6 @@ const std = @import("std");
const UUID = @import("uuid.zig").UUID;
const dataParsing = @import("data-parsing.zig");
pub const parameter_max_file_size_in_bytes = 500; // THe number of bytes than each file can be before splitting
pub const User = struct {
id: UUID,
name: []const u8,

4
src/schemaParser.zig
View File

@ -1,7 +1,7 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
const Toker = @import("tokenizers/schemaTokenizer.zig").Tokenizer;
const Token = @import("tokenizers/schemaTokenizer.zig").Token;
const Toker = @import("schemaTokenizer.zig").Tokenizer;
const Token = @import("schemaTokenizer.zig").Token;
pub const Parser = struct {
file: std.fs.File,

2
src/schemaTokenizer.zig
View File

@ -157,7 +157,6 @@ pub const Tokenizer = struct {
test "keywords" {
try testTokenize("int float str date", &.{ .type_int, .type_float, .type_str, .type_date });
std.debug.print("Schema type OK\n", .{});
}
test "basic query" {
@ -182,7 +181,6 @@ test "basic query" {
.identifier,
.r_paren,
});
std.debug.print("Schema OK\n", .{});
}
fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {

3
src/uuid.zig
View File

@ -165,9 +165,6 @@ test "check to_string works" {
uuid1.to_string(&string1);
uuid1.to_string(&string2);
std.debug.print("\nUUID {s} \n", .{uuid1});
std.debug.print("\nFirst call to_string {s} \n", .{string1});
std.debug.print("Second call to_string {s} \n", .{string2});
try testing.expectEqual(string1, string2);
}

2
src/ziqlTokenizer.zig
View File

@ -345,7 +345,6 @@ pub const Tokenizer = struct {
test "keywords" {
try testTokenize("GRAB UPDATE ADD DELETE IN", &.{ .keyword_grab, .keyword_update, .keyword_add, .keyword_delete, .keyword_in });
std.debug.print("ZiQL keywords OK\n", .{});
}
test "basic query" {
@ -354,7 +353,6 @@ test "basic query" {
try testTokenize("GRAB User [1; name] {}", &.{ .keyword_grab, .identifier, .l_bracket, .number_literal, .semicolon, .identifier, .r_bracket, .l_brace, .r_brace });
try testTokenize("GRAB User{}|ASCENDING name|", &.{ .keyword_grab, .identifier, .l_brace, .r_brace, .pipe, .identifier, .identifier, .pipe });
try testTokenize("DELETE User[1]{name='Adrien'}|ASCENDING name, age|", &.{ .keyword_delete, .identifier, .l_bracket, .number_literal, .r_bracket, .l_brace, .identifier, .equal, .string_literal, .r_brace, .pipe, .identifier, .identifier, .comma, .identifier, .pipe });
std.debug.print("ZiQL query OK\n", .{});
}
fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {