APPEND work, need to check next array manipulation

This commit is contained in:
Adrien Bouvais 2025-01-27 20:38:03 +01:00
parent baf6cd2284
commit 15ec304fe1
8 changed files with 210 additions and 85 deletions

View File

@ -4,7 +4,10 @@ const dtype = @import("dtype");
const ConditionValue = @import("../dataStructure/filter.zig").ConditionValue;
const ArrayCondition = @import("../ziql/parts//newData.zig").ArrayCondition;
pub fn updateData(allocator: std.mem.Allocator, condition: ArrayCondition, input: *zid.Data, data: []ConditionValue) !void {
// This shouldn't be here, to move somewhere, idk yet
/// Update an array based on keyword like append or remove
pub fn updateData(allocator: std.mem.Allocator, condition: ArrayCondition, input: *zid.Data, data: ConditionValue) !void {
switch (condition) {
.append => try append(allocator, input, data),
.pop => pop(input),
@ -50,100 +53,93 @@ fn clear(input: *zid.Data) void {
}
}
fn allocForAppend(allocator: std.mem.Allocator, input: *zid.Data, data: []ConditionValue) []zid.Data {
switch (input.*) {
.UUIDArray => {
var total: usize = 0;
for (data) |d| total += d.link_array.count();
return try allocator.alloc(zid.Data, total);
},
else => return try allocator.alloc(zid.Data, data.len),
}
}
// I think I could use meta programming here by adding the type as argument
fn append(allocator: std.mem.Allocator, input: *zid.Data, data: []ConditionValue) !void {
// TODO: Update the remaining type like int
fn append(allocator: std.mem.Allocator, input: *zid.Data, data: ConditionValue) !void {
switch (input.*) {
.IntArray => {
// 1. Make a list of the right type from ConditionValue
var array = std.ArrayList(i32).init(allocator);
defer array.deinit();
for (data) |d| try array.append(d.int);
// 2. Encode the new array
const new_array = try zid.allocEncodArray.Int(allocator, array.items);
// 3. Add the new array at the end of the old one without the first 4 bytes that are the number of value in the array
var updated_array = std.ArrayList(u8).init(allocator);
try updated_array.appendSlice(input.IntArray);
try updated_array.appendSlice(new_array[4..]);
// 4. Update the number of value in the array
const new_len = input.size() + data.len;
@memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
switch (data) {
.int => |v| {
try updated_array.appendSlice(std.mem.asBytes(&v));
const new_len = input.size() - 8 + @sizeOf(i32);
@memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
},
.int_array => |v| {
const new_array = try zid.allocEncodArray.Int(allocator, v);
try updated_array.appendSlice(new_array[8..]);
const new_len = input.size() + new_array.len - 16;
@memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
},
else => unreachable,
}
// 5. Update the input
input.*.IntArray = try updated_array.toOwnedSlice();
},
.FloatArray => {
var array = std.ArrayList(f64).init(allocator);
defer array.deinit();
for (data) |d| try array.append(d.float);
try array.appendSlice(data.float_array);
const new_array = try zid.allocEncodArray.Float(allocator, array.items);
var updated_array = std.ArrayList(u8).init(allocator);
try updated_array.appendSlice(input.FloatArray);
try updated_array.appendSlice(new_array[4..]);
const new_len = input.size() + data.len;
try updated_array.appendSlice(new_array[8..]);
const new_len = input.size() + new_array.len - 16;
@memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
input.*.FloatArray = try updated_array.toOwnedSlice();
},
.UnixArray => {
var array = std.ArrayList(u64).init(allocator);
defer array.deinit();
for (data) |d| try array.append(d.unix);
try array.appendSlice(data.unix_array);
const new_array = try zid.allocEncodArray.Unix(allocator, array.items);
var updated_array = std.ArrayList(u8).init(allocator);
try updated_array.appendSlice(input.UnixArray);
try updated_array.appendSlice(new_array[4..]);
const new_len = input.size() + data.len;
try updated_array.appendSlice(new_array[8..]);
const new_len = input.size() + new_array.len - 16;
@memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
input.*.UnixArray = try updated_array.toOwnedSlice();
},
.BoolArray => {
var array = std.ArrayList(bool).init(allocator);
defer array.deinit();
for (data) |d| try array.append(d.bool_);
try array.appendSlice(data.bool_array);
const new_array = try zid.allocEncodArray.Bool(allocator, array.items);
var updated_array = std.ArrayList(u8).init(allocator);
try updated_array.appendSlice(input.BoolArray);
try updated_array.appendSlice(new_array[4..]);
const new_len = input.size() + data.len;
try updated_array.appendSlice(new_array[8..]);
const new_len = input.size() + new_array.len - 16;
@memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
input.*.BoolArray = try updated_array.toOwnedSlice();
},
.StrArray => {
var array = std.ArrayList([]const u8).init(allocator);
defer array.deinit();
for (data) |d| try array.append(d.str);
try array.appendSlice(data.str_array);
const new_array = try zid.allocEncodArray.Str(allocator, array.items);
var updated_array = std.ArrayList(u8).init(allocator);
try updated_array.appendSlice(input.StrArray);
try updated_array.appendSlice(new_array[4..]);
const new_len = input.size() + data.len;
try updated_array.appendSlice(new_array[8..]);
const new_len = input.size() + new_array.len - 16;
@memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
input.*.StrArray = try updated_array.toOwnedSlice();
},
.UUIDArray => { // If input is a UUID array, that mean all data are also UUIDArray. There should be only one UUIDArray in data as it is use like that "friends APPEND {name = 'Bob'}"
.UUIDArray => { // If input is a UUID array, that mean all data are also UUIDArray
var array = std.ArrayList([16]u8).init(allocator);
defer array.deinit();
for (data) |d| {
var iter = d.link_array.keyIterator();
while (iter.next()) |uuid| try array.append(uuid.bytes);
}
var iter = data.link_array.keyIterator();
while (iter.next()) |uuid| try array.append(uuid.bytes);
const new_array = try zid.allocEncodArray.UUID(allocator, array.items);
var updated_array = std.ArrayList(u8).init(allocator);
try updated_array.appendSlice(input.UUIDArray);
try updated_array.appendSlice(new_array[4..]);
const new_len = input.size() + array.items.len;
try updated_array.appendSlice(new_array[8..]);
const new_len = input.size() + new_array.len - 16;
@memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
input.*.UUIDArray = try updated_array.toOwnedSlice();
},
@ -156,7 +152,7 @@ fn append(allocator: std.mem.Allocator, input: *zid.Data, data: []ConditionValue
// So I could just memcopy the remaining of the bytes at the current position, so it overwrite the value to remove
// Like if I want to re;ove 3 in [1 2 3 4 5], it would become [1 2 4 5 5]. Then I dont take the last value when I return.
// But that mean I keep in memory useless data, so maybe not
fn remove(allocator: std.mem.Allocator, input: *zid.Data, data: []ConditionValue) !void {
fn remove(allocator: std.mem.Allocator, input: *zid.Data, data: ConditionValue) !void {
var iter = try zid.ArrayIterator.init(input.*);
switch (input.*) {
.IntArray => {
@ -199,7 +195,7 @@ fn remove(allocator: std.mem.Allocator, input: *zid.Data, data: []ConditionValue
}
}
fn removeat(allocator: std.mem.Allocator, input: *zid.Data, data: []ConditionValue) !void {
fn removeat(allocator: std.mem.Allocator, input: *zid.Data, data: ConditionValue) !void {
var iter = try zid.ArrayIterator.init(input.*);
switch (input.*) {
.IntArray => {
@ -257,15 +253,16 @@ fn removeat(allocator: std.mem.Allocator, input: *zid.Data, data: []ConditionVal
}
}
// Should just use a map.contain
fn in(x: zid.Data, y: []ConditionValue) bool {
// TODO: Use a map.contain for the ConditionValue
// Specially because I end up iterate over the list for all entity, when I just need to make the map one time for all
fn in(x: zid.Data, y: ConditionValue) bool {
switch (x) {
.Int => |v| for (y) |z| if (v == z.int) return true,
.Float => |v| for (y) |z| if (v == z.float) return true,
.Unix => |v| for (y) |z| if (v == z.unix) return true,
.Bool => |v| for (y) |z| if (v == z.bool_) return true,
.Str => |v| for (y) |z| if (std.mem.eql(u8, z.str, v)) return true,
.UUID => |v| for (y) |z| if (z.link_array.contains(dtype.UUID{ .bytes = v })) return true,
.Int => |v| for (y.int_array) |z| if (v == z) return true,
.Float => |v| for (y.float_array) |z| if (v == z) return true,
.Unix => |v| for (y.unix_array) |z| if (v == z) return true,
.Bool => |v| for (y.bool_array) |z| if (v == z) return true,
.Str => |v| for (y.str_array) |z| if (std.mem.eql(u8, z, v)) return true,
.UUID => |v| if (y.link_array.contains(dtype.UUID{ .bytes = v })) return true,
else => unreachable,
}
return false;

View File

@ -189,12 +189,11 @@ fn updateEntitiesOneFile(
zid.deleteFile(new_path, dir) catch {};
return;
};
new_data_buff[i] = row[i];
},
else => {},
};
log.debug("{d} {any}\n\n", .{ new_data_buff.len, new_data_buff });
new_writer.write(new_data_buff) catch {
zid.deleteFile(new_path, dir) catch {};
return;

View File

@ -30,7 +30,7 @@ test "Clear" {
// Basic
// ===============================================================
test "ADD" {
test "ADD" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "ADD User (name = 'Bob', email='bob@email.com', age=55, scores=[ 1 ], best_friend=none, friends=none, bday=2000/01/01, a_time=12:04, last_order=2000/01/01-12:45)");
try testParsing(db, "ADD User (name = 'Bob', email='bob@email.com', age=55, scores=[ 666, 123, 331 ], best_friend=none, friends=none, bday=2000/11/01, a_time=12:04:54, last_order=2000/01/01-12:45)");
@ -44,7 +44,7 @@ test "ADD" {
try testParsing(db, "GRAB User {}");
}
test "ADD batch" {
test "ADD batch" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "ADD User (name = 'ewq', email='ewq@email.com', age=22, scores=[ ], best_friend=none, friends=none, bday=2000/01/01, a_time=12:04, last_order=2000/01/01-12:45) (name = 'Roger', email='roger@email.com', age=10, scores=[ 1, 11, 111, 123, 562345, 123451234, 34623465234, 12341234 ], best_friend=none, friends=none, bday=2000/01/01, a_time=12:04, last_order=2000/01/01-12:45)");
try testParsing(db, "ADD User (name = 'qwe', email='qwe@email.com', age=57, scores=[ ], best_friend=none, friends=none, bday=2000/01/01, a_time=12:04, last_order=2000/01/01-12:45) ('Rodrigo', 'bob@email.com', 55, [ 1 ], {name = 'qwe'}, none, 2000/01/01, 12:04, 2000/01/01-12:45)");
@ -53,26 +53,26 @@ test "ADD batch" {
try testParsing(db, "GRAB User {}");
}
test "GRAB filter with string" {
test "GRAB filter with string" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "GRAB User {name = 'Bob'}");
try testParsing(db, "GRAB User {name != 'Brittany Rogers'}");
}
test "GRAB with additional data" {
test "GRAB with additional data" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "GRAB User [1] {age < 18}");
try testParsing(db, "GRAB User [id, name] {age < 18}");
try testParsing(db, "GRAB User [100; name, age] {age < 18}");
}
test "UPDATE" {
test "UPDATE" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "UPDATE User [1] {name = 'Bob'} TO (email='new@gmail.com')");
try testParsing(db, "GRAB User {}");
}
test "GRAB filter with int" {
test "GRAB filter with int" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "GRAB User {age = 18}");
try testParsing(db, "GRAB User {age > -18}");
@ -82,14 +82,14 @@ test "GRAB filter with int" {
try testParsing(db, "GRAB User {age != 18}");
}
test "GRAB filter with date" {
test "GRAB filter with date" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "GRAB User {bday > 2000/01/01}");
try testParsing(db, "GRAB User {a_time < 08:00}");
try testParsing(db, "GRAB User {last_order > 2000/01/01-12:45}");
}
test "Specific query" {
test "Specific query" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "GRAB User");
try testParsing(db, "GRAB User {}");
@ -99,46 +99,56 @@ test "Specific query" {
// Array manipulation
// ===============================================================
test "GRAB name IN" {
test "GRAB name IN" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "GRAB User {name IN ['Bob', 'Bobinou']}");
try testParsing(db, "GRAB User {name IN ['Bob', 'Bobibou']}");
}
test "UPDATE APPEND" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "UPDATE User {name IN ['Bob', 'Bobibou']} TO (scores APPEND [69])");
try testParsing(db, "GRAB User {name IN ['Bob', 'Bobibou']}");
try testParsing(db, "UPDATE User {name IN ['Bob']} TO (scores APPEND [69, 123, 123, 11, 22, 44, 51235])");
try testParsing(db, "GRAB User {name IN ['Bob', 'Bobibou']}");
try testParsing(db, "UPDATE User {name IN ['Bob', 'Bobibou']} TO (scores APPEND 1)");
try testParsing(db, "GRAB User {name IN ['Bob', 'Bobibou']}");
}
// Single Struct Relationship
// ===============================================================
test "UPDATE relationship" {
test "UPDATE relationship" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "UPDATE User [1] {name='Bob'} TO (best_friend = {name='Boba'} )");
try testParsing(db, "GRAB User {}");
}
test "GRAB Relationship Filter" {
test "GRAB Relationship Filter" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "GRAB User {best_friend IN {name = 'Bob'}}");
try testParsing(db, "GRAB User {best_friend IN {name = 'Boba'}}");
}
test "GRAB Relationship AdditionalData" {
test "GRAB Relationship AdditionalData" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "GRAB User [name, friends] {}");
try testParsing(db, "GRAB User [name, best_friend] {}");
}
test "GRAB Relationship Sub AdditionalData" {
test "GRAB Relationship Sub AdditionalData" { // OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "GRAB User [name, friends [name]] {}");
try testParsing(db, "GRAB User [name, best_friend [name, friends [age]]] {}");
}
test "GRAB Relationship AdditionalData Filtered" {
test "GRAB Relationship AdditionalData Filtered" { // FIXME: NOT OK
const db = DB{ .path = "test1", .schema = "schema/test" };
try testParsing(db, "GRAB User [2; name, best_friend] {name = 'Bob'}");
try testParsing(db, "GRAB User [2; name, best_friend] {best_friend IN {}}");
try testParsing(db, "GRAB User [2; name, best_friend] {best_friend !IN {}}");
}
test "GRAB Relationship dot" {
test "GRAB Relationship dot" { // TODO: Make this a reality
// DO I add this ? I'm not sure about this feature
const db = DB{ .path = "test1", .schema = "schema/test" };
// try testParsing(db, "GRAB User.best_friend {}");

View File

@ -17,7 +17,7 @@ pub fn myLog(
args: anytype,
) void {
_ = scope;
if (message_level == .debug) {
if (message_level != .debug) {
std.debug.print(format, args);
std.debug.print("\n", .{});
}

View File

@ -2,8 +2,6 @@ const std = @import("std");
const log = std.log.scoped(.thread);
const U64 = std.atomic.Value(u64);
// Remove the use waitgroup instead
pub const Self = @This();
processed_struct: U64 = U64.init(0),

View File

@ -20,8 +20,8 @@ pub fn init(allocator: std.mem.Allocator) !ThreadEngine {
};
const cpu_core = if (CPU_CORE == 0) std.Thread.getCpuCount() catch 1 else CPU_CORE;
log.debug(" Using {d} cpu core.", .{cpu_core});
log.debug(" Using {d}Mb stack size.", .{std.Thread.SpawnConfig.default_stack_size / 1024 / 1024});
log.debug("Using {d} cpu core.", .{cpu_core});
log.debug("Using {d}Mb stack size.", .{std.Thread.SpawnConfig.default_stack_size / 1024 / 1024});
const thread_pool = try allocator.create(std.Thread.Pool);
try thread_pool.init(std.Thread.Pool.Options{

View File

@ -57,6 +57,7 @@ pub const State = enum {
expect_comma_OR_end,
add_member_to_map,
add_array_to_map,
expect_new_array,
};
pub const Self = @This();

View File

@ -1,5 +1,6 @@
const std = @import("std");
const config = @import("config");
const DataType = @import("dtype").DataType;
const Allocator = std.mem.Allocator;
const ConditionValue = @import("../../dataStructure/filter.zig").ConditionValue;
const printError = @import("../../utils.zig").printError;
@ -14,6 +15,10 @@ const Self = @import("../parser.zig");
// Or maybe just an array, it can be an array of 1 value.
// Like that I just need do add some switch on the enum to make it work
// I dont really like that ValueOrArray. I like how it work but not how I implemented it.
// I need to see if I can just make it a bit more simple and readable.
// Maybe make it's own file ?
pub const ValueOrArray = union(enum) {
value: ConditionValue,
array: ArrayUpdate,
@ -23,7 +28,7 @@ pub const ArrayCondition = enum { append, clear, pop, remove, removeat };
pub const ArrayUpdate = struct {
condition: ArrayCondition,
data: []ConditionValue,
data: ConditionValue,
};
/// Take the tokenizer and return a map of the ADD action.
@ -97,37 +102,46 @@ pub fn parseNewData(
},
.expect_equal => switch (token.tag) {
// TODO: Implement stuff to manipulate array like APPEND or REMOVE
.equal => state = .expect_new_value,
.keyword_pop => if (for_update) {} else return printError(
.keyword_pop => if (for_update) {
state = .expect_new_array;
} else return printError(
"Error: Can only manipulate array with UPDATE.",
ZipponError.SynthaxError,
self.toker.buffer,
token.loc.start,
token.loc.end,
),
.keyword_clear => if (for_update) {} else return printError(
.keyword_clear => if (for_update) {
state = .expect_new_array;
} else return printError(
"Error: Can only manipulate array with UPDATE.",
ZipponError.SynthaxError,
self.toker.buffer,
token.loc.start,
token.loc.end,
),
.keyword_append => if (for_update) {} else return printError(
.keyword_append => if (for_update) {
state = .expect_new_array;
} else return printError(
"Error: Can only manipulate array with UPDATE.",
ZipponError.SynthaxError,
self.toker.buffer,
token.loc.start,
token.loc.end,
),
.keyword_remove => if (for_update) {} else return printError(
.keyword_remove => if (for_update) {
state = .expect_new_array;
} else return printError(
"Error: Can only manipulate array with UPDATE.",
ZipponError.SynthaxError,
self.toker.buffer,
token.loc.start,
token.loc.end,
),
.keyword_remove_at => if (for_update) {} else return printError(
.keyword_remove_at => if (for_update) {
state = .expect_new_array;
} else return printError(
"Error: Can only manipulate array with UPDATE.",
ZipponError.SynthaxError,
self.toker.buffer,
@ -156,6 +170,112 @@ pub fn parseNewData(
state = .expect_comma_OR_end;
},
.expect_new_array => { // This is what is call after array manipulation keyword
const member_data_type = self.schema_engine.memberName2DataType(struct_name, member_name) catch return ZipponError.StructNotFound;
const new_data_type: DataType = switch (token.tag) {
.l_bracket => switch (member_data_type) {
.int, .int_array => .int_array,
.float, .float_array => .float_array,
.str, .str_array => .str_array,
.bool, .bool_array => .bool_array,
.date, .date_array => .date_array,
.time, .time_array => .time_array,
.datetime, .datetime_array => .datetime_array,
.link, .link_array => .link_array,
else => unreachable,
},
.int_literal => switch (member_data_type) {
.int, .int_array => .int,
else => return printError(
"Error, expecting int or int array.",
ZipponError.SynthaxError,
self.toker.buffer,
token.loc.start,
token.loc.end,
),
},
.float_literal => switch (member_data_type) {
.float, .float_array => .float,
else => return printError(
"Error, expecting float or float array.",
ZipponError.SynthaxError,
self.toker.buffer,
token.loc.start,
token.loc.end,
),
},
.string_literal => switch (member_data_type) {
.str, .str_array => .str,
else => return printError(
"Error, expecting str or str array.",
ZipponError.SynthaxError,
self.toker.buffer,
token.loc.start,
token.loc.end,
),
},
.bool_literal_false, .bool_literal_true => switch (member_data_type) {
.bool, .bool_array => .bool,
else => return printError(
"Error, expecting bool or bool array.",
ZipponError.SynthaxError,
self.toker.buffer,
token.loc.start,
token.loc.end,
),
},
.date_literal => switch (member_data_type) {
.date, .date_array => .date,
.datetime, .datetime_array => .datetime,
else => return printError(
"Error, expecting date or date array.",
ZipponError.SynthaxError,
self.toker.buffer,
token.loc.start,
token.loc.end,
),
},
.time_literal => switch (member_data_type) {
.time, .time_array => .time,
else => return printError(
"Error, expecting time or time array.",
ZipponError.SynthaxError,
self.toker.buffer,
token.loc.start,
token.loc.end,
),
},
.datetime_literal => switch (member_data_type) {
.datetime, .datetime_array => .datetime,
else => return printError(
"Error, expecting datetime or datetime array.",
ZipponError.SynthaxError,
self.toker.buffer,
token.loc.start,
token.loc.end,
),
},
else => return printError(
"Error, expecting value or array.",
ZipponError.SynthaxError,
self.toker.buffer,
token.loc.start,
token.loc.end,
),
};
map.put(
member_name,
// TODO: Get right keyword
// TODO: Update ValueOrArray.array to use a single ConditionValue value instead of current array of it
ValueOrArray{ .array = .{ .condition = .append, .data = try self.parseConditionValue(allocator, struct_name, member_name, new_data_type, &token) } },
) catch return ZipponError.MemoryError;
if (member_data_type == .link or member_data_type == .link_array) {
token = self.toker.last_token;
keep_next = true;
}
state = .expect_comma_OR_end;
},
.expect_comma_OR_end => switch (token.tag) {
.r_paren => state = .end,
.comma => state = .expect_member_OR_value,