Compilable array manipulation, still some errors

src/file/array.zig

@@ -0,0 +1,272 @@
+const std = @import("std");
+const zid = @import("ZipponData");
+const dtype = @import("dtype");
+const ConditionValue = @import("../dataStructure/filter.zig").ConditionValue;
+const ArrayCondition = @import("../ziql/parts//newData.zig").ArrayCondition;
+
+pub fn updateData(allocator: std.mem.Allocator, condition: ArrayCondition, input: *zid.Data, data: []ConditionValue) !void {
+    switch (condition) {
+        .append => try append(allocator, input, data),
+        .pop => pop(input),
+        .clear => clear(input),
+        .remove => try remove(allocator, input, data),
+        .removeat => try removeat(allocator, input, data),
+    }
+}
+
+fn pop(input: *zid.Data) void {
+    switch (input.*) {
+        .IntArray => |v| if (v.len > 4) {
+            input.*.IntArray = v[0 .. v.len - input.size()];
+        },
+        .FloatArray => |v| if (v.len > 4) {
+            input.*.FloatArray = v[0 .. v.len - input.size()];
+        },
+        .UnixArray => |v| if (v.len > 4) {
+            input.*.UnixArray = v[0 .. v.len - input.size()];
+        },
+        .UUIDArray => |v| if (v.len > 4) {
+            input.*.UUIDArray = v[0 .. v.len - input.size()];
+        },
+        .BoolArray => |v| if (v.len > 4) {
+            input.*.BoolArray = v[0 .. v.len - input.size()];
+        },
+        .StrArray => |v| if (v.len > 4) {
+            input.*.StrArray = v[0 .. v.len - input.size()];
+        },
+        else => unreachable,
+    }
+}
+
+fn clear(input: *zid.Data) void {
+    switch (input.*) {
+        .IntArray => input.*.IntArray = zid.allocEncodArray.Empty(),
+        .FloatArray => input.*.FloatArray = zid.allocEncodArray.Empty(),
+        .UnixArray => input.*.UnixArray = zid.allocEncodArray.Empty(),
+        .UUIDArray => input.*.UUIDArray = zid.allocEncodArray.Empty(),
+        .BoolArray => input.*.BoolArray = zid.allocEncodArray.Empty(),
+        .StrArray => input.*.StrArray = zid.allocEncodArray.Empty(),
+        else => unreachable,
+    }
+}
+
+fn allocForAppend(allocator: std.mem.Allocator, input: *zid.Data, data: []ConditionValue) []zid.Data {
+    switch (input.*) {
+        .UUIDArray => {
+            var total: usize = 0;
+            for (data) |d| total += d.link_array.count();
+            return try allocator.alloc(zid.Data, total);
+        },
+        else => return try allocator.alloc(zid.Data, data.len),
+    }
+}
+// I think I could use meta programming here by adding the type as argument
+fn append(allocator: std.mem.Allocator, input: *zid.Data, data: []ConditionValue) !void {
+    switch (input.*) {
+        .IntArray => {
+            // 1. Make a list of the right type from ConditionValue
+            var array = std.ArrayList(i32).init(allocator);
+            defer array.deinit();
+            for (data) |d| try array.append(d.int);
+
+            // 2. Encode the new array
+            const new_array = try zid.allocEncodArray.Int(allocator, array.items);
+
+            // 3. Add the new array at the end of the old one without the first 4 bytes that are the number of value in the array
+            var updated_array = std.ArrayList(u8).init(allocator);
+            try updated_array.appendSlice(input.IntArray);
+            try updated_array.appendSlice(new_array[4..]);
+
+            // 4. Update the number of value in the array
+            const new_len = input.size() + data.len;
+            @memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
+
+            // 5. Update the input
+            input.*.IntArray = try updated_array.toOwnedSlice();
+        },
+        .FloatArray => {
+            var array = std.ArrayList(f64).init(allocator);
+            defer array.deinit();
+            for (data) |d| try array.append(d.float);
+            const new_array = try zid.allocEncodArray.Float(allocator, array.items);
+            var updated_array = std.ArrayList(u8).init(allocator);
+            try updated_array.appendSlice(input.FloatArray);
+            try updated_array.appendSlice(new_array[4..]);
+            const new_len = input.size() + data.len;
+            @memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
+            input.*.FloatArray = try updated_array.toOwnedSlice();
+        },
+        .UnixArray => {
+            var array = std.ArrayList(u64).init(allocator);
+            defer array.deinit();
+            for (data) |d| try array.append(d.unix);
+            const new_array = try zid.allocEncodArray.Unix(allocator, array.items);
+            var updated_array = std.ArrayList(u8).init(allocator);
+            try updated_array.appendSlice(input.UnixArray);
+            try updated_array.appendSlice(new_array[4..]);
+            const new_len = input.size() + data.len;
+            @memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
+            input.*.UnixArray = try updated_array.toOwnedSlice();
+        },
+        .BoolArray => {
+            var array = std.ArrayList(bool).init(allocator);
+            defer array.deinit();
+            for (data) |d| try array.append(d.bool_);
+            const new_array = try zid.allocEncodArray.Bool(allocator, array.items);
+            var updated_array = std.ArrayList(u8).init(allocator);
+            try updated_array.appendSlice(input.BoolArray);
+            try updated_array.appendSlice(new_array[4..]);
+            const new_len = input.size() + data.len;
+            @memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
+            input.*.BoolArray = try updated_array.toOwnedSlice();
+        },
+        .StrArray => {
+            var array = std.ArrayList([]const u8).init(allocator);
+            defer array.deinit();
+            for (data) |d| try array.append(d.str);
+            const new_array = try zid.allocEncodArray.Str(allocator, array.items);
+            var updated_array = std.ArrayList(u8).init(allocator);
+            try updated_array.appendSlice(input.StrArray);
+            try updated_array.appendSlice(new_array[4..]);
+            const new_len = input.size() + data.len;
+            @memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
+            input.*.StrArray = try updated_array.toOwnedSlice();
+        },
+        .UUIDArray => { // If input is a UUID array, that mean all data are also UUIDArray. There should be only one UUIDArray in data as it is use like that "friends APPEND {name = 'Bob'}"
+            var array = std.ArrayList([16]u8).init(allocator);
+            defer array.deinit();
+            for (data) |d| {
+                var iter = d.link_array.keyIterator();
+                while (iter.next()) |uuid| try array.append(uuid.bytes);
+            }
+            const new_array = try zid.allocEncodArray.UUID(allocator, array.items);
+            var updated_array = std.ArrayList(u8).init(allocator);
+            try updated_array.appendSlice(input.UUIDArray);
+            try updated_array.appendSlice(new_array[4..]);
+            const new_len = input.size() + array.items.len;
+            @memcpy(updated_array.items[0..@sizeOf(u64)], std.mem.asBytes(&new_len));
+            input.*.UUIDArray = try updated_array.toOwnedSlice();
+        },
+        else => unreachable,
+    }
+}
+
+// TODO: Change the array for a map to speed up thing
+// And also I dont really need to realoc anything, only append need because here it can only go lower
+// So I could just memcopy the remaining of the bytes at the current position, so it overwrite the value to remove
+// Like if I want to re;ove 3 in [1 2 3 4 5], it would become [1 2 4 5 5]. Then I dont take the last value when I return.
+// But that mean I keep in memory useless data, so maybe not
+fn remove(allocator: std.mem.Allocator, input: *zid.Data, data: []ConditionValue) !void {
+    var iter = try zid.ArrayIterator.init(input.*);
+    switch (input.*) {
+        .IntArray => {
+            var array = std.ArrayList(i32).init(allocator);
+            defer array.deinit();
+            while (iter.next()) |v| if (!in(v, data)) try array.append(v.Int);
+            input.*.IntArray = try zid.allocEncodArray.Int(allocator, array.items);
+        },
+        .FloatArray => {
+            var array = std.ArrayList(f64).init(allocator);
+            defer array.deinit();
+            while (iter.next()) |v| if (!in(v, data)) try array.append(v.Float);
+            input.*.FloatArray = try zid.allocEncodArray.Float(allocator, array.items);
+        },
+        .UnixArray => {
+            var array = std.ArrayList(u64).init(allocator);
+            defer array.deinit();
+            while (iter.next()) |v| if (!in(v, data)) try array.append(v.Unix);
+            input.*.UnixArray = try zid.allocEncodArray.Unix(allocator, array.items);
+        },
+        .BoolArray => {
+            var array = std.ArrayList(bool).init(allocator);
+            defer array.deinit();
+            while (iter.next()) |v| if (!in(v, data)) try array.append(v.Bool);
+            input.*.BoolArray = try zid.allocEncodArray.Bool(allocator, array.items);
+        },
+        .StrArray => {
+            var array = std.ArrayList([]const u8).init(allocator);
+            defer array.deinit();
+            while (iter.next()) |v| if (!in(v, data)) try array.append(v.Str);
+            input.*.StrArray = try zid.allocEncodArray.Str(allocator, array.items);
+        },
+        .UUIDArray => {
+            var array = std.ArrayList([16]u8).init(allocator);
+            defer array.deinit();
+            while (iter.next()) |v| if (!in(v, data)) try array.append(v.UUID);
+            input.*.UUIDArray = try zid.allocEncodArray.UUID(allocator, array.items);
+        },
+        else => unreachable,
+    }
+}
+
+fn removeat(allocator: std.mem.Allocator, input: *zid.Data, data: []ConditionValue) !void {
+    var iter = try zid.ArrayIterator.init(input.*);
+    switch (input.*) {
+        .IntArray => {
+            var array = std.ArrayList(i32).init(allocator);
+            defer array.deinit();
+            var i: i32 = 0; // Maybe use usize because here it limite the size of the array
+            while (iter.next()) |v| {
+                defer i += 1;
+                if (!in(zid.Data{ .Int = i }, data)) try array.append(v.Int);
+            }
+            input.*.IntArray = try zid.allocEncodArray.Int(allocator, array.items);
+        },
+        .FloatArray => {
+            var array = std.ArrayList(f64).init(allocator);
+            defer array.deinit();
+            var i: i32 = 0; // Maybe use usize because here it limite the size of the array
+            while (iter.next()) |v| {
+                defer i += 1;
+                if (!in(zid.Data{ .Int = i }, data)) try array.append(v.Float);
+            }
+            input.*.FloatArray = try zid.allocEncodArray.Float(allocator, array.items);
+        },
+        .UnixArray => {
+            var array = std.ArrayList(u64).init(allocator);
+            defer array.deinit();
+            var i: i32 = 0; // Maybe use usize because here it limite the size of the array
+            while (iter.next()) |v| {
+                defer i += 1;
+                if (!in(zid.Data{ .Int = i }, data)) try array.append(v.Unix);
+            }
+            input.*.UnixArray = try zid.allocEncodArray.Unix(allocator, array.items);
+        },
+        .BoolArray => {
+            var array = std.ArrayList(bool).init(allocator);
+            defer array.deinit();
+            var i: i32 = 0; // Maybe use usize because here it limite the size of the array
+            while (iter.next()) |v| {
+                defer i += 1;
+                if (!in(zid.Data{ .Int = i }, data)) try array.append(v.Bool);
+            }
+            input.*.BoolArray = try zid.allocEncodArray.Bool(allocator, array.items);
+        },
+        .StrArray => {
+            var array = std.ArrayList([]const u8).init(allocator);
+            defer array.deinit();
+            var i: i32 = 0; // Maybe use usize because here it limite the size of the array
+            while (iter.next()) |v| {
+                defer i += 1;
+                if (!in(zid.Data{ .Int = i }, data)) try array.append(v.Str);
+            }
+            input.*.StrArray = try zid.allocEncodArray.Str(allocator, array.items);
+        },
+        .UUIDArray => unreachable, // I cant do that for removeat because link don't really have order
+        else => unreachable,
+    }
+}
+
+// Should just use a map.contain
+fn in(x: zid.Data, y: []ConditionValue) bool {
+    switch (x) {
+        .Int => |v| for (y) |z| if (v == z.int) return true,
+        .Float => |v| for (y) |z| if (v == z.float) return true,
+        .Unix => |v| for (y) |z| if (v == z.unix) return true,
+        .Bool => |v| for (y) |z| if (v == z.bool_) return true,
+        .Str => |v| for (y) |z| if (std.mem.eql(u8, z.str, v)) return true,
+        .UUID => |v| for (y) |z| if (z.link_array.contains(dtype.UUID{ .bytes = v })) return true,
+        else => unreachable,
+    }
+    return false;
+}

src/file/core.zig

@@ -1,4 +1,5 @@
 const std = @import("std");
+const config = @import("config");
 const Pool = std.Thread.Pool;
 const SchemaEngine = @import("../schema/core.zig");
 
@@ -6,6 +7,7 @@ const ZipponError = @import("error").ZipponError;
 const log = std.log.scoped(.fileEngine);
 
 var path_to_ZipponDB_dir_buffer: [1024]u8 = undefined;
+pub var data_buffer: [config.BUFFER_SIZE]u8 = undefined;
 
 /// Manage everything that is relate to read or write in files
 /// Or even get stats, whatever. If it touch files, it's here
@@ -19,14 +21,25 @@ pub usingnamespace @import("read.zig");
 pub usingnamespace @import("write.zig");
 pub usingnamespace @import("dump.zig");
 
-allocator: std.mem.Allocator = std.heap.page_allocator,
+arena: *std.heap.ArenaAllocator,
+allocator: std.mem.Allocator,
 path_to_ZipponDB_dir: []const u8,
 thread_pool: *Pool, // same pool as the ThreadEngine
 schema_engine: SchemaEngine = undefined, // This is init after the FileEngine and I attach after. Do I need to init after tho ?
 
-pub fn init(path: []const u8, thread_pool: *Pool) ZipponError!Self {
+pub fn init(allocator: std.mem.Allocator, path: []const u8, thread_pool: *Pool) ZipponError!Self {
+    const arena = allocator.create(std.heap.ArenaAllocator) catch return ZipponError.MemoryError;
+    arena.* = std.heap.ArenaAllocator.init(allocator);
     return Self{
+        .arena = arena,
+        .allocator = arena.allocator(),
         .path_to_ZipponDB_dir = std.fmt.bufPrint(&path_to_ZipponDB_dir_buffer, "{s}", .{path}) catch return ZipponError.MemoryError,
         .thread_pool = thread_pool,
     };
 }
+
+pub fn deinit(self: *Self) void {
+    const parent_allocator = self.arena.child_allocator;
+    self.arena.deinit();
+    parent_allocator.destroy(self.arena);
+}

src/file/read.zig

@@ -28,13 +28,9 @@ var path_buffer: [1024]u8 = undefined;
 /// Use a struct name to populate a list with all UUID of this struct
 /// TODO: Multi thread that too
 pub fn getNumberOfEntityAndFile(self: *Self, struct_name: []const u8) ZipponError!struct { entity: usize, file: usize } {
-    var arena = std.heap.ArenaAllocator.init(self.allocator);
-    defer arena.deinit();
-    const allocator = arena.allocator();
-
     const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name);
-    const to_parse = try self.allFileIndex(allocator, struct_name);
-    defer allocator.free(to_parse);
+    const to_parse = try self.allFileIndex(self.allocator, struct_name);
+    defer self.allocator.free(to_parse);
 
     return .{ .entity = sstruct.uuid_file_index.map.count(), .file = to_parse.len };
 }

src/file/utils.zig

@@ -5,6 +5,7 @@ const Self = @import("core.zig").Self;
 const ZipponError = @import("error").ZipponError;
 const Allocator = std.mem.Allocator;
 const ConditionValue = @import("../dataStructure/filter.zig").ConditionValue;
+const ValueOrArray = @import("../ziql/parts/newData.zig").ValueOrArray;
 const dtype = @import("dtype");
 const UUID = dtype.UUID;
 const zid = @import("ZipponData");
@@ -115,7 +116,7 @@ pub fn orderedNewData(
     self: *Self,
     allocator: Allocator,
     struct_name: []const u8,
-    map: std.StringHashMap(ConditionValue),
+    map: std.StringHashMap(ValueOrArray),
 ) ZipponError![]zid.Data {
     const members = try self.schema_engine.structName2structMembers(struct_name);
     var datas = allocator.alloc(zid.Data, (members.len)) catch return ZipponError.MemoryError;
@@ -125,7 +126,7 @@ pub fn orderedNewData(
 
     for (members, 0..) |member, i| {
         if (i == 0) continue; // Skip the id
-        datas[i] = try string2Data(allocator, map.get(member).?);
+        datas[i] = try string2Data(allocator, map.get(member).?.value);
     }
 
     return datas;

src/file/write.zig

@@ -2,6 +2,7 @@ const std = @import("std");
 const config = @import("config");
 const utils = @import("../utils.zig");
 const zid = @import("ZipponData");
+const updateData = @import("array.zig").updateData;
 const Allocator = std.mem.Allocator;
 const Self = @import("core.zig").Self;
 const ZipponError = @import("error").ZipponError;
@@ -14,6 +15,7 @@ const RelationMap = @import("../dataStructure/relationMap.zig");
 const JsonString = RelationMap.JsonString;
 const EntityWriter = @import("entityWriter.zig");
 const ThreadSyncContext = @import("../thread/context.zig");
+const ValueOrArray = @import("../ziql/parts/newData.zig").ValueOrArray;
 
 const dtype = @import("dtype");
 const s2t = dtype.s2t;
@@ -27,10 +29,10 @@ var path_buffer: [1024]u8 = undefined;
 pub fn addEntity(
     self: *Self,
     struct_name: []const u8,
-    maps: []std.StringHashMap(ConditionValue),
+    maps: []std.StringHashMap(ValueOrArray),
     writer: anytype,
 ) ZipponError!void {
-    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    var arena = std.heap.ArenaAllocator.init(self.allocator);
     defer arena.deinit();
     const allocator = arena.allocator();
 
@@ -62,15 +64,17 @@ pub fn addEntity(
     data_writer.flush() catch return ZipponError.ZipponDataError;
 }
 
+const UpdatePosibility = enum { fix, vari, stay };
+
 pub fn updateEntities(
     self: *Self,
     struct_name: []const u8,
     filter: ?Filter,
-    map: std.StringHashMap(ConditionValue),
+    map: std.StringHashMap(ValueOrArray),
     writer: anytype,
     additional_data: *AdditionalData,
 ) ZipponError!void {
-    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    var arena = std.heap.ArenaAllocator.init(self.allocator);
     defer arena.deinit();
     const allocator = arena.allocator();
 
@@ -91,21 +95,32 @@ pub fn updateEntities(
         list.* = std.ArrayList(u8).init(allocator);
     }
 
-    var new_data_buff = allocator.alloc(zid.Data, sstruct.members.len) catch return ZipponError.MemoryError;
+    var index_switch = std.ArrayList(UpdatePosibility).init(allocator);
+    defer index_switch.deinit();
 
-    // Convert the map to an array of ZipponData Data type, to be use with ZipponData writter
-    for (sstruct.members, 0..) |member, i| {
-        if (!map.contains(member)) continue;
-        new_data_buff[i] = try @import("utils.zig").string2Data(allocator, map.get(member).?);
+    // If the member name is not in the map, it stay
+    // Otherwise it need to be update. For that 2 scenarios:
+    // - Update all entities with a const .fix
+    // - Update entities base on themself .vari
+    // FIXME: I'm not sure that id is in the array, need to check, also need to check to prevent updating it
+    for (sstruct.members) |member| {
+        if (map.get(member)) |voa| {
+            switch (voa) {
+                .value => index_switch.append(.fix) catch return ZipponError.MemoryError,
+                .array => index_switch.append(.vari) catch return ZipponError.MemoryError,
+            }
+        } else {
+            index_switch.append(.stay) catch return ZipponError.MemoryError;
+        }
     }
 
     // Spawn threads for each file
     for (to_parse, 0..) |file_index, i| {
         self.thread_pool.spawn(updateEntitiesOneFile, .{
-            new_data_buff,
             sstruct,
             filter,
-            &map,
+            map,
+            index_switch.items,
             thread_writer_list[i].writer(),
             file_index,
             dir,
@@ -125,10 +140,10 @@ pub fn updateEntities(
 }
 
 fn updateEntitiesOneFile(
-    new_data_buff: []zid.Data,
     sstruct: SchemaStruct,
     filter: ?Filter,
-    map: *const std.StringHashMap(ConditionValue),
+    map: std.StringHashMap(ValueOrArray),
+    index_switch: []UpdatePosibility,
     writer: anytype,
     file_index: u64,
     dir: std.fs.Dir,
@@ -140,6 +155,20 @@ fn updateEntitiesOneFile(
     defer fa.reset();
     const allocator = fa.allocator();
 
+    var new_data_buff = allocator.alloc(zid.Data, index_switch.len) catch |err| {
+        sync_context.logError("Cant init new data buff", err);
+        return;
+    };
+
+    // First I fill the one that are updated by a const
+    for (index_switch, 0..) |is, i| switch (is) {
+        .fix => new_data_buff[i] = @import("utils.zig").string2Data(allocator, map.get(sstruct.members[i]).?.value) catch |err| {
+            sync_context.logError("Writting data", err);
+            return;
+        },
+        else => {},
+    };
+
     const path = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{file_index}) catch |err| {
         sync_context.logError("Error creating file path", err);
         return;
@@ -176,11 +205,18 @@ fn updateEntitiesOneFile(
     }) |row| {
         if (!finish_writing and (filter == null or filter.?.evaluate(row))) {
             // Add the unchanged Data in the new_data_buff
-            new_data_buff[0] = row[0];
-            for (sstruct.members, 0..) |member, i| {
-                if (map.contains(member)) continue;
-                new_data_buff[i] = row[i];
-            }
+            for (index_switch, 0..) |is, i| switch (is) {
+                .stay => new_data_buff[i] = row[i],
+                .vari => {
+                    const x = map.get(sstruct.members[i]).?.array;
+                    updateData(allocator, x.condition, &row[i], x.data) catch |err| {
+                        sync_context.logError("Error updating data", err);
+                        zid.deleteFile(new_path, dir) catch {};
+                        return;
+                    };
+                },
+                else => {},
+            };
 
             log.debug("{d} {any}\n\n", .{ new_data_buff.len, new_data_buff });
 
@@ -233,7 +269,7 @@ pub fn deleteEntities(
     writer: anytype,
     additional_data: *AdditionalData,
 ) ZipponError!void {
-    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    var arena = std.heap.ArenaAllocator.init(self.allocator);
     defer arena.deinit();
     const allocator = arena.allocator();