MrBounty
13caff2fd7
Can now use command dump csv Will see later to add json and zid format as well as loading it
1219 lines
46 KiB
Zig
1219 lines
46 KiB
Zig
const std = @import("std");
|
|
const utils = @import("stuffs/utils.zig");
|
|
const zid = @import("ZipponData");
|
|
const U64 = std.atomic.Value(u64);
|
|
const Pool = std.Thread.Pool;
|
|
const Allocator = std.mem.Allocator;
|
|
const SchemaEngine = @import("schemaEngine.zig").SchemaEngine;
|
|
const SchemaStruct = @import("schemaEngine.zig").SchemaStruct;
|
|
const ThreadSyncContext = @import("threadEngine.zig").ThreadSyncContext;
|
|
const EntityWriter = @import("entityWriter.zig").EntityWriter;
|
|
|
|
const dtype = @import("dtype");
|
|
const s2t = dtype.s2t;
|
|
const UUID = dtype.UUID;
|
|
const DateTime = dtype.DateTime;
|
|
const DataType = dtype.DataType;
|
|
|
|
const AdditionalData = @import("stuffs/additionalData.zig").AdditionalData;
|
|
const Filter = @import("stuffs/filter.zig").Filter;
|
|
const RelationMap = @import("stuffs/relationMap.zig").RelationMap;
|
|
const JsonString = @import("stuffs/relationMap.zig").JsonString;
|
|
const ConditionValue = @import("stuffs/filter.zig").ConditionValue;
|
|
|
|
const ZipponError = @import("stuffs/errors.zig").ZipponError;
|
|
|
|
const config = @import("config");
|
|
const BUFFER_SIZE = config.BUFFER_SIZE;
|
|
const OUT_BUFFER_SIZE = config.OUT_BUFFER_SIZE;
|
|
const MAX_FILE_SIZE = config.MAX_FILE_SIZE;
|
|
const RESET_LOG_AT_RESTART = config.RESET_LOG_AT_RESTART;
|
|
const CPU_CORE = config.CPU_CORE;
|
|
|
|
// TODO: Cute this into smaller modules: core, file_management, data_operation, utils
|
|
|
|
const log = std.log.scoped(.fileEngine);
|
|
|
|
var path_buffer: [1024]u8 = undefined;
|
|
var path_to_ZipponDB_dir_buffer: [1024]u8 = undefined;
|
|
|
|
/// Manage everything that is relate to read or write in files
|
|
/// Or even get stats, whatever. If it touch files, it's here
|
|
pub const FileEngine = @This();
|
|
|
|
path_to_ZipponDB_dir: []const u8,
|
|
thread_pool: *Pool, // same pool as the ThreadEngine
|
|
schema_engine: SchemaEngine = undefined, // This is init after the FileEngine and I attach after. Do I need to init after tho ?
|
|
|
|
pub fn init(path: []const u8, thread_pool: *Pool) ZipponError!FileEngine {
|
|
return FileEngine{
|
|
.path_to_ZipponDB_dir = std.fmt.bufPrint(&path_to_ZipponDB_dir_buffer, "{s}", .{path}) catch return ZipponError.MemoryError,
|
|
.thread_pool = thread_pool,
|
|
};
|
|
}
|
|
|
|
// --------------------Other--------------------
|
|
|
|
pub fn readSchemaFile(sub_path: []const u8, buffer: []u8) ZipponError!usize {
|
|
const file = std.fs.cwd().openFile(sub_path, .{}) catch return ZipponError.CantOpenFile;
|
|
defer file.close();
|
|
|
|
const len = file.readAll(buffer) catch return ZipponError.ReadError;
|
|
return len;
|
|
}
|
|
|
|
pub fn writeDbMetrics(self: *FileEngine, buffer: *std.ArrayList(u8)) ZipponError!void {
|
|
const main_dir = std.fs.cwd().openDir(self.path_to_ZipponDB_dir, .{ .iterate = true }) catch return ZipponError.CantOpenDir;
|
|
|
|
const writer = buffer.writer();
|
|
writer.print("Database path: {s}\n", .{self.path_to_ZipponDB_dir}) catch return ZipponError.WriteError;
|
|
const main_size = utils.getDirTotalSize(main_dir) catch 0;
|
|
writer.print("Total size: {d:.2}Mb\n", .{@as(f64, @floatFromInt(main_size)) / 1024.0 / 1024.0}) catch return ZipponError.WriteError;
|
|
|
|
const log_dir = main_dir.openDir("LOG", .{ .iterate = true }) catch return ZipponError.CantOpenDir;
|
|
const log_size = utils.getDirTotalSize(log_dir) catch 0;
|
|
writer.print("LOG: {d:.2}Mb\n", .{@as(f64, @floatFromInt(log_size)) / 1024.0 / 1024.0}) catch return ZipponError.WriteError;
|
|
|
|
const backup_dir = main_dir.openDir("BACKUP", .{ .iterate = true }) catch return ZipponError.CantOpenDir;
|
|
const backup_size = utils.getDirTotalSize(backup_dir) catch 0;
|
|
writer.print("BACKUP: {d:.2}Mb\n", .{@as(f64, @floatFromInt(backup_size)) / 1024.0 / 1024.0}) catch return ZipponError.WriteError;
|
|
|
|
const data_dir = main_dir.openDir("DATA", .{ .iterate = true }) catch return ZipponError.CantOpenDir;
|
|
const data_size = utils.getDirTotalSize(data_dir) catch 0;
|
|
writer.print("DATA: {d:.2}Mb\n", .{@as(f64, @floatFromInt(data_size)) / 1024.0 / 1024.0}) catch return ZipponError.WriteError;
|
|
|
|
var iter = data_dir.iterate();
|
|
while (iter.next() catch return ZipponError.DirIterError) |entry| {
|
|
if (entry.kind != .directory) continue;
|
|
const sub_dir = data_dir.openDir(entry.name, .{ .iterate = true }) catch return ZipponError.CantOpenDir;
|
|
const size = utils.getDirTotalSize(sub_dir) catch 0;
|
|
writer.print(" {s}: {d:.}Mb {d} entities\n", .{
|
|
entry.name,
|
|
@as(f64, @floatFromInt(size)) / 1024.0 / 1024.0,
|
|
try self.getNumberOfEntity(entry.name),
|
|
}) catch return ZipponError.WriteError;
|
|
}
|
|
}
|
|
|
|
// --------------------Init folder and files--------------------
|
|
|
|
/// Create the main folder. Including DATA, LOG and BACKUP
|
|
pub fn createMainDirectories(self: *FileEngine) ZipponError!void {
|
|
var path_buff = std.fmt.bufPrint(&path_buffer, "{s}", .{self.path_to_ZipponDB_dir}) catch return ZipponError.MemoryError;
|
|
|
|
const cwd = std.fs.cwd();
|
|
|
|
cwd.makeDir(path_buff) catch |err| switch (err) {
|
|
error.PathAlreadyExists => {},
|
|
else => return ZipponError.CantMakeDir,
|
|
};
|
|
|
|
path_buff = std.fmt.bufPrint(&path_buffer, "{s}/DATA", .{self.path_to_ZipponDB_dir}) catch return ZipponError.MemoryError;
|
|
|
|
cwd.makeDir(path_buff) catch |err| switch (err) {
|
|
error.PathAlreadyExists => {},
|
|
else => return ZipponError.CantMakeDir,
|
|
};
|
|
|
|
path_buff = std.fmt.bufPrint(&path_buffer, "{s}/BACKUP", .{self.path_to_ZipponDB_dir}) catch return ZipponError.MemoryError;
|
|
|
|
cwd.makeDir(path_buff) catch |err| switch (err) {
|
|
error.PathAlreadyExists => {},
|
|
else => return ZipponError.CantMakeDir,
|
|
};
|
|
|
|
path_buff = std.fmt.bufPrint(&path_buffer, "{s}/LOG", .{self.path_to_ZipponDB_dir}) catch return ZipponError.MemoryError;
|
|
|
|
cwd.makeDir(path_buff) catch |err| switch (err) {
|
|
error.PathAlreadyExists => {},
|
|
else => return ZipponError.CantMakeDir,
|
|
};
|
|
|
|
path_buff = std.fmt.bufPrint(&path_buffer, "{s}/LOG/log", .{self.path_to_ZipponDB_dir}) catch return ZipponError.MemoryError;
|
|
|
|
if (RESET_LOG_AT_RESTART) {
|
|
_ = cwd.createFile(path_buff, .{}) catch return ZipponError.CantMakeFile;
|
|
} else {
|
|
_ = std.fs.cwd().openFile(path_buff, .{}) catch {
|
|
_ = cwd.createFile(path_buff, .{}) catch return ZipponError.CantMakeFile;
|
|
};
|
|
}
|
|
}
|
|
|
|
/// Request a path to a schema file and then create the struct folder
|
|
/// TODO: Check if some data already exist and if so ask if the user want to delete it and make a backup
|
|
pub fn createStructDirectories(self: *FileEngine, struct_array: []SchemaStruct) ZipponError!void {
|
|
var data_dir = try utils.printOpenDir("{s}/DATA", .{self.path_to_ZipponDB_dir}, .{});
|
|
defer data_dir.close();
|
|
|
|
for (struct_array) |schema_struct| {
|
|
data_dir.makeDir(schema_struct.name) catch |err| switch (err) {
|
|
error.PathAlreadyExists => continue,
|
|
else => return ZipponError.CantMakeDir,
|
|
};
|
|
const struct_dir = data_dir.openDir(schema_struct.name, .{}) catch return ZipponError.CantOpenDir;
|
|
|
|
zid.createFile("0.zid", struct_dir) catch return ZipponError.CantMakeFile;
|
|
}
|
|
}
|
|
|
|
// --------------------Read and parse files--------------------
|
|
|
|
/// Use a struct name to populate a list with all UUID of this struct
|
|
/// TODO: Multi thread that too
|
|
pub fn getNumberOfEntity(self: *FileEngine, struct_name: []const u8) ZipponError!usize {
|
|
var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
|
|
defer arena.deinit();
|
|
const allocator = arena.allocator();
|
|
|
|
const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name);
|
|
const max_file_index = try self.maxFileIndex(sstruct.name);
|
|
var count: usize = 0;
|
|
|
|
const dir = try utils.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{});
|
|
|
|
for (0..(max_file_index + 1)) |i| {
|
|
const path_buff = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{i}) catch return ZipponError.MemoryError;
|
|
|
|
var iter = zid.DataIterator.init(allocator, path_buff, dir, sstruct.zid_schema) catch return ZipponError.ZipponDataError;
|
|
defer iter.deinit();
|
|
|
|
while (iter.next() catch return ZipponError.ZipponDataError) |_| count += 1;
|
|
}
|
|
|
|
return count;
|
|
}
|
|
|
|
const UUIDFileIndex = @import("stuffs/UUIDFileIndex.zig").UUIDIndexMap;
|
|
|
|
/// Populate a map with all UUID bytes as key and file index as value
|
|
/// This map is store in the SchemaStruct to then by using a list of UUID, get a list of file_index to parse
|
|
pub fn populateFileIndexUUIDMap(
|
|
self: *FileEngine,
|
|
sstruct: SchemaStruct,
|
|
map: *UUIDFileIndex,
|
|
) ZipponError!void {
|
|
var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
|
|
defer arena.deinit();
|
|
const allocator = arena.allocator();
|
|
|
|
const max_file_index = try self.maxFileIndex(sstruct.name);
|
|
|
|
const dir = try utils.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{});
|
|
|
|
// Multi-threading setup
|
|
var sync_context = ThreadSyncContext.init(
|
|
0,
|
|
max_file_index + 1,
|
|
);
|
|
|
|
// Create a thread-safe writer for each file
|
|
var thread_writer_list = allocator.alloc(std.ArrayList(UUID), max_file_index + 1) catch return ZipponError.MemoryError;
|
|
defer {
|
|
for (thread_writer_list) |list| list.deinit();
|
|
allocator.free(thread_writer_list);
|
|
}
|
|
|
|
for (thread_writer_list) |*list| {
|
|
list.* = std.ArrayList(UUID).init(allocator);
|
|
}
|
|
|
|
// Spawn threads for each file
|
|
for (0..(max_file_index + 1)) |file_index| {
|
|
self.thread_pool.spawn(populateFileIndexUUIDMapOneFile, .{
|
|
sstruct,
|
|
&thread_writer_list[file_index],
|
|
file_index,
|
|
dir,
|
|
&sync_context,
|
|
}) catch return ZipponError.ThreadError;
|
|
}
|
|
|
|
// Wait for all threads to complete
|
|
while (!sync_context.isComplete()) {
|
|
std.time.sleep(10_000_000);
|
|
}
|
|
|
|
// Combine results
|
|
for (thread_writer_list, 0..) |list, file_index| {
|
|
for (list.items) |uuid| map.put(uuid, file_index) catch return ZipponError.MemoryError;
|
|
}
|
|
}
|
|
|
|
fn populateFileIndexUUIDMapOneFile(
|
|
sstruct: SchemaStruct,
|
|
list: *std.ArrayList(UUID),
|
|
file_index: u64,
|
|
dir: std.fs.Dir,
|
|
sync_context: *ThreadSyncContext,
|
|
) void {
|
|
var data_buffer: [BUFFER_SIZE]u8 = undefined;
|
|
var fa = std.heap.FixedBufferAllocator.init(&data_buffer);
|
|
defer fa.reset();
|
|
const allocator = fa.allocator();
|
|
|
|
const path = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{file_index}) catch |err| {
|
|
sync_context.logError("Error creating file path", err);
|
|
return;
|
|
};
|
|
|
|
var iter = zid.DataIterator.init(allocator, path, dir, sstruct.zid_schema) catch |err| {
|
|
sync_context.logError("Error initializing DataIterator", err);
|
|
return;
|
|
};
|
|
defer iter.deinit();
|
|
|
|
while (iter.next() catch |err| {
|
|
sync_context.logError("Error initializing DataIterator", err);
|
|
return;
|
|
}) |row| {
|
|
list.*.append(UUID{ .bytes = row[0].UUID }) catch |err| {
|
|
sync_context.logError("Error initializing DataIterator", err);
|
|
return;
|
|
};
|
|
}
|
|
|
|
_ = sync_context.completeThread();
|
|
}
|
|
|
|
/// Use a struct name and filter to populate a map with all UUID bytes as key and void as value
|
|
/// This map is use as value for the ConditionValue of links, so I can do a `contains` on it.
|
|
pub fn populateVoidUUIDMap(
|
|
self: *FileEngine,
|
|
struct_name: []const u8,
|
|
filter: ?Filter,
|
|
map: *std.AutoHashMap(UUID, void),
|
|
additional_data: *AdditionalData,
|
|
) ZipponError!void {
|
|
var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
|
|
defer arena.deinit();
|
|
const allocator = arena.allocator();
|
|
|
|
const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name);
|
|
const max_file_index = try self.maxFileIndex(sstruct.name);
|
|
|
|
const dir = try utils.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{});
|
|
|
|
// Multi-threading setup
|
|
var sync_context = ThreadSyncContext.init(
|
|
additional_data.limit,
|
|
max_file_index + 1,
|
|
);
|
|
|
|
// Create a thread-safe writer for each file
|
|
var thread_writer_list = allocator.alloc(std.ArrayList(UUID), max_file_index + 1) catch return ZipponError.MemoryError;
|
|
|
|
for (thread_writer_list) |*list| {
|
|
list.* = std.ArrayList(UUID).init(allocator);
|
|
}
|
|
|
|
// Spawn threads for each file
|
|
for (0..(max_file_index + 1)) |file_index| {
|
|
self.thread_pool.spawn(populateVoidUUIDMapOneFile, .{
|
|
sstruct,
|
|
filter,
|
|
&thread_writer_list[file_index],
|
|
file_index,
|
|
dir,
|
|
&sync_context,
|
|
}) catch return ZipponError.ThreadError;
|
|
}
|
|
|
|
// Wait for all threads to complete
|
|
while (!sync_context.isComplete()) {
|
|
std.time.sleep(10_000_000);
|
|
}
|
|
|
|
// Combine results
|
|
for (thread_writer_list) |list| {
|
|
for (list.items) |uuid| _ = map.getOrPut(uuid) catch return ZipponError.MemoryError;
|
|
}
|
|
|
|
if (additional_data.limit == 0) return;
|
|
|
|
if (map.count() > additional_data.limit) {
|
|
log.err("Found {d} entity in populateVoidUUIDMap but max is: {d}", .{ map.count(), additional_data.limit });
|
|
var iter = map.iterator();
|
|
while (iter.next()) |entry| {
|
|
log.debug("{s}", .{UUID.format_bytes(entry.key_ptr.bytes)});
|
|
}
|
|
}
|
|
}
|
|
|
|
fn populateVoidUUIDMapOneFile(
|
|
sstruct: SchemaStruct,
|
|
filter: ?Filter,
|
|
list: *std.ArrayList(UUID),
|
|
file_index: u64,
|
|
dir: std.fs.Dir,
|
|
sync_context: *ThreadSyncContext,
|
|
) void {
|
|
var data_buffer: [BUFFER_SIZE]u8 = undefined;
|
|
var fa = std.heap.FixedBufferAllocator.init(&data_buffer);
|
|
defer fa.reset();
|
|
const allocator = fa.allocator();
|
|
|
|
const path = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{file_index}) catch |err| {
|
|
sync_context.logError("Error creating file path", err);
|
|
return;
|
|
};
|
|
|
|
var iter = zid.DataIterator.init(allocator, path, dir, sstruct.zid_schema) catch |err| {
|
|
sync_context.logError("Error initializing DataIterator", err);
|
|
return;
|
|
};
|
|
defer iter.deinit();
|
|
|
|
while (iter.next() catch |err| {
|
|
sync_context.logError("Error in iter next", err);
|
|
return;
|
|
}) |row| {
|
|
if (sync_context.checkStructLimit()) break;
|
|
if (filter == null or filter.?.evaluate(row)) {
|
|
list.*.append(UUID{ .bytes = row[0].UUID }) catch |err| {
|
|
sync_context.logError("Error initializing DataIterator", err);
|
|
return;
|
|
};
|
|
|
|
if (sync_context.incrementAndCheckStructLimit()) break;
|
|
}
|
|
}
|
|
|
|
_ = sync_context.completeThread();
|
|
}
|
|
|
|
/// Take a filter, parse all file and if one struct if validate by the filter, write it in a JSON format to the writer
|
|
/// filter can be null. This will return all of them
|
|
pub fn parseEntities(
|
|
self: *FileEngine,
|
|
struct_name: []const u8,
|
|
filter: ?Filter,
|
|
additional_data: *AdditionalData,
|
|
entry_allocator: Allocator,
|
|
) ZipponError![]const u8 {
|
|
var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
|
|
defer arena.deinit();
|
|
const allocator = arena.allocator();
|
|
|
|
var buff = std.ArrayList(u8).init(entry_allocator);
|
|
defer buff.deinit();
|
|
const writer = buff.writer();
|
|
|
|
const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name);
|
|
const max_file_index = try self.maxFileIndex(sstruct.name);
|
|
|
|
log.debug("Max file index {d}", .{max_file_index});
|
|
|
|
// If there is no member to find, that mean we need to return all members, so let's populate additional data with all of them
|
|
if (additional_data.childrens.items.len == 0)
|
|
additional_data.populateWithEverythingExceptLink(sstruct.members, sstruct.types) catch return ZipponError.MemoryError;
|
|
|
|
// Do I populate the relationMap directly in the thread or do I do it on the string at the end ?
|
|
// I think it is better at the end, like that I dont need to create a deplicate of each map for the number of file
|
|
const relation_maps = try self.schema_engine.relationMapArrayInit(allocator, struct_name, additional_data.*);
|
|
|
|
// Open the dir that contain all files
|
|
const dir = try utils.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{ .access_sub_paths = false });
|
|
|
|
// Multi thread stuffs
|
|
var sync_context = ThreadSyncContext.init(
|
|
additional_data.limit,
|
|
max_file_index + 1,
|
|
);
|
|
|
|
// Do an array of writer for each thread
|
|
// Could I create just the number of max cpu ? Because if I have 1000 files, I do 1000 list
|
|
// But at the end, only the number of use CPU/Thread will use list simultanously
|
|
// So I could pass list from a thread to another technicly
|
|
var thread_writer_list = allocator.alloc(std.ArrayList(u8), max_file_index + 1) catch return ZipponError.MemoryError;
|
|
|
|
// Start parsing all file in multiple thread
|
|
for (0..(max_file_index + 1)) |file_index| {
|
|
thread_writer_list[file_index] = std.ArrayList(u8).init(allocator);
|
|
|
|
self.thread_pool.spawn(parseEntitiesOneFile, .{
|
|
thread_writer_list[file_index].writer(),
|
|
file_index,
|
|
dir,
|
|
sstruct.zid_schema,
|
|
filter,
|
|
additional_data.*,
|
|
try self.schema_engine.structName2DataType(struct_name),
|
|
&sync_context,
|
|
}) catch return ZipponError.ThreadError;
|
|
}
|
|
|
|
// Wait for all thread to either finish or return an error
|
|
while (!sync_context.isComplete()) {
|
|
std.time.sleep(100_000); // Check every 0.1ms
|
|
}
|
|
|
|
// Append all writer to each other
|
|
writer.writeByte('[') catch return ZipponError.WriteError;
|
|
for (thread_writer_list) |list| writer.writeAll(list.items) catch return ZipponError.WriteError;
|
|
writer.writeByte(']') catch return ZipponError.WriteError;
|
|
|
|
// Now I need to do the relation stuff, meaning parsing new files to get the relationship value
|
|
// Without relationship to return, this function is basically finish here
|
|
|
|
// Here I take the JSON string and I parse it to find all {|<>|} and add them to the relation map with an empty JsonString
|
|
for (relation_maps) |*relation_map| try relation_map.populate(buff.items);
|
|
|
|
// I then call parseEntitiesRelationMap on each
|
|
// This will update the buff items to be the same Json but with {|<[16]u8>|} replaced with the right Json
|
|
for (relation_maps) |*relation_map| try self.parseEntitiesRelationMap(allocator, relation_map.struct_name, relation_map, &buff);
|
|
|
|
return buff.toOwnedSlice() catch return ZipponError.MemoryError;
|
|
}
|
|
|
|
fn parseEntitiesOneFile(
|
|
writer: anytype,
|
|
file_index: u64,
|
|
dir: std.fs.Dir,
|
|
zid_schema: []zid.DType,
|
|
filter: ?Filter,
|
|
additional_data: AdditionalData,
|
|
data_types: []const DataType,
|
|
sync_context: *ThreadSyncContext,
|
|
) void {
|
|
var data_buffer: [BUFFER_SIZE]u8 = undefined;
|
|
var fa = std.heap.FixedBufferAllocator.init(&data_buffer);
|
|
defer fa.reset();
|
|
const allocator = fa.allocator();
|
|
|
|
const path = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{file_index}) catch |err| {
|
|
sync_context.logError("Error creating file path", err);
|
|
return;
|
|
};
|
|
|
|
var iter = zid.DataIterator.init(allocator, path, dir, zid_schema) catch |err| {
|
|
sync_context.logError("Error initializing DataIterator", err);
|
|
return;
|
|
};
|
|
|
|
while (iter.next() catch |err| {
|
|
sync_context.logError("Error in iter next", err);
|
|
return;
|
|
}) |row| {
|
|
if (sync_context.checkStructLimit()) break;
|
|
if (filter) |f| if (!f.evaluate(row)) continue;
|
|
|
|
EntityWriter.writeEntityJSON(
|
|
writer,
|
|
row,
|
|
additional_data,
|
|
data_types,
|
|
) catch |err| {
|
|
sync_context.logError("Error writing entity", err);
|
|
return;
|
|
};
|
|
if (sync_context.incrementAndCheckStructLimit()) break;
|
|
}
|
|
|
|
_ = sync_context.completeThread();
|
|
}
|
|
|
|
// Receive a map of UUID -> empty JsonString
|
|
// Will parse the files and update the value to the JSON string of the entity that represent the key
|
|
// Will then write the input with the JSON in the map looking for {|<>|}
|
|
// Once the new input received, call parseEntitiesRelationMap again the string still contain {|<>|} because of sub relationship
|
|
// The buffer contain the string with {|<>|} and need to be updated at the end
|
|
// TODO: Use the new function in SchemaEngine to reduce the number of files to parse
|
|
pub fn parseEntitiesRelationMap(
|
|
self: *FileEngine,
|
|
parent_allocator: Allocator,
|
|
struct_name: []const u8,
|
|
relation_map: *RelationMap,
|
|
buff: *std.ArrayList(u8),
|
|
) ZipponError!void {
|
|
var arena = std.heap.ArenaAllocator.init(parent_allocator);
|
|
defer arena.deinit();
|
|
const allocator = arena.allocator();
|
|
|
|
var new_buff = std.ArrayList(u8).init(allocator);
|
|
defer new_buff.deinit();
|
|
const writer = new_buff.writer();
|
|
|
|
const relation_maps = try self.schema_engine.relationMapArrayInit(
|
|
allocator,
|
|
struct_name,
|
|
relation_map.additional_data,
|
|
);
|
|
|
|
const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name);
|
|
const to_parse = try self.schema_engine.fileListToParse(allocator, struct_name, relation_map.map.*);
|
|
|
|
// If there is no member to find, that mean we need to return all members, so let's populate additional data with all of them
|
|
if (relation_map.additional_data.childrens.items.len == 0) {
|
|
relation_map.additional_data.populateWithEverythingExceptLink(
|
|
sstruct.members,
|
|
sstruct.types,
|
|
) catch return ZipponError.MemoryError;
|
|
}
|
|
|
|
// Open the dir that contain all files
|
|
const dir = try utils.printOpenDir(
|
|
"{s}/DATA/{s}",
|
|
.{ self.path_to_ZipponDB_dir, sstruct.name },
|
|
.{ .access_sub_paths = false },
|
|
);
|
|
|
|
// Multi thread stuffs
|
|
var sync_context = ThreadSyncContext.init(
|
|
relation_map.additional_data.limit,
|
|
to_parse.len,
|
|
);
|
|
|
|
// Do one writer for each thread otherwise it create error by writing at the same time
|
|
var thread_map_list = allocator.alloc(
|
|
std.AutoHashMap([16]u8, JsonString),
|
|
to_parse.len,
|
|
) catch return ZipponError.MemoryError;
|
|
|
|
// Start parsing all file in multiple thread
|
|
for (to_parse, 0..) |file_index, i| {
|
|
thread_map_list[i] = relation_map.map.cloneWithAllocator(allocator) catch return ZipponError.MemoryError;
|
|
|
|
self.thread_pool.spawn(parseEntitiesRelationMapOneFile, .{
|
|
&thread_map_list[i],
|
|
file_index,
|
|
dir,
|
|
sstruct.zid_schema,
|
|
relation_map.additional_data,
|
|
try self.schema_engine.structName2DataType(struct_name),
|
|
&sync_context,
|
|
}) catch return ZipponError.ThreadError;
|
|
}
|
|
|
|
// Wait for all thread to either finish or return an error
|
|
while (!sync_context.isComplete()) {
|
|
std.time.sleep(100_000); // Check every 0.1ms
|
|
}
|
|
|
|
// Now here I should have a list of copy of the map with all UUID a bit everywhere
|
|
|
|
// Put all in the same map
|
|
for (thread_map_list) |map| {
|
|
var iter = map.iterator();
|
|
while (iter.next()) |entry| {
|
|
if (entry.value_ptr.init) relation_map.*.map.put(entry.key_ptr.*, entry.value_ptr.*) catch return ZipponError.MemoryError;
|
|
}
|
|
}
|
|
|
|
// Here I write the new string and update the buff to have the new version
|
|
try EntityWriter.updateWithRelation(writer, buff.items, relation_map.map.*);
|
|
buff.clearRetainingCapacity();
|
|
buff.writer().writeAll(new_buff.items) catch return ZipponError.WriteError;
|
|
|
|
// Now here I need to iterate if buff.items still have {|<>|}
|
|
|
|
// Here I take the JSON string and I parse it to find all {|<>|} and add them to the relation map with an empty JsonString
|
|
for (relation_maps) |*sub_relation_map| try sub_relation_map.populate(buff.items);
|
|
|
|
// I then call parseEntitiesRelationMap on each
|
|
// This will update the buff items to be the same Json but with {|<[16]u8>|} replaced with the right Json
|
|
for (relation_maps) |*sub_relation_map| try self.parseEntitiesRelationMap(allocator, sub_relation_map.struct_name, sub_relation_map, buff);
|
|
}
|
|
|
|
fn parseEntitiesRelationMapOneFile(
|
|
map: *std.AutoHashMap([16]u8, JsonString),
|
|
file_index: u64,
|
|
dir: std.fs.Dir,
|
|
zid_schema: []zid.DType,
|
|
additional_data: AdditionalData,
|
|
data_types: []const DataType,
|
|
sync_context: *ThreadSyncContext,
|
|
) void {
|
|
var data_buffer: [BUFFER_SIZE]u8 = undefined;
|
|
var fa = std.heap.FixedBufferAllocator.init(&data_buffer);
|
|
defer fa.reset();
|
|
const allocator = fa.allocator();
|
|
|
|
const parent_alloc = map.allocator;
|
|
var string_list = std.ArrayList(u8).init(allocator);
|
|
const writer = string_list.writer();
|
|
|
|
const path = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{file_index}) catch |err| {
|
|
sync_context.logError("Error creating file path", err);
|
|
return;
|
|
};
|
|
|
|
var iter = zid.DataIterator.init(allocator, path, dir, zid_schema) catch |err| {
|
|
sync_context.logError("Error initializing DataIterator", err);
|
|
return;
|
|
};
|
|
|
|
while (iter.next() catch |err| {
|
|
sync_context.logError("Error in iter next", err);
|
|
return;
|
|
}) |row| {
|
|
if (sync_context.checkStructLimit()) break;
|
|
if (!map.contains(row[0].UUID)) continue;
|
|
defer string_list.clearRetainingCapacity();
|
|
|
|
EntityWriter.writeEntityJSON(
|
|
writer,
|
|
row,
|
|
additional_data,
|
|
data_types,
|
|
) catch |err| {
|
|
sync_context.logError("Error writing entity", err);
|
|
return;
|
|
};
|
|
map.put(row[0].UUID, JsonString{
|
|
.slice = parent_alloc.dupe(u8, string_list.items) catch |err| {
|
|
sync_context.logError("Error duping data", err);
|
|
return;
|
|
},
|
|
.init = true,
|
|
}) catch |err| {
|
|
sync_context.logError("Error writing entity", err);
|
|
return;
|
|
};
|
|
|
|
if (sync_context.incrementAndCheckStructLimit()) break;
|
|
}
|
|
|
|
_ = sync_context.completeThread();
|
|
}
|
|
|
|
// --------------------Change existing files--------------------
|
|
|
|
// TODO: Make it in batch too
|
|
pub fn addEntity(
|
|
self: *FileEngine,
|
|
struct_name: []const u8,
|
|
maps: []std.StringHashMap(ConditionValue),
|
|
writer: anytype,
|
|
) ZipponError!void {
|
|
var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
|
|
defer arena.deinit();
|
|
const allocator = arena.allocator();
|
|
|
|
var file_index = try self.getFirstUsableIndexFile(struct_name); // TODO: Speed up this
|
|
var path = std.fmt.bufPrint(&path_buffer, "{s}/DATA/{s}/{d}.zid", .{ self.path_to_ZipponDB_dir, struct_name, file_index }) catch return ZipponError.MemoryError;
|
|
|
|
var data_writer = zid.DataWriter.init(path, null) catch return ZipponError.ZipponDataError;
|
|
defer data_writer.deinit();
|
|
|
|
const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name);
|
|
|
|
for (maps) |map| {
|
|
const data = try self.orderedNewData(allocator, struct_name, map);
|
|
data_writer.write(data) catch return ZipponError.ZipponDataError;
|
|
sstruct.uuid_file_index.map.*.put(UUID{ .bytes = data[0].UUID }, file_index) catch return ZipponError.MemoryError;
|
|
writer.print("\"{s}\", ", .{UUID.format_bytes(data[0].UUID)}) catch return ZipponError.WriteError;
|
|
|
|
const file_stat = data_writer.fileStat() catch return ZipponError.ZipponDataError;
|
|
if (file_stat.size > MAX_FILE_SIZE) {
|
|
file_index = try self.getFirstUsableIndexFile(struct_name);
|
|
data_writer.flush() catch return ZipponError.ZipponDataError;
|
|
data_writer.deinit();
|
|
|
|
path = std.fmt.bufPrint(&path_buffer, "{s}/DATA/{s}/{d}.zid", .{ self.path_to_ZipponDB_dir, struct_name, file_index }) catch return ZipponError.MemoryError;
|
|
data_writer = zid.DataWriter.init(path, null) catch return ZipponError.ZipponDataError;
|
|
}
|
|
}
|
|
|
|
data_writer.flush() catch return ZipponError.ZipponDataError;
|
|
}
|
|
|
|
pub fn updateEntities(
|
|
self: *FileEngine,
|
|
struct_name: []const u8,
|
|
filter: ?Filter,
|
|
map: std.StringHashMap(ConditionValue),
|
|
writer: anytype,
|
|
additional_data: *AdditionalData,
|
|
) ZipponError!void {
|
|
var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
|
|
defer arena.deinit();
|
|
const allocator = arena.allocator();
|
|
|
|
const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name);
|
|
const max_file_index = try self.maxFileIndex(sstruct.name);
|
|
|
|
const dir = try utils.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{});
|
|
|
|
// Multi-threading setup
|
|
var sync_context = ThreadSyncContext.init(
|
|
additional_data.limit,
|
|
max_file_index + 1,
|
|
);
|
|
|
|
// Create a thread-safe writer for each file
|
|
var thread_writer_list = allocator.alloc(std.ArrayList(u8), max_file_index + 1) catch return ZipponError.MemoryError;
|
|
for (thread_writer_list) |*list| {
|
|
list.* = std.ArrayList(u8).init(allocator);
|
|
}
|
|
|
|
var new_data_buff = allocator.alloc(zid.Data, sstruct.members.len) catch return ZipponError.MemoryError;
|
|
|
|
// Convert the map to an array of ZipponData Data type, to be use with ZipponData writter
|
|
for (sstruct.members, 0..) |member, i| {
|
|
if (!map.contains(member)) continue;
|
|
new_data_buff[i] = try string2Data(allocator, map.get(member).?);
|
|
}
|
|
|
|
// Spawn threads for each file
|
|
for (0..(max_file_index + 1)) |file_index| {
|
|
self.thread_pool.spawn(updateEntitiesOneFile, .{
|
|
new_data_buff,
|
|
sstruct,
|
|
filter,
|
|
&map,
|
|
thread_writer_list[file_index].writer(),
|
|
file_index,
|
|
dir,
|
|
&sync_context,
|
|
}) catch return ZipponError.ThreadError;
|
|
}
|
|
|
|
// Wait for all threads to complete
|
|
while (!sync_context.isComplete()) {
|
|
std.time.sleep(100_000); // Check every 0.1ms
|
|
}
|
|
|
|
// Combine results
|
|
writer.writeByte('[') catch return ZipponError.WriteError;
|
|
for (thread_writer_list) |list| {
|
|
writer.writeAll(list.items) catch return ZipponError.WriteError;
|
|
}
|
|
writer.writeByte(']') catch return ZipponError.WriteError;
|
|
}
|
|
|
|
fn updateEntitiesOneFile(
|
|
new_data_buff: []zid.Data,
|
|
sstruct: SchemaStruct,
|
|
filter: ?Filter,
|
|
map: *const std.StringHashMap(ConditionValue),
|
|
writer: anytype,
|
|
file_index: u64,
|
|
dir: std.fs.Dir,
|
|
sync_context: *ThreadSyncContext,
|
|
) void {
|
|
log.debug("{any}\n", .{@TypeOf(writer)});
|
|
var data_buffer: [BUFFER_SIZE]u8 = undefined;
|
|
var fa = std.heap.FixedBufferAllocator.init(&data_buffer);
|
|
defer fa.reset();
|
|
const allocator = fa.allocator();
|
|
|
|
const path = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{file_index}) catch |err| {
|
|
sync_context.logError("Error creating file path", err);
|
|
return;
|
|
};
|
|
|
|
var iter = zid.DataIterator.init(allocator, path, dir, sstruct.zid_schema) catch |err| {
|
|
sync_context.logError("Error initializing DataIterator", err);
|
|
return;
|
|
};
|
|
defer iter.deinit();
|
|
|
|
const new_path = std.fmt.allocPrint(allocator, "{d}.zid.new", .{file_index}) catch |err| {
|
|
sync_context.logError("Error creating new file path", err);
|
|
return;
|
|
};
|
|
defer allocator.free(new_path);
|
|
|
|
zid.createFile(new_path, dir) catch |err| {
|
|
sync_context.logError("Error creating new file", err);
|
|
return;
|
|
};
|
|
|
|
var new_writer = zid.DataWriter.init(new_path, dir) catch |err| {
|
|
sync_context.logError("Error initializing DataWriter", err);
|
|
zid.deleteFile(new_path, dir) catch {};
|
|
return;
|
|
};
|
|
defer new_writer.deinit();
|
|
|
|
var finish_writing = false;
|
|
while (iter.next() catch |err| {
|
|
sync_context.logError("Parsing files", err);
|
|
return;
|
|
}) |row| {
|
|
if (!finish_writing and (filter == null or filter.?.evaluate(row))) {
|
|
// Add the unchanged Data in the new_data_buff
|
|
new_data_buff[0] = row[0];
|
|
for (sstruct.members, 0..) |member, i| {
|
|
if (map.contains(member)) continue;
|
|
new_data_buff[i] = row[i];
|
|
}
|
|
|
|
log.debug("{d} {any}\n\n", .{ new_data_buff.len, new_data_buff });
|
|
|
|
new_writer.write(new_data_buff) catch |err| {
|
|
sync_context.logError("Error initializing DataWriter", err);
|
|
zid.deleteFile(new_path, dir) catch {};
|
|
return;
|
|
};
|
|
|
|
writer.print("\"{s}\",", .{UUID.format_bytes(row[0].UUID)}) catch |err| {
|
|
sync_context.logError("Error initializing DataWriter", err);
|
|
zid.deleteFile(new_path, dir) catch {};
|
|
return;
|
|
};
|
|
|
|
finish_writing = sync_context.incrementAndCheckStructLimit();
|
|
} else {
|
|
new_writer.write(row) catch |err| {
|
|
sync_context.logError("Error initializing DataWriter", err);
|
|
zid.deleteFile(new_path, dir) catch {};
|
|
return;
|
|
};
|
|
}
|
|
}
|
|
|
|
new_writer.flush() catch |err| {
|
|
sync_context.logError("Error initializing DataWriter", err);
|
|
zid.deleteFile(new_path, dir) catch {};
|
|
return;
|
|
};
|
|
|
|
dir.deleteFile(path) catch |err| {
|
|
sync_context.logError("Error deleting old file", err);
|
|
return;
|
|
};
|
|
|
|
dir.rename(new_path, path) catch |err| {
|
|
sync_context.logError("Error initializing DataWriter", err);
|
|
return;
|
|
};
|
|
|
|
_ = sync_context.completeThread();
|
|
}
|
|
|
|
/// Delete all entity based on the filter. Will also write a JSON format list of all UUID deleted into the buffer
|
|
pub fn deleteEntities(
|
|
self: *FileEngine,
|
|
struct_name: []const u8,
|
|
filter: ?Filter,
|
|
writer: anytype,
|
|
additional_data: *AdditionalData,
|
|
) ZipponError!void {
|
|
var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
|
|
defer arena.deinit();
|
|
const allocator = arena.allocator();
|
|
|
|
const sstruct = try self.schema_engine.structName2SchemaStruct(struct_name);
|
|
const max_file_index = try self.maxFileIndex(sstruct.name);
|
|
|
|
const dir = try utils.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{});
|
|
|
|
// Multi-threading setup
|
|
var sync_context = ThreadSyncContext.init(
|
|
additional_data.limit,
|
|
max_file_index + 1,
|
|
);
|
|
|
|
// Create a thread-safe writer for each file
|
|
var thread_writer_list = allocator.alloc(std.ArrayList(u8), max_file_index + 1) catch return ZipponError.MemoryError;
|
|
for (thread_writer_list) |*list| {
|
|
list.* = std.ArrayList(u8).init(allocator);
|
|
}
|
|
|
|
// Spawn threads for each file
|
|
for (0..(max_file_index + 1)) |file_index| {
|
|
self.thread_pool.spawn(deleteEntitiesOneFile, .{
|
|
sstruct,
|
|
filter,
|
|
thread_writer_list[file_index].writer(),
|
|
file_index,
|
|
dir,
|
|
&sync_context,
|
|
}) catch return ZipponError.ThreadError;
|
|
}
|
|
|
|
// Wait for all threads to complete
|
|
while (!sync_context.isComplete()) {
|
|
std.time.sleep(100_000); // Check every 0.1ms
|
|
}
|
|
|
|
// Combine results
|
|
writer.writeByte('[') catch return ZipponError.WriteError;
|
|
for (thread_writer_list) |list| {
|
|
writer.writeAll(list.items) catch return ZipponError.WriteError;
|
|
}
|
|
writer.writeByte(']') catch return ZipponError.WriteError;
|
|
|
|
// Update UUID file index map FIXME: Stop doing that and just remove UUID from the map itself instead of reparsing everything at the end
|
|
sstruct.uuid_file_index.map.clearRetainingCapacity();
|
|
_ = sstruct.uuid_file_index.arena.reset(.free_all);
|
|
try self.populateFileIndexUUIDMap(sstruct, sstruct.uuid_file_index);
|
|
}
|
|
|
|
fn deleteEntitiesOneFile(
|
|
sstruct: SchemaStruct,
|
|
filter: ?Filter,
|
|
writer: anytype,
|
|
file_index: u64,
|
|
dir: std.fs.Dir,
|
|
sync_context: *ThreadSyncContext,
|
|
) void {
|
|
var data_buffer: [BUFFER_SIZE]u8 = undefined;
|
|
var fa = std.heap.FixedBufferAllocator.init(&data_buffer);
|
|
defer fa.reset();
|
|
const allocator = fa.allocator();
|
|
|
|
const path = std.fmt.allocPrint(allocator, "{d}.zid", .{file_index}) catch |err| {
|
|
sync_context.logError("Error creating file path", err);
|
|
return;
|
|
};
|
|
|
|
var iter = zid.DataIterator.init(allocator, path, dir, sstruct.zid_schema) catch |err| {
|
|
sync_context.logError("Error initializing DataIterator", err);
|
|
return;
|
|
};
|
|
defer iter.deinit();
|
|
|
|
const new_path = std.fmt.allocPrint(allocator, "{d}.zid.new", .{file_index}) catch |err| {
|
|
sync_context.logError("Error creating file path", err);
|
|
return;
|
|
};
|
|
|
|
zid.createFile(new_path, dir) catch |err| {
|
|
sync_context.logError("Error creating new file", err);
|
|
return;
|
|
};
|
|
|
|
var new_writer = zid.DataWriter.init(new_path, dir) catch |err| {
|
|
sync_context.logError("Error initializing DataWriter", err);
|
|
return;
|
|
};
|
|
errdefer new_writer.deinit();
|
|
|
|
var finish_writing = false;
|
|
while (iter.next() catch |err| {
|
|
sync_context.logError("Error during iter", err);
|
|
return;
|
|
}) |row| {
|
|
if (!finish_writing and (filter == null or filter.?.evaluate(row))) {
|
|
// _ = sstruct.uuid_file_index.map.remove(UUID{ .bytes = row[0].UUID }); FIXME: This doesnt work in multithread because they try to remove at the same time
|
|
writer.print("{{\"{s}\"}},", .{UUID.format_bytes(row[0].UUID)}) catch |err| {
|
|
sync_context.logError("Error writting", err);
|
|
return;
|
|
};
|
|
|
|
finish_writing = sync_context.incrementAndCheckStructLimit();
|
|
} else {
|
|
new_writer.write(row) catch |err| {
|
|
sync_context.logError("Error writing unchanged data", err);
|
|
return;
|
|
};
|
|
}
|
|
}
|
|
|
|
new_writer.flush() catch |err| {
|
|
sync_context.logError("Error flushing new writer", err);
|
|
return;
|
|
};
|
|
|
|
dir.deleteFile(path) catch |err| {
|
|
sync_context.logError("Error deleting old file", err);
|
|
return;
|
|
};
|
|
|
|
const file_stat = new_writer.fileStat() catch |err| {
|
|
sync_context.logError("Error getting new file stat", err);
|
|
return;
|
|
};
|
|
new_writer.deinit();
|
|
if (file_index != 0 and file_stat.size == 0) dir.deleteFile(new_path) catch |err| {
|
|
sync_context.logError("Error deleting empty new file", err);
|
|
return;
|
|
} else {
|
|
dir.rename(new_path, path) catch |err| {
|
|
sync_context.logError("Error renaming new file", err);
|
|
return;
|
|
};
|
|
}
|
|
|
|
sync_context.completeThread();
|
|
}
|
|
|
|
// TODO: Make a function that take a list of UUID and remove all instance in relationship
|
|
// It is to remove when they are deleted
|
|
|
|
// --------------------ZipponData utils--------------------
|
|
|
|
//TODO: Update to make it use ConditionValue
|
|
fn string2Data(allocator: Allocator, value: ConditionValue) ZipponError!zid.Data {
|
|
switch (value) {
|
|
.int => |v| return zid.Data.initInt(v),
|
|
.float => |v| return zid.Data.initFloat(v),
|
|
.bool_ => |v| return zid.Data.initBool(v),
|
|
.unix => |v| return zid.Data.initUnix(v),
|
|
.str => |v| return zid.Data.initStr(v),
|
|
.link => |v| {
|
|
if (v.count() > 0) {
|
|
var iter = v.keyIterator();
|
|
return zid.Data.initUUID(iter.next().?.bytes);
|
|
} else {
|
|
const uuid = UUID.parse("00000000-0000-0000-0000-000000000000") catch return ZipponError.InvalidUUID;
|
|
return zid.Data.initUUID(uuid.bytes);
|
|
}
|
|
},
|
|
.link_array => |v| {
|
|
var iter = v.keyIterator();
|
|
var items = std.ArrayList([16]u8).init(allocator);
|
|
defer items.deinit();
|
|
|
|
while (iter.next()) |uuid| {
|
|
items.append(uuid.bytes) catch return ZipponError.MemoryError;
|
|
}
|
|
return zid.Data.initUUIDArray(zid.allocEncodArray.UUID(allocator, items.items) catch return ZipponError.AllocEncodError);
|
|
},
|
|
.self => |v| return zid.Data.initUUID(v.bytes),
|
|
.int_array => |v| return zid.Data.initIntArray(zid.allocEncodArray.Int(allocator, v) catch return ZipponError.AllocEncodError),
|
|
.float_array => |v| return zid.Data.initFloatArray(zid.allocEncodArray.Float(allocator, v) catch return ZipponError.AllocEncodError),
|
|
.str_array => |v| return zid.Data.initStrArray(zid.allocEncodArray.Str(allocator, v) catch return ZipponError.AllocEncodError),
|
|
.bool_array => |v| return zid.Data.initBoolArray(zid.allocEncodArray.Bool(allocator, v) catch return ZipponError.AllocEncodError),
|
|
.unix_array => |v| return zid.Data.initUnixArray(zid.allocEncodArray.Unix(allocator, v) catch return ZipponError.AllocEncodError),
|
|
}
|
|
}
|
|
|
|
/// Take a map from the parseNewData and return an ordered array of Data to be use in a DataWriter
|
|
/// TODO: Optimize and maybe put it somewhere else than fileEngine
|
|
fn orderedNewData(
|
|
self: *FileEngine,
|
|
allocator: Allocator,
|
|
struct_name: []const u8,
|
|
map: std.StringHashMap(ConditionValue),
|
|
) ZipponError![]zid.Data {
|
|
const members = try self.schema_engine.structName2structMembers(struct_name);
|
|
var datas = allocator.alloc(zid.Data, (members.len)) catch return ZipponError.MemoryError;
|
|
|
|
const new_uuid = UUID.init();
|
|
datas[0] = zid.Data.initUUID(new_uuid.bytes);
|
|
|
|
for (members, 0..) |member, i| {
|
|
if (i == 0) continue; // Skip the id
|
|
datas[i] = try string2Data(allocator, map.get(member).?);
|
|
}
|
|
|
|
return datas;
|
|
}
|
|
|
|
// --------------------Dump--------------------
|
|
pub fn dumpDb(self: FileEngine, parent_allocator: Allocator, path: []const u8, format: enum { csv, json, zid }) ZipponError!void {
|
|
std.fs.cwd().makeDir(path) catch |err| switch (err) {
|
|
error.PathAlreadyExists => {},
|
|
else => return ZipponError.CantMakeDir,
|
|
};
|
|
|
|
var arena = std.heap.ArenaAllocator.init(parent_allocator);
|
|
defer arena.deinit();
|
|
const allocator = arena.allocator();
|
|
|
|
const dir = std.fs.cwd().openDir(path, .{}) catch return ZipponError.CantOpenDir;
|
|
|
|
for (self.schema_engine.struct_array) |sstruct| {
|
|
const file_name = std.fmt.bufPrint(&path_buffer, "{s}.{s}", .{ sstruct.name, @tagName(format) }) catch return ZipponError.MemoryError;
|
|
const file = dir.createFile(file_name, .{}) catch return ZipponError.CantMakeFile;
|
|
defer file.close();
|
|
|
|
var writer = std.io.bufferedWriter(file.writer());
|
|
EntityWriter.writeHeaderCsv(writer.writer(), sstruct.members, ';') catch return ZipponError.WriteError;
|
|
|
|
const struct_dir = try utils.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, sstruct.name }, .{});
|
|
|
|
const file_indexs = try self.allFileIndex(allocator, sstruct.name);
|
|
for (file_indexs) |file_index| {
|
|
var data_buffer: [BUFFER_SIZE]u8 = undefined;
|
|
var fa = std.heap.FixedBufferAllocator.init(&data_buffer);
|
|
defer fa.reset();
|
|
const data_allocator = fa.allocator();
|
|
|
|
const zid_path = std.fmt.bufPrint(&path_buffer, "{d}.zid", .{file_index}) catch return ZipponError.MemoryError;
|
|
|
|
var iter = zid.DataIterator.init(data_allocator, zid_path, struct_dir, sstruct.zid_schema) catch return ZipponError.ZipponDataError;
|
|
while (iter.next() catch return ZipponError.ZipponDataError) |row| {
|
|
EntityWriter.writeEntityCsv(
|
|
writer.writer(),
|
|
row,
|
|
sstruct.types,
|
|
';',
|
|
) catch return ZipponError.WriteError;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// --------------------Schema utils--------------------
|
|
|
|
/// Get the index of the first file that is bellow the size limit. If not found, create a new file
|
|
/// TODO: Need some serious speed up. I should keep in memory a file->size as a hashmap and use that instead
|
|
fn getFirstUsableIndexFile(self: FileEngine, struct_name: []const u8) ZipponError!usize {
|
|
var member_dir = try utils.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, struct_name }, .{ .iterate = true });
|
|
defer member_dir.close();
|
|
|
|
var i: usize = 0;
|
|
var iter = member_dir.iterate();
|
|
while (iter.next() catch return ZipponError.DirIterError) |entry| {
|
|
i += 1;
|
|
const file_stat = member_dir.statFile(entry.name) catch return ZipponError.FileStatError;
|
|
if (file_stat.size < MAX_FILE_SIZE) {
|
|
// Cant I just return i ? It is supossed that files are ordered. I think I already check and it is not
|
|
log.debug("{s}\n\n", .{entry.name});
|
|
return std.fmt.parseInt(usize, entry.name[0..(entry.name.len - 4)], 10) catch return ZipponError.InvalidFileIndex; // INFO: Hardcoded len of file extension
|
|
}
|
|
}
|
|
|
|
const path = std.fmt.bufPrint(&path_buffer, "{s}/DATA/{s}/{d}.zid", .{ self.path_to_ZipponDB_dir, struct_name, i }) catch return ZipponError.MemoryError;
|
|
zid.createFile(path, null) catch return ZipponError.ZipponDataError;
|
|
|
|
return i;
|
|
}
|
|
|
|
/// Iterate over all file of a struct and return the index of the last file.
|
|
/// E.g. a struct with 0.csv and 1.csv it return 1.
|
|
/// FIXME: I use 0..file_index but because now I delete empty file, I can end up trying to parse an empty file. So I need to delete that
|
|
/// And do something that return a list of file to parse instead
|
|
fn maxFileIndex(self: FileEngine, struct_name: []const u8) ZipponError!usize {
|
|
var dir = try utils.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, struct_name }, .{ .iterate = true });
|
|
defer dir.close();
|
|
|
|
var count: usize = 0;
|
|
|
|
var iter = dir.iterate();
|
|
while (iter.next() catch return ZipponError.DirIterError) |entry| {
|
|
if (entry.kind != .file) continue;
|
|
count += 1;
|
|
}
|
|
return count - 1;
|
|
}
|
|
|
|
fn allFileIndex(self: FileEngine, allocator: Allocator, struct_name: []const u8) ZipponError![]usize {
|
|
var dir = try utils.printOpenDir("{s}/DATA/{s}", .{ self.path_to_ZipponDB_dir, struct_name }, .{ .iterate = true });
|
|
defer dir.close();
|
|
|
|
var array = std.ArrayList(usize).init(allocator);
|
|
|
|
var iter = dir.iterate();
|
|
while (iter.next() catch return ZipponError.DirIterError) |entry| {
|
|
if (entry.kind != .file) continue;
|
|
const index = std.fmt.parseInt(usize, entry.name[0..(entry.name.len - 4)], 10) catch return ZipponError.InvalidFileIndex;
|
|
array.append(index) catch return ZipponError.MemoryError;
|
|
}
|
|
return array.toOwnedSlice() catch return ZipponError.MemoryError;
|
|
}
|
|
|
|
pub fn isSchemaFileInDir(self: *FileEngine) bool {
|
|
_ = utils.printOpenFile("{s}/schema", .{self.path_to_ZipponDB_dir}, .{}) catch return false;
|
|
return true;
|
|
}
|
|
|
|
pub fn writeSchemaFile(self: *FileEngine, null_terminated_schema_buff: [:0]const u8) ZipponError!void {
|
|
var zippon_dir = std.fs.cwd().openDir(self.path_to_ZipponDB_dir, .{}) catch return ZipponError.MemoryError;
|
|
defer zippon_dir.close();
|
|
|
|
zippon_dir.deleteFile("schema") catch |err| switch (err) {
|
|
error.FileNotFound => {},
|
|
else => return ZipponError.DeleteFileError,
|
|
};
|
|
|
|
var file = zippon_dir.createFile("schema", .{}) catch return ZipponError.CantMakeFile;
|
|
defer file.close();
|
|
file.writeAll(null_terminated_schema_buff) catch return ZipponError.WriteError;
|
|
}
|