adrien/ZipponDB
1
0
Fork 0
Code Actions Releases Activity

Started to use fixed lenght alloc for performance

Browse Source 
For very importatnt stuff like the writter that write data when parsing,
started to use fixed length because it take the majority of the time to
write, not to parse =/

Gonna need to improve that
This commit is contained in:
Adrien Bouvais 2024-11-02 22:12:47 +01:00
parent aaa1cb2589
commit dba73ce113
4 changed files with 61 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/config.zig
View File

@ -1,4 +1,4 @@
pub const BUFFER_SIZE = 1024 * 50; // Line limit when parsing file
pub const BUFFER_SIZE = 1024 * 64 * 64; // Line limit when parsing file and other buffers
pub const MAX_FILE_SIZE = 5e+6; // 5Mb
pub const CSV_DELIMITER = ';'; // TODO: Delete

50
src/fileEngine.zig
View File

@ -3,6 +3,7 @@ const utils = @import("stuffs/utils.zig");
const dtype = @import("dtype");
const s2t = dtype.s2t;
const zid = @import("ZipponData");
const time = std.time;
const Allocator = std.mem.Allocator;
const UUID = dtype.UUID;
@ -297,7 +298,7 @@ pub const FileEngine = struct {
self: *FileEngine,
struct_name: []const u8,
filter: ?Filter,
buffer: *std.ArrayList(u8),
writer: anytype,
additional_data: *AdditionalData,
) FileEngineError!void {
const sstruct = try self.structName2SchemaStruct(struct_name);
@ -317,13 +318,18 @@ pub const FileEngine = struct {
additional_data.populateWithEverything(self.allocator, sstruct.members) catch return FileEngineError.MemoryError;
}
var writer = buffer.writer();
var data_buffer: [BUFFER_SIZE]u8 = undefined;
var fa = std.heap.FixedBufferAllocator.init(&data_buffer);
defer fa.reset();
const data_allocator = fa.allocator();
writer.writeAll("[") catch return FileEngineError.WriteError;
for (0..(max_file_index + 1)) |file_index| { // TODO: Multi thread that
self.allocator.free(path_buff);
path_buff = std.fmt.allocPrint(self.allocator, "{d}.zid", .{file_index}) catch return FileEngineError.MemoryError;
var iter = zid.DataIterator.init(self.allocator, path_buff, dir, sstruct.zid_schema) catch return FileEngineError.ZipponDataError;
fa.reset();
var iter = zid.DataIterator.init(data_allocator, path_buff, dir, sstruct.zid_schema) catch return FileEngineError.ZipponDataError;
defer iter.deinit();
blk: while (iter.next() catch return FileEngineError.ZipponDataError) |row| {
@ -361,7 +367,6 @@ pub const FileEngine = struct {
if (additional_data.entity_count_to_find != 0 and total_currently_found >= additional_data.entity_count_to_find) break :blk;
}
}
writer.writeAll("]") catch return FileEngineError.WriteError;
}
@ -399,7 +404,7 @@ pub const FileEngine = struct {
self: *FileEngine,
struct_name: []const u8,
map: std.StringHashMap([]const u8),
buffer: *std.ArrayList(u8),
writer: anytype,
) FileEngineError!void {
const uuid = UUID.init();
@ -412,18 +417,20 @@ pub const FileEngine = struct {
) catch return FileEngineError.MemoryError;
defer self.allocator.free(path);
var arena = std.heap.ArenaAllocator.init(self.allocator);
defer arena.deinit();
const data = try self.orderedNewData(arena.allocator(), struct_name, map);
var data_buffer: [BUFFER_SIZE]u8 = undefined;
var fa = std.heap.FixedBufferAllocator.init(&data_buffer);
defer fa.reset();
const data_allocator = fa.allocator();
const data = try self.orderedNewData(data_allocator, struct_name, map);
var data_writer = zid.DataWriter.init(path, null) catch return FileEngineError.ZipponDataError;
data_writer.write(data) catch return FileEngineError.ZipponDataError;
data_writer.flush() catch return FileEngineError.ZipponDataError;
var writer = buffer.writer();
writer.writeByte('{') catch return FileEngineError.WriteError;
writer.writeByte('[') catch return FileEngineError.WriteError;
writer.print("\"{s}\"", .{uuid.format_uuid()}) catch return FileEngineError.WriteError;
writer.writeAll("}, ") catch return FileEngineError.WriteError;
writer.writeAll("], ") catch return FileEngineError.WriteError;
}
pub fn updateEntities(
@ -431,7 +438,7 @@ pub const FileEngine = struct {
struct_name: []const u8,
filter: ?Filter,
map: std.StringHashMap([]const u8),
buffer: *std.ArrayList(u8),
writer: anytype,
additional_data: *AdditionalData,
) FileEngineError!void {
const sstruct = try self.structName2SchemaStruct(struct_name);
@ -456,7 +463,6 @@ pub const FileEngine = struct {
new_data_buff[i] = try string2Data(self.allocator, dt, map.get(member).?);
}
var writer = buffer.writer();
writer.writeAll("[") catch return FileEngineError.WriteError;
for (0..(max_file_index + 1)) |file_index| { // TODO: Multi thread that
if (additional_data.entity_count_to_find != 0 and total_currently_found >= additional_data.entity_count_to_find) break;
@ -483,8 +489,6 @@ pub const FileEngine = struct {
new_data_buff[i] = row[i];
}
std.debug.print("{any}\n\n", .{new_data_buff});
new_writer.write(new_data_buff) catch return FileEngineError.WriteError;
writer.writeByte('{') catch return FileEngineError.WriteError;
writer.print("\"{s}\"", .{UUID.format_bytes(row[0].UUID)}) catch return FileEngineError.WriteError;
@ -502,8 +506,6 @@ pub const FileEngine = struct {
dir.rename(new_path_buff, path_buff) catch return FileEngineError.RenameFileError;
}
writer.writeAll("]") catch return FileEngineError.WriteError;
for (try self.structName2structMembers(struct_name), 1..) |member, i| {
if (!map.contains(member)) continue;
@ -524,7 +526,7 @@ pub const FileEngine = struct {
self: *FileEngine,
struct_name: []const u8,
filter: ?Filter,
buffer: *std.ArrayList(u8),
writer: anytype,
additional_data: *AdditionalData,
) FileEngineError!void {
const sstruct = try self.structName2SchemaStruct(struct_name);
@ -539,7 +541,6 @@ pub const FileEngine = struct {
defer self.allocator.free(path_buff);
const dir = std.fs.cwd().openDir(path_buff, .{}) catch return FileEngineError.CantOpenDir;
var writer = buffer.writer();
writer.writeAll("[") catch return FileEngineError.WriteError;
for (0..(max_file_index + 1)) |file_index| { // TODO: Multi thread that
self.allocator.free(path_buff);
@ -642,7 +643,13 @@ pub const FileEngine = struct {
}
/// Take a map from the parseNewData and return an ordered array of Data to be use in a DataWriter
fn orderedNewData(self: *FileEngine, allocator: Allocator, struct_name: []const u8, map: std.StringHashMap([]const u8)) FileEngineError![]const zid.Data {
/// TODO: Optimize
fn orderedNewData(
self: *FileEngine,
allocator: Allocator,
struct_name: []const u8,
map: std.StringHashMap([]const u8),
) FileEngineError![]zid.Data {
const members = try self.structName2structMembers(struct_name);
const types = try self.structName2DataType(struct_name);
@ -660,6 +667,7 @@ pub const FileEngine = struct {
// --------------------Schema utils--------------------
/// Get the index of the first file that is bellow the size limit. If not found, create a new file
/// TODO: Need some serious speed up. I should keep in memory a file->size as a hashmap and use that instead
fn getFirstUsableIndexFile(self: FileEngine, struct_name: []const u8) FileEngineError!usize {
var path = std.fmt.allocPrint(
self.allocator,
@ -682,7 +690,7 @@ pub const FileEngine = struct {
}
}
i += 1;
self.allocator.free(path);
path = std.fmt.allocPrint(
self.allocator,
"{s}/DATA/{s}/{d}.zid",

2
src/stuffs/utils.zig
View File

@ -37,6 +37,8 @@ const stdout = std.io.getStdOut().writer();
// Maybe create a struct for that
pub fn send(comptime format: []const u8, args: anytype) void {
if (false) return;
stdout.print(format, args) catch |err| {
log.err("Can't send: {any}", .{err});
stdout.print("\x03\n", .{}) catch {};

47
src/ziqlParser.zig
View File

@ -23,6 +23,10 @@ const printError = @import("stuffs/utils.zig").printError;
const ZiQlParserError = @import("stuffs/errors.zig").ZiQlParserError;
const ZipponError = @import("stuffs/errors.zig").ZipponError;
const BUFFER_SIZE = @import("config.zig").BUFFER_SIZE;
const log = std.log.scoped(.ziqlParser);
const State = enum {
start,
invalid,
@ -85,6 +89,11 @@ pub const Parser = struct {
var struct_name: []const u8 = undefined;
var action: enum { GRAB, ADD, UPDATE, DELETE } = undefined;
var out_buff: [BUFFER_SIZE]u8 = undefined;
var fa = std.heap.FixedBufferAllocator.init(&out_buff);
defer fa.reset();
const out_allocator = self.allocator;
var token = self.toker.next();
var keep_next = false; // Use in the loop to prevent to get the next token when continue. Just need to make it true and it is reset at every loop
@ -177,18 +186,18 @@ pub const Parser = struct {
var filter = try self.parseFilter(struct_name, false);
defer filter.deinit();
var buff = std.ArrayList(u8).init(self.allocator);
var buff = std.ArrayList(u8).init(out_allocator);
defer buff.deinit();
try self.file_engine.parseEntities(struct_name, filter, &buff, &additional_data);
try self.file_engine.parseEntities(struct_name, filter, &buff.writer(), &additional_data);
send("{s}", .{buff.items});
state = .end;
},
.eof => {
var buff = std.ArrayList(u8).init(self.allocator);
var buff = std.ArrayList(u8).init(out_allocator);
defer buff.deinit();
try self.file_engine.parseEntities(struct_name, null, &buff, &additional_data);
try self.file_engine.parseEntities(struct_name, null, &buff.writer(), &additional_data);
send("{s}", .{buff.items});
state = .end;
},
@ -207,9 +216,6 @@ pub const Parser = struct {
var filter = try self.parseFilter(struct_name, false);
defer filter.deinit();
var uuids = std.ArrayList(UUID).init(self.allocator);
defer uuids.deinit();
token = self.toker.last();
if (token.tag != .keyword_to) return printError(
@ -233,10 +239,10 @@ pub const Parser = struct {
defer data_map.deinit();
try self.parseNewData(&data_map, struct_name);
var buff = std.ArrayList(u8).init(self.allocator);
var buff = std.ArrayList(u8).init(out_allocator);
defer buff.deinit();
try self.file_engine.updateEntities(struct_name, filter, data_map, &buff, &additional_data);
try self.file_engine.updateEntities(struct_name, filter, data_map, &buff.writer(), &additional_data);
send("{s}", .{buff.items});
state = .end;
},
@ -258,10 +264,10 @@ pub const Parser = struct {
defer data_map.deinit();
try self.parseNewData(&data_map, struct_name);
var buff = std.ArrayList(u8).init(self.allocator);
var buff = std.ArrayList(u8).init(out_allocator);
defer buff.deinit();
try self.file_engine.updateEntities(struct_name, null, data_map, &buff, &additional_data);
try self.file_engine.updateEntities(struct_name, null, data_map, &buff.writer(), &additional_data);
send("{s}", .{buff.items});
state = .end;
},
@ -279,18 +285,18 @@ pub const Parser = struct {
var filter = try self.parseFilter(struct_name, false);
defer filter.deinit();
var buff = std.ArrayList(u8).init(self.allocator);
var buff = std.ArrayList(u8).init(out_allocator);
defer buff.deinit();
try self.file_engine.deleteEntities(struct_name, filter, &buff, &additional_data);
try self.file_engine.deleteEntities(struct_name, filter, &buff.writer(), &additional_data);
send("{s}", .{buff.items});
state = .end;
},
.eof => {
var buff = std.ArrayList(u8).init(self.allocator);
var buff = std.ArrayList(u8).init(out_allocator);
defer buff.deinit();
try self.file_engine.deleteEntities(struct_name, null, &buff, &additional_data);
try self.file_engine.deleteEntities(struct_name, null, &buff.writer(), &additional_data);
send("{s}", .{buff.items});
state = .end;
},
@ -341,10 +347,16 @@ pub const Parser = struct {
token.loc.end,
);
}
var buff = std.ArrayList(u8).init(self.allocator);
var buff = std.ArrayList(u8).init(out_allocator);
defer buff.deinit();
self.file_engine.writeEntity(struct_name, data_map, &buff) catch return ZipponError.CantWriteEntity;
token = self.toker.last_token;
log.info("Token end of add: {s} {any}\n", .{ self.toker.getTokenSlice(token), token.tag });
if (token.tag == .identifier and std.mem.eql(u8, self.toker.getTokenSlice(token), "MULTIPLE")) {
for (0..1_000_000) |_| self.file_engine.writeEntity(struct_name, data_map, &buff.writer()) catch return ZipponError.CantWriteEntity;
} else {
self.file_engine.writeEntity(struct_name, data_map, &buff.writer()) catch return ZipponError.CantWriteEntity;
}
send("{s}", .{buff.items});
state = .end;
},
@ -1105,5 +1117,4 @@ fn testParseFilter(source: [:0]const u8) !void {
defer filter.deinit();
std.debug.print("{s}\n", .{source});
filter.debugPrint();
std.debug.print("\n", .{});
}