adrien/ZipponDB
1
0
Fork 0
Code Actions Releases Activity

Speed up date

Browse Source 
Date was taking a long time for parsing when using ADD in batch, speeded
up by like x50
This commit is contained in:
Adrien Bouvais 2025-01-11 15:27:17 +01:00
parent 7d012b527f
commit 1495e779c9
3 changed files with 30 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
lib/types/date.zig
View File

@ -24,19 +24,20 @@ pub const DateTime = struct {
} }
pub fn now() Self { pub fn now() Self {
return epoch_unix.addYears(1970).addMs(@as(u64, @intCast(std.time.milliTimestamp()))); return epoch_unix.addMs(@as(u64, @intCast(std.time.milliTimestamp())));
} }
/// Caller asserts that this is > epoch /// Caller asserts that this is > epoch
pub fn init(year: u16, month: u16, day: u16, hr: u16, min: u16, sec: u16, ms: u16) Self { pub fn init(year: u16, month: u8, day: u8, hr: u8, min: u8, sec: u8, ms: u16) Self {
return epoch_unix return Self{
.addYears(year - epoch_unix.years) .years = if (year < 1970) 1970 else year,
.addMonths(month) .months = month,
.addDays(day) .days = day,
.addHours(hr) .hours = hr,
.addMins(min) .minutes = min,
.addSecs(sec) .seconds = sec,
.addMs(ms); .ms = ms,
};
} }
pub const epoch_unix = Self{ pub const epoch_unix = Self{
@ -46,7 +47,7 @@ pub const DateTime = struct {
.hours = 0, .hours = 0,
.days = 0, .days = 0,
.months = 0, .months = 0,
.years = 0, .years = 1970,
}; };
pub fn eql(self: Self, other: Self) bool { pub fn eql(self: Self, other: Self) bool {

20
lib/types/stringToType.zig
View File

@ -45,8 +45,8 @@ pub fn parseDate(value_str: []const u8) DateTime {
if (std.mem.eql(u8, value_str, "NOW")) return DateTime.now(); if (std.mem.eql(u8, value_str, "NOW")) return DateTime.now();
const year: u16 = std.fmt.parseInt(u16, value_str[0..4], 10) catch 0; const year: u16 = std.fmt.parseInt(u16, value_str[0..4], 10) catch 0;
const month: u16 = std.fmt.parseInt(u16, value_str[5..7], 10) catch 0; const month: u8 = std.fmt.parseInt(u8, value_str[5..7], 10) catch 0;
const day: u16 = std.fmt.parseInt(u16, value_str[8..10], 10) catch 0; const day: u8 = std.fmt.parseInt(u8, value_str[8..10], 10) catch 0;
return DateTime.init(year, month - 1, day - 1, 0, 0, 0, 0); return DateTime.init(year, month - 1, day - 1, 0, 0, 0, 0);
} }
@ -78,9 +78,9 @@ pub fn parseArrayDateUnix(allocator: std.mem.Allocator, array_str: []const u8) !
pub fn parseTime(value_str: []const u8) DateTime { pub fn parseTime(value_str: []const u8) DateTime {
if (std.mem.eql(u8, value_str, "NOW")) return DateTime.now(); if (std.mem.eql(u8, value_str, "NOW")) return DateTime.now();
const hours: u16 = std.fmt.parseInt(u16, value_str[0..2], 10) catch 0; const hours: u8 = std.fmt.parseInt(u8, value_str[0..2], 10) catch 0;
const minutes: u16 = std.fmt.parseInt(u16, value_str[3..5], 10) catch 0; const minutes: u8 = std.fmt.parseInt(u8, value_str[3..5], 10) catch 0;
const seconds: u16 = if (value_str.len > 6) std.fmt.parseInt(u16, value_str[6..8], 10) catch 0 else 0; const seconds: u8 = if (value_str.len > 6) std.fmt.parseInt(u8, value_str[6..8], 10) catch 0 else 0;
const milliseconds: u16 = if (value_str.len > 9) std.fmt.parseInt(u16, value_str[9..13], 10) catch 0 else 0; const milliseconds: u16 = if (value_str.len > 9) std.fmt.parseInt(u16, value_str[9..13], 10) catch 0 else 0;
return DateTime.init(0, 0, 0, hours, minutes, seconds, milliseconds); return DateTime.init(0, 0, 0, hours, minutes, seconds, milliseconds);
@ -114,11 +114,11 @@ pub fn parseDatetime(value_str: []const u8) DateTime {
if (std.mem.eql(u8, value_str, "NOW")) return DateTime.now(); if (std.mem.eql(u8, value_str, "NOW")) return DateTime.now();
const year: u16 = std.fmt.parseInt(u16, value_str[0..4], 10) catch 0; const year: u16 = std.fmt.parseInt(u16, value_str[0..4], 10) catch 0;
const month: u16 = std.fmt.parseInt(u16, value_str[5..7], 10) catch 0; const month: u8 = std.fmt.parseInt(u8, value_str[5..7], 10) catch 0;
const day: u16 = std.fmt.parseInt(u16, value_str[8..10], 10) catch 0; const day: u8 = std.fmt.parseInt(u8, value_str[8..10], 10) catch 0;
const hours: u16 = std.fmt.parseInt(u16, value_str[11..13], 10) catch 0; const hours: u8 = std.fmt.parseInt(u8, value_str[11..13], 10) catch 0;
const minutes: u16 = std.fmt.parseInt(u16, value_str[14..16], 10) catch 0; const minutes: u8 = std.fmt.parseInt(u8, value_str[14..16], 10) catch 0;
const seconds: u16 = if (value_str.len > 17) std.fmt.parseInt(u16, value_str[17..19], 10) catch 0 else 0; const seconds: u8 = if (value_str.len > 17) std.fmt.parseInt(u8, value_str[17..19], 10) catch 0 else 0;
const milliseconds: u16 = if (value_str.len > 20) std.fmt.parseInt(u16, value_str[20..24], 10) catch 0 else 0; const milliseconds: u16 = if (value_str.len > 20) std.fmt.parseInt(u16, value_str[20..24], 10) catch 0 else 0;
return DateTime.init(year, month - 1, day - 1, hours, minutes, seconds, milliseconds); return DateTime.init(year, month - 1, day - 1, hours, minutes, seconds, milliseconds);

12
src/ziqlParser.zig
View File

@ -322,15 +322,19 @@ pub const Parser = struct {
var maps = std.ArrayList(std.StringHashMap(ConditionValue)).init(allocator); var maps = std.ArrayList(std.StringHashMap(ConditionValue)).init(allocator);
defer maps.deinit(); defer maps.deinit();
var local_arena = std.heap.ArenaAllocator.init(allocator);
defer local_arena.deinit();
const local_allocator = arena.allocator();
var data_map = std.StringHashMap(ConditionValue).init(allocator); var data_map = std.StringHashMap(ConditionValue).init(allocator);
defer data_map.deinit(); defer data_map.deinit();
while (true) { // I could multithread that as it do take a long time for big benchmark while (true) { // I could multithread that as it do take a long time for big benchmark
data_map.clearRetainingCapacity(); data_map.clearRetainingCapacity();
try self.parseNewData(allocator, &data_map, struct_name, &order, ordered); try self.parseNewData(local_allocator, &data_map, struct_name, &order, ordered);
ordered = true; ordered = true;
var error_message_buffer = std.ArrayList(u8).init(allocator); var error_message_buffer = std.ArrayList(u8).init(local_allocator);
defer error_message_buffer.deinit(); defer error_message_buffer.deinit();
const error_message_buffer_writer = error_message_buffer.writer(); const error_message_buffer_writer = error_message_buffer.writer();
@ -350,12 +354,12 @@ pub const Parser = struct {
); );
} }
maps.append(data_map.clone() catch return ZipponError.MemoryError) catch return ZipponError.MemoryError; maps.append(data_map.cloneWithAllocator(local_allocator) catch return ZipponError.MemoryError) catch return ZipponError.MemoryError;
if (maps.items.len >= 1_000) { if (maps.items.len >= 1_000) {
self.file_engine.addEntity(struct_name, maps.items, &buff.writer()) catch return ZipponError.CantWriteEntity; self.file_engine.addEntity(struct_name, maps.items, &buff.writer()) catch return ZipponError.CantWriteEntity;
for (maps.items) |*map| map.deinit();
maps.clearRetainingCapacity(); maps.clearRetainingCapacity();
_ = local_arena.reset(.retain_capacity);
} }
token = self.toker.last_token; token = self.toker.last_token;