From 280b3b3c3a2acaca86cc3c835cff33bc7f318e1f Mon Sep 17 00:00:00 2001 From: MrBounty Date: Mon, 4 Nov 2024 22:52:44 +0100 Subject: [PATCH] Update docs --- Benchmark.md | 1 + README.md | 27 ++++++++++++++++++++------- Technical docs.md | 23 +++++++++++++++++++---- build.zig.zon | 4 ++-- test_runner.zig | 7 ------- 5 files changed, 42 insertions(+), 20 deletions(-) diff --git a/Benchmark.md b/Benchmark.md index 44d55f5..ae6fbb5 100644 --- a/Benchmark.md +++ b/Benchmark.md @@ -53,6 +53,7 @@ This take 127MB space on disk, sperated into 24 files of 5.2MB ## TODO +- [ ] Benchmark per files size, to find the optimal one. For 10kB, 5MB, 100MB, 1GB - [ ] Create a build command to benchmark. For 1_000, 1_000_000, 50_000_000 users - [ ] Create a random dataset - [ ] Do simple query, get average and +- time by set of 25 query diff --git a/README.md b/README.md index e39bba5..ca8553c 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,8 @@ Comment ( ***Note: `[]` before the type means a list/array of this type.*** +***Note: Members order matter for now!*** + ### Migration to a new schema - Not yet implemented In the future, you will be able to update the schema, such as adding a new member to a struct, and update the database. For the moment, you can't change the schema once it's initialized. @@ -144,11 +146,24 @@ Here an example where I create a new `Comment` that I then append to the list of ADD Comment (content='Hello world', at=NOW, like_by=[]) => added_comment => UPDATE User {id = '000'} TO (comments APPEND added_comment) ``` -The name between `=>` is the variable name of the list of UUID used for the next queries, you can have multiple one if the link has more than 2 queries. You can also just use one `=>` but the list of UUID is discarded in that case. +The name between `=>` is the variable name of the list of UUID used for the next queries, you can have multiple one if the link has more than 2 queries. +You can also just use one `=>` but the list of UUID is discarded in that case. + +This can be use with GRAB too. So you can create variable before making the query. Here an example: +```js +GRAB User {name = 'Bob'} => bobs => +GRAB User {age > 18} => adults => +GRAB User {IN adults AND !IN bobs} +``` + +Which is the same as: +```js +GRAB User {name != 'Bob' AND age > 18} +``` # Data types -There is 5 data types for the moment: +There is 8 data types: - `int`: 64 bit integer - `float`: 64 bit float. Need to have a dot, `1.` is a float `1` is an integer. - `bool`: Boolean, can be `true` or `false` @@ -160,8 +175,6 @@ There is 5 data types for the moment: All data types can be an array of those types using `[]` in front of it. So `[]int` is an array of integer. -All data types can also be `null`. Except arrays that can only be empty. - # Why I created it ? Well, the first reason is to learn both Zig and databases. @@ -189,10 +202,10 @@ TODO: Create a tech doc of what is happening inside. #### v0.2 - Usable - [ ] Relationships -- [ ] Optimized data file +- [X] Custom data file - [X] Date - [ ] Linked query -- [ ] Optimization +- [ ] Query optimization - [X] Logs #### v0.3 - QoL @@ -239,4 +252,4 @@ TODO: Create a tech doc of what is happening inside. - [ ] Schema visualization - [ ] Dashboard metrics -Let's see where it (or my brain) start explode ;) +Let's see where it (or my brain) start to explode ;) diff --git a/Technical docs.md b/Technical docs.md index 82c6f93..58aeb3e 100644 --- a/Technical docs.md +++ b/Technical docs.md @@ -93,9 +93,9 @@ And that's basicly it, the entire `Parser` work like that. It is fairly easy to Note that the `ZiQLParser` use different methods for parsing: - **parse:** The main one that will then use the other. -- **parseFilter:** This will populate an array of `UUID` based on what is between `{}`. -- **parseCondition:** Create a `Condition` struct based on a part of what is between `{}`. E.g. `name = 'Bob'`. -- **parseAdditionalData:** Populate the `AdditionalData` struct that represent what is between `[]`. +- **parseFilter:** This will create a `Filter`, this is a tree that contain all condition in the query, what is between `{}`. +- **parseCondition:** Create a `Condition` based on a part of what is between `{}`. E.g. `name = 'Bob'`. +- **parseAdditionalData:** Populate the `AdditionalData` that represent what is between `[]`. - **parseNewData:** Return a string map with key as member name and value as value of what is between `()`. E.g. `(name = 'Bob')` will return a map with one key `name` with the value `Bob`. - **parseOption:** Not done yet. Parse what is between `||` @@ -103,4 +103,19 @@ Note that the `ZiQLParser` use different methods for parsing: The `FileEngine` is that is managing files, everything that need to read or write into files is here. -I am not goind into too much detail here as I think this will change in the futur. +I am not goind into too much detail here as I think this will change in the future. + +# Multi-threading + +How do I do multi-threading ? Basically all struct are saved in multiples `.zid` files. Each files have +a size limit defined in the config and a new one is created when no previous one is found with space left. + +When I run a GRAB query and parse all files and evaluate each struct, I use a thread pool and give a file +to each thread. Each thread have it's own buffered writer and once all finished, I concatenate all writer +and send it. + +The only atomic value share by all threads are the number of founded struct (to stop thread if enough are found when +[10] is use). And the number of finished thread, so I know when I can concatenate and send stuffs. + +Like that this keep things simple and easy to implement. I dont have parallel thread that run different +that need to access the same file. diff --git a/build.zig.zon b/build.zig.zon index ecdd1eb..a5b8cd8 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -3,8 +3,8 @@ .version = "0.1.4", .dependencies = .{ .ZipponData = .{ - .url = "git+https://github.com/MrBounty/ZipponData#2ec9cc00e0d798e741d63f91cde18af0f9bf1bce", - .hash = "12206c4cac549a5d1beab62fe1c45388cec0bcc5aac96da8175eccd8abbeb6d41913", + .url = "git+https://github.com/MrBounty/ZipponData#237a1f546e8e0bd68c786081ef454694244e6221", + .hash = "12207d024d13697ab989ec54bbb3e24e24485da5ceb29f91101cacf43c98aac30ca4", }, }, .paths = .{ diff --git a/test_runner.zig b/test_runner.zig index 6824108..ffb58d0 100644 --- a/test_runner.zig +++ b/test_runner.zig @@ -274,13 +274,6 @@ const Env = struct { } }; -pub fn panic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, ret_addr: ?usize) noreturn { - if (current_test) |ct| { - std.debug.print("\x1b[31m{s}\npanic running \"{s}\"\n{s}\x1b[0m\n", .{ BORDER, ct, BORDER }); - } - std.builtin.panic(msg, error_return_trace, ret_addr); -} - fn isUnnamed(t: std.builtin.TestFn) bool { const marker = ".test_"; const test_name = t.name;