2025-07-21 19:45:16 +00:00

1717 lines
64 KiB
HTML

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="canonical" href="https://MrBounty.github.io/ZipponDB/Technical%20docs/">
<link rel="prev" href="../Benchmark/">
<link rel="next" href="../build/">
<link rel="icon" href="../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.6.15">
<title>Technical - Documentation</title>
<link rel="stylesheet" href="../assets/stylesheets/main.342714a4.min.css">
<link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<link rel="stylesheet" href="../stylesheets/extra.css">
<script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
<link href="../assets/stylesheets/glightbox.min.css" rel="stylesheet"/><style>
html.glightbox-open { overflow: initial; height: 100%; }
.gslide-title { margin-top: 0px; user-select: text; }
.gslide-desc { color: #666; user-select: text; }
.gslide-image img { background: white; }
.gscrollbar-fixer { padding-right: 15px; }
.gdesc-inner { font-size: 0.75rem; }
body[data-md-color-scheme="slate"] .gdesc-inner { background: var(--md-default-bg-color);}
body[data-md-color-scheme="slate"] .gslide-title { color: var(--md-default-fg-color);}
body[data-md-color-scheme="slate"] .gslide-desc { color: var(--md-default-fg-color);}</style> <script src="../assets/javascripts/glightbox.min.js"></script></head>
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="amber">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#intro" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href=".." title="Documentation" class="md-header__button md-logo" aria-label="Documentation" data-md-component="logo">
<img src="../images/banner.png" alt="logo">
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
Documentation
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Technical
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="amber" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_0">
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
</label>
<input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="custom" data-md-color-accent="amber" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_1">
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_0" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12s-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
</label>
</form>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
<div class="md-search__suggest" data-md-component="search-suggest"></div>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://github.com/MrBounty/ZipponDB" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81"/></svg>
</div>
<div class="md-source__repository">
GitHub
</div>
</a>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href=".." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item">
<a href="../Quickstart/" class="md-tabs__link">
Quickstart
</a>
</li>
<li class="md-tabs__item">
<a href="../Schema/" class="md-tabs__link">
Learn
</a>
</li>
<li class="md-tabs__item md-tabs__item--active">
<a href="../Benchmark/" class="md-tabs__link">
Technical
</a>
</li>
<li class="md-tabs__item">
<a href="../about/" class="md-tabs__link">
About
</a>
</li>
<li class="md-tabs__item">
<a href="../Roadmap/" class="md-tabs__link">
Roadmap
</a>
</li>
<li class="md-tabs__item">
<a href="../release/" class="md-tabs__link">
Release
</a>
</li>
</ul>
</div>
</nav>
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href=".." title="Documentation" class="md-nav__button md-logo" aria-label="Documentation" data-md-component="logo">
<img src="../images/banner.png" alt="logo">
</a>
Documentation
</label>
<div class="md-nav__source">
<a href="https://github.com/MrBounty/ZipponDB" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81"/></svg>
</div>
<div class="md-source__repository">
GitHub
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href=".." class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../Quickstart/" class="md-nav__link">
<span class="md-ellipsis">
Quickstart
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_3" >
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
<span class="md-ellipsis">
Learn
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3">
<span class="md-nav__icon md-icon"></span>
Learn
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../Schema/" class="md-nav__link">
<span class="md-ellipsis">
Schema
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_3_2" >
<label class="md-nav__link" for="__nav_3_2" id="__nav_3_2_label" tabindex="0">
<span class="md-ellipsis">
ZipponQL
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_2">
<span class="md-nav__icon md-icon"></span>
ZipponQL
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../ziql/intro/" class="md-nav__link">
<span class="md-ellipsis">
Intro
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../ziql/vssql/" class="md-nav__link">
<span class="md-ellipsis">
Vs SQL
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../ziql/grab/" class="md-nav__link">
<span class="md-ellipsis">
GRAB
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../ziql/add/" class="md-nav__link">
<span class="md-ellipsis">
ADD
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../ziql/update/" class="md-nav__link">
<span class="md-ellipsis">
UPDATE
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../ziql/delete/" class="md-nav__link">
<span class="md-ellipsis">
DELETE
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../Data%20type/" class="md-nav__link">
<span class="md-ellipsis">
Data types
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../cli/" class="md-nav__link">
<span class="md-ellipsis">
Command Line Interface
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../logs/" class="md-nav__link">
<span class="md-ellipsis">
Logs
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../interface/" class="md-nav__link">
<span class="md-ellipsis">
Interface
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="">
<span class="md-ellipsis">
Technical
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_4">
<span class="md-nav__icon md-icon"></span>
Technical
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../Benchmark/" class="md-nav__link">
<span class="md-ellipsis">
Benchmark
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
Technical
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Technical
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#tokenizers" class="md-nav__link">
<span class="md-ellipsis">
Tokenizers
</span>
</a>
<nav class="md-nav" aria-label="Tokenizers">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#available-tokenizers" class="md-nav__link">
<span class="md-ellipsis">
Available Tokenizers
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#parser" class="md-nav__link">
<span class="md-ellipsis">
Parser
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#file-parsing" class="md-nav__link">
<span class="md-ellipsis">
File parsing
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#engines" class="md-nav__link">
<span class="md-ellipsis">
Engines
</span>
</a>
<nav class="md-nav" aria-label="Engines">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#dbengine" class="md-nav__link">
<span class="md-ellipsis">
DBEngine
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#fileengine" class="md-nav__link">
<span class="md-ellipsis">
FileEngine
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#schemaengine" class="md-nav__link">
<span class="md-ellipsis">
SchemaEngine
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#threadengine" class="md-nav__link">
<span class="md-ellipsis">
ThreadEngine
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#multi-threading" class="md-nav__link">
<span class="md-ellipsis">
Multi-threading
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#data-structures" class="md-nav__link">
<span class="md-ellipsis">
Data Structures
</span>
</a>
<nav class="md-nav" aria-label="Data Structures">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#additionaldata" class="md-nav__link">
<span class="md-ellipsis">
AdditionalData
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#filters" class="md-nav__link">
<span class="md-ellipsis">
Filters
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#condition" class="md-nav__link">
<span class="md-ellipsis">
Condition
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#newdata" class="md-nav__link">
<span class="md-ellipsis">
NewData
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#relationmap" class="md-nav__link">
<span class="md-ellipsis">
RelationMap
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#entitywriter" class="md-nav__link">
<span class="md-ellipsis">
EntityWriter
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../build/" class="md-nav__link">
<span class="md-ellipsis">
Build
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../ZipponData/" class="md-nav__link">
<span class="md-ellipsis">
ZipponData
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../Single_file/" class="md-nav__link">
<span class="md-ellipsis">
Single file
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../about/" class="md-nav__link">
<span class="md-ellipsis">
About
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../Roadmap/" class="md-nav__link">
<span class="md-ellipsis">
Roadmap
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../release/" class="md-nav__link">
<span class="md-ellipsis">
Release
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#tokenizers" class="md-nav__link">
<span class="md-ellipsis">
Tokenizers
</span>
</a>
<nav class="md-nav" aria-label="Tokenizers">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#available-tokenizers" class="md-nav__link">
<span class="md-ellipsis">
Available Tokenizers
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#parser" class="md-nav__link">
<span class="md-ellipsis">
Parser
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#file-parsing" class="md-nav__link">
<span class="md-ellipsis">
File parsing
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#engines" class="md-nav__link">
<span class="md-ellipsis">
Engines
</span>
</a>
<nav class="md-nav" aria-label="Engines">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#dbengine" class="md-nav__link">
<span class="md-ellipsis">
DBEngine
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#fileengine" class="md-nav__link">
<span class="md-ellipsis">
FileEngine
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#schemaengine" class="md-nav__link">
<span class="md-ellipsis">
SchemaEngine
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#threadengine" class="md-nav__link">
<span class="md-ellipsis">
ThreadEngine
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#multi-threading" class="md-nav__link">
<span class="md-ellipsis">
Multi-threading
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#data-structures" class="md-nav__link">
<span class="md-ellipsis">
Data Structures
</span>
</a>
<nav class="md-nav" aria-label="Data Structures">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#additionaldata" class="md-nav__link">
<span class="md-ellipsis">
AdditionalData
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#filters" class="md-nav__link">
<span class="md-ellipsis">
Filters
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#condition" class="md-nav__link">
<span class="md-ellipsis">
Condition
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#newdata" class="md-nav__link">
<span class="md-ellipsis">
NewData
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#relationmap" class="md-nav__link">
<span class="md-ellipsis">
RelationMap
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#entitywriter" class="md-nav__link">
<span class="md-ellipsis">
EntityWriter
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="intro">Intro</h1>
<p>TODO</p>
<p><strong><em>Note: Code snippets in this documentation are simplified examples and may not represent the actual codebase.</em></strong></p>
<h2 id="tokenizers">Tokenizers</h2>
<p>Tokenizers are responsible for converting a buffer string into a list of tokens. Each token has a <code>Tag</code> enum that represents its type, such as <code>equal</code> for the <code>=</code> symbol, and a <code>Loc</code> struct with start and end indices that represent its position in the buffer.</p>
<p>All tokenizers work similarly and are based on the <a href="https://github.com/ziglang/zig/blob/master/lib/std/zig/tokenizer.zig">zig tokenizer.</a> They have two main methods: next, which returns the next token, and getTokenSlice, which returns the slice of the buffer that represents the token.</p>
<p>Here's an example of how to use a tokenizer:
<div class="language-zig highlight"><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="kr">const</span><span class="w"> </span><span class="n">toker</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">Tokenizer</span><span class="p">.</span><span class="n">init</span><span class="p">(</span><span class="n">buff</span><span class="p">);</span>
</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="kr">const</span><span class="w"> </span><span class="n">token</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">toker</span><span class="p">.</span><span class="n">next</span><span class="p">();</span>
</span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="n">std</span><span class="p">.</span><span class="n">debug</span><span class="p">.</span><span class="n">print</span><span class="p">(</span><span class="s">&quot;{s}&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">.{</span><span class="n">toker</span><span class="p">.</span><span class="n">getTokenSlice</span><span class="p">(</span><span class="n">token</span><span class="p">)});</span>
</span></code></pre></div></p>
<p>Tokenizers are often used in a loop until the <code>end</code> tag is reached. In each iteration, the next token is retrieved and processed based on its tag. Here's a simple example:
<div class="language-zig highlight"><pre><span></span><code><span id="__span-1-1"><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="kr">const</span><span class="w"> </span><span class="n">toker</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">Tokenizer</span><span class="p">.</span><span class="n">init</span><span class="p">(</span><span class="n">buff</span><span class="p">);</span>
</span><span id="__span-1-2"><a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="kr">var</span><span class="w"> </span><span class="n">token</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">toker</span><span class="p">.</span><span class="n">next</span><span class="p">();</span>
</span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a><span class="k">while</span><span class="w"> </span><span class="p">(</span><span class="n">token</span><span class="p">.</span><span class="n">tag</span><span class="w"> </span><span class="o">!=</span><span class="w"> </span><span class="p">.</span><span class="n">end</span><span class="p">)</span><span class="w"> </span><span class="o">:</span><span class="w"> </span><span class="p">(</span><span class="n">token</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">toker</span><span class="p">.</span><span class="n">next</span><span class="p">())</span><span class="w"> </span><span class="k">switch</span><span class="w"> </span><span class="p">(</span><span class="n">token</span><span class="p">.</span><span class="n">tag</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
</span><span id="__span-1-4"><a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a><span class="w"> </span><span class="p">.</span><span class="n">equal</span><span class="w"> </span><span class="o">=&gt;</span><span class="w"> </span><span class="n">std</span><span class="p">.</span><span class="n">debug</span><span class="p">.</span><span class="n">print</span><span class="p">(</span><span class="s">&quot;{s}&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">.{</span><span class="n">toker</span><span class="p">.</span><span class="n">getTokenSlice</span><span class="p">(</span><span class="n">token</span><span class="p">)}),</span>
</span><span id="__span-1-5"><a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a><span class="w"> </span><span class="k">else</span><span class="w"> </span><span class="o">=&gt;</span><span class="w"> </span><span class="p">{},</span>
</span><span id="__span-1-6"><a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a><span class="p">}</span>
</span></code></pre></div></p>
<h3 id="available-tokenizers">Available Tokenizers</h3>
<p>There are four different tokenizers in ZipponDB:</p>
<ul>
<li><strong>ZiQL:</strong> Tokenizer for the query language.</li>
<li><strong>cli:</strong> Tokenizer the commands.</li>
<li><strong>schema:</strong> Tokenizer for the schema file.</li>
</ul>
<p>Each tokenizer has its own set of tags and parsing rules, but they all work similarly.</p>
<h2 id="parser">Parser</h2>
<p>Parsers are the next step after tokenization. They take tokens and perform actions or raise errors. There are three parsers in ZipponDB: one for ZiQL, one for schema files, and one for CLI commands.</p>
<p>A parser has a <code>State</code> enum and a <code>Tokenizer</code> instance as members, and a parse method that processes tokens until the <code>end</code> state is reached.</p>
<p>Here's an example of how a parser works:
<div class="language-zig highlight"><pre><span></span><code><span id="__span-2-1"><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="kr">var</span><span class="w"> </span><span class="n">state</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">.</span><span class="n">start</span><span class="p">;</span>
</span><span id="__span-2-2"><a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a><span class="kr">var</span><span class="w"> </span><span class="n">token</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">self</span><span class="p">.</span><span class="n">toker</span><span class="p">.</span><span class="n">next</span><span class="p">();</span>
</span><span id="__span-2-3"><a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a><span class="k">while</span><span class="w"> </span><span class="p">(</span><span class="n">state</span><span class="w"> </span><span class="o">!=</span><span class="w"> </span><span class="p">.</span><span class="n">end</span><span class="p">)</span><span class="w"> </span><span class="o">:</span><span class="w"> </span><span class="p">(</span><span class="n">token</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">self</span><span class="p">.</span><span class="n">toker</span><span class="p">.</span><span class="n">next</span><span class="p">())</span><span class="w"> </span><span class="k">switch</span><span class="w"> </span><span class="p">(</span><span class="n">state</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
</span><span id="__span-2-4"><a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a><span class="w"> </span><span class="p">.</span><span class="n">start</span><span class="w"> </span><span class="o">=&gt;</span><span class="w"> </span><span class="k">switch</span><span class="w"> </span><span class="p">(</span><span class="n">token</span><span class="p">.</span><span class="n">tag</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
</span><span id="__span-2-5"><a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a><span class="w"> </span><span class="p">.</span><span class="n">identifier</span><span class="w"> </span><span class="o">=&gt;</span><span class="w"> </span><span class="n">self</span><span class="p">.</span><span class="n">addStruct</span><span class="p">(</span><span class="n">token</span><span class="p">),</span>
</span><span id="__span-2-6"><a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a><span class="w"> </span><span class="k">else</span><span class="w"> </span><span class="o">=&gt;</span><span class="w"> </span><span class="n">printError</span><span class="p">(</span><span class="s">&quot;Error: Expected a struct name.&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">token</span><span class="p">),</span>
</span><span id="__span-2-7"><a id="__codelineno-2-7" name="__codelineno-2-7" href="#__codelineno-2-7"></a><span class="w"> </span><span class="p">},</span>
</span><span id="__span-2-8"><a id="__codelineno-2-8" name="__codelineno-2-8" href="#__codelineno-2-8"></a><span class="w"> </span><span class="k">else</span><span class="w"> </span><span class="o">=&gt;</span><span class="w"> </span><span class="p">{},</span>
</span><span id="__span-2-9"><a id="__codelineno-2-9" name="__codelineno-2-9" href="#__codelineno-2-9"></a><span class="p">}</span>
</span></code></pre></div></p>
<p>The parser's state is updated based on the combination of the current state and token tag. This process continues until the <code>end</code> state is reached.</p>
<p>The ZiQL parser uses different methods for parsing:</p>
<ul>
<li><code>parse</code>: The main parsing method that calls other methods.</li>
<li><code>parseFilter</code>: Creates a filter tree from the query.</li>
<li><code>parseCondition</code>: Creates a condition from a part of the query.</li>
<li><code>parseAdditionalData</code>: Populates additional data from the query.</li>
<li><code>parseNewData</code>: Returns a string map with key-value pairs from the query.</li>
<li><code>parseOption</code>: Not implemented yet.</li>
</ul>
<h2 id="file-parsing">File parsing</h2>
<p>File parsing is done through a small library that I did named <a href="https://github.com/MrBounty/ZipponData">ZipponData</a>. </p>
<p>It is minimal and fast, it can parse 1_000_000 entity in 0.3s on one thread
on a 7 7800X3D at around 4.5GHz with a Samsung SSD 980 PRO 2TB (up to 7,000/5,100MB/s for read/write speed).</p>
<p>To read a file, you create an iterator for a single file and then you can iterate with <code>.next()</code>. It will return an array of <code>Data</code>. This make everything very easy to use.</p>
<div class="language-zig highlight"><pre><span></span><code><span id="__span-3-1"><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="kr">const</span><span class="w"> </span><span class="n">std</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">@import</span><span class="p">(</span><span class="s">&quot;std&quot;</span><span class="p">);</span>
</span><span id="__span-3-2"><a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a>
</span><span id="__span-3-3"><a id="__codelineno-3-3" name="__codelineno-3-3" href="#__codelineno-3-3"></a><span class="kr">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="n">main</span><span class="p">()</span><span class="w"> </span><span class="o">!</span><span class="kt">void</span><span class="w"> </span><span class="p">{</span>
</span><span id="__span-3-4"><a id="__codelineno-3-4" name="__codelineno-3-4" href="#__codelineno-3-4"></a><span class="w"> </span><span class="kr">const</span><span class="w"> </span><span class="n">allocator</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">std</span><span class="p">.</span><span class="n">testing</span><span class="p">.</span><span class="n">allocator</span><span class="p">;</span>
</span><span id="__span-3-5"><a id="__codelineno-3-5" name="__codelineno-3-5" href="#__codelineno-3-5"></a>
</span><span id="__span-3-6"><a id="__codelineno-3-6" name="__codelineno-3-6" href="#__codelineno-3-6"></a><span class="w"> </span><span class="c1">// 0. Make a temporary directory</span>
</span><span id="__span-3-7"><a id="__codelineno-3-7" name="__codelineno-3-7" href="#__codelineno-3-7"></a><span class="w"> </span><span class="k">try</span><span class="w"> </span><span class="n">std</span><span class="p">.</span><span class="n">fs</span><span class="p">.</span><span class="n">cwd</span><span class="p">().</span><span class="n">makeDir</span><span class="p">(</span><span class="s">&quot;tmp&quot;</span><span class="p">);</span>
</span><span id="__span-3-8"><a id="__codelineno-3-8" name="__codelineno-3-8" href="#__codelineno-3-8"></a><span class="w"> </span><span class="kr">const</span><span class="w"> </span><span class="n">dir</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="k">try</span><span class="w"> </span><span class="n">std</span><span class="p">.</span><span class="n">fs</span><span class="p">.</span><span class="n">cwd</span><span class="p">().</span><span class="n">openDir</span><span class="p">(</span><span class="s">&quot;tmp&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">.{});</span>
</span><span id="__span-3-9"><a id="__codelineno-3-9" name="__codelineno-3-9" href="#__codelineno-3-9"></a>
</span><span id="__span-3-10"><a id="__codelineno-3-10" name="__codelineno-3-10" href="#__codelineno-3-10"></a><span class="w"> </span><span class="c1">// 1. Create a file</span>
</span><span id="__span-3-11"><a id="__codelineno-3-11" name="__codelineno-3-11" href="#__codelineno-3-11"></a><span class="w"> </span><span class="k">try</span><span class="w"> </span><span class="n">createFile</span><span class="p">(</span><span class="s">&quot;test&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">dir</span><span class="p">);</span>
</span><span id="__span-3-12"><a id="__codelineno-3-12" name="__codelineno-3-12" href="#__codelineno-3-12"></a>
</span><span id="__span-3-13"><a id="__codelineno-3-13" name="__codelineno-3-13" href="#__codelineno-3-13"></a><span class="w"> </span><span class="c1">// 2. Create some Data</span>
</span><span id="__span-3-14"><a id="__codelineno-3-14" name="__codelineno-3-14" href="#__codelineno-3-14"></a><span class="w"> </span><span class="kr">const</span><span class="w"> </span><span class="n">data</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">[</span><span class="n">_</span><span class="p">]</span><span class="n">Data</span><span class="p">{</span>
</span><span id="__span-3-15"><a id="__codelineno-3-15" name="__codelineno-3-15" href="#__codelineno-3-15"></a><span class="w"> </span><span class="n">Data</span><span class="p">.</span><span class="n">initInt</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span>
</span><span id="__span-3-16"><a id="__codelineno-3-16" name="__codelineno-3-16" href="#__codelineno-3-16"></a><span class="w"> </span><span class="n">Data</span><span class="p">.</span><span class="n">initFloat</span><span class="p">(</span><span class="mf">3.14159</span><span class="p">),</span>
</span><span id="__span-3-17"><a id="__codelineno-3-17" name="__codelineno-3-17" href="#__codelineno-3-17"></a><span class="w"> </span><span class="n">Data</span><span class="p">.</span><span class="n">initInt</span><span class="p">(</span><span class="o">-</span><span class="mi">5</span><span class="p">),</span>
</span><span id="__span-3-18"><a id="__codelineno-3-18" name="__codelineno-3-18" href="#__codelineno-3-18"></a><span class="w"> </span><span class="n">Data</span><span class="p">.</span><span class="n">initStr</span><span class="p">(</span><span class="s">&quot;Hello world&quot;</span><span class="p">),</span>
</span><span id="__span-3-19"><a id="__codelineno-3-19" name="__codelineno-3-19" href="#__codelineno-3-19"></a><span class="w"> </span><span class="n">Data</span><span class="p">.</span><span class="n">initBool</span><span class="p">(</span><span class="kc">true</span><span class="p">),</span>
</span><span id="__span-3-20"><a id="__codelineno-3-20" name="__codelineno-3-20" href="#__codelineno-3-20"></a><span class="w"> </span><span class="n">Data</span><span class="p">.</span><span class="n">initUnix</span><span class="p">(</span><span class="mi">2021</span><span class="p">),</span>
</span><span id="__span-3-21"><a id="__codelineno-3-21" name="__codelineno-3-21" href="#__codelineno-3-21"></a><span class="w"> </span><span class="p">};</span>
</span><span id="__span-3-22"><a id="__codelineno-3-22" name="__codelineno-3-22" href="#__codelineno-3-22"></a>
</span><span id="__span-3-23"><a id="__codelineno-3-23" name="__codelineno-3-23" href="#__codelineno-3-23"></a><span class="w"> </span><span class="c1">// 3. Create a DataWriter</span>
</span><span id="__span-3-24"><a id="__codelineno-3-24" name="__codelineno-3-24" href="#__codelineno-3-24"></a><span class="w"> </span><span class="kr">var</span><span class="w"> </span><span class="n">dwriter</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="k">try</span><span class="w"> </span><span class="n">DataWriter</span><span class="p">.</span><span class="n">init</span><span class="p">(</span><span class="s">&quot;test&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">dir</span><span class="p">);</span>
</span><span id="__span-3-25"><a id="__codelineno-3-25" name="__codelineno-3-25" href="#__codelineno-3-25"></a><span class="w"> </span><span class="k">defer</span><span class="w"> </span><span class="n">dwriter</span><span class="p">.</span><span class="n">deinit</span><span class="p">();</span><span class="w"> </span><span class="c1">// This just close the file</span>
</span><span id="__span-3-26"><a id="__codelineno-3-26" name="__codelineno-3-26" href="#__codelineno-3-26"></a>
</span><span id="__span-3-27"><a id="__codelineno-3-27" name="__codelineno-3-27" href="#__codelineno-3-27"></a><span class="w"> </span><span class="c1">// 4. Write some data</span>
</span><span id="__span-3-28"><a id="__codelineno-3-28" name="__codelineno-3-28" href="#__codelineno-3-28"></a><span class="w"> </span><span class="k">try</span><span class="w"> </span><span class="n">dwriter</span><span class="p">.</span><span class="n">write</span><span class="p">(</span><span class="o">&amp;</span><span class="n">data</span><span class="p">);</span>
</span><span id="__span-3-29"><a id="__codelineno-3-29" name="__codelineno-3-29" href="#__codelineno-3-29"></a><span class="w"> </span><span class="k">try</span><span class="w"> </span><span class="n">dwriter</span><span class="p">.</span><span class="n">write</span><span class="p">(</span><span class="o">&amp;</span><span class="n">data</span><span class="p">);</span>
</span><span id="__span-3-30"><a id="__codelineno-3-30" name="__codelineno-3-30" href="#__codelineno-3-30"></a><span class="w"> </span><span class="k">try</span><span class="w"> </span><span class="n">dwriter</span><span class="p">.</span><span class="n">flush</span><span class="p">();</span><span class="w"> </span><span class="c1">// Dont forget to flush !</span>
</span><span id="__span-3-31"><a id="__codelineno-3-31" name="__codelineno-3-31" href="#__codelineno-3-31"></a>
</span><span id="__span-3-32"><a id="__codelineno-3-32" name="__codelineno-3-32" href="#__codelineno-3-32"></a><span class="w"> </span><span class="c1">// 5. Create a schema</span>
</span><span id="__span-3-33"><a id="__codelineno-3-33" name="__codelineno-3-33" href="#__codelineno-3-33"></a><span class="w"> </span><span class="c1">// A schema is how the iterator will parse the file. </span>
</span><span id="__span-3-34"><a id="__codelineno-3-34" name="__codelineno-3-34" href="#__codelineno-3-34"></a><span class="w"> </span><span class="c1">// If you are wrong here, it will return wrong/random data</span>
</span><span id="__span-3-35"><a id="__codelineno-3-35" name="__codelineno-3-35" href="#__codelineno-3-35"></a><span class="w"> </span><span class="c1">// And most likely an error when iterating in the while loop</span>
</span><span id="__span-3-36"><a id="__codelineno-3-36" name="__codelineno-3-36" href="#__codelineno-3-36"></a><span class="w"> </span><span class="kr">const</span><span class="w"> </span><span class="n">schema</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="o">&amp;</span><span class="p">[</span><span class="n">_</span><span class="p">]</span><span class="n">DType</span><span class="p">{</span>
</span><span id="__span-3-37"><a id="__codelineno-3-37" name="__codelineno-3-37" href="#__codelineno-3-37"></a><span class="w"> </span><span class="p">.</span><span class="n">Int</span><span class="p">,</span>
</span><span id="__span-3-38"><a id="__codelineno-3-38" name="__codelineno-3-38" href="#__codelineno-3-38"></a><span class="w"> </span><span class="p">.</span><span class="n">Float</span><span class="p">,</span>
</span><span id="__span-3-39"><a id="__codelineno-3-39" name="__codelineno-3-39" href="#__codelineno-3-39"></a><span class="w"> </span><span class="p">.</span><span class="n">Int</span><span class="p">,</span>
</span><span id="__span-3-40"><a id="__codelineno-3-40" name="__codelineno-3-40" href="#__codelineno-3-40"></a><span class="w"> </span><span class="p">.</span><span class="n">Str</span><span class="p">,</span>
</span><span id="__span-3-41"><a id="__codelineno-3-41" name="__codelineno-3-41" href="#__codelineno-3-41"></a><span class="w"> </span><span class="p">.</span><span class="n">Bool</span><span class="p">,</span>
</span><span id="__span-3-42"><a id="__codelineno-3-42" name="__codelineno-3-42" href="#__codelineno-3-42"></a><span class="w"> </span><span class="p">.</span><span class="n">Unix</span><span class="p">,</span>
</span><span id="__span-3-43"><a id="__codelineno-3-43" name="__codelineno-3-43" href="#__codelineno-3-43"></a><span class="w"> </span><span class="p">};</span>
</span><span id="__span-3-44"><a id="__codelineno-3-44" name="__codelineno-3-44" href="#__codelineno-3-44"></a>
</span><span id="__span-3-45"><a id="__codelineno-3-45" name="__codelineno-3-45" href="#__codelineno-3-45"></a><span class="w"> </span><span class="c1">// 6. Create a DataIterator</span>
</span><span id="__span-3-46"><a id="__codelineno-3-46" name="__codelineno-3-46" href="#__codelineno-3-46"></a><span class="w"> </span><span class="kr">var</span><span class="w"> </span><span class="n">iter</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="k">try</span><span class="w"> </span><span class="n">DataIterator</span><span class="p">.</span><span class="n">init</span><span class="p">(</span><span class="n">allocator</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;test&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">dir</span><span class="p">,</span><span class="w"> </span><span class="n">schema</span><span class="p">);</span>
</span><span id="__span-3-47"><a id="__codelineno-3-47" name="__codelineno-3-47" href="#__codelineno-3-47"></a><span class="w"> </span><span class="k">defer</span><span class="w"> </span><span class="n">iter</span><span class="p">.</span><span class="n">deinit</span><span class="p">();</span>
</span><span id="__span-3-48"><a id="__codelineno-3-48" name="__codelineno-3-48" href="#__codelineno-3-48"></a>
</span><span id="__span-3-49"><a id="__codelineno-3-49" name="__codelineno-3-49" href="#__codelineno-3-49"></a><span class="w"> </span><span class="c1">// 7. Iterate over data</span>
</span><span id="__span-3-50"><a id="__codelineno-3-50" name="__codelineno-3-50" href="#__codelineno-3-50"></a><span class="w"> </span><span class="k">while</span><span class="w"> </span><span class="p">(</span><span class="k">try</span><span class="w"> </span><span class="n">iter</span><span class="p">.</span><span class="n">next</span><span class="p">())</span><span class="w"> </span><span class="o">|</span><span class="n">row</span><span class="o">|</span><span class="w"> </span><span class="p">{</span>
</span><span id="__span-3-51"><a id="__codelineno-3-51" name="__codelineno-3-51" href="#__codelineno-3-51"></a><span class="w"> </span><span class="n">std</span><span class="p">.</span><span class="n">debug</span><span class="p">.</span><span class="n">print</span><span class="p">(</span><span class="s">&quot;Row: {any}</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">.{</span><span class="w"> </span><span class="n">row</span><span class="w"> </span><span class="p">});</span>
</span><span id="__span-3-52"><a id="__codelineno-3-52" name="__codelineno-3-52" href="#__codelineno-3-52"></a><span class="w"> </span><span class="p">}</span>
</span><span id="__span-3-53"><a id="__codelineno-3-53" name="__codelineno-3-53" href="#__codelineno-3-53"></a>
</span><span id="__span-3-54"><a id="__codelineno-3-54" name="__codelineno-3-54" href="#__codelineno-3-54"></a><span class="w"> </span><span class="c1">// 8. Delete the file (Optional ofc)</span>
</span><span id="__span-3-55"><a id="__codelineno-3-55" name="__codelineno-3-55" href="#__codelineno-3-55"></a><span class="w"> </span><span class="k">try</span><span class="w"> </span><span class="n">deleteFile</span><span class="p">(</span><span class="s">&quot;test&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">dir</span><span class="p">);</span>
</span><span id="__span-3-56"><a id="__codelineno-3-56" name="__codelineno-3-56" href="#__codelineno-3-56"></a><span class="w"> </span><span class="k">try</span><span class="w"> </span><span class="n">std</span><span class="p">.</span><span class="n">fs</span><span class="p">.</span><span class="n">cwd</span><span class="p">().</span><span class="n">deleteDir</span><span class="p">(</span><span class="s">&quot;tmp&quot;</span><span class="p">);</span>
</span><span id="__span-3-57"><a id="__codelineno-3-57" name="__codelineno-3-57" href="#__codelineno-3-57"></a><span class="p">}</span>
</span></code></pre></div>
<h2 id="engines">Engines</h2>
<p>ZipponDB segregate responsability with Engines.</p>
<p>For example the <code>FileEngine</code> is the only place where files used, for both writting and reading. This simplify refactoring, testing, etc.</p>
<h3 id="dbengine">DBEngine</h3>
<p>This is just a wrapper around all other Engines to keep them at the same place. This doesnt do anything except storing other Engines.</p>
<p>This can be find in <code>main.zig</code>, in the <code>main</code> function.</p>
<h3 id="fileengine">FileEngine</h3>
<p>The <code>FileEngine</code> is responsible for managing files, including reading and writing.</p>
<p>Most methods will parse all files of a struct and evaluate them with a filter and do stuff if <code>true</code>. For example <code>parseEntities</code> will parse all entities and if the filter return <code>true</code>,
will write using the writer argument a JSON object with the entity's data.</p>
<p>Those methods are usually sperated into 2 methods. The main one and a <code>OneFile</code> version, e.g. <code>parseEntitiesOneFile</code>. The main one will call a thread for each file using multiple <code>OneFile</code> version.
This is how multi-threading is done.</p>
<h3 id="schemaengine">SchemaEngine</h3>
<p>The <code>SchemaEngine</code> manage everything related to schemas. </p>
<p>This is mostly used to store a list of <code>SchemaStruct</code>, with is just one struct as defined in the schema. With all member names, data types, links, etc.</p>
<p>This is also here that I store the <code>UUIDFileIndex</code>, that is a map of UUID to file index. So I can quickly check if a UUID exist and in which file it is store.
This work well but use a bit too much memory for me, around 220MB for 1_000_000 entities. I tried doing a Radix Trie but it doesn't use that much less memory, maybe I did a mistake somewhere.</p>
<h3 id="threadengine">ThreadEngine</h3>
<p>The <code>ThreadEngine</code> manage the thread pool of the database.</p>
<p>This is also where is stored the <code>ThreadSyncContext</code> that is use for each <code>OneFile</code> version of parsing methods in the <code>FileEngine</code>. This is the only atomix value currently used in the database.</p>
<h2 id="multi-threading">Multi-threading</h2>
<p>ZipponDB uses multi-threading to improve performance. Each struct is saved in multiple <code>.zid</code> files, and a thread pool is used to process files concurrently. Each thread has its own buffered writer, and the results are concatenated and sent once all threads finish.</p>
<p>The only shared atomic values between threads are the number of found structs and the number of finished threads. This approach keeps things simple and easy to implement, avoiding parallel threads accessing the same file.</p>
<h2 id="data-structures">Data Structures</h2>
<h3 id="additionaldata">AdditionalData</h3>
<p>AdditionalData keep what is between <code>[]</code>. It is composed of 2 struct <code>AdditionalData</code> and <code>AdditionalDataMember</code>.</p>
<p><code>AdditionalDataMember</code> have the name of the member, it's position in the schema file and an <code>AdditionalData</code>.</p>
<p><code>AdditionalData</code> have a limit (the first part <code>[100]</code>), and a list of <code>AdditionalDataMember</code>.</p>
<h3 id="filters">Filters</h3>
<p>A filter is a series of condition. It use a tree approach, by that I mean the filter have a root node that is either a condition or have 2 others nodes (left and right).</p>
<p>For example the filter <code>{name = 'Bob'}</code> have one root node that is the condition. So when I evaluate the struct, I just check this condition.</p>
<p>Now for like <code>{name = 'Bob' AND age &gt; 0}</code>, the root node have as left node the condition <code>name = 'Bob'</code> and right node the condition <code>age &gt; 0</code>.</p>
<p>To look like that:
<div class="language-text highlight"><pre><span></span><code><span id="__span-4-1"><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a> AND
</span><span id="__span-4-2"><a id="__codelineno-4-2" name="__codelineno-4-2" href="#__codelineno-4-2"></a> / \
</span><span id="__span-4-3"><a id="__codelineno-4-3" name="__codelineno-4-3" href="#__codelineno-4-3"></a> OR OR
</span><span id="__span-4-4"><a id="__codelineno-4-4" name="__codelineno-4-4" href="#__codelineno-4-4"></a> / \ / \
</span><span id="__span-4-5"><a id="__codelineno-4-5" name="__codelineno-4-5" href="#__codelineno-4-5"></a> name name age age
</span><span id="__span-4-6"><a id="__codelineno-4-6" name="__codelineno-4-6" href="#__codelineno-4-6"></a> =&#39;A&#39; =&#39;B&#39; &gt;80 &lt;20
</span></code></pre></div></p>
<h3 id="condition">Condition</h3>
<p>A condition is part of filters. It is one 'unit of condition'. For example <code>name = 'Bob'</code> is one condition.
<code>name = 'Bob' and age &gt; 0</code> are 2 conditions and one filter. It is created inside <code>parseCondition</code> in the <code>ziqlParser</code>.</p>
<p>A condition have those infos:</p>
<ul>
<li>value: ConditionValue. E.g. <code>32</code></li>
<li>operation: ComparisonOperator. E.g. <code>equal</code> or <code>in</code></li>
<li>data_type: DataType. E.g. <code>int</code> or <code>str</code></li>
<li>data_index: usize. This is the index in when parsing returned by zid <code>DataIterator</code></li>
</ul>
<h3 id="newdata">NewData</h3>
<p>NewData is a map with member name as key and ConditionValue as value, it is created when parsing and is use to add data into a file.
I transform ConditionValue into Zid Data. Maybe I can directly do a map member name -&gt; zid Data ?</p>
<h3 id="relationmap">RelationMap</h3>
<p>A <code>RelationMap</code> is use when I need to return relationship. Let's say we have this query <code>GRAB User [orders [date]]</code>.</p>
<p>The <code>RelationMap</code> have a struct_name, here Order. A member name, here orders. A map with UUID as key and a string as value.</p>
<p>When I first init the map, I am parsing the first struct (here User). So it populate the map with empty string for entities that I want to return.
Here it will be UUID of Order. </p>
<p>Then I parse Order file and add the string to the right UUID, skipping UUID that are not in the map.
Once that done, I parse the JSON response that I generated when parsing User. Where a relationship should be, there is <code>{&lt;|[16]u8|&gt;}</code>, where <code>[16]u8</code> is the UUID of the entity that shoud be here.
So now I can just replace it by the right key in the map.</p>
<h2 id="entitywriter">EntityWriter</h2>
<p>This is responsable to transform the raw Data into a JSON, Table or other output format to send send to end user.
Basically the last step before sending.</p>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
Back to top
</button>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"base": "..", "features": ["search.suggest", "navigation.tabs", "navigation.sections", "navigation.expand", "navigation.top", "content.code.annotate"], "search": "../assets/javascripts/workers/search.d50fe291.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../assets/javascripts/bundle.56ea9cef.min.js"></script>
<script id="init-glightbox">const lightbox = GLightbox({"touchNavigation": true, "loop": false, "zoomable": true, "draggable": true, "openEffect": "zoom", "closeEffect": "zoom", "slideEffect": "slide"});
document$.subscribe(() => { lightbox.reload() });
</script></body>
</html>