Autodoc: new decl search system (#15475)

New search system is based on a Radix Tree. The Radix Tree contains a shallow list of all decl names (ie no paths), plus some suffixes, split by following the official style guide (eg "HashMapUnmanaged" also produces "MapUnmanaged" and "Unmanaged", same with snake_case and camelCase names).

Additionally, the search system uses the decl graph data to recognize hierarchical relationships between decls, allowing you to zero on a target namespace for search. As an example "fs create" will score highe all things related to the creation of files and directories inside of `std.fs`, while still showing (but with lower score) matches from `std.Bulild`. 

As another example "fs windows" will prioritize windows-related results in `std.fs`, while "windows fs" will prioritize fs-related results in `std.windows`.

lib/docs/index.html

@@ -24,6 +24,7 @@
         --search-bg-color: #f3f3f3;
         --search-bg-color-focus: #ffffff;
         --search-sh-color: rgba(0, 0, 0, 0.18);
+        --search-other-results-color: rgb(100, 100, 100);
         --help-sh-color: rgba(0, 0, 0, 0.75);
         --help-bg-color: #aaa;
       }
@@ -83,6 +84,7 @@
 
       .flex-right {
         display: flex;
+        flex-direction: column;
         overflow: auto;
         -webkit-overflow-scrolling: touch;
         flex-grow: 1;
@@ -200,7 +202,7 @@
       }
 
       #guides {
-        padding: 1rem 0.7rem 2.4rem 1.4rem;
+        padding: 0rem 0.7rem 2.4rem 1.4rem;
         box-sizing: border-box;
         font-size: 1rem;
         background-color: var(--bg-color);
@@ -209,7 +211,7 @@
 
       /* docs section */
       .docs {
-        padding: 1rem 0.7rem 2.4rem 1.4rem;
+        padding: 0rem 0.7rem 2.4rem 1.4rem;
         font-size: 1rem;
         background-color: var(--bg-color);
         overflow-wrap: break-word;
@@ -248,6 +250,37 @@
         left: 5px;
       }
       
+      .other-results {
+        line-height: 1em;
+        position: relative;
+        outline: 0;
+        border: 0;
+        color: var(--search-other-results-color);
+        text-align: center;
+        height: 1.5em;
+        opacity: .5;
+      }
+      .other-results:before {
+        content: '';
+        background: var(--search-other-results-color);
+        position: absolute;
+        left: 0;
+        top: 50%;
+        width: 100%;
+        height: 1px;
+      }
+        
+      .other-results:after {
+        content: "other results";
+        position: relative;
+        display: inline-block;
+        padding: 0 .5em;
+        line-height: 1.5em;
+        color: var(--search-other-results-color);
+        background-color: var(--bg-color);
+      }
+      
+      
       .docs a {
         color: var(--link-color);
       }
@@ -505,6 +538,7 @@
           --search-bg-color: #3c3c3c;
           --search-bg-color-focus: #000;
           --search-sh-color: rgba(255, 255, 255, 0.28);
+          --search-other-results-color: rgba(255, 255, 255, 0.28);
           --help-sh-color: rgba(142, 142, 142, 0.5);
           --help-bg-color: #333;
         }
@@ -717,22 +751,38 @@
               <h2><span>Zig Version</span></h2>
               <p class="str" id="tdZigVer"></p>
             </div>
-            <div>
-              <input id="privDeclsBox" type="checkbox"/>
-              <label for="privDeclsBox">Internal Doc Mode</label>
-            </div>
           </div>
         </nav>
       </div>
       <div class="flex-right">
+        <div class="wrap">
+          <section class="docs" style="padding-top: 1.5rem; padding-bottom:0;">
+          <div style="position: relative">
+            <span id="searchPlaceholder"><kbd>s</kbd> to search, <kbd>?</kbd> for more options</span>
+            <input type="search" class="search" id="search" autocomplete="off" spellcheck="false" disabled>
+          </div>
+          </section>
+        </div>
+        <div id="sectSearchResults" class="docs hidden">
+          <h2>Search Results</h2>
+          <ul id="listSearchResults"></ul>
+          <p id="sectSearchAllResultsLink" class="hidden"><a href="">show all results</a></p>
+        </div>
+        <div id="sectSearchNoResults" class="docs hidden">
+          <h2>No Results Found</h2>
+          <p>Here are some things you can try:</p>
+          <ul>
+            <li>Check out the <a id="langRefLink">Language Reference</a> for the language itself.</li>
+            <li>Check out the <a href="https://ziglang.org/learn/">Learn page</a> for other helpful resources for learning Zig.</li>
+            <li>Use your search engine.</li>
+          </ul>
+          <p>Press <kbd>?</kbd> to see keyboard shortcuts and <kbd>Esc</kbd> to return.</p>
+        </div>
         <div id="guides" class="wrap hidden">
+          <div id="activeGuide" class="hidden"></div>
         </div> 
         <div id="docs" class="wrap hidden">
           <section class="docs">
-            <div style="position: relative">
-              <span id="searchPlaceholder"><kbd>s</kbd> to search, <kbd>?</kbd> for more options</span>
-              <input type="search" class="search" id="search" autocomplete="off" spellcheck="false" disabled>
-            </div>
             <p id="status">Loading...</p>
             <div id="sectNav" class="hidden"><ul id="listNav"></ul></div>
             <div id="fnProto" class="hidden">
@@ -762,21 +812,6 @@
               </div>
               <div id="tableFnErrors"><dl id="listFnErrors"></dl></div>
             </div>
-            <div id="sectSearchResults" class="hidden">
-              <h2>Search Results</h2>
-              <ul id="listSearchResults"></ul>
-              <p id="sectSearchAllResultsLink" class="hidden"><a href="">show all results</a></p>
-            </div>
-            <div id="sectSearchNoResults" class="hidden">
-              <h2>No Results Found</h2>
-              <p>Here are some things you can try:</p>
-              <ul>
-                <li>Check out the <a id="langRefLink">Language Reference</a> for the language itself.</li>
-                <li>Check out the <a href="https://ziglang.org/learn/">Learn page</a> for other helpful resources for learning Zig.</li>
-                <li>Use your search engine.</li>
-              </ul>
-              <p>Press <kbd>?</kbd> to see keyboard shortcuts and <kbd>Esc</kbd> to return.</p>
-            </div>
             <div id="sectFields" class="hidden">
               <h2>Fields</h2>
               <div id="listFields"></div>
@@ -852,6 +887,7 @@
       </div>
     </div>
     <script src="data.js"></script>
+    <script src="commonmark.js"></script>
     <script src="main.js"></script>
   </body>
 </html>

lib/docs/main.js

@@ -4,7 +4,6 @@ var zigAnalysis;
 
 const NAV_MODES = {
   API: "#A;",
-  API_INTERNAL: "#a;",
   GUIDES: "#G;",
 };
 
@@ -56,12 +55,13 @@ const NAV_MODES = {
   const domSectSearchResults = document.getElementById("sectSearchResults");
   const domSectSearchAllResultsLink = document.getElementById("sectSearchAllResultsLink");
   const domDocs = document.getElementById("docs");
-  const domGuides = document.getElementById("guides");
+  const domGuidesSection = document.getElementById("guides");
+  const domActiveGuide = document.getElementById("activeGuide");
+  
   const domListSearchResults = document.getElementById("listSearchResults");
   const domSectSearchNoResults = document.getElementById("sectSearchNoResults");
   const domSectInfo = document.getElementById("sectInfo");
   // const domTdTarget = (document.getElementById("tdTarget"));
-  const domPrivDeclsBox = document.getElementById("privDeclsBox");
   const domTdZigVer = document.getElementById("tdZigVer");
   const domHdrName = document.getElementById("hdrName");
   const domHelpModal = document.getElementById("helpModal");
@@ -83,15 +83,16 @@ const NAV_MODES = {
   let typeTypeId = findTypeTypeId();
   let pointerSizeEnum = { One: 0, Many: 1, Slice: 2, C: 3 };
 
+  let declSearchIndex = new RadixTree();
+  window.search = declSearchIndex;
+
   // for each module, is an array with modules to get to this one
   let canonModPaths = computeCanonicalModulePaths();
 
   // for each decl, is an array with {declNames, modNames} to get to this one
-
   let canonDeclPaths = null; // lazy; use getCanonDeclPath
 
   // for each type, is an array with {declNames, modNames} to get to this one
-
   let canonTypeDecls = null; // lazy; use getCanonTypeDecl
 
   let curNav = {
@@ -122,6 +123,12 @@ const NAV_MODES = {
   // map of decl index to list of comptime fn calls
   // let nodesToCallsMap = indexNodesToCalls();
 
+  let guidesSearchIndex = {}; 
+  window.guideSearch = guidesSearchIndex;
+  parseGuides();
+
+  
+
   domSearch.disabled = false;
   domSearch.addEventListener("keydown", onSearchKeyDown, false);
   domSearch.addEventListener("focus", ev => {
@@ -139,31 +146,6 @@ const NAV_MODES = {
     onHashChange();
   }
 
-  domPrivDeclsBox.addEventListener(
-    "change",
-    function () {
-      if (this.checked != curNav.showPrivDecls) {
-        if (
-          this.checked &&
-          location.hash.length > 1 &&
-          location.hash[1] != "*"
-        ) {
-          location.hash = "#*" + location.hash.substring(1);
-          return;
-        }
-        if (
-          !this.checked &&
-          location.hash.length > 1 &&
-          location.hash[1] == "*"
-        ) {
-          location.hash = "#" + location.hash.substring(2);
-          return;
-        }
-      }
-    },
-    false
-  );
-
   if (location.hash == "") {
     location.hash = "#A;";
   }
@@ -189,7 +171,6 @@ const NAV_MODES = {
     let suffix = " - Zig";
     switch (curNav.mode) {
       case NAV_MODES.API:
-      case NAV_MODES.API_INTERNAL:
         let list = curNav.modNames.concat(curNav.declNames);
         if (list.length === 0) {
           document.title = zigAnalysis.modules[zigAnalysis.rootMod].name + suffix;
@@ -284,8 +265,11 @@ const NAV_MODES = {
 
   function resolveValue(value) {
     let i = 0;
-    while (i < 1000) {
+    while (true) {
       i += 1;
+      if (i >= 10000) {
+        throw "resolveValue quota exceeded"
+      }
 
       if ("refPath" in value.expr) {
         value = { expr: value.expr.refPath[value.expr.refPath.length - 1] };
@@ -307,8 +291,31 @@ const NAV_MODES = {
 
       return value;
     }
-    console.assert(false);
+  }
-    return {};
+
+  function resolveGenericRet(genericFunc) {
+    if (genericFunc.generic_ret == null) return null;
+    let result = resolveValue({expr: genericFunc.generic_ret});
+    
+    let i = 0;
+    while (true) {
+      i += 1;
+      if (i >= 10000) {
+        throw "resolveGenericRet quota exceeded"
+      }
+
+      if ("call" in result.expr) {
+        let call = zigAnalysis.calls[result.expr.call];
+        let resolvedFunc = resolveValue({ expr: call.func });
+        if (!("type" in resolvedFunc.expr)) return null;
+        let callee = getType(resolvedFunc.expr.type);
+        if (!callee.generic_ret) return null;
+        result = resolveValue({ expr: callee.generic_ret });
+        continue;
+      }
+
+      return result;
+    }
   }
 
   //    function typeOfDecl(decl){
@@ -401,8 +408,12 @@ const NAV_MODES = {
     domGuideSwitch.classList.add("active");
     domApiSwitch.classList.remove("active");
     domDocs.classList.add("hidden");
-    domGuides.classList.remove("hidden");
+    domGuidesSection.classList.remove("hidden");
+    domActiveGuide.classList.add("hidden");
     domApiMenu.classList.add("hidden");
+    domSectSearchResults.classList.add("hidden");
+    domSectSearchAllResultsLink.classList.add("hidden");
+    domSectSearchNoResults.classList.add("hidden");
 
     // sidebar guides list
     const section_list = zigAnalysis.guide_sections;
@@ -417,7 +428,7 @@ const NAV_MODES = {
           const guide = section.guides[i];
           let liDom = domGuides.children[i];
           let aDom = liDom.children[0];
-          aDom.textContent = guide.name;
+          aDom.textContent = guide.title;
           aDom.setAttribute("href", NAV_MODES.GUIDES + guide.name);
           if (guide.name === curNav.activeGuide) {
             aDom.classList.add("active");
@@ -431,6 +442,11 @@ const NAV_MODES = {
       domGuidesMenu.classList.remove("hidden");
     }
 
+    
+    if (curNavSearch !== "") {
+          return renderSearchGuides();
+    }
+
     // main content
     let activeGuide = undefined;
     outer: for (let i = 0; i < zigAnalysis.guide_sections.length; i += 1) {
@@ -447,7 +463,7 @@ const NAV_MODES = {
     if (activeGuide == undefined) {
       const root_file_idx = zigAnalysis.modules[zigAnalysis.rootMod].file;
       const root_file_name = getFile(root_file_idx).name;
-      domGuides.innerHTML = markdown(`
+      domActiveGuide.innerHTML = markdown(`
           # Zig Guides
           These autodocs don't contain any guide.
 
@@ -461,29 +477,38 @@ const NAV_MODES = {
           You can add guides by specifying which markdown files to include
           in the top level doc comment of your root file, like so:
 
-          (At the top of \`${root_file_name}\`)
+          (At the top of *${root_file_name}*)
           \`\`\`
           //!zig-autodoc-guide: intro.md
           //!zig-autodoc-guide: quickstart.md
-          //!zig-autodoc-section: Advanced topics
+          //!zig-autodoc-guide: advanced-docs/advanced-stuff.md
-          //!zig-autodoc-guide: ../advanced-docs/advanced-stuff.md
           \`\`\`
 
+          You can also create sections to group guides together:
+
+          \`\`\`
+          //!zig-autodoc-section: CLI Usage
+          //!zig-autodoc-guide: cli-basics.md
+          //!zig-autodoc-guide: cli-advanced.md
+          \`\`\`
+           
+        
           **Note that this feature is still under heavy development so expect bugs**
           **and missing features!**
 
           Happy writing!
         `);
     } else {
-      domGuides.innerHTML = markdown(activeGuide.body);
+      domActiveGuide.innerHTML = markdown(activeGuide.body);
     }
+    domActiveGuide.classList.remove("hidden");
   }
 
   function renderApi() {
     // set Api mode
     domApiSwitch.classList.add("active");
     domGuideSwitch.classList.remove("active");
-    domGuides.classList.add("hidden");
+    domGuidesSection.classList.add("hidden");
     domDocs.classList.remove("hidden");
     domApiMenu.classList.remove("hidden");
     domGuidesMenu.classList.add("hidden");
@@ -520,10 +545,8 @@ const NAV_MODES = {
     renderInfo();
     renderModList();
 
-    domPrivDeclsBox.checked = curNav.mode == NAV_MODES.API_INTERNAL;
-
     if (curNavSearch !== "") {
-      return renderSearch();
+          return renderSearchAPI();
     }
 
     let rootMod = zigAnalysis.modules[zigAnalysis.rootMod];
@@ -542,6 +565,7 @@ const NAV_MODES = {
     curNav.declObjs = [currentType];
     for (let i = 0; i < curNav.declNames.length; i += 1) {
       let childDecl = findSubDecl(currentType, curNav.declNames[i]);
+      window.last_decl = childDecl;
       if (childDecl == null) {
         return render404();
       }
@@ -605,7 +629,6 @@ const NAV_MODES = {
   function render() {
     switch (curNav.mode) {
       case NAV_MODES.API:
-      case NAV_MODES.API_INTERNAL:
         return renderApi();
       case NAV_MODES.GUIDES:
         return renderGuides();
@@ -3123,22 +3146,21 @@ const NAV_MODES = {
     const mode = location.hash.substring(0, 3);
     let query = location.hash.substring(3);
 
+    let qpos = query.indexOf("?");
+    let nonSearchPart;
+    if (qpos === -1) {
+      nonSearchPart = query;
+    } else {
+      nonSearchPart = query.substring(0, qpos);
+      curNavSearch = decodeURIComponent(query.substring(qpos + 1));
+    }
+
     const DEFAULT_HASH = NAV_MODES.API + zigAnalysis.modules[zigAnalysis.rootMod].name;
     switch (mode) {
       case NAV_MODES.API:
-      case NAV_MODES.API_INTERNAL:
+       // #A;MODULE:decl.decl.decl?search-term
-        // #A;MODULE:decl.decl.decl?search-term
         curNav.mode = mode;
 
-        let qpos = query.indexOf("?");
-        let nonSearchPart;
-        if (qpos === -1) {
-          nonSearchPart = query;
-        } else {
-          nonSearchPart = query.substring(0, qpos);
-          curNavSearch = decodeURIComponent(query.substring(qpos + 1));
-        }
-
         let parts = nonSearchPart.split(":");
         if (parts[0] == "") {
           location.hash = DEFAULT_HASH;
@@ -3152,14 +3174,18 @@ const NAV_MODES = {
 
         return;
       case NAV_MODES.GUIDES:
+        
         const sections = zigAnalysis.guide_sections;
-        if (sections.length != 0 && sections[0].guides.length != 0 && query == "") {
+        if (sections.length != 0 && sections[0].guides.length != 0 && nonSearchPart == "") {
           location.hash = NAV_MODES.GUIDES + sections[0].guides[0].name;
+          if (qpos != -1) {
+            location.hash += query.substring(qpos);
+          }
           return;
         }
 
         curNav.mode = mode;
-        curNav.activeGuide = query;
+        curNav.activeGuide = nonSearchPart;
 
         return;
       default:
@@ -3230,22 +3256,6 @@ const NAV_MODES = {
       }
     }
 
-    if (parentType.privDecls) {
-      for (let i = 0; i < parentType.privDecls.length; i += 1) {
-        let declIndex = parentType.privDecls[i];
-        let childDecl = getDecl(declIndex);
-        if (childDecl.name === childName) {
-          childDecl.find_subdecl_idx = declIndex;
-          return childDecl;
-        } else if (childDecl.is_uns) {
-          let declValue = resolveValue(childDecl.value);
-          if (!("type" in declValue.expr)) continue;
-          let uns_container = getType(declValue.expr.type);
-          let uns_res = findSubDecl(uns_container, childName);
-          if (uns_res !== null) return uns_res;
-        }
-      }
-    }
     return null;
   }
 
@@ -3276,6 +3286,12 @@ const NAV_MODES = {
         });
       }
     }
+
+    for (let i = 0; i < zigAnalysis.modules.length; i += 1) {
+      const p = zigAnalysis.modules[i];
+      // TODO
+      // declSearchIndex.add(p.name, {moduleId: i});
+    }
     return list;
   }
 
@@ -3291,6 +3307,7 @@ const NAV_MODES = {
       let stack = [
         {
           declNames: [],
+          declIndexes: [],
           type: getType(mod.main),
         },
       ];
@@ -3334,19 +3351,28 @@ const NAV_MODES = {
         }
       }
     }
+    window.cdp = list;
     return list;
   }
 
 function addDeclToSearchResults(decl, declIndex, modNames, item, list, stack) {
   let declVal = resolveValue(decl.value);
   let declNames = item.declNames.concat([decl.name]);
+  let declIndexes = item.declIndexes.concat([declIndex]);
 
   if (list[declIndex] != null) return;
   list[declIndex] = {
     modNames: modNames,
     declNames: declNames,
+    declIndexes: declIndexes,
   };
 
+  // add to search index
+  {
+    declSearchIndex.add(decl.name, {declIndex});
+  }
+  
+  
   if ("type" in declVal.expr) {
     let value = getType(declVal.expr.type);
     if (declCanRepresentTypeKind(value.kind)) {
@@ -3356,18 +3382,20 @@ function addDeclToSearchResults(decl, declIndex, modNames, item, list, stack) {
     if (isContainerType(value)) {
       stack.push({
         declNames: declNames,
+        declIndexes: declIndexes,
         type: value,
       });
     }
 
     // Generic function
-    if (value.kind == typeKinds.Fn && value.generic_ret != null) {
+    if (typeIsGenericFn(declVal.expr.type)) {
-      let resolvedVal = resolveValue({ expr: value.generic_ret });
+      let ret = resolveGenericRet(value);
-      if ("type" in resolvedVal.expr) {
+      if (ret != null && "type" in ret.expr) {
-        let generic_type = getType(resolvedVal.expr.type);
+        let generic_type = getType(ret.expr.type);
         if (isContainerType(generic_type)) {
           stack.push({
             declNames: declNames,
+            declIndexes: declIndexes,
             type: generic_type,
           });
         }
@@ -3419,6 +3447,53 @@ function addDeclToSearchResults(decl, declIndex, modNames, item, list, stack) {
     return markdown(shortDesc(docs));
   }
 
+  function parseGuides() {
+    for (let j = 0; j < zigAnalysis.guide_sections.length; j+=1){
+      const section = zigAnalysis.guide_sections[j];
+      for (let i = 0; i < section.guides.length; i+=1){
+        let reader = new commonmark.Parser({smart: true});
+        const guide = section.guides[i];
+        const ast = reader.parse(guide.body);        
+        
+        // Find the first text thing to use as a sidebar title
+        guide.title = "[empty guide]";
+        {
+          let walker = ast.walker();
+          let event, node;
+          while ((event = walker.next())) {
+            node = event.node;
+            if (node.type === 'text') {
+              guide.title = node.literal;
+              break;
+            }
+          }        
+        }
+        // Index this guide
+        {
+          let walker = ast.walker();
+          let event, node;
+          while ((event = walker.next())) {
+            node = event.node;
+            if (event.entering == true && node.type === 'text') {
+                indexTextForGuide(j, i, node);          
+            }
+          }        
+        }
+      }  
+    }
+  }
+
+  function indexTextForGuide(section_idx, guide_idx, node){
+    const terms = node.literal.split(" ");
+    for (let i = 0; i < terms.length; i += 1){
+      const t = terms[i];
+      if (!guidesSearchIndex[t]) guidesSearchIndex[t] = new Set();
+      node.guide = {section_idx, guide_idx};
+      guidesSearchIndex[t].add(node);
+    }  
+  }
+  
+
   function markdown(input, contextType) {
     const raw_lines = input.split("\n"); // zig allows no '\r', so we don't need to split on CR
 
@@ -3842,6 +3917,7 @@ function addDeclToSearchResults(decl, declIndex, modNames, item, list, stack) {
     }
   }
   
+
   function onSearchKeyDown(ev) {
     switch (getKeyString(ev)) {
       case "Enter":
@@ -3982,16 +4058,207 @@ function addDeclToSearchResults(decl, declIndex, modNames, item, list, stack) {
     clearAsyncSearch();
     let oldHash = location.hash;
     let parts = oldHash.split("?");
-    let newPart2 = domSearch.value === "" ? "" : "?" + domSearch.value;
+    // TODO: make a tooltip that shows the user that we've replaced their dots
+    let box_text = domSearch.value.replaceAll(".", " ");
+    let newPart2 = box_text === "" ? "" : "?" + box_text;
     location.replace(parts.length === 1 ? oldHash + newPart2 : parts[0] + newPart2);
   }
   function getSearchTerms() {
     let list = curNavSearch.trim().split(/[ \r\n\t]+/);
-    list.sort();
     return list;
   }
 
-  function renderSearch() {
+  function renderSearchGuides() {
+    const searchTrimmed = false;
+    let ignoreCase = curNavSearch.toLowerCase() === curNavSearch;
+    
+    let terms = getSearchTerms();
+    let matchedItems = new Set();
+
+    for (let i = 0; i < terms.length; i += 1) {
+      const nodes = guidesSearchIndex[terms[i]];
+      if (nodes) {
+        for (const n of nodes) {      
+          matchedItems.add(n);
+        }
+      }
+    }
+
+    
+    
+    if (matchedItems.size !== 0) {
+      // Build up the list of search results
+      let matchedItemsHTML = "";
+
+      for (const node of matchedItems) {
+        const text = node.literal;
+        const href = "";
+
+        matchedItemsHTML += "<li><a href=\"" + href + "\">" + text + "</a></li>";
+      }
+
+      // Replace the search results using our newly constructed HTML string
+      domListSearchResults.innerHTML = matchedItemsHTML;
+      if (searchTrimmed) {
+        domSectSearchAllResultsLink.classList.remove("hidden");
+      }
+      renderSearchCursor();
+
+      domSectSearchResults.classList.remove("hidden");
+    } else {
+      domSectSearchNoResults.classList.remove("hidden");
+    }
+  }
+
+  function renderSearchAPI(){
+    if (canonDeclPaths == null) {
+      canonDeclPaths = computeCanonDeclPaths();
+    }
+    let declSet = new Set();
+    let otherDeclSet = new Set(); // for low quality results
+    let declScores = {};
+    
+    let ignoreCase = curNavSearch.toLowerCase() === curNavSearch;
+    let term_list = getSearchTerms();
+    for (let i = 0; i < term_list.length; i += 1) {
+      let term = term_list[i];
+      let result = declSearchIndex.search(term.toLowerCase());
+      if (result == null) {
+        domSectSearchNoResults.classList.remove("hidden");
+        domSectSearchResults.classList.add("hidden");
+        return;
+      }
+
+      let termSet = new Set();
+      let termOtherSet = new Set();
+
+      for (let list of [result.full, result.partial]) {
+        for (let r of list) {
+          const d = r.declIndex;
+          const decl = getDecl(d);
+          const canonPath = getCanonDeclPath(d);
+
+          // collect unconditionally for the first term
+          if (i == 0) {
+            declSet.add(d);
+          } else {
+            // path intersection for subsequent terms
+            let found = false;
+            for (let p of canonPath.declIndexes) {
+              if (declSet.has(p)) {
+                found = true;
+                break;
+              }   
+            }
+            if (!found) {
+              otherDeclSet.add(d);
+            } else {
+              termSet.add(d);
+            }
+          }
+          
+          if (declScores[d] == undefined) declScores[d] = 0;
+        
+          // scores (lower is better)
+          let decl_name = decl.name;
+          if (ignoreCase) decl_name = decl_name.toLowerCase();
+
+          // shallow path are preferable
+          const path_depth = canonPath.declNames.length * 50;
+          // matching the start of a decl name is good
+          const match_from_start = decl_name.startsWith(term) ? -term.length * (1 -ignoreCase) : (decl_name.length - term.length) + 1; 
+          // being a perfect match is good
+          const is_full_match = (list == result.full) ? -decl_name.length * (2 - ignoreCase) : decl_name.length - term.length;
+          // matching the end of a decl name is good
+          const matches_the_end = decl_name.endsWith(term) ? -term.length * (1 - ignoreCase) : (decl_name.length - term.length) + 1;
+          // explicitly penalizing scream case decls
+          const decl_is_scream_case = decl.name.toUpperCase() != decl.name ? 0 : decl.name.length;  
+        
+          const score =  path_depth 
+            + match_from_start 
+            + is_full_match 
+            + matches_the_end 
+            + decl_is_scream_case;
+
+          declScores[d] += score;        
+        }
+      }
+      if (i != 0) {
+        for (let d of declSet) {
+          if (termSet.has(d)) continue;
+          let found = false;
+          for (let p of getCanonDeclPath(d).declIndexes) {
+            if (termSet.has(p) || otherDeclSet.has(p)) {
+              found = true;
+              break;
+            }   
+          }
+          if (found) {
+            declScores[d] = declScores[d] / term_list.length;
+          }
+          
+          termOtherSet.add(d);
+        }
+        declSet = termSet;
+        for (let d of termOtherSet) {
+          otherDeclSet.add(d);
+        }
+        
+      }
+    }
+
+    let matchedItems = {
+      high_quality: [],
+      low_quality: [],
+    };
+    for (let idx of declSet) {
+      matchedItems.high_quality.push({points: declScores[idx], declIndex: idx})
+    }
+    for (let idx of otherDeclSet) {
+      matchedItems.low_quality.push({points: declScores[idx], declIndex: idx})
+    }
+    
+    matchedItems.high_quality.sort(function (a, b) {
+      let cmp = operatorCompare(a.points, b.points);
+      return cmp;
+    });
+    matchedItems.low_quality.sort(function (a, b) {
+      let cmp = operatorCompare(a.points, b.points);
+      return cmp;
+    });
+    
+    // Build up the list of search results
+    let matchedItemsHTML = "";
+
+    for (let list of [matchedItems.high_quality, matchedItems.low_quality]) {
+      if (list == matchedItems.low_quality && list.length > 0) {
+        matchedItemsHTML += "<hr class='other-results'>"
+      }
+      for (let result of list) {
+        const points = result.points;
+        const match = result.declIndex;
+
+        let canonPath = getCanonDeclPath(match);
+        if (canonPath == null) continue;
+
+        let lastModName = canonPath.modNames[canonPath.modNames.length - 1];
+        let text = lastModName + "." + canonPath.declNames.join(".");
+      
+
+        const href = navLink(canonPath.modNames, canonPath.declNames);
+
+        matchedItemsHTML += "<li><a href=\"" + href + "\">" +  text + "</a></li>";
+      }
+    }
+
+    // Replace the search results using our newly constructed HTML string
+    domListSearchResults.innerHTML = matchedItemsHTML;
+    renderSearchCursor();
+
+    domSectSearchResults.classList.remove("hidden");
+  }
+   
+  function renderSearchAPIOld() {
     let matchedItems = [];
     let ignoreCase = curNavSearch.toLowerCase() === curNavSearch;
     let terms = getSearchTerms();
@@ -4063,7 +4330,7 @@ function addDeclToSearchResults(decl, declIndex, modNames, item, list, stack) {
       });
 
       let searchTrimmed = false;
-      const searchTrimResultsMaxItems = 200;
+      const searchTrimResultsMaxItems = 60;
       if (searchTrimResults && matchedItems.length > searchTrimResultsMaxItems) {
         matchedItems = matchedItems.slice(0, searchTrimResultsMaxItems);
         searchTrimmed = true;
@@ -4314,3 +4581,255 @@ function toggleExpand(event) {
     parent.parentElement.parentElement.scrollIntoView(true);
   }
 }
+
+function RadixTree() {
+  this.root = null;
+
+  RadixTree.prototype.search = function (query) {
+    return this.root.search(query);
+    
+  }
+
+  RadixTree.prototype.add = function (declName, value) {
+    if (this.root == null) {
+      this.root = new Node(declName.toLowerCase(), null, [value]);
+    } else {
+      this.root.add(declName.toLowerCase(), value);
+    }
+
+    const not_scream_case = declName.toUpperCase() != declName;
+    let found_separator = false;
+    for (let i = 1; i < declName.length; i +=1) {
+      if (declName[i] == '_' || declName[i] == '.') {
+        found_separator = true;
+        continue;
+      }
+
+
+      if (found_separator || (declName[i].toLowerCase() !== declName[i])) {
+        if (declName.length > i+1 
+          && declName[i+1].toLowerCase() != declName[i+1]) continue;
+        let suffix = declName.slice(i);
+        this.root.add(suffix.toLowerCase(), value);
+        found_separator = false;
+      }
+    }
+  }
+  
+  function Node(labels, next, values) {
+    this.labels = labels; 
+    this.next = next;
+    this.values = values;
+  }
+
+  Node.prototype.isCompressed = function () {
+    return !Array.isArray(this.next);
+  }
+
+  Node.prototype.search = function (word) {
+    let full_matches = [];
+    let partial_matches = [];
+    let subtree_root = null;
+    
+    let cn = this;
+    char_loop: for (let i = 0; i < word.length;) {
+      if (cn.isCompressed()) {
+        for (let j = 0; j < cn.labels.length; j += 1) {
+          let current_idx = i+j;
+
+          if (current_idx == word.length) {
+            partial_matches = cn.values;
+            subtree_root = cn.next;
+            break char_loop; 
+          }
+          
+          if (word[current_idx] != cn.labels[j]) return null;
+        }
+
+        // the full label matched
+        let new_idx = i + cn.labels.length;
+        if (new_idx == word.length) {
+          full_matches = cn.values;
+          subtree_root = cn.next;
+          break char_loop;
+        }
+        
+        
+        i = new_idx;
+        cn = cn.next;
+        continue;
+      } else {
+        for (let j = 0; j < cn.labels.length; j += 1) {
+          if (word[i] == cn.labels[j]) {
+            if (i == word.length - 1) { 
+              full_matches = cn.values[j];
+              subtree_root = cn.next[j];
+              break char_loop;
+            }
+              
+            let next = cn.next[j];
+            if (next == null) return null;
+            cn = next;
+            i += 1;
+            continue char_loop; 
+          } 
+        }
+        
+        // didn't find a match
+        return null;
+      }
+    }
+    
+    // Match was found, let's collect all other 
+    // partial matches from the subtree
+    let stack = [subtree_root];
+    let node;
+    while (node = stack.pop()) {
+      if (node.isCompressed()) {
+        partial_matches = partial_matches.concat(node.values);
+        if (node.next != null) {
+          stack.push(node.next);
+        }
+      } else {
+        for (let v of node.values) {
+          partial_matches = partial_matches.concat(v);
+        }
+        
+        for (let n of node.next) {
+          if (n != null) stack.push(n);
+        }
+      }
+    }
+
+    return {full: full_matches, partial: partial_matches};
+  }
+
+  Node.prototype.add = function (word, value) {
+    let cn = this;
+    char_loop: for (let i = 0; i < word.length;) {
+      if (cn.isCompressed()) {
+        for(let j = 0; j < cn.labels.length; j += 1) {
+          let current_idx = i+j;
+
+          if (current_idx == word.length) {
+            if (j < cn.labels.length - 1) {
+              let node = new Node(cn.labels.slice(j), cn.next, cn.values);
+              cn.labels = cn.labels.slice(0, j);
+              cn.next = node;
+              cn.values = [];
+            }
+            cn.values.push(value);
+            return;
+          }
+          
+          if (word[current_idx] == cn.labels[j]) continue;
+          
+          // if we're here, a mismatch was found
+          if (j != cn.labels.length - 1) {
+            // create a suffix node
+            const label_suffix = cn.labels.slice(j+1);
+            let node = new Node(label_suffix, cn.next, [...cn.values]);
+            cn.next = node;
+            cn.values = [];
+          }
+
+          // turn current node into a split node
+          let node = null;
+          let word_values = [];
+          if (current_idx == word.length - 1) {
+            // mismatch happened in the last character of word
+            // meaning that the current node should hold its value
+            word_values.push(value);
+          } else {
+             node = new Node(word.slice(current_idx+1), null, [value]);
+          }
+        
+          cn.labels = cn.labels[j] + word[current_idx];
+          cn.next = [cn.next, node];          
+          cn.values = [cn.values, word_values];
+
+          if (j != 0) {
+            // current node must be turned into a prefix node
+            let splitNode = new Node(cn.labels, cn.next, cn.values);
+            cn.labels = word.slice(i, current_idx);
+            cn.next = splitNode;
+            cn.values = [];
+          }
+
+          return;
+        }        
+        // label matched fully with word, are there any more chars?
+        const new_idx = i + cn.labels.length;
+        if (new_idx == word.length) {
+          cn.values.push(value);
+          return;
+        } else {
+          if (cn.next == null) {
+            let node = new Node(word.slice(new_idx), null, [value]);
+            cn.next = node;
+            return;
+          } else {
+            cn = cn.next;
+            i = new_idx;
+            continue;
+          }
+        } 
+      } else { // node is not compressed
+        let letter = word[i];
+        for (let j = 0; j < cn.labels.length; j += 1) {
+          if (letter == cn.labels[j]) {
+            if (i == word.length - 1) {
+              cn.values[j].push(value);
+              return;
+            }
+            if (cn.next[j] == null) {
+              let node = new Node(word.slice(i+1), null, [value]);
+              cn.next[j] = node;
+              return;
+            } else {
+              cn = cn.next[j];
+              i += 1;
+              continue char_loop;
+            }
+          }
+        }
+
+        // if we're here we didn't find a match
+        cn.labels += letter;
+        if (i == word.length - 1) {
+          cn.next.push(null);
+          cn.values.push([value]);
+        } else {
+          let node = new Node(word.slice(i+1), null, [value]);
+          cn.next.push(node);
+          cn.values.push([]);
+        }
+        return;
+      }
+    }
+  }
+}
+
+// RADIX TREE:
+
+// apple 
+// appliance
+
+// "appl" => [
+//   'e', => $
+//   'i' => "ance" => $
+// ]
+
+// OUR STUFF:
+
+// AutoHashMap
+// AutoArrayHashMap
+
+// "Auto" => [
+//   'A', => "rrayHashMap" => $
+//   'H'  => "ashMap" => $
+// ]
+
+// BUT!
+
+// We want to be able to search "Hash", for example!

src/Autodoc.zig

@@ -234,12 +234,13 @@ pub fn generateZirData(self: *Autodoc) !void {
     };
 
     const tldoc_comment = try self.getTLDocComment(file);
+    const cleaned_tldoc_comment = try self.findGuidePaths(file, tldoc_comment);
+    defer self.arena.free(cleaned_tldoc_comment);
     try self.ast_nodes.append(self.arena, .{
         .name = "(root)",
-        .docs = tldoc_comment,
+        .docs = cleaned_tldoc_comment,
     });
     try self.files.put(self.arena, file, main_type_index);
-    try self.findGuidePaths(file, tldoc_comment);
 
     _ = try self.walkInstruction(file, &root_scope, .{}, Zir.main_struct_inst, false);
 
@@ -349,6 +350,7 @@ pub fn generateZirData(self: *Autodoc) !void {
     var docs_dir = try self.comp_module.comp.zig_lib_directory.handle.openDir("docs", .{});
     defer docs_dir.close();
     try docs_dir.copyFile("main.js", output_dir, "main.js", .{});
+    try docs_dir.copyFile("commonmark.js", output_dir, "commonmark.js", .{});
     try docs_dir.copyFile("index.html", output_dir, "index.html", .{});
 }
 
@@ -4754,14 +4756,20 @@ fn getTLDocComment(self: *Autodoc, file: *File) ![]const u8 {
     return comment.items;
 }
 
-fn findGuidePaths(self: *Autodoc, file: *File, str: []const u8) !void {
+/// Returns the doc comment cleared of autodoc directives.
+fn findGuidePaths(self: *Autodoc, file: *File, str: []const u8) ![]const u8 {
     const guide_prefix = "zig-autodoc-guide:";
     const section_prefix = "zig-autodoc-section:";
 
     try self.guide_sections.append(self.arena, .{}); // add a default section
     var current_section = &self.guide_sections.items[self.guide_sections.items.len - 1];
 
-    var it = std.mem.tokenize(u8, str, "\n");
+    var clean_docs: std.ArrayListUnmanaged(u8) = .{};
+    errdefer clean_docs.deinit(self.arena);
+
+    // TODO: this algo is kinda inefficient
+
+    var it = std.mem.split(u8, str, "\n");
     while (it.next()) |line| {
         const trimmed_line = std.mem.trim(u8, line, " ");
         if (std.mem.startsWith(u8, trimmed_line, guide_prefix)) {
@@ -4775,8 +4783,13 @@ fn findGuidePaths(self: *Autodoc, file: *File, str: []const u8) !void {
                 .name = trimmed_section_name,
             });
             current_section = &self.guide_sections.items[self.guide_sections.items.len - 1];
+        } else {
+            try clean_docs.appendSlice(self.arena, line);
+            try clean_docs.append(self.arena, '\n');
         }
     }
+
+    return clean_docs.toOwnedSlice(self.arena);
 }
 
 fn addGuide(self: *Autodoc, file: *File, guide_path: []const u8, section: *Section) !void {