From 1fb308ceeea0259ad021d67945ea5adc10960a85 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <superjoe30@gmail.com>
Date: Fri, 9 Feb 2018 13:08:02 -0500
Subject: [PATCH] self hosted compiler: move tokenization and parsing to std
 lib

---
 CMakeLists.txt                             |  4 ++++
 build.zig                                  |  4 ----
 src-self-hosted/main.zig                   |  5 -----
 src-self-hosted/module.zig                 |  6 +++---
 std/index.zig                              |  2 ++
 {src-self-hosted => std/zig}/ast.zig       |  4 ++--
 std/zig/index.zig                          | 11 +++++++++++
 {src-self-hosted => std/zig}/parser.zig    |  8 ++++----
 {src-self-hosted => std/zig}/tokenizer.zig |  2 +-
 9 files changed, 27 insertions(+), 19 deletions(-)
 rename {src-self-hosted => std/zig}/ast.zig (98%)
 create mode 100644 std/zig/index.zig
 rename {src-self-hosted => std/zig}/parser.zig (99%)
 rename {src-self-hosted => std/zig}/tokenizer.zig (99%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 012ad075aa..46a1ecbb0d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -477,6 +477,10 @@ set(ZIG_STD_FILES
     "special/panic.zig"
     "special/test_runner.zig"
     "unicode.zig"
+    "zig/ast.zig"
+    "zig/index.zig"
+    "zig/parser.zig"
+    "zig/tokenizer.zig"
 )
 
 set(ZIG_C_HEADER_FILES
diff --git a/build.zig b/build.zig
index 5b27a96c31..facbe1f209 100644
--- a/build.zig
+++ b/build.zig
@@ -108,10 +108,6 @@ pub fn build(b: &Builder) !void {
         "std/special/compiler_rt/index.zig", "compiler-rt", "Run the compiler_rt tests",
         with_lldb));
 
-    test_step.dependOn(tests.addPkgTests(b, test_filter,
-        "src-self-hosted/main.zig", "fmt", "Run the fmt tests",
-        with_lldb));
-
     test_step.dependOn(tests.addCompareOutputTests(b, test_filter));
     test_step.dependOn(tests.addBuildExampleTests(b, test_filter));
     test_step.dependOn(tests.addCompileErrorTests(b, test_filter));
diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig
index 9adf294ad9..7b4a5e959e 100644
--- a/src-self-hosted/main.zig
+++ b/src-self-hosted/main.zig
@@ -622,8 +622,3 @@ fn findZigLibDir(allocator: &mem.Allocator) ![]u8 {
 
     return error.FileNotFound;
 }
-
-test "import tests" {
-    _ = @import("tokenizer.zig");
-    _ = @import("parser.zig");
-}
diff --git a/src-self-hosted/module.zig b/src-self-hosted/module.zig
index d95e54b269..245c395ece 100644
--- a/src-self-hosted/module.zig
+++ b/src-self-hosted/module.zig
@@ -8,9 +8,9 @@ const c = @import("c.zig");
 const builtin = @import("builtin");
 const Target = @import("target.zig").Target;
 const warn = std.debug.warn;
-const Tokenizer = @import("tokenizer.zig").Tokenizer;
-const Token = @import("tokenizer.zig").Token;
-const Parser = @import("parser.zig").Parser;
+const Tokenizer = std.zig.Tokenizer;
+const Token = std.zig.Token;
+const Parser = std.zig.Parser;
 const ArrayList = std.ArrayList;
 
 pub const Module = struct {
diff --git a/std/index.zig b/std/index.zig
index b5a80cba23..8d292c2f5c 100644
--- a/std/index.zig
+++ b/std/index.zig
@@ -28,6 +28,7 @@ pub const os = @import("os/index.zig");
 pub const rand = @import("rand.zig");
 pub const sort = @import("sort.zig");
 pub const unicode = @import("unicode.zig");
+pub const zig = @import("zig/index.zig");
 
 test "std" {
     // run tests from these
@@ -58,4 +59,5 @@ test "std" {
     _ = @import("rand.zig");
     _ = @import("sort.zig");
     _ = @import("unicode.zig");
+    _ = @import("zig/index.zig");
 }
diff --git a/src-self-hosted/ast.zig b/std/zig/ast.zig
similarity index 98%
rename from src-self-hosted/ast.zig
rename to std/zig/ast.zig
index b63c0a347e..a966c0316e 100644
--- a/src-self-hosted/ast.zig
+++ b/std/zig/ast.zig
@@ -1,7 +1,7 @@
-const std = @import("std");
+const std = @import("../index.zig");
 const assert = std.debug.assert;
 const ArrayList = std.ArrayList;
-const Token = @import("tokenizer.zig").Token;
+const Token = std.zig.Token;
 const mem = std.mem;
 
 pub const Node = struct {
diff --git a/std/zig/index.zig b/std/zig/index.zig
new file mode 100644
index 0000000000..32699935d9
--- /dev/null
+++ b/std/zig/index.zig
@@ -0,0 +1,11 @@
+const tokenizer = @import("tokenizer.zig");
+pub const Token = tokenizer.Token;
+pub const Tokenizer = tokenizer.Tokenizer;
+pub const Parser = @import("parser.zig").Parser;
+pub const ast = @import("ast.zig");
+
+test "std.zig tests" {
+    _ = @import("tokenizer.zig");
+    _ = @import("parser.zig");
+    _ = @import("ast.zig");
+}
diff --git a/src-self-hosted/parser.zig b/std/zig/parser.zig
similarity index 99%
rename from src-self-hosted/parser.zig
rename to std/zig/parser.zig
index 7157cfbaf9..4fcdddf63d 100644
--- a/src-self-hosted/parser.zig
+++ b/std/zig/parser.zig
@@ -1,10 +1,10 @@
-const std = @import("std");
+const std = @import("../index.zig");
 const assert = std.debug.assert;
 const ArrayList = std.ArrayList;
 const mem = std.mem;
-const ast = @import("ast.zig");
-const Tokenizer = @import("tokenizer.zig").Tokenizer;
-const Token = @import("tokenizer.zig").Token;
+const ast = std.zig.ast;
+const Tokenizer = std.zig.Tokenizer;
+const Token = std.zig.Token;
 const builtin = @import("builtin");
 const io = std.io;
 
diff --git a/src-self-hosted/tokenizer.zig b/std/zig/tokenizer.zig
similarity index 99%
rename from src-self-hosted/tokenizer.zig
rename to std/zig/tokenizer.zig
index f1f87abfc9..e9bfb21633 100644
--- a/src-self-hosted/tokenizer.zig
+++ b/std/zig/tokenizer.zig
@@ -1,4 +1,4 @@
-const std = @import("std");
+const std = @import("../index.zig");
 const mem = std.mem;
 
 pub const Token = struct {