From f466e539ef27d1cf90e2b163d5afbcf0bffc0aa5 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <superjoe30@gmail.com>
Date: Fri, 8 Dec 2017 23:56:07 -0500
Subject: [PATCH] tokenizing libc hello world

---
 src-self-hosted/main.zig | 90 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)

diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig
index 680de4d041..a826780bc1 100644
--- a/src-self-hosted/main.zig
+++ b/src-self-hosted/main.zig
@@ -90,6 +90,10 @@ const Token = struct {
         Arrow,
         Colon,
         Slash,
+        Comma,
+        Ampersand,
+        AmpersandEqual,
+        NumberLiteral,
         Keyword_align,
         Keyword_and,
         Keyword_asm,
@@ -189,6 +193,13 @@ const Tokenizer = struct {
         Minus,
         Slash,
         LineComment,
+        Zero,
+        NumberLiteral,
+        NumberDot,
+        FloatFraction,
+        FloatExponentUnsigned,
+        FloatExponentNumber,
+        Ampersand,
     };
 
     pub fn next(self: &Tokenizer) -> Token {
@@ -241,6 +252,11 @@ const Tokenizer = struct {
                         self.index += 1;
                         break;
                     },
+                    ',' => {
+                        result.id = Token.Id.Comma;
+                        self.index += 1;
+                        break;
+                    },
                     ':' => {
                         result.id = Token.Id.Colon;
                         self.index += 1;
@@ -272,12 +288,34 @@ const Tokenizer = struct {
                     '/' => {
                         state = State.Slash;
                     },
+                    '&' => {
+                        state = State.Ampersand;
+                    },
+                    '0' => {
+                        state = State.Zero;
+                        result.id = Token.Id.NumberLiteral;
+                    },
+                    '1'...'9' => {
+                        state = State.NumberLiteral;
+                        result.id = Token.Id.NumberLiteral;
+                    },
                     else => {
                         result.id = Token.Id.Invalid;
                         self.index += 1;
                         break;
                     },
                 },
+                State.Ampersand => switch (c) {
+                    '=' => {
+                        result.id = Token.Id.AmpersandEqual;
+                        self.index += 1;
+                        break;
+                    },
+                    else => {
+                        result.id = Token.Id.Ampersand;
+                        break;
+                    },
+                },
                 State.Identifier => switch (c) {
                     'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
                     else => {
@@ -353,6 +391,58 @@ const Tokenizer = struct {
                     },
                     else => {},
                 },
+                State.Zero => switch (c) {
+                    'b', 'o', 'x' => {
+                        state = State.NumberLiteral;
+                    },
+                    else => {
+                        // reinterpret as a normal number
+                        self.index -= 1;
+                        state = State.NumberLiteral;
+                    },
+                },
+                State.NumberLiteral => switch (c) {
+                    '.' => {
+                        state = State.NumberDot;
+                    },
+                    'p', 'P', 'e', 'E' => {
+                        state = State.FloatExponentUnsigned;
+                    },
+                    '0'...'9', 'a'...'f', 'A'...'F' => {},
+                    else => break,
+                },
+                State.NumberDot => switch (c) {
+                    '.' => {
+                        self.index -= 1;
+                        state = State.Start;
+                        break;
+                    },
+                    else => {
+                        self.index -= 1;
+                        state = State.FloatFraction;
+                    },
+                },
+                State.FloatFraction => switch (c) {
+                    'p', 'P', 'e', 'E' => {
+                        state = State.FloatExponentUnsigned;
+                    },
+                    '0'...'9', 'a'...'f', 'A'...'F' => {},
+                    else => break,
+                },
+                State.FloatExponentUnsigned => switch (c) {
+                    '+', '-' => {
+                        state = State.FloatExponentNumber;
+                    },
+                    else => {
+                        // reinterpret as a normal exponent number
+                        self.index -= 1;
+                        state = State.FloatExponentNumber;
+                    }
+                },
+                State.FloatExponentNumber => switch (c) {
+                    '0'...'9', 'a'...'f', 'A'...'F' => {},
+                    else => break,
+                },
             }
         }
         result.end = self.index;