mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2026-02-13 21:08:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

tokenizing libc hello world

Browse Source
This commit is contained in:
Andrew Kelley 2017-12-08 23:56:07 -05:00
parent d431b0fb99
commit f466e539ef
1 changed files with 90 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
src-self-hosted/main.zig
View File

@ -90,6 +90,10 @@ const Token = struct {
Arrow,
Colon,
Slash,
Comma,
Ampersand,
AmpersandEqual,
NumberLiteral,
Keyword_align,
Keyword_and,
Keyword_asm,
@ -189,6 +193,13 @@ const Tokenizer = struct {
Minus,
Slash,
LineComment,
Zero,
NumberLiteral,
NumberDot,
FloatFraction,
FloatExponentUnsigned,
FloatExponentNumber,
Ampersand,
};
pub fn next(self: &Tokenizer) -> Token {
@ -241,6 +252,11 @@ const Tokenizer = struct {
self.index += 1;
break;
},
',' => {
result.id = Token.Id.Comma;
self.index += 1;
break;
},
':' => {
result.id = Token.Id.Colon;
self.index += 1;
@ -272,12 +288,34 @@ const Tokenizer = struct {
'/' => {
state = State.Slash;
},
'&' => {
state = State.Ampersand;
},
'0' => {
state = State.Zero;
result.id = Token.Id.NumberLiteral;
},
'1'...'9' => {
state = State.NumberLiteral;
result.id = Token.Id.NumberLiteral;
},
else => {
result.id = Token.Id.Invalid;
self.index += 1;
break;
},
},
State.Ampersand => switch (c) {
'=' => {
result.id = Token.Id.AmpersandEqual;
self.index += 1;
break;
},
else => {
result.id = Token.Id.Ampersand;
break;
},
},
State.Identifier => switch (c) {
'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
else => {
@ -353,6 +391,58 @@ const Tokenizer = struct {
},
else => {},
},
State.Zero => switch (c) {
'b', 'o', 'x' => {
state = State.NumberLiteral;
},
else => {
// reinterpret as a normal number
self.index -= 1;
state = State.NumberLiteral;
},
},
State.NumberLiteral => switch (c) {
'.' => {
state = State.NumberDot;
},
'p', 'P', 'e', 'E' => {
state = State.FloatExponentUnsigned;
},
'0'...'9', 'a'...'f', 'A'...'F' => {},
else => break,
},
State.NumberDot => switch (c) {
'.' => {
self.index -= 1;
state = State.Start;
break;
},
else => {
self.index -= 1;
state = State.FloatFraction;
},
},
State.FloatFraction => switch (c) {
'p', 'P', 'e', 'E' => {
state = State.FloatExponentUnsigned;
},
'0'...'9', 'a'...'f', 'A'...'F' => {},
else => break,
},
State.FloatExponentUnsigned => switch (c) {
'+', '-' => {
state = State.FloatExponentNumber;
},
else => {
// reinterpret as a normal exponent number
self.index -= 1;
state = State.FloatExponentNumber;
}
},
State.FloatExponentNumber => switch (c) {
'0'...'9', 'a'...'f', 'A'...'F' => {},
else => break,
},
}
}
result.end = self.index;