mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2026-01-09 08:55:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

std.zig.tokenizer: support hex escape in char literals

Browse Source
This commit is contained in:
Andrew Kelley 2018-05-24 21:51:58 -04:00
parent 43085417be
commit b74dda34b6
2 changed files with 47 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
std/zig/parser_test.zig
View File

@ -1,3 +1,16 @@
test "zig fmt: float literal with exponent" {
try testCanonical(
\\test "aoeu" {
\\ switch (state) {
\\ TermState.Start => switch (c) {
\\ '\x1b' => state = TermState.Escape,
\\ else => try out.writeByte(c),
\\ },
\\ }
\\}
\\
);
}
test "zig fmt: float literal with exponent" {
try testCanonical(
\\pub const f64_true_min = 4.94065645841246544177e-324;

34
std/zig/tokenizer.zig
View File

@ -220,6 +220,8 @@ pub const Tokenizer = struct {
MultilineStringLiteralLineBackslash,
CharLiteral,
CharLiteralBackslash,
CharLiteralEscape1,
CharLiteralEscape2,
CharLiteralEnd,
Backslash,
Equal,
@ -612,11 +614,34 @@ pub const Tokenizer = struct {
result.id = Token.Id.Invalid;
break;
},
'x' => {
state = State.CharLiteralEscape1;
},
else => {
state = State.CharLiteralEnd;
},
},
State.CharLiteralEscape1 => switch (c) {
'0'...'9', 'a'...'z', 'A'...'F' => {
state = State.CharLiteralEscape2;
},
else => {
result.id = Token.Id.Invalid;
break;
},
},
State.CharLiteralEscape2 => switch (c) {
'0'...'9', 'a'...'z', 'A'...'F' => {
state = State.CharLiteralEnd;
},
else => {
result.id = Token.Id.Invalid;
break;
},
},
State.CharLiteralEnd => switch (c) {
'\'' => {
result.id = Token.Id.CharLiteral;
@ -988,6 +1013,8 @@ pub const Tokenizer = struct {
State.MultilineStringLiteralLineBackslash,
State.CharLiteral,
State.CharLiteralBackslash,
State.CharLiteralEscape1,
State.CharLiteralEscape2,
State.CharLiteralEnd,
State.StringLiteralBackslash => {
result.id = Token.Id.Invalid;
@ -1127,6 +1154,13 @@ test "tokenizer" {
});
}
test "tokenizer - char literal with hex escape" {
testTokenize( \\'\x1b'
, []Token.Id {
Token.Id.CharLiteral,
});
}
test "tokenizer - float literal e exponent" {
testTokenize("a = 4.94065645841246544177e-324;\n", []Token.Id {
Token.Id.Identifier,