diff --git a/std/zig/parser_test.zig b/std/zig/parser_test.zig index c11db78775..d114179bf7 100644 --- a/std/zig/parser_test.zig +++ b/std/zig/parser_test.zig @@ -1,3 +1,16 @@ +test "zig fmt: float literal with exponent" { + try testCanonical( + \\test "aoeu" { + \\ switch (state) { + \\ TermState.Start => switch (c) { + \\ '\x1b' => state = TermState.Escape, + \\ else => try out.writeByte(c), + \\ }, + \\ } + \\} + \\ + ); +} test "zig fmt: float literal with exponent" { try testCanonical( \\pub const f64_true_min = 4.94065645841246544177e-324; diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index afea7fc899..f4cd847dff 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -220,6 +220,8 @@ pub const Tokenizer = struct { MultilineStringLiteralLineBackslash, CharLiteral, CharLiteralBackslash, + CharLiteralEscape1, + CharLiteralEscape2, CharLiteralEnd, Backslash, Equal, @@ -612,11 +614,34 @@ pub const Tokenizer = struct { result.id = Token.Id.Invalid; break; }, + 'x' => { + state = State.CharLiteralEscape1; + }, else => { state = State.CharLiteralEnd; }, }, + State.CharLiteralEscape1 => switch (c) { + '0'...'9', 'a'...'z', 'A'...'F' => { + state = State.CharLiteralEscape2; + }, + else => { + result.id = Token.Id.Invalid; + break; + }, + }, + + State.CharLiteralEscape2 => switch (c) { + '0'...'9', 'a'...'z', 'A'...'F' => { + state = State.CharLiteralEnd; + }, + else => { + result.id = Token.Id.Invalid; + break; + }, + }, + State.CharLiteralEnd => switch (c) { '\'' => { result.id = Token.Id.CharLiteral; @@ -988,6 +1013,8 @@ pub const Tokenizer = struct { State.MultilineStringLiteralLineBackslash, State.CharLiteral, State.CharLiteralBackslash, + State.CharLiteralEscape1, + State.CharLiteralEscape2, State.CharLiteralEnd, State.StringLiteralBackslash => { result.id = Token.Id.Invalid; @@ -1127,6 +1154,13 @@ test "tokenizer" { }); } +test "tokenizer - char literal with hex escape" { + testTokenize( \\'\x1b' + , []Token.Id { + Token.Id.CharLiteral, + }); +} + test "tokenizer - float literal e exponent" { testTokenize("a = 4.94065645841246544177e-324;\n", []Token.Id { Token.Id.Identifier,