From 9de2c1b76e2b38e35bb17cd0a3943a61c730da76 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 28 May 2021 13:26:16 -0700 Subject: [PATCH] stage1: tokenizer: backport recent stage2 changes This backports ec10595b65490126e1b7d231f655be6d7d198acf stage2 tokenizer logic to stage1. --- src/stage1/tokenizer.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/stage1/tokenizer.cpp b/src/stage1/tokenizer.cpp index b1bae1865b..49d5d80fb0 100644 --- a/src/stage1/tokenizer.cpp +++ b/src/stage1/tokenizer.cpp @@ -1290,12 +1290,9 @@ void tokenize(const char *source, Tokenization *out) { case DIGIT: t.state = TokenizeState_float_fraction_dec; break; - case ALPHA_EXCEPT_E: + default: invalid_char_error(&t, c); break; - default: - t.state = TokenizeState_start; - continue; } break; case TokenizeState_num_dot_hex: @@ -1316,12 +1313,9 @@ void tokenize(const char *source, Tokenization *out) { t.out->ids.last() = TokenIdFloatLiteral; t.state = TokenizeState_float_fraction_hex; break; - case ALPHA_EXCEPT_HEX_AND_P: + default: invalid_char_error(&t, c); break; - default: - t.state = TokenizeState_start; - continue; } break; case TokenizeState_float_fraction_dec_no_underscore: