diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 33abf35062..8ce0ce33ab 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -189,6 +189,7 @@ enum TokenizeState { TokenizeStateLineString, TokenizeStateLineStringEnd, TokenizeStateLineStringContinue, + TokenizeStateLineStringContinueC, TokenizeStateSawEq, TokenizeStateSawBang, TokenizeStateSawLessThan, @@ -932,6 +933,30 @@ void tokenize(Buf *buf, Tokenization *out) { switch (c) { case WHITESPACE: break; + case 'c': + if (!t.cur_tok->data.str_lit.is_c_str) { + t.pos -= 1; + end_token(&t); + t.state = TokenizeStateStart; + break; + } + t.state = TokenizeStateLineStringContinueC; + break; + case '\\': + if (t.cur_tok->data.str_lit.is_c_str) { + tokenize_error(&t, "invalid character: '%c'", c); + } + t.state = TokenizeStateLineStringContinue; + break; + default: + t.pos -= 1; + end_token(&t); + t.state = TokenizeStateStart; + continue; + } + break; + case TokenizeStateLineStringContinueC: + switch (c) { case '\\': t.state = TokenizeStateLineStringContinue; break; @@ -970,6 +995,11 @@ void tokenize(Buf *buf, Tokenization *out) { t.cur_tok->data.str_lit.is_c_str = true; t.state = TokenizeStateString; break; + case '\\': + set_token_id(&t, t.cur_tok, TokenIdStringLiteral); + t.cur_tok->data.str_lit.is_c_str = true; + t.state = TokenizeStateSawBackslash; + break; case SYMBOL_CHAR: t.state = TokenizeStateSymbol; buf_append_char(&t.cur_tok->data.str_lit.str, c); @@ -1386,6 +1416,7 @@ void tokenize(Buf *buf, Tokenization *out) { case TokenizeStateSawDotDot: case TokenizeStateSawBackslash: case TokenizeStateLineStringContinue: + case TokenizeStateLineStringContinueC: tokenize_error(&t, "unexpected EOF"); break; case TokenizeStateLineComment: diff --git a/test/self_hosted.zig b/test/self_hosted.zig index 6c9fd2d5f5..5eab6e42ff 100644 --- a/test/self_hosted.zig +++ b/test/self_hosted.zig @@ -693,6 +693,17 @@ fn multiline_string() { assert(str.eql(s1, s2)); } +#attribute("test") +fn multiline_c_string() { + const s1 = + c\\one + c\\two) + c\\three + ; + const s2 = c"one\ntwo)\nthree"; + assert(cstr.cmp(s1, s2) == 0); +} + #attribute("test")