mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2025-12-06 14:23:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

support C-style comments, plus nesting

Browse Source
This commit is contained in:
Andrew Kelley 2015-11-27 00:40:26 -07:00
parent 4060ae93fb
commit 821907317e
4 changed files with 124 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -31,7 +31,7 @@ readable, safe, optimal, and concise code to solve any computing problem.
## Roadmap ## Roadmap
* C style comments. * empty function and return with no expression
* Simple .so library * Simple .so library
* Multiple files * Multiple files
* figure out integers * figure out integers

103
src/tokenizer.cpp
View File

@ -87,6 +87,19 @@
case DIGIT: \ case DIGIT: \
case '_' case '_'
enum TokenizeState {
TokenizeStateStart,
TokenizeStateSymbol,
TokenizeStateNumber,
TokenizeStateString,
TokenizeStateSawDash,
TokenizeStateSawSlash,
TokenizeStateLineComment,
TokenizeStateMultiLineComment,
TokenizeStateMultiLineCommentSlash,
TokenizeStateMultiLineCommentStar,
};
struct Tokenize { struct Tokenize {
Buf *buf; Buf *buf;
@ -96,6 +109,7 @@ struct Tokenize {
int line; int line;
int column; int column;
Token *cur_tok; Token *cur_tok;
int multi_line_comment_count;
}; };
__attribute__ ((format (printf, 2, 3))) __attribute__ ((format (printf, 2, 3)))
@ -222,10 +236,80 @@ ZigList<Token> *tokenize(Buf *buf) {
begin_token(&t, TokenIdNumberSign); begin_token(&t, TokenIdNumberSign);
end_token(&t); end_token(&t);
break; break;
case '/':
t.state = TokenizeStateSawSlash;
break;
default: default:
tokenize_error(&t, "invalid character: '%c'", c); tokenize_error(&t, "invalid character: '%c'", c);
} }
break; break;
case TokenizeStateSawSlash:
switch (c) {
case '/':
t.state = TokenizeStateLineComment;
break;
case '*':
t.state = TokenizeStateMultiLineComment;
t.multi_line_comment_count = 1;
break;
default:
tokenize_error(&t, "invalid character: '%c'", c);
break;
}
break;
case TokenizeStateLineComment:
switch (c) {
case '\n':
t.state = TokenizeStateStart;
break;
default:
// do nothing
break;
}
break;
case TokenizeStateMultiLineComment:
switch (c) {
case '*':
t.state = TokenizeStateMultiLineCommentStar;
break;
case '/':
t.state = TokenizeStateMultiLineCommentSlash;
break;
default:
// do nothing
break;
}
break;
case TokenizeStateMultiLineCommentSlash:
switch (c) {
case '*':
t.state = TokenizeStateMultiLineComment;
t.multi_line_comment_count += 1;
break;
case '/':
break;
default:
t.state = TokenizeStateMultiLineComment;
break;
}
break;
case TokenizeStateMultiLineCommentStar:
switch (c) {
case '/':
t.multi_line_comment_count -= 1;
if (t.multi_line_comment_count == 0) {
t.state = TokenizeStateStart;
} else {
t.state = TokenizeStateMultiLineComment;
}
break;
case '*':
break;
default:
t.state = TokenizeStateMultiLineComment;
break;
}
break;
case TokenizeStateSymbol: case TokenizeStateSymbol:
switch (c) { switch (c) {
case SYMBOL_CHAR: case SYMBOL_CHAR:
@ -295,7 +379,18 @@ ZigList<Token> *tokenize(Buf *buf) {
case TokenizeStateSawDash: case TokenizeStateSawDash:
end_token(&t); end_token(&t);
break; break;
case TokenizeStateSawSlash:
tokenize_error(&t, "unexpected EOF");
break;
case TokenizeStateLineComment:
break;
case TokenizeStateMultiLineComment:
case TokenizeStateMultiLineCommentSlash:
case TokenizeStateMultiLineCommentStar:
tokenize_error(&t, "unterminated multi-line comment");
break;
} }
t.pos = -1;
begin_token(&t, TokenIdEof); begin_token(&t, TokenIdEof);
end_token(&t); end_token(&t);
assert(!t.cur_tok); assert(!t.cur_tok);
@ -333,9 +428,11 @@ static const char * token_name(Token *token) {
void print_tokens(Buf *buf, ZigList<Token> *tokens) { void print_tokens(Buf *buf, ZigList<Token> *tokens) {
for (int i = 0; i < tokens->length; i += 1) { for (int i = 0; i < tokens->length; i += 1) {
Token *token = &tokens->at(i); Token *token = &tokens->at(i);
printf("%s ", token_name(token)); fprintf(stderr, "%s ", token_name(token));
fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stdout); if (token->start_pos >= 0) {
printf("\n"); fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stderr);
}
fprintf(stderr, "\n");
} }
} }

8
src/tokenizer.hpp
View File

@ -43,14 +43,6 @@ struct Token {
int start_column; int start_column;
}; };
enum TokenizeState {
TokenizeStateStart,
TokenizeStateSymbol,
TokenizeStateNumber,
TokenizeStateString,
TokenizeStateSawDash,
};
ZigList<Token> *tokenize(Buf *buf); ZigList<Token> *tokenize(Buf *buf);
void print_tokens(Buf *buf, ZigList<Token> *tokens); void print_tokens(Buf *buf, ZigList<Token> *tokens);

25
test/standalone.cpp
View File

@ -75,6 +75,26 @@ static void add_all_test_cases(void) {
exit(0); exit(0);
} }
)SOURCE", "OK\n"); )SOURCE", "OK\n");
add_simple_case("comments", R"SOURCE(
#link("c")
extern {
fn puts(s: *mut u8) -> i32;
fn exit(code: i32) -> unreachable;
}
/**
* multi line doc comment
*/
fn another_function() -> i32 { return 0; }
/// this is a documentation comment
/// doc comment line 2
fn _start() -> unreachable {
puts(/* mid-line comment /* nested */ */ "OK");
exit(0);
}
)SOURCE", "OK\n");
} }
static void run_test(TestCase *test_case) { static void run_test(TestCase *test_case) {
@ -83,11 +103,12 @@ static void run_test(TestCase *test_case) {
Buf zig_stderr = BUF_INIT; Buf zig_stderr = BUF_INIT;
Buf zig_stdout = BUF_INIT; Buf zig_stdout = BUF_INIT;
int return_code; int return_code;
os_exec_process("./zig", test_case->compiler_args, &return_code, &zig_stderr, &zig_stdout); static const char *zig_exe = "./zig";
os_exec_process(zig_exe, test_case->compiler_args, &return_code, &zig_stderr, &zig_stdout);
if (return_code != 0) { if (return_code != 0) {
printf("\nCompile failed with return code %d:\n", return_code); printf("\nCompile failed with return code %d:\n", return_code);
printf("zig"); printf("%s", zig_exe);
for (int i = 0; i < test_case->compiler_args.length; i += 1) { for (int i = 0; i < test_case->compiler_args.length; i += 1) {
printf(" %s", test_case->compiler_args.at(i)); printf(" %s", test_case->compiler_args.at(i));
} }