From 6df9e9fe47ad79f362a57279751374b6966782ed Mon Sep 17 00:00:00 2001 From: frmdstryr Date: Sun, 29 Dec 2019 10:41:26 -0500 Subject: [PATCH] Cleanup c_tokenizer.zig tests --- src-self-hosted/c_tokenizer.zig | 215 +++++++++++++++++++++----------- 1 file changed, 143 insertions(+), 72 deletions(-) diff --git a/src-self-hosted/c_tokenizer.zig b/src-self-hosted/c_tokenizer.zig index 1bc3c5f25c..e82a2b5aad 100644 --- a/src-self-hosted/c_tokenizer.zig +++ b/src-self-hosted/c_tokenizer.zig @@ -24,21 +24,26 @@ pub const CToken = struct { RParen, Eof, Dot, - Asterisk, - Ampersand, - And, - Or, - Bang, - Tilde, - Shl, - Shr, - Lt, - Gt, - Increment, - Decrement, + Asterisk, // * + Ampersand, // & + And, // && + Assign, // = + Or, // || + Bang, // ! + Tilde, // ~ + Shl, // << + Shr, // >> + Lt, // < + Lte, // <= + Gt, // > + Gte, // >= + Eq, // == + Ne, // != + Increment, // ++ + Decrement, // -- Comma, Fn, - Arrow, + Arrow, // -> LBrace, RBrace, Pipe, @@ -239,6 +244,8 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*: GotMinus, GotAmpersand, GotPipe, + GotBang, + GotEq, CharLit, OpenComment, Comment, @@ -298,6 +305,8 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*: .GotPlus, .GotAmpersand, .GotPipe, + .GotBang, + .GotEq, => { return result; }, @@ -387,12 +396,16 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*: }, '!' => { result.id = .Bang; - state = .Done; + state = .GotBang; }, '~' => { result.id = .Tilde; state = .Done; }, + '=' => { + result.id = .Assign; + state = .GotEq; + }, ',' => { result.id = .Comma; state = .Done; @@ -438,9 +451,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*: result.id = .Decrement; state = .Done; }, - else => { - return result; - }, + else => return result, } }, .GotPlus => { @@ -449,9 +460,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*: result.id = .Increment; state = .Done; }, - else => { - return result; - }, + else => return result, } }, .GotLt => { @@ -460,9 +469,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*: result.id = .Shl; state = .Done; }, - else => { - return result; + '=' => { + result.id = .Lte; + state = .Done; }, + else => return result, } }, .GotGt => { @@ -471,9 +482,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*: result.id = .Shr; state = .Done; }, - else => { - return result; + '=' => { + result.id = .Gte; + state = .Done; }, + else => return result, } }, .GotPipe => { @@ -482,9 +495,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*: result.id = .Or; state = .Done; }, - else => { - return result; - }, + else => return result, } }, .GotAmpersand => { @@ -493,9 +504,25 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*: result.id = .And; state = .Done; }, - else => { - return result; + else => return result, + } + }, + .GotBang => { + switch (c) { + '=' => { + result.id = .Ne; + state = .Done; }, + else => return result, + } + }, + .GotEq => { + switch (c) { + '=' => { + result.id = .Eq; + state = .Done; + }, + else => return result, } }, .Float => { @@ -802,57 +829,101 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*: unreachable; } + +fn expectTokens(tl: *TokenList, src: [*:0]const u8, expected: []CToken) void { + tokenizeCMacro(tl, src) catch unreachable; + var it = tl.iterator(0); + for (expected) |t| { + var tok = it.next().?; + std.testing.expectEqual(t.id, tok.id); + if (t.bytes.len > 0) { + //std.debug.warn(" {} = {}\n", .{tok.bytes, t.bytes}); + std.testing.expectEqualSlices(u8, tok.bytes, t.bytes); + } + if (t.num_lit_suffix != .None) { + std.testing.expectEqual(t.num_lit_suffix, tok.num_lit_suffix); + } + } + std.testing.expect(it.next() == null); + tl.shrink(0); +} + + test "tokenize macro" { var tl = TokenList.init(std.heap.page_allocator); defer tl.deinit(); - const src = "TEST(0\n"; - try tokenizeCMacro(&tl, src); - var it = tl.iterator(0); - expect(it.next().?.id == .Identifier); - expect(it.next().?.id == .Fn); - expect(it.next().?.id == .LParen); - expect(std.mem.eql(u8, it.next().?.bytes, "0")); - expect(it.next().?.id == .Eof); - expect(it.next() == null); - tl.shrink(0); + expectTokens(&tl, "TEST(0\n", &[_]CToken{ + ctoken(.Identifier, "TEST"), + ctoken(.Fn, ""), + ctoken(.LParen, ""), + ctoken(.NumLitInt, "0"), + ctoken(.Eof, ""), + }); - const src2 = "__FLT_MIN_10_EXP__ -37\n"; - try tokenizeCMacro(&tl, src2); - it = tl.iterator(0); - expect(std.mem.eql(u8, it.next().?.bytes, "__FLT_MIN_10_EXP__")); - expect(it.next().?.id == .Minus); - expect(std.mem.eql(u8, it.next().?.bytes, "37")); - expect(it.next().?.id == .Eof); - expect(it.next() == null); - tl.shrink(0); + expectTokens(&tl, "__FLT_MIN_10_EXP__ -37\n", &[_]CToken{ + ctoken(.Identifier, "__FLT_MIN_10_EXP__"), + ctoken(.Minus, ""), + ctoken(.NumLitInt, "37"), + ctoken(.Eof, ""), + }); - const src3 = "__llvm__ 1\n#define"; - try tokenizeCMacro(&tl, src3); - it = tl.iterator(0); - expect(std.mem.eql(u8, it.next().?.bytes, "__llvm__")); - expect(std.mem.eql(u8, it.next().?.bytes, "1")); - expect(it.next().?.id == .Eof); - expect(it.next() == null); - tl.shrink(0); + expectTokens(&tl, "__llvm__ 1\n#define", &[_]CToken{ + ctoken(.Identifier, "__llvm__"), + ctoken(.NumLitInt, "1"), + ctoken(.Eof, ""), - const src4 = "TEST 2"; - try tokenizeCMacro(&tl, src4); - it = tl.iterator(0); - expect(it.next().?.id == .Identifier); - expect(std.mem.eql(u8, it.next().?.bytes, "2")); - expect(it.next().?.id == .Eof); - expect(it.next() == null); - tl.shrink(0); + }); - const src5 = "FOO 0ull"; - try tokenizeCMacro(&tl, src5); - it = tl.iterator(0); - expect(it.next().?.id == .Identifier); - expect(std.mem.eql(u8, it.next().?.bytes, "0")); - expect(it.next().?.id == .Eof); - expect(it.next() == null); - tl.shrink(0); + expectTokens(&tl, "TEST 2", &[_]CToken{ + ctoken(.Identifier, "TEST"), + ctoken(.NumLitInt, "2"), + ctoken(.Eof, ""), + + }); + + expectTokens(&tl, "FOO 0ull", &[_]CToken{ + ctoken(.Identifier, "FOO"), + cnumtoken(.LLU, "0"), + ctoken(.Eof, ""), + + }); + +} + + + +test "tokenize macro ops" { + var tl = TokenList.init(std.heap.page_allocator); + defer tl.deinit(); + + expectTokens(&tl, "ADD A + B", &[_]CToken{ + ctoken(.Identifier, "ADD"), + ctoken(.Identifier, "A"), + ctoken(.Plus, ""), + ctoken(.Identifier, "B"), + ctoken(.Eof, ""), + }); + + expectTokens(&tl, "ADD (A) + B", &[_]CToken{ + ctoken(.Identifier, "ADD"), + ctoken(.LParen, ""), + ctoken(.Identifier, "A"), + ctoken(.RParen, ""), + ctoken(.Plus, ""), + ctoken(.Identifier, "B"), + ctoken(.Eof, ""), + }); + + expectTokens(&tl, "ADD (A) + B", &[_]CToken{ + ctoken(.Identifier, "ADD"), + ctoken(.LParen, ""), + ctoken(.Identifier, "A"), + ctoken(.RParen, ""), + ctoken(.Plus, ""), + ctoken(.Identifier, "B"), + ctoken(.Eof, ""), + }); } test "escape sequences" {