From 5118caf5ab2599ca61b57f68a89aa2d094f51981 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 00:53:00 +0200 Subject: [PATCH 01/54] Added a lot of test cases --- std/zig/parser.zig | 150 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 149 insertions(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 62c62ed185..3b532d7030 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1648,6 +1648,13 @@ test "zig fmt" { \\ ); + try testCanonical( + \\extern fn f1(s: [][]align(1) []const []volatile u8) c_int; + \\extern fn f2(s: []align(1) const []align(1) volatile []const volatile u8) c_int; + \\extern fn f3(s: []align(1) const volatile u8) c_int; + \\ + ); + try testCanonical( \\fn f1(a: bool, b: bool) bool { \\ a != b; @@ -1716,7 +1723,7 @@ test "zig fmt" { try testCanonical( \\test "prefix operators" { - \\ --%~??!*&0; + \\ try return --%~??!*&0; \\} \\ ); @@ -1730,4 +1737,145 @@ test "zig fmt" { \\} \\ ); + + try testCanonical( + \\test "test index" { + \\ a[0]; + \\ a[0 + 5]; + \\ a[0..]; + \\ a[0..5]; + \\} + \\ + ); + + try testCanonical( + \\test "test array" { + \\ const a : [2]u8 = [2]u8{ 1, 2 }; + \\ const a : [2]u8 = []u8{ 1, 2 }; + \\ const a : [0]u8 = []u8{}; + \\} + \\ + ); + +// PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType + try testCanonical( + \\test "values" { + \\ 1; + \\ 1.0; + \\ "string"; + \\ c"cstring"; + \\ \\ Multi + \\ \\ line + \\ \\ string + \\ ; + \\ 'c'; + \\ true; + \\ false; + \\ null; + \\ undefined; + \\ error; + \\ this; + \\ unreachable; + \\ suspend; + \\} + \\ + ); + + try testCanonical( + \\test "percendence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ !a{}; + \\ !(a{}); + \\ a + b{}; + \\ (a + b){}; + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\ a = b or c; + \\ (a = b) or c; + \\} + \\ + ); + + try testCanonical( + \\const S = struct { + \\ const Self = this; + \\ f1: u8, + \\ + \\ fn method(self: &Self) Self { + \\ return *self; + \\ } + \\ + \\ f2: u8, + \\}; + \\ + \\const Ps = packed struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8, + \\}; + \\ + \\const Es = extern struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8, + \\}; + \\ + ); + + try testCanonical( + \\const E = enum { + \\ Ok, + \\ SomethingElse = 0, + \\}; + \\ + \\const E2 = enum(u8) { + \\ Ok, + \\ SomethingElse = 255, + \\ SomethingThird, + \\}; + \\ + \\const Ee = extern enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird, + \\}; + \\ + \\const Ep = packed enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird, + \\}; + \\ + ); + + try testCanonical( + \\const a1 = []u8{ }; + \\const a2 = []u8{ 1, 2, 3, 4 }; + \\const s1 = S{ }; + \\const s2 = S{ .a = 1, .b = 2, }; + \\ + ); + + try testCanonical(@embedFile("ast.zig")); + try testCanonical(@embedFile("index.zig")); + try testCanonical(@embedFile("parser.zig")); + try testCanonical(@embedFile("tokenizer.zig")); } From 596f4b6002d3ed29adc8777b0417952cb79ad888 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 14:00:49 +0200 Subject: [PATCH 02/54] Fixed review commented code --- std/zig/parser.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3b532d7030..b9246033dd 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1750,9 +1750,9 @@ test "zig fmt" { try testCanonical( \\test "test array" { - \\ const a : [2]u8 = [2]u8{ 1, 2 }; - \\ const a : [2]u8 = []u8{ 1, 2 }; - \\ const a : [0]u8 = []u8{}; + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{}; \\} \\ ); @@ -1782,7 +1782,7 @@ test "zig fmt" { ); try testCanonical( - \\test "percendence" { + \\test "precendence" { \\ a!b(); \\ (a!b)(); \\ !a!b; From 26e56f2fab7c56829d51db1afb10acb0306a101f Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 14:18:09 +0200 Subject: [PATCH 03/54] Each test now have it's own test name --- std/zig/parser.zig | 122 +++++++++++++++++++++++++++++---------------- 1 file changed, 78 insertions(+), 44 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index b9246033dd..9b493d3b4b 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1557,7 +1557,7 @@ fn testCanonical(source: []const u8) !void { } } -test "zig fmt" { +test "zig fmt: get stdout or fail" { try testCanonical( \\const std = @import("std"); \\ @@ -1568,7 +1568,9 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: preserve spacing" { try testCanonical( \\const std = @import("std"); \\ @@ -1581,25 +1583,33 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: return types" { try testCanonical( \\pub fn main() !void {} \\pub fn main() var {} \\pub fn main() i32 {} \\ ); +} +test "zig fmt: imports" { try testCanonical( \\const std = @import("std"); \\const std = @import(); \\ ); +} +test "zig fmt: extern function" { try testCanonical( \\extern fn puts(s: &const u8) c_int; \\ ); +} +test "zig fmt: global declarations" { try testCanonical( \\const a = b; \\pub const a = b; @@ -1611,66 +1621,71 @@ test "zig fmt" { \\pub var a: i32 = b; \\ ); +} +test "zig fmt: extern declaration" { try testCanonical( \\extern var foo: c_int; \\ ); +} - try testCanonical( +test "zig fmt: alignment" { + try testCanonical( \\var foo: c_int align(1); \\ ); +} +test "zig fmt: C main" { try testCanonical( \\fn main(argc: c_int, argv: &&u8) c_int { \\ const a = b; \\} \\ ); +} +test "zig fmt: return" { try testCanonical( \\fn foo(argc: c_int, argv: &&u8) c_int { \\ return 0; \\} \\ ); +} +test "zig fmt: pointer attributes" { try testCanonical( \\extern fn f1(s: &align(&u8) u8) c_int; + \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; + \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; + \\extern fn f4(s: &align(1) const volatile u8) c_int; \\ ); +} +test "zig fmt: slice attributes" { try testCanonical( - \\extern fn f1(s: &&align(1) &const &volatile u8) c_int; - \\extern fn f2(s: &align(1) const &align(1) volatile &const volatile u8) c_int; - \\extern fn f3(s: &align(1) const volatile u8) c_int; + \\extern fn f1(s: &align(&u8) u8) c_int; + \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; + \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; + \\extern fn f4(s: &align(1) const volatile u8) c_int; \\ ); +} - try testCanonical( - \\extern fn f1(s: [][]align(1) []const []volatile u8) c_int; - \\extern fn f2(s: []align(1) const []align(1) volatile []const volatile u8) c_int; - \\extern fn f3(s: []align(1) const volatile u8) c_int; - \\ - ); - - try testCanonical( - \\fn f1(a: bool, b: bool) bool { - \\ a != b; - \\ return a == b; - \\} - \\ - ); - - try testCanonical( +test "zig fmt: test declaration" { + try testCanonical( \\test "test name" { \\ const a = 1; \\ var b = 1; \\} \\ ); +} +test "zig fmt: infix operators" { try testCanonical( \\test "infix operators" { \\ var i = undefined; @@ -1720,14 +1735,18 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: prefix operators" { try testCanonical( \\test "prefix operators" { \\ try return --%~??!*&0; \\} \\ ); +} +test "zig fmt: call expression" { try testCanonical( \\test "test calls" { \\ a(); @@ -1737,27 +1756,9 @@ test "zig fmt" { \\} \\ ); +} - try testCanonical( - \\test "test index" { - \\ a[0]; - \\ a[0 + 5]; - \\ a[0..]; - \\ a[0..5]; - \\} - \\ - ); - - try testCanonical( - \\test "test array" { - \\ const a: [2]u8 = [2]u8{ 1, 2 }; - \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{}; - \\} - \\ - ); - -// PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType +test "zig fmt: values" { try testCanonical( \\test "values" { \\ 1; @@ -1780,9 +1781,34 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: indexing" { try testCanonical( - \\test "precendence" { + \\test "test index" { + \\ a[0]; + \\ a[0 + 5]; + \\ a[0..]; + \\ a[0..5]; + \\} + \\ + ); +} + +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{}; + \\} + \\ + ); +} + +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { \\ a!b(); \\ (a!b)(); \\ !a!b; @@ -1811,7 +1837,9 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: struct declaration" { try testCanonical( \\const S = struct { \\ const Self = this; @@ -1839,8 +1867,10 @@ test "zig fmt" { \\}; \\ ); +} - try testCanonical( +test "zig fmt: enum declaration" { + try testCanonical( \\const E = enum { \\ Ok, \\ SomethingElse = 0, @@ -1865,7 +1895,9 @@ test "zig fmt" { \\}; \\ ); +} +test "zig fmt: container initializers" { try testCanonical( \\const a1 = []u8{ }; \\const a2 = []u8{ 1, 2, 3, 4 }; @@ -1873,9 +1905,11 @@ test "zig fmt" { \\const s2 = S{ .a = 1, .b = 2, }; \\ ); +} +test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); try testCanonical(@embedFile("parser.zig")); try testCanonical(@embedFile("tokenizer.zig")); -} +} \ No newline at end of file From cda35093537221abe7c148c57b3a28892046b8b2 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 15:39:51 +0200 Subject: [PATCH 04/54] Added test cases to cover all of zigs syntax --- std/zig/parser.zig | 352 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 345 insertions(+), 7 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 9b493d3b4b..22e87dd178 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1602,13 +1602,6 @@ test "zig fmt: imports" { ); } -test "zig fmt: extern function" { - try testCanonical( - \\extern fn puts(s: &const u8) c_int; - \\ - ); -} - test "zig fmt: global declarations" { try testCanonical( \\const a = b; @@ -1758,6 +1751,21 @@ test "zig fmt: call expression" { ); } +test "zig fmt: var args" { + try testCanonical( + \\fn print(args: ...) void {} + \\ + ); +} + +test "zig fmt: extern function" { + try testCanonical( + \\extern fn puts(s: &const u8) c_int; + \\extern "c" fn puts(s: &const u8) c_int; + \\ + ); +} + test "zig fmt: values" { try testCanonical( \\test "values" { @@ -1897,6 +1905,41 @@ test "zig fmt: enum declaration" { ); } +test "zig fmt: union declaration" { + try testCanonical( + \\const U = union { + \\ Int: u8, + \\ Float: f32, + \\ Bool: bool, + \\}; + \\ + \\const Ue = union(enum) { + \\ Int: u8, + \\ Float: f32, + \\ Bool: bool, + \\}; + \\ + \\const E = enum { + \\ Int, + \\ Float, + \\ Bool, + \\}; + \\ + \\const Ue2 = union(E) { + \\ Int: u8, + \\ Float: f32, + \\ Bool: bool, + \\}; + \\ + \\const Eu = extern union { + \\ Int: u8, + \\ Float: f32, + \\ Bool: bool, + \\}; + \\ + ); +} + test "zig fmt: container initializers" { try testCanonical( \\const a1 = []u8{ }; @@ -1907,6 +1950,301 @@ test "zig fmt: container initializers" { ); } +test "zig fmt: switch" { + try testCanonical( + \\test "switch" { + \\ switch (0) { + \\ 0 => {}, + \\ 1 => unreachable, + \\ 2, 3 => {}, + \\ 4 ... 7 => {}, + \\ 1 + 4 * 3 + 22 => {}, + \\ else => { + \\ const a = 1; + \\ const b = a; + \\ }, + \\ } + \\ + \\ const res = switch (0) { + \\ 0 => 0, + \\ 1 => 2, + \\ else => 4, + \\ }; + \\ + \\ const Union = union(enum) { + \\ Int: i64, + \\ Float: f64, + \\ }; + \\ + \\ const u = Union { .Int = 0 }; + \\ switch (u) { + \\ Union.Int => |int| {}, + \\ Union.Float => |*float| unreachable, + \\ } + \\} + \\ + ); +} + +test "zig fmt: while" { + try testCanonical( + \\test "while" { + \\ while (10 < 1) { + \\ unreachable; + \\ } + \\ + \\ while (10 < 1) + \\ unreachable; + \\ + \\ var i: usize = 0; + \\ while (i < 10) : (i += 1) { + \\ continue; + \\ } + \\ + \\ i = 0; + \\ while (i < 10) : (i += 1) + \\ continue; + \\ + \\ i = 0; + \\ var j usize = 0; + \\ while (i < 10) : ({ i += 1; j += 1; }) { + \\ continue; + \\ } + \\ + \\ var a: ?u8 = 2; + \\ while (a) |v| : (a = null) { + \\ continue; + \\ } + \\ + \\ while (a) |v| : (a = null) + \\ unreachable; + \\ + \\ label: while (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ const res = while (0 < 10) { + \\ break 7; + \\ } else { + \\ unreachable; + \\ } + \\ + \\ var a: error!u8 = 0; + \\ while (a) |v| { + \\ a = error.Err; + \\ } else |err| { + \\ i = 1; + \\ } + \\ + \\ comptime var k: usize = 0; + \\ inline while (i < 10) (i += 1) + \\ j += 2; + \\} + \\ + ); +} + +test "zig fmt: for" { + try testCanonical( + \\test "for" { + \\ const a = []u8{ 1, 2, 3 }; + \\ for (a) |v| { + \\ continue; + \\ } + \\ + \\ for (a) |v| + \\ continue; + \\ + \\ for (a) |*v| + \\ continue; + \\ + \\ for (a) |v, i| { + \\ continue; + \\ } + \\ + \\ for (a) |v, i| + \\ continue; + \\ + \\ const res = for (a) |v, i| { + \\ breal v; + \\ } else { + \\ unreachable; + \\ } + \\ + \\ var num: usize = 0; + \\ inline for (a) |v, i| { + \\ num += v; + \\ num += i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: if" { + try testCanonical( + \\test "if" { + \\ if (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ if (10 < 0) + \\ unreachable; + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else if (5 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ const is_world_broken = if (10 < 0) true else false; + \\ + \\ const a: ?u8 = 10; + \\ const b: ?u8 = null; + \\ if (a) |v| { + \\ const some = v; + \\ } else if (b) |*v| { + \\ unreachable; + \\ } else { + \\ const some = 10; + \\ } + \\ + \\ const non_null_a = if (a) |v| v else 0; + \\ + \\ const a_err: error!u8 = 0; + \\ if (a_err) |v| { + \\ const p = v; + \\ } else |err| { + \\ unreachable; + \\ } + \\} + \\ + ); +} + +test "zig fmt: defer" { + try testCanonical( + \\test "defer" { + \\ var i: usize = 0; + \\ defer i = 1; + \\ defer { + \\ i += 2; + \\ i *= i; + \\ } + \\ + \\ errdefer i += 3; + \\ errdefer { + \\ i += 2; + \\ i /= i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: error!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch |err| return; + \\} + \\ + ); +} + +test "zig fmt: comptime" { + try testCanonical( + \\fn a() u8 { + \\ return 5; + \\} + \\ + \\fn b(comptime i: u8) u8 { + \\ return i; + \\} + \\ + \\const av = comptime a(); + \\const av2 = comptime blk: { + \\ var res = a(); + \\ res *= b(2); + \\ break :blk res; + \\}; + \\ + \\comptime { + \\ _ = a(); + \\} + \\ + \\test "comptime" { + \\ const av3 = comptime a(); + \\ const av4 = comptime blk: { + \\ var res = a(); + \\ res *= a(); + \\ break :blk res; + \\ }; + \\ + \\ comptime var i = 0; + \\ comptime { + \\ i = a(); + \\ i += b(i); + \\ } + \\} + \\ + ); +} + +test "zig fmt: fn type" { + try testCanonical( + \\fn a(i: u8) u8 { + \\ return i + 1; + \\} + \\ + \\const ap: fn(u8) u8 = a; + \\ + ); +} + +test "zig fmt: inline asm" { + try testCanonical( + \\pub fn syscall1(number: usize, arg1: usize) usize { + \\ return asm volatile ("syscall" + \\ : [ret] "={rax}" (-> usize) + \\ : [number] "{rax}" (number), + \\ [arg1] "{rdi}" (arg1) + \\ : "rcx", "r11"); + \\} + \\ + ); +} + +test "zig fmt: coroutines" { + try testCanonical( + \\async fn simpleAsyncFn() void { + \\ x += 1; + \\ suspend; + \\ x += 1; + \\ suspend |p| { + \\ } + \\ const p = async simpleAsyncFn() cache unreachable; + \\ await p; + \\} + \\ + \\test "coroutine suspend, resume, cancel" { + \\ const p = try async testAsyncSeq(); + \\ resume p; + \\ cancel p; + \\} + \\ + ); +} + test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); From 4793c3397e17169317d2feded9ca6901ff0e99e8 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 17:46:29 +0200 Subject: [PATCH 05/54] std.zig.parser now handles lib name for extern var and fn --- std/zig/ast.zig | 3 +- std/zig/parser.zig | 162 ++++++++++++++++++++++++++++----------------- 2 files changed, 102 insertions(+), 63 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 715a333c0f..e64e5931c3 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -624,7 +624,7 @@ pub const NodeLineComment = struct { pub const NodeTestDecl = struct { base: Node, test_token: Token, - name_token: Token, + name: &Node, body_node: &Node, pub fn iterate(self: &NodeTestDecl, index: usize) ?&Node { @@ -644,4 +644,3 @@ pub const NodeTestDecl = struct { return self.body_node.lastToken(); } }; - diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 22e87dd178..1bad72aea4 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -55,6 +55,7 @@ pub const Parser = struct { const TopLevelDeclCtx = struct { visib_token: ?Token, extern_token: ?Token, + lib_name: ?&ast.Node, }; const DestPtr = union(enum) { @@ -183,8 +184,9 @@ pub const Parser = struct { if (lbrace.id != Token.Id.LBrace) return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); + const name = try self.createStringLiteral(arena, name_token); const block = try self.createBlock(arena, token); - const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, name_token, block); + const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, &name.base, block); try stack.append(State { .Block = block }); continue; }, @@ -202,10 +204,22 @@ pub const Parser = struct { State.TopLevelExtern => |visib_token| { const token = self.getNextToken(); if (token.id == Token.Id.Keyword_extern) { + const lib_name_token = self.getNextToken(); + const lib_name = blk: { + if (lib_name_token.id == Token.Id.StringLiteral) { + const res = try self.createStringLiteral(arena, lib_name_token); + break :blk &res.base; + } else { + self.putBackToken(lib_name_token); + break :blk null; + } + }; + stack.append(State { .TopLevelDecl = TopLevelDeclCtx { .visib_token = visib_token, .extern_token = token, + .lib_name = lib_name, }, }) catch unreachable; continue; @@ -215,6 +229,7 @@ pub const Parser = struct { .TopLevelDecl = TopLevelDeclCtx { .visib_token = visib_token, .extern_token = null, + .lib_name = null, }, }) catch unreachable; continue; @@ -226,7 +241,7 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const var_decl_node = try self.createAttachVarDecl(arena, &root_node.decls, ctx.visib_token, - token, (?Token)(null), ctx.extern_token); + token, (?Token)(null), ctx.extern_token, ctx.lib_name); try stack.append(State { .VarDecl = var_decl_node }); continue; }, @@ -234,20 +249,17 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, token, - ctx.extern_token, (?Token)(null), ctx.visib_token, (?Token)(null)); + ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); continue; }, - Token.Id.StringLiteral => { - @panic("TODO extern with string literal"); - }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { stack.append(State.TopLevel) catch unreachable; const fn_token = try self.eatToken(Token.Id.Keyword_fn); // TODO shouldn't need this cast const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, fn_token, - ctx.extern_token, (?Token)(token), (?Token)(null), (?Token)(null)); + ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); continue; @@ -437,11 +449,7 @@ pub const Parser = struct { continue; }, Token.Id.StringLiteral => { - const node = try arena.create(ast.NodeStringLiteral); - *node = ast.NodeStringLiteral { - .base = self.initNode(ast.Node.Id.StringLiteral), - .token = token, - }; + const node = try self.createStringLiteral(arena, token); try stack.append(State { .Operand = &node.base }); @@ -722,7 +730,7 @@ pub const Parser = struct { if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(comptime_token), (?Token)(null)); + mut_token, (?Token)(comptime_token), (?Token)(null), null); try stack.append(State { .VarDecl = var_decl }); continue; } @@ -736,7 +744,7 @@ pub const Parser = struct { if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(null), (?Token)(null)); + mut_token, (?Token)(null), (?Token)(null), null); try stack.append(State { .VarDecl = var_decl }); continue; } @@ -852,7 +860,7 @@ pub const Parser = struct { } fn createVarDecl(self: &Parser, arena: &mem.Allocator, visib_token: &const ?Token, mut_token: &const Token, - comptime_token: &const ?Token, extern_token: &const ?Token) !&ast.NodeVarDecl + comptime_token: &const ?Token, extern_token: &const ?Token, lib_name: ?&ast.Node) !&ast.NodeVarDecl { const node = try arena.create(ast.NodeVarDecl); @@ -865,7 +873,7 @@ pub const Parser = struct { .type_node = null, .align_node = null, .init_node = null, - .lib_name = null, + .lib_name = lib_name, // initialized later .name_token = undefined, .eq_token = undefined, @@ -874,7 +882,18 @@ pub const Parser = struct { return node; } - fn createTestDecl(self: &Parser, arena: &mem.Allocator, test_token: &const Token, name_token: &const Token, + fn createStringLiteral(self: &Parser, arena: &mem.Allocator, token: &const Token) !&ast.NodeStringLiteral { + const node = try arena.create(ast.NodeStringLiteral); + + assert(token.id == Token.Id.StringLiteral); + *node = ast.NodeStringLiteral { + .base = self.initNode(ast.Node.Id.StringLiteral), + .token = *token, + }; + return node; + } + + fn createTestDecl(self: &Parser, arena: &mem.Allocator, test_token: &const Token, name: &ast.Node, block: &ast.NodeBlock) !&ast.NodeTestDecl { const node = try arena.create(ast.NodeTestDecl); @@ -882,14 +901,14 @@ pub const Parser = struct { *node = ast.NodeTestDecl { .base = self.initNode(ast.Node.Id.TestDecl), .test_token = *test_token, - .name_token = *name_token, + .name = name, .body_node = &block.base, }; return node; } fn createFnProto(self: &Parser, arena: &mem.Allocator, fn_token: &const Token, extern_token: &const ?Token, - cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto + lib_name: ?&ast.Node, cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto { const node = try arena.create(ast.NodeFnProto); @@ -905,7 +924,7 @@ pub const Parser = struct { .inline_token = *inline_token, .cc_token = *cc_token, .body_node = null, - .lib_name = null, + .lib_name = lib_name, .align_expr = null, }; return node; @@ -1015,27 +1034,27 @@ pub const Parser = struct { } fn createAttachFnProto(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), fn_token: &const Token, - extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token, + extern_token: &const ?Token, lib_name: ?&ast.Node, cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto { - const node = try self.createFnProto(arena, fn_token, extern_token, cc_token, visib_token, inline_token); + const node = try self.createFnProto(arena, fn_token, extern_token, lib_name, cc_token, visib_token, inline_token); try list.append(&node.base); return node; } fn createAttachVarDecl(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), visib_token: &const ?Token, mut_token: &const Token, comptime_token: &const ?Token, - extern_token: &const ?Token) !&ast.NodeVarDecl + extern_token: &const ?Token, lib_name: ?&ast.Node) !&ast.NodeVarDecl { - const node = try self.createVarDecl(arena, visib_token, mut_token, comptime_token, extern_token); + const node = try self.createVarDecl(arena, visib_token, mut_token, comptime_token, extern_token, lib_name); try list.append(&node.base); return node; } fn createAttachTestDecl(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), - test_token: &const Token, name_token: &const Token, block: &ast.NodeBlock) !&ast.NodeTestDecl + test_token: &const Token, name: &ast.Node, block: &ast.NodeBlock) !&ast.NodeTestDecl { - const node = try self.createTestDecl(arena, test_token, name_token, block); + const node = try self.createTestDecl(arena, test_token, name, block); try list.append(&node.base); return node; } @@ -1169,23 +1188,6 @@ pub const Parser = struct { switch (decl.id) { ast.Node.Id.FnProto => { const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", decl); - if (fn_proto.visib_token) |visib_token| { - switch (visib_token.id) { - Token.Id.Keyword_pub => try stream.print("pub "), - Token.Id.Keyword_export => try stream.print("export "), - else => unreachable, - } - } - if (fn_proto.extern_token) |extern_token| { - try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token)); - } - try stream.print("fn"); - - if (fn_proto.name_token) |name_token| { - try stream.print(" {}", self.tokenizer.getTokenSlice(name_token)); - } - - try stream.print("("); if (fn_proto.body_node == null) { try stack.append(RenderState { .Text = ";" }); @@ -1201,6 +1203,27 @@ pub const Parser = struct { try stack.append(RenderState { .Text = ", " }); } } + + try stack.append(RenderState { .Text = "(" }); + if (fn_proto.name_token) |name_token| { + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(name_token) }); + } + + try stack.append(RenderState { .Text = "fn " }); + if (fn_proto.lib_name) |lib_name| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = lib_name }); + } + if (fn_proto.extern_token) |extern_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); + } + + if (fn_proto.visib_token) |visib_token| { + assert(visib_token.id == Token.Id.Keyword_pub or visib_token.id == Token.Id.Keyword_export); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) }); + } }, ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); @@ -1208,29 +1231,16 @@ pub const Parser = struct { }, ast.Node.Id.TestDecl => { const test_decl = @fieldParentPtr(ast.NodeTestDecl, "base", decl); - try stream.print("test {} ", self.tokenizer.getTokenSlice(test_decl.name_token)); + try stream.print("test "); try stack.append(RenderState { .Expression = test_decl.body_node }); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = test_decl.name }); }, else => unreachable, } }, RenderState.VarDecl => |var_decl| { - if (var_decl.visib_token) |visib_token| { - try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token)); - } - if (var_decl.extern_token) |extern_token| { - try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token)); - if (var_decl.lib_name != null) { - @panic("TODO"); - } - } - if (var_decl.comptime_token) |comptime_token| { - try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token)); - } - try stream.print("{} ", self.tokenizer.getTokenSlice(var_decl.mut_token)); - try stream.print("{}", self.tokenizer.getTokenSlice(var_decl.name_token)); - try stack.append(RenderState { .Text = ";" }); if (var_decl.init_node) |init_node| { try stack.append(RenderState { .Expression = init_node }); @@ -1242,8 +1252,30 @@ pub const Parser = struct { try stack.append(RenderState { .Text = " align(" }); } if (var_decl.type_node) |type_node| { - try stream.print(": "); try stack.append(RenderState { .Expression = type_node }); + try stack.append(RenderState { .Text = ": " }); + } + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(var_decl.name_token) }); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(var_decl.mut_token) }); + + if (var_decl.comptime_token) |comptime_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(comptime_token) }); + } + + if (var_decl.extern_token) |extern_token| { + if (var_decl.lib_name != null) { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = ??var_decl.lib_name }); + } + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); + } + + if (var_decl.visib_token) |visib_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) }); } }, @@ -1612,6 +1644,14 @@ test "zig fmt: global declarations" { \\pub const a: i32 = b; \\var a: i32 = b; \\pub var a: i32 = b; + \\extern const a: i32 = b; + \\pub extern const a: i32 = b; + \\extern var a: i32 = b; + \\pub extern var a: i32 = b; + \\extern "a" const a: i32 = b; + \\pub extern "a" const a: i32 = b; + \\extern "a" var a: i32 = b; + \\pub extern "a" var a: i32 = b; \\ ); } @@ -2232,7 +2272,7 @@ test "zig fmt: coroutines" { \\ x += 1; \\ suspend |p| { \\ } - \\ const p = async simpleAsyncFn() cache unreachable; + \\ const p = async simpleAsyncFn() catch unreachable; \\ await p; \\} \\ From 4d8f9e2295bb672f70550b3bb5a4cb667f68bb70 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 21:04:54 +0200 Subject: [PATCH 06/54] std.zig.parser now parses multi line strings --- std/zig/ast.zig | 21 +++++++++++++++++ std/zig/parser.zig | 52 +++++++++++++++++++++++++++++++++++++++---- std/zig/tokenizer.zig | 40 ++++++++++++++++++++++++++++++++- 3 files changed, 108 insertions(+), 5 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index e64e5931c3..75003b06cd 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,6 +20,7 @@ pub const Node = struct { IntegerLiteral, FloatLiteral, StringLiteral, + MultilineStringLiteral, UndefinedLiteral, BuiltinCall, Call, @@ -40,6 +41,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), + Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), @@ -61,6 +63,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), + Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), @@ -82,6 +85,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), + Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), @@ -587,6 +591,23 @@ pub const NodeStringLiteral = struct { } }; +pub const NodeMultilineStringLiteral = struct { + base: Node, + tokens: ArrayList(Token), + + pub fn iterate(self: &NodeMultilineStringLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeMultilineStringLiteral) Token { + return self.tokens.at(0); + } + + pub fn lastToken(self: &NodeMultilineStringLiteral) Token { + return self.tokens.at(self.tokens.len - 1); + } +}; + pub const NodeUndefinedLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 1bad72aea4..fa09a26dbb 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -456,6 +456,30 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.MultilineStringLiteralLine => { + const node = try arena.create(ast.NodeMultilineStringLiteral); + *node = ast.NodeMultilineStringLiteral { + .base = self.initNode(ast.Node.Id.MultilineStringLiteral), + .tokens = ArrayList(Token).init(arena), + }; + try node.tokens.append(token); + + while (true) { + const multiline_str = self.getNextToken(); + if (multiline_str.id != Token.Id.MultilineStringLiteralLine) { + self.putBackToken(multiline_str); + break; + } + + try node.tokens.append(multiline_str); + } + + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), } @@ -1427,6 +1451,20 @@ pub const Parser = struct { const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token)); }, + ast.Node.Id.MultilineStringLiteral => { + const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); + try stream.print("\n"); + + var i : usize = 0; + indent += 4; + while (i < multiline_str_literal.tokens.len) : (i += 1) { + const t = multiline_str_literal.tokens.at(i); + try stream.writeByteNTimes(' ', indent); + try stream.print("{}", self.tokenizer.getTokenSlice(t)); + } + try stream.writeByteNTimes(' ', indent); + indent -= 4; + }, ast.Node.Id.UndefinedLiteral => { const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(undefined_literal.token)); @@ -1806,6 +1844,16 @@ test "zig fmt: extern function" { ); } +test "zig fmt: multiline string" { + try testCanonical( + \\const s = + \\ \\ something + \\ \\ something else + \\ ; + \\ + ); +} + test "zig fmt: values" { try testCanonical( \\test "values" { @@ -1813,10 +1861,6 @@ test "zig fmt: values" { \\ 1.0; \\ "string"; \\ c"cstring"; - \\ \\ Multi - \\ \\ line - \\ \\ string - \\ ; \\ 'c'; \\ true; \\ false; diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 7a13d89975..5647fcb866 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -72,6 +72,7 @@ pub const Token = struct { Invalid, Identifier, StringLiteral: StrLitKind, + MultilineStringLiteralLine: StrLitKind, StringIdentifier, Eof, Builtin, @@ -225,6 +226,9 @@ pub const Tokenizer = struct { C, StringLiteral, StringLiteralBackslash, + MultilineStringLiteralLine, + MultilineStringLiteralLineBackslash, + Backslash, Equal, Bang, Pipe, @@ -352,6 +356,10 @@ pub const Tokenizer = struct { '^' => { state = State.Caret; }, + '\\' => { + state = State.Backslash; + result.id = Token.Id { .MultilineStringLiteralLine = Token.StrLitKind.Normal }; + }, '{' => { result.id = Token.Id.LBrace; self.index += 1; @@ -532,8 +540,17 @@ pub const Tokenizer = struct { 'a'...'z', 'A'...'Z', '_', '0'...'9' => {}, else => break, }, + State.Backslash => switch (c) { + '\\' => { + state = State.MultilineStringLiteralLine; + }, + else => break, + }, State.C => switch (c) { - '\\' => @panic("TODO"), + '\\' => { + state = State.Backslash; + result.id = Token.Id { .MultilineStringLiteralLine = Token.StrLitKind.C }; + }, '"' => { state = State.StringLiteral; result.id = Token.Id { .StringLiteral = Token.StrLitKind.C }; @@ -562,6 +579,24 @@ pub const Tokenizer = struct { }, }, + State.MultilineStringLiteralLine => switch (c) { + '\\' => { + state = State.MultilineStringLiteralLineBackslash; + }, + '\n' => { + self.index += 1; + break; + }, + else => self.checkLiteralCharacter(), + }, + + State.MultilineStringLiteralLineBackslash => switch (c) { + '\n' => break, // Look for this error later. + else => { + state = State.MultilineStringLiteralLine; + }, + }, + State.Bang => switch (c) { '=' => { result.id = Token.Id.BangEqual; @@ -811,6 +846,7 @@ pub const Tokenizer = struct { State.FloatFraction, State.FloatExponentNumber, State.StringLiteral, // find this error later + State.MultilineStringLiteralLine, State.Builtin => {}, State.Identifier => { @@ -825,6 +861,8 @@ pub const Tokenizer = struct { State.NumberDot, State.FloatExponentUnsigned, State.SawAtSign, + State.Backslash, + State.MultilineStringLiteralLineBackslash, State.StringLiteralBackslash => { result.id = Token.Id.Invalid; }, From 975dc5a390490794df367b6a9869609a8b14d8f8 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 21:28:40 +0200 Subject: [PATCH 07/54] std.zig.parser now parses char literals --- std/zig/ast.zig | 21 ++++++++++++++++++++ std/zig/parser.zig | 16 +++++++++++++++ std/zig/tokenizer.zig | 46 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 75003b06cd..1c84e12642 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -21,6 +21,7 @@ pub const Node = struct { FloatLiteral, StringLiteral, MultilineStringLiteral, + CharLiteral, UndefinedLiteral, BuiltinCall, Call, @@ -42,6 +43,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index), + Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), @@ -64,6 +66,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(), + Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), @@ -86,6 +89,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(), + Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), @@ -608,6 +612,23 @@ pub const NodeMultilineStringLiteral = struct { } }; +pub const NodeCharLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeCharLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeCharLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeCharLiteral) Token { + return self.token; + } +}; + pub const NodeUndefinedLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index fa09a26dbb..a05328660b 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -456,6 +456,18 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.CharLiteral => { + const node = try arena.create(ast.NodeCharLiteral); + *node = ast.NodeCharLiteral { + .base = self.initNode(ast.Node.Id.CharLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, Token.Id.MultilineStringLiteralLine => { const node = try arena.create(ast.NodeMultilineStringLiteral); *node = ast.NodeMultilineStringLiteral { @@ -1451,6 +1463,10 @@ pub const Parser = struct { const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token)); }, + ast.Node.Id.CharLiteral => { + const char_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(char_literal.token)); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 5647fcb866..64eb627590 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -73,6 +73,7 @@ pub const Token = struct { Identifier, StringLiteral: StrLitKind, MultilineStringLiteralLine: StrLitKind, + CharLiteral, StringIdentifier, Eof, Builtin, @@ -228,6 +229,9 @@ pub const Tokenizer = struct { StringLiteralBackslash, MultilineStringLiteralLine, MultilineStringLiteralLineBackslash, + CharLiteral, + CharLiteralBackslash, + CharLiteralEnd, Backslash, Equal, Bang, @@ -294,6 +298,9 @@ pub const Tokenizer = struct { state = State.StringLiteral; result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal }; }, + '\'' => { + state = State.CharLiteral; + }, 'a'...'b', 'd'...'z', 'A'...'Z', '_' => { state = State.Identifier; result.id = Token.Id.Identifier; @@ -579,6 +586,35 @@ pub const Tokenizer = struct { }, }, + State.CharLiteral => switch (c) { + '\\' => { + state = State.CharLiteralBackslash; + }, + '\'' => break, // Look for this error later. + else => { + if (c < 0x20 or c == 0x7f) + break; // Look for this error later. + + state = State.CharLiteralEnd; + } + }, + + State.CharLiteralBackslash => switch (c) { + '\n' => break, // Look for this error later. + else => { + state = State.CharLiteralEnd; + }, + }, + + State.CharLiteralEnd => switch (c) { + '\'' => { + result.id = Token.Id.CharLiteral; + self.index += 1; + break; + }, + else => break, // Look for this error later. + }, + State.MultilineStringLiteralLine => switch (c) { '\\' => { state = State.MultilineStringLiteralLineBackslash; @@ -847,6 +883,7 @@ pub const Tokenizer = struct { State.FloatExponentNumber, State.StringLiteral, // find this error later State.MultilineStringLiteralLine, + State.CharLiteralEnd, State.Builtin => {}, State.Identifier => { @@ -863,6 +900,8 @@ pub const Tokenizer = struct { State.SawAtSign, State.Backslash, State.MultilineStringLiteralLineBackslash, + State.CharLiteral, + State.CharLiteralBackslash, State.StringLiteralBackslash => { result.id = Token.Id.Invalid; }, @@ -1006,9 +1045,16 @@ test "tokenizer" { }); } +test "tokenizer - chars" { + testTokenize("'c'", []Token.Id {Token.Id.CharLiteral}); +} + test "tokenizer - invalid token characters" { testTokenize("#", []Token.Id{Token.Id.Invalid}); testTokenize("`", []Token.Id{Token.Id.Invalid}); + testTokenize("'c", []Token.Id {Token.Id.Invalid}); + testTokenize("'", []Token.Id {Token.Id.Invalid}); + testTokenize("''", []Token.Id {Token.Id.Invalid}); } test "tokenizer - invalid literal/comment characters" { From aabf7cf57e62379d5f21aa013d0735c5427eace2 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 22:10:49 +0200 Subject: [PATCH 08/54] std.zig.parser now parses null and bool literals --- std/zig/ast.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ std/zig/parser.zig | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 1c84e12642..6c06b5423d 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -22,6 +22,8 @@ pub const Node = struct { StringLiteral, MultilineStringLiteral, CharLiteral, + BoolLiteral, + NullLiteral, UndefinedLiteral, BuiltinCall, Call, @@ -44,6 +46,8 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index), Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).iterate(index), + Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).iterate(index), + Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), @@ -67,6 +71,8 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(), Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).firstToken(), + Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).firstToken(), + Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), @@ -90,6 +96,8 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(), Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).lastToken(), + Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).lastToken(), + Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), @@ -629,6 +637,40 @@ pub const NodeCharLiteral = struct { } }; +pub const NodeBoolLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeBoolLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeBoolLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeBoolLiteral) Token { + return self.token; + } +}; + +pub const NodeNullLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeNullLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeNullLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeNullLiteral) Token { + return self.token; + } +}; + pub const NodeUndefinedLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index a05328660b..6c0a1f3f19 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -427,6 +427,30 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_true, Token.Id.Keyword_false => { + const node = try arena.create(ast.NodeBoolLiteral); + *node = ast.NodeBoolLiteral { + .base = self.initNode(ast.Node.Id.BoolLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, + Token.Id.Keyword_null => { + const node = try arena.create(ast.NodeNullLiteral); + *node = ast.NodeNullLiteral { + .base = self.initNode(ast.Node.Id.NullLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -1467,6 +1491,14 @@ pub const Parser = struct { const char_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(char_literal.token)); }, + ast.Node.Id.BoolLiteral => { + const bool_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(bool_literal.token)); + }, + ast.Node.Id.NullLiteral => { + const null_literal = @fieldParentPtr(ast.NodeNullLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(null_literal.token)); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); From df09c01f7f141a384010d17ab23db3b36316f5b6 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 22:48:12 +0200 Subject: [PATCH 09/54] std.zig.parser now parses error, this and unreachable --- std/zig/ast.zig | 63 ++++++++++++++++++++++++++++++++++++++++++++++ std/zig/parser.zig | 48 +++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 6c06b5423d..8a9a072d67 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -25,6 +25,9 @@ pub const Node = struct { BoolLiteral, NullLiteral, UndefinedLiteral, + ThisLiteral, + Unreachable, + ErrorType, BuiltinCall, Call, LineComment, @@ -49,6 +52,9 @@ pub const Node = struct { Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).iterate(index), Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), + Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).iterate(index), + Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), + Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), @@ -74,6 +80,9 @@ pub const Node = struct { Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).firstToken(), Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), + Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).firstToken(), + Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), + Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), @@ -99,6 +108,9 @@ pub const Node = struct { Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).lastToken(), Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), + Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).lastToken(), + Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), + Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), @@ -688,6 +700,57 @@ pub const NodeUndefinedLiteral = struct { } }; +pub const NodeThisLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeThisLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeThisLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeThisLiteral) Token { + return self.token; + } +}; + +pub const NodeUnreachable = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeUnreachable, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeUnreachable) Token { + return self.token; + } + + pub fn lastToken(self: &NodeUnreachable) Token { + return self.token; + } +}; + +pub const NodeErrorType = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeErrorType, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeErrorType) Token { + return self.token; + } + + pub fn lastToken(self: &NodeErrorType) Token { + return self.token; + } +}; + pub const NodeLineComment = struct { base: Node, lines: ArrayList(Token), diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 6c0a1f3f19..0b8a15d3dd 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -451,6 +451,42 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_this => { + const node = try arena.create(ast.NodeThisLiteral); + *node = ast.NodeThisLiteral { + .base = self.initNode(ast.Node.Id.ThisLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, + Token.Id.Keyword_unreachable => { + const node = try arena.create(ast.NodeUnreachable); + *node = ast.NodeUnreachable { + .base = self.initNode(ast.Node.Id.Unreachable), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, + Token.Id.Keyword_error => { + const node = try arena.create(ast.NodeErrorType); + *node = ast.NodeErrorType { + .base = self.initNode(ast.Node.Id.ErrorType), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -1499,6 +1535,18 @@ pub const Parser = struct { const null_literal = @fieldParentPtr(ast.NodeNullLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(null_literal.token)); }, + ast.Node.Id.ThisLiteral => { + const this_literal = @fieldParentPtr(ast.NodeThisLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(this_literal.token)); + }, + ast.Node.Id.Unreachable => { + const unreachable_node = @fieldParentPtr(ast.NodeUnreachable, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(unreachable_node.token)); + }, + ast.Node.Id.ErrorType => { + const error_type = @fieldParentPtr(ast.NodeErrorType, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(error_type.token)); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); From b9093185f748293064e7eeaaa07e7479099420ed Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sun, 1 Apr 2018 22:02:51 +0200 Subject: [PATCH 10/54] std.zig.parser now parses slicing and array access --- std/zig/ast.zig | 66 +++++++++++++++++++++++++++++++ std/zig/parser.zig | 92 +++++++++++++++++++++++++++++++++++++++++++ std/zig/tokenizer.zig | 12 ++++++ 3 files changed, 170 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 8a9a072d67..680e617962 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -30,6 +30,8 @@ pub const Node = struct { ErrorType, BuiltinCall, Call, + ArrayAccess, + SliceExpression, LineComment, TestDecl, }; @@ -57,6 +59,8 @@ pub const Node = struct { Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), + Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).iterate(index), + Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), }; @@ -85,6 +89,8 @@ pub const Node = struct { Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), + Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).firstToken(), + Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), }; @@ -113,6 +119,8 @@ pub const Node = struct { Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), + Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).lastToken(), + Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), }; @@ -598,6 +606,64 @@ pub const NodeCall = struct { } }; +pub const NodeArrayAccess = struct { + base: Node, + expr: &Node, + index: &Node, + rbracket_token: Token, + + pub fn iterate(self: &NodeArrayAccess, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + if (i < 1) return self.index; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeArrayAccess) Token { + return self.expr.firstToken(); + } + + pub fn lastToken(self: &NodeArrayAccess) Token { + return self.rbracket_token; + } +}; + +pub const NodeSliceExpression = struct { + base: Node, + expr: &Node, + start: &Node, + end: ?&Node, + rbracket_token: Token, + + pub fn iterate(self: &NodeSliceExpression, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.callee; + i -= 1; + + if (i < 1) return self.start; + i -= 1; + + if (i < 1) return self.end; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeSliceExpression) Token { + return self.expr.firstToken(); + } + + pub fn lastToken(self: &NodeSliceExpression) Token { + return self.rbracket_token; + } +}; + pub const NodeStringLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0b8a15d3dd..d0ab7e17d8 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -88,6 +88,7 @@ pub const Parser = struct { InfixOp: &ast.NodeInfixOp, PrefixOp: &ast.NodePrefixOp, SuffixOp: &ast.Node, + SliceOrArrayAccess, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, TypeExpr: DestPtr, VarDecl: &ast.NodeVarDecl, @@ -590,6 +591,11 @@ pub const Parser = struct { }); continue; + } else if (token.id == Token.Id.LBracket) { + try stack.append(State.SliceOrArrayAccess); + try stack.append(State.ExpectOperand); + continue; + // TODO: Parse postfix operator } else { // no postfix/infix operator after this operand. @@ -603,6 +609,53 @@ pub const Parser = struct { try dest_ptr.store(expression); break; }, + State.SliceOrArrayAccess => { + var rbracket_or_ellipsis2_token = self.getNextToken(); + + switch (rbracket_or_ellipsis2_token.id) { + Token.Id.Ellipsis2 => { + const node = try arena.create(ast.NodeSliceExpression); + *node = ast.NodeSliceExpression { + .base = self.initNode(ast.Node.Id.SliceExpression), + .expr = undefined, + .start = expression, + .end = null, + .rbracket_token = undefined, + }; + + try stack.append(State { .SuffixOp = &node.base }); + try stack.append(State.AfterOperand); + + const rbracket_token = self.getNextToken(); + if (rbracket_token.id != Token.Id.RBracket) { + self.putBackToken(rbracket_token); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RBracket, + .ptr = &node.rbracket_token, + } + }); + try stack.append(State { .Expression = DestPtr { .NullableField = &node.end } }); + } else { + node.rbracket_token = rbracket_token; + } + break; + }, + Token.Id.RBracket => { + const node = try arena.create(ast.NodeArrayAccess); + *node = ast.NodeArrayAccess { + .base = self.initNode(ast.Node.Id.ArrayAccess), + .expr = undefined, + .index = expression, + .rbracket_token = token, + }; + try stack.append(State { .SuffixOp = &node.base }); + try stack.append(State.AfterOperand); + break; + }, + else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id)) + } + }, State.InfixOp => |infix_op| { infix_op.rhs = expression; infix_op.lhs = popSuffixOp(&stack); @@ -857,6 +910,7 @@ pub const Parser = struct { State.PrefixOp => unreachable, State.SuffixOp => unreachable, State.Operand => unreachable, + State.SliceOrArrayAccess => unreachable, } } } @@ -874,6 +928,18 @@ pub const Parser = struct { left_leaf_ptr = &call.callee; continue; }, + ast.Node.Id.ArrayAccess => { + const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", suffix_op); + *left_leaf_ptr = &arr_access.base; + left_leaf_ptr = &arr_access.expr; + continue; + }, + ast.Node.Id.SliceExpression => { + const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", suffix_op); + *left_leaf_ptr = &slice_expr.base; + left_leaf_ptr = &slice_expr.expr; + continue; + }, else => unreachable, } }, @@ -1594,6 +1660,24 @@ pub const Parser = struct { try stack.append(RenderState { .Text = "("}); try stack.append(RenderState { .Expression = call.callee }); }, + ast.Node.Id.ArrayAccess => { + const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", base); + try stack.append(RenderState { .Text = "]"}); + try stack.append(RenderState { .Expression = arr_access.index}); + try stack.append(RenderState { .Text = "["}); + try stack.append(RenderState { .Expression = arr_access.expr }); + }, + ast.Node.Id.SliceExpression => { + const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", base); + try stack.append(RenderState { .Text = "]"}); + if (slice_expr.end) |end| { + try stack.append(RenderState { .Expression = end}); + } + try stack.append(RenderState { .Text = ".."}); + try stack.append(RenderState { .Expression = slice_expr.start}); + try stack.append(RenderState { .Text = "["}); + try stack.append(RenderState { .Expression = slice_expr.expr}); + }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), @@ -1978,6 +2062,14 @@ test "zig fmt: indexing" { \\ a[0 + 5]; \\ a[0..]; \\ a[0..5]; + \\ a[a[0]]; + \\ a[a[0..]]; + \\ a[a[0..5]]; + \\ a[a[0]..]; + \\ a[a[0..5]..]; + \\ a[a[0]..a[0]]; + \\ a[a[0..5]..a[0]]; + \\ a[a[0..5]..a[0..5]]; \\} \\ ); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 64eb627590..1014bacb30 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -91,6 +91,8 @@ pub const Token = struct { PercentEqual, LBrace, RBrace, + LBracket, + RBracket, Period, Ellipsis2, Ellipsis3, @@ -327,6 +329,16 @@ pub const Tokenizer = struct { self.index += 1; break; }, + '[' => { + result.id = Token.Id.LBracket; + self.index += 1; + break; + }, + ']' => { + result.id = Token.Id.RBracket; + self.index += 1; + break; + }, ';' => { result.id = Token.Id.Semicolon; self.index += 1; From a2330d0ea366e0ced7fa917d3bf1ccd022e932c3 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 10:54:19 +0200 Subject: [PATCH 11/54] std.zig.parser now parses slice and array types --- std/zig/ast.zig | 5 +++++ std/zig/parser.zig | 50 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 680e617962..1e62bd26a8 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -475,9 +475,12 @@ pub const NodePrefixOp = struct { Negation, NegationWrap, Return, + ArrayType: &Node, + SliceType: AddrOfInfo, Try, UnwrapMaybe, }; + const AddrOfInfo = struct { align_expr: ?&Node, bit_offset_start_token: ?Token, @@ -502,6 +505,8 @@ pub const NodePrefixOp = struct { PrefixOp.Negation, PrefixOp.NegationWrap, PrefixOp.Return, + PrefixOp.ArrayType, + PrefixOp.SliceType, PrefixOp.Try, PrefixOp.UnwrapMaybe => {}, } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index d0ab7e17d8..671f454f5a 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -385,6 +385,35 @@ pub const Parser = struct { try stack.append(State.ExpectOperand); continue; }, + Token.Id.LBracket => { + const rbracket_token = self.getNextToken(); + if (rbracket_token.id == Token.Id.RBracket) { + const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ + .SliceType = ast.NodePrefixOp.AddrOfInfo { + .align_expr = null, + .bit_offset_start_token = null, + .bit_offset_end_token = null, + .const_token = null, + .volatile_token = null, + } + }); + try stack.append(State { .PrefixOp = prefix_op }); + try stack.append(State.ExpectOperand); + try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf }); + continue; + } + + self.putBackToken(rbracket_token); + + const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ + .ArrayType = undefined, + }); + try stack.append(State { .PrefixOp = prefix_op }); + try stack.append(State.ExpectOperand); + try stack.append(State { .ExpectToken = Token.Id.RBracket }); + try stack.append(State { .Expression = DestPtr { .Field = &prefix_op.op.ArrayType } }); + + }, Token.Id.Ampersand => { const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ .AddrOf = ast.NodePrefixOp.AddrOfInfo { @@ -1567,6 +1596,25 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = align_expr}); } }, + ast.NodePrefixOp.PrefixOp.SliceType => |addr_of_info| { + try stream.write("[]"); + if (addr_of_info.volatile_token != null) { + try stack.append(RenderState { .Text = "volatile "}); + } + if (addr_of_info.const_token != null) { + try stack.append(RenderState { .Text = "const "}); + } + if (addr_of_info.align_expr) |align_expr| { + try stream.print("align("); + try stack.append(RenderState { .Text = ") "}); + try stack.append(RenderState { .Expression = align_expr}); + } + }, + ast.NodePrefixOp.PrefixOp.ArrayType => |array_index| { + try stack.append(RenderState { .Text = "]"}); + try stack.append(RenderState { .Expression = array_index}); + try stack.append(RenderState { .Text = "["}); + }, ast.NodePrefixOp.PrefixOp.BitNot => try stream.write("~"), ast.NodePrefixOp.PrefixOp.BoolNot => try stream.write("!"), ast.NodePrefixOp.PrefixOp.Deref => try stream.write("*"), @@ -2522,4 +2570,4 @@ test "zig fmt: zig fmt" { try testCanonical(@embedFile("index.zig")); try testCanonical(@embedFile("parser.zig")); try testCanonical(@embedFile("tokenizer.zig")); -} \ No newline at end of file +} From 22e38ffb54775c2e523665ac739f2a895d8757ad Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 11:18:18 +0200 Subject: [PATCH 12/54] std.zig.tokenizer fixed tokens having wrong column and line --- std/zig/tokenizer.zig | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 1014bacb30..2e40999920 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -271,6 +271,7 @@ pub const Tokenizer = struct { self.pending_invalid_token = null; return token; } + const start_index = self.index; var state = State.Start; var result = Token { .id = Token.Id.Eof, @@ -279,7 +280,7 @@ pub const Tokenizer = struct { .line = self.line, .column = self.column, }; - while (self.index < self.buffer.len) { + while (self.index < self.buffer.len) : (self.index += 1) { const c = self.buffer[self.index]; switch (state) { State.Start => switch (c) { @@ -877,14 +878,6 @@ pub const Tokenizer = struct { else => break, }, } - - self.index += 1; - if (c == '\n') { - self.line += 1; - self.column = 0; - } else { - self.column += 1; - } } else if (self.index == self.buffer.len) { switch (state) { State.Start, @@ -983,6 +976,16 @@ pub const Tokenizer = struct { }, } } + + for (self.buffer[start_index..self.index]) |c| { + if (c == '\n') { + self.line += 1; + self.column = 0; + } else { + self.column += 1; + } + } + if (result.id == Token.Id.Eof) { if (self.pending_invalid_token) |token| { self.pending_invalid_token = null; From b424cd75ab9df69f44fe47c10ded3fd25c2d27bb Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 12:33:06 +0200 Subject: [PATCH 13/54] std.zig.parser refactored call, slice and array access to be suffix op --- std/zig/ast.zig | 183 +++++++++++++++++++++------------------------ std/zig/parser.zig | 161 +++++++++++++++++++-------------------- 2 files changed, 160 insertions(+), 184 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 1e62bd26a8..380dd95aee 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -17,6 +17,7 @@ pub const Node = struct { Block, InfixOp, PrefixOp, + SuffixOp, IntegerLiteral, FloatLiteral, StringLiteral, @@ -29,9 +30,6 @@ pub const Node = struct { Unreachable, ErrorType, BuiltinCall, - Call, - ArrayAccess, - SliceExpression, LineComment, TestDecl, }; @@ -46,6 +44,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), + Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), @@ -58,9 +57,6 @@ pub const Node = struct { Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), - Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), - Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).iterate(index), - Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), }; @@ -76,6 +72,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), + Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), @@ -88,9 +85,6 @@ pub const Node = struct { Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), - Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), - Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).firstToken(), - Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), }; @@ -106,6 +100,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), + Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), @@ -118,9 +113,6 @@ pub const Node = struct { Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), - Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), - Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).lastToken(), - Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), }; @@ -493,20 +485,28 @@ pub const NodePrefixOp = struct { var i = index; switch (self.op) { + PrefixOp.SliceType => |addr_of_info| { + if (addr_of_info.align_expr) |align_expr| { + if (i < 1) return align_expr; + i -= 1; + } + }, PrefixOp.AddrOf => |addr_of_info| { if (addr_of_info.align_expr) |align_expr| { if (i < 1) return align_expr; i -= 1; } }, + PrefixOp.ArrayType => |size_expr| { + if (i < 1) return size_expr; + i -= 1; + }, PrefixOp.BitNot, PrefixOp.BoolNot, PrefixOp.Deref, PrefixOp.Negation, PrefixOp.NegationWrap, PrefixOp.Return, - PrefixOp.ArrayType, - PrefixOp.SliceType, PrefixOp.Try, PrefixOp.UnwrapMaybe => {}, } @@ -526,6 +526,76 @@ pub const NodePrefixOp = struct { } }; +pub const NodeSuffixOp = struct { + base: Node, + lhs: &Node, + op: SuffixOp, + rtoken: Token, + + const SuffixOp = union(enum) { + Call: CallInfo, + ArrayAccess: &Node, + Slice: SliceRange, + ArrayInitializer: ArrayList(&Node), + StructInitializer: ArrayList(&Node), + }; + + const CallInfo = struct { + params: ArrayList(&Node), + is_async: bool, + }; + + const SliceRange = struct { + start: &Node, + end: ?&Node, + }; + + pub fn iterate(self: &NodeSuffixOp, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.lhs; + i -= 1; + + switch (self.op) { + SuffixOp.Call => |call_info| { + if (i < call_info.params.len) return call_info.params.at(i); + i -= call_info.params.len; + }, + SuffixOp.ArrayAccess => |index_expr| { + if (i < 1) return index_expr; + i -= 1; + }, + SuffixOp.Slice => |range| { + if (i < 1) return range.start; + i -= 1; + + if (range.end) |end| { + if (i < 1) return end; + i -= 1; + } + }, + SuffixOp.ArrayInitializer => |exprs| { + if (i < exprs.len) return exprs.at(i); + i -= exprs.len; + }, + SuffixOp.StructInitializer => |fields| { + if (i < fields.len) return fields.at(i); + i -= fields.len; + }, + } + + return null; + } + + pub fn firstToken(self: &NodeSuffixOp) Token { + return self.lhs.firstToken(); + } + + pub fn lastToken(self: &NodeSuffixOp) Token { + return self.rtoken; + } +}; + pub const NodeIntegerLiteral = struct { base: Node, token: Token, @@ -584,91 +654,6 @@ pub const NodeBuiltinCall = struct { } }; -pub const NodeCall = struct { - base: Node, - callee: &Node, - params: ArrayList(&Node), - rparen_token: Token, - - pub fn iterate(self: &NodeCall, index: usize) ?&Node { - var i = index; - - if (i < 1) return self.callee; - i -= 1; - - if (i < self.params.len) return self.params.at(i); - i -= self.params.len; - - return null; - } - - pub fn firstToken(self: &NodeCall) Token { - return self.callee.firstToken(); - } - - pub fn lastToken(self: &NodeCall) Token { - return self.rparen_token; - } -}; - -pub const NodeArrayAccess = struct { - base: Node, - expr: &Node, - index: &Node, - rbracket_token: Token, - - pub fn iterate(self: &NodeArrayAccess, index: usize) ?&Node { - var i = index; - - if (i < 1) return self.expr; - i -= 1; - - if (i < 1) return self.index; - i -= 1; - - return null; - } - - pub fn firstToken(self: &NodeArrayAccess) Token { - return self.expr.firstToken(); - } - - pub fn lastToken(self: &NodeArrayAccess) Token { - return self.rbracket_token; - } -}; - -pub const NodeSliceExpression = struct { - base: Node, - expr: &Node, - start: &Node, - end: ?&Node, - rbracket_token: Token, - - pub fn iterate(self: &NodeSliceExpression, index: usize) ?&Node { - var i = index; - - if (i < 1) return self.callee; - i -= 1; - - if (i < 1) return self.start; - i -= 1; - - if (i < 1) return self.end; - i -= 1; - - return null; - } - - pub fn firstToken(self: &NodeSliceExpression) Token { - return self.expr.firstToken(); - } - - pub fn lastToken(self: &NodeSliceExpression) Token { - return self.rbracket_token; - } -}; - pub const NodeStringLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 671f454f5a..6d5ec54bc8 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -87,7 +87,7 @@ pub const Parser = struct { AfterOperand, InfixOp: &ast.NodeInfixOp, PrefixOp: &ast.NodePrefixOp, - SuffixOp: &ast.Node, + SuffixOp: &ast.NodeSuffixOp, SliceOrArrayAccess, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, TypeExpr: DestPtr, @@ -602,20 +602,19 @@ pub const Parser = struct { } else if (token.id == Token.Id.LParen) { self.putBackToken(token); - const node = try arena.create(ast.NodeCall); - *node = ast.NodeCall { - .base = self.initNode(ast.Node.Id.Call), - .callee = undefined, - .params = ArrayList(&ast.Node).init(arena), - .rparen_token = undefined, - }; - try stack.append(State { .SuffixOp = &node.base }); + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .Call = ast.NodeSuffixOp.CallInfo { + .params = ArrayList(&ast.Node).init(arena), + .is_async = false, // TODO: ASYNC + } + }); + try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); - try stack.append(State {.ExprListItemOrEnd = &node.params }); + try stack.append(State {.ExprListItemOrEnd = &node.op.Call.params }); try stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.LParen, - .ptr = &node.rparen_token, + .ptr = &node.rtoken, }, }); continue; @@ -643,16 +642,14 @@ pub const Parser = struct { switch (rbracket_or_ellipsis2_token.id) { Token.Id.Ellipsis2 => { - const node = try arena.create(ast.NodeSliceExpression); - *node = ast.NodeSliceExpression { - .base = self.initNode(ast.Node.Id.SliceExpression), - .expr = undefined, - .start = expression, - .end = null, - .rbracket_token = undefined, - }; + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .Slice = ast.NodeSuffixOp.SliceRange { + .start = expression, + .end = null, + } + }); - try stack.append(State { .SuffixOp = &node.base }); + try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); const rbracket_token = self.getNextToken(); @@ -661,24 +658,21 @@ pub const Parser = struct { try stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.RBracket, - .ptr = &node.rbracket_token, + .ptr = &node.rtoken, } }); - try stack.append(State { .Expression = DestPtr { .NullableField = &node.end } }); + try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } }); } else { - node.rbracket_token = rbracket_token; + node.rtoken = rbracket_token; } break; }, Token.Id.RBracket => { - const node = try arena.create(ast.NodeArrayAccess); - *node = ast.NodeArrayAccess { - .base = self.initNode(ast.Node.Id.ArrayAccess), - .expr = undefined, - .index = expression, - .rbracket_token = token, - }; - try stack.append(State { .SuffixOp = &node.base }); + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .ArrayAccess = expression + }); + node.rtoken = token; + try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); break; }, @@ -950,27 +944,8 @@ pub const Parser = struct { while (true) { switch (stack.pop()) { State.SuffixOp => |suffix_op| { - switch (suffix_op.id) { - ast.Node.Id.Call => { - const call = @fieldParentPtr(ast.NodeCall, "base", suffix_op); - *left_leaf_ptr = &call.base; - left_leaf_ptr = &call.callee; - continue; - }, - ast.Node.Id.ArrayAccess => { - const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", suffix_op); - *left_leaf_ptr = &arr_access.base; - left_leaf_ptr = &arr_access.expr; - continue; - }, - ast.Node.Id.SliceExpression => { - const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", suffix_op); - *left_leaf_ptr = &slice_expr.base; - left_leaf_ptr = &slice_expr.expr; - continue; - }, - else => unreachable, - } + *left_leaf_ptr = &suffix_op.base; + left_leaf_ptr = &suffix_op.lhs; }, State.Operand => |operand| { *left_leaf_ptr = operand; @@ -1172,6 +1147,18 @@ pub const Parser = struct { return node; } + fn createSuffixOp(self: &Parser, arena: &mem.Allocator, op: &const ast.NodeSuffixOp.SuffixOp) !&ast.NodeSuffixOp { + const node = try arena.create(ast.NodeSuffixOp); + + *node = ast.NodeSuffixOp { + .base = self.initNode(ast.Node.Id.SuffixOp), + .lhs = undefined, + .op = *op, + .rtoken = undefined, + }; + return node; + } + fn createIdentifier(self: &Parser, arena: &mem.Allocator, name_token: &const Token) !&ast.NodeIdentifier { const node = try arena.create(ast.NodeIdentifier); @@ -1625,6 +1612,43 @@ pub const Parser = struct { ast.NodePrefixOp.PrefixOp.UnwrapMaybe => try stream.write("??"), } }, + ast.Node.Id.SuffixOp => { + const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", base); + + switch (suffix_op.op) { + ast.NodeSuffixOp.SuffixOp.Call => |call_info| { + try stack.append(RenderState { .Text = ")"}); + var i = call_info.params.len; + while (i != 0) { + i -= 1; + const param_node = call_info.params.at(i); + try stack.append(RenderState { .Expression = param_node}); + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + try stack.append(RenderState { .Text = "("}); + }, + ast.NodeSuffixOp.SuffixOp.ArrayAccess => |index_expr| { + try stack.append(RenderState { .Text = "]"}); + try stack.append(RenderState { .Expression = index_expr}); + try stack.append(RenderState { .Text = "["}); + }, + ast.NodeSuffixOp.SuffixOp.Slice => |range| { + try stack.append(RenderState { .Text = "]"}); + if (range.end) |end| { + try stack.append(RenderState { .Expression = end}); + } + try stack.append(RenderState { .Text = ".."}); + try stack.append(RenderState { .Expression = range.start}); + try stack.append(RenderState { .Text = "["}); + }, + ast.NodeSuffixOp.SuffixOp.StructInitializer => @panic("TODO: StructInitializer"), + ast.NodeSuffixOp.SuffixOp.ArrayInitializer => @panic("TODO: ArrayInitializer"), + } + + try stack.append(RenderState { .Expression = suffix_op.lhs }); + }, ast.Node.Id.IntegerLiteral => { const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token)); @@ -1693,39 +1717,6 @@ pub const Parser = struct { } } }, - ast.Node.Id.Call => { - const call = @fieldParentPtr(ast.NodeCall, "base", base); - try stack.append(RenderState { .Text = ")"}); - var i = call.params.len; - while (i != 0) { - i -= 1; - const param_node = call.params.at(i); - try stack.append(RenderState { .Expression = param_node}); - if (i != 0) { - try stack.append(RenderState { .Text = ", " }); - } - } - try stack.append(RenderState { .Text = "("}); - try stack.append(RenderState { .Expression = call.callee }); - }, - ast.Node.Id.ArrayAccess => { - const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", base); - try stack.append(RenderState { .Text = "]"}); - try stack.append(RenderState { .Expression = arr_access.index}); - try stack.append(RenderState { .Text = "["}); - try stack.append(RenderState { .Expression = arr_access.expr }); - }, - ast.Node.Id.SliceExpression => { - const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", base); - try stack.append(RenderState { .Text = "]"}); - if (slice_expr.end) |end| { - try stack.append(RenderState { .Expression = end}); - } - try stack.append(RenderState { .Text = ".."}); - try stack.append(RenderState { .Expression = slice_expr.start}); - try stack.append(RenderState { .Text = "["}); - try stack.append(RenderState { .Expression = slice_expr.expr}); - }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), From 0b9247fb636a47add80f7c4ec194df436629df91 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 14:20:34 +0200 Subject: [PATCH 14/54] std.zig.parser Refactor: * Slice/Array access is now not parsed in the expr contruction loop * State.ExprListItemOrEnd now takes a token id for the end token --- std/zig/ast.zig | 30 ++++++++- std/zig/parser.zig | 149 +++++++++++++++++++++++++++------------------ 2 files changed, 118 insertions(+), 61 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 380dd95aee..2ec305429c 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -18,6 +18,7 @@ pub const Node = struct { InfixOp, PrefixOp, SuffixOp, + FieldInitializer, IntegerLiteral, FloatLiteral, StringLiteral, @@ -45,6 +46,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), + Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), @@ -73,6 +75,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), + Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), @@ -101,6 +104,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), + Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), @@ -526,6 +530,30 @@ pub const NodePrefixOp = struct { } }; +pub const NodeFieldInitializer = struct { + base: Node, + dot_token: Token, + name_token: Token, + expr: &Node, + + pub fn iterate(self: &NodeFieldInitializer, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeFieldInitializer) Token { + return self.dot_token; + } + + pub fn lastToken(self: &NodeFieldInitializer) Token { + return self.expr.lastToken(); + } +}; + pub const NodeSuffixOp = struct { base: Node, lhs: &Node, @@ -537,7 +565,7 @@ pub const NodeSuffixOp = struct { ArrayAccess: &Node, Slice: SliceRange, ArrayInitializer: ArrayList(&Node), - StructInitializer: ArrayList(&Node), + StructInitializer: ArrayList(&NodeFieldInitializer), }; const CallInfo = struct { diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 6d5ec54bc8..43cc35fb6d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -77,6 +77,12 @@ pub const Parser = struct { ptr: &Token, }; + const ExprListState = struct { + list: &ArrayList(&ast.Node), + end: Token.Id, + ptr: &Token, + }; + const State = union(enum) { TopLevel, TopLevelExtern: ?Token, @@ -88,7 +94,7 @@ pub const Parser = struct { InfixOp: &ast.NodeInfixOp, PrefixOp: &ast.NodePrefixOp, SuffixOp: &ast.NodeSuffixOp, - SliceOrArrayAccess, + SliceOrArrayAccess: &ast.NodeSuffixOp, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, TypeExpr: DestPtr, VarDecl: &ast.NodeVarDecl, @@ -104,8 +110,8 @@ pub const Parser = struct { FnDef: &ast.NodeFnProto, Block: &ast.NodeBlock, Statement: &ast.NodeBlock, - ExprListItemOrEnd: &ArrayList(&ast.Node), - ExprListCommaOrEnd: &ArrayList(&ast.Node), + ExprListItemOrEnd: ExprListState, + ExprListCommaOrEnd: ExprListState, }; /// Returns an AST tree, allocated with the parser's allocator. @@ -529,13 +535,14 @@ pub const Parser = struct { .Operand = &node.base }); try stack.append(State.AfterOperand); - try stack.append(State {.ExprListItemOrEnd = &node.params }); try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.LParen, + .ExprListItemOrEnd = ExprListState { + .list = &node.params, + .end = Token.Id.RParen, .ptr = &node.rparen_token, - }, + } }); + try stack.append(State { .ExpectToken = Token.Id.LParen, }); continue; }, Token.Id.StringLiteral => { @@ -587,6 +594,46 @@ pub const Parser = struct { } }, + State.SliceOrArrayAccess => |node| { + var token = self.getNextToken(); + + switch (token.id) { + Token.Id.Ellipsis2 => { + const start = node.op.ArrayAccess; + node.op = ast.NodeSuffixOp.SuffixOp { + .Slice = ast.NodeSuffixOp.SliceRange { + .start = start, + .end = undefined, + } + }; + try stack.append(State { .SuffixOp = node }); + try stack.append(State.AfterOperand); + + const rbracket_token = self.getNextToken(); + if (rbracket_token.id != Token.Id.RBracket) { + self.putBackToken(rbracket_token); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RBracket, + .ptr = &node.rtoken, + } + }); + try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } }); + } else { + node.rtoken = rbracket_token; + } + continue; + }, + Token.Id.RBracket => { + node.rtoken = token; + try stack.append(State { .SuffixOp = node }); + try stack.append(State.AfterOperand); + continue; + }, + else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id)) + } + }, + State.AfterOperand => { // we'll either get an infix operator (like != or ^), // or a postfix operator (like () or {}), @@ -610,7 +657,13 @@ pub const Parser = struct { }); try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); - try stack.append(State {.ExprListItemOrEnd = &node.op.Call.params }); + try stack.append(State { + .ExprListItemOrEnd = ExprListState { + .list = &node.op.Call.params, + .end = Token.Id.RParen, + .ptr = &node.rtoken, + } + }); try stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.LParen, @@ -620,8 +673,17 @@ pub const Parser = struct { continue; } else if (token.id == Token.Id.LBracket) { - try stack.append(State.SliceOrArrayAccess); - try stack.append(State.ExpectOperand); + const node = try arena.create(ast.NodeSuffixOp); + *node = ast.NodeSuffixOp { + .base = self.initNode(ast.Node.Id.SuffixOp), + .lhs = undefined, + .op = ast.NodeSuffixOp.SuffixOp { + .ArrayAccess = undefined, + }, + .rtoken = undefined, + }; + try stack.append(State { .SliceOrArrayAccess = node }); + try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); continue; // TODO: Parse postfix operator @@ -637,48 +699,6 @@ pub const Parser = struct { try dest_ptr.store(expression); break; }, - State.SliceOrArrayAccess => { - var rbracket_or_ellipsis2_token = self.getNextToken(); - - switch (rbracket_or_ellipsis2_token.id) { - Token.Id.Ellipsis2 => { - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .Slice = ast.NodeSuffixOp.SliceRange { - .start = expression, - .end = null, - } - }); - - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); - - const rbracket_token = self.getNextToken(); - if (rbracket_token.id != Token.Id.RBracket) { - self.putBackToken(rbracket_token); - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.RBracket, - .ptr = &node.rtoken, - } - }); - try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } }); - } else { - node.rtoken = rbracket_token; - } - break; - }, - Token.Id.RBracket => { - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .ArrayAccess = expression - }); - node.rtoken = token; - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); - break; - }, - else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id)) - } - }, State.InfixOp => |infix_op| { infix_op.rhs = expression; infix_op.lhs = popSuffixOp(&stack); @@ -697,26 +717,31 @@ pub const Parser = struct { } }, - State.ExprListItemOrEnd => |params| { + State.ExprListItemOrEnd => |expr_list_state| { var token = self.getNextToken(); switch (token.id) { Token.Id.RParen => continue, else => { self.putBackToken(token); - stack.append(State { .ExprListCommaOrEnd = params }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.List = params} }); + stack.append(State { .ExprListCommaOrEnd = expr_list_state }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.List = expr_list_state.list} }); }, } }, - State.ExprListCommaOrEnd => |params| { + State.ExprListCommaOrEnd => |expr_list_state| { var token = self.getNextToken(); switch (token.id) { Token.Id.Comma => { - stack.append(State { .ExprListItemOrEnd = params }) catch unreachable; + stack.append(State { .ExprListItemOrEnd = expr_list_state }) catch unreachable; + }, + else => { + const IdTag = @TagType(Token.Id); + if (IdTag(expr_list_state.end) == token.id) + continue; + + return self.parseError(token, "expected ',' or {}, found {}", @tagName(expr_list_state.end), @tagName(token.id)); }, - Token.Id.RParen => continue, - else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)), } }, @@ -933,7 +958,6 @@ pub const Parser = struct { State.PrefixOp => unreachable, State.SuffixOp => unreachable, State.Operand => unreachable, - State.SliceOrArrayAccess => unreachable, } } } @@ -1649,6 +1673,11 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = suffix_op.lhs }); }, + ast.Node.Id.FieldInitializer => { + const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base); + try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token)); + try stack.append(RenderState { .Expression = field_init.expr }); + }, ast.Node.Id.IntegerLiteral => { const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token)); From 5c82ed2ea9360ea0931ba789a4ffb73d689b6a72 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 14:53:27 +0200 Subject: [PATCH 15/54] std.zig.parser now parses initializers... Or, it would, if it worked --- std/zig/ast.zig | 4 +- std/zig/parser.zig | 149 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 125 insertions(+), 28 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 2ec305429c..c4b4ef983a 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -532,7 +532,7 @@ pub const NodePrefixOp = struct { pub const NodeFieldInitializer = struct { base: Node, - dot_token: Token, + period_token: Token, name_token: Token, expr: &Node, @@ -546,7 +546,7 @@ pub const NodeFieldInitializer = struct { } pub fn firstToken(self: &NodeFieldInitializer) Token { - return self.dot_token; + return self.period_token; } pub fn lastToken(self: &NodeFieldInitializer) Token { diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 43cc35fb6d..19ed4af4bf 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -75,13 +75,16 @@ pub const Parser = struct { const ExpectTokenSave = struct { id: Token.Id, ptr: &Token, + }; - const ExprListState = struct { - list: &ArrayList(&ast.Node), - end: Token.Id, - ptr: &Token, - }; + fn ListState(comptime T: type) type { + return struct { + list: &ArrayList(T), + end: Token.Id, + ptr: &Token, + }; + } const State = union(enum) { TopLevel, @@ -110,8 +113,10 @@ pub const Parser = struct { FnDef: &ast.NodeFnProto, Block: &ast.NodeBlock, Statement: &ast.NodeBlock, - ExprListItemOrEnd: ExprListState, - ExprListCommaOrEnd: ExprListState, + ExprListItemOrEnd: ListState(&ast.Node), + ExprListCommaOrEnd: ListState(&ast.Node), + FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer), + FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer), }; /// Returns an AST tree, allocated with the parser's allocator. @@ -536,7 +541,7 @@ pub const Parser = struct { }); try stack.append(State.AfterOperand); try stack.append(State { - .ExprListItemOrEnd = ExprListState { + .ExprListItemOrEnd = ListState(&ast.Node) { .list = &node.params, .end = Token.Id.RParen, .ptr = &node.rparen_token, @@ -647,8 +652,6 @@ pub const Parser = struct { continue; } else if (token.id == Token.Id.LParen) { - self.putBackToken(token); - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { .Call = ast.NodeSuffixOp.CallInfo { .params = ArrayList(&ast.Node).init(arena), @@ -658,18 +661,12 @@ pub const Parser = struct { try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); try stack.append(State { - .ExprListItemOrEnd = ExprListState { + .ExprListItemOrEnd = ListState(&ast.Node) { .list = &node.op.Call.params, .end = Token.Id.RParen, .ptr = &node.rtoken, } }); - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.LParen, - .ptr = &node.rtoken, - }, - }); continue; } else if (token.id == Token.Id.LBracket) { @@ -686,6 +683,47 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); continue; + // TODO: This is the initializer parsing code. It doesn't work because of + // the ambiguity between function bodies and initializers: + // fn main() void {} or fn main() (void {}) + } else if (false) { //(token.id == Token.Id.LBrace) { + const next = self.getNextToken(); + + switch (next.id) { + Token.Id.Period => { + self.putBackToken(token); + + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), + }); + + try stack.append(State { + .FieldInitListItemOrEnd = ListState(&ast.NodeFieldInitializer) { + .list = &node.op.StructInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } + }); + continue; + }, + else => { + self.putBackToken(token); + + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .ArrayInitializer = ArrayList(&ast.Node).init(arena), + }); + + try stack.append(State { + .ExprListItemOrEnd = ListState(&ast.Node) { + .list = &node.op.ArrayInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } + }); + continue; + }, + } + // TODO: Parse postfix operator } else { // no postfix/infix operator after this operand. @@ -717,30 +755,89 @@ pub const Parser = struct { } }, - State.ExprListItemOrEnd => |expr_list_state| { + State.ExprListItemOrEnd => |list_state| { + var token = self.getNextToken(); + + const IdTag = @TagType(Token.Id); + if (IdTag(list_state.end) == token.id) { + *list_state.ptr = token; + continue; + } + + self.putBackToken(token); + stack.append(State { .ExprListCommaOrEnd = list_state }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.List = list_state.list} }); + }, + + State.ExprListCommaOrEnd => |list_state| { var token = self.getNextToken(); switch (token.id) { - Token.Id.RParen => continue, + Token.Id.Comma => { + stack.append(State { .ExprListItemOrEnd = list_state }) catch unreachable; + }, else => { - self.putBackToken(token); - stack.append(State { .ExprListCommaOrEnd = expr_list_state }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.List = expr_list_state.list} }); + const IdTag = @TagType(Token.Id); + if (IdTag(list_state.end) == token.id) { + *list_state.ptr = token; + continue; + } + + return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id)); }, } }, - State.ExprListCommaOrEnd => |expr_list_state| { + State.FieldInitListItemOrEnd => |list_state| { + var token = self.getNextToken(); + + const IdTag = @TagType(Token.Id); + if (IdTag(list_state.end) == token.id){ + *list_state.ptr = token; + continue; + } + + self.putBackToken(token); + + const node = try arena.create(ast.NodeFieldInitializer); + *node = ast.NodeFieldInitializer { + .base = self.initNode(ast.Node.Id.FieldInitializer), + .period_token = undefined, + .name_token = undefined, + .expr = undefined, + }; + try list_state.list.append(node); + + stack.append(State { .FieldInitListCommaOrEnd = list_state }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = &node.expr} }); + try stack.append(State { .ExpectToken = Token.Id.Equal }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Identifier, + .ptr = &node.name_token, + } + }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Period, + .ptr = &node.period_token, + } + }); + }, + + State.FieldInitListCommaOrEnd => |list_state| { var token = self.getNextToken(); switch (token.id) { Token.Id.Comma => { - stack.append(State { .ExprListItemOrEnd = expr_list_state }) catch unreachable; + stack.append(State { .FieldInitListItemOrEnd = list_state }) catch unreachable; }, else => { const IdTag = @TagType(Token.Id); - if (IdTag(expr_list_state.end) == token.id) + if (IdTag(list_state.end) == token.id) { + *list_state.ptr = token; continue; + } - return self.parseError(token, "expected ',' or {}, found {}", @tagName(expr_list_state.end), @tagName(token.id)); + return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id)); }, } }, From 9d69e94bbad0b1ff23999584f5f632cfe8db3656 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 15:16:32 +0200 Subject: [PATCH 16/54] std.zig.parser now parses grouped expressions * I also moved some tests down, as they fail in ways I can't fix yet --- std/zig/ast.zig | 28 +++++++++++ std/zig/parser.zig | 115 ++++++++++++++++++++++++++++----------------- 2 files changed, 99 insertions(+), 44 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index c4b4ef983a..32a8a7f110 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -18,6 +18,7 @@ pub const Node = struct { InfixOp, PrefixOp, SuffixOp, + GroupedExpression, FieldInitializer, IntegerLiteral, FloatLiteral, @@ -46,6 +47,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), + Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).iterate(index), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), @@ -75,6 +77,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), + Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).firstToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), @@ -104,6 +107,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), + Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).lastToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), @@ -624,6 +628,30 @@ pub const NodeSuffixOp = struct { } }; +pub const NodeGroupedExpression = struct { + base: Node, + lparen: Token, + expr: &Node, + rparen: Token, + + pub fn iterate(self: &NodeGroupedExpression, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeGroupedExpression) Token { + return self.lparen; + } + + pub fn lastToken(self: &NodeGroupedExpression) Token { + return self.rparen; + } +}; + pub const NodeIntegerLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 19ed4af4bf..61552f2a3d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -594,6 +594,27 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.LParen => { + const node = try arena.create(ast.NodeGroupedExpression); + *node = ast.NodeGroupedExpression { + .base = self.initNode(ast.Node.Id.GroupedExpression), + .lparen = token, + .expr = undefined, + .rparen = undefined, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RParen, + .ptr = &node.rparen, + } + }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), } @@ -1770,6 +1791,12 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = suffix_op.lhs }); }, + ast.Node.Id.GroupedExpression => { + const grouped_expr = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); + try stack.append(RenderState { .Text = ")"}); + try stack.append(RenderState { .Expression = grouped_expr.expr }); + try stack.append(RenderState { .Text = "("}); + }, ast.Node.Id.FieldInitializer => { const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base); try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token)); @@ -2240,50 +2267,6 @@ test "zig fmt: indexing" { ); } -test "zig fmt: arrays" { - try testCanonical( - \\test "test array" { - \\ const a: [2]u8 = [2]u8{ 1, 2 }; - \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{}; - \\} - \\ - ); -} - -test "zig fmt: precedence" { - try testCanonical( - \\test "precedence" { - \\ a!b(); - \\ (a!b)(); - \\ !a!b; - \\ !(a!b); - \\ !a{}; - \\ !(a{}); - \\ a + b{}; - \\ (a + b){}; - \\ a << b + c; - \\ (a << b) + c; - \\ a & b << c; - \\ (a & b) << c; - \\ a ^ b & c; - \\ (a ^ b) & c; - \\ a | b ^ c; - \\ (a | b) ^ c; - \\ a == b | c; - \\ (a == b) | c; - \\ a and b == c; - \\ (a and b) == c; - \\ a or b and c; - \\ (a or b) and c; - \\ (a or b) and c; - \\ a = b or c; - \\ (a = b) or c; - \\} - \\ - ); -} - test "zig fmt: struct declaration" { try testCanonical( \\const S = struct { @@ -2682,6 +2665,50 @@ test "zig fmt: coroutines" { ); } +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{}; + \\} + \\ + ); +} + +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ !a{}; + \\ !(a{}); + \\ a + b{}; + \\ (a + b){}; + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\ a = b or c; + \\ (a = b) or c; + \\} + \\ + ); +} + test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); From 40f35e997a2155df8141ec309e43d6dad9785965 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 15:17:26 +0200 Subject: [PATCH 17/54] std.zig.parser moved container initializer tests down --- std/zig/parser.zig | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 61552f2a3d..bcb66a6e62 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2360,16 +2360,6 @@ test "zig fmt: union declaration" { ); } -test "zig fmt: container initializers" { - try testCanonical( - \\const a1 = []u8{ }; - \\const a2 = []u8{ 1, 2, 3, 4 }; - \\const s1 = S{ }; - \\const s2 = S{ .a = 1, .b = 2, }; - \\ - ); -} - test "zig fmt: switch" { try testCanonical( \\test "switch" { @@ -2676,6 +2666,16 @@ test "zig fmt: arrays" { ); } +test "zig fmt: container initializers" { + try testCanonical( + \\const a1 = []u8{ }; + \\const a2 = []u8{ 1, 2, 3, 4 }; + \\const s1 = S{ }; + \\const s2 = S{ .a = 1, .b = 2, }; + \\ + ); +} + test "zig fmt: precedence" { try testCanonical( \\test "precedence" { From 4fae452684c108275e9d3003b4927c8c8d41c59b Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 15:33:22 +0200 Subject: [PATCH 18/54] std.zig.parser Refactored top level decl parsing * Now, the arraylist from the root node is passed through the states. * This allows us to reuse the code for enums, unions and structs --- std/zig/parser.zig | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index bcb66a6e62..99ea89bf97 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -53,6 +53,7 @@ pub const Parser = struct { } const TopLevelDeclCtx = struct { + decls: &ArrayList(&ast.Node), visib_token: ?Token, extern_token: ?Token, lib_name: ?&ast.Node, @@ -75,7 +76,6 @@ pub const Parser = struct { const ExpectTokenSave = struct { id: Token.Id, ptr: &Token, - }; fn ListState(comptime T: type) type { @@ -88,7 +88,7 @@ pub const Parser = struct { const State = union(enum) { TopLevel, - TopLevelExtern: ?Token, + TopLevelExtern: TopLevelDeclCtx, TopLevelDecl: TopLevelDeclCtx, Expression: DestPtr, ExpectOperand, @@ -182,7 +182,14 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_pub, Token.Id.Keyword_export => { - stack.append(State { .TopLevelExtern = token }) catch unreachable; + stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &root_node.decls, + .visib_token = token, + .extern_token = null, + .lib_name = null, + } + }) catch unreachable; continue; }, Token.Id.Keyword_test => { @@ -208,12 +215,19 @@ pub const Parser = struct { }, else => { self.putBackToken(token); - stack.append(State { .TopLevelExtern = null }) catch unreachable; + stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &root_node.decls, + .visib_token = null, + .extern_token = null, + .lib_name = null, + } + }) catch unreachable; continue; }, } }, - State.TopLevelExtern => |visib_token| { + State.TopLevelExtern => |ctx| { const token = self.getNextToken(); if (token.id == Token.Id.Keyword_extern) { const lib_name_token = self.getNextToken(); @@ -229,7 +243,8 @@ pub const Parser = struct { stack.append(State { .TopLevelDecl = TopLevelDeclCtx { - .visib_token = visib_token, + .decls = ctx.decls, + .visib_token = ctx.visib_token, .extern_token = token, .lib_name = lib_name, }, @@ -237,13 +252,7 @@ pub const Parser = struct { continue; } self.putBackToken(token); - stack.append(State { - .TopLevelDecl = TopLevelDeclCtx { - .visib_token = visib_token, - .extern_token = null, - .lib_name = null, - }, - }) catch unreachable; + stack.append(State { .TopLevelDecl = ctx }) catch unreachable; continue; }, State.TopLevelDecl => |ctx| { @@ -252,7 +261,7 @@ pub const Parser = struct { Token.Id.Keyword_var, Token.Id.Keyword_const => { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts - const var_decl_node = try self.createAttachVarDecl(arena, &root_node.decls, ctx.visib_token, + const var_decl_node = try self.createAttachVarDecl(arena, ctx.decls, ctx.visib_token, token, (?Token)(null), ctx.extern_token, ctx.lib_name); try stack.append(State { .VarDecl = var_decl_node }); continue; @@ -260,7 +269,7 @@ pub const Parser = struct { Token.Id.Keyword_fn => { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts - const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, token, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, token, ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); @@ -270,7 +279,7 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; const fn_token = try self.eatToken(Token.Id.Keyword_fn); // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, fn_token, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, fn_token, ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); From d602f12df8f12c3a363abec8c1976f8bc84eb2be Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 15:59:14 +0200 Subject: [PATCH 19/54] std.zig.ast Added ContainerDecl --- std/zig/ast.zig | 69 +++++++++++++++++++++++++++++++++++++++++++++- std/zig/parser.zig | 4 +++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 32a8a7f110..767a797cab 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -11,6 +11,7 @@ pub const Node = struct { pub const Id = enum { Root, VarDecl, + ContainerDecl, Identifier, FnProto, ParamDecl, @@ -40,6 +41,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), + Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), @@ -70,6 +72,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), + Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), @@ -100,6 +103,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), + Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), @@ -196,6 +200,69 @@ pub const NodeVarDecl = struct { } }; +pub const NodeContainerDecl = struct { + base: Node, + kind_token: Token, + init_arg_expr: InitArg, + kind: Kind, + decls: ArrayList(&Node), + rbrace_token: Token, + + // TODO: Different array lists for each kind. + const Kind = union(enum) { + Struct: ArrayList(&Node), + Enum: ArrayList(&Node), + Union: ArrayList(&Node), + }; + + const InitArg = union(enum) { + None, + Enum, + Type: &Node, + }; + + pub fn iterate(self: &NodeContainerDecl, index: usize) ?&Node { + var i = index; + + switch (self.init_arg_expr) { + InitArg.Type => |t| { + if (i < 1) return t; + i -= 1; + }, + InitArg.None, + InitArg.Enum => { } + } + + if (i < self.decls.len) return self.decls.at(i); + i -= self.decls.len; + + switch (self.kind) { + Kind.Struct => |fields| { + if (i < fields.len) return fields.at(i); + i -= fields.len; + }, + Kind.Enum => |tags| { + if (i < tags.len) return tags.at(i); + i -= tags.len; + }, + Kind.Union => |tags| { + if (i < tags.len) return tags.at(i); + i -= tags.len; + }, + } + + return null; + } + + pub fn firstToken(self: &NodeContainerDecl) Token { + return self.kind_token; + } + + pub fn lastToken(self: &NodeContainerDecl) Token { + return self.rbrace_token; + } +}; + pub const NodeIdentifier = struct { base: Node, name_token: Token, @@ -611,7 +678,7 @@ pub const NodeSuffixOp = struct { i -= exprs.len; }, SuffixOp.StructInitializer => |fields| { - if (i < fields.len) return fields.at(i); + if (i < fields.len) return &fields.at(i).base; i -= fields.len; }, } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 99ea89bf97..0b4e55913c 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1806,6 +1806,10 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = grouped_expr.expr }); try stack.append(RenderState { .Text = "("}); }, + ast.Node.Id.ContainerDecl => { + const container_decl = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); + @panic("TODO: ContainerDecl"); + }, ast.Node.Id.FieldInitializer => { const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base); try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token)); From ec611bf8b47af0db5244af644040907bfcb63945 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 20:00:02 +0200 Subject: [PATCH 20/54] std.zig.parser now parses regular enums, unions and struct * Still missing packed, and extern --- std/zig/ast.zig | 106 ++++++++++++++-- std/zig/parser.zig | 305 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 351 insertions(+), 60 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 767a797cab..484bc59f16 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -12,6 +12,9 @@ pub const Node = struct { Root, VarDecl, ContainerDecl, + StructField, + UnionTag, + EnumTag, Identifier, FnProto, ParamDecl, @@ -42,6 +45,9 @@ pub const Node = struct { Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), + Id.StructField => @fieldParentPtr(NodeStructField, "base", base).iterate(index), + Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).iterate(index), + Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).iterate(index), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), @@ -73,6 +79,9 @@ pub const Node = struct { Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), + Id.StructField => @fieldParentPtr(NodeStructField, "base", base).firstToken(), + Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).firstToken(), + Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).firstToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), @@ -104,6 +113,9 @@ pub const Node = struct { Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), + Id.StructField => @fieldParentPtr(NodeStructField, "base", base).lastToken(), + Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).lastToken(), + Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).lastToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), @@ -203,16 +215,15 @@ pub const NodeVarDecl = struct { pub const NodeContainerDecl = struct { base: Node, kind_token: Token, - init_arg_expr: InitArg, kind: Kind, - decls: ArrayList(&Node), + init_arg_expr: InitArg, + fields_and_decls: ArrayList(&Node), rbrace_token: Token, - // TODO: Different array lists for each kind. - const Kind = union(enum) { - Struct: ArrayList(&Node), - Enum: ArrayList(&Node), - Union: ArrayList(&Node), + const Kind = enum { + Struct, + Enum, + Union, }; const InitArg = union(enum) { @@ -263,6 +274,87 @@ pub const NodeContainerDecl = struct { } }; +pub const NodeStructField = struct { + base: Node, + name_token: Token, + type_expr: &Node, + + pub fn iterate(self: &NodeStructField, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.type_expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeStructField) Token { + return self.name_token; + } + + pub fn lastToken(self: &NodeStructField) Token { + return self.type_expr.lastToken(); + } +}; + +pub const NodeUnionTag = struct { + base: Node, + name_token: Token, + type_expr: ?&Node, + + pub fn iterate(self: &NodeUnionTag, index: usize) ?&Node { + var i = index; + + if (self.type_expr) |type_expr| { + if (i < 1) return type_expr; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeUnionTag) Token { + return self.name_token; + } + + pub fn lastToken(self: &NodeUnionTag) Token { + if (self.type_expr) |type_expr| { + return type_expr.lastToken(); + } + + return self.name_token; + } +}; + +pub const NodeEnumTag = struct { + base: Node, + name_token: Token, + value: ?&Node, + + pub fn iterate(self: &NodeEnumTag, index: usize) ?&Node { + var i = index; + + if (self.value) |value| { + if (i < 1) return value; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeEnumTag) Token { + return self.name_token; + } + + pub fn lastToken(self: &NodeEnumTag) Token { + if (self.value) |value| { + return value.lastToken(); + } + + return self.name_token; + } +}; + pub const NodeIdentifier = struct { base: Node, name_token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0b4e55913c..fc00ba5f4e 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -90,6 +90,7 @@ pub const Parser = struct { TopLevel, TopLevelExtern: TopLevelDeclCtx, TopLevelDecl: TopLevelDeclCtx, + ContainerDecl: &ast.NodeContainerDecl, Expression: DestPtr, ExpectOperand, Operand: &ast.Node, @@ -117,6 +118,7 @@ pub const Parser = struct { ExprListCommaOrEnd: ListState(&ast.Node), FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer), FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer), + FieldListCommaOrEnd: &ast.NodeContainerDecl, }; /// Returns an AST tree, allocated with the parser's allocator. @@ -181,17 +183,6 @@ pub const Parser = struct { State.TopLevel => { const token = self.getNextToken(); switch (token.id) { - Token.Id.Keyword_pub, Token.Id.Keyword_export => { - stack.append(State { - .TopLevelExtern = TopLevelDeclCtx { - .decls = &root_node.decls, - .visib_token = token, - .extern_token = null, - .lib_name = null, - } - }) catch unreachable; - continue; - }, Token.Id.Keyword_test => { stack.append(State.TopLevel) catch unreachable; @@ -213,16 +204,29 @@ pub const Parser = struct { root_node.eof_token = token; return Tree {.root_node = root_node, .arena_allocator = arena_allocator}; }, + Token.Id.Keyword_pub, Token.Id.Keyword_export => { + stack.append(State.TopLevel) catch unreachable; + try stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &root_node.decls, + .visib_token = token, + .extern_token = null, + .lib_name = null, + } + }); + continue; + }, else => { self.putBackToken(token); - stack.append(State { + stack.append(State.TopLevel) catch unreachable; + try stack.append(State { .TopLevelExtern = TopLevelDeclCtx { .decls = &root_node.decls, .visib_token = null, .extern_token = null, .lib_name = null, } - }) catch unreachable; + }); continue; }, } @@ -259,7 +263,6 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_var, Token.Id.Keyword_const => { - stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const var_decl_node = try self.createAttachVarDecl(arena, ctx.decls, ctx.visib_token, token, (?Token)(null), ctx.extern_token, ctx.lib_name); @@ -267,7 +270,6 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_fn => { - stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const fn_proto = try self.createAttachFnProto(arena, ctx.decls, token, ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); @@ -276,7 +278,6 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { - stack.append(State.TopLevel) catch unreachable; const fn_token = try self.eatToken(Token.Id.Keyword_fn); // TODO shouldn't need this cast const fn_proto = try self.createAttachFnProto(arena, ctx.decls, fn_token, @@ -336,6 +337,101 @@ pub const Parser = struct { } return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id)); }, + + State.ContainerDecl => |container_decl| { + const token = self.getNextToken(); + + switch (token.id) { + Token.Id.Identifier => { + switch (container_decl.kind) { + ast.NodeContainerDecl.Kind.Struct => { + const node = try arena.create(ast.NodeStructField); + *node = ast.NodeStructField { + .base = self.initNode(ast.Node.Id.StructField), + .name_token = token, + .type_expr = undefined, + }; + try container_decl.fields_and_decls.append(&node.base); + + try stack.append(State { .FieldListCommaOrEnd = container_decl }); + try stack.append(State { .Expression = DestPtr { .Field = &node.type_expr } }); + try stack.append(State { .ExpectToken = Token.Id.Colon }); + continue; + }, + ast.NodeContainerDecl.Kind.Union => { + const node = try arena.create(ast.NodeUnionTag); + *node = ast.NodeUnionTag { + .base = self.initNode(ast.Node.Id.UnionTag), + .name_token = token, + .type_expr = null, + }; + try container_decl.fields_and_decls.append(&node.base); + + try stack.append(State { .FieldListCommaOrEnd = container_decl }); + + const next = self.getNextToken(); + if (next.id != Token.Id.Colon) { + self.putBackToken(next); + continue; + } + + try stack.append(State { .Expression = DestPtr { .NullableField = &node.type_expr } }); + continue; + }, + ast.NodeContainerDecl.Kind.Enum => { + const node = try arena.create(ast.NodeEnumTag); + *node = ast.NodeEnumTag { + .base = self.initNode(ast.Node.Id.EnumTag), + .name_token = token, + .value = null, + }; + try container_decl.fields_and_decls.append(&node.base); + + try stack.append(State { .FieldListCommaOrEnd = container_decl }); + + const next = self.getNextToken(); + if (next.id != Token.Id.Equal) { + self.putBackToken(next); + continue; + } + + try stack.append(State { .Expression = DestPtr { .NullableField = &node.value } }); + continue; + }, + } + }, + Token.Id.Keyword_pub, Token.Id.Keyword_export => { + stack.append(State{ .ContainerDecl = container_decl }) catch unreachable; + try stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &container_decl.fields_and_decls, + .visib_token = token, + .extern_token = null, + .lib_name = null, + } + }); + continue; + }, + Token.Id.RBrace => { + container_decl.rbrace_token = token; + continue; + }, + else => { + self.putBackToken(token); + stack.append(State{ .ContainerDecl = container_decl }) catch unreachable; + try stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &container_decl.fields_and_decls, + .visib_token = null, + .extern_token = null, + .lib_name = null, + } + }); + continue; + } + } + }, + State.ExpectToken => |token_id| { _ = try self.eatToken(token_id); continue; @@ -537,6 +633,53 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => { + const node = try arena.create(ast.NodeContainerDecl); + *node = ast.NodeContainerDecl { + .base = self.initNode(ast.Node.Id.ContainerDecl), + .kind_token = token, + .kind = switch (token.id) { + Token.Id.Keyword_struct => ast.NodeContainerDecl.Kind.Struct, + Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, + Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, + else => unreachable, + }, + .init_arg_expr = undefined, + .fields_and_decls = ArrayList(&ast.Node).init(arena), + .rbrace_token = undefined, + }; + + try stack.append(State { .Operand = &node.base }); + try stack.append(State.AfterOperand); + try stack.append(State { .ContainerDecl = node }); + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + + const lparen = self.getNextToken(); + if (lparen.id != Token.Id.LParen) { + self.putBackToken(lparen); + node.init_arg_expr = ast.NodeContainerDecl.InitArg.None; + continue; + } + + try stack.append(State { .ExpectToken = Token.Id.RParen }); + + const init_arg_token = self.getNextToken(); + switch (init_arg_token.id) { + Token.Id.Keyword_enum => { + node.init_arg_expr = ast.NodeContainerDecl.InitArg.Enum; + }, + else => { + self.putBackToken(lparen); + node.init_arg_expr = ast.NodeContainerDecl.InitArg { .Type = undefined }; + try stack.append(State { + .Expression = DestPtr { + .Field = &node.init_arg_expr.Type + } + }); + }, + } + continue; + }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -799,24 +942,6 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr{.List = list_state.list} }); }, - State.ExprListCommaOrEnd => |list_state| { - var token = self.getNextToken(); - switch (token.id) { - Token.Id.Comma => { - stack.append(State { .ExprListItemOrEnd = list_state }) catch unreachable; - }, - else => { - const IdTag = @TagType(Token.Id); - if (IdTag(list_state.end) == token.id) { - *list_state.ptr = token; - continue; - } - - return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id)); - }, - } - }, - State.FieldInitListItemOrEnd => |list_state| { var token = self.getNextToken(); @@ -854,22 +979,20 @@ pub const Parser = struct { }); }, - State.FieldInitListCommaOrEnd => |list_state| { - var token = self.getNextToken(); - switch (token.id) { - Token.Id.Comma => { - stack.append(State { .FieldInitListItemOrEnd = list_state }) catch unreachable; - }, - else => { - const IdTag = @TagType(Token.Id); - if (IdTag(list_state.end) == token.id) { - *list_state.ptr = token; - continue; - } + State.ExprListCommaOrEnd => |list_state| { + try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .ExprListItemOrEnd = list_state }); + continue; + }, - return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id)); - }, - } + State.FieldInitListCommaOrEnd => |list_state| { + try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .FieldInitListItemOrEnd = list_state }); + continue; + }, + + State.FieldListCommaOrEnd => |container_decl| { + try self.commaOrEnd(&stack, Token.Id.RBrace, &container_decl.rbrace_token, + State { .ContainerDecl = container_decl }); + continue; }, State.AddrOfModifiers => |addr_of_info| { @@ -1089,6 +1212,24 @@ pub const Parser = struct { } } + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void { + var token = self.getNextToken(); + switch (token.id) { + Token.Id.Comma => { + stack.append(state_after_comma) catch unreachable; + }, + else => { + const IdTag = @TagType(Token.Id); + if (IdTag(*end) == token.id) { + *ptr = token; + return; + } + + return self.parseError(token, "expected ',' or {}, found {}", @tagName(*end), @tagName(token.id)); + }, + } + } + fn popSuffixOp(stack: &ArrayList(State)) &ast.Node { var expression: &ast.Node = undefined; var left_leaf_ptr: &&ast.Node = &expression; @@ -1806,10 +1947,6 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = grouped_expr.expr }); try stack.append(RenderState { .Text = "("}); }, - ast.Node.Id.ContainerDecl => { - const container_decl = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); - @panic("TODO: ContainerDecl"); - }, ast.Node.Id.FieldInitializer => { const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base); try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token)); @@ -1851,6 +1988,46 @@ pub const Parser = struct { const error_type = @fieldParentPtr(ast.NodeErrorType, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(error_type.token)); }, + ast.Node.Id.ContainerDecl => { + const container_decl = @fieldParentPtr(ast.NodeContainerDecl, "base", base); + try stream.print("{} {{", self.tokenizer.getTokenSlice(container_decl.kind_token)); + try stack.append(RenderState { .Text = "}"}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent }); + + const fields_and_decls = container_decl.fields_and_decls.toSliceConst(); + var i = fields_and_decls.len; + while (i != 0) { + i -= 1; + const node = fields_and_decls[i]; + if (i != 0) { + switch (node.id) { + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag => { + try stack.append(RenderState { .Text = "," }); + }, + else => { } + } + } + try stack.append(RenderState { .Expression = node}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + if (i != 0) { + const prev_node = fields_and_decls[i - 1]; + const prev_line_index = prev_node.lastToken().line; + const this_line_index = node.firstToken().line; + if (this_line_index - prev_line_index >= 2) { + break :blk "\n\n"; + } + } + break :blk "\n"; + }, + }); + } + try stack.append(RenderState { .Indent = indent + indent_delta}); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); @@ -1883,6 +2060,28 @@ pub const Parser = struct { } } }, + ast.Node.Id.StructField => { + const field = @fieldParentPtr(ast.NodeStructField, "base", base); + try stream.print("{}:", self.tokenizer.getTokenSlice(field.name_token)); + }, + ast.Node.Id.UnionTag => { + const tag = @fieldParentPtr(ast.NodeUnionTag, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); + + if (tag.type_expr) |type_expr| { + try stream.print(": "); + try stack.append(RenderState { .Expression = type_expr}); + } + }, + ast.Node.Id.EnumTag => { + const tag = @fieldParentPtr(ast.NodeEnumTag, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); + + if (tag.value) |value| { + try stream.print(" = "); + try stack.append(RenderState { .Expression = value}); + } + }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), From 09cf82361999c1a22819467e02fc68fd22ca188e Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 09:57:37 +0200 Subject: [PATCH 21/54] std.zig.parser now parses container decls --- std/zig/ast.zig | 11 +- std/zig/parser.zig | 262 +++++++++++++++++++++++++++++---------------- 2 files changed, 177 insertions(+), 96 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 484bc59f16..1d42d721d9 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -214,12 +214,19 @@ pub const NodeVarDecl = struct { pub const NodeContainerDecl = struct { base: Node, - kind_token: Token, + ltoken: Token, + layout: Layout, kind: Kind, init_arg_expr: InitArg, fields_and_decls: ArrayList(&Node), rbrace_token: Token, + const Layout = enum { + Auto, + Extern, + Packed, + }; + const Kind = enum { Struct, Enum, @@ -266,7 +273,7 @@ pub const NodeContainerDecl = struct { } pub fn firstToken(self: &NodeContainerDecl) Token { - return self.kind_token; + return self.ltoken; } pub fn lastToken(self: &NodeContainerDecl) Token { diff --git a/std/zig/parser.zig b/std/zig/parser.zig index fc00ba5f4e..8a4999a4ee 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -59,6 +59,11 @@ pub const Parser = struct { lib_name: ?&ast.Node, }; + const ContainerExternCtx = struct { + ltoken: Token, + layout: ast.NodeContainerDecl.Layout, + }; + const DestPtr = union(enum) { Field: &&ast.Node, NullableField: &?&ast.Node, @@ -90,6 +95,7 @@ pub const Parser = struct { TopLevel, TopLevelExtern: TopLevelDeclCtx, TopLevelDecl: TopLevelDeclCtx, + ContainerExtern: ContainerExternCtx, ContainerDecl: &ast.NodeContainerDecl, Expression: DestPtr, ExpectOperand, @@ -338,6 +344,63 @@ pub const Parser = struct { return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id)); }, + State.ContainerExtern => |ctx| { + const token = self.getNextToken(); + + const node = try arena.create(ast.NodeContainerDecl); + *node = ast.NodeContainerDecl { + .base = self.initNode(ast.Node.Id.ContainerDecl), + .ltoken = ctx.ltoken, + .layout = ctx.layout, + .kind = switch (token.id) { + Token.Id.Keyword_struct => ast.NodeContainerDecl.Kind.Struct, + Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, + Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, + else => { + return self.parseError(token, "expected {}, {} or {}, found {}", + @tagName(Token.Id.Keyword_struct), + @tagName(Token.Id.Keyword_union), + @tagName(Token.Id.Keyword_enum), + @tagName(token.id)); + }, + }, + .init_arg_expr = undefined, + .fields_and_decls = ArrayList(&ast.Node).init(arena), + .rbrace_token = undefined, + }; + + try stack.append(State { .Operand = &node.base }); + try stack.append(State.AfterOperand); + try stack.append(State { .ContainerDecl = node }); + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + + const lparen = self.getNextToken(); + if (lparen.id != Token.Id.LParen) { + self.putBackToken(lparen); + node.init_arg_expr = ast.NodeContainerDecl.InitArg.None; + continue; + } + + try stack.append(State { .ExpectToken = Token.Id.RParen }); + + const init_arg_token = self.getNextToken(); + switch (init_arg_token.id) { + Token.Id.Keyword_enum => { + node.init_arg_expr = ast.NodeContainerDecl.InitArg.Enum; + }, + else => { + self.putBackToken(init_arg_token); + node.init_arg_expr = ast.NodeContainerDecl.InitArg { .Type = undefined }; + try stack.append(State { + .Expression = DestPtr { + .Field = &node.init_arg_expr.Type + } + }); + }, + } + continue; + }, + State.ContainerDecl => |container_decl| { const token = self.getNextToken(); @@ -633,52 +696,30 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_packed => { + try stack.append(State { + .ContainerExtern = ContainerExternCtx { + .ltoken = token, + .layout = ast.NodeContainerDecl.Layout.Packed, + }, + }); + }, + Token.Id.Keyword_extern => { + try stack.append(State { + .ContainerExtern = ContainerExternCtx { + .ltoken = token, + .layout = ast.NodeContainerDecl.Layout.Extern, + }, + }); + }, Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => { - const node = try arena.create(ast.NodeContainerDecl); - *node = ast.NodeContainerDecl { - .base = self.initNode(ast.Node.Id.ContainerDecl), - .kind_token = token, - .kind = switch (token.id) { - Token.Id.Keyword_struct => ast.NodeContainerDecl.Kind.Struct, - Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, - Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, - else => unreachable, + self.putBackToken(token); + try stack.append(State { + .ContainerExtern = ContainerExternCtx { + .ltoken = token, + .layout = ast.NodeContainerDecl.Layout.Auto, }, - .init_arg_expr = undefined, - .fields_and_decls = ArrayList(&ast.Node).init(arena), - .rbrace_token = undefined, - }; - - try stack.append(State { .Operand = &node.base }); - try stack.append(State.AfterOperand); - try stack.append(State { .ContainerDecl = node }); - try stack.append(State { .ExpectToken = Token.Id.LBrace }); - - const lparen = self.getNextToken(); - if (lparen.id != Token.Id.LParen) { - self.putBackToken(lparen); - node.init_arg_expr = ast.NodeContainerDecl.InitArg.None; - continue; - } - - try stack.append(State { .ExpectToken = Token.Id.RParen }); - - const init_arg_token = self.getNextToken(); - switch (init_arg_token.id) { - Token.Id.Keyword_enum => { - node.init_arg_expr = ast.NodeContainerDecl.InitArg.Enum; - }, - else => { - self.putBackToken(lparen); - node.init_arg_expr = ast.NodeContainerDecl.InitArg { .Type = undefined }; - try stack.append(State { - .Expression = DestPtr { - .Field = &node.init_arg_expr.Type - } - }); - }, - } - continue; + }); }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); @@ -1706,6 +1747,29 @@ pub const Parser = struct { try stack.append(RenderState { .Text = " " }); try stack.append(RenderState { .Expression = test_decl.name }); }, + ast.Node.Id.StructField => { + const field = @fieldParentPtr(ast.NodeStructField, "base", decl); + try stream.print("{}: ", self.tokenizer.getTokenSlice(field.name_token)); + try stack.append(RenderState { .Expression = field.type_expr}); + }, + ast.Node.Id.UnionTag => { + const tag = @fieldParentPtr(ast.NodeUnionTag, "base", decl); + try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); + + if (tag.type_expr) |type_expr| { + try stream.print(": "); + try stack.append(RenderState { .Expression = type_expr}); + } + }, + ast.Node.Id.EnumTag => { + const tag = @fieldParentPtr(ast.NodeEnumTag, "base", decl); + try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); + + if (tag.value) |value| { + try stream.print(" = "); + try stack.append(RenderState { .Expression = value}); + } + }, else => unreachable, } }, @@ -1990,27 +2054,30 @@ pub const Parser = struct { }, ast.Node.Id.ContainerDecl => { const container_decl = @fieldParentPtr(ast.NodeContainerDecl, "base", base); - try stream.print("{} {{", self.tokenizer.getTokenSlice(container_decl.kind_token)); + + switch (container_decl.layout) { + ast.NodeContainerDecl.Layout.Packed => try stream.print("packed "), + ast.NodeContainerDecl.Layout.Extern => try stream.print("extern "), + ast.NodeContainerDecl.Layout.Auto => { }, + } + + switch (container_decl.kind) { + ast.NodeContainerDecl.Kind.Struct => try stream.print("struct"), + ast.NodeContainerDecl.Kind.Enum => try stream.print("enum"), + ast.NodeContainerDecl.Kind.Union => try stream.print("union"), + } + try stack.append(RenderState { .Text = "}"}); try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Text = "\n"}); const fields_and_decls = container_decl.fields_and_decls.toSliceConst(); var i = fields_and_decls.len; while (i != 0) { i -= 1; const node = fields_and_decls[i]; - if (i != 0) { - switch (node.id) { - ast.Node.Id.StructField, - ast.Node.Id.UnionTag, - ast.Node.Id.EnumTag => { - try stack.append(RenderState { .Text = "," }); - }, - else => { } - } - } - try stack.append(RenderState { .Expression = node}); + try stack.append(RenderState { .TopLevelDecl = node}); try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Text = blk: { @@ -2025,22 +2092,43 @@ pub const Parser = struct { break :blk "\n"; }, }); + + if (i != 0) { + const prev_node = fields_and_decls[i - 1]; + switch (prev_node.id) { + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag => { + try stack.append(RenderState { .Text = "," }); + }, + else => { } + } + } } try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "{"}); + + switch (container_decl.init_arg_expr) { + ast.NodeContainerDecl.InitArg.None => try stack.append(RenderState { .Text = " "}), + ast.NodeContainerDecl.InitArg.Enum => try stack.append(RenderState { .Text = "(enum) "}), + ast.NodeContainerDecl.InitArg.Type => |type_expr| { + try stack.append(RenderState { .Text = ") "}); + try stack.append(RenderState { .Expression = type_expr}); + try stack.append(RenderState { .Text = "("}); + }, + } }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); var i : usize = 0; - indent += 4; while (i < multiline_str_literal.tokens.len) : (i += 1) { const t = multiline_str_literal.tokens.at(i); - try stream.writeByteNTimes(' ', indent); + try stream.writeByteNTimes(' ', indent + indent_delta); try stream.print("{}", self.tokenizer.getTokenSlice(t)); } - try stream.writeByteNTimes(' ', indent); - indent -= 4; + try stream.writeByteNTimes(' ', indent + indent_delta); }, ast.Node.Id.UndefinedLiteral => { const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base); @@ -2060,31 +2148,12 @@ pub const Parser = struct { } } }, - ast.Node.Id.StructField => { - const field = @fieldParentPtr(ast.NodeStructField, "base", base); - try stream.print("{}:", self.tokenizer.getTokenSlice(field.name_token)); - }, - ast.Node.Id.UnionTag => { - const tag = @fieldParentPtr(ast.NodeUnionTag, "base", base); - try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); - - if (tag.type_expr) |type_expr| { - try stream.print(": "); - try stack.append(RenderState { .Expression = type_expr}); - } - }, - ast.Node.Id.EnumTag => { - const tag = @fieldParentPtr(ast.NodeEnumTag, "base", base); - try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); - - if (tag.value) |value| { - try stream.print(" = "); - try stack.append(RenderState { .Expression = value}); - } - }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag, ast.Node.Id.Root, ast.Node.Id.VarDecl, ast.Node.Id.TestDecl, @@ -2489,21 +2558,21 @@ test "zig fmt: struct declaration" { \\ return *self; \\ } \\ - \\ f2: u8, + \\ f2: u8 \\}; \\ \\const Ps = packed struct { \\ a: u8, \\ b: u8, \\ - \\ c: u8, + \\ c: u8 \\}; \\ \\const Es = extern struct { \\ a: u8, \\ b: u8, \\ - \\ c: u8, + \\ c: u8 \\}; \\ ); @@ -2513,25 +2582,25 @@ test "zig fmt: enum declaration" { try testCanonical( \\const E = enum { \\ Ok, - \\ SomethingElse = 0, + \\ SomethingElse = 0 \\}; \\ \\const E2 = enum(u8) { \\ Ok, \\ SomethingElse = 255, - \\ SomethingThird, + \\ SomethingThird \\}; \\ \\const Ee = extern enum { \\ Ok, \\ SomethingElse, - \\ SomethingThird, + \\ SomethingThird \\}; \\ \\const Ep = packed enum { \\ Ok, \\ SomethingElse, - \\ SomethingThird, + \\ SomethingThird \\}; \\ ); @@ -2542,31 +2611,36 @@ test "zig fmt: union declaration" { \\const U = union { \\ Int: u8, \\ Float: f32, - \\ Bool: bool, + \\ None, + \\ Bool: bool \\}; \\ \\const Ue = union(enum) { \\ Int: u8, \\ Float: f32, - \\ Bool: bool, + \\ None, + \\ Bool: bool \\}; \\ \\const E = enum { \\ Int, \\ Float, - \\ Bool, + \\ None, + \\ Bool \\}; \\ \\const Ue2 = union(E) { \\ Int: u8, \\ Float: f32, - \\ Bool: bool, + \\ None, + \\ Bool: bool \\}; \\ \\const Eu = extern union { \\ Int: u8, \\ Float: f32, - \\ Bool: bool, + \\ None, + \\ Bool: bool \\}; \\ ); From 020724cfa0677bf42f48957d0ca7a474f6fe31e0 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 10:27:38 +0200 Subject: [PATCH 22/54] std.zig.tokenizer Tokens now don't contain a line and column field. * Instead, this information is optained by asking the tokenizer. * getTokenLocation takes a start_index, so relative loc can be optained --- std/zig/parser.zig | 23 ++++++++++------------ std/zig/tokenizer.zig | 44 +++++++++++++++---------------------------- 2 files changed, 25 insertions(+), 42 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 8a4999a4ee..f7db8fd93d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1571,12 +1571,12 @@ pub const Parser = struct { } fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) { - const loc = self.tokenizer.getTokenLocation(token); - warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args); + const loc = self.tokenizer.getTokenLocation(0, token); + warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args); warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]); { var i: usize = 0; - while (i < token.column) : (i += 1) { + while (i < loc.column) : (i += 1) { warn(" "); } } @@ -1679,9 +1679,8 @@ pub const Parser = struct { try stack.append(RenderState { .Text = blk: { const prev_node = root_node.decls.at(i - 1); - const prev_line_index = prev_node.lastToken().line; - const this_line_index = decl.firstToken().line; - if (this_line_index - prev_line_index >= 2) { + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, decl.firstToken()); + if (loc.line >= 2) { break :blk "\n\n"; } break :blk "\n"; @@ -1858,10 +1857,9 @@ pub const Parser = struct { try stack.append(RenderState { .Text = blk: { if (i != 0) { - const prev_statement_node = block.statements.items[i - 1]; - const prev_line_index = prev_statement_node.lastToken().line; - const this_line_index = statement_node.firstToken().line; - if (this_line_index - prev_line_index >= 2) { + const prev_node = block.statements.items[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, statement_node.firstToken()); + if (loc.line >= 2) { break :blk "\n\n"; } } @@ -2083,9 +2081,8 @@ pub const Parser = struct { .Text = blk: { if (i != 0) { const prev_node = fields_and_decls[i - 1]; - const prev_line_index = prev_node.lastToken().line; - const this_line_index = node.firstToken().line; - if (this_line_index - prev_line_index >= 2) { + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { break :blk "\n\n"; } } diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 2e40999920..3427678341 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -5,8 +5,6 @@ pub const Token = struct { id: Id, start: usize, end: usize, - line: usize, - column: usize, const KeywordId = struct { bytes: []const u8, @@ -180,28 +178,34 @@ pub const Token = struct { pub const Tokenizer = struct { buffer: []const u8, index: usize, - line: usize, - column: usize, pending_invalid_token: ?Token, - pub const LineLocation = struct { + pub const Location = struct { + line: usize, + column: usize, line_start: usize, line_end: usize, }; - pub fn getTokenLocation(self: &Tokenizer, token: &const Token) LineLocation { - var loc = LineLocation { - .line_start = 0, + pub fn getTokenLocation(self: &Tokenizer, start_index: usize, token: &const Token) Location { + var loc = Location { + .line = 0, + .column = 0, + .line_start = start_index, .line_end = self.buffer.len, }; - for (self.buffer) |c, i| { - if (i == token.start) { - loc.line_end = i; + for (self.buffer[start_index..]) |c, i| { + if (i + start_index == token.start) { + loc.line_end = i + start_index; while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {} return loc; } if (c == '\n') { + loc.line += 1; + loc.column = 0; loc.line_start = i + 1; + } else { + loc.column += 1; } } return loc; @@ -216,8 +220,6 @@ pub const Tokenizer = struct { return Tokenizer { .buffer = buffer, .index = 0, - .line = 0, - .column = 0, .pending_invalid_token = null, }; } @@ -277,8 +279,6 @@ pub const Tokenizer = struct { .id = Token.Id.Eof, .start = self.index, .end = undefined, - .line = self.line, - .column = self.column, }; while (self.index < self.buffer.len) : (self.index += 1) { const c = self.buffer[self.index]; @@ -286,12 +286,9 @@ pub const Tokenizer = struct { State.Start => switch (c) { ' ' => { result.start = self.index + 1; - result.column += 1; }, '\n' => { result.start = self.index + 1; - result.line += 1; - result.column = 0; }, 'c' => { state = State.C; @@ -977,15 +974,6 @@ pub const Tokenizer = struct { } } - for (self.buffer[start_index..self.index]) |c| { - if (c == '\n') { - self.line += 1; - self.column = 0; - } else { - self.column += 1; - } - } - if (result.id == Token.Id.Eof) { if (self.pending_invalid_token) |token| { self.pending_invalid_token = null; @@ -1009,8 +997,6 @@ pub const Tokenizer = struct { .id = Token.Id.Invalid, .start = self.index, .end = self.index + invalid_length, - .line = self.line, - .column = self.column, }; } From ca0085c46dc5acb6ea93e35192b7b52294381d75 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 10:54:48 +0200 Subject: [PATCH 23/54] std.zig.parser now parses error set declarations --- std/zig/ast.zig | 47 ++++++++++++------- std/zig/parser.zig | 111 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 137 insertions(+), 21 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 1d42d721d9..efb959c7f2 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -11,6 +11,7 @@ pub const Node = struct { pub const Id = enum { Root, VarDecl, + ErrorSetDecl, ContainerDecl, StructField, UnionTag, @@ -44,6 +45,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), + Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).iterate(index), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).iterate(index), Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).iterate(index), @@ -78,6 +80,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), + Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).firstToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).firstToken(), Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).firstToken(), @@ -112,6 +115,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), + Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).lastToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).lastToken(), Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).lastToken(), @@ -212,6 +216,30 @@ pub const NodeVarDecl = struct { } }; +pub const NodeErrorSetDecl = struct { + base: Node, + error_token: Token, + decls: ArrayList(&NodeIdentifier), + rbrace_token: Token, + + pub fn iterate(self: &NodeErrorSetDecl, index: usize) ?&Node { + var i = index; + + if (i < self.decls.len) return self.decls.at(i); + i -= self.decls.len; + + return null; + } + + pub fn firstToken(self: &NodeErrorSetDecl) Token { + return self.error_token; + } + + pub fn lastToken(self: &NodeErrorSetDecl) Token { + return self.rbrace_token; + } +}; + pub const NodeContainerDecl = struct { base: Node, ltoken: Token, @@ -251,23 +279,8 @@ pub const NodeContainerDecl = struct { InitArg.Enum => { } } - if (i < self.decls.len) return self.decls.at(i); - i -= self.decls.len; - - switch (self.kind) { - Kind.Struct => |fields| { - if (i < fields.len) return fields.at(i); - i -= fields.len; - }, - Kind.Enum => |tags| { - if (i < tags.len) return tags.at(i); - i -= tags.len; - }, - Kind.Union => |tags| { - if (i < tags.len) return tags.at(i); - i -= tags.len; - }, - } + if (i < self.fields_and_decls.len) return self.fields_and_decls.at(i); + i -= self.fields_and_decls.len; return null; } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index f7db8fd93d..8177700fb5 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -685,11 +685,66 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_error => { - const node = try arena.create(ast.NodeErrorType); - *node = ast.NodeErrorType { - .base = self.initNode(ast.Node.Id.ErrorType), - .token = token, + const next = self.getNextToken(); + + if (next.id != Token.Id.LBrace) { + self.putBackToken(next); + const node = try arena.create(ast.NodeErrorType); + *node = ast.NodeErrorType { + .base = self.initNode(ast.Node.Id.ErrorType), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + } + + const node = try arena.create(ast.NodeErrorSetDecl); + *node = ast.NodeErrorSetDecl { + .base = self.initNode(ast.Node.Id.ErrorSetDecl), + .error_token = token, + .decls = ArrayList(&ast.NodeIdentifier).init(arena), + .rbrace_token = undefined, }; + + while (true) { + const t = self.getNextToken(); + switch (t.id) { + Token.Id.RBrace => { + node.rbrace_token = t; + break; + }, + Token.Id.Identifier => { + try node.decls.append( + try self.createIdentifier(arena, t) + ); + }, + else => { + return self.parseError(token, "expected {} or {}, found {}", + @tagName(Token.Id.RBrace), + @tagName(Token.Id.Identifier), + @tagName(token.id)); + } + } + + const t2 = self.getNextToken(); + switch (t2.id) { + Token.Id.RBrace => { + node.rbrace_token = t; + break; + }, + Token.Id.Comma => continue, + else => { + return self.parseError(token, "expected {} or {}, found {}", + @tagName(Token.Id.RBrace), + @tagName(Token.Id.Comma), + @tagName(token.id)); + } + } + } + try stack.append(State { .Operand = &node.base }); @@ -2115,6 +2170,42 @@ pub const Parser = struct { }, } }, + ast.Node.Id.ErrorSetDecl => { + const err_set_decl = @fieldParentPtr(ast.NodeErrorSetDecl, "base", base); + try stream.print("error "); + + try stack.append(RenderState { .Text = "}"}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Text = "\n"}); + + const decls = err_set_decl.decls.toSliceConst(); + var i = decls.len; + while (i != 0) { + i -= 1; + const node = decls[i]; + try stack.append(RenderState { .Expression = &node.base}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + if (i != 0) { + const prev_node = decls[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { + break :blk "\n\n"; + } + } + break :blk "\n"; + }, + }); + + if (i != 0) { + try stack.append(RenderState { .Text = "," }); + } + } + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "{"}); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); @@ -2643,6 +2734,18 @@ test "zig fmt: union declaration" { ); } +test "zig fmt: error set declaration" { + try testCanonical( + \\const E = error { + \\ A, + \\ B, + \\ + \\ C + \\}; + \\ + ); +} + test "zig fmt: switch" { try testCanonical( \\test "switch" { From 744416ce0cd23c84e33965fc8171c0d1aea0659c Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 14:58:51 +0200 Subject: [PATCH 24/54] std.zig.parser should now parse operators with precedence. * This haven't been tested yet --- std/zig/parser.zig | 227 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 177 insertions(+), 50 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 8177700fb5..3f7b5f4e31 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1000,24 +1000,58 @@ pub const Parser = struct { var expression = popSuffixOp(&stack); while (true) { - switch (stack.pop()) { - State.Expression => |dest_ptr| { - // we're done - try dest_ptr.store(expression); + const s = stack.pop(); + if (s == State.Expression) { + const dest_ptr = s.Expression; + // we're done + try dest_ptr.store(expression); + break; + } + + var placement_ptr = &expression; + var rhs : &&ast.Node = undefined; + const node = blk: { + switch (s) { + State.InfixOp => |infix_op| { + infix_op.lhs = popSuffixOp(&stack); + infix_op.rhs = expression; + rhs = &infix_op.rhs; + break :blk &infix_op.base; + }, + State.PrefixOp => |prefix_op| { + prefix_op.rhs = expression; + rhs = &prefix_op.rhs; + break :blk &prefix_op.base; + }, + else => unreachable, + } + }; + const node_perc = precedence(node); + + while (true) { + const perc = precedence(*placement_ptr); + + if (node_perc > perc) { + *placement_ptr = node; break; - }, - State.InfixOp => |infix_op| { - infix_op.rhs = expression; - infix_op.lhs = popSuffixOp(&stack); - expression = &infix_op.base; - continue; - }, - State.PrefixOp => |prefix_op| { - prefix_op.rhs = expression; - expression = &prefix_op.base; - continue; - }, - else => unreachable, + } + + switch ((*placement_ptr).id) { + ast.Node.Id.SuffixOp => { + const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", *placement_ptr); + placement_ptr = &suffix_op.lhs; + *rhs = suffix_op.lhs; + }, + ast.Node.Id.InfixOp => { + const infix_op = @fieldParentPtr(ast.NodeInfixOp, "base", *placement_ptr); + placement_ptr = &infix_op.lhs; + *rhs = infix_op.lhs; + }, + else => { + *placement_ptr = node; + break; + }, + } } } continue; @@ -1308,6 +1342,99 @@ pub const Parser = struct { } } + fn precedence(node: &ast.Node) u8 { + switch (node.id) { + ast.Node.Id.PrefixOp => { + const prefix_op = @fieldParentPtr(ast.NodePrefixOp, "base", node); + switch (prefix_op.op) { + ast.NodePrefixOp.PrefixOp.ArrayType, + ast.NodePrefixOp.PrefixOp.SliceType => return 1, + + ast.NodePrefixOp.PrefixOp.BoolNot, + ast.NodePrefixOp.PrefixOp.Negation, + ast.NodePrefixOp.PrefixOp.NegationWrap, + ast.NodePrefixOp.PrefixOp.BitNot, + ast.NodePrefixOp.PrefixOp.Deref, + ast.NodePrefixOp.PrefixOp.AddrOf, + ast.NodePrefixOp.PrefixOp.UnwrapMaybe => return 3, + + ast.NodePrefixOp.PrefixOp.Try, + ast.NodePrefixOp.PrefixOp.Return => return 255, + } + }, + ast.Node.Id.SuffixOp => { + const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", node); + switch (suffix_op.op) { + ast.NodeSuffixOp.SuffixOp.Call, + ast.NodeSuffixOp.SuffixOp.Slice, + ast.NodeSuffixOp.SuffixOp.ArrayAccess => return 2, + + ast.NodeSuffixOp.SuffixOp.ArrayInitializer, + ast.NodeSuffixOp.SuffixOp.StructInitializer => return 5, + } + }, + ast.Node.Id.InfixOp => { + const infix_op = @fieldParentPtr(ast.NodeInfixOp, "base", node); + switch (infix_op.op) { + ast.NodeInfixOp.InfixOp.Period => return 2, + + ast.NodeInfixOp.InfixOp.ErrorUnion => return 4, + + ast.NodeInfixOp.InfixOp.Div, + ast.NodeInfixOp.InfixOp.ArrayMult, + ast.NodeInfixOp.InfixOp.Mod, + ast.NodeInfixOp.InfixOp.Mult, + ast.NodeInfixOp.InfixOp.MultWrap => return 6, + + ast.NodeInfixOp.InfixOp.Add, + ast.NodeInfixOp.InfixOp.AddWrap, + ast.NodeInfixOp.InfixOp.ArrayCat, + ast.NodeInfixOp.InfixOp.Sub, + ast.NodeInfixOp.InfixOp.SubWrap => return 7, + + ast.NodeInfixOp.InfixOp.BitShiftLeft, + ast.NodeInfixOp.InfixOp.BitShiftRight => return 8, + + ast.NodeInfixOp.InfixOp.BitAnd => return 9, + + ast.NodeInfixOp.InfixOp.BitXor => return 10, + + ast.NodeInfixOp.InfixOp.BitOr => return 11, + + ast.NodeInfixOp.InfixOp.EqualEqual, + ast.NodeInfixOp.InfixOp.BangEqual, + ast.NodeInfixOp.InfixOp.GreaterOrEqual, + ast.NodeInfixOp.InfixOp.GreaterThan, + ast.NodeInfixOp.InfixOp.LessOrEqual, + ast.NodeInfixOp.InfixOp.LessThan => return 12, + + ast.NodeInfixOp.InfixOp.BoolAnd => return 13, + + ast.NodeInfixOp.InfixOp.BoolOr => return 14, + + ast.NodeInfixOp.InfixOp.UnwrapMaybe => return 15, + + ast.NodeInfixOp.InfixOp.Assign, + ast.NodeInfixOp.InfixOp.AssignBitAnd, + ast.NodeInfixOp.InfixOp.AssignBitOr, + ast.NodeInfixOp.InfixOp.AssignBitShiftLeft, + ast.NodeInfixOp.InfixOp.AssignBitShiftRight, + ast.NodeInfixOp.InfixOp.AssignBitXor, + ast.NodeInfixOp.InfixOp.AssignDiv, + ast.NodeInfixOp.InfixOp.AssignMinus, + ast.NodeInfixOp.InfixOp.AssignMinusWrap, + ast.NodeInfixOp.InfixOp.AssignMod, + ast.NodeInfixOp.InfixOp.AssignPlus, + ast.NodeInfixOp.InfixOp.AssignPlusWrap, + ast.NodeInfixOp.InfixOp.AssignTimes, + ast.NodeInfixOp.InfixOp.AssignTimesWarp, + ast.NodeInfixOp.InfixOp.MergeErrorSets => return 16, + } + }, + else => return 0, + } + } + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { @@ -2549,6 +2676,39 @@ test "zig fmt: infix operators" { ); } +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\ a = b or c; + \\ (a = b) or c; + \\} + \\ + //\\ !a{}; + //\\ !(a{}); + //\\ a + b{}; + //\\ (a + b){}; + ); +} + test "zig fmt: prefix operators" { try testCanonical( \\test "prefix operators" { @@ -3062,39 +3222,6 @@ test "zig fmt: container initializers" { ); } -test "zig fmt: precedence" { - try testCanonical( - \\test "precedence" { - \\ a!b(); - \\ (a!b)(); - \\ !a!b; - \\ !(a!b); - \\ !a{}; - \\ !(a{}); - \\ a + b{}; - \\ (a + b){}; - \\ a << b + c; - \\ (a << b) + c; - \\ a & b << c; - \\ (a & b) << c; - \\ a ^ b & c; - \\ (a ^ b) & c; - \\ a | b ^ c; - \\ (a | b) ^ c; - \\ a == b | c; - \\ (a == b) | c; - \\ a and b == c; - \\ (a and b) == c; - \\ a or b and c; - \\ (a or b) and c; - \\ (a or b) and c; - \\ a = b or c; - \\ (a = b) or c; - \\} - \\ - ); -} - test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); From 779247ba114492a28287bc2026719091b7022e1d Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 23:36:55 +0200 Subject: [PATCH 25/54] std.zig Major Refactor * parser now parses expression like the C++ compiler does * This makes initializers work * Added control flow expression (only return is parsed) * Added catch parsing (It doesn't quite work) * The parse can now specify states as optional. * The parse will roll back on error if states are optional * This can be overriden by State.Required --- std/zig/ast.zig | 60 +- std/zig/parser.zig | 1691 ++++++++++++++++++++++++----------------- std/zig/tokenizer.zig | 10 + 3 files changed, 1077 insertions(+), 684 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index efb959c7f2..4aaeef8dab 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -24,6 +24,7 @@ pub const Node = struct { PrefixOp, SuffixOp, GroupedExpression, + ControlFlowExpression, FieldInitializer, IntegerLiteral, FloatLiteral, @@ -58,6 +59,7 @@ pub const Node = struct { Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).iterate(index), + Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).iterate(index), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), @@ -93,6 +95,7 @@ pub const Node = struct { Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).firstToken(), + Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).firstToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), @@ -128,6 +131,7 @@ pub const Node = struct { Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).lastToken(), + Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).lastToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), @@ -531,7 +535,7 @@ pub const NodeInfixOp = struct { op: InfixOp, rhs: &Node, - const InfixOp = enum { + const InfixOp = union(enum) { Add, AddWrap, ArrayCat, @@ -558,6 +562,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, + Catch: ?&NodeIdentifier, Div, EqualEqual, ErrorUnion, @@ -651,9 +656,9 @@ pub const NodePrefixOp = struct { BitNot, BoolNot, Deref, + MaybeType, Negation, NegationWrap, - Return, ArrayType: &Node, SliceType: AddrOfInfo, Try, @@ -754,6 +759,7 @@ pub const NodeSuffixOp = struct { const CallInfo = struct { params: ArrayList(&Node), is_async: bool, + allocator: ?&Node, }; const SliceRange = struct { @@ -831,6 +837,56 @@ pub const NodeGroupedExpression = struct { } }; +pub const NodeControlFlowExpression = struct { + base: Node, + ltoken: Token, + kind: Kind, + rhs: ?&Node, + + const Kind = union(enum) { + Break: ?Token, + Continue: ?Token, + Return, + }; + + pub fn iterate(self: &NodeControlFlowExpression, index: usize) ?&Node { + var i = index; + + if (self.rhs) |rhs| { + if (i < 1) return rhs; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeControlFlowExpression) Token { + return self.ltoken; + } + + pub fn lastToken(self: &NodeControlFlowExpression) Token { + if (self.rhs) |rhs| { + return rhs.lastToken(); + } + + switch (self.kind) { + Kind.Break => |maybe_blk_token| { + if (maybe_blk_token) |blk_token| { + return blk_token; + } + }, + Kind.Continue => |maybe_blk_token| { + if (maybe_blk_token) |blk_token| { + return blk_token; + } + }, + Kind.Return => return self.ltoken, + } + + return self.ltoken; + } +}; + pub const NodeIntegerLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3f7b5f4e31..1a8e7bde88 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -60,6 +60,7 @@ pub const Parser = struct { }; const ContainerExternCtx = struct { + dest_ptr: DestPtr, ltoken: Token, layout: ast.NodeContainerDecl.Layout, }; @@ -67,13 +68,18 @@ pub const Parser = struct { const DestPtr = union(enum) { Field: &&ast.Node, NullableField: &?&ast.Node, - List: &ArrayList(&ast.Node), - pub fn store(self: &const DestPtr, value: &ast.Node) !void { + pub fn store(self: &const DestPtr, value: &ast.Node) void { switch (*self) { DestPtr.Field => |ptr| *ptr = value, DestPtr.NullableField => |ptr| *ptr = value, - DestPtr.List => |list| try list.append(value), + } + } + + pub fn get(self: &const DestPtr) &ast.Node { + switch (*self) { + DestPtr.Field => |ptr| return *ptr, + DestPtr.NullableField => |ptr| return ??*ptr, } } }; @@ -97,19 +103,13 @@ pub const Parser = struct { TopLevelDecl: TopLevelDeclCtx, ContainerExtern: ContainerExternCtx, ContainerDecl: &ast.NodeContainerDecl, - Expression: DestPtr, - ExpectOperand, - Operand: &ast.Node, - AfterOperand, - InfixOp: &ast.NodeInfixOp, - PrefixOp: &ast.NodePrefixOp, - SuffixOp: &ast.NodeSuffixOp, SliceOrArrayAccess: &ast.NodeSuffixOp, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, - TypeExpr: DestPtr, VarDecl: &ast.NodeVarDecl, VarDeclAlign: &ast.NodeVarDecl, VarDeclEq: &ast.NodeVarDecl, + IfToken: @TagType(Token.Id), + IfTokenSave: ExpectTokenSave, ExpectToken: @TagType(Token.Id), ExpectTokenSave: ExpectTokenSave, FnProto: &ast.NodeFnProto, @@ -125,6 +125,48 @@ pub const Parser = struct { FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer), FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, + + /// A state that can be appended before any other State. If an error occures, + /// the parser will first try looking for the closest optional state. If an + /// optional state is found, the parser will revert to the state it was in + /// when the optional was added. This will polute the arena allocator with + /// "leaked" nodes. TODO: Figure out if it's nessesary to handle leaked nodes. + Optional: Parser, + + /// Optional can be reverted by adding the Required state to the stack. + Required, + + Expression: DestPtr, + AssignmentExpressionBegin: DestPtr, + AssignmentExpressionEnd: DestPtr, + UnwrapExpressionBegin: DestPtr, + UnwrapExpressionEnd: DestPtr, + BoolOrExpressionBegin: DestPtr, + BoolOrExpressionEnd: DestPtr, + BoolAndExpressionBegin: DestPtr, + BoolAndExpressionEnd: DestPtr, + ComparisonExpressionBegin: DestPtr, + ComparisonExpressionEnd: DestPtr, + BinaryOrExpressionBegin: DestPtr, + BinaryOrExpressionEnd: DestPtr, + BinaryXorExpressionBegin: DestPtr, + BinaryXorExpressionEnd: DestPtr, + BinaryAndExpressionBegin: DestPtr, + BinaryAndExpressionEnd: DestPtr, + BitShiftExpressionBegin: DestPtr, + BitShiftExpressionEnd: DestPtr, + AdditionExpressionBegin: DestPtr, + AdditionExpressionEnd: DestPtr, + MultiplyExpressionBegin: DestPtr, + MultiplyExpressionEnd: DestPtr, + CurlySuffixExpressionBegin: DestPtr, + CurlySuffixExpressionEnd: DestPtr, + TypeExprBegin: DestPtr, + TypeExprEnd: DestPtr, + PrefixOpExpression: DestPtr, + SuffixOpExpressionBegin: DestPtr, + SuffixOpExpressionEnd: DestPtr, + PrimaryExpression: DestPtr, }; /// Returns an AST tree, allocated with the parser's allocator. @@ -193,12 +235,16 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; const name_token = self.getNextToken(); - if (name_token.id != Token.Id.StringLiteral) - return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.StringLiteral), @tagName(name_token.id)); + if (name_token.id != Token.Id.StringLiteral) { + try self.parseError(&stack, token, "expected {}, found {}", @tagName(Token.Id.StringLiteral), @tagName(name_token.id)); + continue; + } const lbrace = self.getNextToken(); - if (lbrace.id != Token.Id.LBrace) - return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); + if (lbrace.id != Token.Id.LBrace) { + try self.parseError(&stack, token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); + continue; + } const name = try self.createStringLiteral(arena, name_token); const block = try self.createBlock(arena, token); @@ -284,28 +330,35 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { - const fn_token = try self.eatToken(Token.Id.Keyword_fn); // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, ctx.decls, fn_token, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Keyword_fn, + .ptr = &fn_proto.fn_token, + } + }); + continue; + }, + else => { + try self.parseError(&stack, token, "expected variable declaration or function, found {}", @tagName(token.id)); continue; }, - else => return self.parseError(token, "expected variable declaration or function, found {}", @tagName(token.id)), } }, State.VarDecl => |var_decl| { - var_decl.name_token = try self.eatToken(Token.Id.Identifier); stack.append(State { .VarDeclAlign = var_decl }) catch unreachable; - - const next_token = self.getNextToken(); - if (next_token.id == Token.Id.Colon) { - try stack.append(State { .TypeExpr = DestPtr {.NullableField = &var_decl.type_node} }); - continue; - } - - self.putBackToken(next_token); + try stack.append(State { .TypeExprBegin = DestPtr {.NullableField = &var_decl.type_node} }); + try stack.append(State { .IfToken = Token.Id.Colon }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Identifier, + .ptr = &var_decl.name_token, + } + }); continue; }, State.VarDeclAlign => |var_decl| { @@ -313,9 +366,9 @@ pub const Parser = struct { const next_token = self.getNextToken(); if (next_token.id == Token.Id.Keyword_align) { - _ = try self.eatToken(Token.Id.LParen); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr{.NullableField = &var_decl.align_node} }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); continue; } @@ -341,7 +394,8 @@ pub const Parser = struct { var_decl.semicolon_token = token; continue; } - return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id)); + try self.parseError(&stack, token, "expected '=' or ';', found {}", @tagName(token.id)); + continue; }, State.ContainerExtern => |ctx| { @@ -357,20 +411,20 @@ pub const Parser = struct { Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, else => { - return self.parseError(token, "expected {}, {} or {}, found {}", + try self.parseError(&stack, token, "expected {}, {} or {}, found {}", @tagName(Token.Id.Keyword_struct), @tagName(Token.Id.Keyword_union), @tagName(Token.Id.Keyword_enum), @tagName(token.id)); + continue; }, }, .init_arg_expr = undefined, .fields_and_decls = ArrayList(&ast.Node).init(arena), .rbrace_token = undefined, }; + ctx.dest_ptr.store(&node.base); - try stack.append(State { .Operand = &node.base }); - try stack.append(State.AfterOperand); try stack.append(State { .ContainerDecl = node }); try stack.append(State { .ExpectToken = Token.Id.LBrace }); @@ -496,144 +550,569 @@ pub const Parser = struct { }, State.ExpectToken => |token_id| { - _ = try self.eatToken(token_id); + _ = (try self.eatToken(&stack, token_id)) ?? continue; continue; }, State.ExpectTokenSave => |expect_token_save| { - *expect_token_save.ptr = try self.eatToken(expect_token_save.id); + *expect_token_save.ptr = (try self.eatToken(&stack, expect_token_save.id)) ?? continue; continue; }, - State.Expression => |dest_ptr| { - // save the dest_ptr for later - stack.append(state) catch unreachable; - try stack.append(State.ExpectOperand); + State.IfToken => |token_id| { + const token = self.getNextToken(); + if (@TagType(Token.Id)(token.id) != token_id) { + self.putBackToken(token); + _ = stack.pop(); + continue; + } continue; }, - State.ExpectOperand => { - // we'll either get an operand (like 1 or x), - // or a prefix operator (like ~ or return). + + State.IfTokenSave => |if_token_save| { + const token = self.getNextToken(); + if (@TagType(Token.Id)(token.id) != if_token_save.id) { + self.putBackToken(token); + _ = stack.pop(); + continue; + } + + *if_token_save.ptr = token; + continue; + }, + + State.Optional, + State.Required => { }, + + State.Expression => |dest_ptr| { const token = self.getNextToken(); switch (token.id) { - Token.Id.Keyword_return => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.Return) }); - try stack.append(State.ExpectOperand); - continue; - }, Token.Id.Keyword_try => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.Try) }); - try stack.append(State.ExpectOperand); + const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp.Try); + dest_ptr.store(&node.base); + + stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; continue; }, - Token.Id.Minus => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.Negation) }); - try stack.append(State.ExpectOperand); + Token.Id.Keyword_return => { + const node = try self.createControlFlowExpr(arena, token, ast.NodeControlFlowExpression.Kind.Return); + dest_ptr.store(&node.base); + + stack.append(State { .Optional = *self }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); continue; }, - Token.Id.MinusPercent => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.NegationWrap) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_break => { + @panic("TODO: break"); }, - Token.Id.Tilde => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.BitNot) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_continue => { + @panic("TODO: break"); }, - Token.Id.QuestionMarkQuestionMark => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.UnwrapMaybe) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_cancel => { + @panic("TODO: cancel"); }, - Token.Id.Bang => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.BoolNot) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_resume => { + @panic("TODO: resume"); }, - Token.Id.Asterisk => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.Deref) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_await => { + @panic("TODO: await"); }, - Token.Id.LBracket => { - const rbracket_token = self.getNextToken(); - if (rbracket_token.id == Token.Id.RBracket) { - const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ - .SliceType = ast.NodePrefixOp.AddrOfInfo { - .align_expr = null, - .bit_offset_start_token = null, - .bit_offset_end_token = null, - .const_token = null, - .volatile_token = null, - } - }); - try stack.append(State { .PrefixOp = prefix_op }); - try stack.append(State.ExpectOperand); - try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf }); + Token.Id.Keyword_suspend => { + @panic("TODO: suspend"); + }, + else => { + self.putBackToken(token); + stack.append(State { .AssignmentExpressionBegin = dest_ptr }) catch unreachable; + continue; + } + } + }, + + State.AssignmentExpressionBegin => |dest_ptr| { + try stack.append(State { .AssignmentExpressionEnd = dest_ptr }); + stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; + continue; + }, + + State.AssignmentExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToAssignment(token.id)) |ass_id| { + const node = try self.createInfixOp(arena, token, ass_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .UnwrapExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.UnwrapExpressionBegin => |dest_ptr| { + stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BoolOrExpressionBegin = dest_ptr }); + continue; + }, + + State.UnwrapExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_catch => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp { .Catch = null }); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); + + const next = self.getNextToken(); + if (next.id != Token.Id.Pipe) { + self.putBackToken(next); continue; } - self.putBackToken(rbracket_token); - - const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ - .ArrayType = undefined, - }); - try stack.append(State { .PrefixOp = prefix_op }); - try stack.append(State.ExpectOperand); - try stack.append(State { .ExpectToken = Token.Id.RBracket }); - try stack.append(State { .Expression = DestPtr { .Field = &prefix_op.op.ArrayType } }); - - }, - Token.Id.Ampersand => { - const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ - .AddrOf = ast.NodePrefixOp.AddrOfInfo { - .align_expr = null, - .bit_offset_start_token = null, - .bit_offset_end_token = null, - .const_token = null, - .volatile_token = null, + node.op.Catch = try self.createIdentifier(arena, undefined); + try stack.append(State { .ExpectToken = Token.Id.Pipe }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Identifier, + .ptr = &(??node.op.Catch).name_token } }); - try stack.append(State { .PrefixOp = prefix_op }); - try stack.append(State.ExpectOperand); - try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf }); continue; }, - Token.Id.Identifier => { - try stack.append(State { - .Operand = &(try self.createIdentifier(arena, token)).base + Token.Id.QuestionMarkQuestionMark => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.UnwrapMaybe); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BoolOrExpressionBegin => |dest_ptr| { + stack.append(State { .BoolOrExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BoolAndExpressionBegin = dest_ptr }); + continue; + }, + + State.BoolOrExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_or => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BoolOr); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BoolOrExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BoolAndExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BoolAndExpressionBegin => |dest_ptr| { + stack.append(State { .BoolAndExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .ComparisonExpressionBegin = dest_ptr }); + continue; + }, + + State.BoolAndExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_and => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BoolAnd); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BoolAndExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .ComparisonExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.ComparisonExpressionBegin => |dest_ptr| { + stack.append(State { .ComparisonExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryOrExpressionBegin = dest_ptr }); + continue; + }, + + State.ComparisonExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToComparison(token.id)) |comp_id| { + const node = try self.createInfixOp(arena, token, comp_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .ComparisonExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryOrExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.BinaryOrExpressionBegin => |dest_ptr| { + stack.append(State { .BinaryOrExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryXorExpressionBegin = dest_ptr }); + continue; + }, + + State.BinaryOrExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Pipe => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BitOr); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BinaryOrExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryXorExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BinaryXorExpressionBegin => |dest_ptr| { + stack.append(State { .BinaryXorExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryAndExpressionBegin = dest_ptr }); + continue; + }, + + State.BinaryXorExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Caret => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BitXor); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BinaryXorExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryAndExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BinaryAndExpressionBegin => |dest_ptr| { + stack.append(State { .BinaryAndExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BitShiftExpressionBegin = dest_ptr }); + continue; + }, + + State.BinaryAndExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Ampersand => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BitAnd); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BinaryAndExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BitShiftExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BitShiftExpressionBegin => |dest_ptr| { + stack.append(State { .BitShiftExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .AdditionExpressionBegin = dest_ptr }); + continue; + }, + + State.BitShiftExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToBitShift(token.id)) |bitshift_id| { + const node = try self.createInfixOp(arena, token, bitshift_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BitShiftExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .AdditionExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.AdditionExpressionBegin => |dest_ptr| { + stack.append(State { .AdditionExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .MultiplyExpressionBegin = dest_ptr }); + continue; + }, + + State.AdditionExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToAddition(token.id)) |add_id| { + const node = try self.createInfixOp(arena, token, add_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .AdditionExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .MultiplyExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.MultiplyExpressionBegin => |dest_ptr| { + stack.append(State { .MultiplyExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .CurlySuffixExpressionBegin = dest_ptr }); + continue; + }, + + State.MultiplyExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToMultiply(token.id)) |mult_id| { + const node = try self.createInfixOp(arena, token, mult_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .MultiplyExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .CurlySuffixExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.CurlySuffixExpressionBegin => |dest_ptr| { + stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .TypeExprBegin = dest_ptr }); + continue; + }, + + State.CurlySuffixExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (token.id != Token.Id.LBrace) { + self.putBackToken(token); + continue; + } + + const next = self.getNextToken(); + self.putBackToken(token); + switch (next.id) { + Token.Id.Period => { + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), + }); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { + .FieldInitListItemOrEnd = ListState(&ast.NodeFieldInitializer) { + .list = &node.op.StructInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } }); - try stack.append(State.AfterOperand); continue; }, + else => { + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .ArrayInitializer = ArrayList(&ast.Node).init(arena), + }); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { + .ExprListItemOrEnd = ListState(&ast.Node) { + .list = &node.op.ArrayInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } + }); + continue; + }, + } + }, + + State.TypeExprBegin => |dest_ptr| { + const token = self.getNextToken(); + if (token.id == Token.Id.Keyword_var) { + @panic("TODO param with type var"); + } + self.putBackToken(token); + + stack.append(State { .TypeExprEnd = dest_ptr }) catch unreachable; + try stack.append(State { .PrefixOpExpression = dest_ptr }); + continue; + }, + + State.TypeExprEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Bang => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.ErrorUnion); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .TypeExprEnd = dest_ptr }) catch unreachable; + try stack.append(State { .PrefixOpExpression = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.PrefixOpExpression => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToPrefixOp(token.id)) |prefix_id| { + const node = try self.createPrefixOp(arena, token, prefix_id); + dest_ptr.store(&node.base); + + stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; + if (node.op == ast.NodePrefixOp.PrefixOp.AddrOf) { + try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); + } + continue; + } else { + self.putBackToken(token); + stack.append(State { .SuffixOpExpressionBegin = dest_ptr }) catch unreachable; + continue; + } + }, + + State.SuffixOpExpressionBegin => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_async => { + @panic("TODO: Parse async"); + }, + else => { + self.putBackToken(token); + stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .PrimaryExpression = dest_ptr }); + continue; + } + } + }, + + State.SuffixOpExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.LParen => { + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .Call = ast.NodeSuffixOp.CallInfo { + .params = ArrayList(&ast.Node).init(arena), + .is_async = false, // TODO: ASYNC + .allocator = null, + } + }); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { + .ExprListItemOrEnd = ListState(&ast.Node) { + .list = &node.op.Call.params, + .end = Token.Id.RParen, + .ptr = &node.rtoken, + } + }); + continue; + }, + Token.Id.LBracket => { + const node = try arena.create(ast.NodeSuffixOp); + *node = ast.NodeSuffixOp { + .base = self.initNode(ast.Node.Id.SuffixOp), + .lhs = undefined, + .op = ast.NodeSuffixOp.SuffixOp { + .ArrayAccess = undefined, + }, + .rtoken = undefined, + }; + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .SliceOrArrayAccess = node }); + try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); + continue; + }, + Token.Id.Period => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .SuffixOpExpressionBegin = DestPtr { .Field = &node.rhs }}); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.PrimaryExpression => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { Token.Id.IntegerLiteral => { - try stack.append(State { - .Operand = &(try self.createIntegerLiteral(arena, token)).base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&(try self.createIntegerLiteral(arena, token)).base); continue; }, Token.Id.FloatLiteral => { - try stack.append(State { - .Operand = &(try self.createFloatLiteral(arena, token)).base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&(try self.createFloatLiteral(arena, token)).base); + continue; + }, + Token.Id.StringLiteral => { + dest_ptr.store(&(try self.createStringLiteral(arena, token)).base); + continue; + }, + Token.Id.CharLiteral => { + const node = try arena.create(ast.NodeCharLiteral); + *node = ast.NodeCharLiteral { + .base = self.initNode(ast.Node.Id.CharLiteral), + .token = token, + }; + dest_ptr.store(&node.base); continue; }, Token.Id.Keyword_undefined => { - try stack.append(State { - .Operand = &(try self.createUndefined(arena, token)).base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&(try self.createUndefined(arena, token)).base); continue; }, Token.Id.Keyword_true, Token.Id.Keyword_false => { @@ -642,10 +1121,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.BoolLiteral), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; }, Token.Id.Keyword_null => { @@ -654,10 +1130,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.NullLiteral), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; }, Token.Id.Keyword_this => { @@ -666,10 +1139,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.ThisLiteral), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; }, Token.Id.Keyword_unreachable => { @@ -678,12 +1148,97 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.Unreachable), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; }, + Token.Id.MultilineStringLiteralLine => { + const node = try arena.create(ast.NodeMultilineStringLiteral); + *node = ast.NodeMultilineStringLiteral { + .base = self.initNode(ast.Node.Id.MultilineStringLiteral), + .tokens = ArrayList(Token).init(arena), + }; + dest_ptr.store(&node.base); + try node.tokens.append(token); + + while (true) { + const multiline_str = self.getNextToken(); + if (multiline_str.id != Token.Id.MultilineStringLiteralLine) { + self.putBackToken(multiline_str); + break; + } + + try node.tokens.append(multiline_str); + } + continue; + }, + Token.Id.LParen => { + const node = try arena.create(ast.NodeGroupedExpression); + *node = ast.NodeGroupedExpression { + .base = self.initNode(ast.Node.Id.GroupedExpression), + .lparen = token, + .expr = undefined, + .rparen = undefined, + }; + dest_ptr.store(&node.base); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RParen, + .ptr = &node.rparen, + } + }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, + Token.Id.Identifier => { + dest_ptr.store(&(try self.createIdentifier(arena, token)).base); + continue; + }, + Token.Id.Builtin => { + const node = try arena.create(ast.NodeBuiltinCall); + *node = ast.NodeBuiltinCall { + .base = self.initNode(ast.Node.Id.BuiltinCall), + .builtin_token = token, + .params = ArrayList(&ast.Node).init(arena), + .rparen_token = undefined, + }; + dest_ptr.store(&node.base); + try stack.append(State { + .ExprListItemOrEnd = ListState(&ast.Node) { + .list = &node.params, + .end = Token.Id.RParen, + .ptr = &node.rparen_token, + } + }); + try stack.append(State { .ExpectToken = Token.Id.LParen, }); + continue; + }, + Token.Id.LBracket => { + const rbracket_token = self.getNextToken(); + if (rbracket_token.id == Token.Id.RBracket) { + const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ + .SliceType = ast.NodePrefixOp.AddrOfInfo { + .align_expr = null, + .bit_offset_start_token = null, + .bit_offset_end_token = null, + .const_token = null, + .volatile_token = null, + } + }); + dest_ptr.store(&node.base); + try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); + continue; + } + + self.putBackToken(rbracket_token); + + const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ + .ArrayType = undefined, + }); + dest_ptr.store(&node.base); + try stack.append(State { .ExpectToken = Token.Id.RBracket }); + try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayType } }); + + }, Token.Id.Keyword_error => { const next = self.getNextToken(); @@ -694,10 +1249,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.ErrorType), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; } @@ -708,6 +1260,7 @@ pub const Parser = struct { .decls = ArrayList(&ast.NodeIdentifier).init(arena), .rbrace_token = undefined, }; + dest_ptr.store(&node.base); while (true) { const t = self.getNextToken(); @@ -722,10 +1275,11 @@ pub const Parser = struct { ); }, else => { - return self.parseError(token, "expected {} or {}, found {}", + try self.parseError(&stack, token, "expected {} or {}, found {}", @tagName(Token.Id.RBrace), @tagName(Token.Id.Identifier), @tagName(token.id)); + continue; } } @@ -737,23 +1291,20 @@ pub const Parser = struct { }, Token.Id.Comma => continue, else => { - return self.parseError(token, "expected {} or {}, found {}", + try self.parseError(&stack, token, "expected {} or {}, found {}", @tagName(Token.Id.RBrace), @tagName(Token.Id.Comma), @tagName(token.id)); + continue; } } } - - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); continue; }, Token.Id.Keyword_packed => { try stack.append(State { .ContainerExtern = ContainerExternCtx { + .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Packed, }, @@ -762,6 +1313,7 @@ pub const Parser = struct { Token.Id.Keyword_extern => { try stack.append(State { .ContainerExtern = ContainerExternCtx { + .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Extern, }, @@ -771,100 +1323,25 @@ pub const Parser = struct { self.putBackToken(token); try stack.append(State { .ContainerExtern = ContainerExternCtx { + .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Auto, }, }); }, - Token.Id.Builtin => { - const node = try arena.create(ast.NodeBuiltinCall); - *node = ast.NodeBuiltinCall { - .base = self.initNode(ast.Node.Id.BuiltinCall), - .builtin_token = token, - .params = ArrayList(&ast.Node).init(arena), - .rparen_token = undefined, - }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); - try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { - .list = &node.params, - .end = Token.Id.RParen, - .ptr = &node.rparen_token, - } - }); - try stack.append(State { .ExpectToken = Token.Id.LParen, }); - continue; + Token.Id.LBrace => { + @panic("TODO: Block expr"); }, - Token.Id.StringLiteral => { - const node = try self.createStringLiteral(arena, token); - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); - continue; + Token.Id.Keyword_fn => { + @panic("TODO: fn proto"); }, - Token.Id.CharLiteral => { - const node = try arena.create(ast.NodeCharLiteral); - *node = ast.NodeCharLiteral { - .base = self.initNode(ast.Node.Id.CharLiteral), - .token = token, - }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); - continue; + Token.Id.Keyword_asm => { + @panic("TODO: inline asm"); }, - Token.Id.MultilineStringLiteralLine => { - const node = try arena.create(ast.NodeMultilineStringLiteral); - *node = ast.NodeMultilineStringLiteral { - .base = self.initNode(ast.Node.Id.MultilineStringLiteral), - .tokens = ArrayList(Token).init(arena), - }; - try node.tokens.append(token); - - while (true) { - const multiline_str = self.getNextToken(); - if (multiline_str.id != Token.Id.MultilineStringLiteralLine) { - self.putBackToken(multiline_str); - break; - } - - try node.tokens.append(multiline_str); - } - - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + else => { + try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; - }, - Token.Id.LParen => { - const node = try arena.create(ast.NodeGroupedExpression); - *node = ast.NodeGroupedExpression { - .base = self.initNode(ast.Node.Id.GroupedExpression), - .lparen = token, - .expr = undefined, - .rparen = undefined, - }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.RParen, - .ptr = &node.rparen, - } - }); - try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); - continue; - }, - - else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), + } } }, @@ -880,8 +1357,6 @@ pub const Parser = struct { .end = undefined, } }; - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); const rbracket_token = self.getNextToken(); if (rbracket_token.id != Token.Id.RBracket) { @@ -900,161 +1375,12 @@ pub const Parser = struct { }, Token.Id.RBracket => { node.rtoken = token; - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); continue; }, - else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id)) - } - }, - - State.AfterOperand => { - // we'll either get an infix operator (like != or ^), - // or a postfix operator (like () or {}), - // otherwise this expression is done (like on a ; or else). - var token = self.getNextToken(); - if (tokenIdToInfixOp(token.id)) |infix_id| { - try stack.append(State { - .InfixOp = try self.createInfixOp(arena, token, infix_id) - }); - try stack.append(State.ExpectOperand); + else => { + try self.parseError(&stack, token, "expected ']' or '..', found {}", @tagName(token.id)); continue; - - } else if (token.id == Token.Id.LParen) { - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .Call = ast.NodeSuffixOp.CallInfo { - .params = ArrayList(&ast.Node).init(arena), - .is_async = false, // TODO: ASYNC - } - }); - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); - try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { - .list = &node.op.Call.params, - .end = Token.Id.RParen, - .ptr = &node.rtoken, - } - }); - continue; - - } else if (token.id == Token.Id.LBracket) { - const node = try arena.create(ast.NodeSuffixOp); - *node = ast.NodeSuffixOp { - .base = self.initNode(ast.Node.Id.SuffixOp), - .lhs = undefined, - .op = ast.NodeSuffixOp.SuffixOp { - .ArrayAccess = undefined, - }, - .rtoken = undefined, - }; - try stack.append(State { .SliceOrArrayAccess = node }); - try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); - continue; - - // TODO: This is the initializer parsing code. It doesn't work because of - // the ambiguity between function bodies and initializers: - // fn main() void {} or fn main() (void {}) - } else if (false) { //(token.id == Token.Id.LBrace) { - const next = self.getNextToken(); - - switch (next.id) { - Token.Id.Period => { - self.putBackToken(token); - - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), - }); - - try stack.append(State { - .FieldInitListItemOrEnd = ListState(&ast.NodeFieldInitializer) { - .list = &node.op.StructInitializer, - .end = Token.Id.RBrace, - .ptr = &node.rtoken, - } - }); - continue; - }, - else => { - self.putBackToken(token); - - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .ArrayInitializer = ArrayList(&ast.Node).init(arena), - }); - - try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { - .list = &node.op.ArrayInitializer, - .end = Token.Id.RBrace, - .ptr = &node.rtoken, - } - }); - continue; - }, } - - // TODO: Parse postfix operator - } else { - // no postfix/infix operator after this operand. - self.putBackToken(token); - - var expression = popSuffixOp(&stack); - while (true) { - const s = stack.pop(); - if (s == State.Expression) { - const dest_ptr = s.Expression; - // we're done - try dest_ptr.store(expression); - break; - } - - var placement_ptr = &expression; - var rhs : &&ast.Node = undefined; - const node = blk: { - switch (s) { - State.InfixOp => |infix_op| { - infix_op.lhs = popSuffixOp(&stack); - infix_op.rhs = expression; - rhs = &infix_op.rhs; - break :blk &infix_op.base; - }, - State.PrefixOp => |prefix_op| { - prefix_op.rhs = expression; - rhs = &prefix_op.rhs; - break :blk &prefix_op.base; - }, - else => unreachable, - } - }; - const node_perc = precedence(node); - - while (true) { - const perc = precedence(*placement_ptr); - - if (node_perc > perc) { - *placement_ptr = node; - break; - } - - switch ((*placement_ptr).id) { - ast.Node.Id.SuffixOp => { - const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", *placement_ptr); - placement_ptr = &suffix_op.lhs; - *rhs = suffix_op.lhs; - }, - ast.Node.Id.InfixOp => { - const infix_op = @fieldParentPtr(ast.NodeInfixOp, "base", *placement_ptr); - placement_ptr = &infix_op.lhs; - *rhs = infix_op.lhs; - }, - else => { - *placement_ptr = node; - break; - }, - } - } - } - continue; } }, @@ -1069,7 +1395,7 @@ pub const Parser = struct { self.putBackToken(token); stack.append(State { .ExprListCommaOrEnd = list_state }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.List = list_state.list} }); + try stack.append(State { .Expression = DestPtr{ .Field = try list_state.list.addOne() } }); }, State.FieldInitListItemOrEnd => |list_state| { @@ -1130,21 +1456,30 @@ pub const Parser = struct { switch (token.id) { Token.Id.Keyword_align => { stack.append(state) catch unreachable; - if (addr_of_info.align_expr != null) return self.parseError(token, "multiple align qualifiers"); - _ = try self.eatToken(Token.Id.LParen); + if (addr_of_info.align_expr != null) { + try self.parseError(&stack, token, "multiple align qualifiers"); + continue; + } try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr{.NullableField = &addr_of_info.align_expr} }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); continue; }, Token.Id.Keyword_const => { stack.append(state) catch unreachable; - if (addr_of_info.const_token != null) return self.parseError(token, "duplicate qualifier: const"); + if (addr_of_info.const_token != null) { + try self.parseError(&stack, token, "duplicate qualifier: const"); + continue; + } addr_of_info.const_token = token; continue; }, Token.Id.Keyword_volatile => { stack.append(state) catch unreachable; - if (addr_of_info.volatile_token != null) return self.parseError(token, "duplicate qualifier: volatile"); + if (addr_of_info.volatile_token != null) { + try self.parseError(&stack, token, "duplicate qualifier: volatile"); + continue; + } addr_of_info.volatile_token = token; continue; }, @@ -1155,17 +1490,6 @@ pub const Parser = struct { } }, - State.TypeExpr => |dest_ptr| { - const token = self.getNextToken(); - if (token.id == Token.Id.Keyword_var) { - @panic("TODO param with type var"); - } - self.putBackToken(token); - - stack.append(State { .Expression = dest_ptr }) catch unreachable; - continue; - }, - State.FnProto => |fn_proto| { stack.append(State { .FnProtoAlign = fn_proto }) catch unreachable; try stack.append(State { .ParamDecl = fn_proto }); @@ -1201,14 +1525,14 @@ pub const Parser = struct { Token.Id.Bang => { fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined }; stack.append(State { - .TypeExpr = DestPtr {.Field = &fn_proto.return_type.InferErrorSet}, + .TypeExprBegin = DestPtr {.Field = &fn_proto.return_type.InferErrorSet}, }) catch unreachable; }, else => { self.putBackToken(token); fn_proto.return_type = ast.NodeFnProto.ReturnType { .Explicit = undefined }; stack.append(State { - .TypeExpr = DestPtr {.Field = &fn_proto.return_type.Explicit}, + .TypeExprBegin = DestPtr {.Field = &fn_proto.return_type.Explicit}, }) catch unreachable; }, } @@ -1251,7 +1575,7 @@ pub const Parser = struct { stack.append(State { .ParamDecl = fn_proto }) catch unreachable; try stack.append(State.ParamDeclComma); try stack.append(State { - .TypeExpr = DestPtr {.Field = ¶m_decl.type_node} + .TypeExprBegin = DestPtr {.Field = ¶m_decl.type_node} }); continue; }, @@ -1264,7 +1588,10 @@ pub const Parser = struct { continue; }, Token.Id.Comma => continue, - else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)), + else => { + try self.parseError(&stack, token, "expected ',' or ')', found {}", @tagName(token.id)); + continue; + }, } }, @@ -1278,7 +1605,10 @@ pub const Parser = struct { continue; }, Token.Id.Semicolon => continue, - else => return self.parseError(token, "expected ';' or '{{', found {}", @tagName(token.id)), + else => { + try self.parseError(&stack, token, "expected ';' or '{{', found {}", @tagName(token.id)); + continue; + }, } }, @@ -1329,112 +1659,13 @@ pub const Parser = struct { } stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.List = &block.statements} }); + try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); continue; }, - - // These are data, not control flow. - State.InfixOp => unreachable, - State.PrefixOp => unreachable, - State.SuffixOp => unreachable, - State.Operand => unreachable, } } } - fn precedence(node: &ast.Node) u8 { - switch (node.id) { - ast.Node.Id.PrefixOp => { - const prefix_op = @fieldParentPtr(ast.NodePrefixOp, "base", node); - switch (prefix_op.op) { - ast.NodePrefixOp.PrefixOp.ArrayType, - ast.NodePrefixOp.PrefixOp.SliceType => return 1, - - ast.NodePrefixOp.PrefixOp.BoolNot, - ast.NodePrefixOp.PrefixOp.Negation, - ast.NodePrefixOp.PrefixOp.NegationWrap, - ast.NodePrefixOp.PrefixOp.BitNot, - ast.NodePrefixOp.PrefixOp.Deref, - ast.NodePrefixOp.PrefixOp.AddrOf, - ast.NodePrefixOp.PrefixOp.UnwrapMaybe => return 3, - - ast.NodePrefixOp.PrefixOp.Try, - ast.NodePrefixOp.PrefixOp.Return => return 255, - } - }, - ast.Node.Id.SuffixOp => { - const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", node); - switch (suffix_op.op) { - ast.NodeSuffixOp.SuffixOp.Call, - ast.NodeSuffixOp.SuffixOp.Slice, - ast.NodeSuffixOp.SuffixOp.ArrayAccess => return 2, - - ast.NodeSuffixOp.SuffixOp.ArrayInitializer, - ast.NodeSuffixOp.SuffixOp.StructInitializer => return 5, - } - }, - ast.Node.Id.InfixOp => { - const infix_op = @fieldParentPtr(ast.NodeInfixOp, "base", node); - switch (infix_op.op) { - ast.NodeInfixOp.InfixOp.Period => return 2, - - ast.NodeInfixOp.InfixOp.ErrorUnion => return 4, - - ast.NodeInfixOp.InfixOp.Div, - ast.NodeInfixOp.InfixOp.ArrayMult, - ast.NodeInfixOp.InfixOp.Mod, - ast.NodeInfixOp.InfixOp.Mult, - ast.NodeInfixOp.InfixOp.MultWrap => return 6, - - ast.NodeInfixOp.InfixOp.Add, - ast.NodeInfixOp.InfixOp.AddWrap, - ast.NodeInfixOp.InfixOp.ArrayCat, - ast.NodeInfixOp.InfixOp.Sub, - ast.NodeInfixOp.InfixOp.SubWrap => return 7, - - ast.NodeInfixOp.InfixOp.BitShiftLeft, - ast.NodeInfixOp.InfixOp.BitShiftRight => return 8, - - ast.NodeInfixOp.InfixOp.BitAnd => return 9, - - ast.NodeInfixOp.InfixOp.BitXor => return 10, - - ast.NodeInfixOp.InfixOp.BitOr => return 11, - - ast.NodeInfixOp.InfixOp.EqualEqual, - ast.NodeInfixOp.InfixOp.BangEqual, - ast.NodeInfixOp.InfixOp.GreaterOrEqual, - ast.NodeInfixOp.InfixOp.GreaterThan, - ast.NodeInfixOp.InfixOp.LessOrEqual, - ast.NodeInfixOp.InfixOp.LessThan => return 12, - - ast.NodeInfixOp.InfixOp.BoolAnd => return 13, - - ast.NodeInfixOp.InfixOp.BoolOr => return 14, - - ast.NodeInfixOp.InfixOp.UnwrapMaybe => return 15, - - ast.NodeInfixOp.InfixOp.Assign, - ast.NodeInfixOp.InfixOp.AssignBitAnd, - ast.NodeInfixOp.InfixOp.AssignBitOr, - ast.NodeInfixOp.InfixOp.AssignBitShiftLeft, - ast.NodeInfixOp.InfixOp.AssignBitShiftRight, - ast.NodeInfixOp.InfixOp.AssignBitXor, - ast.NodeInfixOp.InfixOp.AssignDiv, - ast.NodeInfixOp.InfixOp.AssignMinus, - ast.NodeInfixOp.InfixOp.AssignMinusWrap, - ast.NodeInfixOp.InfixOp.AssignMod, - ast.NodeInfixOp.InfixOp.AssignPlus, - ast.NodeInfixOp.InfixOp.AssignPlusWrap, - ast.NodeInfixOp.InfixOp.AssignTimes, - ast.NodeInfixOp.InfixOp.AssignTimesWarp, - ast.NodeInfixOp.InfixOp.MergeErrorSets => return 16, - } - }, - else => return 0, - } - } - fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { @@ -1448,74 +1679,104 @@ pub const Parser = struct { return; } - return self.parseError(token, "expected ',' or {}, found {}", @tagName(*end), @tagName(token.id)); + try self.parseError(stack, token, "expected ',' or {}, found {}", @tagName(*end), @tagName(token.id)); }, } } - fn popSuffixOp(stack: &ArrayList(State)) &ast.Node { - var expression: &ast.Node = undefined; - var left_leaf_ptr: &&ast.Node = &expression; - while (true) { - switch (stack.pop()) { - State.SuffixOp => |suffix_op| { - *left_leaf_ptr = &suffix_op.base; - left_leaf_ptr = &suffix_op.lhs; - }, - State.Operand => |operand| { - *left_leaf_ptr = operand; - break; - }, - else => unreachable, - } - } - - return expression; + fn tokenIdToAssignment(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.AmpersandEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitAnd), + Token.Id.AngleBracketAngleBracketLeftEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitShiftLeft), + Token.Id.AngleBracketAngleBracketRightEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitShiftRight), + Token.Id.AsteriskEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignTimes), + Token.Id.AsteriskPercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignTimesWarp), + Token.Id.CaretEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitXor), + Token.Id.Equal => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Assign), + Token.Id.MinusEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignMinus), + Token.Id.MinusPercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignMinusWrap), + Token.Id.PercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignMod), + Token.Id.PipeEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitOr), + Token.Id.PlusEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignPlus), + Token.Id.PlusPercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignPlusWrap), + Token.Id.SlashEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignDiv), + else => null, + }; } - fn tokenIdToInfixOp(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + fn tokenIdToComparison(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' return switch (*id) { - Token.Id.Ampersand => ast.NodeInfixOp.InfixOp.BitAnd, - Token.Id.AmpersandEqual => ast.NodeInfixOp.InfixOp.AssignBitAnd, - Token.Id.AngleBracketAngleBracketLeft => ast.NodeInfixOp.InfixOp.BitShiftLeft, - Token.Id.AngleBracketAngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftLeft, - Token.Id.AngleBracketAngleBracketRight => ast.NodeInfixOp.InfixOp.BitShiftRight, - Token.Id.AngleBracketAngleBracketRightEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftRight, - Token.Id.AngleBracketLeft => ast.NodeInfixOp.InfixOp.LessThan, - Token.Id.AngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.LessOrEqual, - Token.Id.AngleBracketRight => ast.NodeInfixOp.InfixOp.GreaterThan, - Token.Id.AngleBracketRightEqual => ast.NodeInfixOp.InfixOp.GreaterOrEqual, - Token.Id.Asterisk => ast.NodeInfixOp.InfixOp.Mult, - Token.Id.AsteriskAsterisk => ast.NodeInfixOp.InfixOp.ArrayMult, - Token.Id.AsteriskEqual => ast.NodeInfixOp.InfixOp.AssignTimes, - Token.Id.AsteriskPercent => ast.NodeInfixOp.InfixOp.MultWrap, - Token.Id.AsteriskPercentEqual => ast.NodeInfixOp.InfixOp.AssignTimesWarp, - Token.Id.Bang => ast.NodeInfixOp.InfixOp.ErrorUnion, - Token.Id.BangEqual => ast.NodeInfixOp.InfixOp.BangEqual, - Token.Id.Caret => ast.NodeInfixOp.InfixOp.BitXor, - Token.Id.CaretEqual => ast.NodeInfixOp.InfixOp.AssignBitXor, - Token.Id.Equal => ast.NodeInfixOp.InfixOp.Assign, - Token.Id.EqualEqual => ast.NodeInfixOp.InfixOp.EqualEqual, - Token.Id.Keyword_and => ast.NodeInfixOp.InfixOp.BoolAnd, - Token.Id.Keyword_or => ast.NodeInfixOp.InfixOp.BoolOr, - Token.Id.Minus => ast.NodeInfixOp.InfixOp.Sub, - Token.Id.MinusEqual => ast.NodeInfixOp.InfixOp.AssignMinus, - Token.Id.MinusPercent => ast.NodeInfixOp.InfixOp.SubWrap, - Token.Id.MinusPercentEqual => ast.NodeInfixOp.InfixOp.AssignMinusWrap, - Token.Id.Percent => ast.NodeInfixOp.InfixOp.Mod, - Token.Id.PercentEqual => ast.NodeInfixOp.InfixOp.AssignMod, - Token.Id.Period => ast.NodeInfixOp.InfixOp.Period, - Token.Id.Pipe => ast.NodeInfixOp.InfixOp.BitOr, - Token.Id.PipeEqual => ast.NodeInfixOp.InfixOp.AssignBitOr, - Token.Id.PipePipe => ast.NodeInfixOp.InfixOp.MergeErrorSets, - Token.Id.Plus => ast.NodeInfixOp.InfixOp.Add, - Token.Id.PlusEqual => ast.NodeInfixOp.InfixOp.AssignPlus, - Token.Id.PlusPercent => ast.NodeInfixOp.InfixOp.AddWrap, - Token.Id.PlusPercentEqual => ast.NodeInfixOp.InfixOp.AssignPlusWrap, - Token.Id.PlusPlus => ast.NodeInfixOp.InfixOp.ArrayCat, - Token.Id.QuestionMarkQuestionMark => ast.NodeInfixOp.InfixOp.UnwrapMaybe, - Token.Id.Slash => ast.NodeInfixOp.InfixOp.Div, - Token.Id.SlashEqual => ast.NodeInfixOp.InfixOp.AssignDiv, + Token.Id.BangEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.BangEqual), + Token.Id.EqualEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.EqualEqual), + Token.Id.AngleBracketLeft => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.LessThan), + Token.Id.AngleBracketLeftEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.LessOrEqual), + Token.Id.AngleBracketRight => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.GreaterThan), + Token.Id.AngleBracketRightEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.GreaterOrEqual), + else => null, + }; + } + + fn tokenIdToBitShift(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.AngleBracketAngleBracketLeft => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.BitShiftLeft), + Token.Id.AngleBracketAngleBracketRight => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.BitShiftRight), + else => null, + }; + } + + fn tokenIdToAddition(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.Minus => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Sub), + Token.Id.MinusPercent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.SubWrap), + Token.Id.Plus => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Add), + Token.Id.PlusPercent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AddWrap), + Token.Id.PlusPlus => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.ArrayCat), + else => null, + }; + } + + fn tokenIdToMultiply(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.Slash => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Div), + Token.Id.Asterisk => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Mult), + Token.Id.AsteriskAsterisk => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.ArrayMult), + Token.Id.AsteriskPercent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.MultWrap), + Token.Id.Percent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Mod), + Token.Id.PipePipe => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.MergeErrorSets), + else => null, + }; + } + + fn tokenIdToPrefixOp(id: &const Token.Id) ?ast.NodePrefixOp.PrefixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.Bang => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.BoolNot), + Token.Id.Tilde => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.BitNot), + Token.Id.Minus => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.Negation), + Token.Id.MinusPercent => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.NegationWrap), + Token.Id.Asterisk => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.Deref), + Token.Id.Ampersand => ast.NodePrefixOp.PrefixOp { + .AddrOf = ast.NodePrefixOp.AddrOfInfo { + .align_expr = null, + .bit_offset_start_token = null, + .bit_offset_end_token = null, + .const_token = null, + .volatile_token = null, + }, + }, + Token.Id.QuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.MaybeType), + Token.Id.QuestionMarkQuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.UnwrapMaybe), else => null, }; } @@ -1637,6 +1898,19 @@ pub const Parser = struct { return node; } + fn createControlFlowExpr(self: &Parser, arena: &mem.Allocator, ltoken: &const Token, + kind: &const ast.NodeControlFlowExpression.Kind) !&ast.NodeControlFlowExpression + { + const node = try arena.create(ast.NodeControlFlowExpression); + *node = ast.NodeControlFlowExpression { + .base = self.initNode(ast.Node.Id.ControlFlowExpression), + .ltoken = *ltoken, + .kind = *kind, + .rhs = undefined, + }; + return node; + } + fn createInfixOp(self: &Parser, arena: &mem.Allocator, op_token: &const Token, op: &const ast.NodeInfixOp.InfixOp) !&ast.NodeInfixOp { const node = try arena.create(ast.NodeInfixOp); @@ -1752,39 +2026,56 @@ pub const Parser = struct { return node; } - fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) { - const loc = self.tokenizer.getTokenLocation(0, token); - warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args); - warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]); - { - var i: usize = 0; - while (i < loc.column) : (i += 1) { - warn(" "); + fn parseError(self: &Parser, stack: &ArrayList(State), token: &const Token, comptime fmt: []const u8, args: ...) !void { + // Before reporting an error. We pop the stack to see if our state was optional + self.revertIfOptional(stack) catch { + const loc = self.tokenizer.getTokenLocation(0, token); + warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args); + warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]); + { + var i: usize = 0; + while (i < loc.column) : (i += 1) { + warn(" "); + } } - } - { - const caret_count = token.end - token.start; - var i: usize = 0; - while (i < caret_count) : (i += 1) { - warn("~"); + { + const caret_count = token.end - token.start; + var i: usize = 0; + while (i < caret_count) : (i += 1) { + warn("~"); + } } - } - warn("\n"); - return error.ParseError; + warn("\n"); + return error.ParseError; + }; } - fn expectToken(self: &Parser, token: &const Token, id: @TagType(Token.Id)) !void { - if (token.id != id) { - return self.parseError(token, "expected {}, found {}", @tagName(id), @tagName(token.id)); - } - } - - fn eatToken(self: &Parser, id: @TagType(Token.Id)) !Token { + fn eatToken(self: &Parser, stack: &ArrayList(State), id: @TagType(Token.Id)) !?Token { const token = self.getNextToken(); - try self.expectToken(token, id); + if (token.id != id) { + try self.parseError(stack, token, "expected {}, found {}", @tagName(id), @tagName(token.id)); + return null; + } return token; } + fn revertIfOptional(self: &Parser, stack: &ArrayList(State)) !void { + while (stack.popOrNull()) |state| { + switch (state) { + State.Optional => |revert| { + *self = state.Optional; + return; + }, + State.Required => { + return error.StateRequired; + }, + else => { } + } + } + + return error.NoOptionalStateFound; + } + fn putBackToken(self: &Parser, token: &const Token) void { self.put_back_tokens[self.put_back_count] = *token; self.put_back_count += 1; @@ -2054,51 +2345,62 @@ pub const Parser = struct { ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); - const text = switch (prefix_op_node.op) { - ast.NodeInfixOp.InfixOp.Add => " + ", - ast.NodeInfixOp.InfixOp.AddWrap => " +% ", - ast.NodeInfixOp.InfixOp.ArrayCat => " ++ ", - ast.NodeInfixOp.InfixOp.ArrayMult => " ** ", - ast.NodeInfixOp.InfixOp.Assign => " = ", - ast.NodeInfixOp.InfixOp.AssignBitAnd => " &= ", - ast.NodeInfixOp.InfixOp.AssignBitOr => " |= ", - ast.NodeInfixOp.InfixOp.AssignBitShiftLeft => " <<= ", - ast.NodeInfixOp.InfixOp.AssignBitShiftRight => " >>= ", - ast.NodeInfixOp.InfixOp.AssignBitXor => " ^= ", - ast.NodeInfixOp.InfixOp.AssignDiv => " /= ", - ast.NodeInfixOp.InfixOp.AssignMinus => " -= ", - ast.NodeInfixOp.InfixOp.AssignMinusWrap => " -%= ", - ast.NodeInfixOp.InfixOp.AssignMod => " %= ", - ast.NodeInfixOp.InfixOp.AssignPlus => " += ", - ast.NodeInfixOp.InfixOp.AssignPlusWrap => " +%= ", - ast.NodeInfixOp.InfixOp.AssignTimes => " *= ", - ast.NodeInfixOp.InfixOp.AssignTimesWarp => " *%= ", - ast.NodeInfixOp.InfixOp.BangEqual => " != ", - ast.NodeInfixOp.InfixOp.BitAnd => " & ", - ast.NodeInfixOp.InfixOp.BitOr => " | ", - ast.NodeInfixOp.InfixOp.BitShiftLeft => " << ", - ast.NodeInfixOp.InfixOp.BitShiftRight => " >> ", - ast.NodeInfixOp.InfixOp.BitXor => " ^ ", - ast.NodeInfixOp.InfixOp.BoolAnd => " and ", - ast.NodeInfixOp.InfixOp.BoolOr => " or ", - ast.NodeInfixOp.InfixOp.Div => " / ", - ast.NodeInfixOp.InfixOp.EqualEqual => " == ", - ast.NodeInfixOp.InfixOp.ErrorUnion => "!", - ast.NodeInfixOp.InfixOp.GreaterOrEqual => " >= ", - ast.NodeInfixOp.InfixOp.GreaterThan => " > ", - ast.NodeInfixOp.InfixOp.LessOrEqual => " <= ", - ast.NodeInfixOp.InfixOp.LessThan => " < ", - ast.NodeInfixOp.InfixOp.MergeErrorSets => " || ", - ast.NodeInfixOp.InfixOp.Mod => " % ", - ast.NodeInfixOp.InfixOp.Mult => " * ", - ast.NodeInfixOp.InfixOp.MultWrap => " *% ", - ast.NodeInfixOp.InfixOp.Period => ".", - ast.NodeInfixOp.InfixOp.Sub => " - ", - ast.NodeInfixOp.InfixOp.SubWrap => " -% ", - ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ", - }; - try stack.append(RenderState { .Text = text }); + if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { + if (prefix_op_node.op.Catch) |payload| { + try stack.append(RenderState { .Text = "|" }); + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = "|" }); + } + try stack.append(RenderState { .Text = " catch " }); + } else { + const text = switch (prefix_op_node.op) { + ast.NodeInfixOp.InfixOp.Add => " + ", + ast.NodeInfixOp.InfixOp.AddWrap => " +% ", + ast.NodeInfixOp.InfixOp.ArrayCat => " ++ ", + ast.NodeInfixOp.InfixOp.ArrayMult => " ** ", + ast.NodeInfixOp.InfixOp.Assign => " = ", + ast.NodeInfixOp.InfixOp.AssignBitAnd => " &= ", + ast.NodeInfixOp.InfixOp.AssignBitOr => " |= ", + ast.NodeInfixOp.InfixOp.AssignBitShiftLeft => " <<= ", + ast.NodeInfixOp.InfixOp.AssignBitShiftRight => " >>= ", + ast.NodeInfixOp.InfixOp.AssignBitXor => " ^= ", + ast.NodeInfixOp.InfixOp.AssignDiv => " /= ", + ast.NodeInfixOp.InfixOp.AssignMinus => " -= ", + ast.NodeInfixOp.InfixOp.AssignMinusWrap => " -%= ", + ast.NodeInfixOp.InfixOp.AssignMod => " %= ", + ast.NodeInfixOp.InfixOp.AssignPlus => " += ", + ast.NodeInfixOp.InfixOp.AssignPlusWrap => " +%= ", + ast.NodeInfixOp.InfixOp.AssignTimes => " *= ", + ast.NodeInfixOp.InfixOp.AssignTimesWarp => " *%= ", + ast.NodeInfixOp.InfixOp.BangEqual => " != ", + ast.NodeInfixOp.InfixOp.BitAnd => " & ", + ast.NodeInfixOp.InfixOp.BitOr => " | ", + ast.NodeInfixOp.InfixOp.BitShiftLeft => " << ", + ast.NodeInfixOp.InfixOp.BitShiftRight => " >> ", + ast.NodeInfixOp.InfixOp.BitXor => " ^ ", + ast.NodeInfixOp.InfixOp.BoolAnd => " and ", + ast.NodeInfixOp.InfixOp.BoolOr => " or ", + ast.NodeInfixOp.InfixOp.Div => " / ", + ast.NodeInfixOp.InfixOp.EqualEqual => " == ", + ast.NodeInfixOp.InfixOp.ErrorUnion => "!", + ast.NodeInfixOp.InfixOp.GreaterOrEqual => " >= ", + ast.NodeInfixOp.InfixOp.GreaterThan => " > ", + ast.NodeInfixOp.InfixOp.LessOrEqual => " <= ", + ast.NodeInfixOp.InfixOp.LessThan => " < ", + ast.NodeInfixOp.InfixOp.MergeErrorSets => " || ", + ast.NodeInfixOp.InfixOp.Mod => " % ", + ast.NodeInfixOp.InfixOp.Mult => " * ", + ast.NodeInfixOp.InfixOp.MultWrap => " *% ", + ast.NodeInfixOp.InfixOp.Period => ".", + ast.NodeInfixOp.InfixOp.Sub => " - ", + ast.NodeInfixOp.InfixOp.SubWrap => " -% ", + ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ", + else => unreachable, + }; + + try stack.append(RenderState { .Text = text }); + } try stack.append(RenderState { .Expression = prefix_op_node.lhs }); }, ast.Node.Id.PrefixOp => { @@ -2143,9 +2445,9 @@ pub const Parser = struct { ast.NodePrefixOp.PrefixOp.Deref => try stream.write("*"), ast.NodePrefixOp.PrefixOp.Negation => try stream.write("-"), ast.NodePrefixOp.PrefixOp.NegationWrap => try stream.write("-%"), - ast.NodePrefixOp.PrefixOp.Return => try stream.write("return "), ast.NodePrefixOp.PrefixOp.Try => try stream.write("try "), ast.NodePrefixOp.PrefixOp.UnwrapMaybe => try stream.write("??"), + ast.NodePrefixOp.PrefixOp.MaybeType => try stream.write("?"), } }, ast.Node.Id.SuffixOp => { @@ -2185,6 +2487,32 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = suffix_op.lhs }); }, + ast.Node.Id.ControlFlowExpression => { + const flow_expr = @fieldParentPtr(ast.NodeControlFlowExpression, "base", base); + switch (flow_expr.kind) { + ast.NodeControlFlowExpression.Kind.Break => |maybe_blk_token| { + try stream.print("break"); + if (maybe_blk_token) |blk_token| { + try stream.print(" :{}", self.tokenizer.getTokenSlice(blk_token)); + } + }, + ast.NodeControlFlowExpression.Kind.Continue => |maybe_blk_token| { + try stream.print("continue"); + if (maybe_blk_token) |blk_token| { + try stream.print(" :{}", self.tokenizer.getTokenSlice(blk_token)); + } + }, + ast.NodeControlFlowExpression.Kind.Return => { + try stream.print("return"); + }, + + } + + if (flow_expr.rhs) |rhs| { + try stream.print(" "); + try stack.append(RenderState { .Expression = rhs }); + } + }, ast.Node.Id.GroupedExpression => { const grouped_expr = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); try stack.append(RenderState { .Text = ")"}); @@ -2770,7 +3098,6 @@ test "zig fmt: values" { \\ error; \\ this; \\ unreachable; - \\ suspend; \\} \\ ); @@ -2906,6 +3233,38 @@ test "zig fmt: error set declaration" { ); } +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: error!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch |err| return; + \\} + \\ + ); +} + +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{}; + \\} + \\ + ); +} + +test "zig fmt: container initializers" { + try testCanonical( + \\const a1 = []u8{ }; + \\const a2 = []u8{ 1, 2, 3, 4 }; + \\const s1 = S{ }; + \\const s2 = S{ .a = 1, .b = 2, }; + \\ + ); +} + test "zig fmt: switch" { try testCanonical( \\test "switch" { @@ -3106,17 +3465,6 @@ test "zig fmt: defer" { ); } -test "zig fmt: catch" { - try testCanonical( - \\test "catch" { - \\ const a: error!u8 = 0; - \\ _ = a catch return; - \\ _ = a catch |err| return; - \\} - \\ - ); -} - test "zig fmt: comptime" { try testCanonical( \\fn a() u8 { @@ -3201,27 +3549,6 @@ test "zig fmt: coroutines" { ); } -test "zig fmt: arrays" { - try testCanonical( - \\test "test array" { - \\ const a: [2]u8 = [2]u8{ 1, 2 }; - \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{}; - \\} - \\ - ); -} - -test "zig fmt: container initializers" { - try testCanonical( - \\const a1 = []u8{ }; - \\const a2 = []u8{ 1, 2, 3, 4 }; - \\const s1 = S{ }; - \\const s2 = S{ .a = 1, .b = 2, }; - \\ - ); -} - test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 3427678341..87597adff1 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -15,8 +15,11 @@ pub const Token = struct { KeywordId{.bytes="align", .id = Id.Keyword_align}, KeywordId{.bytes="and", .id = Id.Keyword_and}, KeywordId{.bytes="asm", .id = Id.Keyword_asm}, + KeywordId{.bytes="async", .id = Id.Keyword_async}, + KeywordId{.bytes="await", .id = Id.Keyword_await}, KeywordId{.bytes="break", .id = Id.Keyword_break}, KeywordId{.bytes="catch", .id = Id.Keyword_catch}, + KeywordId{.bytes="cancel", .id = Id.Keyword_cancel}, KeywordId{.bytes="comptime", .id = Id.Keyword_comptime}, KeywordId{.bytes="const", .id = Id.Keyword_const}, KeywordId{.bytes="continue", .id = Id.Keyword_continue}, @@ -37,10 +40,12 @@ pub const Token = struct { KeywordId{.bytes="or", .id = Id.Keyword_or}, KeywordId{.bytes="packed", .id = Id.Keyword_packed}, KeywordId{.bytes="pub", .id = Id.Keyword_pub}, + KeywordId{.bytes="resume", .id = Id.Keyword_resume}, KeywordId{.bytes="return", .id = Id.Keyword_return}, KeywordId{.bytes="section", .id = Id.Keyword_section}, KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc}, KeywordId{.bytes="struct", .id = Id.Keyword_struct}, + KeywordId{.bytes="suspend", .id = Id.Keyword_suspend}, KeywordId{.bytes="switch", .id = Id.Keyword_switch}, KeywordId{.bytes="test", .id = Id.Keyword_test}, KeywordId{.bytes="this", .id = Id.Keyword_this}, @@ -134,7 +139,10 @@ pub const Token = struct { Keyword_align, Keyword_and, Keyword_asm, + Keyword_async, + Keyword_await, Keyword_break, + Keyword_cancel, Keyword_catch, Keyword_comptime, Keyword_const, @@ -156,10 +164,12 @@ pub const Token = struct { Keyword_or, Keyword_packed, Keyword_pub, + Keyword_resume, Keyword_return, Keyword_section, Keyword_stdcallcc, Keyword_struct, + Keyword_suspend, Keyword_switch, Keyword_test, Keyword_this, From e45de607d6437428d82f69711a8cc8f338d019c8 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 6 Apr 2018 08:56:28 +0200 Subject: [PATCH 26/54] std.zig.parser: Initializers are now parsed and fmt correctly --- std/zig/parser.zig | 127 +++++++++++++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 45 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 1a8e7bde88..5d2a30caee 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -249,7 +249,7 @@ pub const Parser = struct { const name = try self.createStringLiteral(arena, name_token); const block = try self.createBlock(arena, token); const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, &name.base, block); - try stack.append(State { .Block = block }); + stack.append(State { .Block = block }) catch unreachable; continue; }, Token.Id.Eof => { @@ -318,14 +318,14 @@ pub const Parser = struct { // TODO shouldn't need these casts const var_decl_node = try self.createAttachVarDecl(arena, ctx.decls, ctx.visib_token, token, (?Token)(null), ctx.extern_token, ctx.lib_name); - try stack.append(State { .VarDecl = var_decl_node }); + stack.append(State { .VarDecl = var_decl_node }) catch unreachable; continue; }, Token.Id.Keyword_fn => { // TODO shouldn't need these casts const fn_proto = try self.createAttachFnProto(arena, ctx.decls, token, ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); - try stack.append(State { .FnDef = fn_proto }); + stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); continue; }, @@ -333,7 +333,7 @@ pub const Parser = struct { // TODO shouldn't need this cast const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); - try stack.append(State { .FnDef = fn_proto }); + stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); try stack.append(State { .ExpectTokenSave = ExpectTokenSave { @@ -425,7 +425,7 @@ pub const Parser = struct { }; ctx.dest_ptr.store(&node.base); - try stack.append(State { .ContainerDecl = node }); + stack.append(State { .ContainerDecl = node }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.LBrace }); const lparen = self.getNextToken(); @@ -470,7 +470,7 @@ pub const Parser = struct { }; try container_decl.fields_and_decls.append(&node.base); - try stack.append(State { .FieldListCommaOrEnd = container_decl }); + stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.type_expr } }); try stack.append(State { .ExpectToken = Token.Id.Colon }); continue; @@ -484,7 +484,7 @@ pub const Parser = struct { }; try container_decl.fields_and_decls.append(&node.base); - try stack.append(State { .FieldListCommaOrEnd = container_decl }); + stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; const next = self.getNextToken(); if (next.id != Token.Id.Colon) { @@ -504,7 +504,7 @@ pub const Parser = struct { }; try container_decl.fields_and_decls.append(&node.base); - try stack.append(State { .FieldListCommaOrEnd = container_decl }); + stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; const next = self.getNextToken(); if (next.id != Token.Id.Equal) { @@ -629,8 +629,8 @@ pub const Parser = struct { }, State.AssignmentExpressionBegin => |dest_ptr| { - try stack.append(State { .AssignmentExpressionEnd = dest_ptr }); - stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; + stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .UnwrapExpressionBegin = dest_ptr }); continue; }, @@ -926,7 +926,6 @@ pub const Parser = struct { } const next = self.getNextToken(); - self.putBackToken(token); switch (next.id) { Token.Id.Period => { const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { @@ -943,6 +942,7 @@ pub const Parser = struct { .ptr = &node.rtoken, } }); + self.putBackToken(next); continue; }, else => { @@ -960,6 +960,7 @@ pub const Parser = struct { .ptr = &node.rtoken, } }); + self.putBackToken(next); continue; }, } @@ -1180,12 +1181,12 @@ pub const Parser = struct { .rparen = undefined, }; dest_ptr.store(&node.base); - try stack.append(State { + stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.RParen, .ptr = &node.rparen, } - }); + }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); continue; }, @@ -1202,17 +1203,18 @@ pub const Parser = struct { .rparen_token = undefined, }; dest_ptr.store(&node.base); - try stack.append(State { + stack.append(State { .ExprListItemOrEnd = ListState(&ast.Node) { .list = &node.params, .end = Token.Id.RParen, .ptr = &node.rparen_token, } - }); + }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.LParen, }); continue; }, Token.Id.LBracket => { + // TODO: option("align" "(" Expression option(":" Integer ":" Integer) ")")) option("const") option("volatile") const rbracket_token = self.getNextToken(); if (rbracket_token.id == Token.Id.RBracket) { const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ @@ -1225,7 +1227,8 @@ pub const Parser = struct { } }); dest_ptr.store(&node.base); - try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); + stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; + try stack.append(State { .AddrOfModifiers = &node.op.SliceType }); continue; } @@ -1235,6 +1238,7 @@ pub const Parser = struct { .ArrayType = undefined, }); dest_ptr.store(&node.base); + stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.RBracket }); try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayType } }); @@ -1302,32 +1306,32 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_packed => { - try stack.append(State { + stack.append(State { .ContainerExtern = ContainerExternCtx { .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Packed, }, - }); + }) catch unreachable; }, Token.Id.Keyword_extern => { - try stack.append(State { + stack.append(State { .ContainerExtern = ContainerExternCtx { .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Extern, }, - }); + }) catch unreachable; }, Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => { self.putBackToken(token); - try stack.append(State { + stack.append(State { .ContainerExtern = ContainerExternCtx { .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Auto, }, - }); + }) catch unreachable; }, Token.Id.LBrace => { @panic("TODO: Block expr"); @@ -1361,12 +1365,12 @@ pub const Parser = struct { const rbracket_token = self.getNextToken(); if (rbracket_token.id != Token.Id.RBracket) { self.putBackToken(rbracket_token); - try stack.append(State { + stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.RBracket, .ptr = &node.rtoken, } - }); + }) catch unreachable; try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } }); } else { node.rtoken = rbracket_token; @@ -1638,7 +1642,7 @@ pub const Parser = struct { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), mut_token, (?Token)(comptime_token), (?Token)(null), null); - try stack.append(State { .VarDecl = var_decl }); + stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; } self.putBackToken(mut_token); @@ -1652,7 +1656,7 @@ pub const Parser = struct { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), mut_token, (?Token)(null), (?Token)(null), null); - try stack.append(State { .VarDecl = var_decl }); + stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; } self.putBackToken(mut_token); @@ -2132,6 +2136,7 @@ pub const Parser = struct { Expression: &ast.Node, VarDecl: &ast.NodeVarDecl, Statement: &ast.Node, + FieldInitializer: &ast.NodeFieldInitializer, PrintIndent, Indent: usize, }; @@ -2246,6 +2251,12 @@ pub const Parser = struct { } }, + RenderState.FieldInitializer => |field_init| { + try stream.print(".{}", self.tokenizer.getTokenSlice(field_init.name_token)); + try stream.print(" = "); + try stack.append(RenderState { .Expression = field_init.expr }); + }, + RenderState.VarDecl => |var_decl| { try stack.append(RenderState { .Text = ";" }); if (var_decl.init_node) |init_node| { @@ -2481,8 +2492,34 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = range.start}); try stack.append(RenderState { .Text = "["}); }, - ast.NodeSuffixOp.SuffixOp.StructInitializer => @panic("TODO: StructInitializer"), - ast.NodeSuffixOp.SuffixOp.ArrayInitializer => @panic("TODO: ArrayInitializer"), + ast.NodeSuffixOp.SuffixOp.StructInitializer => |field_inits| { + try stack.append(RenderState { .Text = " }"}); + var i = field_inits.len; + while (i != 0) { + i -= 1; + const field_init = field_inits.at(i); + try stack.append(RenderState { .FieldInitializer = field_init }); + try stack.append(RenderState { .Text = " " }); + if (i != 0) { + try stack.append(RenderState { .Text = "," }); + } + } + try stack.append(RenderState { .Text = "{"}); + }, + ast.NodeSuffixOp.SuffixOp.ArrayInitializer => |exprs| { + try stack.append(RenderState { .Text = " }"}); + var i = exprs.len; + while (i != 0) { + i -= 1; + const expr = exprs.at(i); + try stack.append(RenderState { .Expression = expr }); + try stack.append(RenderState { .Text = " " }); + if (i != 0) { + try stack.append(RenderState { .Text = "," }); + } + } + try stack.append(RenderState { .Text = "{"}); + }, } try stack.append(RenderState { .Expression = suffix_op.lhs }); @@ -3011,6 +3048,10 @@ test "zig fmt: precedence" { \\ (a!b)(); \\ !a!b; \\ !(a!b); + \\ !a{ }; + \\ !(a{ }); + \\ a + b{ }; + \\ (a + b){ }; \\ a << b + c; \\ (a << b) + c; \\ a & b << c; @@ -3030,10 +3071,6 @@ test "zig fmt: precedence" { \\ (a = b) or c; \\} \\ - //\\ !a{}; - //\\ !(a{}); - //\\ a + b{}; - //\\ (a + b){}; ); } @@ -3233,23 +3270,12 @@ test "zig fmt: error set declaration" { ); } -test "zig fmt: catch" { - try testCanonical( - \\test "catch" { - \\ const a: error!u8 = 0; - \\ _ = a catch return; - \\ _ = a catch |err| return; - \\} - \\ - ); -} - test "zig fmt: arrays" { try testCanonical( \\test "test array" { \\ const a: [2]u8 = [2]u8{ 1, 2 }; \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{}; + \\ const a: [0]u8 = []u8{ }; \\} \\ ); @@ -3260,7 +3286,18 @@ test "zig fmt: container initializers" { \\const a1 = []u8{ }; \\const a2 = []u8{ 1, 2, 3, 4 }; \\const s1 = S{ }; - \\const s2 = S{ .a = 1, .b = 2, }; + \\const s2 = S{ .a = 1, .b = 2 }; + \\ + ); +} + +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: error!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch |err| return; + \\} \\ ); } From f667744d44b0fc3599c85c01d3fcf3b63c4e68a6 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 6 Apr 2018 09:36:11 +0200 Subject: [PATCH 27/54] std.zig.parser Fixed: * Parsing of the optional expression in contrl flow expr * Rendering of catch expressions --- std/zig/parser.zig | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 5d2a30caee..dfc3b2f82c 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -89,6 +89,15 @@ pub const Parser = struct { ptr: &Token, }; + const RevertState = struct { + parser: Parser, + tokenizer: Tokenizer, + + // We expect, that if something is optional, then there is a field, + // that needs to be set to null, when we revert. + ptr: &?&ast.Node, + }; + fn ListState(comptime T: type) type { return struct { list: &ArrayList(T), @@ -131,7 +140,7 @@ pub const Parser = struct { /// optional state is found, the parser will revert to the state it was in /// when the optional was added. This will polute the arena allocator with /// "leaked" nodes. TODO: Figure out if it's nessesary to handle leaked nodes. - Optional: Parser, + Optional: RevertState, /// Optional can be reverted by adding the Required state to the stack. Required, @@ -598,7 +607,13 @@ pub const Parser = struct { const node = try self.createControlFlowExpr(arena, token, ast.NodeControlFlowExpression.Kind.Return); dest_ptr.store(&node.base); - stack.append(State { .Optional = *self }) catch unreachable; + stack.append(State { + .Optional = RevertState { + .parser = *self, + .tokenizer = *self.tokenizer, + .ptr = &node.rhs, + } + }) catch unreachable; try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); continue; }, @@ -673,7 +688,7 @@ pub const Parser = struct { continue; } - node.op.Catch = try self.createIdentifier(arena, undefined); + node.op.Catch = try self.createIdentifier(arena, Token(undefined)); try stack.append(State { .ExpectToken = Token.Id.Pipe }); try stack.append(State { .ExpectTokenSave = ExpectTokenSave { @@ -1910,7 +1925,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.ControlFlowExpression), .ltoken = *ltoken, .kind = *kind, - .rhs = undefined, + .rhs = null, }; return node; } @@ -2067,7 +2082,9 @@ pub const Parser = struct { while (stack.popOrNull()) |state| { switch (state) { State.Optional => |revert| { - *self = state.Optional; + *self = revert.parser; + *self.tokenizer = revert.tokenizer; + *revert.ptr = null; return; }, State.Required => { @@ -2359,7 +2376,7 @@ pub const Parser = struct { if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { if (prefix_op_node.op.Catch) |payload| { - try stack.append(RenderState { .Text = "|" }); + try stack.append(RenderState { .Text = "| " }); try stack.append(RenderState { .Expression = &payload.base }); try stack.append(RenderState { .Text = "|" }); } @@ -2956,6 +2973,10 @@ test "zig fmt: return" { \\ return 0; \\} \\ + \\fn bar() void { + \\ return; + \\} + \\ ); } From 820de1716b83ee99ed94461996ca592a302eccae Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 6 Apr 2018 15:37:49 +0200 Subject: [PATCH 28/54] std.zig.parser now parses labeled blocks. * There is also some code for switch range parsing --- std/zig/ast.zig | 88 +++++++++++++++++++++++-- std/zig/parser.zig | 161 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 215 insertions(+), 34 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 4aaeef8dab..bc066a9922 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,6 +20,8 @@ pub const Node = struct { FnProto, ParamDecl, Block, + Switch, + SwitchCase, InfixOp, PrefixOp, SuffixOp, @@ -55,6 +57,8 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), + Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), + Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -91,6 +95,8 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), + Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), + Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -127,6 +133,8 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), + Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), + Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -506,9 +514,10 @@ pub const NodeParamDecl = struct { pub const NodeBlock = struct { base: Node, - begin_token: Token, - end_token: Token, + label: ?Token, + lbrace: Token, statements: ArrayList(&Node), + rbrace: Token, pub fn iterate(self: &NodeBlock, index: usize) ?&Node { var i = index; @@ -520,11 +529,80 @@ pub const NodeBlock = struct { } pub fn firstToken(self: &NodeBlock) Token { - return self.begin_token; + if (self.label) |label| { + return label; + } + + return self.lbrace; } pub fn lastToken(self: &NodeBlock) Token { - return self.end_token; + return self.rbrace; + } +}; + +pub const NodeSwitch = struct { + base: Node, + switch_token: Token, + expr: &Node, + cases: ArrayList(&NodeSwitchCase), + rbrace: Token, + + pub fn iterate(self: &NodeSwitch, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + if (i < self.cases.len) return self.cases.at(i); + i -= self.cases.len; + + return null; + } + + pub fn firstToken(self: &NodeSwitch) Token { + return self.switch_token; + } + + pub fn lastToken(self: &NodeSwitch) Token { + return self.rbrace; + } +}; + +pub const NodeSwitchCase = struct { + base: Node, + items: ArrayList(&Node), + capture: ?Capture, + expr: &Node, + + const Capture = struct { + symbol: &NodeIdentifier, + is_ptr: bool, + }; + + pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { + var i = index; + + if (i < self.items.len) return self.items.at(i); + i -= self.items.len; + + if (self.capture) |capture| { + if (i < 1) return &capture.base; + i -= 1; + } + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeSwitchCase) Token { + return self.items.at(0).firstToken(); + } + + pub fn lastToken(self: &NodeSwitchCase) Token { + return self.expr.lastToken(); } }; @@ -575,6 +653,7 @@ pub const NodeInfixOp = struct { Mult, MultWrap, Period, + Range, Sub, SubWrap, UnwrapMaybe, @@ -625,6 +704,7 @@ pub const NodeInfixOp = struct { InfixOp.Mult, InfixOp.MultWrap, InfixOp.Period, + InfixOp.Range, InfixOp.Sub, InfixOp.SubWrap, InfixOp.UnwrapMaybe => {}, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index dfc3b2f82c..3337968d69 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -146,6 +146,8 @@ pub const Parser = struct { Required, Expression: DestPtr, + RangeExpressionBegin: DestPtr, + RangeExpressionEnd: DestPtr, AssignmentExpressionBegin: DestPtr, AssignmentExpressionEnd: DestPtr, UnwrapExpressionBegin: DestPtr, @@ -256,7 +258,7 @@ pub const Parser = struct { } const name = try self.createStringLiteral(arena, name_token); - const block = try self.createBlock(arena, token); + const block = try self.createBlock(arena, (?Token)(null), token); const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, &name.base, block); stack.append(State { .Block = block }) catch unreachable; continue; @@ -643,6 +645,27 @@ pub const Parser = struct { } }, + State.RangeExpressionBegin => |dest_ptr| { + stack.append(State { .RangeExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .Expression = dest_ptr }); + continue; + }, + + State.RangeExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (token.id == Token.Id.Ellipsis3) { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Range); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; + continue; + } else { + self.putBackToken(token); + continue; + } + }, + State.AssignmentExpressionBegin => |dest_ptr| { stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .UnwrapExpressionBegin = dest_ptr }); @@ -1205,10 +1228,6 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); continue; }, - Token.Id.Identifier => { - dest_ptr.store(&(try self.createIdentifier(arena, token)).base); - continue; - }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -1348,8 +1367,32 @@ pub const Parser = struct { }, }) catch unreachable; }, + Token.Id.Identifier => { + const next = self.getNextToken(); + if (next.id != Token.Id.Colon) { + self.putBackToken(next); + dest_ptr.store(&(try self.createIdentifier(arena, token)).base); + continue; + } + + const block = try self.createBlock(arena, (?Token)(token), Token(undefined)); + dest_ptr.store(&block.base); + + stack.append(State { .Block = block }) catch unreachable; + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.LBrace, + .ptr = &block.lbrace, + } + }); + continue; + }, Token.Id.LBrace => { - @panic("TODO: Block expr"); + const block = try self.createBlock(arena, (?Token)(null), token); + dest_ptr.store(&block.base); + + stack.append(State { .Block = block }) catch unreachable; + continue; }, Token.Id.Keyword_fn => { @panic("TODO: fn proto"); @@ -1618,7 +1661,7 @@ pub const Parser = struct { const token = self.getNextToken(); switch(token.id) { Token.Id.LBrace => { - const block = try self.createBlock(arena, token); + const block = try self.createBlock(arena, (?Token)(null), token); fn_proto.body_node = &block.base; stack.append(State { .Block = block }) catch unreachable; continue; @@ -1635,7 +1678,7 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.RBrace => { - block.end_token = token; + block.rbrace = token; continue; }, else => { @@ -1648,38 +1691,64 @@ pub const Parser = struct { }, State.Statement => |block| { - { - // Look for comptime var, comptime const - const comptime_token = self.getNextToken(); - if (comptime_token.id == Token.Id.Keyword_comptime) { + const next = self.getNextToken(); + switch (next.id) { + Token.Id.Keyword_comptime => { const mut_token = self.getNextToken(); if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(comptime_token), (?Token)(null), null); + mut_token, (?Token)(next), (?Token)(null), null); stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; + } else { + self.putBackToken(mut_token); + @panic("TODO: comptime block"); } - self.putBackToken(mut_token); - } - self.putBackToken(comptime_token); - } - { - // Look for const, var - const mut_token = self.getNextToken(); - if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { - // TODO shouldn't need these casts + }, + Token.Id.Keyword_var, Token.Id.Keyword_const => { const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(null), (?Token)(null), null); + next, (?Token)(null), (?Token)(null), null); stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; + }, + Token.Id.Identifier => { + const maybe_colon = self.getNextToken(); + if (maybe_colon.id != Token.Id.Colon) { + self.putBackToken(maybe_colon); + self.putBackToken(next); + stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); + continue; + } + + const inner_block = try self.createBlock(arena, (?Token)(next), Token(undefined)); + try block.statements.append(&inner_block.base); + + stack.append(State { .Block = inner_block }) catch unreachable; + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.LBrace, + .ptr = &inner_block.lbrace, + } + }); + continue; + }, + Token.Id.LBrace => { + const inner_block = try self.createBlock(arena, (?Token)(null), next); + try block.statements.append(&inner_block.base); + + stack.append(State { .Block = inner_block }) catch unreachable; + continue; + }, + else => { + self.putBackToken(next); + stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); + continue; } - self.putBackToken(mut_token); } - stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); - continue; }, } } @@ -1905,14 +1974,15 @@ pub const Parser = struct { return node; } - fn createBlock(self: &Parser, arena: &mem.Allocator, begin_token: &const Token) !&ast.NodeBlock { + fn createBlock(self: &Parser, arena: &mem.Allocator, label: &const ?Token, lbrace: &const Token) !&ast.NodeBlock { const node = try arena.create(ast.NodeBlock); *node = ast.NodeBlock { .base = self.initNode(ast.Node.Id.Block), - .begin_token = *begin_token, - .end_token = undefined, + .label = *label, + .lbrace = *lbrace, .statements = ArrayList(&ast.Node).init(arena), + .rbrace = undefined, }; return node; } @@ -2340,6 +2410,10 @@ pub const Parser = struct { }, ast.Node.Id.Block => { const block = @fieldParentPtr(ast.NodeBlock, "base", base); + if (block.label) |label| { + try stream.print("{}: ", self.tokenizer.getTokenSlice(label)); + } + if (block.statements.len == 0) { try stream.write("{}"); } else { @@ -2747,6 +2821,8 @@ pub const Parser = struct { }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), + ast.Node.Id.Switch => @panic("TODO switch"), + ast.Node.Id.SwitchCase => @panic("TODO switch case"), ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -2791,6 +2867,9 @@ pub const Parser = struct { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base); try stack.append(RenderState { .VarDecl = var_decl}); }, + ast.Node.Id.Block => { + try stack.append(RenderState { .Expression = base}); + }, else => { try stack.append(RenderState { .Text = ";"}); try stack.append(RenderState { .Expression = base}); @@ -3323,6 +3402,28 @@ test "zig fmt: catch" { ); } +test "zig fmt: blocks" { + try testCanonical( + \\test "blocks" { + \\ { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ blk: { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ const r = blk: { + \\ const a = 0; + \\ const b = 0; + \\ }; + \\} + \\ + ); +} + test "zig fmt: switch" { try testCanonical( \\test "switch" { From bdff5bfa3e9f6ab490771b54109cb200b180b4da Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 7 Apr 2018 01:38:38 +0200 Subject: [PATCH 29/54] std.zig.parser now parses switch --- std/zig/ast.zig | 21 ++++ std/zig/parser.zig | 260 ++++++++++++++++++++++++++++++++++++------ std/zig/tokenizer.zig | 6 + 3 files changed, 251 insertions(+), 36 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index bc066a9922..ca384efedd 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -22,6 +22,7 @@ pub const Node = struct { Block, Switch, SwitchCase, + SwitchElse, InfixOp, PrefixOp, SuffixOp, @@ -59,6 +60,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -97,6 +99,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -135,6 +138,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -606,6 +610,23 @@ pub const NodeSwitchCase = struct { } }; +pub const NodeSwitchElse = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeSwitchElse, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeSwitchElse) Token { + return self.token; + } + + pub fn lastToken(self: &NodeSwitchElse) Token { + return self.token; + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3337968d69..09f7827505 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -98,10 +98,15 @@ pub const Parser = struct { ptr: &?&ast.Node, }; - fn ListState(comptime T: type) type { + const ExprListCtx = struct { + list: &ArrayList(&ast.Node), + end: Token.Id, + ptr: &Token, + }; + + fn ListSave(comptime T: type) type { return struct { list: &ArrayList(T), - end: Token.Id, ptr: &Token, }; } @@ -129,11 +134,16 @@ pub const Parser = struct { FnDef: &ast.NodeFnProto, Block: &ast.NodeBlock, Statement: &ast.NodeBlock, - ExprListItemOrEnd: ListState(&ast.Node), - ExprListCommaOrEnd: ListState(&ast.Node), - FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer), - FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer), + ExprListItemOrEnd: ExprListCtx, + ExprListCommaOrEnd: ExprListCtx, + FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer), + FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, + SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), + SwitchCaseCapture: &?ast.NodeSwitchCase.Capture, + SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), + SwitchCaseItem: &ArrayList(&ast.Node), + SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), /// A state that can be appended before any other State. If an error occures, /// the parser will first try looking for the closest optional state. If an @@ -245,17 +255,8 @@ pub const Parser = struct { Token.Id.Keyword_test => { stack.append(State.TopLevel) catch unreachable; - const name_token = self.getNextToken(); - if (name_token.id != Token.Id.StringLiteral) { - try self.parseError(&stack, token, "expected {}, found {}", @tagName(Token.Id.StringLiteral), @tagName(name_token.id)); - continue; - } - - const lbrace = self.getNextToken(); - if (lbrace.id != Token.Id.LBrace) { - try self.parseError(&stack, token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); - continue; - } + const name_token = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + const lbrace = (try self.eatToken(&stack, Token.Id.LBrace)) ?? continue; const name = try self.createStringLiteral(arena, name_token); const block = try self.createBlock(arena, (?Token)(null), token); @@ -974,9 +975,8 @@ pub const Parser = struct { stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { - .FieldInitListItemOrEnd = ListState(&ast.NodeFieldInitializer) { + .FieldInitListItemOrEnd = ListSave(&ast.NodeFieldInitializer) { .list = &node.op.StructInitializer, - .end = Token.Id.RBrace, .ptr = &node.rtoken, } }); @@ -992,7 +992,7 @@ pub const Parser = struct { stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { + .ExprListItemOrEnd = ExprListCtx { .list = &node.op.ArrayInitializer, .end = Token.Id.RBrace, .ptr = &node.rtoken, @@ -1084,7 +1084,7 @@ pub const Parser = struct { stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { + .ExprListItemOrEnd = ExprListCtx { .list = &node.op.Call.params, .end = Token.Id.RParen, .ptr = &node.rtoken, @@ -1238,7 +1238,7 @@ pub const Parser = struct { }; dest_ptr.store(&node.base); stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { + .ExprListItemOrEnd = ExprListCtx { .list = &node.params, .end = Token.Id.RParen, .ptr = &node.rparen_token, @@ -1400,6 +1400,43 @@ pub const Parser = struct { Token.Id.Keyword_asm => { @panic("TODO: inline asm"); }, + Token.Id.Keyword_if => { + @panic("TODO: inline if"); + }, + Token.Id.Keyword_while => { + @panic("TODO: inline while"); + }, + Token.Id.Keyword_for => { + @panic("TODO: inline for"); + }, + Token.Id.Keyword_switch => { + const node = try arena.create(ast.NodeSwitch); + *node = ast.NodeSwitch { + .base = self.initNode(ast.Node.Id.Switch), + .switch_token = token, + .expr = undefined, + .cases = ArrayList(&ast.NodeSwitchCase).init(arena), + .rbrace = undefined, + }; + dest_ptr.store(&node.base); + + stack.append(State { + .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) { + .list = &node.cases, + .ptr = &node.rbrace, + }, + }) catch unreachable; + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + }, + Token.Id.Keyword_comptime => { + @panic("TODO: inline comptime"); + }, + Token.Id.Keyword_suspend => { + @panic("TODO: inline suspend"); + }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; @@ -1463,8 +1500,7 @@ pub const Parser = struct { State.FieldInitListItemOrEnd => |list_state| { var token = self.getNextToken(); - const IdTag = @TagType(Token.Id); - if (IdTag(list_state.end) == token.id){ + if (token.id == Token.Id.RBrace){ *list_state.ptr = token; continue; } @@ -1497,13 +1533,82 @@ pub const Parser = struct { }); }, + State.SwitchCaseOrEnd => |list_state| { + var token = self.getNextToken(); + + if (token.id == Token.Id.RBrace){ + *list_state.ptr = token; + continue; + } + + self.putBackToken(token); + + const node = try arena.create(ast.NodeSwitchCase); + *node = ast.NodeSwitchCase { + .base = self.initNode(ast.Node.Id.SwitchCase), + .items = ArrayList(&ast.Node).init(arena), + .capture = null, + .expr = undefined, + }; + try list_state.list.append(node); + stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; + try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); + try stack.append(State { .SwitchCaseCapture = &node.capture }); + + const maybe_else = self.getNextToken(); + if (maybe_else.id == Token.Id.Keyword_else) { + const else_node = try arena.create(ast.NodeSwitchElse); + *else_node = ast.NodeSwitchElse { + .base = self.initNode(ast.Node.Id.SwitchElse), + .token = maybe_else, + }; + try node.items.append(&else_node.base); + try stack.append(State { .ExpectToken = Token.Id.EqualAngleBracketRight }); + continue; + } else { + self.putBackToken(maybe_else); + try stack.append(State { .SwitchCaseItem = &node.items }); + continue; + } + }, + + State.SwitchCaseCapture => |capture| { + const token = self.getNextToken(); + if (token.id != Token.Id.Pipe) { + self.putBackToken(token); + continue; + } + + const is_ptr = blk: { + const asterik = self.getNextToken(); + if (asterik.id == Token.Id.Asterisk) { + break :blk true; + } else { + self.putBackToken(asterik); + break :blk false; + } + }; + + const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + _ = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + *capture = ast.NodeSwitchCase.Capture { + .symbol = try self.createIdentifier(arena, ident), + .is_ptr = is_ptr + }; + }, + + State.SwitchCaseItem => |case_items| { + stack.append(State { .SwitchCaseItemCommaOrEnd = case_items }) catch unreachable; + try stack.append(State { .RangeExpressionBegin = DestPtr{ .Field = try case_items.addOne() } }); + }, + State.ExprListCommaOrEnd => |list_state| { try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .ExprListItemOrEnd = list_state }); continue; }, State.FieldInitListCommaOrEnd => |list_state| { - try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .FieldInitListItemOrEnd = list_state }); + try self.commaOrEnd(&stack, Token.Id.RBrace, list_state.ptr, State { .FieldInitListItemOrEnd = list_state }); continue; }, @@ -1513,6 +1618,16 @@ pub const Parser = struct { continue; }, + State.SwitchCaseCommaOrEnd => |list_state| { + try self.commaOrEnd(&stack, Token.Id.RBrace, list_state.ptr, State { .SwitchCaseOrEnd = list_state }); + continue; + }, + + State.SwitchCaseItemCommaOrEnd => |case_items| { + try self.commaOrEnd(&stack, Token.Id.EqualAngleBracketRight, null, State { .SwitchCaseItem = case_items }); + continue; + }, + State.AddrOfModifiers => |addr_of_info| { var token = self.getNextToken(); switch (token.id) { @@ -1741,6 +1856,11 @@ pub const Parser = struct { stack.append(State { .Block = inner_block }) catch unreachable; continue; }, + Token.Id.Keyword_switch => { + self.putBackToken(next); + stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }) catch unreachable; + continue; + }, else => { self.putBackToken(next); stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; @@ -1754,7 +1874,7 @@ pub const Parser = struct { } } - fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void { + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, maybe_ptr: ?&Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { Token.Id.Comma => { @@ -1763,7 +1883,9 @@ pub const Parser = struct { else => { const IdTag = @TagType(Token.Id); if (IdTag(*end) == token.id) { - *ptr = token; + if (maybe_ptr) |ptr| { + *ptr = token; + } return; } @@ -2498,7 +2620,8 @@ pub const Parser = struct { ast.NodeInfixOp.InfixOp.Sub => " - ", ast.NodeInfixOp.InfixOp.SubWrap => " -% ", ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ", - else => unreachable, + ast.NodeInfixOp.InfixOp.Range => " ... ", + ast.NodeInfixOp.InfixOp.Catch => unreachable, }; try stack.append(RenderState { .Text = text }); @@ -2821,8 +2944,73 @@ pub const Parser = struct { }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), - ast.Node.Id.Switch => @panic("TODO switch"), - ast.Node.Id.SwitchCase => @panic("TODO switch case"), + ast.Node.Id.Switch => { + const switch_node = @fieldParentPtr(ast.NodeSwitch, "base", base); + try stream.print("{} (", self.tokenizer.getTokenSlice(switch_node.switch_token)); + + try stack.append(RenderState { .Text = "}"}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Text = "\n"}); + + const cases = switch_node.cases.toSliceConst(); + var i = cases.len; + while (i != 0) { + i -= 1; + const node = cases[i]; + try stack.append(RenderState { .Expression = &node.base}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + if (i != 0) { + const prev_node = cases[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { + break :blk "\n\n"; + } + } + break :blk "\n"; + }, + }); + + if (i != 0) { + try stack.append(RenderState { .Text = "," }); + } + } + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = ") {"}); + try stack.append(RenderState { .Expression = switch_node.expr }); + }, + ast.Node.Id.SwitchCase => { + const switch_case = @fieldParentPtr(ast.NodeSwitchCase, "base", base); + + try stack.append(RenderState { .Expression = switch_case.expr }); + if (switch_case.capture) |capture| { + try stack.append(RenderState { .Text = "| "}); + try stack.append(RenderState { .Expression = &capture.symbol.base }); + + if (capture.is_ptr) { + try stack.append(RenderState { .Text = "*"}); + } + try stack.append(RenderState { .Text = "|"}); + } + try stack.append(RenderState { .Text = " => "}); + + const items = switch_case.items.toSliceConst(); + var i = items.len; + while (i != 0) { + i -= 1; + try stack.append(RenderState { .Expression = items[i] }); + + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + }, + ast.Node.Id.SwitchElse => { + const switch_else = @fieldParentPtr(ast.NodeSwitchElse, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(switch_else.token)); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -2867,7 +3055,7 @@ pub const Parser = struct { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base); try stack.append(RenderState { .VarDecl = var_decl}); }, - ast.Node.Id.Block => { + ast.Node.Id.Block, ast.Node.Id.Switch => { try stack.append(RenderState { .Expression = base}); }, else => { @@ -3436,24 +3624,24 @@ test "zig fmt: switch" { \\ else => { \\ const a = 1; \\ const b = a; - \\ }, + \\ } \\ } \\ \\ const res = switch (0) { \\ 0 => 0, \\ 1 => 2, - \\ else => 4, + \\ else => 4 \\ }; \\ \\ const Union = union(enum) { \\ Int: i64, - \\ Float: f64, + \\ Float: f64 \\ }; \\ - \\ const u = Union { .Int = 0 }; + \\ const u = Union{ .Int = 0 }; \\ switch (u) { \\ Union.Int => |int| {}, - \\ Union.Float => |*float| unreachable, + \\ Union.Float => |*float| unreachable \\ } \\} \\ diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 87597adff1..422aa20629 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -86,6 +86,7 @@ pub const Token = struct { PipeEqual, Equal, EqualEqual, + EqualAngleBracketRight, BangEqual, LParen, RParen, @@ -688,6 +689,11 @@ pub const Tokenizer = struct { self.index += 1; break; }, + '>' => { + result.id = Token.Id.EqualAngleBracketRight; + self.index += 1; + break; + }, else => { result.id = Token.Id.Equal; break; From e4d0b46c0c07f3151b4959c5e2e0d4c304981ead Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sun, 8 Apr 2018 17:05:08 +0200 Subject: [PATCH 30/54] std.zig.parser WIP generalizing parsing of payloads * Note, it doesn't work :) --- std/zig/ast.zig | 108 ++++++++++++++++++++++++++++++++++++++++----- std/zig/parser.zig | 104 ++++++++++++++++++++++++++----------------- 2 files changed, 161 insertions(+), 51 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index ca384efedd..0d060eb685 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,9 +20,11 @@ pub const Node = struct { FnProto, ParamDecl, Block, + Payload, Switch, SwitchCase, SwitchElse, + While, InfixOp, PrefixOp, SuffixOp, @@ -58,9 +60,11 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), + Id.While => @fieldParentPtr(NodeWhile, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -97,9 +101,11 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), + Id.While => @fieldParentPtr(NodeWhile, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -136,9 +142,11 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), - Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), + Id.While => @fieldParentPtr(NodeWhile, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -545,6 +553,31 @@ pub const NodeBlock = struct { } }; +pub const NodePayload = struct { + base: Node, + lpipe: Token, + is_ptr: bool, + symbol: &NodeIdentifier, + rpipe: Token, + + pub fn iterate(self: &NodePayload, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.symbol.base; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodePayload) Token { + return self.lpipe; + } + + pub fn lastToken(self: &NodePayload) Token { + return self.rpipe; + } +}; + pub const NodeSwitch = struct { base: Node, switch_token: Token, @@ -576,22 +609,17 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - capture: ?Capture, + payload: ?&Node, expr: &Node, - const Capture = struct { - symbol: &NodeIdentifier, - is_ptr: bool, - }; - pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { var i = index; if (i < self.items.len) return self.items.at(i); i -= self.items.len; - if (self.capture) |capture| { - if (i < 1) return &capture.base; + if (self.payload) |payload| { + if (i < 1) return payload; i -= 1; } @@ -627,6 +655,66 @@ pub const NodeSwitchElse = struct { } }; +pub const NodeWhile = struct { + base: Node, + while_token: Token, + condition: &Node, + payload: ?&NodePayload, + continue_expr: ?&Node, + body: &Node, + @"else": ?Else, + + const Else = struct { + capture: ?&NodeIdentifier, + body: &Node, + }; + + + pub fn iterate(self: &NodeWhile, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.condition; + i -= 1; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (self.continue_expr) |continue_expr| { + if (i < 1) return continue_expr; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + if (self.@"else") |@"else"| { + if (@"else".capture) |capture| { + if (i < 1) return &capture.base; + i -= 1; + } + + if (i < 1) return @"else".body; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeWhile) Token { + return self.while_token; + } + + pub fn lastToken(self: &NodeWhile) Token { + if (self.@"else") |@"else"| { + return @"else".body.lastToken(); + } + + return self.body.lastToken(); + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, @@ -661,7 +749,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&NodeIdentifier, + Catch: ?&Node, Div, EqualEqual, ErrorUnion, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 09f7827505..0d8b8a3248 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -104,6 +104,11 @@ pub const Parser = struct { ptr: &Token, }; + const ElseCtx = struct { + payload: ?DestPtr, + body: DestPtr, + }; + fn ListSave(comptime T: type) type { return struct { list: &ArrayList(T), @@ -140,7 +145,7 @@ pub const Parser = struct { FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - SwitchCaseCapture: &?ast.NodeSwitchCase.Capture, + Payload: DestPtr, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -635,9 +640,6 @@ pub const Parser = struct { Token.Id.Keyword_await => { @panic("TODO: await"); }, - Token.Id.Keyword_suspend => { - @panic("TODO: suspend"); - }, else => { self.putBackToken(token); stack.append(State { .AssignmentExpressionBegin = dest_ptr }) catch unreachable; @@ -705,21 +707,14 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - - const next = self.getNextToken(); - if (next.id != Token.Id.Pipe) { - self.putBackToken(next); - continue; - } - - node.op.Catch = try self.createIdentifier(arena, Token(undefined)); - try stack.append(State { .ExpectToken = Token.Id.Pipe }); try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.Identifier, - .ptr = &(??node.op.Catch).name_token + .Optional = RevertState { + .tokenizer = *self.tokenizer, + .parser = *self, + .ptr = &node.op.Catch, } }); + try stack.append(State { .Payload = DestPtr { .NullableField = &node.op.Catch } }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1404,12 +1399,25 @@ pub const Parser = struct { @panic("TODO: inline if"); }, Token.Id.Keyword_while => { + const node = try arena.create(ast.NodeWhile); + *node = ast.NodeWhile { + .base = self.initNode(ast.Node.Id.While), + .while_token = token, + .condition = undefined, + .payload = null, + .continue_expr = null, + .body = undefined, + .@"else" = null, + }; + dest_ptr.store(&node.base); + @panic("TODO: inline while"); }, Token.Id.Keyword_for => { @panic("TODO: inline for"); }, Token.Id.Keyword_switch => { + @breakpoint(); const node = try arena.create(ast.NodeSwitch); *node = ast.NodeSwitch { .base = self.initNode(ast.Node.Id.Switch), @@ -1434,9 +1442,6 @@ pub const Parser = struct { Token.Id.Keyword_comptime => { @panic("TODO: inline comptime"); }, - Token.Id.Keyword_suspend => { - @panic("TODO: inline suspend"); - }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; @@ -1547,13 +1552,21 @@ pub const Parser = struct { *node = ast.NodeSwitchCase { .base = self.initNode(ast.Node.Id.SwitchCase), .items = ArrayList(&ast.Node).init(arena), - .capture = null, + .payload = null, .expr = undefined, }; try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { .SwitchCaseCapture = &node.capture }); + try stack.append(State { + .Optional = RevertState { + .tokenizer = *self.tokenizer, + .parser = *self, + .ptr = &node.payload, + } + }); + try stack.append(State { .Payload = DestPtr { .NullableField = &node.payload } }); + try stack.append(State.Required); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1572,12 +1585,8 @@ pub const Parser = struct { } }, - State.SwitchCaseCapture => |capture| { - const token = self.getNextToken(); - if (token.id != Token.Id.Pipe) { - self.putBackToken(token); - continue; - } + State.Payload => |dest_ptr| { + const lpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; const is_ptr = blk: { const asterik = self.getNextToken(); @@ -1590,11 +1599,16 @@ pub const Parser = struct { }; const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - _ = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - *capture = ast.NodeSwitchCase.Capture { + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodePayload); + *node = ast.NodePayload { + .base = self.initNode(ast.Node.Id.Payload), + .lpipe = lpipe, + .is_ptr = is_ptr, .symbol = try self.createIdentifier(arena, ident), - .is_ptr = is_ptr + .rpipe = rpipe }; + dest_ptr.store(&node.base); }, State.SwitchCaseItem => |case_items| { @@ -1856,6 +1870,8 @@ pub const Parser = struct { stack.append(State { .Block = inner_block }) catch unreachable; continue; }, + Token.Id.Keyword_suspend, Token.Id.Keyword_if, + Token.Id.Keyword_while, Token.Id.Keyword_for, Token.Id.Keyword_switch => { self.putBackToken(next); stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }) catch unreachable; @@ -2572,9 +2588,8 @@ pub const Parser = struct { if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { if (prefix_op_node.op.Catch) |payload| { - try stack.append(RenderState { .Text = "| " }); - try stack.append(RenderState { .Expression = &payload.base }); - try stack.append(RenderState { .Text = "|" }); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = payload }); } try stack.append(RenderState { .Text = " catch " }); } else { @@ -2764,6 +2779,17 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = rhs }); } }, + ast.Node.Id.Payload => { + const payload = @fieldParentPtr(ast.NodePayload, "base", base); + try stack.append(RenderState { .Text = "|"}); + try stack.append(RenderState { .Expression = &payload.symbol.base }); + + if (payload.is_ptr) { + try stack.append(RenderState { .Text = "*"}); + } + + try stack.append(RenderState { .Text = "|"}); + }, ast.Node.Id.GroupedExpression => { const grouped_expr = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); try stack.append(RenderState { .Text = ")"}); @@ -2985,14 +3011,9 @@ pub const Parser = struct { const switch_case = @fieldParentPtr(ast.NodeSwitchCase, "base", base); try stack.append(RenderState { .Expression = switch_case.expr }); - if (switch_case.capture) |capture| { - try stack.append(RenderState { .Text = "| "}); - try stack.append(RenderState { .Expression = &capture.symbol.base }); - - if (capture.is_ptr) { - try stack.append(RenderState { .Text = "*"}); - } - try stack.append(RenderState { .Text = "|"}); + if (switch_case.payload) |payload| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = payload }); } try stack.append(RenderState { .Text = " => "}); @@ -3011,6 +3032,7 @@ pub const Parser = struct { const switch_else = @fieldParentPtr(ast.NodeSwitchElse, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(switch_else.token)); }, + ast.Node.Id.While => @panic("TODO: Render while"), ast.Node.Id.StructField, ast.Node.Id.UnionTag, From e260c8ca632fb2569f99d182dd6d1daea2b6df63 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 11:11:18 +0200 Subject: [PATCH 31/54] std.zig.parser now parses while loops and labeled break and continue --- std/zig/ast.zig | 62 ++++-- std/zig/parser.zig | 541 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 468 insertions(+), 135 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 0d060eb685..22b14fe363 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -21,6 +21,7 @@ pub const Node = struct { ParamDecl, Block, Payload, + Else, Switch, SwitchCase, SwitchElse, @@ -61,6 +62,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), + Id.Else => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), @@ -102,6 +104,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), + Id.Else => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), @@ -143,6 +146,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), + Id.Else => @fieldParentPtr(NodeElse, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), @@ -578,6 +582,35 @@ pub const NodePayload = struct { } }; +pub const NodeElse = struct { + base: Node, + else_token: Token, + payload: ?&NodePayload, + body: &Node, + + pub fn iterate(self: &NodeElse, index: usize) ?&Node { + var i = index; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeElse) Token { + return self.else_token; + } + + pub fn lastToken(self: &NodeElse) Token { + return self.body.lastToken(); + } +}; + pub const NodeSwitch = struct { base: Node, switch_token: Token, @@ -609,7 +642,7 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - payload: ?&Node, + payload: ?&NodePayload, expr: &Node, pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { @@ -657,18 +690,14 @@ pub const NodeSwitchElse = struct { pub const NodeWhile = struct { base: Node, + label: ?Token, + inline_token: ?Token, while_token: Token, condition: &Node, payload: ?&NodePayload, continue_expr: ?&Node, body: &Node, - @"else": ?Else, - - const Else = struct { - capture: ?&NodeIdentifier, - body: &Node, - }; - + @"else": ?&NodeElse, pub fn iterate(self: &NodeWhile, index: usize) ?&Node { var i = index; @@ -690,12 +719,7 @@ pub const NodeWhile = struct { i -= 1; if (self.@"else") |@"else"| { - if (@"else".capture) |capture| { - if (i < 1) return &capture.base; - i -= 1; - } - - if (i < 1) return @"else".body; + if (i < 1) return &@"else".base; i -= 1; } @@ -703,6 +727,14 @@ pub const NodeWhile = struct { } pub fn firstToken(self: &NodeWhile) Token { + if (self.label) |label| { + return label; + } + + if (self.inline_token) |inline_token| { + return inline_token; + } + return self.while_token; } @@ -749,7 +781,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&Node, + Catch: ?&NodePayload, Div, EqualEqual, ErrorUnion, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0d8b8a3248..7c8fbb4e52 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -116,6 +116,24 @@ pub const Parser = struct { }; } + const LabelCtx = struct { + label: ?Token, + dest_ptr: DestPtr, + }; + + const InlineCtx = struct { + label: ?Token, + inline_token: ?Token, + dest_ptr: DestPtr, + }; + + const LoopCtx = struct { + label: ?Token, + inline_token: ?Token, + loop_token: Token, + dest_ptr: DestPtr, + }; + const State = union(enum) { TopLevel, TopLevelExtern: TopLevelDeclCtx, @@ -137,15 +155,22 @@ pub const Parser = struct { ParamDecl: &ast.NodeFnProto, ParamDeclComma, FnDef: &ast.NodeFnProto, + LabeledExpression: LabelCtx, + Inline: InlineCtx, + While: LoopCtx, + For: LoopCtx, Block: &ast.NodeBlock, + Else: &?&ast.NodeElse, + WhileContinueExpr: &?&ast.Node, Statement: &ast.NodeBlock, + Semicolon: &const &const ast.Node, ExprListItemOrEnd: ExprListCtx, ExprListCommaOrEnd: ExprListCtx, FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer), FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - Payload: DestPtr, + Payload: &?&ast.NodePayload, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -157,9 +182,6 @@ pub const Parser = struct { /// "leaked" nodes. TODO: Figure out if it's nessesary to handle leaked nodes. Optional: RevertState, - /// Optional can be reverted by adding the Required state to the stack. - Required, - Expression: DestPtr, RangeExpressionBegin: DestPtr, RangeExpressionEnd: DestPtr, @@ -598,8 +620,7 @@ pub const Parser = struct { continue; }, - State.Optional, - State.Required => { }, + State.Optional => { }, State.Expression => |dest_ptr| { const token = self.getNextToken(); @@ -626,10 +647,51 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_break => { - @panic("TODO: break"); + const label = blk: { + const colon = self.getNextToken(); + if (colon.id != Token.Id.Colon) { + self.putBackToken(colon); + break :blk null; + } + + break :blk (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + }; + + const node = try self.createControlFlowExpr(arena, token, + ast.NodeControlFlowExpression.Kind { + .Break = label, + } + ); + dest_ptr.store(&node.base); + + stack.append(State { + .Optional = RevertState { + .parser = *self, + .tokenizer = *self.tokenizer, + .ptr = &node.rhs, + } + }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); + continue; }, Token.Id.Keyword_continue => { - @panic("TODO: break"); + const label = blk: { + const colon = self.getNextToken(); + if (colon.id != Token.Id.Colon) { + self.putBackToken(colon); + break :blk null; + } + + break :blk (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + }; + + const node = try self.createControlFlowExpr(arena, token, + ast.NodeControlFlowExpression.Kind { + .Continue = label, + } + ); + dest_ptr.store(&node.base); + continue; }, Token.Id.Keyword_cancel => { @panic("TODO: cancel"); @@ -707,14 +769,7 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - try stack.append(State { - .Optional = RevertState { - .tokenizer = *self.tokenizer, - .parser = *self, - .ptr = &node.op.Catch, - } - }); - try stack.append(State { .Payload = DestPtr { .NullableField = &node.op.Catch } }); + try stack.append(State { .Payload = &node.op.Catch }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1370,16 +1425,12 @@ pub const Parser = struct { continue; } - const block = try self.createBlock(arena, (?Token)(token), Token(undefined)); - dest_ptr.store(&block.base); - - stack.append(State { .Block = block }) catch unreachable; - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.LBrace, - .ptr = &block.lbrace, + stack.append(State { + .LabeledExpression = LabelCtx { + .label = token, + .dest_ptr = dest_ptr } - }); + }) catch unreachable; continue; }, Token.Id.LBrace => { @@ -1398,26 +1449,31 @@ pub const Parser = struct { Token.Id.Keyword_if => { @panic("TODO: inline if"); }, + Token.Id.Keyword_inline => { + stack.append(State { + .Inline = InlineCtx { + .label = null, + .inline_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; + }, Token.Id.Keyword_while => { - const node = try arena.create(ast.NodeWhile); - *node = ast.NodeWhile { - .base = self.initNode(ast.Node.Id.While), - .while_token = token, - .condition = undefined, - .payload = null, - .continue_expr = null, - .body = undefined, - .@"else" = null, - }; - dest_ptr.store(&node.base); - - @panic("TODO: inline while"); + stack.append(State { + .While = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; }, Token.Id.Keyword_for => { @panic("TODO: inline for"); }, Token.Id.Keyword_switch => { - @breakpoint(); const node = try arena.create(ast.NodeSwitch); *node = ast.NodeSwitch { .base = self.initNode(ast.Node.Id.Switch), @@ -1558,15 +1614,7 @@ pub const Parser = struct { try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { - .Optional = RevertState { - .tokenizer = *self.tokenizer, - .parser = *self, - .ptr = &node.payload, - } - }); - try stack.append(State { .Payload = DestPtr { .NullableField = &node.payload } }); - try stack.append(State.Required); + try stack.append(State { .Payload = &node.payload }); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1585,32 +1633,6 @@ pub const Parser = struct { } }, - State.Payload => |dest_ptr| { - const lpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - - const is_ptr = blk: { - const asterik = self.getNextToken(); - if (asterik.id == Token.Id.Asterisk) { - break :blk true; - } else { - self.putBackToken(asterik); - break :blk false; - } - }; - - const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodePayload); - *node = ast.NodePayload { - .base = self.initNode(ast.Node.Id.Payload), - .lpipe = lpipe, - .is_ptr = is_ptr, - .symbol = try self.createIdentifier(arena, ident), - .rpipe = rpipe - }; - dest_ptr.store(&node.base); - }, - State.SwitchCaseItem => |case_items| { stack.append(State { .SwitchCaseItemCommaOrEnd = case_items }) catch unreachable; try stack.append(State { .RangeExpressionBegin = DestPtr{ .Field = try case_items.addOne() } }); @@ -1642,6 +1664,68 @@ pub const Parser = struct { continue; }, + State.Else => |dest| { + const else_token = self.getNextToken(); + if (else_token.id != Token.Id.Keyword_else) { + self.putBackToken(else_token); + continue; + } + + const node = try arena.create(ast.NodeElse); + *node = ast.NodeElse { + .base = self.initNode(ast.Node.Id.Else), + .else_token = else_token, + .payload = null, + .body = undefined, + }; + *dest = node; + + stack.append(State { .Expression = DestPtr { .Field = &node.body } }) catch unreachable; + try stack.append(State { .Payload = &node.payload }); + }, + + State.WhileContinueExpr => |dest| { + const colon = self.getNextToken(); + if (colon.id != Token.Id.Colon) { + self.putBackToken(colon); + continue; + } + + _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + stack.append(State { .ExpectToken = Token.Id.RParen }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .NullableField = dest } }); + }, + + State.Payload => |dest| { + const lpipe = self.getNextToken(); + if (lpipe.id != Token.Id.Pipe) { + self.putBackToken(lpipe); + continue; + } + + const is_ptr = blk: { + const asterik = self.getNextToken(); + if (asterik.id == Token.Id.Asterisk) { + break :blk true; + } else { + self.putBackToken(asterik); + break :blk false; + } + }; + + const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodePayload); + *node = ast.NodePayload { + .base = self.initNode(ast.Node.Id.Payload), + .lpipe = lpipe, + .is_ptr = is_ptr, + .symbol = try self.createIdentifier(arena, ident), + .rpipe = rpipe + }; + *dest = node; + }, + State.AddrOfModifiers => |addr_of_info| { var token = self.getNextToken(); switch (token.id) { @@ -1803,6 +1887,114 @@ pub const Parser = struct { } }, + State.LabeledExpression => |ctx| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.LBrace => { + const block = try self.createBlock(arena, (?Token)(ctx.label), token); + ctx.dest_ptr.store(&block.base); + + stack.append(State { .Block = block }) catch unreachable; + continue; + }, + Token.Id.Keyword_while => { + stack.append(State { + .While = LoopCtx { + .label = ctx.label, + .inline_token = null, + .loop_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_for => { + stack.append(State { + .For = LoopCtx { + .label = ctx.label, + .inline_token = null, + .loop_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_inline => { + stack.append(State { + .Inline = InlineCtx { + .label = ctx.label, + .inline_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + else => { + try self.parseError(&stack, token, "expected 'while', 'for', 'inline' or '{{', found {}", @tagName(token.id)); + continue; + }, + } + }, + + State.Inline => |ctx| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_while => { + stack.append(State { + .While = LoopCtx { + .inline_token = ctx.inline_token, + .label = ctx.label, + .loop_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_for => { + stack.append(State { + .For = LoopCtx { + .inline_token = ctx.inline_token, + .label = ctx.label, + .loop_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + else => { + try self.parseError(&stack, token, "expected 'while' or 'for', found {}", @tagName(token.id)); + continue; + }, + } + }, + + State.While => |ctx| { + const node = try arena.create(ast.NodeWhile); + *node = ast.NodeWhile { + .base = self.initNode(ast.Node.Id.While), + .label = ctx.label, + .inline_token = ctx.inline_token, + .while_token = ctx.loop_token, + .condition = undefined, + .payload = null, + .continue_expr = null, + .body = undefined, + .@"else" = null, + }; + ctx.dest_ptr.store(&node.base); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .WhileContinueExpr = &node.continue_expr }); + try stack.append(State { .Payload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + }, + + State.For => |ctx| { + }, + State.Block => |block| { const token = self.getNextToken(); switch (token.id) { @@ -1841,28 +2033,6 @@ pub const Parser = struct { stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; }, - Token.Id.Identifier => { - const maybe_colon = self.getNextToken(); - if (maybe_colon.id != Token.Id.Colon) { - self.putBackToken(maybe_colon); - self.putBackToken(next); - stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); - continue; - } - - const inner_block = try self.createBlock(arena, (?Token)(next), Token(undefined)); - try block.statements.append(&inner_block.base); - - stack.append(State { .Block = inner_block }) catch unreachable; - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.LBrace, - .ptr = &inner_block.lbrace, - } - }); - continue; - }, Token.Id.LBrace => { const inner_block = try self.createBlock(arena, (?Token)(null), next); try block.statements.append(&inner_block.base); @@ -1870,22 +2040,53 @@ pub const Parser = struct { stack.append(State { .Block = inner_block }) catch unreachable; continue; }, - Token.Id.Keyword_suspend, Token.Id.Keyword_if, - Token.Id.Keyword_while, Token.Id.Keyword_for, - Token.Id.Keyword_switch => { - self.putBackToken(next); - stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }) catch unreachable; - continue; - }, else => { self.putBackToken(next); - stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); + const statememt = try block.statements.addOne(); + stack.append(State { .Semicolon = statememt }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = statememt } }); continue; } } }, + + State.Semicolon => |node_ptr| { + const node = *node_ptr; + switch (node.id) { + ast.Node.Id.Root, + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag, + ast.Node.Id.ParamDecl, + ast.Node.Id.Block, + ast.Node.Id.Payload, + ast.Node.Id.Switch, + ast.Node.Id.SwitchCase, + ast.Node.Id.SwitchElse, + ast.Node.Id.FieldInitializer, + ast.Node.Id.LineComment, + ast.Node.Id.TestDecl => continue, + ast.Node.Id.While => { + const while_node = @fieldParentPtr(ast.NodeWhile, "base", node); + if (while_node.@"else") |@"else"| { + stack.append(State { .Semicolon = &@"else".base }) catch unreachable; + continue; + } + + stack.append(State { .Semicolon = &while_node.body }) catch unreachable; + continue; + }, + ast.Node.Id.Else => { + const else_node = @fieldParentPtr(ast.NodeElse, "base", node); + stack.append(State { .Semicolon = &else_node.body }) catch unreachable; + continue; + }, + else => { + _ = (try self.eatToken(&stack, Token.Id.Semicolon)) ?? continue; + } + } + } } } } @@ -2295,9 +2496,6 @@ pub const Parser = struct { *revert.ptr = null; return; }, - State.Required => { - return error.StateRequired; - }, else => { } } } @@ -2361,6 +2559,7 @@ pub const Parser = struct { Expression: &ast.Node, VarDecl: &ast.NodeVarDecl, Statement: &ast.Node, + Semicolon: &ast.Node, FieldInitializer: &ast.NodeFieldInitializer, PrintIndent, Indent: usize, @@ -2589,7 +2788,7 @@ pub const Parser = struct { if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { if (prefix_op_node.op.Catch) |payload| { try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Expression = payload }); + try stack.append(RenderState { .Expression = &payload.base }); } try stack.append(RenderState { .Text = " catch " }); } else { @@ -3013,7 +3212,7 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = switch_case.expr }); if (switch_case.payload) |payload| { try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Expression = payload }); + try stack.append(RenderState { .Expression = &payload.base }); } try stack.append(RenderState { .Text = " => "}); @@ -3032,7 +3231,74 @@ pub const Parser = struct { const switch_else = @fieldParentPtr(ast.NodeSwitchElse, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(switch_else.token)); }, - ast.Node.Id.While => @panic("TODO: Render while"), + ast.Node.Id.Else => { + const else_node = @fieldParentPtr(ast.NodeElse, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(else_node.else_token)); + + if (else_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Expression = else_node.body }); + } else { + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Expression = else_node.body }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + } + + if (else_node.payload) |payload| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = &payload.base }); + } + }, + ast.Node.Id.While => { + const while_node = @fieldParentPtr(ast.NodeWhile, "base", base); + if (while_node.label) |label| { + try stream.print("{}: ", self.tokenizer.getTokenSlice(label)); + } + + if (while_node.inline_token) |inline_token| { + try stream.print("{} ", self.tokenizer.getTokenSlice(inline_token)); + } + + try stream.print("{} ", self.tokenizer.getTokenSlice(while_node.while_token)); + + if (while_node.@"else") |@"else"| { + try stack.append(RenderState { .Expression = &@"else".base }); + + if (while_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState { .Text = "\n" }); + } + } + + if (while_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Expression = while_node.body }); + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Expression = while_node.body }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + } + + if (while_node.continue_expr) |continue_expr| { + try stack.append(RenderState { .Text = ")" }); + try stack.append(RenderState { .Expression = continue_expr }); + try stack.append(RenderState { .Text = ": (" }); + try stack.append(RenderState { .Text = " " }); + } + + if (while_node.payload) |payload| { + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = " " }); + } + + try stack.append(RenderState { .Text = ")" }); + try stack.append(RenderState { .Expression = while_node.condition }); + try stack.append(RenderState { .Text = "(" }); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -3077,13 +3343,45 @@ pub const Parser = struct { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base); try stack.append(RenderState { .VarDecl = var_decl}); }, - ast.Node.Id.Block, ast.Node.Id.Switch => { - try stack.append(RenderState { .Expression = base}); + else => { + try stack.append(RenderState { .Semicolon = base }); + try stack.append(RenderState { .Expression = base }); + }, + } + }, + RenderState.Semicolon => |base| { + switch (base.id) { + ast.Node.Id.Root, + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag, + ast.Node.Id.ParamDecl, + ast.Node.Id.Block, + ast.Node.Id.Payload, + ast.Node.Id.Switch, + ast.Node.Id.SwitchCase, + ast.Node.Id.SwitchElse, + ast.Node.Id.FieldInitializer, + ast.Node.Id.LineComment, + ast.Node.Id.TestDecl => {}, + ast.Node.Id.While => { + const while_node = @fieldParentPtr(ast.NodeWhile, "base", base); + if (while_node.@"else") |@"else"| { + stack.append(RenderState { .Semicolon = &@"else".base }) catch unreachable; + continue; + } + + stack.append(RenderState { .Semicolon = while_node.body }) catch unreachable; + continue; + }, + ast.Node.Id.Else => { + const else_node = @fieldParentPtr(ast.NodeElse, "base", base); + stack.append(RenderState { .Semicolon = else_node.body }) catch unreachable; + continue; }, else => { - try stack.append(RenderState { .Text = ";"}); - try stack.append(RenderState { .Expression = base}); - }, + try stack.append(RenderState { .Text = ";" }); + } } }, RenderState.Indent => |new_indent| indent = new_indent, @@ -3690,8 +3988,11 @@ test "zig fmt: while" { \\ continue; \\ \\ i = 0; - \\ var j usize = 0; - \\ while (i < 10) : ({ i += 1; j += 1; }) { + \\ var j: usize = 0; + \\ while (i < 10) : ({ + \\ i += 1; + \\ j += 1; + \\ }) { \\ continue; \\ } \\ @@ -3711,7 +4012,7 @@ test "zig fmt: while" { \\ break 7; \\ } else { \\ unreachable; - \\ } + \\ }; \\ \\ var a: error!u8 = 0; \\ while (a) |v| { @@ -3721,7 +4022,7 @@ test "zig fmt: while" { \\ } \\ \\ comptime var k: usize = 0; - \\ inline while (i < 10) (i += 1) + \\ inline while (i < 10) : (i += 1) \\ j += 2; \\} \\ From e24409ebe0f50be9e01810a5f61bb4c09db57d28 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 11:17:57 +0200 Subject: [PATCH 32/54] std.zig.parser unified code for rendering and parsing semicolon in statements --- std/zig/parser.zig | 109 +++++++++++++++++---------------------------- 1 file changed, 40 insertions(+), 69 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 7c8fbb4e52..0fd757a01f 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2053,44 +2053,49 @@ pub const Parser = struct { State.Semicolon => |node_ptr| { const node = *node_ptr; - switch (node.id) { - ast.Node.Id.Root, - ast.Node.Id.StructField, - ast.Node.Id.UnionTag, - ast.Node.Id.EnumTag, - ast.Node.Id.ParamDecl, - ast.Node.Id.Block, - ast.Node.Id.Payload, - ast.Node.Id.Switch, - ast.Node.Id.SwitchCase, - ast.Node.Id.SwitchElse, - ast.Node.Id.FieldInitializer, - ast.Node.Id.LineComment, - ast.Node.Id.TestDecl => continue, - ast.Node.Id.While => { - const while_node = @fieldParentPtr(ast.NodeWhile, "base", node); - if (while_node.@"else") |@"else"| { - stack.append(State { .Semicolon = &@"else".base }) catch unreachable; - continue; - } - - stack.append(State { .Semicolon = &while_node.body }) catch unreachable; - continue; - }, - ast.Node.Id.Else => { - const else_node = @fieldParentPtr(ast.NodeElse, "base", node); - stack.append(State { .Semicolon = &else_node.body }) catch unreachable; - continue; - }, - else => { - _ = (try self.eatToken(&stack, Token.Id.Semicolon)) ?? continue; - } + if (requireSemiColon(node)) { + _ = (try self.eatToken(&stack, Token.Id.Semicolon)) ?? continue; } } } } } + fn requireSemiColon(node: &const ast.Node) bool { + var n = node; + while (true) { + switch (n.id) { + ast.Node.Id.Root, + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag, + ast.Node.Id.ParamDecl, + ast.Node.Id.Block, + ast.Node.Id.Payload, + ast.Node.Id.Switch, + ast.Node.Id.SwitchCase, + ast.Node.Id.SwitchElse, + ast.Node.Id.FieldInitializer, + ast.Node.Id.LineComment, + ast.Node.Id.TestDecl => return false, + ast.Node.Id.While => { + const while_node = @fieldParentPtr(ast.NodeWhile, "base", n); + if (while_node.@"else") |@"else"| { + n = @"else".base; + continue; + } + + n = while_node.body; + }, + ast.Node.Id.Else => { + const else_node = @fieldParentPtr(ast.NodeElse, "base", n); + n = else_node.body; + }, + else => return true, + } + } + } + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, maybe_ptr: ?&Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { @@ -2559,7 +2564,6 @@ pub const Parser = struct { Expression: &ast.Node, VarDecl: &ast.NodeVarDecl, Statement: &ast.Node, - Semicolon: &ast.Node, FieldInitializer: &ast.NodeFieldInitializer, PrintIndent, Indent: usize, @@ -3344,46 +3348,13 @@ pub const Parser = struct { try stack.append(RenderState { .VarDecl = var_decl}); }, else => { - try stack.append(RenderState { .Semicolon = base }); + if (requireSemiColon(base)) { + try stack.append(RenderState { .Text = ";" }); + } try stack.append(RenderState { .Expression = base }); }, } }, - RenderState.Semicolon => |base| { - switch (base.id) { - ast.Node.Id.Root, - ast.Node.Id.StructField, - ast.Node.Id.UnionTag, - ast.Node.Id.EnumTag, - ast.Node.Id.ParamDecl, - ast.Node.Id.Block, - ast.Node.Id.Payload, - ast.Node.Id.Switch, - ast.Node.Id.SwitchCase, - ast.Node.Id.SwitchElse, - ast.Node.Id.FieldInitializer, - ast.Node.Id.LineComment, - ast.Node.Id.TestDecl => {}, - ast.Node.Id.While => { - const while_node = @fieldParentPtr(ast.NodeWhile, "base", base); - if (while_node.@"else") |@"else"| { - stack.append(RenderState { .Semicolon = &@"else".base }) catch unreachable; - continue; - } - - stack.append(RenderState { .Semicolon = while_node.body }) catch unreachable; - continue; - }, - ast.Node.Id.Else => { - const else_node = @fieldParentPtr(ast.NodeElse, "base", base); - stack.append(RenderState { .Semicolon = else_node.body }) catch unreachable; - continue; - }, - else => { - try stack.append(RenderState { .Text = ";" }); - } - } - }, RenderState.Indent => |new_indent| indent = new_indent, RenderState.PrintIndent => try stream.writeByteNTimes(' ', indent), } From 7dd55a8007c540415d3e704e490609dd86bea924 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 11:48:25 +0200 Subject: [PATCH 33/54] std.zig.parser now parses for loops --- std/zig/ast.zig | 150 +++++++++++++++++++++++++++++---- std/zig/parser.zig | 202 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 319 insertions(+), 33 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 22b14fe363..c97e33ba10 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,12 +20,15 @@ pub const Node = struct { FnProto, ParamDecl, Block, - Payload, + ErrorPayload, + ValuePayload, + ValueIndexPayload, Else, Switch, SwitchCase, SwitchElse, While, + For, InfixOp, PrefixOp, SuffixOp, @@ -61,12 +64,15 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), - Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), + Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).iterate(index), + Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).iterate(index), + Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).iterate(index), Id.Else => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), Id.While => @fieldParentPtr(NodeWhile, "base", base).iterate(index), + Id.For => @fieldParentPtr(NodeFor, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -103,12 +109,15 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), - Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), + Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).firstToken(), + Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).firstToken(), + Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).firstToken(), Id.Else => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), Id.While => @fieldParentPtr(NodeWhile, "base", base).firstToken(), + Id.For => @fieldParentPtr(NodeFor, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -145,12 +154,15 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), - Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), + Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).lastToken(), + Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).lastToken(), + Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).lastToken(), Id.Else => @fieldParentPtr(NodeElse, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), Id.While => @fieldParentPtr(NodeWhile, "base", base).lastToken(), + Id.For => @fieldParentPtr(NodeFor, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -557,27 +569,82 @@ pub const NodeBlock = struct { } }; -pub const NodePayload = struct { +pub const NodeErrorPayload = struct { base: Node, lpipe: Token, - is_ptr: bool, - symbol: &NodeIdentifier, + error_symbol: &NodeIdentifier, rpipe: Token, - pub fn iterate(self: &NodePayload, index: usize) ?&Node { + pub fn iterate(self: &NodeErrorPayload, index: usize) ?&Node { var i = index; - if (i < 1) return &self.symbol.base; + if (i < 1) return &self.error_symbol.base; i -= 1; return null; } - pub fn firstToken(self: &NodePayload) Token { + pub fn firstToken(self: &NodeErrorPayload) Token { return self.lpipe; } - pub fn lastToken(self: &NodePayload) Token { + pub fn lastToken(self: &NodeErrorPayload) Token { + return self.rpipe; + } +}; + +pub const NodeValuePayload = struct { + base: Node, + lpipe: Token, + is_ptr: bool, + value_symbol: &NodeIdentifier, + rpipe: Token, + + pub fn iterate(self: &NodeValuePayload, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.value_symbol.base; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeValuePayload) Token { + return self.lpipe; + } + + pub fn lastToken(self: &NodeValuePayload) Token { + return self.rpipe; + } +}; + +pub const NodeValueIndexPayload = struct { + base: Node, + lpipe: Token, + is_ptr: bool, + value_symbol: &NodeIdentifier, + index_symbol: ?&NodeIdentifier, + rpipe: Token, + + pub fn iterate(self: &NodeValueIndexPayload, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.value_symbol.base; + i -= 1; + + if (self.index_symbol) |index_symbol| { + if (i < 1) return &index_symbol.base; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeValueIndexPayload) Token { + return self.lpipe; + } + + pub fn lastToken(self: &NodeValueIndexPayload) Token { return self.rpipe; } }; @@ -585,7 +652,7 @@ pub const NodePayload = struct { pub const NodeElse = struct { base: Node, else_token: Token, - payload: ?&NodePayload, + payload: ?&NodeErrorPayload, body: &Node, pub fn iterate(self: &NodeElse, index: usize) ?&Node { @@ -642,7 +709,7 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - payload: ?&NodePayload, + payload: ?&NodeValuePayload, expr: &Node, pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { @@ -694,7 +761,7 @@ pub const NodeWhile = struct { inline_token: ?Token, while_token: Token, condition: &Node, - payload: ?&NodePayload, + payload: ?&NodeValuePayload, continue_expr: ?&Node, body: &Node, @"else": ?&NodeElse, @@ -747,6 +814,59 @@ pub const NodeWhile = struct { } }; +pub const NodeFor = struct { + base: Node, + label: ?Token, + inline_token: ?Token, + for_token: Token, + array_expr: &Node, + payload: ?&NodeValueIndexPayload, + body: &Node, + @"else": ?&NodeElse, + + pub fn iterate(self: &NodeFor, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.array_expr; + i -= 1; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + if (self.@"else") |@"else"| { + if (i < 1) return &@"else".base; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeFor) Token { + if (self.label) |label| { + return label; + } + + if (self.inline_token) |inline_token| { + return inline_token; + } + + return self.for_token; + } + + pub fn lastToken(self: &NodeFor) Token { + if (self.@"else") |@"else"| { + return @"else".body.lastToken(); + } + + return self.body.lastToken(); + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, @@ -781,7 +901,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&NodePayload, + Catch: ?&NodeErrorPayload, Div, EqualEqual, ErrorUnion, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0fd757a01f..2b2afaeabb 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -170,7 +170,9 @@ pub const Parser = struct { FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - Payload: &?&ast.NodePayload, + ErrorPayload: &?&ast.NodeErrorPayload, + ValuePayload: &?&ast.NodeValuePayload, + ValueIndexPayload: &?&ast.NodeValueIndexPayload, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -769,7 +771,7 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - try stack.append(State { .Payload = &node.op.Catch }); + try stack.append(State { .ErrorPayload = &node.op.Catch }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1471,7 +1473,15 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_for => { - @panic("TODO: inline for"); + stack.append(State { + .For = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; }, Token.Id.Keyword_switch => { const node = try arena.create(ast.NodeSwitch); @@ -1614,7 +1624,7 @@ pub const Parser = struct { try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { .Payload = &node.payload }); + try stack.append(State { .ValuePayload = &node.payload }); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1681,7 +1691,7 @@ pub const Parser = struct { *dest = node; stack.append(State { .Expression = DestPtr { .Field = &node.body } }) catch unreachable; - try stack.append(State { .Payload = &node.payload }); + try stack.append(State { .ErrorPayload = &node.payload }); }, State.WhileContinueExpr => |dest| { @@ -1696,7 +1706,26 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr { .NullableField = dest } }); }, - State.Payload => |dest| { + State.ErrorPayload => |dest| { + const lpipe = self.getNextToken(); + if (lpipe.id != Token.Id.Pipe) { + self.putBackToken(lpipe); + continue; + } + + const error_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodeErrorPayload); + *node = ast.NodeErrorPayload { + .base = self.initNode(ast.Node.Id.ErrorPayload), + .lpipe = lpipe, + .error_symbol = try self.createIdentifier(arena, error_symbol), + .rpipe = rpipe + }; + *dest = node; + }, + + State.ValuePayload => |dest| { const lpipe = self.getNextToken(); if (lpipe.id != Token.Id.Pipe) { self.putBackToken(lpipe); @@ -1713,14 +1742,56 @@ pub const Parser = struct { } }; - const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const value_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodePayload); - *node = ast.NodePayload { - .base = self.initNode(ast.Node.Id.Payload), + const node = try arena.create(ast.NodeValuePayload); + *node = ast.NodeValuePayload { + .base = self.initNode(ast.Node.Id.ValuePayload), .lpipe = lpipe, .is_ptr = is_ptr, - .symbol = try self.createIdentifier(arena, ident), + .value_symbol = try self.createIdentifier(arena, value_symbol), + .rpipe = rpipe + }; + *dest = node; + }, + + State.ValueIndexPayload => |dest| { + const lpipe = self.getNextToken(); + if (lpipe.id != Token.Id.Pipe) { + self.putBackToken(lpipe); + continue; + } + + const is_ptr = blk: { + const asterik = self.getNextToken(); + if (asterik.id == Token.Id.Asterisk) { + break :blk true; + } else { + self.putBackToken(asterik); + break :blk false; + } + }; + + const value_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const index_symbol = blk: { + const comma = self.getNextToken(); + if (comma.id != Token.Id.Comma) { + self.putBackToken(comma); + break :blk null; + } + + const symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + break :blk try self.createIdentifier(arena, symbol); + }; + + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodeValueIndexPayload); + *node = ast.NodeValueIndexPayload { + .base = self.initNode(ast.Node.Id.ValueIndexPayload), + .lpipe = lpipe, + .is_ptr = is_ptr, + .value_symbol = try self.createIdentifier(arena, value_symbol), + .index_symbol = index_symbol, .rpipe = rpipe }; *dest = node; @@ -1986,13 +2057,32 @@ pub const Parser = struct { stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); try stack.append(State { .WhileContinueExpr = &node.continue_expr }); - try stack.append(State { .Payload = &node.payload }); + try stack.append(State { .ValuePayload = &node.payload }); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); try stack.append(State { .ExpectToken = Token.Id.LParen }); }, State.For => |ctx| { + const node = try arena.create(ast.NodeFor); + *node = ast.NodeFor { + .base = self.initNode(ast.Node.Id.For), + .label = ctx.label, + .inline_token = ctx.inline_token, + .for_token = ctx.loop_token, + .array_expr = undefined, + .payload = null, + .body = undefined, + .@"else" = null, + }; + ctx.dest_ptr.store(&node.base); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .ValueIndexPayload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.array_expr } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); }, State.Block => |block| { @@ -2071,7 +2161,9 @@ pub const Parser = struct { ast.Node.Id.EnumTag, ast.Node.Id.ParamDecl, ast.Node.Id.Block, - ast.Node.Id.Payload, + ast.Node.Id.ErrorPayload, + ast.Node.Id.ValuePayload, + ast.Node.Id.ValueIndexPayload, ast.Node.Id.Switch, ast.Node.Id.SwitchCase, ast.Node.Id.SwitchElse, @@ -2087,6 +2179,15 @@ pub const Parser = struct { n = while_node.body; }, + ast.Node.Id.For => { + const for_node = @fieldParentPtr(ast.NodeFor, "base", n); + if (for_node.@"else") |@"else"| { + n = @"else".base; + continue; + } + + n = for_node.body; + }, ast.Node.Id.Else => { const else_node = @fieldParentPtr(ast.NodeElse, "base", n); n = else_node.body; @@ -2982,10 +3083,33 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = rhs }); } }, - ast.Node.Id.Payload => { - const payload = @fieldParentPtr(ast.NodePayload, "base", base); + ast.Node.Id.ErrorPayload => { + const payload = @fieldParentPtr(ast.NodeErrorPayload, "base", base); try stack.append(RenderState { .Text = "|"}); - try stack.append(RenderState { .Expression = &payload.symbol.base }); + try stack.append(RenderState { .Expression = &payload.error_symbol.base }); + try stack.append(RenderState { .Text = "|"}); + }, + ast.Node.Id.ValuePayload => { + const payload = @fieldParentPtr(ast.NodeValuePayload, "base", base); + try stack.append(RenderState { .Text = "|"}); + try stack.append(RenderState { .Expression = &payload.value_symbol.base }); + + if (payload.is_ptr) { + try stack.append(RenderState { .Text = "*"}); + } + + try stack.append(RenderState { .Text = "|"}); + }, + ast.Node.Id.ValueIndexPayload => { + const payload = @fieldParentPtr(ast.NodeValueIndexPayload, "base", base); + try stack.append(RenderState { .Text = "|"}); + + if (payload.index_symbol) |index_symbol| { + try stack.append(RenderState { .Expression = &index_symbol.base }); + try stack.append(RenderState { .Text = ", "}); + } + + try stack.append(RenderState { .Expression = &payload.value_symbol.base }); if (payload.is_ptr) { try stack.append(RenderState { .Text = "*"}); @@ -3303,6 +3427,48 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = while_node.condition }); try stack.append(RenderState { .Text = "(" }); }, + ast.Node.Id.For => { + const for_node = @fieldParentPtr(ast.NodeFor, "base", base); + if (for_node.label) |label| { + try stream.print("{}: ", self.tokenizer.getTokenSlice(label)); + } + + if (for_node.inline_token) |inline_token| { + try stream.print("{} ", self.tokenizer.getTokenSlice(inline_token)); + } + + try stream.print("{} ", self.tokenizer.getTokenSlice(for_node.for_token)); + + if (for_node.@"else") |@"else"| { + try stack.append(RenderState { .Expression = &@"else".base }); + + if (for_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState { .Text = "\n" }); + } + } + + if (for_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Expression = for_node.body }); + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Expression = for_node.body }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + } + + if (for_node.payload) |payload| { + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = " " }); + } + + try stack.append(RenderState { .Text = ")" }); + try stack.append(RenderState { .Expression = for_node.array_expr }); + try stack.append(RenderState { .Text = "(" }); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -4022,10 +4188,10 @@ test "zig fmt: for" { \\ continue; \\ \\ const res = for (a) |v, i| { - \\ breal v; + \\ break v; \\ } else { \\ unreachable; - \\ } + \\ }; \\ \\ var num: usize = 0; \\ inline for (a) |v, i| { From c19f5a2356b441e3dde95e85e029fec6ef4f836d Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 12:51:18 +0200 Subject: [PATCH 34/54] std.zig.parser now parses if statements --- std/zig/ast.zig | 47 +++++++++++++++++++++ std/zig/parser.zig | 101 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 137 insertions(+), 11 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index c97e33ba10..9e077a3567 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -29,6 +29,7 @@ pub const Node = struct { SwitchElse, While, For, + If, InfixOp, PrefixOp, SuffixOp, @@ -73,6 +74,7 @@ pub const Node = struct { Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), Id.While => @fieldParentPtr(NodeWhile, "base", base).iterate(index), Id.For => @fieldParentPtr(NodeFor, "base", base).iterate(index), + Id.If => @fieldParentPtr(NodeIf, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -118,6 +120,7 @@ pub const Node = struct { Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), Id.While => @fieldParentPtr(NodeWhile, "base", base).firstToken(), Id.For => @fieldParentPtr(NodeFor, "base", base).firstToken(), + Id.If => @fieldParentPtr(NodeIf, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -163,6 +166,7 @@ pub const Node = struct { Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), Id.While => @fieldParentPtr(NodeWhile, "base", base).lastToken(), Id.For => @fieldParentPtr(NodeFor, "base", base).lastToken(), + Id.If => @fieldParentPtr(NodeIf, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -867,6 +871,49 @@ pub const NodeFor = struct { } }; +pub const NodeIf = struct { + base: Node, + if_token: Token, + condition: &Node, + payload: ?&NodeValuePayload, + body: &Node, + @"else": ?&NodeElse, + + pub fn iterate(self: &NodeIf, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.condition; + i -= 1; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + if (self.@"else") |@"else"| { + if (i < 1) return &@"else".base; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeIf) Token { + return self.if_token; + } + + pub fn lastToken(self: &NodeIf) Token { + if (self.@"else") |@"else"| { + return @"else".body.lastToken(); + } + + return self.body.lastToken(); + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 2b2afaeabb..09383a313b 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1449,7 +1449,24 @@ pub const Parser = struct { @panic("TODO: inline asm"); }, Token.Id.Keyword_if => { - @panic("TODO: inline if"); + const node = try arena.create(ast.NodeIf); + *node = ast.NodeIf { + .base = self.initNode(ast.Node.Id.If), + .if_token = token, + .condition = undefined, + .payload = null, + .body = undefined, + .@"else" = null, + }; + dest_ptr.store(&node.base); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .ValuePayload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + continue; }, Token.Id.Keyword_inline => { stack.append(State { @@ -2188,6 +2205,15 @@ pub const Parser = struct { n = for_node.body; }, + ast.Node.Id.If => { + const if_node = @fieldParentPtr(ast.NodeIf, "base", n); + if (if_node.@"else") |@"else"| { + n = @"else".base; + continue; + } + + n = if_node.body; + }, ast.Node.Id.Else => { const else_node = @fieldParentPtr(ast.NodeElse, "base", n); n = else_node.body; @@ -3363,14 +3389,19 @@ pub const Parser = struct { const else_node = @fieldParentPtr(ast.NodeElse, "base", base); try stream.print("{} ", self.tokenizer.getTokenSlice(else_node.else_token)); - if (else_node.body.id == ast.Node.Id.Block) { - try stack.append(RenderState { .Expression = else_node.body }); - } else { - try stack.append(RenderState { .Indent = indent }); - try stack.append(RenderState { .Expression = else_node.body }); - try stack.append(RenderState.PrintIndent); - try stack.append(RenderState { .Indent = indent + indent_delta }); - try stack.append(RenderState { .Text = "\n" }); + switch (else_node.body.id) { + ast.Node.Id.Block, ast.Node.Id.If, + ast.Node.Id.For, ast.Node.Id.While, + ast.Node.Id.Switch => { + try stack.append(RenderState { .Expression = else_node.body }); + }, + else => { + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Expression = else_node.body }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + } } if (else_node.payload) |payload| { @@ -3396,6 +3427,7 @@ pub const Parser = struct { if (while_node.body.id == ast.Node.Id.Block) { try stack.append(RenderState { .Text = " " }); } else { + try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Text = "\n" }); } } @@ -3445,6 +3477,7 @@ pub const Parser = struct { if (for_node.body.id == ast.Node.Id.Block) { try stack.append(RenderState { .Text = " " }); } else { + try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Text = "\n" }); } } @@ -3469,6 +3502,53 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = for_node.array_expr }); try stack.append(RenderState { .Text = "(" }); }, + ast.Node.Id.If => { + const if_node = @fieldParentPtr(ast.NodeIf, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(if_node.if_token)); + + switch (if_node.body.id) { + ast.Node.Id.Block, ast.Node.Id.If, + ast.Node.Id.For, ast.Node.Id.While, + ast.Node.Id.Switch => { + if (if_node.@"else") |@"else"| { + try stack.append(RenderState { .Expression = &@"else".base }); + + if (if_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Text = "\n" }); + } + } + }, + else => { + if (if_node.@"else") |@"else"| { + try stack.append(RenderState { .Expression = @"else".body }); + + if (@"else".payload) |payload| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = &payload.base }); + } + + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(@"else".else_token) }); + try stack.append(RenderState { .Text = " " }); + } + } + } + + try stack.append(RenderState { .Expression = if_node.body }); + try stack.append(RenderState { .Text = " " }); + + if (if_node.payload) |payload| { + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = " " }); + } + + try stack.append(RenderState { .Text = ")" }); + try stack.append(RenderState { .Expression = if_node.condition }); + try stack.append(RenderState { .Text = "(" }); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -4210,8 +4290,7 @@ test "zig fmt: if" { \\ unreachable; \\ } \\ - \\ if (10 < 0) - \\ unreachable; + \\ if (10 < 0) unreachable; \\ \\ if (10 < 0) { \\ unreachable; From d04346d2ac04c079c1722db691d1536c3d717735 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 13:07:46 +0200 Subject: [PATCH 35/54] ast.zig.parser now parses defer statements --- std/zig/ast.zig | 33 +++++++++++++++++++++++++++++++++ std/zig/parser.zig | 27 +++++++++++++++++++++++++++ std/zig/tokenizer.zig | 2 ++ 3 files changed, 62 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 9e077a3567..29eaa56be4 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,6 +20,7 @@ pub const Node = struct { FnProto, ParamDecl, Block, + Defer, ErrorPayload, ValuePayload, ValueIndexPayload, @@ -65,6 +66,7 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), + Id.Defer => @fieldParentPtr(NodeDefer, "base", base).iterate(index), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).iterate(index), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).iterate(index), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).iterate(index), @@ -111,6 +113,7 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), + Id.Defer => @fieldParentPtr(NodeDefer, "base", base).firstToken(), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).firstToken(), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).firstToken(), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).firstToken(), @@ -157,6 +160,7 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), + Id.Defer => @fieldParentPtr(NodeDefer, "base", base).lastToken(), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).lastToken(), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).lastToken(), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).lastToken(), @@ -573,6 +577,35 @@ pub const NodeBlock = struct { } }; +pub const NodeDefer = struct { + base: Node, + defer_token: Token, + kind: Kind, + expr: &Node, + + const Kind = enum { + Error, + Unconditional, + }; + + pub fn iterate(self: &NodeDefer, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeDefer) Token { + return self.defer_token; + } + + pub fn lastToken(self: &NodeDefer) Token { + return self.expr.lastToken(); + } +}; + pub const NodeErrorPayload = struct { base: Node, lpipe: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 09383a313b..b783af3ea7 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2140,6 +2140,24 @@ pub const Parser = struct { stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; }, + Token.Id.Keyword_defer, Token.Id.Keyword_errdefer => { + const node = try arena.create(ast.NodeDefer); + *node = ast.NodeDefer { + .base = self.initNode(ast.Node.Id.Defer), + .defer_token = next, + .kind = switch (next.id) { + Token.Id.Keyword_defer => ast.NodeDefer.Kind.Unconditional, + Token.Id.Keyword_errdefer => ast.NodeDefer.Kind.Error, + else => unreachable, + }, + .expr = undefined, + }; + try block.statements.append(&node.base); + + stack.append(State { .Semicolon = &node.base }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = &node.expr } }); + continue; + }, Token.Id.LBrace => { const inner_block = try self.createBlock(arena, (?Token)(null), next); try block.statements.append(&inner_block.base); @@ -2218,6 +2236,10 @@ pub const Parser = struct { const else_node = @fieldParentPtr(ast.NodeElse, "base", n); n = else_node.body; }, + ast.Node.Id.Defer => { + const defer_node = @fieldParentPtr(ast.NodeDefer, "base", n); + n = defer_node.expr; + }, else => return true, } } @@ -2912,6 +2934,11 @@ pub const Parser = struct { } } }, + ast.Node.Id.Defer => { + const defer_node = @fieldParentPtr(ast.NodeDefer, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(defer_node.defer_token)); + try stack.append(RenderState { .Expression = defer_node.expr }); + }, ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 422aa20629..4cce31baeb 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -26,6 +26,7 @@ pub const Token = struct { KeywordId{.bytes="defer", .id = Id.Keyword_defer}, KeywordId{.bytes="else", .id = Id.Keyword_else}, KeywordId{.bytes="enum", .id = Id.Keyword_enum}, + KeywordId{.bytes="errdefer", .id = Id.Keyword_errdefer}, KeywordId{.bytes="error", .id = Id.Keyword_error}, KeywordId{.bytes="export", .id = Id.Keyword_export}, KeywordId{.bytes="extern", .id = Id.Keyword_extern}, @@ -151,6 +152,7 @@ pub const Token = struct { Keyword_defer, Keyword_else, Keyword_enum, + Keyword_errdefer, Keyword_error, Keyword_export, Keyword_extern, From 7d32c9521fff3d2a18293d38562636c5a0e3408e Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 13:24:47 +0200 Subject: [PATCH 36/54] std.zig.parser now parses comptime --- std/zig/ast.zig | 27 +++++++++++++++++++++++++++ std/zig/parser.zig | 42 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 29eaa56be4..2da819556c 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -21,6 +21,7 @@ pub const Node = struct { ParamDecl, Block, Defer, + Comptime, ErrorPayload, ValuePayload, ValueIndexPayload, @@ -67,6 +68,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).iterate(index), + Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).iterate(index), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).iterate(index), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).iterate(index), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).iterate(index), @@ -114,6 +116,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).firstToken(), + Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).firstToken(), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).firstToken(), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).firstToken(), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).firstToken(), @@ -161,6 +164,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).lastToken(), + Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).lastToken(), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).lastToken(), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).lastToken(), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).lastToken(), @@ -606,6 +610,29 @@ pub const NodeDefer = struct { } }; +pub const NodeComptime = struct { + base: Node, + comptime_token: Token, + expr: &Node, + + pub fn iterate(self: &NodeComptime, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeComptime) Token { + return self.comptime_token; + } + + pub fn lastToken(self: &NodeComptime) Token { + return self.expr.lastToken(); + } +}; + pub const NodeErrorPayload = struct { base: Node, lpipe: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index b783af3ea7..aa04ad68d3 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -309,6 +309,18 @@ pub const Parser = struct { }); continue; }, + Token.Id.Keyword_comptime => { + const node = try arena.create(ast.NodeComptime); + *node = ast.NodeComptime { + .base = self.initNode(ast.Node.Id.Comptime), + .comptime_token = token, + .expr = undefined, + }; + try root_node.decls.append(&node.base); + stack.append(State.TopLevel) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, else => { self.putBackToken(token); stack.append(State.TopLevel) catch unreachable; @@ -1523,7 +1535,15 @@ pub const Parser = struct { try stack.append(State { .ExpectToken = Token.Id.LParen }); }, Token.Id.Keyword_comptime => { - @panic("TODO: inline comptime"); + const node = try arena.create(ast.NodeComptime); + *node = ast.NodeComptime { + .base = self.initNode(ast.Node.Id.Comptime), + .comptime_token = token, + .expr = undefined, + }; + dest_ptr.store(&node.base); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); @@ -2131,7 +2151,10 @@ pub const Parser = struct { continue; } else { self.putBackToken(mut_token); - @panic("TODO: comptime block"); + self.putBackToken(next); + const statememt = try block.statements.addOne(); + stack.append(State { .Semicolon = statememt }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = statememt } }); } }, Token.Id.Keyword_var, Token.Id.Keyword_const => { @@ -2240,6 +2263,10 @@ pub const Parser = struct { const defer_node = @fieldParentPtr(ast.NodeDefer, "base", n); n = defer_node.expr; }, + ast.Node.Id.Comptime => { + const comptime_node = @fieldParentPtr(ast.NodeComptime, "base", n); + n = comptime_node.expr; + }, else => return true, } } @@ -2824,6 +2851,12 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = value}); } }, + ast.Node.Id.Comptime => { + if (requireSemiColon(decl)) { + try stack.append(RenderState { .Text = ";" }); + } + try stack.append(RenderState { .Expression = decl }); + }, else => unreachable, } }, @@ -2939,6 +2972,11 @@ pub const Parser = struct { try stream.print("{} ", self.tokenizer.getTokenSlice(defer_node.defer_token)); try stack.append(RenderState { .Expression = defer_node.expr }); }, + ast.Node.Id.Comptime => { + const comptime_node = @fieldParentPtr(ast.NodeComptime, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_node.comptime_token)); + try stack.append(RenderState { .Expression = comptime_node.expr }); + }, ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); From aa552633cc0bc66d46f61ea2105f7a4392df37be Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 14:02:03 +0200 Subject: [PATCH 37/54] std.zig.parser now parses fn types --- std/zig/parser.zig | 134 +++++++++++++++++++++++++++++++++------------ 1 file changed, 99 insertions(+), 35 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index aa04ad68d3..7bdff56b85 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1413,6 +1413,17 @@ pub const Parser = struct { }) catch unreachable; }, Token.Id.Keyword_extern => { + const next = self.getNextToken(); + if (next.id == Token.Id.Keyword_fn) { + // TODO shouldn't need this cast + const fn_proto = try self.createFnProto(arena, next, + (?Token)(token), (?&ast.Node)(null), (?Token)(null), (?Token)(null), (?Token)(null)); + dest_ptr.store(&fn_proto.base); + stack.append(State { .FnProto = fn_proto }) catch unreachable; + continue; + } + + self.putBackToken(next); stack.append(State { .ContainerExtern = ContainerExternCtx { .dest_ptr = dest_ptr, @@ -1455,7 +1466,26 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_fn => { - @panic("TODO: fn proto"); + // TODO shouldn't need these casts + const fn_proto = try self.createFnProto(arena, token, + (?Token)(null), (?&ast.Node)(null), (?Token)(null), (?Token)(null), (?Token)(null)); + dest_ptr.store(&fn_proto.base); + stack.append(State { .FnProto = fn_proto }) catch unreachable; + continue; + }, + Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { + // TODO shouldn't need this cast + const fn_proto = try self.createFnProto(arena, undefined, + (?Token)(null), (?&ast.Node)(null), (?Token)(token), (?Token)(null), (?Token)(null)); + dest_ptr.store(&fn_proto.base); + stack.append(State { .FnProto = fn_proto }) catch unreachable; + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Keyword_fn, + .ptr = &fn_proto.fn_token, + } + }); + continue; }, Token.Id.Keyword_asm => { @panic("TODO: inline asm"); @@ -2781,41 +2811,14 @@ pub const Parser = struct { ast.Node.Id.FnProto => { const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", decl); - if (fn_proto.body_node == null) { - try stack.append(RenderState { .Text = ";" }); + if (fn_proto.body_node) |body_node| { + stack.append(RenderState { .Expression = body_node}) catch unreachable; + try stack.append(RenderState { .Text = " "}); + } else { + stack.append(RenderState { .Text = ";" }) catch unreachable; } - try stack.append(RenderState { .FnProtoRParen = fn_proto}); - var i = fn_proto.params.len; - while (i != 0) { - i -= 1; - const param_decl_node = fn_proto.params.items[i]; - try stack.append(RenderState { .ParamDecl = param_decl_node}); - if (i != 0) { - try stack.append(RenderState { .Text = ", " }); - } - } - - try stack.append(RenderState { .Text = "(" }); - if (fn_proto.name_token) |name_token| { - try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(name_token) }); - } - - try stack.append(RenderState { .Text = "fn " }); - if (fn_proto.lib_name) |lib_name| { - try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Expression = lib_name }); - } - if (fn_proto.extern_token) |extern_token| { - try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); - } - - if (fn_proto.visib_token) |visib_token| { - assert(visib_token.id == Token.Id.Keyword_pub or visib_token.id == Token.Id.Keyword_export); - try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) }); - } + try stack.append(RenderState { .Expression = decl }); }, ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); @@ -3386,7 +3389,65 @@ pub const Parser = struct { } } }, - ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), + ast.Node.Id.FnProto => { + const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", base); + + switch (fn_proto.return_type) { + ast.NodeFnProto.ReturnType.Explicit => |node| { + try stack.append(RenderState { .Expression = node}); + }, + ast.NodeFnProto.ReturnType.Infer => { + try stack.append(RenderState { .Text = "var"}); + }, + ast.NodeFnProto.ReturnType.InferErrorSet => |node| { + try stack.append(RenderState { .Expression = node}); + try stack.append(RenderState { .Text = "!"}); + }, + } + + if (fn_proto.align_expr != null) { + @panic("TODO"); + } + + try stack.append(RenderState { .Text = ") " }); + var i = fn_proto.params.len; + while (i != 0) { + i -= 1; + const param_decl_node = fn_proto.params.items[i]; + try stack.append(RenderState { .ParamDecl = param_decl_node}); + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + + try stack.append(RenderState { .Text = "(" }); + if (fn_proto.name_token) |name_token| { + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(name_token) }); + try stack.append(RenderState { .Text = " " }); + } + + try stack.append(RenderState { .Text = "fn" }); + + if (fn_proto.cc_token) |cc_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(cc_token) }); + } + + if (fn_proto.lib_name) |lib_name| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = lib_name }); + } + if (fn_proto.extern_token) |extern_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); + } + + if (fn_proto.visib_token) |visib_token| { + assert(visib_token.id == Token.Id.Keyword_pub or visib_token.id == Token.Id.Keyword_export); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) }); + } + }, ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), ast.Node.Id.Switch => { const switch_node = @fieldParentPtr(ast.NodeSwitch, "base", base); @@ -4461,6 +4522,9 @@ test "zig fmt: fn type" { \\ return i + 1; \\} \\ + \\const a: fn(u8) u8 = undefined; + \\const b: extern fn(u8) u8 = undefined; + \\const c: nakedcc fn(u8) u8 = undefined; \\const ap: fn(u8) u8 = a; \\ ); From a09bb408a20fcfc0575a9298b318803b07517b0a Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 15:40:16 +0200 Subject: [PATCH 38/54] std.zig.parser now parses asm expressions * We cannot render asm expressions yet --- std/zig/ast.zig | 41 ++++++++++++++ std/zig/parser.zig | 138 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 178 insertions(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 2da819556c..d048f4ed43 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -47,6 +47,7 @@ pub const Node = struct { NullLiteral, UndefinedLiteral, ThisLiteral, + Asm, Unreachable, ErrorType, BuiltinCall, @@ -94,6 +95,7 @@ pub const Node = struct { Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).iterate(index), + Id.Asm => @fieldParentPtr(NodeAsm, "base", base).iterate(index), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), @@ -143,6 +145,7 @@ pub const Node = struct { Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).firstToken(), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), + Id.Asm => @fieldParentPtr(NodeAsm, "base", base).firstToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), @@ -190,6 +193,7 @@ pub const Node = struct { Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).lastToken(), + Id.Asm => @fieldParentPtr(NodeAsm, "base", base).lastToken(), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), @@ -1512,6 +1516,43 @@ pub const NodeThisLiteral = struct { } }; +pub const NodeAsm = struct { + base: Node, + asm_token: Token, + is_volatile: bool, + template: Token, + //tokens: ArrayList(AsmToken), + outputs: ArrayList(AsmOutput), + inputs: ArrayList(AsmInput), + cloppers: ArrayList(&NodeStringLiteral), + rparen: Token, + + const AsmOutput = struct { + symbolic_name: Token, + constraint: Token, + variable_name: ?Token, + return_type: ?&Node, + }; + + const AsmInput = struct { + symbolic_name: Token, + constraint: Token, + expr: &Node, + }; + + pub fn iterate(self: &NodeAsm, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeAsm) Token { + return self.asm_token; + } + + pub fn lastToken(self: &NodeAsm) Token { + return self.rparen; + } +}; + pub const NodeUnreachable = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 7bdff56b85..7f5427146a 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -164,6 +164,9 @@ pub const Parser = struct { WhileContinueExpr: &?&ast.Node, Statement: &ast.NodeBlock, Semicolon: &const &const ast.Node, + AsmOutputItems: &ArrayList(ast.NodeAsm.AsmOutput), + AsmInputItems: &ArrayList(ast.NodeAsm.AsmInput), + AsmClopperItems: &ArrayList(&ast.NodeStringLiteral), ExprListItemOrEnd: ExprListCtx, ExprListCommaOrEnd: ExprListCtx, FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer), @@ -1488,7 +1491,44 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_asm => { - @panic("TODO: inline asm"); + const is_volatile = blk: { + const volatile_token = self.getNextToken(); + if (volatile_token.id != Token.Id.Keyword_volatile) { + self.putBackToken(volatile_token); + break :blk false; + } + break :blk true; + }; + _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + const template = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + // TODO parse template + + const node = try arena.create(ast.NodeAsm); + *node = ast.NodeAsm { + .base = self.initNode(ast.Node.Id.Asm), + .asm_token = token, + .is_volatile = is_volatile, + .template = template, + //.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena), + .outputs = ArrayList(ast.NodeAsm.AsmOutput).init(arena), + .inputs = ArrayList(ast.NodeAsm.AsmInput).init(arena), + .cloppers = ArrayList(&ast.NodeStringLiteral).init(arena), + .rparen = undefined, + }; + dest_ptr.store(&node.base); + + stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RParen, + .ptr = &node.rparen, + } + }) catch unreachable; + try stack.append(State { .AsmClopperItems = &node.cloppers }); + try stack.append(State { .IfToken = Token.Id.Colon }); + try stack.append(State { .AsmInputItems = &node.inputs }); + try stack.append(State { .IfToken = Token.Id.Colon }); + try stack.append(State { .AsmOutputItems = &node.outputs }); + try stack.append(State { .IfToken = Token.Id.Colon }); }, Token.Id.Keyword_if => { const node = try arena.create(ast.NodeIf); @@ -1621,6 +1661,84 @@ pub const Parser = struct { } }, + + State.AsmOutputItems => |items| { + const lbracket = self.getNextToken(); + if (lbracket.id != Token.Id.LBracket) { + self.putBackToken(lbracket); + continue; + } + + stack.append(State { .AsmOutputItems = items }) catch unreachable; + try stack.append(State { .IfToken = Token.Id.Comma }); + + const symbolic_name = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + _ = (try self.eatToken(&stack, Token.Id.RBracket)) ?? continue; + const constraint = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + + _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + try stack.append(State { .ExpectToken = Token.Id.RParen }); + + const res = try items.addOne(); + *res = ast.NodeAsm.AsmOutput { + .symbolic_name = symbolic_name, + .constraint = constraint, + .variable_name = null, + .return_type = null, + }; + const symbol_or_arrow = self.getNextToken(); + switch (symbol_or_arrow.id) { + Token.Id.Identifier => res.variable_name = symbol_or_arrow, + Token.Id.Arrow => { + try stack.append(State { .TypeExprBegin = DestPtr { .NullableField = &res.return_type } }); + }, + else => { + try self.parseError(&stack, symbol_or_arrow, "expected '->' or {}, found {}", + @tagName(Token.Id.Identifier), + @tagName(symbol_or_arrow.id)); + continue; + }, + } + }, + + State.AsmInputItems => |items| { + const lbracket = self.getNextToken(); + if (lbracket.id != Token.Id.LBracket) { + self.putBackToken(lbracket); + continue; + } + + stack.append(State { .AsmInputItems = items }) catch unreachable; + try stack.append(State { .IfToken = Token.Id.Comma }); + + const symbolic_name = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + _ = (try self.eatToken(&stack, Token.Id.RBracket)) ?? continue; + const constraint = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + + _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + try stack.append(State { .ExpectToken = Token.Id.RParen }); + + const res = try items.addOne(); + *res = ast.NodeAsm.AsmInput { + .symbolic_name = symbolic_name, + .constraint = constraint, + .expr = undefined, + }; + try stack.append(State { .Expression = DestPtr { .Field = &res.expr } }); + }, + + State.AsmClopperItems => |items| { + const string = self.getNextToken(); + if (string.id != Token.Id.StringLiteral) { + self.putBackToken(string); + continue; + } + + try items.append(try self.createStringLiteral(arena, string)); + stack.append(State { .AsmClopperItems = items }) catch unreachable; + try stack.append(State { .IfToken = Token.Id.Comma }); + }, + State.ExprListItemOrEnd => |list_state| { var token = self.getNextToken(); @@ -3675,6 +3793,24 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = if_node.condition }); try stack.append(RenderState { .Text = "(" }); }, + ast.Node.Id.Asm => { + const asm_node = @fieldParentPtr(ast.NodeAsm, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(asm_node.asm_token)); + + if (asm_node.is_volatile) { + try stream.write("volatile "); + } + + try stream.print("({}", self.tokenizer.getTokenSlice(asm_node.template)); + + try stack.append(RenderState { .Text = ")" }); + @panic("TODO: Render asm"); + //\\ return asm volatile ("syscall" + //\\ : [ret] "={rax}" (-> usize) + //\\ : [number] "{rax}" (number), + //\\ [arg1] "{rdi}" (arg1) + //\\ : "rcx", "r11"); + },, ast.Node.Id.StructField, ast.Node.Id.UnionTag, From 2c7996f4006b94f81fc3b1c1e3f73a4a7a291782 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 09:27:11 +0200 Subject: [PATCH 39/54] std.zig.parser can now render asm expressions --- std/zig/ast.zig | 112 +++++++++++++++++++++++++++----- std/zig/parser.zig | 156 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 228 insertions(+), 40 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index d048f4ed43..86a706f1c7 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -48,6 +48,8 @@ pub const Node = struct { UndefinedLiteral, ThisLiteral, Asm, + AsmInput, + AsmOutput, Unreachable, ErrorType, BuiltinCall, @@ -96,6 +98,8 @@ pub const Node = struct { Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).iterate(index), Id.Asm => @fieldParentPtr(NodeAsm, "base", base).iterate(index), + Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).iterate(index), + Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).iterate(index), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), @@ -146,6 +150,8 @@ pub const Node = struct { Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).firstToken(), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), Id.Asm => @fieldParentPtr(NodeAsm, "base", base).firstToken(), + Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).firstToken(), + Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).firstToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), @@ -194,6 +200,8 @@ pub const Node = struct { Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).lastToken(), Id.Asm => @fieldParentPtr(NodeAsm, "base", base).lastToken(), + Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).lastToken(), + Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).lastToken(), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), @@ -1516,31 +1524,105 @@ pub const NodeThisLiteral = struct { } }; +pub const NodeAsmOutput = struct { + base: Node, + symbolic_name: &NodeIdentifier, + constraint: &NodeStringLiteral, + kind: Kind, + + const Kind = union(enum) { + Variable: &NodeIdentifier, + Return: &Node + }; + + pub fn iterate(self: &NodeAsmOutput, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.symbolic_name.base; + i -= 1; + + if (i < 1) return &self.constraint.base; + i -= 1; + + switch (self.kind) { + Kind.Variable => |variable_name| { + if (i < 1) return &variable_name.base; + i -= 1; + }, + Kind.Return => |return_type| { + if (i < 1) return return_type; + i -= 1; + } + } + + return null; + } + + pub fn firstToken(self: &NodeAsmOutput) Token { + return self.symbolic_name.firstToken(); + } + + pub fn lastToken(self: &NodeAsmOutput) Token { + return switch (self.kind) { + Kind.Variable => |variable_name| variable_name.lastToken(), + Kind.Return => |return_type| return_type.lastToken(), + }; + } +}; + +pub const NodeAsmInput = struct { + base: Node, + symbolic_name: &NodeIdentifier, + constraint: &NodeStringLiteral, + expr: &Node, + + pub fn iterate(self: &NodeAsmInput, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.symbolic_name.base; + i -= 1; + + if (i < 1) return &self.constraint.base; + i -= 1; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeAsmInput) Token { + return self.symbolic_name.firstToken(); + } + + pub fn lastToken(self: &NodeAsmInput) Token { + return self.expr.lastToken(); + } +}; + pub const NodeAsm = struct { base: Node, asm_token: Token, is_volatile: bool, template: Token, //tokens: ArrayList(AsmToken), - outputs: ArrayList(AsmOutput), - inputs: ArrayList(AsmInput), + outputs: ArrayList(&NodeAsmOutput), + inputs: ArrayList(&NodeAsmInput), cloppers: ArrayList(&NodeStringLiteral), rparen: Token, - const AsmOutput = struct { - symbolic_name: Token, - constraint: Token, - variable_name: ?Token, - return_type: ?&Node, - }; - - const AsmInput = struct { - symbolic_name: Token, - constraint: Token, - expr: &Node, - }; - pub fn iterate(self: &NodeAsm, index: usize) ?&Node { + var i = index; + + if (i < self.outputs.len) return &self.outputs.at(index).base; + i -= self.outputs.len; + + if (i < self.inputs.len) return &self.inputs.at(index).base; + i -= self.inputs.len; + + if (i < self.cloppers.len) return &self.cloppers.at(index).base; + i -= self.cloppers.len; + return null; } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 7f5427146a..9851f6cc30 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -164,8 +164,8 @@ pub const Parser = struct { WhileContinueExpr: &?&ast.Node, Statement: &ast.NodeBlock, Semicolon: &const &const ast.Node, - AsmOutputItems: &ArrayList(ast.NodeAsm.AsmOutput), - AsmInputItems: &ArrayList(ast.NodeAsm.AsmInput), + AsmOutputItems: &ArrayList(&ast.NodeAsmOutput), + AsmInputItems: &ArrayList(&ast.NodeAsmInput), AsmClopperItems: &ArrayList(&ast.NodeStringLiteral), ExprListItemOrEnd: ExprListCtx, ExprListCommaOrEnd: ExprListCtx, @@ -1510,8 +1510,8 @@ pub const Parser = struct { .is_volatile = is_volatile, .template = template, //.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena), - .outputs = ArrayList(ast.NodeAsm.AsmOutput).init(arena), - .inputs = ArrayList(ast.NodeAsm.AsmInput).init(arena), + .outputs = ArrayList(&ast.NodeAsmOutput).init(arena), + .inputs = ArrayList(&ast.NodeAsmInput).init(arena), .cloppers = ArrayList(&ast.NodeStringLiteral).init(arena), .rparen = undefined, }; @@ -1679,18 +1679,23 @@ pub const Parser = struct { _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; try stack.append(State { .ExpectToken = Token.Id.RParen }); - const res = try items.addOne(); - *res = ast.NodeAsm.AsmOutput { - .symbolic_name = symbolic_name, - .constraint = constraint, - .variable_name = null, - .return_type = null, + const node = try arena.create(ast.NodeAsmOutput); + *node = ast.NodeAsmOutput { + .base = self.initNode(ast.Node.Id.AsmOutput), + .symbolic_name = try self.createIdentifier(arena, symbolic_name), + .constraint = try self.createStringLiteral(arena, constraint), + .kind = undefined, }; + try items.append(node); + const symbol_or_arrow = self.getNextToken(); switch (symbol_or_arrow.id) { - Token.Id.Identifier => res.variable_name = symbol_or_arrow, + Token.Id.Identifier => { + node.kind = ast.NodeAsmOutput.Kind { .Variable = try self.createIdentifier(arena, symbol_or_arrow) }; + }, Token.Id.Arrow => { - try stack.append(State { .TypeExprBegin = DestPtr { .NullableField = &res.return_type } }); + node.kind = ast.NodeAsmOutput.Kind { .Return = undefined }; + try stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.kind.Return } }); }, else => { try self.parseError(&stack, symbol_or_arrow, "expected '->' or {}, found {}", @@ -1718,13 +1723,15 @@ pub const Parser = struct { _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; try stack.append(State { .ExpectToken = Token.Id.RParen }); - const res = try items.addOne(); - *res = ast.NodeAsm.AsmInput { - .symbolic_name = symbolic_name, - .constraint = constraint, + const node = try arena.create(ast.NodeAsmInput); + *node = ast.NodeAsmInput { + .base = self.initNode(ast.Node.Id.AsmInput), + .symbolic_name = try self.createIdentifier(arena, symbolic_name), + .constraint = try self.createStringLiteral(arena, constraint), .expr = undefined, }; - try stack.append(State { .Expression = DestPtr { .Field = &res.expr } }); + try items.append(node); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); }, State.AsmClopperItems => |items| { @@ -3803,14 +3810,113 @@ pub const Parser = struct { try stream.print("({}", self.tokenizer.getTokenSlice(asm_node.template)); + try stack.append(RenderState { .Indent = indent }); try stack.append(RenderState { .Text = ")" }); - @panic("TODO: Render asm"); - //\\ return asm volatile ("syscall" - //\\ : [ret] "={rax}" (-> usize) - //\\ : [number] "{rax}" (number), - //\\ [arg1] "{rdi}" (arg1) - //\\ : "rcx", "r11"); - },, + { + const cloppers = asm_node.cloppers.toSliceConst(); + var i = cloppers.len; + while (i != 0) { + i -= 1; + try stack.append(RenderState { .Expression = &cloppers[i].base }); + + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + } + try stack.append(RenderState { .Text = ": " }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + { + const inputs = asm_node.inputs.toSliceConst(); + var i = inputs.len; + while (i != 0) { + i -= 1; + const node = inputs[i]; + try stack.append(RenderState { .Expression = &node.base}); + + if (i != 0) { + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + const prev_node = inputs[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { + break :blk "\n\n"; + } + break :blk "\n"; + }, + }); + try stack.append(RenderState { .Text = "," }); + } + } + } + try stack.append(RenderState { .Indent = indent + indent_delta + 2}); + try stack.append(RenderState { .Text = ": "}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "\n" }); + { + const outputs = asm_node.outputs.toSliceConst(); + var i = outputs.len; + while (i != 0) { + i -= 1; + const node = outputs[i]; + try stack.append(RenderState { .Expression = &node.base}); + + if (i != 0) { + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + const prev_node = outputs[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { + break :blk "\n\n"; + } + break :blk "\n"; + }, + }); + try stack.append(RenderState { .Text = "," }); + } + } + } + try stack.append(RenderState { .Indent = indent + indent_delta + 2}); + try stack.append(RenderState { .Text = ": "}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "\n" }); + }, + ast.Node.Id.AsmInput => { + const asm_input = @fieldParentPtr(ast.NodeAsmInput, "base", base); + + try stack.append(RenderState { .Text = ")"}); + try stack.append(RenderState { .Expression = asm_input.expr}); + try stack.append(RenderState { .Text = " ("}); + try stack.append(RenderState { .Expression = &asm_input.constraint.base}); + try stack.append(RenderState { .Text = "] "}); + try stack.append(RenderState { .Expression = &asm_input.symbolic_name.base}); + try stack.append(RenderState { .Text = "["}); + }, + ast.Node.Id.AsmOutput => { + const asm_output = @fieldParentPtr(ast.NodeAsmOutput, "base", base); + + try stack.append(RenderState { .Text = ")"}); + switch (asm_output.kind) { + ast.NodeAsmOutput.Kind.Variable => |variable_name| { + try stack.append(RenderState { .Expression = &variable_name.base}); + }, + ast.NodeAsmOutput.Kind.Return => |return_type| { + try stack.append(RenderState { .Expression = return_type}); + try stack.append(RenderState { .Text = "-> "}); + }, + } + try stack.append(RenderState { .Text = " ("}); + try stack.append(RenderState { .Expression = &asm_output.constraint.base}); + try stack.append(RenderState { .Text = "] "}); + try stack.append(RenderState { .Expression = &asm_output.symbolic_name.base}); + try stack.append(RenderState { .Text = "["}); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -4672,7 +4778,7 @@ test "zig fmt: inline asm" { \\ return asm volatile ("syscall" \\ : [ret] "={rax}" (-> usize) \\ : [number] "{rax}" (number), - \\ [arg1] "{rdi}" (arg1) + \\ [arg1] "{rdi}" (arg1) \\ : "rcx", "r11"); \\} \\ From 5cd69ee6a4b7d02dbc48d93e8f8f95bd608d9d7c Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 09:37:29 +0200 Subject: [PATCH 40/54] std.zig.parser changed assign expr to only be allowed in some contexts * Only allowed in while continue expr and statement expr --- std/zig/parser.zig | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 9851f6cc30..ff3b35c193 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -721,7 +721,7 @@ pub const Parser = struct { }, else => { self.putBackToken(token); - stack.append(State { .AssignmentExpressionBegin = dest_ptr }) catch unreachable; + stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; continue; } } @@ -750,7 +750,7 @@ pub const Parser = struct { State.AssignmentExpressionBegin => |dest_ptr| { stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; - try stack.append(State { .UnwrapExpressionBegin = dest_ptr }); + try stack.append(State { .Expression = dest_ptr }); continue; }, @@ -762,7 +762,7 @@ pub const Parser = struct { dest_ptr.store(&node.base); stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; - try stack.append(State { .UnwrapExpressionBegin = DestPtr { .Field = &node.rhs } }); + try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); continue; } else { self.putBackToken(token); @@ -1895,7 +1895,7 @@ pub const Parser = struct { _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; stack.append(State { .ExpectToken = Token.Id.RParen }) catch unreachable; - try stack.append(State { .Expression = DestPtr { .NullableField = dest } }); + try stack.append(State { .AssignmentExpressionBegin = DestPtr { .NullableField = dest } }); }, State.ErrorPayload => |dest| { @@ -2333,7 +2333,7 @@ pub const Parser = struct { try block.statements.append(&node.base); stack.append(State { .Semicolon = &node.base }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = &node.expr } }); + try stack.append(State { .AssignmentExpressionBegin = DestPtr{.Field = &node.expr } }); continue; }, Token.Id.LBrace => { @@ -2347,7 +2347,7 @@ pub const Parser = struct { self.putBackToken(next); const statememt = try block.statements.addOne(); stack.append(State { .Semicolon = statememt }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = statememt } }); + try stack.append(State { .AssignmentExpressionBegin = DestPtr{.Field = statememt } }); continue; } } @@ -4260,8 +4260,6 @@ test "zig fmt: precedence" { \\ a or b and c; \\ (a or b) and c; \\ (a or b) and c; - \\ a = b or c; - \\ (a = b) or c; \\} \\ ); From f85b9f2bf3ac0c5af7804921d2003c2992558cb7 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 11:25:58 +0200 Subject: [PATCH 41/54] std.zig.parser now parses coroutine code --- std/zig/ast.zig | 157 +++++++++++++++++++------ std/zig/parser.zig | 277 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 345 insertions(+), 89 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 86a706f1c7..9043a66ac3 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -17,14 +17,15 @@ pub const Node = struct { UnionTag, EnumTag, Identifier, + AsyncAttribute, FnProto, ParamDecl, Block, Defer, Comptime, - ErrorPayload, - ValuePayload, - ValueIndexPayload, + Payload, + PointerPayload, + PointerIndexPayload, Else, Switch, SwitchCase, @@ -37,6 +38,7 @@ pub const Node = struct { SuffixOp, GroupedExpression, ControlFlowExpression, + Suspend, FieldInitializer, IntegerLiteral, FloatLiteral, @@ -67,14 +69,15 @@ pub const Node = struct { Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).iterate(index), Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).iterate(index), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index), + Id.AsyncAttribute => @fieldParentPtr(NodeAsyncAttribute, "base", base).iterate(index), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).iterate(index), Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).iterate(index), - Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).iterate(index), - Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).iterate(index), - Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).iterate(index), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), + Id.PointerPayload => @fieldParentPtr(NodePointerPayload, "base", base).iterate(index), + Id.PointerIndexPayload => @fieldParentPtr(NodePointerIndexPayload, "base", base).iterate(index), Id.Else => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), @@ -87,6 +90,7 @@ pub const Node = struct { Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).iterate(index), Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).iterate(index), + Id.Suspend => @fieldParentPtr(NodeSuspend, "base", base).iterate(index), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), @@ -118,14 +122,15 @@ pub const Node = struct { Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).firstToken(), Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).firstToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(), + Id.AsyncAttribute => @fieldParentPtr(NodeAsyncAttribute, "base", base).firstToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).firstToken(), Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).firstToken(), - Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).firstToken(), - Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).firstToken(), - Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).firstToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), + Id.PointerPayload => @fieldParentPtr(NodePointerPayload, "base", base).firstToken(), + Id.PointerIndexPayload => @fieldParentPtr(NodePointerIndexPayload, "base", base).firstToken(), Id.Else => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), @@ -138,6 +143,7 @@ pub const Node = struct { Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).firstToken(), Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).firstToken(), + Id.Suspend => @fieldParentPtr(NodeSuspend, "base", base).firstToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), @@ -169,14 +175,15 @@ pub const Node = struct { Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).lastToken(), Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).lastToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(), + Id.AsyncAttribute => @fieldParentPtr(NodeAsyncAttribute, "base", base).lastToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).lastToken(), Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).lastToken(), - Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).lastToken(), - Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).lastToken(), - Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).lastToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), + Id.PointerPayload => @fieldParentPtr(NodePointerPayload, "base", base).lastToken(), + Id.PointerIndexPayload => @fieldParentPtr(NodePointerIndexPayload, "base", base).lastToken(), Id.Else => @fieldParentPtr(NodeElse, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), @@ -189,6 +196,7 @@ pub const Node = struct { Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).lastToken(), Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).lastToken(), + Id.Suspend => @fieldParentPtr(NodeSuspend, "base", base).lastToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), @@ -456,6 +464,36 @@ pub const NodeIdentifier = struct { } }; +pub const NodeAsyncAttribute = struct { + base: Node, + async_token: Token, + allocator_type: ?&Node, + rangle_bracket: ?Token, + + pub fn iterate(self: &NodeAsyncAttribute, index: usize) ?&Node { + var i = index; + + if (self.allocator_type) |allocator_type| { + if (i < 1) return allocator_type; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeAsyncAttribute) Token { + return self.async_token; + } + + pub fn lastToken(self: &NodeAsyncAttribute) Token { + if (self.rangle_bracket) |rangle_bracket| { + return rangle_bracket; + } + + return self.async_token; + } +}; + pub const NodeFnProto = struct { base: Node, visib_token: ?Token, @@ -467,6 +505,7 @@ pub const NodeFnProto = struct { extern_token: ?Token, inline_token: ?Token, cc_token: ?Token, + async_attr: ?&NodeAsyncAttribute, body_node: ?&Node, lib_name: ?&Node, // populated if this is an extern declaration align_expr: ?&Node, // populated if align(A) is present @@ -511,6 +550,14 @@ pub const NodeFnProto = struct { i -= 1; } + switch (self.call_convetion) { + CallConvetion.Async => |attr| { + if (i < 1) return &attr.base; + i -= 1; + }, + else => {}, + } + return null; } @@ -645,13 +692,13 @@ pub const NodeComptime = struct { } }; -pub const NodeErrorPayload = struct { +pub const NodePayload = struct { base: Node, lpipe: Token, error_symbol: &NodeIdentifier, rpipe: Token, - pub fn iterate(self: &NodeErrorPayload, index: usize) ?&Node { + pub fn iterate(self: &NodePayload, index: usize) ?&Node { var i = index; if (i < 1) return &self.error_symbol.base; @@ -660,23 +707,23 @@ pub const NodeErrorPayload = struct { return null; } - pub fn firstToken(self: &NodeErrorPayload) Token { + pub fn firstToken(self: &NodePayload) Token { return self.lpipe; } - pub fn lastToken(self: &NodeErrorPayload) Token { + pub fn lastToken(self: &NodePayload) Token { return self.rpipe; } }; -pub const NodeValuePayload = struct { +pub const NodePointerPayload = struct { base: Node, lpipe: Token, is_ptr: bool, value_symbol: &NodeIdentifier, rpipe: Token, - pub fn iterate(self: &NodeValuePayload, index: usize) ?&Node { + pub fn iterate(self: &NodePointerPayload, index: usize) ?&Node { var i = index; if (i < 1) return &self.value_symbol.base; @@ -685,16 +732,16 @@ pub const NodeValuePayload = struct { return null; } - pub fn firstToken(self: &NodeValuePayload) Token { + pub fn firstToken(self: &NodePointerPayload) Token { return self.lpipe; } - pub fn lastToken(self: &NodeValuePayload) Token { + pub fn lastToken(self: &NodePointerPayload) Token { return self.rpipe; } }; -pub const NodeValueIndexPayload = struct { +pub const NodePointerIndexPayload = struct { base: Node, lpipe: Token, is_ptr: bool, @@ -702,7 +749,7 @@ pub const NodeValueIndexPayload = struct { index_symbol: ?&NodeIdentifier, rpipe: Token, - pub fn iterate(self: &NodeValueIndexPayload, index: usize) ?&Node { + pub fn iterate(self: &NodePointerIndexPayload, index: usize) ?&Node { var i = index; if (i < 1) return &self.value_symbol.base; @@ -716,11 +763,11 @@ pub const NodeValueIndexPayload = struct { return null; } - pub fn firstToken(self: &NodeValueIndexPayload) Token { + pub fn firstToken(self: &NodePointerIndexPayload) Token { return self.lpipe; } - pub fn lastToken(self: &NodeValueIndexPayload) Token { + pub fn lastToken(self: &NodePointerIndexPayload) Token { return self.rpipe; } }; @@ -728,7 +775,7 @@ pub const NodeValueIndexPayload = struct { pub const NodeElse = struct { base: Node, else_token: Token, - payload: ?&NodeErrorPayload, + payload: ?&NodePayload, body: &Node, pub fn iterate(self: &NodeElse, index: usize) ?&Node { @@ -785,7 +832,7 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - payload: ?&NodeValuePayload, + payload: ?&NodePointerPayload, expr: &Node, pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { @@ -837,7 +884,7 @@ pub const NodeWhile = struct { inline_token: ?Token, while_token: Token, condition: &Node, - payload: ?&NodeValuePayload, + payload: ?&NodePointerPayload, continue_expr: ?&Node, body: &Node, @"else": ?&NodeElse, @@ -896,7 +943,7 @@ pub const NodeFor = struct { inline_token: ?Token, for_token: Token, array_expr: &Node, - payload: ?&NodeValueIndexPayload, + payload: ?&NodePointerIndexPayload, body: &Node, @"else": ?&NodeElse, @@ -947,7 +994,7 @@ pub const NodeIf = struct { base: Node, if_token: Token, condition: &Node, - payload: ?&NodeValuePayload, + payload: ?&NodePointerPayload, body: &Node, @"else": ?&NodeElse, @@ -1020,7 +1067,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&NodeErrorPayload, + Catch: ?&NodePayload, Div, EqualEqual, ErrorUnion, @@ -1113,13 +1160,16 @@ pub const NodePrefixOp = struct { const PrefixOp = union(enum) { AddrOf: AddrOfInfo, + ArrayType: &Node, + Await, BitNot, BoolNot, + Cancel, Deref, MaybeType, Negation, NegationWrap, - ArrayType: &Node, + Resume, SliceType: AddrOfInfo, Try, UnwrapMaybe, @@ -1153,13 +1203,16 @@ pub const NodePrefixOp = struct { if (i < 1) return size_expr; i -= 1; }, + PrefixOp.Await, PrefixOp.BitNot, PrefixOp.BoolNot, + PrefixOp.Cancel, PrefixOp.Deref, PrefixOp.Negation, PrefixOp.NegationWrap, PrefixOp.Return, PrefixOp.Try, + PrefixOp.Resume, PrefixOp.UnwrapMaybe => {}, } @@ -1218,8 +1271,7 @@ pub const NodeSuffixOp = struct { const CallInfo = struct { params: ArrayList(&Node), - is_async: bool, - allocator: ?&Node, + async_attr: ?&NodeAsyncAttribute, }; const SliceRange = struct { @@ -1347,6 +1399,45 @@ pub const NodeControlFlowExpression = struct { } }; +pub const NodeSuspend = struct { + base: Node, + suspend_token: Token, + payload: ?&NodePayload, + body: ?&Node, + + pub fn iterate(self: &NodeSuspend, index: usize) ?&Node { + var i = index; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (self.body) |body| { + if (i < 1) return body; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeSuspend) Token { + return self.suspend_token; + } + + pub fn lastToken(self: &NodeSuspend) Token { + if (self.body) |body| { + return body.lastToken(); + } + + if (self.payload) |payload| { + return payload.lastToken(); + } + + return self.suspend_token; + } +}; + pub const NodeIntegerLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index ff3b35c193..9d6d5de6d3 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -134,6 +134,11 @@ pub const Parser = struct { dest_ptr: DestPtr, }; + const AsyncEndCtx = struct { + dest_ptr: DestPtr, + attribute: &ast.NodeAsyncAttribute, + }; + const State = union(enum) { TopLevel, TopLevelExtern: TopLevelDeclCtx, @@ -173,9 +178,11 @@ pub const Parser = struct { FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - ErrorPayload: &?&ast.NodeErrorPayload, - ValuePayload: &?&ast.NodeValuePayload, - ValueIndexPayload: &?&ast.NodeValueIndexPayload, + SuspendBody: &ast.NodeSuspend, + AsyncEnd: AsyncEndCtx, + Payload: &?&ast.NodePayload, + PointerPayload: &?&ast.NodePointerPayload, + PointerIndexPayload: &?&ast.NodePointerIndexPayload, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -399,6 +406,45 @@ pub const Parser = struct { }); continue; }, + Token.Id.Keyword_async => { + // TODO shouldn't need this cast + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, + ctx.extern_token, ctx.lib_name, (?Token)(null), (?Token)(null), (?Token)(null)); + + const async_node = try arena.create(ast.NodeAsyncAttribute); + *async_node = ast.NodeAsyncAttribute { + .base = self.initNode(ast.Node.Id.AsyncAttribute), + .async_token = token, + .allocator_type = null, + .rangle_bracket = null, + }; + + fn_proto.async_attr = async_node; + stack.append(State { .FnDef = fn_proto }) catch unreachable; + try stack.append(State { .FnProto = fn_proto }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Keyword_fn, + .ptr = &fn_proto.fn_token, + } + }); + + const langle_bracket = self.getNextToken(); + if (langle_bracket.id != Token.Id.AngleBracketLeft) { + self.putBackToken(langle_bracket); + continue; + } + + async_node.rangle_bracket = Token(undefined); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.AngleBracketRight, + .ptr = &??async_node.rangle_bracket, + } + }); + try stack.append(State { .TypeExprBegin = DestPtr { .NullableField = &async_node.allocator_type } }); + continue; + }, else => { try self.parseError(&stack, token, "expected variable declaration or function, found {}", @tagName(token.id)); continue; @@ -711,13 +757,14 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_cancel => { - @panic("TODO: cancel"); + const cancel_node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp.Cancel); + dest_ptr.store(&cancel_node.base); + stack.append(State { .Expression = DestPtr { .Field = &cancel_node.rhs } }) catch unreachable; }, Token.Id.Keyword_resume => { - @panic("TODO: resume"); - }, - Token.Id.Keyword_await => { - @panic("TODO: await"); + const resume_node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp.Resume); + dest_ptr.store(&resume_node.base); + stack.append(State { .Expression = DestPtr { .Field = &resume_node.rhs } }) catch unreachable; }, else => { self.putBackToken(token); @@ -786,7 +833,7 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - try stack.append(State { .ErrorPayload = &node.op.Catch }); + try stack.append(State { .Payload = &node.op.Catch }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1113,6 +1160,9 @@ pub const Parser = struct { try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); } continue; + //Token.Id.Keyword_await => { + // @panic("TODO: await"); + //}, } else { self.putBackToken(token); stack.append(State { .SuffixOpExpressionBegin = dest_ptr }) catch unreachable; @@ -1124,7 +1174,38 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_async => { - @panic("TODO: Parse async"); + const async_node = try arena.create(ast.NodeAsyncAttribute); + *async_node = ast.NodeAsyncAttribute { + .base = self.initNode(ast.Node.Id.AsyncAttribute), + .async_token = token, + .allocator_type = null, + .rangle_bracket = null, + }; + + stack.append(State { + .AsyncEnd = AsyncEndCtx { + .dest_ptr = dest_ptr, + .attribute = async_node, + } + }) catch unreachable; + try stack.append(State { .SuffixOpExpressionEnd = dest_ptr }); + try stack.append(State { .PrimaryExpression = dest_ptr }); + + const langle_bracket = self.getNextToken(); + if (langle_bracket.id != Token.Id.AngleBracketLeft) { + self.putBackToken(langle_bracket); + continue; + } + + async_node.rangle_bracket = Token(undefined); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.AngleBracketRight, + .ptr = &??async_node.rangle_bracket, + } + }); + try stack.append(State { .TypeExprBegin = DestPtr { .NullableField = &async_node.allocator_type } }); + continue; }, else => { self.putBackToken(token); @@ -1142,8 +1223,7 @@ pub const Parser = struct { const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { .Call = ast.NodeSuffixOp.CallInfo { .params = ArrayList(&ast.Node).init(arena), - .is_async = false, // TODO: ASYNC - .allocator = null, + .async_attr = null, } }); node.lhs = dest_ptr.get(); @@ -1257,6 +1337,19 @@ pub const Parser = struct { dest_ptr.store(&node.base); continue; }, + Token.Id.Keyword_suspend => { + const node = try arena.create(ast.NodeSuspend); + *node = ast.NodeSuspend { + .base = self.initNode(ast.Node.Id.Suspend), + .suspend_token = token, + .payload = null, + .body = null, + }; + dest_ptr.store(&node.base); + stack.append(State { .SuspendBody = node }) catch unreachable; + try stack.append(State { .Payload = &node.payload }); + continue; + }, Token.Id.MultilineStringLiteralLine => { const node = try arena.create(ast.NodeMultilineStringLiteral); *node = ast.NodeMultilineStringLiteral { @@ -1477,17 +1570,12 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { + const fn_token = (try self.eatToken(&stack, Token.Id.Keyword_fn)) ?? continue; // TODO shouldn't need this cast - const fn_proto = try self.createFnProto(arena, undefined, + const fn_proto = try self.createFnProto(arena, fn_token, (?Token)(null), (?&ast.Node)(null), (?Token)(token), (?Token)(null), (?Token)(null)); dest_ptr.store(&fn_proto.base); stack.append(State { .FnProto = fn_proto }) catch unreachable; - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.Keyword_fn, - .ptr = &fn_proto.fn_token, - } - }); continue; }, Token.Id.Keyword_asm => { @@ -1544,7 +1632,7 @@ pub const Parser = struct { stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); - try stack.append(State { .ValuePayload = &node.payload }); + try stack.append(State { .PointerPayload = &node.payload }); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); try stack.append(State { .ExpectToken = Token.Id.LParen }); @@ -1816,7 +1904,7 @@ pub const Parser = struct { try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { .ValuePayload = &node.payload }); + try stack.append(State { .PointerPayload = &node.payload }); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1883,7 +1971,7 @@ pub const Parser = struct { *dest = node; stack.append(State { .Expression = DestPtr { .Field = &node.body } }) catch unreachable; - try stack.append(State { .ErrorPayload = &node.payload }); + try stack.append(State { .Payload = &node.payload }); }, State.WhileContinueExpr => |dest| { @@ -1898,7 +1986,41 @@ pub const Parser = struct { try stack.append(State { .AssignmentExpressionBegin = DestPtr { .NullableField = dest } }); }, - State.ErrorPayload => |dest| { + State.SuspendBody => |suspend_node| { + if (suspend_node.payload != null) { + try stack.append(State { .AssignmentExpressionBegin = DestPtr { .NullableField = &suspend_node.body } }); + } + continue; + }, + + State.AsyncEnd => |ctx| { + const node = ctx.dest_ptr.get(); + + switch (node.id) { + ast.Node.Id.FnProto => { + const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", node); + fn_proto.async_attr = ctx.attribute; + }, + ast.Node.Id.SuffixOp => { + const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", node); + if (suffix_op.op == ast.NodeSuffixOp.SuffixOp.Call) { + suffix_op.op.Call.async_attr = ctx.attribute; + continue; + } + + try self.parseError(&stack, node.firstToken(), "expected call or fn proto, found {}.", + @tagName(suffix_op.op)); + continue; + }, + else => { + try self.parseError(&stack, node.firstToken(), "expected call or fn proto, found {}.", + @tagName(node.id)); + continue; + } + } + }, + + State.Payload => |dest| { const lpipe = self.getNextToken(); if (lpipe.id != Token.Id.Pipe) { self.putBackToken(lpipe); @@ -1907,9 +2029,9 @@ pub const Parser = struct { const error_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodeErrorPayload); - *node = ast.NodeErrorPayload { - .base = self.initNode(ast.Node.Id.ErrorPayload), + const node = try arena.create(ast.NodePayload); + *node = ast.NodePayload { + .base = self.initNode(ast.Node.Id.Payload), .lpipe = lpipe, .error_symbol = try self.createIdentifier(arena, error_symbol), .rpipe = rpipe @@ -1917,7 +2039,7 @@ pub const Parser = struct { *dest = node; }, - State.ValuePayload => |dest| { + State.PointerPayload => |dest| { const lpipe = self.getNextToken(); if (lpipe.id != Token.Id.Pipe) { self.putBackToken(lpipe); @@ -1936,9 +2058,9 @@ pub const Parser = struct { const value_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodeValuePayload); - *node = ast.NodeValuePayload { - .base = self.initNode(ast.Node.Id.ValuePayload), + const node = try arena.create(ast.NodePointerPayload); + *node = ast.NodePointerPayload { + .base = self.initNode(ast.Node.Id.PointerPayload), .lpipe = lpipe, .is_ptr = is_ptr, .value_symbol = try self.createIdentifier(arena, value_symbol), @@ -1947,7 +2069,7 @@ pub const Parser = struct { *dest = node; }, - State.ValueIndexPayload => |dest| { + State.PointerIndexPayload => |dest| { const lpipe = self.getNextToken(); if (lpipe.id != Token.Id.Pipe) { self.putBackToken(lpipe); @@ -1977,9 +2099,9 @@ pub const Parser = struct { }; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodeValueIndexPayload); - *node = ast.NodeValueIndexPayload { - .base = self.initNode(ast.Node.Id.ValueIndexPayload), + const node = try arena.create(ast.NodePointerIndexPayload); + *node = ast.NodePointerIndexPayload { + .base = self.initNode(ast.Node.Id.PointerIndexPayload), .lpipe = lpipe, .is_ptr = is_ptr, .value_symbol = try self.createIdentifier(arena, value_symbol), @@ -2249,7 +2371,7 @@ pub const Parser = struct { stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); try stack.append(State { .WhileContinueExpr = &node.continue_expr }); - try stack.append(State { .ValuePayload = &node.payload }); + try stack.append(State { .PointerPayload = &node.payload }); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); try stack.append(State { .ExpectToken = Token.Id.LParen }); @@ -2271,7 +2393,7 @@ pub const Parser = struct { stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); - try stack.append(State { .ValueIndexPayload = &node.payload }); + try stack.append(State { .PointerIndexPayload = &node.payload }); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr { .Field = &node.array_expr } }); try stack.append(State { .ExpectToken = Token.Id.LParen }); @@ -2374,9 +2496,9 @@ pub const Parser = struct { ast.Node.Id.EnumTag, ast.Node.Id.ParamDecl, ast.Node.Id.Block, - ast.Node.Id.ErrorPayload, - ast.Node.Id.ValuePayload, - ast.Node.Id.ValueIndexPayload, + ast.Node.Id.Payload, + ast.Node.Id.PointerPayload, + ast.Node.Id.PointerIndexPayload, ast.Node.Id.Switch, ast.Node.Id.SwitchCase, ast.Node.Id.SwitchElse, @@ -2422,6 +2544,15 @@ pub const Parser = struct { const comptime_node = @fieldParentPtr(ast.NodeComptime, "base", n); n = comptime_node.expr; }, + ast.Node.Id.Suspend => { + const suspend_node = @fieldParentPtr(ast.NodeSuspend, "base", n); + if (suspend_node.body) |body| { + n = body; + continue; + } + + return true; + }, else => return true, } } @@ -2540,6 +2671,7 @@ pub const Parser = struct { }, Token.Id.QuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.MaybeType), Token.Id.QuestionMarkQuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.UnwrapMaybe), + Token.Id.Keyword_await => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.Await), else => null, }; } @@ -2628,6 +2760,7 @@ pub const Parser = struct { .extern_token = *extern_token, .inline_token = *inline_token, .cc_token = *cc_token, + .async_attr = null, .body_node = null, .lib_name = lib_name, .align_expr = null, @@ -3105,6 +3238,30 @@ pub const Parser = struct { try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_node.comptime_token)); try stack.append(RenderState { .Expression = comptime_node.expr }); }, + ast.Node.Id.AsyncAttribute => { + const async_attr = @fieldParentPtr(ast.NodeAsyncAttribute, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(async_attr.async_token)); + + if (async_attr.allocator_type) |allocator_type| { + try stack.append(RenderState { .Text = ">" }); + try stack.append(RenderState { .Expression = allocator_type }); + try stack.append(RenderState { .Text = "<" }); + } + }, + ast.Node.Id.Suspend => { + const suspend_node = @fieldParentPtr(ast.NodeSuspend, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(suspend_node.suspend_token)); + + if (suspend_node.body) |body| { + try stack.append(RenderState { .Expression = body }); + try stack.append(RenderState { .Text = " " }); + } + + if (suspend_node.payload) |payload| { + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = " " }); + } + }, ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); @@ -3211,6 +3368,9 @@ pub const Parser = struct { ast.NodePrefixOp.PrefixOp.Try => try stream.write("try "), ast.NodePrefixOp.PrefixOp.UnwrapMaybe => try stream.write("??"), ast.NodePrefixOp.PrefixOp.MaybeType => try stream.write("?"), + ast.NodePrefixOp.PrefixOp.Await => try stream.write("await "), + ast.NodePrefixOp.PrefixOp.Cancel => try stream.write("cancel "), + ast.NodePrefixOp.PrefixOp.Resume => try stream.write("resume "), } }, ast.Node.Id.SuffixOp => { @@ -3229,11 +3389,18 @@ pub const Parser = struct { } } try stack.append(RenderState { .Text = "("}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); + + if (call_info.async_attr) |async_attr| { + try stack.append(RenderState { .Text = " "}); + try stack.append(RenderState { .Expression = &async_attr.base }); + } }, ast.NodeSuffixOp.SuffixOp.ArrayAccess => |index_expr| { try stack.append(RenderState { .Text = "]"}); try stack.append(RenderState { .Expression = index_expr}); try stack.append(RenderState { .Text = "["}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); }, ast.NodeSuffixOp.SuffixOp.Slice => |range| { try stack.append(RenderState { .Text = "]"}); @@ -3243,6 +3410,7 @@ pub const Parser = struct { try stack.append(RenderState { .Text = ".."}); try stack.append(RenderState { .Expression = range.start}); try stack.append(RenderState { .Text = "["}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); }, ast.NodeSuffixOp.SuffixOp.StructInitializer => |field_inits| { try stack.append(RenderState { .Text = " }"}); @@ -3257,6 +3425,7 @@ pub const Parser = struct { } } try stack.append(RenderState { .Text = "{"}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); }, ast.NodeSuffixOp.SuffixOp.ArrayInitializer => |exprs| { try stack.append(RenderState { .Text = " }"}); @@ -3271,10 +3440,9 @@ pub const Parser = struct { } } try stack.append(RenderState { .Text = "{"}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); }, } - - try stack.append(RenderState { .Expression = suffix_op.lhs }); }, ast.Node.Id.ControlFlowExpression => { const flow_expr = @fieldParentPtr(ast.NodeControlFlowExpression, "base", base); @@ -3302,14 +3470,14 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = rhs }); } }, - ast.Node.Id.ErrorPayload => { - const payload = @fieldParentPtr(ast.NodeErrorPayload, "base", base); + ast.Node.Id.Payload => { + const payload = @fieldParentPtr(ast.NodePayload, "base", base); try stack.append(RenderState { .Text = "|"}); try stack.append(RenderState { .Expression = &payload.error_symbol.base }); try stack.append(RenderState { .Text = "|"}); }, - ast.Node.Id.ValuePayload => { - const payload = @fieldParentPtr(ast.NodeValuePayload, "base", base); + ast.Node.Id.PointerPayload => { + const payload = @fieldParentPtr(ast.NodePointerPayload, "base", base); try stack.append(RenderState { .Text = "|"}); try stack.append(RenderState { .Expression = &payload.value_symbol.base }); @@ -3319,8 +3487,8 @@ pub const Parser = struct { try stack.append(RenderState { .Text = "|"}); }, - ast.Node.Id.ValueIndexPayload => { - const payload = @fieldParentPtr(ast.NodeValueIndexPayload, "base", base); + ast.Node.Id.PointerIndexPayload => { + const payload = @fieldParentPtr(ast.NodePointerIndexPayload, "base", base); try stack.append(RenderState { .Text = "|"}); if (payload.index_symbol) |index_symbol| { @@ -3553,6 +3721,11 @@ pub const Parser = struct { try stack.append(RenderState { .Text = "fn" }); + if (fn_proto.async_attr) |async_attr| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = &async_attr.base }); + } + if (fn_proto.cc_token) |cc_token| { try stack.append(RenderState { .Text = " " }); try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(cc_token) }); @@ -4789,8 +4962,7 @@ test "zig fmt: coroutines" { \\ x += 1; \\ suspend; \\ x += 1; - \\ suspend |p| { - \\ } + \\ suspend |p| {} \\ const p = async simpleAsyncFn() catch unreachable; \\ await p; \\} @@ -4803,10 +4975,3 @@ test "zig fmt: coroutines" { \\ ); } - -test "zig fmt: zig fmt" { - try testCanonical(@embedFile("ast.zig")); - try testCanonical(@embedFile("index.zig")); - try testCanonical(@embedFile("parser.zig")); - try testCanonical(@embedFile("tokenizer.zig")); -} From 34af38e09b33957a9f677e42d57e9cd96f859b76 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 11:35:41 +0200 Subject: [PATCH 42/54] std.zig.tokinizer: fixed failing tests --- std/zig/tokenizer.zig | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 4cce31baeb..7b1f86712a 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -613,17 +613,25 @@ pub const Tokenizer = struct { '\\' => { state = State.CharLiteralBackslash; }, - '\'' => break, // Look for this error later. + '\'' => { + result.id = Token.Id.Invalid; + break; + }, else => { - if (c < 0x20 or c == 0x7f) - break; // Look for this error later. + if (c < 0x20 or c == 0x7f) { + result.id = Token.Id.Invalid; + break; + } state = State.CharLiteralEnd; } }, State.CharLiteralBackslash => switch (c) { - '\n' => break, // Look for this error later. + '\n' => { + result.id = Token.Id.Invalid; + break; + }, else => { state = State.CharLiteralEnd; }, @@ -635,7 +643,10 @@ pub const Tokenizer = struct { self.index += 1; break; }, - else => break, // Look for this error later. + else => { + result.id = Token.Id.Invalid; + break; + }, }, State.MultilineStringLiteralLine => switch (c) { @@ -903,7 +914,6 @@ pub const Tokenizer = struct { State.FloatExponentNumber, State.StringLiteral, // find this error later State.MultilineStringLiteralLine, - State.CharLiteralEnd, State.Builtin => {}, State.Identifier => { @@ -922,6 +932,7 @@ pub const Tokenizer = struct { State.MultilineStringLiteralLineBackslash, State.CharLiteral, State.CharLiteralBackslash, + State.CharLiteralEnd, State.StringLiteralBackslash => { result.id = Token.Id.Invalid; }, @@ -1073,7 +1084,7 @@ test "tokenizer - invalid token characters" { testTokenize("`", []Token.Id{Token.Id.Invalid}); testTokenize("'c", []Token.Id {Token.Id.Invalid}); testTokenize("'", []Token.Id {Token.Id.Invalid}); - testTokenize("''", []Token.Id {Token.Id.Invalid}); + testTokenize("''", []Token.Id {Token.Id.Invalid, Token.Id.Invalid}); } test "tokenizer - invalid literal/comment characters" { @@ -1147,6 +1158,7 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void { var tokenizer = Tokenizer.init(source); for (expected_tokens) |expected_token_id| { const token = tokenizer.next(); + std.debug.warn("{} {}\n", @tagName(expected_token_id), @tagName(token.id)); std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id)); switch (expected_token_id) { Token.Id.StringLiteral => |expected_kind| { From 1b81e406f0ca285738404120178b7b3824cf84bc Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 13:43:20 +0200 Subject: [PATCH 43/54] std.zig: fixed compiler errors --- std/zig/ast.zig | 23 +- std/zig/parser.zig | 1510 ++++++++++++++++++++++---------------------- 2 files changed, 768 insertions(+), 765 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 9043a66ac3..2128b9976f 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -297,7 +297,7 @@ pub const NodeErrorSetDecl = struct { pub fn iterate(self: &NodeErrorSetDecl, index: usize) ?&Node { var i = index; - if (i < self.decls.len) return self.decls.at(i); + if (i < self.decls.len) return &self.decls.at(i).base; i -= self.decls.len; return null; @@ -550,14 +550,6 @@ pub const NodeFnProto = struct { i -= 1; } - switch (self.call_convetion) { - CallConvetion.Async => |attr| { - if (i < 1) return &attr.base; - i -= 1; - }, - else => {}, - } - return null; } @@ -814,7 +806,7 @@ pub const NodeSwitch = struct { if (i < 1) return self.expr; i -= 1; - if (i < self.cases.len) return self.cases.at(i); + if (i < self.cases.len) return &self.cases.at(i).base; i -= self.cases.len; return null; @@ -842,7 +834,7 @@ pub const NodeSwitchCase = struct { i -= self.items.len; if (self.payload) |payload| { - if (i < 1) return payload; + if (i < 1) return &payload.base; i -= 1; } @@ -1093,6 +1085,13 @@ pub const NodeInfixOp = struct { i -= 1; switch (self.op) { + InfixOp.Catch => |maybe_payload| { + if (maybe_payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + }, + InfixOp.Add, InfixOp.AddWrap, InfixOp.ArrayCat, @@ -1208,9 +1207,9 @@ pub const NodePrefixOp = struct { PrefixOp.BoolNot, PrefixOp.Cancel, PrefixOp.Deref, + PrefixOp.MaybeType, PrefixOp.Negation, PrefixOp.NegationWrap, - PrefixOp.Return, PrefixOp.Try, PrefixOp.Resume, PrefixOp.UnwrapMaybe => {}, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 9d6d5de6d3..33408bcc31 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1421,7 +1421,7 @@ pub const Parser = struct { } }); dest_ptr.store(&node.base); - stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; + stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; try stack.append(State { .AddrOfModifiers = &node.op.SliceType }); continue; } @@ -1432,7 +1432,7 @@ pub const Parser = struct { .ArrayType = undefined, }); dest_ptr.store(&node.base); - stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; + stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.RBracket }); try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayType } }); @@ -4222,756 +4222,760 @@ fn testCanonical(source: []const u8) !void { } } -test "zig fmt: get stdout or fail" { - try testCanonical( - \\const std = @import("std"); - \\ - \\pub fn main() !void { - \\ // If this program is run without stdout attached, exit with an error. - \\ // another comment - \\ var stdout_file = try std.io.getStdOut; - \\} - \\ - ); -} +//test "zig fmt: get stdout or fail" { +// try testCanonical( +// \\const std = @import("std"); +// \\ +// \\pub fn main() !void { +// \\ // If this program is run without stdout attached, exit with an error. +// \\ // another comment +// \\ var stdout_file = try std.io.getStdOut; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: preserve spacing" { +// try testCanonical( +// \\const std = @import("std"); +// \\ +// \\pub fn main() !void { +// \\ var stdout_file = try std.io.getStdOut; +// \\ var stdout_file = try std.io.getStdOut; +// \\ +// \\ var stdout_file = try std.io.getStdOut; +// \\ var stdout_file = try std.io.getStdOut; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: return types" { +// try testCanonical( +// \\pub fn main() !void {} +// \\pub fn main() var {} +// \\pub fn main() i32 {} +// \\ +// ); +//} +// +//test "zig fmt: imports" { +// try testCanonical( +// \\const std = @import("std"); +// \\const std = @import(); +// \\ +// ); +//} +// +//test "zig fmt: global declarations" { +// try testCanonical( +// \\const a = b; +// \\pub const a = b; +// \\var a = b; +// \\pub var a = b; +// \\const a: i32 = b; +// \\pub const a: i32 = b; +// \\var a: i32 = b; +// \\pub var a: i32 = b; +// \\extern const a: i32 = b; +// \\pub extern const a: i32 = b; +// \\extern var a: i32 = b; +// \\pub extern var a: i32 = b; +// \\extern "a" const a: i32 = b; +// \\pub extern "a" const a: i32 = b; +// \\extern "a" var a: i32 = b; +// \\pub extern "a" var a: i32 = b; +// \\ +// ); +//} +// +//test "zig fmt: extern declaration" { +// try testCanonical( +// \\extern var foo: c_int; +// \\ +// ); +//} +// +//test "zig fmt: alignment" { +// try testCanonical( +// \\var foo: c_int align(1); +// \\ +// ); +//} +// +//test "zig fmt: C main" { +// try testCanonical( +// \\fn main(argc: c_int, argv: &&u8) c_int { +// \\ const a = b; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: return" { +// try testCanonical( +// \\fn foo(argc: c_int, argv: &&u8) c_int { +// \\ return 0; +// \\} +// \\ +// \\fn bar() void { +// \\ return; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: pointer attributes" { +// try testCanonical( +// \\extern fn f1(s: &align(&u8) u8) c_int; +// \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; +// \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; +// \\extern fn f4(s: &align(1) const volatile u8) c_int; +// \\ +// ); +//} +// +//test "zig fmt: slice attributes" { +// try testCanonical( +// \\extern fn f1(s: &align(&u8) u8) c_int; +// \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; +// \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; +// \\extern fn f4(s: &align(1) const volatile u8) c_int; +// \\ +// ); +//} +// +//test "zig fmt: test declaration" { +// try testCanonical( +// \\test "test name" { +// \\ const a = 1; +// \\ var b = 1; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: infix operators" { +// try testCanonical( +// \\test "infix operators" { +// \\ var i = undefined; +// \\ i = 2; +// \\ i *= 2; +// \\ i |= 2; +// \\ i ^= 2; +// \\ i <<= 2; +// \\ i >>= 2; +// \\ i &= 2; +// \\ i *= 2; +// \\ i *%= 2; +// \\ i -= 2; +// \\ i -%= 2; +// \\ i += 2; +// \\ i +%= 2; +// \\ i /= 2; +// \\ i %= 2; +// \\ _ = i == i; +// \\ _ = i != i; +// \\ _ = i != i; +// \\ _ = i.i; +// \\ _ = i || i; +// \\ _ = i!i; +// \\ _ = i ** i; +// \\ _ = i ++ i; +// \\ _ = i ?? i; +// \\ _ = i % i; +// \\ _ = i / i; +// \\ _ = i *% i; +// \\ _ = i * i; +// \\ _ = i -% i; +// \\ _ = i - i; +// \\ _ = i +% i; +// \\ _ = i + i; +// \\ _ = i << i; +// \\ _ = i >> i; +// \\ _ = i & i; +// \\ _ = i ^ i; +// \\ _ = i | i; +// \\ _ = i >= i; +// \\ _ = i <= i; +// \\ _ = i > i; +// \\ _ = i < i; +// \\ _ = i and i; +// \\ _ = i or i; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: precedence" { +// try testCanonical( +// \\test "precedence" { +// \\ a!b(); +// \\ (a!b)(); +// \\ !a!b; +// \\ !(a!b); +// \\ !a{ }; +// \\ !(a{ }); +// \\ a + b{ }; +// \\ (a + b){ }; +// \\ a << b + c; +// \\ (a << b) + c; +// \\ a & b << c; +// \\ (a & b) << c; +// \\ a ^ b & c; +// \\ (a ^ b) & c; +// \\ a | b ^ c; +// \\ (a | b) ^ c; +// \\ a == b | c; +// \\ (a == b) | c; +// \\ a and b == c; +// \\ (a and b) == c; +// \\ a or b and c; +// \\ (a or b) and c; +// \\ (a or b) and c; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: prefix operators" { +// try testCanonical( +// \\test "prefix operators" { +// \\ try return --%~??!*&0; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: call expression" { +// try testCanonical( +// \\test "test calls" { +// \\ a(); +// \\ a(1); +// \\ a(1, 2); +// \\ a(1, 2) + a(1, 2); +// \\} +// \\ +// ); +//} +// +//test "zig fmt: var args" { +// try testCanonical( +// \\fn print(args: ...) void {} +// \\ +// ); +//} +// +//test "zig fmt: extern function" { +// try testCanonical( +// \\extern fn puts(s: &const u8) c_int; +// \\extern "c" fn puts(s: &const u8) c_int; +// \\ +// ); +//} +// +//test "zig fmt: multiline string" { +// try testCanonical( +// \\const s = +// \\ \\ something +// \\ \\ something else +// \\ ; +// \\ +// ); +//} +// +//test "zig fmt: values" { +// try testCanonical( +// \\test "values" { +// \\ 1; +// \\ 1.0; +// \\ "string"; +// \\ c"cstring"; +// \\ 'c'; +// \\ true; +// \\ false; +// \\ null; +// \\ undefined; +// \\ error; +// \\ this; +// \\ unreachable; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: indexing" { +// try testCanonical( +// \\test "test index" { +// \\ a[0]; +// \\ a[0 + 5]; +// \\ a[0..]; +// \\ a[0..5]; +// \\ a[a[0]]; +// \\ a[a[0..]]; +// \\ a[a[0..5]]; +// \\ a[a[0]..]; +// \\ a[a[0..5]..]; +// \\ a[a[0]..a[0]]; +// \\ a[a[0..5]..a[0]]; +// \\ a[a[0..5]..a[0..5]]; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: struct declaration" { +// try testCanonical( +// \\const S = struct { +// \\ const Self = this; +// \\ f1: u8, +// \\ +// \\ fn method(self: &Self) Self { +// \\ return *self; +// \\ } +// \\ +// \\ f2: u8 +// \\}; +// \\ +// \\const Ps = packed struct { +// \\ a: u8, +// \\ b: u8, +// \\ +// \\ c: u8 +// \\}; +// \\ +// \\const Es = extern struct { +// \\ a: u8, +// \\ b: u8, +// \\ +// \\ c: u8 +// \\}; +// \\ +// ); +//} +// +//test "zig fmt: enum declaration" { +// try testCanonical( +// \\const E = enum { +// \\ Ok, +// \\ SomethingElse = 0 +// \\}; +// \\ +// \\const E2 = enum(u8) { +// \\ Ok, +// \\ SomethingElse = 255, +// \\ SomethingThird +// \\}; +// \\ +// \\const Ee = extern enum { +// \\ Ok, +// \\ SomethingElse, +// \\ SomethingThird +// \\}; +// \\ +// \\const Ep = packed enum { +// \\ Ok, +// \\ SomethingElse, +// \\ SomethingThird +// \\}; +// \\ +// ); +//} +// +//test "zig fmt: union declaration" { +// try testCanonical( +// \\const U = union { +// \\ Int: u8, +// \\ Float: f32, +// \\ None, +// \\ Bool: bool +// \\}; +// \\ +// \\const Ue = union(enum) { +// \\ Int: u8, +// \\ Float: f32, +// \\ None, +// \\ Bool: bool +// \\}; +// \\ +// \\const E = enum { +// \\ Int, +// \\ Float, +// \\ None, +// \\ Bool +// \\}; +// \\ +// \\const Ue2 = union(E) { +// \\ Int: u8, +// \\ Float: f32, +// \\ None, +// \\ Bool: bool +// \\}; +// \\ +// \\const Eu = extern union { +// \\ Int: u8, +// \\ Float: f32, +// \\ None, +// \\ Bool: bool +// \\}; +// \\ +// ); +//} +// +//test "zig fmt: error set declaration" { +// try testCanonical( +// \\const E = error { +// \\ A, +// \\ B, +// \\ +// \\ C +// \\}; +// \\ +// ); +//} +// +//test "zig fmt: arrays" { +// try testCanonical( +// \\test "test array" { +// \\ const a: [2]u8 = [2]u8{ 1, 2 }; +// \\ const a: [2]u8 = []u8{ 1, 2 }; +// \\ const a: [0]u8 = []u8{ }; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: container initializers" { +// try testCanonical( +// \\const a1 = []u8{ }; +// \\const a2 = []u8{ 1, 2, 3, 4 }; +// \\const s1 = S{ }; +// \\const s2 = S{ .a = 1, .b = 2 }; +// \\ +// ); +//} +// +//test "zig fmt: catch" { +// try testCanonical( +// \\test "catch" { +// \\ const a: error!u8 = 0; +// \\ _ = a catch return; +// \\ _ = a catch |err| return; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: blocks" { +// try testCanonical( +// \\test "blocks" { +// \\ { +// \\ const a = 0; +// \\ const b = 0; +// \\ } +// \\ +// \\ blk: { +// \\ const a = 0; +// \\ const b = 0; +// \\ } +// \\ +// \\ const r = blk: { +// \\ const a = 0; +// \\ const b = 0; +// \\ }; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: switch" { +// try testCanonical( +// \\test "switch" { +// \\ switch (0) { +// \\ 0 => {}, +// \\ 1 => unreachable, +// \\ 2, 3 => {}, +// \\ 4 ... 7 => {}, +// \\ 1 + 4 * 3 + 22 => {}, +// \\ else => { +// \\ const a = 1; +// \\ const b = a; +// \\ } +// \\ } +// \\ +// \\ const res = switch (0) { +// \\ 0 => 0, +// \\ 1 => 2, +// \\ else => 4 +// \\ }; +// \\ +// \\ const Union = union(enum) { +// \\ Int: i64, +// \\ Float: f64 +// \\ }; +// \\ +// \\ const u = Union{ .Int = 0 }; +// \\ switch (u) { +// \\ Union.Int => |int| {}, +// \\ Union.Float => |*float| unreachable +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: while" { +// try testCanonical( +// \\test "while" { +// \\ while (10 < 1) { +// \\ unreachable; +// \\ } +// \\ +// \\ while (10 < 1) +// \\ unreachable; +// \\ +// \\ var i: usize = 0; +// \\ while (i < 10) : (i += 1) { +// \\ continue; +// \\ } +// \\ +// \\ i = 0; +// \\ while (i < 10) : (i += 1) +// \\ continue; +// \\ +// \\ i = 0; +// \\ var j: usize = 0; +// \\ while (i < 10) : ({ +// \\ i += 1; +// \\ j += 1; +// \\ }) { +// \\ continue; +// \\ } +// \\ +// \\ var a: ?u8 = 2; +// \\ while (a) |v| : (a = null) { +// \\ continue; +// \\ } +// \\ +// \\ while (a) |v| : (a = null) +// \\ unreachable; +// \\ +// \\ label: while (10 < 0) { +// \\ unreachable; +// \\ } +// \\ +// \\ const res = while (0 < 10) { +// \\ break 7; +// \\ } else { +// \\ unreachable; +// \\ }; +// \\ +// \\ var a: error!u8 = 0; +// \\ while (a) |v| { +// \\ a = error.Err; +// \\ } else |err| { +// \\ i = 1; +// \\ } +// \\ +// \\ comptime var k: usize = 0; +// \\ inline while (i < 10) : (i += 1) +// \\ j += 2; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: for" { +// try testCanonical( +// \\test "for" { +// \\ const a = []u8{ 1, 2, 3 }; +// \\ for (a) |v| { +// \\ continue; +// \\ } +// \\ +// \\ for (a) |v| +// \\ continue; +// \\ +// \\ for (a) |*v| +// \\ continue; +// \\ +// \\ for (a) |v, i| { +// \\ continue; +// \\ } +// \\ +// \\ for (a) |v, i| +// \\ continue; +// \\ +// \\ const res = for (a) |v, i| { +// \\ break v; +// \\ } else { +// \\ unreachable; +// \\ }; +// \\ +// \\ var num: usize = 0; +// \\ inline for (a) |v, i| { +// \\ num += v; +// \\ num += i; +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: if" { +// try testCanonical( +// \\test "if" { +// \\ if (10 < 0) { +// \\ unreachable; +// \\ } +// \\ +// \\ if (10 < 0) unreachable; +// \\ +// \\ if (10 < 0) { +// \\ unreachable; +// \\ } else { +// \\ const a = 20; +// \\ } +// \\ +// \\ if (10 < 0) { +// \\ unreachable; +// \\ } else if (5 < 0) { +// \\ unreachable; +// \\ } else { +// \\ const a = 20; +// \\ } +// \\ +// \\ const is_world_broken = if (10 < 0) true else false; +// \\ +// \\ const a: ?u8 = 10; +// \\ const b: ?u8 = null; +// \\ if (a) |v| { +// \\ const some = v; +// \\ } else if (b) |*v| { +// \\ unreachable; +// \\ } else { +// \\ const some = 10; +// \\ } +// \\ +// \\ const non_null_a = if (a) |v| v else 0; +// \\ +// \\ const a_err: error!u8 = 0; +// \\ if (a_err) |v| { +// \\ const p = v; +// \\ } else |err| { +// \\ unreachable; +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: defer" { +// try testCanonical( +// \\test "defer" { +// \\ var i: usize = 0; +// \\ defer i = 1; +// \\ defer { +// \\ i += 2; +// \\ i *= i; +// \\ } +// \\ +// \\ errdefer i += 3; +// \\ errdefer { +// \\ i += 2; +// \\ i /= i; +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: comptime" { +// try testCanonical( +// \\fn a() u8 { +// \\ return 5; +// \\} +// \\ +// \\fn b(comptime i: u8) u8 { +// \\ return i; +// \\} +// \\ +// \\const av = comptime a(); +// \\const av2 = comptime blk: { +// \\ var res = a(); +// \\ res *= b(2); +// \\ break :blk res; +// \\}; +// \\ +// \\comptime { +// \\ _ = a(); +// \\} +// \\ +// \\test "comptime" { +// \\ const av3 = comptime a(); +// \\ const av4 = comptime blk: { +// \\ var res = a(); +// \\ res *= a(); +// \\ break :blk res; +// \\ }; +// \\ +// \\ comptime var i = 0; +// \\ comptime { +// \\ i = a(); +// \\ i += b(i); +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: fn type" { +// try testCanonical( +// \\fn a(i: u8) u8 { +// \\ return i + 1; +// \\} +// \\ +// \\const a: fn(u8) u8 = undefined; +// \\const b: extern fn(u8) u8 = undefined; +// \\const c: nakedcc fn(u8) u8 = undefined; +// \\const ap: fn(u8) u8 = a; +// \\ +// ); +//} +// +//test "zig fmt: inline asm" { +// try testCanonical( +// \\pub fn syscall1(number: usize, arg1: usize) usize { +// \\ return asm volatile ("syscall" +// \\ : [ret] "={rax}" (-> usize) +// \\ : [number] "{rax}" (number), +// \\ [arg1] "{rdi}" (arg1) +// \\ : "rcx", "r11"); +// \\} +// \\ +// ); +//} +// +//test "zig fmt: coroutines" { +// try testCanonical( +// \\async fn simpleAsyncFn() void { +// \\ x += 1; +// \\ suspend; +// \\ x += 1; +// \\ suspend |p| {} +// \\ const p = async simpleAsyncFn() catch unreachable; +// \\ await p; +// \\} +// \\ +// \\test "coroutine suspend, resume, cancel" { +// \\ const p = try async testAsyncSeq(); +// \\ resume p; +// \\ cancel p; +// \\} +// \\ +// ); +//} -test "zig fmt: preserve spacing" { - try testCanonical( - \\const std = @import("std"); - \\ - \\pub fn main() !void { - \\ var stdout_file = try std.io.getStdOut; - \\ var stdout_file = try std.io.getStdOut; - \\ - \\ var stdout_file = try std.io.getStdOut; - \\ var stdout_file = try std.io.getStdOut; - \\} - \\ - ); -} - -test "zig fmt: return types" { - try testCanonical( - \\pub fn main() !void {} - \\pub fn main() var {} - \\pub fn main() i32 {} - \\ - ); -} - -test "zig fmt: imports" { - try testCanonical( - \\const std = @import("std"); - \\const std = @import(); - \\ - ); -} - -test "zig fmt: global declarations" { - try testCanonical( - \\const a = b; - \\pub const a = b; - \\var a = b; - \\pub var a = b; - \\const a: i32 = b; - \\pub const a: i32 = b; - \\var a: i32 = b; - \\pub var a: i32 = b; - \\extern const a: i32 = b; - \\pub extern const a: i32 = b; - \\extern var a: i32 = b; - \\pub extern var a: i32 = b; - \\extern "a" const a: i32 = b; - \\pub extern "a" const a: i32 = b; - \\extern "a" var a: i32 = b; - \\pub extern "a" var a: i32 = b; - \\ - ); -} - -test "zig fmt: extern declaration" { - try testCanonical( - \\extern var foo: c_int; - \\ - ); -} - -test "zig fmt: alignment" { - try testCanonical( - \\var foo: c_int align(1); - \\ - ); -} - -test "zig fmt: C main" { - try testCanonical( - \\fn main(argc: c_int, argv: &&u8) c_int { - \\ const a = b; - \\} - \\ - ); -} - -test "zig fmt: return" { - try testCanonical( - \\fn foo(argc: c_int, argv: &&u8) c_int { - \\ return 0; - \\} - \\ - \\fn bar() void { - \\ return; - \\} - \\ - ); -} - -test "zig fmt: pointer attributes" { - try testCanonical( - \\extern fn f1(s: &align(&u8) u8) c_int; - \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; - \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; - \\extern fn f4(s: &align(1) const volatile u8) c_int; - \\ - ); -} - -test "zig fmt: slice attributes" { - try testCanonical( - \\extern fn f1(s: &align(&u8) u8) c_int; - \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; - \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; - \\extern fn f4(s: &align(1) const volatile u8) c_int; - \\ - ); -} - -test "zig fmt: test declaration" { - try testCanonical( - \\test "test name" { - \\ const a = 1; - \\ var b = 1; - \\} - \\ - ); -} - -test "zig fmt: infix operators" { - try testCanonical( - \\test "infix operators" { - \\ var i = undefined; - \\ i = 2; - \\ i *= 2; - \\ i |= 2; - \\ i ^= 2; - \\ i <<= 2; - \\ i >>= 2; - \\ i &= 2; - \\ i *= 2; - \\ i *%= 2; - \\ i -= 2; - \\ i -%= 2; - \\ i += 2; - \\ i +%= 2; - \\ i /= 2; - \\ i %= 2; - \\ _ = i == i; - \\ _ = i != i; - \\ _ = i != i; - \\ _ = i.i; - \\ _ = i || i; - \\ _ = i!i; - \\ _ = i ** i; - \\ _ = i ++ i; - \\ _ = i ?? i; - \\ _ = i % i; - \\ _ = i / i; - \\ _ = i *% i; - \\ _ = i * i; - \\ _ = i -% i; - \\ _ = i - i; - \\ _ = i +% i; - \\ _ = i + i; - \\ _ = i << i; - \\ _ = i >> i; - \\ _ = i & i; - \\ _ = i ^ i; - \\ _ = i | i; - \\ _ = i >= i; - \\ _ = i <= i; - \\ _ = i > i; - \\ _ = i < i; - \\ _ = i and i; - \\ _ = i or i; - \\} - \\ - ); -} - -test "zig fmt: precedence" { - try testCanonical( - \\test "precedence" { - \\ a!b(); - \\ (a!b)(); - \\ !a!b; - \\ !(a!b); - \\ !a{ }; - \\ !(a{ }); - \\ a + b{ }; - \\ (a + b){ }; - \\ a << b + c; - \\ (a << b) + c; - \\ a & b << c; - \\ (a & b) << c; - \\ a ^ b & c; - \\ (a ^ b) & c; - \\ a | b ^ c; - \\ (a | b) ^ c; - \\ a == b | c; - \\ (a == b) | c; - \\ a and b == c; - \\ (a and b) == c; - \\ a or b and c; - \\ (a or b) and c; - \\ (a or b) and c; - \\} - \\ - ); -} - -test "zig fmt: prefix operators" { - try testCanonical( - \\test "prefix operators" { - \\ try return --%~??!*&0; - \\} - \\ - ); -} - -test "zig fmt: call expression" { - try testCanonical( - \\test "test calls" { - \\ a(); - \\ a(1); - \\ a(1, 2); - \\ a(1, 2) + a(1, 2); - \\} - \\ - ); -} - -test "zig fmt: var args" { - try testCanonical( - \\fn print(args: ...) void {} - \\ - ); -} - -test "zig fmt: extern function" { - try testCanonical( - \\extern fn puts(s: &const u8) c_int; - \\extern "c" fn puts(s: &const u8) c_int; - \\ - ); -} - -test "zig fmt: multiline string" { - try testCanonical( - \\const s = - \\ \\ something - \\ \\ something else - \\ ; - \\ - ); -} - -test "zig fmt: values" { - try testCanonical( - \\test "values" { - \\ 1; - \\ 1.0; - \\ "string"; - \\ c"cstring"; - \\ 'c'; - \\ true; - \\ false; - \\ null; - \\ undefined; - \\ error; - \\ this; - \\ unreachable; - \\} - \\ - ); -} - -test "zig fmt: indexing" { - try testCanonical( - \\test "test index" { - \\ a[0]; - \\ a[0 + 5]; - \\ a[0..]; - \\ a[0..5]; - \\ a[a[0]]; - \\ a[a[0..]]; - \\ a[a[0..5]]; - \\ a[a[0]..]; - \\ a[a[0..5]..]; - \\ a[a[0]..a[0]]; - \\ a[a[0..5]..a[0]]; - \\ a[a[0..5]..a[0..5]]; - \\} - \\ - ); -} - -test "zig fmt: struct declaration" { - try testCanonical( - \\const S = struct { - \\ const Self = this; - \\ f1: u8, - \\ - \\ fn method(self: &Self) Self { - \\ return *self; - \\ } - \\ - \\ f2: u8 - \\}; - \\ - \\const Ps = packed struct { - \\ a: u8, - \\ b: u8, - \\ - \\ c: u8 - \\}; - \\ - \\const Es = extern struct { - \\ a: u8, - \\ b: u8, - \\ - \\ c: u8 - \\}; - \\ - ); -} - -test "zig fmt: enum declaration" { - try testCanonical( - \\const E = enum { - \\ Ok, - \\ SomethingElse = 0 - \\}; - \\ - \\const E2 = enum(u8) { - \\ Ok, - \\ SomethingElse = 255, - \\ SomethingThird - \\}; - \\ - \\const Ee = extern enum { - \\ Ok, - \\ SomethingElse, - \\ SomethingThird - \\}; - \\ - \\const Ep = packed enum { - \\ Ok, - \\ SomethingElse, - \\ SomethingThird - \\}; - \\ - ); -} - -test "zig fmt: union declaration" { - try testCanonical( - \\const U = union { - \\ Int: u8, - \\ Float: f32, - \\ None, - \\ Bool: bool - \\}; - \\ - \\const Ue = union(enum) { - \\ Int: u8, - \\ Float: f32, - \\ None, - \\ Bool: bool - \\}; - \\ - \\const E = enum { - \\ Int, - \\ Float, - \\ None, - \\ Bool - \\}; - \\ - \\const Ue2 = union(E) { - \\ Int: u8, - \\ Float: f32, - \\ None, - \\ Bool: bool - \\}; - \\ - \\const Eu = extern union { - \\ Int: u8, - \\ Float: f32, - \\ None, - \\ Bool: bool - \\}; - \\ - ); -} - -test "zig fmt: error set declaration" { - try testCanonical( - \\const E = error { - \\ A, - \\ B, - \\ - \\ C - \\}; - \\ - ); -} - -test "zig fmt: arrays" { - try testCanonical( - \\test "test array" { - \\ const a: [2]u8 = [2]u8{ 1, 2 }; - \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{ }; - \\} - \\ - ); -} - -test "zig fmt: container initializers" { - try testCanonical( - \\const a1 = []u8{ }; - \\const a2 = []u8{ 1, 2, 3, 4 }; - \\const s1 = S{ }; - \\const s2 = S{ .a = 1, .b = 2 }; - \\ - ); -} - -test "zig fmt: catch" { - try testCanonical( - \\test "catch" { - \\ const a: error!u8 = 0; - \\ _ = a catch return; - \\ _ = a catch |err| return; - \\} - \\ - ); -} - -test "zig fmt: blocks" { - try testCanonical( - \\test "blocks" { - \\ { - \\ const a = 0; - \\ const b = 0; - \\ } - \\ - \\ blk: { - \\ const a = 0; - \\ const b = 0; - \\ } - \\ - \\ const r = blk: { - \\ const a = 0; - \\ const b = 0; - \\ }; - \\} - \\ - ); -} - -test "zig fmt: switch" { - try testCanonical( - \\test "switch" { - \\ switch (0) { - \\ 0 => {}, - \\ 1 => unreachable, - \\ 2, 3 => {}, - \\ 4 ... 7 => {}, - \\ 1 + 4 * 3 + 22 => {}, - \\ else => { - \\ const a = 1; - \\ const b = a; - \\ } - \\ } - \\ - \\ const res = switch (0) { - \\ 0 => 0, - \\ 1 => 2, - \\ else => 4 - \\ }; - \\ - \\ const Union = union(enum) { - \\ Int: i64, - \\ Float: f64 - \\ }; - \\ - \\ const u = Union{ .Int = 0 }; - \\ switch (u) { - \\ Union.Int => |int| {}, - \\ Union.Float => |*float| unreachable - \\ } - \\} - \\ - ); -} - -test "zig fmt: while" { - try testCanonical( - \\test "while" { - \\ while (10 < 1) { - \\ unreachable; - \\ } - \\ - \\ while (10 < 1) - \\ unreachable; - \\ - \\ var i: usize = 0; - \\ while (i < 10) : (i += 1) { - \\ continue; - \\ } - \\ - \\ i = 0; - \\ while (i < 10) : (i += 1) - \\ continue; - \\ - \\ i = 0; - \\ var j: usize = 0; - \\ while (i < 10) : ({ - \\ i += 1; - \\ j += 1; - \\ }) { - \\ continue; - \\ } - \\ - \\ var a: ?u8 = 2; - \\ while (a) |v| : (a = null) { - \\ continue; - \\ } - \\ - \\ while (a) |v| : (a = null) - \\ unreachable; - \\ - \\ label: while (10 < 0) { - \\ unreachable; - \\ } - \\ - \\ const res = while (0 < 10) { - \\ break 7; - \\ } else { - \\ unreachable; - \\ }; - \\ - \\ var a: error!u8 = 0; - \\ while (a) |v| { - \\ a = error.Err; - \\ } else |err| { - \\ i = 1; - \\ } - \\ - \\ comptime var k: usize = 0; - \\ inline while (i < 10) : (i += 1) - \\ j += 2; - \\} - \\ - ); -} - -test "zig fmt: for" { - try testCanonical( - \\test "for" { - \\ const a = []u8{ 1, 2, 3 }; - \\ for (a) |v| { - \\ continue; - \\ } - \\ - \\ for (a) |v| - \\ continue; - \\ - \\ for (a) |*v| - \\ continue; - \\ - \\ for (a) |v, i| { - \\ continue; - \\ } - \\ - \\ for (a) |v, i| - \\ continue; - \\ - \\ const res = for (a) |v, i| { - \\ break v; - \\ } else { - \\ unreachable; - \\ }; - \\ - \\ var num: usize = 0; - \\ inline for (a) |v, i| { - \\ num += v; - \\ num += i; - \\ } - \\} - \\ - ); -} - -test "zig fmt: if" { - try testCanonical( - \\test "if" { - \\ if (10 < 0) { - \\ unreachable; - \\ } - \\ - \\ if (10 < 0) unreachable; - \\ - \\ if (10 < 0) { - \\ unreachable; - \\ } else { - \\ const a = 20; - \\ } - \\ - \\ if (10 < 0) { - \\ unreachable; - \\ } else if (5 < 0) { - \\ unreachable; - \\ } else { - \\ const a = 20; - \\ } - \\ - \\ const is_world_broken = if (10 < 0) true else false; - \\ - \\ const a: ?u8 = 10; - \\ const b: ?u8 = null; - \\ if (a) |v| { - \\ const some = v; - \\ } else if (b) |*v| { - \\ unreachable; - \\ } else { - \\ const some = 10; - \\ } - \\ - \\ const non_null_a = if (a) |v| v else 0; - \\ - \\ const a_err: error!u8 = 0; - \\ if (a_err) |v| { - \\ const p = v; - \\ } else |err| { - \\ unreachable; - \\ } - \\} - \\ - ); -} - -test "zig fmt: defer" { - try testCanonical( - \\test "defer" { - \\ var i: usize = 0; - \\ defer i = 1; - \\ defer { - \\ i += 2; - \\ i *= i; - \\ } - \\ - \\ errdefer i += 3; - \\ errdefer { - \\ i += 2; - \\ i /= i; - \\ } - \\} - \\ - ); -} - -test "zig fmt: comptime" { - try testCanonical( - \\fn a() u8 { - \\ return 5; - \\} - \\ - \\fn b(comptime i: u8) u8 { - \\ return i; - \\} - \\ - \\const av = comptime a(); - \\const av2 = comptime blk: { - \\ var res = a(); - \\ res *= b(2); - \\ break :blk res; - \\}; - \\ - \\comptime { - \\ _ = a(); - \\} - \\ - \\test "comptime" { - \\ const av3 = comptime a(); - \\ const av4 = comptime blk: { - \\ var res = a(); - \\ res *= a(); - \\ break :blk res; - \\ }; - \\ - \\ comptime var i = 0; - \\ comptime { - \\ i = a(); - \\ i += b(i); - \\ } - \\} - \\ - ); -} - -test "zig fmt: fn type" { - try testCanonical( - \\fn a(i: u8) u8 { - \\ return i + 1; - \\} - \\ - \\const a: fn(u8) u8 = undefined; - \\const b: extern fn(u8) u8 = undefined; - \\const c: nakedcc fn(u8) u8 = undefined; - \\const ap: fn(u8) u8 = a; - \\ - ); -} - -test "zig fmt: inline asm" { - try testCanonical( - \\pub fn syscall1(number: usize, arg1: usize) usize { - \\ return asm volatile ("syscall" - \\ : [ret] "={rax}" (-> usize) - \\ : [number] "{rax}" (number), - \\ [arg1] "{rdi}" (arg1) - \\ : "rcx", "r11"); - \\} - \\ - ); -} - -test "zig fmt: coroutines" { - try testCanonical( - \\async fn simpleAsyncFn() void { - \\ x += 1; - \\ suspend; - \\ x += 1; - \\ suspend |p| {} - \\ const p = async simpleAsyncFn() catch unreachable; - \\ await p; - \\} - \\ - \\test "coroutine suspend, resume, cancel" { - \\ const p = try async testAsyncSeq(); - \\ resume p; - \\ cancel p; - \\} - \\ - ); +test "1" { + try testCanonical(@embedFile("../array_list.zig")); } From 706e0d739e5bb4a6e7a0e4f6e168eb71069e4df2 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 13:49:52 +0200 Subject: [PATCH 44/54] std.zig.parser readded all tests * Ops! --- std/zig/parser.zig | 1527 ++++++++++++++++++++++---------------------- 1 file changed, 761 insertions(+), 766 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 33408bcc31..5f7412b5c2 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1119,12 +1119,6 @@ pub const Parser = struct { }, State.TypeExprBegin => |dest_ptr| { - const token = self.getNextToken(); - if (token.id == Token.Id.Keyword_var) { - @panic("TODO param with type var"); - } - self.putBackToken(token); - stack.append(State { .TypeExprEnd = dest_ptr }) catch unreachable; try stack.append(State { .PrefixOpExpression = dest_ptr }); continue; @@ -1160,9 +1154,6 @@ pub const Parser = struct { try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); } continue; - //Token.Id.Keyword_await => { - // @panic("TODO: await"); - //}, } else { self.putBackToken(token); stack.append(State { .SuffixOpExpressionBegin = dest_ptr }) catch unreachable; @@ -1408,7 +1399,6 @@ pub const Parser = struct { continue; }, Token.Id.LBracket => { - // TODO: option("align" "(" Expression option(":" Integer ":" Integer) ")")) option("const") option("volatile") const rbracket_token = self.getNextToken(); if (rbracket_token.id == Token.Id.RBracket) { const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ @@ -4222,760 +4212,765 @@ fn testCanonical(source: []const u8) !void { } } -//test "zig fmt: get stdout or fail" { -// try testCanonical( -// \\const std = @import("std"); -// \\ -// \\pub fn main() !void { -// \\ // If this program is run without stdout attached, exit with an error. -// \\ // another comment -// \\ var stdout_file = try std.io.getStdOut; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: preserve spacing" { -// try testCanonical( -// \\const std = @import("std"); -// \\ -// \\pub fn main() !void { -// \\ var stdout_file = try std.io.getStdOut; -// \\ var stdout_file = try std.io.getStdOut; -// \\ -// \\ var stdout_file = try std.io.getStdOut; -// \\ var stdout_file = try std.io.getStdOut; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: return types" { -// try testCanonical( -// \\pub fn main() !void {} -// \\pub fn main() var {} -// \\pub fn main() i32 {} -// \\ -// ); -//} -// -//test "zig fmt: imports" { -// try testCanonical( -// \\const std = @import("std"); -// \\const std = @import(); -// \\ -// ); -//} -// -//test "zig fmt: global declarations" { -// try testCanonical( -// \\const a = b; -// \\pub const a = b; -// \\var a = b; -// \\pub var a = b; -// \\const a: i32 = b; -// \\pub const a: i32 = b; -// \\var a: i32 = b; -// \\pub var a: i32 = b; -// \\extern const a: i32 = b; -// \\pub extern const a: i32 = b; -// \\extern var a: i32 = b; -// \\pub extern var a: i32 = b; -// \\extern "a" const a: i32 = b; -// \\pub extern "a" const a: i32 = b; -// \\extern "a" var a: i32 = b; -// \\pub extern "a" var a: i32 = b; -// \\ -// ); -//} -// -//test "zig fmt: extern declaration" { -// try testCanonical( -// \\extern var foo: c_int; -// \\ -// ); -//} -// -//test "zig fmt: alignment" { -// try testCanonical( -// \\var foo: c_int align(1); -// \\ -// ); -//} -// -//test "zig fmt: C main" { -// try testCanonical( -// \\fn main(argc: c_int, argv: &&u8) c_int { -// \\ const a = b; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: return" { -// try testCanonical( -// \\fn foo(argc: c_int, argv: &&u8) c_int { -// \\ return 0; -// \\} -// \\ -// \\fn bar() void { -// \\ return; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: pointer attributes" { -// try testCanonical( -// \\extern fn f1(s: &align(&u8) u8) c_int; -// \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; -// \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; -// \\extern fn f4(s: &align(1) const volatile u8) c_int; -// \\ -// ); -//} -// -//test "zig fmt: slice attributes" { -// try testCanonical( -// \\extern fn f1(s: &align(&u8) u8) c_int; -// \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; -// \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; -// \\extern fn f4(s: &align(1) const volatile u8) c_int; -// \\ -// ); -//} -// -//test "zig fmt: test declaration" { -// try testCanonical( -// \\test "test name" { -// \\ const a = 1; -// \\ var b = 1; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: infix operators" { -// try testCanonical( -// \\test "infix operators" { -// \\ var i = undefined; -// \\ i = 2; -// \\ i *= 2; -// \\ i |= 2; -// \\ i ^= 2; -// \\ i <<= 2; -// \\ i >>= 2; -// \\ i &= 2; -// \\ i *= 2; -// \\ i *%= 2; -// \\ i -= 2; -// \\ i -%= 2; -// \\ i += 2; -// \\ i +%= 2; -// \\ i /= 2; -// \\ i %= 2; -// \\ _ = i == i; -// \\ _ = i != i; -// \\ _ = i != i; -// \\ _ = i.i; -// \\ _ = i || i; -// \\ _ = i!i; -// \\ _ = i ** i; -// \\ _ = i ++ i; -// \\ _ = i ?? i; -// \\ _ = i % i; -// \\ _ = i / i; -// \\ _ = i *% i; -// \\ _ = i * i; -// \\ _ = i -% i; -// \\ _ = i - i; -// \\ _ = i +% i; -// \\ _ = i + i; -// \\ _ = i << i; -// \\ _ = i >> i; -// \\ _ = i & i; -// \\ _ = i ^ i; -// \\ _ = i | i; -// \\ _ = i >= i; -// \\ _ = i <= i; -// \\ _ = i > i; -// \\ _ = i < i; -// \\ _ = i and i; -// \\ _ = i or i; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: precedence" { -// try testCanonical( -// \\test "precedence" { -// \\ a!b(); -// \\ (a!b)(); -// \\ !a!b; -// \\ !(a!b); -// \\ !a{ }; -// \\ !(a{ }); -// \\ a + b{ }; -// \\ (a + b){ }; -// \\ a << b + c; -// \\ (a << b) + c; -// \\ a & b << c; -// \\ (a & b) << c; -// \\ a ^ b & c; -// \\ (a ^ b) & c; -// \\ a | b ^ c; -// \\ (a | b) ^ c; -// \\ a == b | c; -// \\ (a == b) | c; -// \\ a and b == c; -// \\ (a and b) == c; -// \\ a or b and c; -// \\ (a or b) and c; -// \\ (a or b) and c; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: prefix operators" { -// try testCanonical( -// \\test "prefix operators" { -// \\ try return --%~??!*&0; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: call expression" { -// try testCanonical( -// \\test "test calls" { -// \\ a(); -// \\ a(1); -// \\ a(1, 2); -// \\ a(1, 2) + a(1, 2); -// \\} -// \\ -// ); -//} -// -//test "zig fmt: var args" { -// try testCanonical( -// \\fn print(args: ...) void {} -// \\ -// ); -//} -// -//test "zig fmt: extern function" { -// try testCanonical( -// \\extern fn puts(s: &const u8) c_int; -// \\extern "c" fn puts(s: &const u8) c_int; -// \\ -// ); -//} -// -//test "zig fmt: multiline string" { -// try testCanonical( -// \\const s = -// \\ \\ something -// \\ \\ something else -// \\ ; -// \\ -// ); -//} -// -//test "zig fmt: values" { -// try testCanonical( -// \\test "values" { -// \\ 1; -// \\ 1.0; -// \\ "string"; -// \\ c"cstring"; -// \\ 'c'; -// \\ true; -// \\ false; -// \\ null; -// \\ undefined; -// \\ error; -// \\ this; -// \\ unreachable; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: indexing" { -// try testCanonical( -// \\test "test index" { -// \\ a[0]; -// \\ a[0 + 5]; -// \\ a[0..]; -// \\ a[0..5]; -// \\ a[a[0]]; -// \\ a[a[0..]]; -// \\ a[a[0..5]]; -// \\ a[a[0]..]; -// \\ a[a[0..5]..]; -// \\ a[a[0]..a[0]]; -// \\ a[a[0..5]..a[0]]; -// \\ a[a[0..5]..a[0..5]]; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: struct declaration" { -// try testCanonical( -// \\const S = struct { -// \\ const Self = this; -// \\ f1: u8, -// \\ -// \\ fn method(self: &Self) Self { -// \\ return *self; -// \\ } -// \\ -// \\ f2: u8 -// \\}; -// \\ -// \\const Ps = packed struct { -// \\ a: u8, -// \\ b: u8, -// \\ -// \\ c: u8 -// \\}; -// \\ -// \\const Es = extern struct { -// \\ a: u8, -// \\ b: u8, -// \\ -// \\ c: u8 -// \\}; -// \\ -// ); -//} -// -//test "zig fmt: enum declaration" { -// try testCanonical( -// \\const E = enum { -// \\ Ok, -// \\ SomethingElse = 0 -// \\}; -// \\ -// \\const E2 = enum(u8) { -// \\ Ok, -// \\ SomethingElse = 255, -// \\ SomethingThird -// \\}; -// \\ -// \\const Ee = extern enum { -// \\ Ok, -// \\ SomethingElse, -// \\ SomethingThird -// \\}; -// \\ -// \\const Ep = packed enum { -// \\ Ok, -// \\ SomethingElse, -// \\ SomethingThird -// \\}; -// \\ -// ); -//} -// -//test "zig fmt: union declaration" { -// try testCanonical( -// \\const U = union { -// \\ Int: u8, -// \\ Float: f32, -// \\ None, -// \\ Bool: bool -// \\}; -// \\ -// \\const Ue = union(enum) { -// \\ Int: u8, -// \\ Float: f32, -// \\ None, -// \\ Bool: bool -// \\}; -// \\ -// \\const E = enum { -// \\ Int, -// \\ Float, -// \\ None, -// \\ Bool -// \\}; -// \\ -// \\const Ue2 = union(E) { -// \\ Int: u8, -// \\ Float: f32, -// \\ None, -// \\ Bool: bool -// \\}; -// \\ -// \\const Eu = extern union { -// \\ Int: u8, -// \\ Float: f32, -// \\ None, -// \\ Bool: bool -// \\}; -// \\ -// ); -//} -// -//test "zig fmt: error set declaration" { -// try testCanonical( -// \\const E = error { -// \\ A, -// \\ B, -// \\ -// \\ C -// \\}; -// \\ -// ); -//} -// -//test "zig fmt: arrays" { -// try testCanonical( -// \\test "test array" { -// \\ const a: [2]u8 = [2]u8{ 1, 2 }; -// \\ const a: [2]u8 = []u8{ 1, 2 }; -// \\ const a: [0]u8 = []u8{ }; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: container initializers" { -// try testCanonical( -// \\const a1 = []u8{ }; -// \\const a2 = []u8{ 1, 2, 3, 4 }; -// \\const s1 = S{ }; -// \\const s2 = S{ .a = 1, .b = 2 }; -// \\ -// ); -//} -// -//test "zig fmt: catch" { -// try testCanonical( -// \\test "catch" { -// \\ const a: error!u8 = 0; -// \\ _ = a catch return; -// \\ _ = a catch |err| return; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: blocks" { -// try testCanonical( -// \\test "blocks" { -// \\ { -// \\ const a = 0; -// \\ const b = 0; -// \\ } -// \\ -// \\ blk: { -// \\ const a = 0; -// \\ const b = 0; -// \\ } -// \\ -// \\ const r = blk: { -// \\ const a = 0; -// \\ const b = 0; -// \\ }; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: switch" { -// try testCanonical( -// \\test "switch" { -// \\ switch (0) { -// \\ 0 => {}, -// \\ 1 => unreachable, -// \\ 2, 3 => {}, -// \\ 4 ... 7 => {}, -// \\ 1 + 4 * 3 + 22 => {}, -// \\ else => { -// \\ const a = 1; -// \\ const b = a; -// \\ } -// \\ } -// \\ -// \\ const res = switch (0) { -// \\ 0 => 0, -// \\ 1 => 2, -// \\ else => 4 -// \\ }; -// \\ -// \\ const Union = union(enum) { -// \\ Int: i64, -// \\ Float: f64 -// \\ }; -// \\ -// \\ const u = Union{ .Int = 0 }; -// \\ switch (u) { -// \\ Union.Int => |int| {}, -// \\ Union.Float => |*float| unreachable -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: while" { -// try testCanonical( -// \\test "while" { -// \\ while (10 < 1) { -// \\ unreachable; -// \\ } -// \\ -// \\ while (10 < 1) -// \\ unreachable; -// \\ -// \\ var i: usize = 0; -// \\ while (i < 10) : (i += 1) { -// \\ continue; -// \\ } -// \\ -// \\ i = 0; -// \\ while (i < 10) : (i += 1) -// \\ continue; -// \\ -// \\ i = 0; -// \\ var j: usize = 0; -// \\ while (i < 10) : ({ -// \\ i += 1; -// \\ j += 1; -// \\ }) { -// \\ continue; -// \\ } -// \\ -// \\ var a: ?u8 = 2; -// \\ while (a) |v| : (a = null) { -// \\ continue; -// \\ } -// \\ -// \\ while (a) |v| : (a = null) -// \\ unreachable; -// \\ -// \\ label: while (10 < 0) { -// \\ unreachable; -// \\ } -// \\ -// \\ const res = while (0 < 10) { -// \\ break 7; -// \\ } else { -// \\ unreachable; -// \\ }; -// \\ -// \\ var a: error!u8 = 0; -// \\ while (a) |v| { -// \\ a = error.Err; -// \\ } else |err| { -// \\ i = 1; -// \\ } -// \\ -// \\ comptime var k: usize = 0; -// \\ inline while (i < 10) : (i += 1) -// \\ j += 2; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: for" { -// try testCanonical( -// \\test "for" { -// \\ const a = []u8{ 1, 2, 3 }; -// \\ for (a) |v| { -// \\ continue; -// \\ } -// \\ -// \\ for (a) |v| -// \\ continue; -// \\ -// \\ for (a) |*v| -// \\ continue; -// \\ -// \\ for (a) |v, i| { -// \\ continue; -// \\ } -// \\ -// \\ for (a) |v, i| -// \\ continue; -// \\ -// \\ const res = for (a) |v, i| { -// \\ break v; -// \\ } else { -// \\ unreachable; -// \\ }; -// \\ -// \\ var num: usize = 0; -// \\ inline for (a) |v, i| { -// \\ num += v; -// \\ num += i; -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: if" { -// try testCanonical( -// \\test "if" { -// \\ if (10 < 0) { -// \\ unreachable; -// \\ } -// \\ -// \\ if (10 < 0) unreachable; -// \\ -// \\ if (10 < 0) { -// \\ unreachable; -// \\ } else { -// \\ const a = 20; -// \\ } -// \\ -// \\ if (10 < 0) { -// \\ unreachable; -// \\ } else if (5 < 0) { -// \\ unreachable; -// \\ } else { -// \\ const a = 20; -// \\ } -// \\ -// \\ const is_world_broken = if (10 < 0) true else false; -// \\ -// \\ const a: ?u8 = 10; -// \\ const b: ?u8 = null; -// \\ if (a) |v| { -// \\ const some = v; -// \\ } else if (b) |*v| { -// \\ unreachable; -// \\ } else { -// \\ const some = 10; -// \\ } -// \\ -// \\ const non_null_a = if (a) |v| v else 0; -// \\ -// \\ const a_err: error!u8 = 0; -// \\ if (a_err) |v| { -// \\ const p = v; -// \\ } else |err| { -// \\ unreachable; -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: defer" { -// try testCanonical( -// \\test "defer" { -// \\ var i: usize = 0; -// \\ defer i = 1; -// \\ defer { -// \\ i += 2; -// \\ i *= i; -// \\ } -// \\ -// \\ errdefer i += 3; -// \\ errdefer { -// \\ i += 2; -// \\ i /= i; -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: comptime" { -// try testCanonical( -// \\fn a() u8 { -// \\ return 5; -// \\} -// \\ -// \\fn b(comptime i: u8) u8 { -// \\ return i; -// \\} -// \\ -// \\const av = comptime a(); -// \\const av2 = comptime blk: { -// \\ var res = a(); -// \\ res *= b(2); -// \\ break :blk res; -// \\}; -// \\ -// \\comptime { -// \\ _ = a(); -// \\} -// \\ -// \\test "comptime" { -// \\ const av3 = comptime a(); -// \\ const av4 = comptime blk: { -// \\ var res = a(); -// \\ res *= a(); -// \\ break :blk res; -// \\ }; -// \\ -// \\ comptime var i = 0; -// \\ comptime { -// \\ i = a(); -// \\ i += b(i); -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: fn type" { -// try testCanonical( -// \\fn a(i: u8) u8 { -// \\ return i + 1; -// \\} -// \\ -// \\const a: fn(u8) u8 = undefined; -// \\const b: extern fn(u8) u8 = undefined; -// \\const c: nakedcc fn(u8) u8 = undefined; -// \\const ap: fn(u8) u8 = a; -// \\ -// ); -//} -// -//test "zig fmt: inline asm" { -// try testCanonical( -// \\pub fn syscall1(number: usize, arg1: usize) usize { -// \\ return asm volatile ("syscall" -// \\ : [ret] "={rax}" (-> usize) -// \\ : [number] "{rax}" (number), -// \\ [arg1] "{rdi}" (arg1) -// \\ : "rcx", "r11"); -// \\} -// \\ -// ); -//} -// -//test "zig fmt: coroutines" { -// try testCanonical( -// \\async fn simpleAsyncFn() void { -// \\ x += 1; -// \\ suspend; -// \\ x += 1; -// \\ suspend |p| {} -// \\ const p = async simpleAsyncFn() catch unreachable; -// \\ await p; -// \\} -// \\ -// \\test "coroutine suspend, resume, cancel" { -// \\ const p = try async testAsyncSeq(); -// \\ resume p; -// \\ cancel p; -// \\} -// \\ -// ); -//} - -test "1" { - try testCanonical(@embedFile("../array_list.zig")); +test "zig fmt: get stdout or fail" { + try testCanonical( + \\const std = @import("std"); + \\ + \\pub fn main() !void { + \\ // If this program is run without stdout attached, exit with an error. + \\ // another comment + \\ var stdout_file = try std.io.getStdOut; + \\} + \\ + ); } + +test "zig fmt: preserve spacing" { + try testCanonical( + \\const std = @import("std"); + \\ + \\pub fn main() !void { + \\ var stdout_file = try std.io.getStdOut; + \\ var stdout_file = try std.io.getStdOut; + \\ + \\ var stdout_file = try std.io.getStdOut; + \\ var stdout_file = try std.io.getStdOut; + \\} + \\ + ); +} + +test "zig fmt: return types" { + try testCanonical( + \\pub fn main() !void {} + \\pub fn main() var {} + \\pub fn main() i32 {} + \\ + ); +} + +test "zig fmt: imports" { + try testCanonical( + \\const std = @import("std"); + \\const std = @import(); + \\ + ); +} + +test "zig fmt: global declarations" { + try testCanonical( + \\const a = b; + \\pub const a = b; + \\var a = b; + \\pub var a = b; + \\const a: i32 = b; + \\pub const a: i32 = b; + \\var a: i32 = b; + \\pub var a: i32 = b; + \\extern const a: i32 = b; + \\pub extern const a: i32 = b; + \\extern var a: i32 = b; + \\pub extern var a: i32 = b; + \\extern "a" const a: i32 = b; + \\pub extern "a" const a: i32 = b; + \\extern "a" var a: i32 = b; + \\pub extern "a" var a: i32 = b; + \\ + ); +} + +test "zig fmt: extern declaration" { + try testCanonical( + \\extern var foo: c_int; + \\ + ); +} + +test "zig fmt: alignment" { + try testCanonical( + \\var foo: c_int align(1); + \\ + ); +} + +test "zig fmt: C main" { + try testCanonical( + \\fn main(argc: c_int, argv: &&u8) c_int { + \\ const a = b; + \\} + \\ + ); +} + +test "zig fmt: return" { + try testCanonical( + \\fn foo(argc: c_int, argv: &&u8) c_int { + \\ return 0; + \\} + \\ + \\fn bar() void { + \\ return; + \\} + \\ + ); +} + +test "zig fmt: pointer attributes" { + try testCanonical( + \\extern fn f1(s: &align(&u8) u8) c_int; + \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; + \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; + \\extern fn f4(s: &align(1) const volatile u8) c_int; + \\ + ); +} + +test "zig fmt: slice attributes" { + try testCanonical( + \\extern fn f1(s: &align(&u8) u8) c_int; + \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; + \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; + \\extern fn f4(s: &align(1) const volatile u8) c_int; + \\ + ); +} + +test "zig fmt: test declaration" { + try testCanonical( + \\test "test name" { + \\ const a = 1; + \\ var b = 1; + \\} + \\ + ); +} + +test "zig fmt: infix operators" { + try testCanonical( + \\test "infix operators" { + \\ var i = undefined; + \\ i = 2; + \\ i *= 2; + \\ i |= 2; + \\ i ^= 2; + \\ i <<= 2; + \\ i >>= 2; + \\ i &= 2; + \\ i *= 2; + \\ i *%= 2; + \\ i -= 2; + \\ i -%= 2; + \\ i += 2; + \\ i +%= 2; + \\ i /= 2; + \\ i %= 2; + \\ _ = i == i; + \\ _ = i != i; + \\ _ = i != i; + \\ _ = i.i; + \\ _ = i || i; + \\ _ = i!i; + \\ _ = i ** i; + \\ _ = i ++ i; + \\ _ = i ?? i; + \\ _ = i % i; + \\ _ = i / i; + \\ _ = i *% i; + \\ _ = i * i; + \\ _ = i -% i; + \\ _ = i - i; + \\ _ = i +% i; + \\ _ = i + i; + \\ _ = i << i; + \\ _ = i >> i; + \\ _ = i & i; + \\ _ = i ^ i; + \\ _ = i | i; + \\ _ = i >= i; + \\ _ = i <= i; + \\ _ = i > i; + \\ _ = i < i; + \\ _ = i and i; + \\ _ = i or i; + \\} + \\ + ); +} + +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ !a{ }; + \\ !(a{ }); + \\ a + b{ }; + \\ (a + b){ }; + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\} + \\ + ); +} + +test "zig fmt: prefix operators" { + try testCanonical( + \\test "prefix operators" { + \\ try return --%~??!*&0; + \\} + \\ + ); +} + +test "zig fmt: call expression" { + try testCanonical( + \\test "test calls" { + \\ a(); + \\ a(1); + \\ a(1, 2); + \\ a(1, 2) + a(1, 2); + \\} + \\ + ); +} + +test "zig fmt: var args" { + try testCanonical( + \\fn print(args: ...) void {} + \\ + ); +} + +test "zig fmt: extern function" { + try testCanonical( + \\extern fn puts(s: &const u8) c_int; + \\extern "c" fn puts(s: &const u8) c_int; + \\ + ); +} + +test "zig fmt: multiline string" { + try testCanonical( + \\const s = + \\ \\ something + \\ \\ something else + \\ ; + \\ + ); +} + +test "zig fmt: values" { + try testCanonical( + \\test "values" { + \\ 1; + \\ 1.0; + \\ "string"; + \\ c"cstring"; + \\ 'c'; + \\ true; + \\ false; + \\ null; + \\ undefined; + \\ error; + \\ this; + \\ unreachable; + \\} + \\ + ); +} + +test "zig fmt: indexing" { + try testCanonical( + \\test "test index" { + \\ a[0]; + \\ a[0 + 5]; + \\ a[0..]; + \\ a[0..5]; + \\ a[a[0]]; + \\ a[a[0..]]; + \\ a[a[0..5]]; + \\ a[a[0]..]; + \\ a[a[0..5]..]; + \\ a[a[0]..a[0]]; + \\ a[a[0..5]..a[0]]; + \\ a[a[0..5]..a[0..5]]; + \\} + \\ + ); +} + +test "zig fmt: struct declaration" { + try testCanonical( + \\const S = struct { + \\ const Self = this; + \\ f1: u8, + \\ + \\ fn method(self: &Self) Self { + \\ return *self; + \\ } + \\ + \\ f2: u8 + \\}; + \\ + \\const Ps = packed struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8 + \\}; + \\ + \\const Es = extern struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8 + \\}; + \\ + ); +} + +test "zig fmt: enum declaration" { + try testCanonical( + \\const E = enum { + \\ Ok, + \\ SomethingElse = 0 + \\}; + \\ + \\const E2 = enum(u8) { + \\ Ok, + \\ SomethingElse = 255, + \\ SomethingThird + \\}; + \\ + \\const Ee = extern enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird + \\}; + \\ + \\const Ep = packed enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird + \\}; + \\ + ); +} + +test "zig fmt: union declaration" { + try testCanonical( + \\const U = union { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool + \\}; + \\ + \\const Ue = union(enum) { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool + \\}; + \\ + \\const E = enum { + \\ Int, + \\ Float, + \\ None, + \\ Bool + \\}; + \\ + \\const Ue2 = union(E) { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool + \\}; + \\ + \\const Eu = extern union { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool + \\}; + \\ + ); +} + +test "zig fmt: error set declaration" { + try testCanonical( + \\const E = error { + \\ A, + \\ B, + \\ + \\ C + \\}; + \\ + ); +} + +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{ }; + \\} + \\ + ); +} + +test "zig fmt: container initializers" { + try testCanonical( + \\const a1 = []u8{ }; + \\const a2 = []u8{ 1, 2, 3, 4 }; + \\const s1 = S{ }; + \\const s2 = S{ .a = 1, .b = 2 }; + \\ + ); +} + +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: error!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch |err| return; + \\} + \\ + ); +} + +test "zig fmt: blocks" { + try testCanonical( + \\test "blocks" { + \\ { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ blk: { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ const r = blk: { + \\ const a = 0; + \\ const b = 0; + \\ }; + \\} + \\ + ); +} + +test "zig fmt: switch" { + try testCanonical( + \\test "switch" { + \\ switch (0) { + \\ 0 => {}, + \\ 1 => unreachable, + \\ 2, 3 => {}, + \\ 4 ... 7 => {}, + \\ 1 + 4 * 3 + 22 => {}, + \\ else => { + \\ const a = 1; + \\ const b = a; + \\ } + \\ } + \\ + \\ const res = switch (0) { + \\ 0 => 0, + \\ 1 => 2, + \\ else => 4 + \\ }; + \\ + \\ const Union = union(enum) { + \\ Int: i64, + \\ Float: f64 + \\ }; + \\ + \\ const u = Union{ .Int = 0 }; + \\ switch (u) { + \\ Union.Int => |int| {}, + \\ Union.Float => |*float| unreachable + \\ } + \\} + \\ + ); +} + +test "zig fmt: while" { + try testCanonical( + \\test "while" { + \\ while (10 < 1) { + \\ unreachable; + \\ } + \\ + \\ while (10 < 1) + \\ unreachable; + \\ + \\ var i: usize = 0; + \\ while (i < 10) : (i += 1) { + \\ continue; + \\ } + \\ + \\ i = 0; + \\ while (i < 10) : (i += 1) + \\ continue; + \\ + \\ i = 0; + \\ var j: usize = 0; + \\ while (i < 10) : ({ + \\ i += 1; + \\ j += 1; + \\ }) { + \\ continue; + \\ } + \\ + \\ var a: ?u8 = 2; + \\ while (a) |v| : (a = null) { + \\ continue; + \\ } + \\ + \\ while (a) |v| : (a = null) + \\ unreachable; + \\ + \\ label: while (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ const res = while (0 < 10) { + \\ break 7; + \\ } else { + \\ unreachable; + \\ }; + \\ + \\ var a: error!u8 = 0; + \\ while (a) |v| { + \\ a = error.Err; + \\ } else |err| { + \\ i = 1; + \\ } + \\ + \\ comptime var k: usize = 0; + \\ inline while (i < 10) : (i += 1) + \\ j += 2; + \\} + \\ + ); +} + +test "zig fmt: for" { + try testCanonical( + \\test "for" { + \\ const a = []u8{ 1, 2, 3 }; + \\ for (a) |v| { + \\ continue; + \\ } + \\ + \\ for (a) |v| + \\ continue; + \\ + \\ for (a) |*v| + \\ continue; + \\ + \\ for (a) |v, i| { + \\ continue; + \\ } + \\ + \\ for (a) |v, i| + \\ continue; + \\ + \\ const res = for (a) |v, i| { + \\ break v; + \\ } else { + \\ unreachable; + \\ }; + \\ + \\ var num: usize = 0; + \\ inline for (a) |v, i| { + \\ num += v; + \\ num += i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: if" { + try testCanonical( + \\test "if" { + \\ if (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ if (10 < 0) unreachable; + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else if (5 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ const is_world_broken = if (10 < 0) true else false; + \\ + \\ const a: ?u8 = 10; + \\ const b: ?u8 = null; + \\ if (a) |v| { + \\ const some = v; + \\ } else if (b) |*v| { + \\ unreachable; + \\ } else { + \\ const some = 10; + \\ } + \\ + \\ const non_null_a = if (a) |v| v else 0; + \\ + \\ const a_err: error!u8 = 0; + \\ if (a_err) |v| { + \\ const p = v; + \\ } else |err| { + \\ unreachable; + \\ } + \\} + \\ + ); +} + +test "zig fmt: defer" { + try testCanonical( + \\test "defer" { + \\ var i: usize = 0; + \\ defer i = 1; + \\ defer { + \\ i += 2; + \\ i *= i; + \\ } + \\ + \\ errdefer i += 3; + \\ errdefer { + \\ i += 2; + \\ i /= i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: comptime" { + try testCanonical( + \\fn a() u8 { + \\ return 5; + \\} + \\ + \\fn b(comptime i: u8) u8 { + \\ return i; + \\} + \\ + \\const av = comptime a(); + \\const av2 = comptime blk: { + \\ var res = a(); + \\ res *= b(2); + \\ break :blk res; + \\}; + \\ + \\comptime { + \\ _ = a(); + \\} + \\ + \\test "comptime" { + \\ const av3 = comptime a(); + \\ const av4 = comptime blk: { + \\ var res = a(); + \\ res *= a(); + \\ break :blk res; + \\ }; + \\ + \\ comptime var i = 0; + \\ comptime { + \\ i = a(); + \\ i += b(i); + \\ } + \\} + \\ + ); +} + +test "zig fmt: fn type" { + try testCanonical( + \\fn a(i: u8) u8 { + \\ return i + 1; + \\} + \\ + \\const a: fn(u8) u8 = undefined; + \\const b: extern fn(u8) u8 = undefined; + \\const c: nakedcc fn(u8) u8 = undefined; + \\const ap: fn(u8) u8 = a; + \\ + ); +} + +test "zig fmt: inline asm" { + try testCanonical( + \\pub fn syscall1(number: usize, arg1: usize) usize { + \\ return asm volatile ("syscall" + \\ : [ret] "={rax}" (-> usize) + \\ : [number] "{rax}" (number), + \\ [arg1] "{rdi}" (arg1) + \\ : "rcx", "r11"); + \\} + \\ + ); +} + +test "zig fmt: coroutines" { + try testCanonical( + \\async fn simpleAsyncFn() void { + \\ x += 1; + \\ suspend; + \\ x += 1; + \\ suspend |p| {} + \\ const p = async simpleAsyncFn() catch unreachable; + \\ await p; + \\} + \\ + \\test "coroutine suspend, resume, cancel" { + \\ const p = try async testAsyncSeq(); + \\ resume p; + \\ cancel p; + \\} + \\ + ); +} + +//{ +// var it = self.link_libs.iterator(); +// while (true) { +// const entry = it.next() ?? break; +// zig_args.append("--library") catch unreachable; +// zig_args.append(entry.key) catch unreachable; +// } +//} From db0812d4b7f856425e0bd26cd6f579468f3ac8ab Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 14:22:01 +0200 Subject: [PATCH 45/54] std.zig.parser: changed block exprs from primary expr to expr --- std/zig/parser.zig | 232 +++++++++++++++++++++++------------------- std/zig/tokenizer.zig | 1 - 2 files changed, 129 insertions(+), 104 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 5f7412b5c2..65e8056ad2 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -766,6 +766,101 @@ pub const Parser = struct { dest_ptr.store(&resume_node.base); stack.append(State { .Expression = DestPtr { .Field = &resume_node.rhs } }) catch unreachable; }, + Token.Id.Keyword_suspend => { + const node = try arena.create(ast.NodeSuspend); + *node = ast.NodeSuspend { + .base = self.initNode(ast.Node.Id.Suspend), + .suspend_token = token, + .payload = null, + .body = null, + }; + dest_ptr.store(&node.base); + stack.append(State { .SuspendBody = node }) catch unreachable; + try stack.append(State { .Payload = &node.payload }); + continue; + }, + Token.Id.Keyword_if => { + const node = try arena.create(ast.NodeIf); + *node = ast.NodeIf { + .base = self.initNode(ast.Node.Id.If), + .if_token = token, + .condition = undefined, + .payload = null, + .body = undefined, + .@"else" = null, + }; + dest_ptr.store(&node.base); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .PointerPayload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + continue; + }, + Token.Id.Keyword_while => { + stack.append(State { + .While = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_for => { + stack.append(State { + .For = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_switch => { + const node = try arena.create(ast.NodeSwitch); + *node = ast.NodeSwitch { + .base = self.initNode(ast.Node.Id.Switch), + .switch_token = token, + .expr = undefined, + .cases = ArrayList(&ast.NodeSwitchCase).init(arena), + .rbrace = undefined, + }; + dest_ptr.store(&node.base); + + stack.append(State { + .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) { + .list = &node.cases, + .ptr = &node.rbrace, + }, + }) catch unreachable; + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + }, + Token.Id.Keyword_comptime => { + const node = try arena.create(ast.NodeComptime); + *node = ast.NodeComptime { + .base = self.initNode(ast.Node.Id.Comptime), + .comptime_token = token, + .expr = undefined, + }; + dest_ptr.store(&node.base); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, + Token.Id.LBrace => { + const block = try self.createBlock(arena, (?Token)(null), token); + dest_ptr.store(&block.base); + + stack.append(State { .Block = block }) catch unreachable; + continue; + }, else => { self.putBackToken(token); stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; @@ -1328,19 +1423,6 @@ pub const Parser = struct { dest_ptr.store(&node.base); continue; }, - Token.Id.Keyword_suspend => { - const node = try arena.create(ast.NodeSuspend); - *node = ast.NodeSuspend { - .base = self.initNode(ast.Node.Id.Suspend), - .suspend_token = token, - .payload = null, - .body = null, - }; - dest_ptr.store(&node.base); - stack.append(State { .SuspendBody = node }) catch unreachable; - try stack.append(State { .Payload = &node.payload }); - continue; - }, Token.Id.MultilineStringLiteralLine => { const node = try arena.create(ast.NodeMultilineStringLiteral); *node = ast.NodeMultilineStringLiteral { @@ -1544,13 +1626,6 @@ pub const Parser = struct { }) catch unreachable; continue; }, - Token.Id.LBrace => { - const block = try self.createBlock(arena, (?Token)(null), token); - dest_ptr.store(&block.base); - - stack.append(State { .Block = block }) catch unreachable; - continue; - }, Token.Id.Keyword_fn => { // TODO shouldn't need these casts const fn_proto = try self.createFnProto(arena, token, @@ -1608,26 +1683,6 @@ pub const Parser = struct { try stack.append(State { .AsmOutputItems = &node.outputs }); try stack.append(State { .IfToken = Token.Id.Colon }); }, - Token.Id.Keyword_if => { - const node = try arena.create(ast.NodeIf); - *node = ast.NodeIf { - .base = self.initNode(ast.Node.Id.If), - .if_token = token, - .condition = undefined, - .payload = null, - .body = undefined, - .@"else" = null, - }; - dest_ptr.store(&node.base); - - stack.append(State { .Else = &node.@"else" }) catch unreachable; - try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); - try stack.append(State { .PointerPayload = &node.payload }); - try stack.append(State { .ExpectToken = Token.Id.RParen }); - try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); - try stack.append(State { .ExpectToken = Token.Id.LParen }); - continue; - }, Token.Id.Keyword_inline => { stack.append(State { .Inline = InlineCtx { @@ -1638,61 +1693,6 @@ pub const Parser = struct { }) catch unreachable; continue; }, - Token.Id.Keyword_while => { - stack.append(State { - .While = LoopCtx { - .label = null, - .inline_token = null, - .loop_token = token, - .dest_ptr = dest_ptr, - } - }) catch unreachable; - continue; - }, - Token.Id.Keyword_for => { - stack.append(State { - .For = LoopCtx { - .label = null, - .inline_token = null, - .loop_token = token, - .dest_ptr = dest_ptr, - } - }) catch unreachable; - continue; - }, - Token.Id.Keyword_switch => { - const node = try arena.create(ast.NodeSwitch); - *node = ast.NodeSwitch { - .base = self.initNode(ast.Node.Id.Switch), - .switch_token = token, - .expr = undefined, - .cases = ArrayList(&ast.NodeSwitchCase).init(arena), - .rbrace = undefined, - }; - dest_ptr.store(&node.base); - - stack.append(State { - .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) { - .list = &node.cases, - .ptr = &node.rbrace, - }, - }) catch unreachable; - try stack.append(State { .ExpectToken = Token.Id.LBrace }); - try stack.append(State { .ExpectToken = Token.Id.RParen }); - try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); - try stack.append(State { .ExpectToken = Token.Id.LParen }); - }, - Token.Id.Keyword_comptime => { - const node = try arena.create(ast.NodeComptime); - *node = ast.NodeComptime { - .base = self.initNode(ast.Node.Id.Comptime), - .comptime_token = token, - .expr = undefined, - }; - dest_ptr.store(&node.base); - try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); - continue; - }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; @@ -4966,11 +4966,37 @@ test "zig fmt: coroutines" { ); } -//{ -// var it = self.link_libs.iterator(); -// while (true) { -// const entry = it.next() ?? break; -// zig_args.append("--library") catch unreachable; -// zig_args.append(entry.key) catch unreachable; -// } -//} +test "zig fmt: coroutines" { + try testCanonical( + \\async fn simpleAsyncFn() void { + \\ x += 1; + \\ suspend; + \\ x += 1; + \\ suspend |p| {} + \\ const p = async simpleAsyncFn() catch unreachable; + \\ await p; + \\} + \\ + \\test "coroutine suspend, resume, cancel" { + \\ const p = try async testAsyncSeq(); + \\ resume p; + \\ cancel p; + \\} + \\ + ); +} + +test "zig fmt: Block after if" { + try testCanonical( + \\test "Block after if" { + \\ if (true) { + \\ const a = 0; + \\ } + \\ + \\ { + \\ const a = 0; + \\ } + \\} + \\ + ); +} diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 7b1f86712a..91fb20974f 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -1158,7 +1158,6 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void { var tokenizer = Tokenizer.init(source); for (expected_tokens) |expected_token_id| { const token = tokenizer.next(); - std.debug.warn("{} {}\n", @tagName(expected_token_id), @tagName(token.id)); std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id)); switch (expected_token_id) { Token.Id.StringLiteral => |expected_kind| { From 3b80e665074cd56d9e24fb8ae0fb9d86e8cd841a Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 14:52:47 +0200 Subject: [PATCH 46/54] std.zig.parser now parses toplevel use --- std/zig/ast.zig | 29 ++++++++++++++++ std/zig/parser.zig | 87 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 93 insertions(+), 23 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 2128b9976f..1096086e37 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -11,6 +11,7 @@ pub const Node = struct { pub const Id = enum { Root, VarDecl, + Use, ErrorSetDecl, ContainerDecl, StructField, @@ -63,6 +64,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), + Id.Use => @fieldParentPtr(NodeUse, "base", base).iterate(index), Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).iterate(index), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).iterate(index), @@ -116,6 +118,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), + Id.Use => @fieldParentPtr(NodeUse, "base", base).firstToken(), Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).firstToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).firstToken(), @@ -169,6 +172,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), + Id.Use => @fieldParentPtr(NodeUse, "base", base).lastToken(), Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).lastToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).lastToken(), @@ -288,6 +292,31 @@ pub const NodeVarDecl = struct { } }; +pub const NodeUse = struct { + base: Node, + visib_token: ?Token, + expr: &Node, + semicolon_token: Token, + + pub fn iterate(self: &NodeUse, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeUse) Token { + if (self.visib_token) |visib_token| return visib_token; + return self.expr.firstToken(); + } + + pub fn lastToken(self: &NodeUse) Token { + return self.semicolon_token; + } +}; + pub const NodeErrorSetDecl = struct { base: Node, error_token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 65e8056ad2..e11bcfef05 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -348,31 +348,54 @@ pub const Parser = struct { }, State.TopLevelExtern => |ctx| { const token = self.getNextToken(); - if (token.id == Token.Id.Keyword_extern) { - const lib_name_token = self.getNextToken(); - const lib_name = blk: { - if (lib_name_token.id == Token.Id.StringLiteral) { - const res = try self.createStringLiteral(arena, lib_name_token); - break :blk &res.base; - } else { - self.putBackToken(lib_name_token); - break :blk null; - } - }; - - stack.append(State { - .TopLevelDecl = TopLevelDeclCtx { - .decls = ctx.decls, + switch (token.id) { + Token.Id.Keyword_use => { + const node = try arena.create(ast.NodeUse); + *node = ast.NodeUse { + .base = self.initNode(ast.Node.Id.Use), .visib_token = ctx.visib_token, - .extern_token = token, - .lib_name = lib_name, - }, - }) catch unreachable; - continue; + .expr = undefined, + .semicolon_token = undefined, + }; + try ctx.decls.append(&node.base); + + stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Semicolon, + .ptr = &node.semicolon_token, + } + }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, + Token.Id.Keyword_extern => { + const lib_name_token = self.getNextToken(); + const lib_name = blk: { + if (lib_name_token.id == Token.Id.StringLiteral) { + const res = try self.createStringLiteral(arena, lib_name_token); + break :blk &res.base; + } else { + self.putBackToken(lib_name_token); + break :blk null; + } + }; + + stack.append(State { + .TopLevelDecl = TopLevelDeclCtx { + .decls = ctx.decls, + .visib_token = ctx.visib_token, + .extern_token = token, + .lib_name = lib_name, + }, + }) catch unreachable; + continue; + }, + else => { + self.putBackToken(token); + stack.append(State { .TopLevelDecl = ctx }) catch unreachable; + continue; + } } - self.putBackToken(token); - stack.append(State { .TopLevelDecl = ctx }) catch unreachable; - continue; }, State.TopLevelDecl => |ctx| { const token = self.getNextToken(); @@ -3068,6 +3091,15 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = decl }); }, + ast.Node.Id.Use => { + const use_decl = @fieldParentPtr(ast.NodeUse, "base", decl); + if (use_decl.visib_token) |visib_token| { + try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token)); + } + try stream.print("use "); + try stack.append(RenderState { .Text = ";" }); + try stack.append(RenderState { .Expression = use_decl.expr }); + }, ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); try stack.append(RenderState { .VarDecl = var_decl}); @@ -4086,6 +4118,7 @@ pub const Parser = struct { ast.Node.Id.EnumTag, ast.Node.Id.Root, ast.Node.Id.VarDecl, + ast.Node.Id.Use, ast.Node.Id.TestDecl, ast.Node.Id.ParamDecl => unreachable, }, @@ -5000,3 +5033,11 @@ test "zig fmt: Block after if" { \\ ); } + +test "zig fmt: use" { + try testCanonical( + \\use @import("std"); + \\pub use @import("std"); + \\ + ); +} From aa09e7b63995639084d25329954b1972a72ad12d Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 15:01:21 +0200 Subject: [PATCH 47/54] std.zig.tokinizer now treats string identifiers as identifiers --- std/zig/parser.zig | 8 ++++++++ std/zig/tokenizer.zig | 5 ++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index e11bcfef05..a01a0d748e 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -5041,3 +5041,11 @@ test "zig fmt: use" { \\ ); } + +test "zig fmt: string identifier" { + try testCanonical( + \\const @"a b" = @"c d".@"e f"; + \\fn @"g h"() void {} + \\ + ); +} diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 91fb20974f..a2c4def9e0 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -78,7 +78,6 @@ pub const Token = struct { StringLiteral: StrLitKind, MultilineStringLiteralLine: StrLitKind, CharLiteral, - StringIdentifier, Eof, Builtin, Bang, @@ -434,7 +433,7 @@ pub const Tokenizer = struct { State.SawAtSign => switch (c) { '"' => { - result.id = Token.Id.StringIdentifier; + result.id = Token.Id.Identifier; state = State.StringLiteral; }, else => { @@ -1136,7 +1135,7 @@ test "tokenizer - string identifier and builtin fns" { , []Token.Id{ Token.Id.Keyword_const, - Token.Id.StringIdentifier, + Token.Id.Identifier, Token.Id.Equal, Token.Id.Builtin, Token.Id.LParen, From db9a9f3a6c248e74953d9f7cbf6bedc78e3c23ff Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 15:16:31 +0200 Subject: [PATCH 48/54] std.zig.parser now parses the `var` type * I parse it as a type in all contexts. This is not how the C++ compiler does it, but I think typechecking should catch this --- std/zig/ast.zig | 23 +++++++++++++++++++++-- std/zig/parser.zig | 30 +++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 1096086e37..d296b3b52b 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -55,6 +55,7 @@ pub const Node = struct { AsmOutput, Unreachable, ErrorType, + VarType, BuiltinCall, LineComment, TestDecl, @@ -108,6 +109,7 @@ pub const Node = struct { Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).iterate(index), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), + Id.VarType => @fieldParentPtr(NodeVarType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), @@ -162,6 +164,7 @@ pub const Node = struct { Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).firstToken(), Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).firstToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), + Id.VarType => @fieldParentPtr(NodeVarType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), @@ -216,6 +219,7 @@ pub const Node = struct { Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).lastToken(), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), + Id.VarType => @fieldParentPtr(NodeVarType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), @@ -541,7 +545,6 @@ pub const NodeFnProto = struct { pub const ReturnType = union(enum) { Explicit: &Node, - Infer: Token, InferErrorSet: &Node, }; @@ -597,7 +600,6 @@ pub const NodeFnProto = struct { // TODO allow this and next prong to share bodies since the types are the same ReturnType.Explicit => |node| return node.lastToken(), ReturnType.InferErrorSet => |node| return node.lastToken(), - ReturnType.Infer => |token| return token, } } }; @@ -1788,6 +1790,23 @@ pub const NodeErrorType = struct { } }; +pub const NodeVarType = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeVarType, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeVarType) Token { + return self.token; + } + + pub fn lastToken(self: &NodeVarType) Token { + return self.token; + } +}; + pub const NodeLineComment = struct { base: Node, lines: ArrayList(Token), diff --git a/std/zig/parser.zig b/std/zig/parser.zig index a01a0d748e..79c90e6e68 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1437,6 +1437,14 @@ pub const Parser = struct { dest_ptr.store(&node.base); continue; }, + Token.Id.Keyword_var => { + const node = try arena.create(ast.NodeVarType); + *node = ast.NodeVarType { + .base = self.initNode(ast.Node.Id.VarType), + .token = token, + }; + dest_ptr.store(&node.base); + }, Token.Id.Keyword_unreachable => { const node = try arena.create(ast.NodeUnreachable); *node = ast.NodeUnreachable { @@ -2192,9 +2200,6 @@ pub const Parser = struct { State.FnProtoReturnType => |fn_proto| { const token = self.getNextToken(); switch (token.id) { - Token.Id.Keyword_var => { - fn_proto.return_type = ast.NodeFnProto.ReturnType { .Infer = token }; - }, Token.Id.Bang => { fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined }; stack.append(State { @@ -3573,6 +3578,10 @@ pub const Parser = struct { const error_type = @fieldParentPtr(ast.NodeErrorType, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(error_type.token)); }, + ast.Node.Id.VarType => { + const var_type = @fieldParentPtr(ast.NodeVarType, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(var_type.token)); + }, ast.Node.Id.ContainerDecl => { const container_decl = @fieldParentPtr(ast.NodeContainerDecl, "base", base); @@ -3711,9 +3720,6 @@ pub const Parser = struct { ast.NodeFnProto.ReturnType.Explicit => |node| { try stack.append(RenderState { .Expression = node}); }, - ast.NodeFnProto.ReturnType.Infer => { - try stack.append(RenderState { .Text = "var"}); - }, ast.NodeFnProto.ReturnType.InferErrorSet => |node| { try stack.append(RenderState { .Expression = node}); try stack.append(RenderState { .Text = "!"}); @@ -4136,9 +4142,6 @@ pub const Parser = struct { ast.NodeFnProto.ReturnType.Explicit => |node| { try stack.append(RenderState { .Expression = node}); }, - ast.NodeFnProto.ReturnType.Infer => { - try stream.print("var"); - }, ast.NodeFnProto.ReturnType.InferErrorSet => |node| { try stream.print("!"); try stack.append(RenderState { .Expression = node}); @@ -4489,6 +4492,15 @@ test "zig fmt: var args" { ); } +test "zig fmt: var type" { + try testCanonical( + \\fn print(args: var) var {} + \\const Var = var; + \\const i: var = 0; + \\ + ); +} + test "zig fmt: extern function" { try testCanonical( \\extern fn puts(s: &const u8) c_int; From b9cccce26d616c2ff570a1e1ba7073d3e8f79672 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 15:56:37 +0200 Subject: [PATCH 49/54] std.zig.ast: fixed none compiling code --- std/zig/ast.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index d296b3b52b..045548d624 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -566,7 +566,6 @@ pub const NodeFnProto = struct { if (i < 1) return node; i -= 1; }, - ReturnType.Infer => {}, } if (self.align_expr) |align_expr| { From c6aa637146c6695f382553f66d311599a90d7425 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 16:33:43 +0200 Subject: [PATCH 50/54] std.zig.parser: removed dublicate "zig fmt: coroutines" test --- std/zig/parser.zig | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 79c90e6e68..76eb2d29d5 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -5011,26 +5011,6 @@ test "zig fmt: coroutines" { ); } -test "zig fmt: coroutines" { - try testCanonical( - \\async fn simpleAsyncFn() void { - \\ x += 1; - \\ suspend; - \\ x += 1; - \\ suspend |p| {} - \\ const p = async simpleAsyncFn() catch unreachable; - \\ await p; - \\} - \\ - \\test "coroutine suspend, resume, cancel" { - \\ const p = try async testAsyncSeq(); - \\ resume p; - \\ cancel p; - \\} - \\ - ); -} - test "zig fmt: Block after if" { try testCanonical( \\test "Block after if" { From 0ba85ea6ff910c7a49ae036625b945c475c0f58c Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 17:46:17 +0200 Subject: [PATCH 51/54] std.zig.parser fixed segfault when parsing cc for fn decl --- std/zig/parser.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 76eb2d29d5..92b461d206 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -417,7 +417,7 @@ pub const Parser = struct { }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, Token(undefined), ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); From 27e881c2d7bebbf06b9932559ff2677ed8b6088b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Apr 2018 21:58:04 -0400 Subject: [PATCH 52/54] fix another undefined deref see 0ba85ea6ff91 --- std/zig/parser.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 92b461d206..5d5b1ceab2 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -431,7 +431,7 @@ pub const Parser = struct { }, Token.Id.Keyword_async => { // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, Token(undefined), ctx.extern_token, ctx.lib_name, (?Token)(null), (?Token)(null), (?Token)(null)); const async_node = try arena.create(ast.NodeAsyncAttribute); From f6c77746d6be1bb238bf632f74c6dfc28de034ed Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Apr 2018 22:24:01 -0400 Subject: [PATCH 53/54] add memmove to builtin.o related: #514 --- std/special/builtin.zig | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/std/special/builtin.zig b/std/special/builtin.zig index 268d0ab545..9de0aa7679 100644 --- a/std/special/builtin.zig +++ b/std/special/builtin.zig @@ -14,26 +14,43 @@ pub fn panic(msg: []const u8, error_return_trace: ?&builtin.StackTrace) noreturn } } -// Note that memset does not return `dest`, like the libc API. -// The semantics of memset is dictated by the corresponding -// LLVM intrinsics, not by the libc API. -export fn memset(dest: ?&u8, c: u8, n: usize) void { +export fn memset(dest: ?&u8, c: u8, n: usize) ?&u8 { @setRuntimeSafety(false); var index: usize = 0; while (index != n) : (index += 1) (??dest)[index] = c; + + return dest; } -// Note that memcpy does not return `dest`, like the libc API. -// The semantics of memcpy is dictated by the corresponding -// LLVM intrinsics, not by the libc API. -export fn memcpy(noalias dest: ?&u8, noalias src: ?&const u8, n: usize) void { +export fn memcpy(noalias dest: ?&u8, noalias src: ?&const u8, n: usize) ?&u8 { @setRuntimeSafety(false); var index: usize = 0; while (index != n) : (index += 1) (??dest)[index] = (??src)[index]; + + return dest; +} + +export fn memmove(dest: ?&u8, src: ?&const u8, n: usize) ?&u8 { + @setRuntimeSafety(false); + + if (@ptrToInt(dest) < @ptrToInt(src)) { + var index: usize = 0; + while (index != n) : (index += 1) { + (??dest)[index] = (??src)[index]; + } + } else { + var index = n; + while (index != 0) { + index -= 1; + (??dest)[index] = (??src)[index]; + } + } + + return dest; } comptime { From 405a2390f09c78fb5db435e97d8ff23c0b44753b Mon Sep 17 00:00:00 2001 From: Josh Wolfe Date: Tue, 10 Apr 2018 22:44:55 -0400 Subject: [PATCH 54/54] zig fmt while-else with no blocks --- std/zig/parser.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 5d5b1ceab2..11b551fec0 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -3839,12 +3839,13 @@ pub const Parser = struct { }, ast.Node.Id.Else => { const else_node = @fieldParentPtr(ast.NodeElse, "base", base); - try stream.print("{} ", self.tokenizer.getTokenSlice(else_node.else_token)); + try stream.print("{}", self.tokenizer.getTokenSlice(else_node.else_token)); switch (else_node.body.id) { ast.Node.Id.Block, ast.Node.Id.If, ast.Node.Id.For, ast.Node.Id.While, ast.Node.Id.Switch => { + try stream.print(" "); try stack.append(RenderState { .Expression = else_node.body }); }, else => { @@ -4805,6 +4806,11 @@ test "zig fmt: while" { \\ unreachable; \\ }; \\ + \\ const res = while (0 < 10) + \\ break 7 + \\ else + \\ unreachable; + \\ \\ var a: error!u8 = 0; \\ while (a) |v| { \\ a = error.Err;