From 866f7dc7d68156d2cb1f3a7edad0882c67943726 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 21 Feb 2021 17:37:10 -0700 Subject: [PATCH] parser: support more recovery test cases --- lib/std/zig/ast.zig | 8 ++ lib/std/zig/parse.zig | 44 +++++--- lib/std/zig/parser_test.zig | 205 ++++++++++++++++++------------------ 3 files changed, 141 insertions(+), 116 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 72c2f82d09..c81386cca4 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -205,6 +205,9 @@ pub const Tree = struct { token_tags[parse_error.token].symbol(), }); }, + .expected_pub_item => { + return stream.writeAll("expected function or variable declaration after pub"); + }, .expected_return_type => { return stream.print("expected return type expression, found '{s}'", .{ token_tags[parse_error.token].symbol(), @@ -265,6 +268,9 @@ pub const Tree = struct { .invalid_align => { return stream.writeAll("alignment not allowed on arrays"); }, + .invalid_and => { + return stream.writeAll("`&&` is invalid; note that `and` is boolean AND"); + }, .invalid_bit_range => { return stream.writeAll("bit range not allowed on slices and arrays"); }, @@ -2316,6 +2322,7 @@ pub const Error = struct { expected_param_list, expected_prefix_expr, expected_primary_type_expr, + expected_pub_item, expected_return_type, expected_semi_or_else, expected_semi_or_lbrace, @@ -2330,6 +2337,7 @@ pub const Error = struct { extra_const_qualifier, extra_volatile_qualifier, invalid_align, + invalid_and, invalid_bit_range, invalid_token, same_line_doc_comment, diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index fc57084ad8..848134cbdc 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -423,7 +423,7 @@ const Parser = struct { while (true) { const tok = p.nextToken(); switch (p.token_tags[tok]) { - // any of these can start a new top level declaration + // Any of these can start a new top level declaration. .keyword_test, .keyword_comptime, .keyword_pub, @@ -436,13 +436,18 @@ const Parser = struct { .keyword_const, .keyword_var, .keyword_fn, - .identifier, => { if (level == 0) { p.tok_i -= 1; return; } }, + .identifier => { + if (p.token_tags[tok + 1] == .comma and level == 0) { + p.tok_i -= 1; + return; + } + }, .comma, .semicolon => { // this decl was likely meant to end here if (level == 0) { @@ -531,10 +536,13 @@ const Parser = struct { fn expectTopLevelDecl(p: *Parser) !Node.Index { const extern_export_inline_token = p.nextToken(); var expect_fn: bool = false; - var exported: bool = false; + var expect_var_or_fn: bool = false; switch (p.token_tags[extern_export_inline_token]) { - .keyword_extern => _ = p.eatToken(.string_literal), - .keyword_export => exported = true, + .keyword_extern => { + _ = p.eatToken(.string_literal); + expect_var_or_fn = true; + }, + .keyword_export => expect_var_or_fn = true, .keyword_inline, .keyword_noinline => expect_fn = true, else => p.tok_i -= 1, } @@ -580,11 +588,12 @@ const Parser = struct { if (thread_local_token != null) { return p.fail(.expected_var_decl); } - - if (exported) { + if (expect_var_or_fn) { return p.fail(.expected_var_decl_or_fn); } - + if (p.token_tags[p.tok_i] != .keyword_usingnamespace) { + return p.fail(.expected_pub_item); + } return p.expectUsingNamespace(); } @@ -599,7 +608,7 @@ const Parser = struct { } fn expectUsingNamespace(p: *Parser) !Node.Index { - const usingnamespace_token = try p.expectToken(.keyword_usingnamespace); + const usingnamespace_token = p.assertToken(.keyword_usingnamespace); const expr = try p.expectExpr(); const semicolon_token = try p.expectToken(.semicolon); return p.addNode(.{ @@ -1346,6 +1355,11 @@ const Parser = struct { }, }); }, + .invalid_ampersands => { + try p.warn(.invalid_and); + p.tok_i += 1; + return p.parseCompareExpr(); + }, else => return res, } } @@ -2283,10 +2297,12 @@ const Parser = struct { if (node == 0) break; res = node; } - const lparen = (try p.expectTokenRecoverable(.l_paren)) orelse { + const lparen = p.nextToken(); + if (p.token_tags[lparen] != .l_paren) { + p.tok_i -= 1; try p.warn(.expected_param_list); return res; - }; + } if (p.eatToken(.r_paren)) |_| { return p.addNode(.{ .tag = .async_call_one, @@ -3769,7 +3785,8 @@ const Parser = struct { /// ExprList <- (Expr COMMA)* Expr? fn parseBuiltinCall(p: *Parser) !Node.Index { const builtin_token = p.assertToken(.builtin); - _ = (try p.expectTokenRecoverable(.l_paren)) orelse { + if (p.token_tags[p.nextToken()] != .l_paren) { + p.tok_i -= 1; try p.warn(.expected_param_list); // Pretend this was an identifier so we can continue parsing. return p.addNode(.{ @@ -3780,7 +3797,7 @@ const Parser = struct { .rhs = undefined, }, }); - }; + } if (p.eatToken(.r_paren)) |_| { return p.addNode(.{ .tag = .builtin_call_two, @@ -4015,6 +4032,7 @@ const Parser = struct { fn expectToken(p: *Parser, tag: Token.Tag) Error!TokenIndex { const token = p.nextToken(); if (p.token_tags[token] != tag) { + p.tok_i -= 1; // Go back so that we can recover properly. return p.failMsg(.{ .tag = .expected_token, .token = token, diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 3b58cb43ea..bc4fc797ef 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3579,7 +3579,7 @@ test "zig fmt: file ends with struct field" { // \\ // , &[_]Error{ // .expected_expr, -// .ExpectedVarDeclOrFn, +// .expected_var_decl_or_fn, // }); //} @@ -4070,24 +4070,24 @@ test "recovery: block statements" { }); } -//test "recovery: missing comma" { -// try testError( -// \\test "" { -// \\ switch (foo) { -// \\ 2 => {} -// \\ 3 => {} -// \\ else => { -// \\ foo && bar +; -// \\ } -// \\ } -// \\} -// , &[_]Error{ -// .expected_token, -// .expected_token, -// .invalid_and, -// .invalid_token, -// }); -//} +test "recovery: missing comma" { + try testError( + \\test "" { + \\ switch (foo) { + \\ 2 => {} + \\ 3 => {} + \\ else => { + \\ foo && bar +; + \\ } + \\ } + \\} + , &[_]Error{ + .expected_token, + .expected_token, + .invalid_and, + .invalid_token, + }); +} test "recovery: extra qualifier" { try testError( @@ -4099,94 +4099,93 @@ test "recovery: extra qualifier" { }); } -//test "recovery: missing return type" { -// try testError( -// \\fn foo() { -// \\ a && b; -// \\} -// \\test "" -// , &[_]Error{ -// .ExpectedReturnType, -// .invalid_and, -// .expected_block, -// }); -//} +test "recovery: missing return type" { + try testError( + \\fn foo() { + \\ a && b; + \\} + \\test "" + , &[_]Error{ + .expected_return_type, + .invalid_and, + .expected_block, + }); +} -//test "recovery: continue after invalid decl" { -// try testError( -// \\fn foo { -// \\ inline; -// \\} -// \\pub test "" { -// \\ async a && b; -// \\} -// , &[_]Error{ -// .expected_token, -// .ExpectedPubItem, -// .ExpectedParamList, -// .invalid_and, -// }); -// try testError( -// \\threadlocal test "" { -// \\ @a && b; -// \\} -// , &[_]Error{ -// .ExpectedVarDecl, -// .ExpectedParamList, -// .invalid_and, -// }); -//} +test "recovery: continue after invalid decl" { + try testError( + \\fn foo { + \\ inline; + \\} + \\pub test "" { + \\ async a && b; + \\} + , &[_]Error{ + .expected_token, + .expected_pub_item, + .expected_param_list, + .invalid_and, + }); + try testError( + \\threadlocal test "" { + \\ @a && b; + \\} + , &[_]Error{ + .expected_var_decl, + .expected_param_list, + .invalid_and, + }); +} -//test "recovery: invalid extern/inline" { -// try testError( -// \\inline test "" { a && b; } -// , &[_]Error{ -// .ExpectedFn, -// .invalid_and, -// }); -// try testError( -// \\extern "" test "" { a && b; } -// , &[_]Error{ -// .ExpectedVarDeclOrFn, -// .invalid_and, -// }); -//} +test "recovery: invalid extern/inline" { + try testError( + \\inline test "" { a && b; } + , &[_]Error{ + .expected_fn, + .invalid_and, + }); + try testError( + \\extern "" test "" { a && b; } + , &[_]Error{ + .expected_var_decl_or_fn, + .invalid_and, + }); +} -//test "recovery: missing semicolon" { -// try testError( -// \\test "" { -// \\ comptime a && b -// \\ c && d -// \\ @foo -// \\} -// , &[_]Error{ -// .invalid_and, -// .expected_token, -// .invalid_and, -// .expected_token, -// .ExpectedParamList, -// .expected_token, -// }); -//} +test "recovery: missing semicolon" { + try testError( + \\test "" { + \\ comptime a && b + \\ c && d + \\ @foo + \\} + , &[_]Error{ + .invalid_and, + .expected_token, + .invalid_and, + .expected_token, + .expected_param_list, + .expected_token, + }); +} -//test "recovery: invalid container members" { -// try testError( -// \\usingnamespace; -// \\foo+ -// \\bar@, -// \\while (a == 2) { test "" {}} -// \\test "" { -// \\ a && b -// \\} -// , &[_]Error{ -// .expected_expr, -// .expected_token, -// .expected_token, -// .expected_container_members, -// .invalid_and, -// .expected_token, -// }); -//} +test "recovery: invalid container members" { + try testError( + \\usingnamespace; + \\foo+ + \\bar@, + \\while (a == 2) { test "" {}} + \\test "" { + \\ a && b + \\} + , &[_]Error{ + .expected_expr, + .expected_token, + .expected_container_members, + .invalid_and, + .expected_token, + }); +} //test "recovery: invalid parameter" { // try testError(