diff --git a/lib/std/c/ast.zig b/lib/std/c/ast.zig index bb6eb54d21..4bb42647cb 100644 --- a/lib/std/c/ast.zig +++ b/lib/std/c/ast.zig @@ -576,7 +576,7 @@ pub const Node = struct { asterisk: ?TokenIndex, static: ?TokenIndex, qual: TypeQual, - // expr: *Expr, + expr: *Expr, }, }, rbracket: TokenIndex, diff --git a/lib/std/c/parse.zig b/lib/std/c/parse.zig index d3a96e9aa2..253df5981b 100644 --- a/lib/std/c/parse.zig +++ b/lib/std/c/parse.zig @@ -75,9 +75,12 @@ pub fn parse(allocator: *Allocator, source: []const u8, options: Options) !*Tree } } + var parse_arena = std.heap.ArenaAllocator.init(allocator); + defer parse_arena.deinit(); + var parser = Parser{ - .symbols = Parser.SymbolList.init(allocator), - .arena = arena, + .scopes = Parser.SymbolList.init(allocator), + .arena = &parse_arena.allocator, .it = &it, .tree = tree, .options = options, @@ -93,11 +96,17 @@ const Parser = struct { it: *TokenIterator, tree: *Tree, - /// only used for scopes - symbols: SymbolList, + arena: *Allocator, + scopes: ScopeList, options: Options, - const SymbolList = std.ArrayList(Symbol); + const ScopeList = std.SegmentedLists(Scope); + const SymbolList = std.SegmentedLists(Symbol); + + const Scope = struct { + kind: ScopeKind, + syms: SymbolList, + }; const Symbol = struct { name: []const u8, @@ -111,21 +120,27 @@ const Parser = struct { Switch, }; - fn pushScope(parser: *Parser, kind: ScopeKind) usize { - return parser.symbols.len; + fn pushScope(parser: *Parser, kind: ScopeKind) !void { + const new = try parser.scopes.addOne(); + new.* = .{ + .kind = kind, + .syms = SymbolList.init(parser.arena), + }; } fn popScope(parser: *Parser, len: usize) void { - parser.symbols.resize(len) catch unreachable; + _ = parser.scopes.pop(); } - fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Type { + fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Symbol { const name = parser.tree.tokenSlice(tok); - const syms = parser.symbols.toSliceConst(); - var i = syms.len; - while (i > 0) : (i -= 1) { - if (mem.eql(u8, name, syms[i].name)) { - return syms[i].ty; + var scope_it = parser.scopes.iterator(parser.scopes.len); + while (scope_it.prev()) |scope| { + var sym_it = scope.syms.iterator(scope.syms.len); + while (sym_it.prev()) |sym| { + if (mem.eql(u8, sym.name, name)) { + return sym; + } } } return null; @@ -137,8 +152,8 @@ const Parser = struct { /// Root <- ExternalDeclaration* eof fn root(parser: *Parser) Allocator.Error!*Node.Root { - const scope = parser.pushScope(.Root); - defer parser.popScope(scope); + try parser.pushScope(.Root); + defer parser.popScope(); const node = try parser.arena.create(Node.Root); node.* = .{ .decls = Node.Root.DeclList.init(parser.arena), @@ -782,8 +797,8 @@ const Parser = struct { .ty = ty, }); if (parser.eatToken(.LBrace)) |lbrace| { - const scope = parser.pushScope(.Block); - defer parser.popScope(scope); + try parser.pushScope(.Block); + defer parser.popScope(); var fields = Node.RecordType.FieldList.init(parser.arena); while (true) { if (parser.eatToken(.RBrace)) |rbrace| { @@ -996,15 +1011,14 @@ const Parser = struct { fn assignmentExpr(parser: *Parser) !*Node {} /// ConstExpr <- ConditionalExpr - fn constExpr(parser: *Parser) Error!*Node { + fn constExpr(parser: *Parser) Error!?*Expr { const start = parser.it.index; const expression = try parser.conditionalExpr(); - // TODO - // if (expression == nullor expression.?.value == null) - // return parser.err(.{ - // .ConsExpr = start, - // }); - return expression.?; + if (expression != null and expression.?.value == .None) + return parser.err(.{ + .ConsExpr = start, + }); + return expression; } /// ConditionalExpr <- LogicalOrExpr (QUESTIONMARK Expr COLON ConditionalExpr)? @@ -1085,8 +1099,8 @@ const Parser = struct { /// CompoundStmt <- LBRACE (Declaration / Stmt)* RBRACE fn compoundStmt(parser: *Parser) Error!?*Node { const lbrace = parser.eatToken(.LBrace) orelse return null; - const scope = parser.pushScope(.Block); - defer parser.popScope(scope); + try parser.pushScope(.Block); + defer parser.popScope(); const body_node = try parser.arena.create(Node.CompoundStmt); body_node.* = .{ .lbrace = lbrace, @@ -1142,8 +1156,8 @@ const Parser = struct { return &node.base; } if (parser.eatToken(.Keyword_while)) |tok| { - const scope = parser.pushScope(.Loop); - defer parser.popScope(scope); + try parser.pushScope(.Loop); + defer parser.popScope(); _ = try parser.expectToken(.LParen); const cond = (try parser.expr()) orelse return parser.err(.{ .ExpectedExpr = .{ .token = parser.it.index }, @@ -1160,8 +1174,8 @@ const Parser = struct { return &node.base; } if (parser.eatToken(.Keyword_do)) |tok| { - const scope = parser.pushScope(.Loop); - defer parser.popScope(scope); + try parser.pushScope(.Loop); + defer parser.popScope(); const body = try parser.stmt(); _ = try parser.expectToken(.LParen); const cond = (try parser.expr()) orelse return parser.err(.{ @@ -1179,8 +1193,8 @@ const Parser = struct { return &node.base; } if (parser.eatToken(.Keyword_for)) |tok| { - const scope = parser.pushScope(.Loop); - defer parser.popScope(scope); + try parser.pushScope(.Loop); + defer parser.popScope(); _ = try parser.expectToken(.LParen); const init = if (try parser.declaration()) |decl| blk:{ // TODO disallow storage class other than auto and register @@ -1203,8 +1217,8 @@ const Parser = struct { return &node.base; } if (parser.eatToken(.Keyword_switch)) |tok| { - const scope = parser.pushScope(.Switch); - defer parser.popScope(scope); + try parser.pushScope(.Switch); + defer parser.popScope(); _ = try parser.expectToken(.LParen); const switch_expr = try parser.exprStmt(); const rparen = try parser.expectToken(.RParen); diff --git a/lib/std/c/tokenizer.zig b/lib/std/c/tokenizer.zig index 7da2e18320..a641529502 100644 --- a/lib/std/c/tokenizer.zig +++ b/lib/std/c/tokenizer.zig @@ -401,7 +401,6 @@ pub const Tokenizer = struct { U, L, StringLiteral, - AfterStringLiteral, CharLiteralStart, CharLiteral, EscapeSequence, @@ -617,7 +616,7 @@ pub const Tokenizer = struct { }, .BackSlash => switch (c) { '\n' => { - state = if (string) .AfterStringLiteral else .Start; + state = .Start; }, '\r' => { state = .BackSlashCr; @@ -632,7 +631,7 @@ pub const Tokenizer = struct { }, .BackSlashCr => switch (c) { '\n' => { - state = if (string) .AfterStringLiteral else .Start; + state = .Start; }, else => { result.id = .Invalid; @@ -696,7 +695,8 @@ pub const Tokenizer = struct { state = .EscapeSequence; }, '"' => { - state = .AfterStringLiteral; + self.index += 1; + break; }, '\n', '\r' => { result.id = .Invalid; @@ -704,22 +704,6 @@ pub const Tokenizer = struct { }, else => {}, }, - .AfterStringLiteral => switch (c) { - '"' => { - state = .StringLiteral; - }, - '\\' => { - state = .BackSlash; - }, - '\n', '\r' => { - if (self.pp_directive) - break; - }, - '\t', '\x0B', '\x0C', ' ' => {}, - else => { - break; - }, - }, .CharLiteralStart => switch (c) { '\\' => { string = false; @@ -1255,7 +1239,7 @@ pub const Tokenizer = struct { } } else if (self.index == self.source.buffer.len) { switch (state) { - .AfterStringLiteral, .Start => {}, + .Start => {}, .u, .u8, .U, .L, .Identifier => { result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier; }, @@ -1322,7 +1306,7 @@ pub const Tokenizer = struct { test "operators" { expectTokens( - \\ ! != | || |= = == + \\ ! != | || |= = == \\ ( ) { } [ ] . .. ... \\ ^ ^= + ++ += - -- -= \\ * *= % %= -> : ; / /= @@ -1505,24 +1489,27 @@ test "line continuation" { .Identifier, .Nl, .{ .StringLiteral = .None }, + .Nl, .Hash, .Keyword_define, .{ .StringLiteral = .None }, .Nl, .{ .StringLiteral = .None }, + .Nl, .Hash, .Keyword_define, .{ .StringLiteral = .None }, + .{ .StringLiteral = .None }, }); } test "string prefix" { expectTokens( - \\"foo" "bar" - \\u"foo" "bar" - \\u8"foo" "bar" - \\U"foo" "bar" - \\L"foo" "bar" + \\"foo" + \\u"foo" + \\u8"foo" + \\U"foo" + \\L"foo" \\'foo' \\u'foo' \\U'foo' @@ -1530,10 +1517,15 @@ test "string prefix" { \\ , &[_]Token.Id{ .{ .StringLiteral = .None }, + .Nl, .{ .StringLiteral = .Utf16 }, + .Nl, .{ .StringLiteral = .Utf8 }, + .Nl, .{ .StringLiteral = .Utf32 }, + .Nl, .{ .StringLiteral = .Wide }, + .Nl, .{ .CharLiteral = .None }, .Nl, .{ .CharLiteral = .Utf16 },