From b92f42d1f4b91bb343558f72af84ea052da4ac98 Mon Sep 17 00:00:00 2001 From: Vexu <15308111+Vexu@users.noreply.github.com> Date: Sat, 12 Oct 2019 12:38:09 +0300 Subject: [PATCH 1/4] implemented container doc comments in stage 2 --- lib/std/zig/ast.zig | 1 - lib/std/zig/parse.zig | 35 ++++++++++++++++----- lib/std/zig/parser_test.zig | 56 +++++++++++++++++++++++++++++++++ lib/std/zig/render.zig | 11 +++++++ lib/std/zig/tokenizer.zig | 14 ++++++++- src-self-hosted/translate_c.zig | 1 - 6 files changed, 108 insertions(+), 10 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index e705aea477..65fbc51606 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -576,7 +576,6 @@ pub const Node = struct { pub const Root = struct { base: Node, - doc_comments: ?*DocComment, decls: DeclList, eof_token: TokenIndex, diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 6d5d4b5f2d..d803beeffa 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -58,13 +58,6 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error node.* = Node.Root{ .base = Node{ .id = .Root }, .decls = undefined, - // TODO: Because zig fmt collapses consecutive comments separated by blank lines into - // a single multi-line comment, it is currently impossible to have a container-level - // doc comment and NO doc comment on the first decl. For now, simply - // ignore the problem and assume that there will be no container-level - // doc comments. - // See: https://github.com/ziglang/zig/issues/2288 - .doc_comments = null, .eof_token = undefined, }; node.decls = parseContainerMembers(arena, it, tree) catch |err| { @@ -94,6 +87,11 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No var list = Node.Root.DeclList.init(arena); while (true) { + if (try parseContainerDocComments(arena, it, tree)) |node| { + try list.push(node); + continue; + } + const doc_comments = try parseDocComment(arena, it, tree); if (try parseTestDecl(arena, it, tree)) |node| { @@ -155,12 +153,35 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No continue; } + // Dangling doc comment + if (doc_comments != null) { + try tree.errors.push(AstError{ + .UnattachedDocComment = AstError.UnattachedDocComment{ .token = doc_comments.?.firstToken() }, + }); + } break; } return list; } +/// Eat a multiline container doc comment +fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { + var lines = Node.DocComment.LineList.init(arena); + while (eatToken(it, .ContainerDocComment)) |line| { + try lines.push(line); + } + + if (lines.len == 0) return null; + + const node = try arena.create(Node.DocComment); + node.* = Node.DocComment{ + .base = Node{ .id = .DocComment }, + .lines = lines, + }; + return &node.base; +} + /// TestDecl <- KEYWORD_test STRINGLITERAL Block fn parseTestDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { const test_token = eatToken(it, .Keyword_test) orelse return null; diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index cabbe823f6..b429a575a3 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -2556,6 +2556,62 @@ test "zig fmt: comments at several places in struct init" { ); } +test "zig fmt: top level doc comments" { + try testCanonical( + \\//! tld 1 + \\//! tld 2 + \\//! tld 3 + \\ + \\// comment + \\ + \\/// A doc + \\const A = struct { + \\ //! A tld 1 + \\ //! A tld 2 + \\ //! A tld 3 + \\}; + \\ + \\/// B doc + \\const B = struct { + \\ //! B tld 1 + \\ //! B tld 2 + \\ //! B tld 3 + \\ + \\ /// b doc + \\ b: u32, + \\}; + \\ + \\/// C doc + \\const C = struct { + \\ //! C tld 1 + \\ //! C tld 2 + \\ //! C tld 3 + \\ + \\ /// c1 doc + \\ c1: u32, + \\ + \\ //! C tld 4 + \\ //! C tld 5 + \\ //! C tld 6 + \\ + \\ /// c2 doc + \\ c2: u32, + \\}; + \\ + ); + try testCanonical( + \\//! Top-level documentation. + \\ + \\/// This is A + \\pub const A = usize; + \\ + ); + try testCanonical( + \\//! Nothing here + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index c824939296..258a5493de 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -299,6 +299,17 @@ fn renderTopLevelDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, i assert(!decl.requireSemiColon()); try renderExpression(allocator, stream, tree, indent, start_col, decl, Space.Newline); }, + + ast.Node.Id.DocComment => { + const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl); + var it = comment.lines.iterator(0); + while (it.next()) |line_token_index| { + try renderToken(tree, stream, line_token_index.*, indent, start_col, Space.Newline); + if (it.peek()) |_| { + try stream.writeByteNTimes(' ', indent); + } + } + }, else => unreachable, } } diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index ae82cb7602..d3646a6a6b 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -142,6 +142,7 @@ pub const Token = struct { FloatLiteral, LineComment, DocComment, + ContainerDocComment, BracketStarBracket, BracketStarCBracket, ShebangLine, @@ -211,6 +212,7 @@ pub const Token = struct { .FloatLiteral => "FloatLiteral", .LineComment => "LineComment", .DocComment => "DocComment", + .ContainerDocComment => "ContainerDocComment", .ShebangLine => "ShebangLine", .Bang => "!", @@ -387,6 +389,7 @@ pub const Tokenizer = struct { LineComment, DocCommentStart, DocComment, + ContainerDocComment, Zero, IntegerLiteral, IntegerLiteralWithRadix, @@ -1076,6 +1079,10 @@ pub const Tokenizer = struct { '/' => { state = State.DocCommentStart; }, + '!' => { + result.id = Token.Id.ContainerDocComment; + state = State.ContainerDocComment; + }, '\n' => break, else => { state = State.LineComment; @@ -1096,7 +1103,7 @@ pub const Tokenizer = struct { self.checkLiteralCharacter(); }, }, - State.LineComment, State.DocComment => switch (c) { + State.LineComment, State.DocComment, State.ContainerDocComment => switch (c) { '\n' => break, else => self.checkLiteralCharacter(), }, @@ -1234,6 +1241,9 @@ pub const Tokenizer = struct { State.DocComment, State.DocCommentStart => { result.id = Token.Id.DocComment; }, + State.ContainerDocComment => { + result.id = Token.Id.ContainerDocComment; + }, State.NumberDot, State.NumberDotHex, @@ -1601,6 +1611,8 @@ test "tokenizer - line comment and doc comment" { testTokenize("/// a", [_]Token.Id{Token.Id.DocComment}); testTokenize("///", [_]Token.Id{Token.Id.DocComment}); testTokenize("////", [_]Token.Id{Token.Id.LineComment}); + testTokenize("//!", [_]Token.Id{Token.Id.ContainerDocComment}); + testTokenize("//!!", [_]Token.Id{Token.Id.ContainerDocComment}); } test "tokenizer - line comment followed by identifier" { diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index f8d3a12343..e919977116 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -174,7 +174,6 @@ pub fn translate( tree.root_node.* = ast.Node.Root{ .base = ast.Node{ .id = ast.Node.Id.Root }, .decls = ast.Node.Root.DeclList.init(arena), - .doc_comments = null, // initialized with the eof token at the end .eof_token = undefined, }; From e509d21f39af726da3c6001b0c20f2c765be07a5 Mon Sep 17 00:00:00 2001 From: Vexu <15308111+Vexu@users.noreply.github.com> Date: Fri, 15 Nov 2019 14:12:14 +0200 Subject: [PATCH 2/4] implemented container doc comments in stage 1 --- src/all_types.hpp | 1 + src/parser.cpp | 23 ++++++++++++++++++++++- src/tokenizer.cpp | 32 ++++++++++++++++++++++++++++++++ src/tokenizer.hpp | 1 + 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index a1e9d76be7..c4178cb130 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -968,6 +968,7 @@ struct AstNodeContainerDecl { AstNode *init_arg_expr; // enum(T), struct(endianness), or union(T), or union(enum(T)) ZigList fields; ZigList decls; + Buf doc_comments; ContainerKind kind; ContainerLayout layout; diff --git a/src/parser.cpp b/src/parser.cpp index 484e145cfa..eef832e32c 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -493,6 +493,7 @@ static AstNode *ast_parse_root(ParseContext *pc) { node->data.container_decl.layout = ContainerLayoutAuto; node->data.container_decl.kind = ContainerKindStruct; node->data.container_decl.is_root = true; + node->data.container_decl.doc_comments = members.doc_comments; return node; } @@ -514,6 +515,21 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) { return first_doc_token; } +static void ast_parse_container_doc_comments(ParseContext *pc, Buf *buf) { + if (buf_len(buf) != 0 && peek_token(pc)->id == TokenIdContainerDocComment) { + buf_append_char(buf, '\n'); + } + Token *doc_token = nullptr; + while ((doc_token = eat_token_if(pc, TokenIdContainerDocComment))) { + if (buf->list.length == 0) { + buf_resize(buf, 0); + } + // chops off '//!' but leaves '\n' + buf_append_mem(buf, buf_ptr(pc->buf) + doc_token->start_pos + 3, + doc_token->end_pos - doc_token->start_pos - 3); + } +} + // ContainerMembers // <- TestDecl ContainerMembers // / TopLevelComptime ContainerMembers @@ -523,7 +539,11 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) { // / static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { AstNodeContainerDecl res = {}; + Buf tld_doc_comment_buf = BUF_INIT; + buf_resize(&tld_doc_comment_buf, 0); for (;;) { + ast_parse_container_doc_comments(pc, &tld_doc_comment_buf); + AstNode *test_decl = ast_parse_test_decl(pc); if (test_decl != nullptr) { res.decls.append(test_decl); @@ -566,7 +586,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { break; } - + res.doc_comments = tld_doc_comment_buf; return res; } @@ -2797,6 +2817,7 @@ static AstNode *ast_parse_container_decl_auto(ParseContext *pc) { res->data.container_decl.fields = members.fields; res->data.container_decl.decls = members.decls; + res->data.container_decl.doc_comments = members.doc_comments; return res; } diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 8b301f85ac..7ece5ff3fe 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -198,6 +198,7 @@ enum TokenizeState { TokenizeStateSawSlash, TokenizeStateSawSlash2, TokenizeStateSawSlash3, + TokenizeStateSawSlashBang, TokenizeStateSawBackslash, TokenizeStateSawPercent, TokenizeStateSawPlus, @@ -209,6 +210,7 @@ enum TokenizeState { TokenizeStateSawBar, TokenizeStateSawBarBar, TokenizeStateDocComment, + TokenizeStateContainerDocComment, TokenizeStateLineComment, TokenizeStateLineString, TokenizeStateLineStringEnd, @@ -938,6 +940,9 @@ void tokenize(Buf *buf, Tokenization *out) { case '/': t.state = TokenizeStateSawSlash3; break; + case '!': + t.state = TokenizeStateSawSlashBang; + break; case '\n': cancel_token(&t); t.state = TokenizeStateStart; @@ -965,6 +970,19 @@ void tokenize(Buf *buf, Tokenization *out) { break; } break; + case TokenizeStateSawSlashBang: + switch (c) { + case '\n': + set_token_id(&t, t.cur_tok, TokenIdContainerDocComment); + end_token(&t); + t.state = TokenizeStateStart; + break; + default: + set_token_id(&t, t.cur_tok, TokenIdContainerDocComment); + t.state = TokenizeStateContainerDocComment; + break; + } + break; case TokenizeStateSawBackslash: switch (c) { case '\\': @@ -1055,6 +1073,17 @@ void tokenize(Buf *buf, Tokenization *out) { break; } break; + case TokenizeStateContainerDocComment: + switch (c) { + case '\n': + end_token(&t); + t.state = TokenizeStateStart; + break; + default: + // do nothing + break; + } + break; case TokenizeStateSymbolFirstC: switch (c) { case '"': @@ -1545,6 +1574,7 @@ void tokenize(Buf *buf, Tokenization *out) { case TokenizeStateSawBarBar: case TokenizeStateLBracket: case TokenizeStateDocComment: + case TokenizeStateContainerDocComment: end_token(&t); break; case TokenizeStateSawDotDot: @@ -1559,6 +1589,7 @@ void tokenize(Buf *buf, Tokenization *out) { case TokenizeStateLineComment: case TokenizeStateSawSlash2: case TokenizeStateSawSlash3: + case TokenizeStateSawSlashBang: break; } if (t.state != TokenizeStateError) { @@ -1606,6 +1637,7 @@ const char * token_name(TokenId id) { case TokenIdDash: return "-"; case TokenIdDivEq: return "/="; case TokenIdDocComment: return "DocComment"; + case TokenIdContainerDocComment: return "ContainerDocComment"; case TokenIdDot: return "."; case TokenIdDotStar: return ".*"; case TokenIdEllipsis2: return ".."; diff --git a/src/tokenizer.hpp b/src/tokenizer.hpp index ee336123bb..149e58b6a7 100644 --- a/src/tokenizer.hpp +++ b/src/tokenizer.hpp @@ -43,6 +43,7 @@ enum TokenId { TokenIdDash, TokenIdDivEq, TokenIdDocComment, + TokenIdContainerDocComment, TokenIdDot, TokenIdDotStar, TokenIdEllipsis2, From 977b6138817c4b09d6d1e47fa3562d9f9e0268a8 Mon Sep 17 00:00:00 2001 From: Vexu <15308111+Vexu@users.noreply.github.com> Date: Fri, 15 Nov 2019 15:03:28 +0200 Subject: [PATCH 3/4] add container doc comments to generated docs --- lib/std/special/docs/index.html | 2 +- lib/std/special/docs/main.js | 31 +++++++++++++++++++------------ lib/std/zig/ast.zig | 1 - src/dump_analysis.cpp | 1 + src/parser.cpp | 8 ++++++-- 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/lib/std/special/docs/index.html b/lib/std/special/docs/index.html index 81df8f30f1..b170ad4a14 100644 --- a/lib/std/special/docs/index.html +++ b/lib/std/special/docs/index.html @@ -484,7 +484,7 @@ doc comments.

- +