mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2026-02-12 20:37:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

zig fmt: start reworking with new memory layout

Browse Source 
* start implementation of ast.Tree.firstToken and lastToken
 * clarify some ast.Node doc comments
 * reimplement renderToken
This commit is contained in:
Andrew Kelley 2021-02-01 17:23:49 -07:00
parent bf8fafc37d
commit 20554d32c0
5 changed files with 6121 additions and 6080 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

300
lib/std/zig/ast.zig
View File

@ -185,24 +185,293 @@ pub const Tree = struct {
}
}
/// Skips over comments.
pub fn prevToken(self: *const Tree, token_index: TokenIndex) TokenIndex {
const token_tags = self.tokens.items(.tag);
var index = token_index - 1;
while (token_tags[index] == .LineComment) {
index -= 1;
pub fn firstToken(tree: Tree, node: Node.Index) TokenIndex {
const tags = tree.nodes.items(.tag);
const datas = tree.nodes.items(.data);
const main_tokens = tree.nodes.items(.main_token);
switch (tags[node]) {
.Root => return 0,
.UsingNamespace,
.TestDecl,
.ErrDefer,
.Defer,
.BoolNot,
.Negation,
.BitNot,
.NegationWrap,
.AddressOf,
.Try,
.Await,
.OptionalType,
.ArrayInitDotTwo,
.ArrayInitDot,
.StructInitDotTwo,
.StructInitDot,
.Switch,
.IfSimple,
.IfSimpleOptional,
.If,
.IfOptional,
.IfError,
.Suspend,
.Resume,
.Continue,
.Break,
.Return,
.AnyFrameType,
.OneToken,
.Identifier,
.EnumLiteral,
.MultilineStringLiteral,
.GroupedExpression,
.BuiltinCallTwo,
.BuiltinCall,
.ErrorSetDecl,
.AnyType,
.Comptime,
.Nosuspend,
.Block,
.AsmSimple,
.Asm,
=> return main_tokens[node],
.Catch,
.FieldAccess,
.UnwrapOptional,
.EqualEqual,
.BangEqual,
.LessThan,
.GreaterThan,
.LessOrEqual,
.GreaterOrEqual,
.AssignMul,
.AssignDiv,
.AssignMod,
.AssignAdd,
.AssignSub,
.AssignBitShiftLeft,
.AssignBitShiftRight,
.AssignBitAnd,
.AssignBitXor,
.AssignBitOr,
.AssignMulWrap,
.AssignAddWrap,
.AssignSubWrap,
.Assign,
.MergeErrorSets,
.Mul,
.Div,
.Mod,
.ArrayMult,
.MulWrap,
.Add,
.Sub,
.ArrayCat,
.AddWrap,
.SubWrap,
.BitShiftLeft,
.BitShiftRight,
.BitAnd,
.BitXor,
.BitOr,
.OrElse,
.BoolAnd,
.BoolOr,
.SliceOpen,
.Slice,
.Deref,
.ArrayAccess,
.ArrayInitOne,
.ArrayInit,
.StructInitOne,
.CallOne,
.Call,
.SwitchCaseOne,
.SwitchRange,
.FnDecl,
=> return tree.firstToken(datas[node].lhs),
.GlobalVarDecl,
.LocalVarDecl,
.SimpleVarDecl,
.AlignedVarDecl,
.ArrayType,
.ArrayTypeSentinel,
.PtrTypeAligned,
.PtrTypeSentinel,
.PtrType,
.SliceType,
.StructInit,
.SwitchCaseMulti,
.WhileSimple,
.WhileSimpleOptional,
.WhileCont,
.WhileContOptional,
.While,
.WhileOptional,
.WhileError,
.ForSimple,
.For,
.FnProtoSimple,
.FnProtoSimpleMulti,
.FnProtoOne,
.FnProto,
.ContainerDecl,
.ContainerDeclArg,
.TaggedUnion,
.TaggedUnionEnumTag,
.ContainerFieldInit,
.ContainerFieldAlign,
.ContainerField,
.AsmOutput,
.AsmInput,
.ErrorValue,
.ErrorUnion,
=> @panic("TODO finish implementing firstToken"),
}
return index;
}
/// Skips over comments.
pub fn nextToken(self: *const Tree, token_index: TokenIndex) TokenIndex {
const token_tags = self.tokens.items(.tag);
var index = token_index + 1;
while (token_tags[index] == .LineComment) {
index += 1;
pub fn lastToken(tree: Tree, node: Node.Index) TokenIndex {
const tags = tree.nodes.items(.tag);
const datas = tree.nodes.items(.data);
const main_tokens = tree.nodes.items(.main_token);
switch (tags[node]) {
.Root,
.UsingNamespace,
.TestDecl,
.ErrDefer,
.Defer,
.BoolNot,
.Negation,
.BitNot,
.NegationWrap,
.AddressOf,
.Try,
.Await,
.OptionalType,
.ArrayInitDotTwo,
.ArrayInitDot,
.StructInitDotTwo,
.StructInitDot,
.Switch,
.IfSimple,
.IfSimpleOptional,
.If,
.IfOptional,
.IfError,
.Suspend,
.Resume,
.Continue,
.Break,
.Return,
.AnyFrameType,
.OneToken,
.Identifier,
.EnumLiteral,
.MultilineStringLiteral,
.GroupedExpression,
.BuiltinCallTwo,
.BuiltinCall,
.ErrorSetDecl,
.AnyType,
.Comptime,
.Nosuspend,
.Block,
.AsmSimple,
.Asm,
.Catch,
.FieldAccess,
.UnwrapOptional,
.EqualEqual,
.BangEqual,
.LessThan,
.GreaterThan,
.LessOrEqual,
.GreaterOrEqual,
.AssignMul,
.AssignDiv,
.AssignMod,
.AssignAdd,
.AssignSub,
.AssignBitShiftLeft,
.AssignBitShiftRight,
.AssignBitAnd,
.AssignBitXor,
.AssignBitOr,
.AssignMulWrap,
.AssignAddWrap,
.AssignSubWrap,
.Assign,
.MergeErrorSets,
.Mul,
.Div,
.Mod,
.ArrayMult,
.MulWrap,
.Add,
.Sub,
.ArrayCat,
.AddWrap,
.SubWrap,
.BitShiftLeft,
.BitShiftRight,
.BitAnd,
.BitXor,
.BitOr,
.OrElse,
.BoolAnd,
.BoolOr,
.SliceOpen,
.Slice,
.Deref,
.ArrayAccess,
.ArrayInitOne,
.ArrayInit,
.StructInitOne,
.CallOne,
.Call,
.SwitchCaseOne,
.SwitchRange,
.FnDecl,
.GlobalVarDecl,
.LocalVarDecl,
.SimpleVarDecl,
.AlignedVarDecl,
.ArrayType,
.ArrayTypeSentinel,
.PtrTypeAligned,
.PtrTypeSentinel,
.PtrType,
.SliceType,
.StructInit,
.SwitchCaseMulti,
.WhileSimple,
.WhileSimpleOptional,
.WhileCont,
.WhileContOptional,
.While,
.WhileOptional,
.WhileError,
.ForSimple,
.For,
.FnProtoSimple,
.FnProtoSimpleMulti,
.FnProtoOne,
.FnProto,
.ContainerDecl,
.ContainerDeclArg,
.TaggedUnion,
.TaggedUnionEnumTag,
.ContainerFieldInit,
.ContainerFieldAlign,
.ContainerField,
.AsmOutput,
.AsmInput,
.ErrorValue,
.ErrorUnion,
=> @panic("TODO finish implementing lastToken"),
}
return index;
}
};
@ -454,7 +723,7 @@ pub const Node = struct {
/// lhs is test name token (must be string literal), if any.
/// rhs is the body node.
TestDecl,
/// lhs is the index into global_var_decl_list.
/// lhs is the index into extra_data.
/// rhs is the initialization expression, if any.
GlobalVarDecl,
/// `var a: x align(y) = rhs`
@ -732,6 +1001,7 @@ pub const Node = struct {
/// `nosuspend lhs`. rhs unused.
Nosuspend,
/// `{}`. `sub_list[lhs..rhs]`.
/// main_token points at the `{`.
Block,
/// `asm(lhs)`. rhs unused.
AsmSimple,

2
lib/std/zig/parse.zig
View File

@ -594,7 +594,7 @@ const Parser = struct {
p.eatToken(.Keyword_var) orelse
return null_node;
const name_token = try p.expectToken(.Identifier);
_ = try p.expectToken(.Identifier);
const type_node: Node.Index = if (p.eatToken(.Colon) == null) 0 else try p.expectTypeExpr();
const align_node = try p.parseByteAlign();
const section_node = try p.parseLinkSection();

7425
lib/std/zig/parser_test.zig
View File

File diff suppressed because it is too large Load Diff

4443
lib/std/zig/render.zig
View File

File diff suppressed because it is too large Load Diff

31
lib/std/zig/tokenizer.zig
View File

@ -195,22 +195,23 @@ pub const Token = struct {
Keyword_volatile,
Keyword_while,
pub fn symbol(tag: Tag) []const u8 {
pub fn lexeme(tag: Tag) ?[]const u8 {
return switch (tag) {
.Invalid => "Invalid",
.Invalid,
.Identifier,
.StringLiteral,
.MultilineStringLiteralLine,
.CharLiteral,
.Eof,
.Builtin,
.IntegerLiteral,
.FloatLiteral,
.DocComment,
.ContainerDocComment,
=> null,
.Invalid_ampersands => "&&",
.Invalid_periodasterisks => ".**",
.Identifier => "Identifier",
.StringLiteral => "StringLiteral",
.MultilineStringLiteralLine => "MultilineStringLiteralLine",
.CharLiteral => "CharLiteral",
.Eof => "Eof",
.Builtin => "Builtin",
.IntegerLiteral => "IntegerLiteral",
.FloatLiteral => "FloatLiteral",
.DocComment => "DocComment",
.ContainerDocComment => "ContainerDocComment",
.Bang => "!",
.Pipe => "|",
.PipePipe => "||",
@ -319,6 +320,10 @@ pub const Token = struct {
.Keyword_while => "while",
};
}
pub fn symbol(tag: Tag) []const u8 {
return tag.lexeme() orelse @tagName(tag);
}
};
};