mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2026-01-26 09:15:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

stage1: rework tokenizer to match stage2

Browse Source 
* Extracts AstGen logic from ir.cpp into astgen.cpp. Reduces the
   largest file of stage1 from 33,551 lines to 25,510.
 * tokenizer: rework it completely to match the stage2 tokenizer logic.
   They can now be maintained together; when one is changed, the other
   can be changed in the same way.
   - Each token now takes up 13 bytes instead of 64 bytes. The tokenizer
     does not parse char literals, string literals, integer literals,
     etc into meaningful data. Instead, that happens during parsing or
     astgen.
   - no longer store line offsets. Error messages scan source
     files to find the line/column as needed (same as stage2).
   - main loop: instead of checking the loop, handle a null byte
     explicitly in the switch statements. This is a nice improvement
     that we may want to backport to stage2.
   - delete some dead tokens, artifacts of past syntax that no longer
     exists.
 * Parser: fix a TODO by parsing builtin functions as tokens rather than
   `@` as a separate token. This is how stage2 does it.
 * Remove some debugging infrastructure. These will need to be redone,
   if at all, as the code migrates to match stage2.
   - remove the ast_render code.
   - remove the IR debugging stuff
   - remove teh token printing code
This commit is contained in:
Andrew Kelley 2021-05-27 16:32:35 -07:00
parent 673ae5b457
commit 2a990d6966
25 changed files with 10445 additions and 11427 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists.txt
View File

@ -292,7 +292,7 @@ set(ZIG0_SOURCES
set(STAGE1_SOURCES
"${CMAKE_SOURCE_DIR}/src/stage1/analyze.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/ast_render.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/astgen.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/bigfloat.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/bigint.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/buffer.cpp"
@ -307,11 +307,11 @@ set(STAGE1_SOURCES
"${CMAKE_SOURCE_DIR}/src/stage1/os.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/parser.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/range_set.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/softfloat_ext.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/stage1.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/target.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/tokenizer.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/util.cpp"
"${CMAKE_SOURCE_DIR}/src/stage1/softfloat_ext.cpp"
)
set(OPTIMIZED_C_SOURCES
"${CMAKE_SOURCE_DIR}/src/stage1/parse_f128.c"

1
lib/std/zig/string_literal.zig
View File

@ -107,6 +107,7 @@ pub fn parseAppend(buf: *std.ArrayList(u8), bytes: []const u8) error{OutOfMemory
const hex_str = slice[index + 2 .. index_end];
if (std.fmt.parseUnsigned(u32, hex_str, 16)) |uint| {
if (uint <= 0x10ffff) {
// TODO this incorrectly depends on endianness
try buf.appendSlice(std.mem.toBytes(uint)[0..]);
state = State.Start;
index = index_end; // loop-header increments

2
src/stage1.zig
View File

@ -252,6 +252,8 @@ const Error = extern enum {
ZigIsTheCCompiler,
FileBusy,
Locked,
InvalidCharacter,
UnicodePointTooLarge,
};
// ABI warning

102
src/stage1/all_types.hpp
View File

@ -670,7 +670,7 @@ enum NodeType {
NodeTypeIntLiteral,
NodeTypeStringLiteral,
NodeTypeCharLiteral,
NodeTypeSymbol,
NodeTypeIdentifier,
NodeTypePrefixOpExpr,
NodeTypePointerType,
NodeTypeFnCallExpr,
@ -710,6 +710,7 @@ enum NodeType {
NodeTypeAwaitExpr,
NodeTypeSuspend,
NodeTypeAnyFrameType,
// main_token points to the identifier.
NodeTypeEnumLiteral,
NodeTypeAnyTypeField,
};
@ -733,7 +734,8 @@ struct AstNodeFnProto {
AstNode *section_expr;
// populated if the "callconv(S)" is present
AstNode *callconv_expr;
Buf doc_comments;
TokenIndex doc_comments;
// This is set based only on the existence of a noinline or inline keyword.
// This is then resolved to an is_noinline bool and (potentially .Inline)
@ -755,8 +757,8 @@ struct AstNodeFnDef {
struct AstNodeParamDecl {
Buf *name;
AstNode *type;
Token *anytype_token;
Buf doc_comments;
TokenIndex doc_comments;
TokenIndex anytype_token;
bool is_noalias;
bool is_comptime;
bool is_var_args;
@ -799,9 +801,9 @@ struct AstNodeVariableDeclaration {
AstNode *align_expr;
// populated if the "section(S)" is present
AstNode *section_expr;
Token *threadlocal_tok;
Buf doc_comments;
TokenIndex doc_comments;
TokenIndex threadlocal_tok;
VisibMod visib_mod;
bool is_const;
bool is_comptime;
@ -935,13 +937,14 @@ struct AstNodePrefixOpExpr {
};
struct AstNodePointerType {
Token *star_token;
TokenIndex star_token;
TokenIndex allow_zero_token;
TokenIndex bit_offset_start;
TokenIndex host_int_bytes;
AstNode *sentinel;
AstNode *align_expr;
BigInt *bit_offset_start;
BigInt *host_int_bytes;
AstNode *op_expr;
Token *allow_zero_token;
bool is_const;
bool is_volatile;
};
@ -956,7 +959,7 @@ struct AstNodeArrayType {
AstNode *sentinel;
AstNode *child_type;
AstNode *align_expr;
Token *allow_zero_token;
TokenIndex allow_zero_token;
bool is_const;
bool is_volatile;
};
@ -974,11 +977,11 @@ struct AstNodeIfBoolExpr {
struct AstNodeTryExpr {
Buf *var_symbol;
bool var_is_ptr;
AstNode *target_node;
AstNode *then_node;
AstNode *else_node;
Buf *err_symbol;
bool var_is_ptr;
};
struct AstNodeTestExpr {
@ -993,12 +996,12 @@ struct AstNodeWhileExpr {
Buf *name;
AstNode *condition;
Buf *var_symbol;
bool var_is_ptr;
AstNode *continue_expr;
AstNode *body;
AstNode *else_node;
Buf *err_symbol;
bool is_inline;
bool var_is_ptr;
};
struct AstNodeForExpr {
@ -1070,7 +1073,7 @@ struct AsmToken {
};
struct AstNodeAsmExpr {
Token *volatile_token;
TokenIndex volatile_token;
AstNode *asm_template;
ZigList<AsmOutput*> output_list;
ZigList<AsmInput*> input_list;
@ -1094,7 +1097,7 @@ struct AstNodeContainerDecl {
AstNode *init_arg_expr; // enum(T), struct(endianness), or union(T), or union(enum(T))
ZigList<AstNode *> fields;
ZigList<AstNode *> decls;
Buf doc_comments;
TokenIndex doc_comments;
ContainerKind kind;
ContainerLayout layout;
@ -1103,7 +1106,7 @@ struct AstNodeContainerDecl {
};
struct AstNodeErrorSetField {
Buf doc_comments;
TokenIndex doc_comments;
AstNode *field_name;
};
@ -1118,28 +1121,8 @@ struct AstNodeStructField {
AstNode *value;
// populated if the "align(A)" is present
AstNode *align_expr;
Buf doc_comments;
Token *comptime_token;
};
struct AstNodeStringLiteral {
Buf *buf;
};
struct AstNodeCharLiteral {
uint32_t value;
};
struct AstNodeFloatLiteral {
BigFloat *bigfloat;
// overflow is true if when parsing the number, we discovered it would not
// fit without losing data in a double
bool overflow;
};
struct AstNodeIntLiteral {
BigInt *bigint;
TokenIndex doc_comments;
TokenIndex comptime_token;
};
struct AstNodeStructValueField {
@ -1158,19 +1141,6 @@ struct AstNodeContainerInitExpr {
ContainerInitKind kind;
};
struct AstNodeNullLiteral {
};
struct AstNodeUndefinedLiteral {
};
struct AstNodeThisLiteral {
};
struct AstNodeSymbolExpr {
Buf *symbol;
};
struct AstNodeBoolLiteral {
bool value;
};
@ -1188,13 +1158,6 @@ struct AstNodeContinueExpr {
Buf *name;
};
struct AstNodeUnreachableExpr {
};
struct AstNodeErrorType {
};
struct AstNodeAwaitExpr {
AstNode *expr;
};
@ -1207,16 +1170,10 @@ struct AstNodeAnyFrameType {
AstNode *payload_type; // can be NULL
};
struct AstNodeEnumLiteral {
Token *period;
Token *identifier;
};
struct AstNode {
enum NodeType type;
TokenIndex main_token;
bool already_traced_this_node;
size_t line;
size_t column;
ZigType *owner;
union {
AstNodeFnDef fn_def;
@ -1252,30 +1209,19 @@ struct AstNode {
AstNodePtrDerefExpr ptr_deref_expr;
AstNodeContainerDecl container_decl;
AstNodeStructField struct_field;
AstNodeStringLiteral string_literal;
AstNodeCharLiteral char_literal;
AstNodeFloatLiteral float_literal;
AstNodeIntLiteral int_literal;
AstNodeContainerInitExpr container_init_expr;
AstNodeStructValueField struct_val_field;
AstNodeNullLiteral null_literal;
AstNodeUndefinedLiteral undefined_literal;
AstNodeThisLiteral this_literal;
AstNodeSymbolExpr symbol_expr;
AstNodeBoolLiteral bool_literal;
AstNodeBreakExpr break_expr;
AstNodeContinueExpr continue_expr;
AstNodeUnreachableExpr unreachable_expr;
AstNodeArrayType array_type;
AstNodeInferredArrayType inferred_array_type;
AstNodeErrorType error_type;
AstNodeErrorSetDecl err_set_decl;
AstNodeErrorSetField err_set_field;
AstNodeResumeExpr resume_expr;
AstNodeAwaitExpr await_expr;
AstNodeSuspend suspend;
AstNodeAnyFrameType anyframe_type;
AstNodeEnumLiteral enum_literal;
} data;
// This is a function for use in the debugger to print
@ -1409,9 +1355,11 @@ struct ZigPackage {
struct RootStruct {
ZigPackage *package;
Buf *path; // relative to root_package->root_src_dir
ZigList<size_t> *line_offsets;
Buf *source_code;
ZigLLVMDIFile *di_file;
size_t token_count;
TokenId *token_ids;
TokenLoc *token_locs;
};
enum StructSpecial {

155
src/stage1/analyze.cpp
View File

@ -6,9 +6,9 @@
*/
#include "analyze.hpp"
#include "ast_render.hpp"
#include "codegen.hpp"
#include "error.hpp"
#include "astgen.hpp"
#include "ir.hpp"
#include "ir_print.hpp"
#include "os.hpp"
@ -41,41 +41,44 @@ static bool is_top_level_struct(ZigType *import) {
return import->id == ZigTypeIdStruct && import->data.structure.root_struct != nullptr;
}
static ErrorMsg *add_error_note_token(CodeGen *g, ErrorMsg *parent_msg, ZigType *owner, Token *token, Buf *msg) {
static ErrorMsg *add_error_note_token(CodeGen *g, ErrorMsg *parent_msg, ZigType *owner,
TokenIndex token, Buf *msg)
{
assert(is_top_level_struct(owner));
RootStruct *root_struct = owner->data.structure.root_struct;
ErrorMsg *err = err_msg_create_with_line(root_struct->path, token->start_line, token->start_column,
root_struct->source_code, root_struct->line_offsets, msg);
uint32_t byte_offset = root_struct->token_locs[token].offset;
ErrorMsg *err = err_msg_create_with_offset(root_struct->path, byte_offset,
buf_ptr(root_struct->source_code), msg);
err_msg_add_note(parent_msg, err);
return err;
}
ErrorMsg *add_token_error(CodeGen *g, ZigType *owner, Token *token, Buf *msg) {
ErrorMsg *add_token_error_offset(CodeGen *g, ZigType *owner, TokenIndex token, Buf *msg,
uint32_t bad_index)
{
assert(is_top_level_struct(owner));
RootStruct *root_struct = owner->data.structure.root_struct;
ErrorMsg *err = err_msg_create_with_line(root_struct->path, token->start_line, token->start_column,
root_struct->source_code, root_struct->line_offsets, msg);
uint32_t byte_offset = root_struct->token_locs[token].offset + bad_index;
ErrorMsg *err = err_msg_create_with_offset(root_struct->path, byte_offset,
buf_ptr(root_struct->source_code), msg);
g->errors.append(err);
g->trace_err = err;
return err;
}
ErrorMsg *add_token_error(CodeGen *g, ZigType *owner, TokenIndex token, Buf *msg) {
return add_token_error_offset(g, owner, token, msg, 0);
}
ErrorMsg *add_node_error(CodeGen *g, AstNode *node, Buf *msg) {
Token fake_token;
fake_token.start_line = node->line;
fake_token.start_column = node->column;
node->already_traced_this_node = true;
return add_token_error(g, node->owner, &fake_token, msg);
return add_token_error(g, node->owner, node->main_token, msg);
}
ErrorMsg *add_error_note(CodeGen *g, ErrorMsg *parent_msg, const AstNode *node, Buf *msg) {
Token fake_token;
fake_token.start_line = node->line;
fake_token.start_column = node->column;
return add_error_note_token(g, parent_msg, node->owner, &fake_token, msg);
return add_error_note_token(g, parent_msg, node->owner, node->main_token, msg);
}
ZigType *new_type_table_entry(ZigTypeId id) {
@ -913,13 +916,27 @@ ZigType *get_slice_type(CodeGen *g, ZigType *ptr_type) {
return entry;
}
ZigType *get_opaque_type(CodeGen *g, Scope *scope, AstNode *source_node, const char *full_name, Buf *bare_name) {
static uint32_t node_line_onebased(AstNode *node) {
RootStruct *root_struct = node->owner->data.structure.root_struct;
assert(node->main_token < root_struct->token_count);
return root_struct->token_locs[node->main_token].line + 1;
}
static uint32_t node_column_onebased(AstNode *node) {
RootStruct *root_struct = node->owner->data.structure.root_struct;
assert(node->main_token < root_struct->token_count);
return root_struct->token_locs[node->main_token].column + 1;
}
ZigType *get_opaque_type(CodeGen *g, Scope *scope, AstNode *source_node, const char *full_name,
Buf *bare_name)
{
ZigType *entry = new_type_table_entry(ZigTypeIdOpaque);
buf_init_from_str(&entry->name, full_name);
ZigType *import = scope ? get_scope_import(scope) : nullptr;
unsigned line = source_node ? (unsigned)(source_node->line + 1) : 0;
unsigned line = source_node ? node_line_onebased(source_node) : 0;
// Note: duplicated in get_partial_container_type
entry->llvm_type = LLVMInt8Type();
@ -1142,7 +1159,7 @@ ZigType *get_partial_container_type(CodeGen *g, Scope *scope, ContainerKind kind
break;
case ContainerKindOpaque: {
ZigType *import = scope ? get_scope_import(scope) : nullptr;
unsigned line = decl_node ? (unsigned)(decl_node->line + 1) : 0;
unsigned line = decl_node ? node_line_onebased(decl_node) : 0;
// Note: duplicated in get_opaque_type
entry->llvm_type = LLVMInt8Type();
entry->llvm_di_type = ZigLLVMCreateDebugForwardDeclType(g->dbuilder,
@ -1584,8 +1601,9 @@ static OnePossibleValue type_val_resolve_has_one_possible_value(CodeGen *g, ZigV
ZigType *analyze_type_expr(CodeGen *g, Scope *scope, AstNode *node) {
Error err;
// Hot path for simple identifiers, to avoid unnecessary memory allocations.
if (node->type == NodeTypeSymbol) {
Buf *variable_name = node->data.symbol_expr.symbol;
if (node->type == NodeTypeIdentifier) {
RootStruct *root_struct = node->owner->data.structure.root_struct;
Buf *variable_name = token_identifier_buf(root_struct, node->main_token);
if (buf_eql_str(variable_name, "_"))
goto abort_hot_path;
ZigType *primitive_type;
@ -2034,7 +2052,7 @@ static ZigType *analyze_fn_type(CodeGen *g, AstNode *proto_node, Scope *child_sc
buf_sprintf("var args only allowed in functions with C calling convention"));
return g->builtin_types.entry_invalid;
}
} else if (param_node->data.param_decl.anytype_token != nullptr) {
} else if (param_node->data.param_decl.anytype_token != 0) {
if (!calling_convention_allows_zig_types(fn_type_id.cc)) {
add_node_error(g, param_node,
buf_sprintf("parameter of type 'anytype' not allowed in function with calling convention '%s'",
@ -3021,7 +3039,7 @@ static Error resolve_struct_zero_bits(CodeGen *g, ZigType *struct_type) {
field_node = decl_node->data.container_decl.fields.at(i);
type_struct_field->name = field_node->data.struct_field.name;
type_struct_field->decl_node = field_node;
if (field_node->data.struct_field.comptime_token != nullptr) {
if (field_node->data.struct_field.comptime_token != 0) {
if (field_node->data.struct_field.value == nullptr) {
add_token_error(g, field_node->owner,
field_node->data.struct_field.comptime_token,
@ -4042,7 +4060,7 @@ void scan_decls(CodeGen *g, ScopeDecls *decls_scope, AstNode *node) {
case NodeTypeBoolLiteral:
case NodeTypeNullLiteral:
case NodeTypeUndefinedLiteral:
case NodeTypeSymbol:
case NodeTypeIdentifier:
case NodeTypePrefixOpExpr:
case NodeTypePointerType:
case NodeTypeIfBoolExpr:
@ -4238,7 +4256,7 @@ static void resolve_decl_var(CodeGen *g, TldVar *tld_var, bool allow_lazy) {
bool is_const = var_decl->is_const;
bool is_extern = var_decl->is_extern;
bool is_export = var_decl->is_export;
bool is_thread_local = var_decl->threadlocal_tok != nullptr;
bool is_thread_local = var_decl->threadlocal_tok != 0;
ZigType *explicit_type = nullptr;
if (var_decl->type) {
@ -5225,8 +5243,6 @@ static void analyze_fn_body(CodeGen *g, ZigFn *fn_table_entry) {
}
if (g->verbose_ir) {
fprintf(stderr, "\n");
ast_render(stderr, fn_table_entry->body_node, 4);
fprintf(stderr, "\nfn %s() { // (IR)\n", buf_ptr(&fn_table_entry->symbol_name));
ir_print_src(g, stderr, fn_table_entry->ir_executable, 4);
fprintf(stderr, "}\n");
@ -5238,33 +5254,17 @@ static void analyze_fn_body(CodeGen *g, ZigFn *fn_table_entry) {
ZigType *add_source_file(CodeGen *g, ZigPackage *package, Buf *resolved_path, Buf *source_code,
SourceKind source_kind)
{
if (g->verbose_tokenize) {
fprintf(stderr, "\nOriginal Source (%s):\n", buf_ptr(resolved_path));
fprintf(stderr, "----------------\n");
fprintf(stderr, "%s\n", buf_ptr(source_code));
fprintf(stderr, "\nTokens:\n");
fprintf(stderr, "---------\n");
}
Tokenization tokenization = {0};
tokenize(source_code, &tokenization);
tokenize(buf_ptr(source_code), &tokenization);
if (tokenization.err) {
ErrorMsg *err = err_msg_create_with_line(resolved_path, tokenization.err_line, tokenization.err_column,
source_code, tokenization.line_offsets, tokenization.err);
ErrorMsg *err = err_msg_create_with_offset(resolved_path, tokenization.err_byte_offset,
buf_ptr(source_code), tokenization.err);
print_err_msg(err, g->err_color);
exit(1);
}
if (g->verbose_tokenize) {
print_tokens(source_code, tokenization.tokens);
fprintf(stderr, "\nAST:\n");
fprintf(stderr, "------\n");
}
Buf *src_dirname = buf_alloc();
Buf *src_basename = buf_alloc();
os_path_split(resolved_path, src_dirname, src_basename);
@ -5297,9 +5297,20 @@ ZigType *add_source_file(CodeGen *g, ZigPackage *package, Buf *resolved_path, Bu
RootStruct *root_struct = heap::c_allocator.create<RootStruct>();
root_struct->package = package;
root_struct->source_code = source_code;
root_struct->line_offsets = tokenization.line_offsets;
root_struct->path = resolved_path;
root_struct->di_file = ZigLLVMCreateFile(g->dbuilder, buf_ptr(src_basename), buf_ptr(src_dirname));
assert(tokenization.ids.length == tokenization.locs.length);
size_t token_count = tokenization.ids.length;
root_struct->token_count = token_count;
root_struct->token_ids = g->pass1_arena->allocate<TokenId>(token_count);
memcpy(root_struct->token_ids, tokenization.ids.items, token_count * sizeof(TokenId));
root_struct->token_locs = g->pass1_arena->allocate<TokenLoc>(token_count);
memcpy(root_struct->token_locs, tokenization.locs.items, token_count * sizeof(TokenLoc));
tokenization.ids.deinit();
tokenization.locs.deinit();
ZigType *import_entry = get_root_container_type(g, buf_ptr(namespace_name), bare_name, root_struct);
if (source_kind == SourceKindRoot) {
assert(g->root_import == nullptr);
@ -5307,14 +5318,11 @@ ZigType *add_source_file(CodeGen *g, ZigPackage *package, Buf *resolved_path, Bu
}
g->import_table.put(resolved_path, import_entry);
AstNode *root_node = ast_parse(source_code, tokenization.tokens, import_entry, g->err_color);
AstNode *root_node = ast_parse(source_code, import_entry, g->err_color);
assert(root_node != nullptr);
assert(root_node->type == NodeTypeContainerDecl);
import_entry->data.structure.decl_node = root_node;
import_entry->data.structure.decls_scope->base.source_node = root_node;
if (g->verbose_ast) {
ast_print(stderr, root_node, 0);
}
for (size_t decl_i = 0; decl_i < root_node->data.container_decl.decls.length; decl_i += 1) {
AstNode *top_level_decl = root_node->data.container_decl.decls.at(decl_i);
@ -6254,9 +6262,12 @@ ReqCompTime type_requires_comptime(CodeGen *g, ZigType *ty) {
zig_unreachable();
}
void init_const_str_lit(CodeGen *g, ZigValue *const_val, Buf *str) {
void init_const_str_lit(CodeGen *g, ZigValue *const_val, Buf *str, bool move_str) {
auto entry = g->string_literals_table.maybe_get(str);
if (entry != nullptr) {
if (move_str) {
buf_destroy(str);
}
memcpy(const_val, entry->value, sizeof(ZigValue));
return;
}
@ -6280,7 +6291,7 @@ void init_const_str_lit(CodeGen *g, ZigValue *const_val, Buf *str) {
ZigValue *create_const_str_lit(CodeGen *g, Buf *str) {
ZigValue *const_val = g->pass1_arena->create<ZigValue>();
init_const_str_lit(g, const_val, str);
init_const_str_lit(g, const_val, str, false);
return const_val;
}
@ -8626,7 +8637,7 @@ static void resolve_llvm_types_struct(CodeGen *g, ZigType *struct_type, ResolveS
ZigType *import = get_scope_import(scope);
di_file = import->data.structure.root_struct->di_file;
di_scope = ZigLLVMFileToScope(di_file);
line = decl_node->line + 1;
line = node_line_onebased(decl_node);
} else {
di_file = nullptr;
di_scope = ZigLLVMCompileUnitToScope(g->compile_unit);
@ -8830,7 +8841,7 @@ static void resolve_llvm_types_struct(CodeGen *g, ZigType *struct_type, ResolveS
unsigned line;
if (decl_node != nullptr) {
AstNode *field_node = field->decl_node;
line = field_node->line + 1;
line = node_line_onebased(field_node);
} else {
line = 0;
}
@ -8881,7 +8892,7 @@ static ZigLLVMDIType *make_empty_namespace_llvm_di_type(CodeGen *g, ZigType *imp
return ZigLLVMCreateDebugStructType(g->dbuilder,
ZigLLVMFileToScope(import->data.structure.root_struct->di_file),
name,
import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
debug_size_in_bits,
debug_align_in_bits,
ZigLLVM_DIFlags_Zero,
@ -8926,7 +8937,7 @@ static void resolve_llvm_types_enum(CodeGen *g, ZigType *enum_type, ResolveStatu
uint64_t tag_debug_align_in_bits = 8*tag_int_type->abi_align;
ZigLLVMDIType *tag_di_type = ZigLLVMCreateDebugEnumerationType(g->dbuilder,
ZigLLVMFileToScope(import->data.structure.root_struct->di_file), buf_ptr(&enum_type->name),
import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
tag_debug_size_in_bits,
tag_debug_align_in_bits,
di_enumerators, field_count,
@ -8966,12 +8977,12 @@ static void resolve_llvm_types_union(CodeGen *g, ZigType *union_type, ResolveSta
if (union_type->data.unionation.resolve_status < ResolveStatusLLVMFwdDecl) {
union_type->llvm_type = LLVMStructCreateNamed(LLVMGetGlobalContext(), buf_ptr(&union_type->name));
size_t line = decl_node ? decl_node->line : 0;
unsigned line = decl_node ? node_line_onebased(decl_node) : 0;
unsigned dwarf_kind = ZigLLVMTag_DW_structure_type();
union_type->llvm_di_type = ZigLLVMCreateReplaceableCompositeType(g->dbuilder,
dwarf_kind, buf_ptr(&union_type->name),
ZigLLVMFileToScope(import->data.structure.root_struct->di_file),
import->data.structure.root_struct->di_file, (unsigned)(line + 1));
import->data.structure.root_struct->di_file, line);
union_type->data.unionation.resolve_status = ResolveStatusLLVMFwdDecl;
if (ResolveStatusLLVMFwdDecl >= wanted_resolve_status) return;
@ -8992,7 +9003,7 @@ static void resolve_llvm_types_union(CodeGen *g, ZigType *union_type, ResolveSta
AstNode *field_node = union_field->decl_node;
union_inner_di_types[union_field->gen_index] = ZigLLVMCreateDebugMemberType(g->dbuilder,
ZigLLVMTypeToScope(union_type->llvm_di_type), buf_ptr(union_field->enum_field->name),
import->data.structure.root_struct->di_file, (unsigned)(field_node->line + 1),
import->data.structure.root_struct->di_file, node_line_onebased(field_node),
store_size_in_bits,
abi_align_in_bits,
0,
@ -9022,7 +9033,7 @@ static void resolve_llvm_types_union(CodeGen *g, ZigType *union_type, ResolveSta
// create debug type for union
ZigLLVMDIType *replacement_di_type = ZigLLVMCreateDebugUnionType(g->dbuilder,
ZigLLVMFileToScope(import->data.structure.root_struct->di_file), buf_ptr(&union_type->name),
import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
union_type->data.unionation.union_abi_size * 8,
most_aligned_union_member->align * 8,
ZigLLVM_DIFlags_Zero, union_inner_di_types,
@ -9057,7 +9068,7 @@ static void resolve_llvm_types_union(CodeGen *g, ZigType *union_type, ResolveSta
// create debug type for union
ZigLLVMDIType *union_di_type = ZigLLVMCreateDebugUnionType(g->dbuilder,
ZigLLVMTypeToScope(union_type->llvm_di_type), "AnonUnion",
import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
most_aligned_union_member->type_entry->size_in_bits, 8*most_aligned_union_member->align,
ZigLLVM_DIFlags_Zero, union_inner_di_types, gen_field_count, 0, "");
@ -9068,7 +9079,7 @@ static void resolve_llvm_types_union(CodeGen *g, ZigType *union_type, ResolveSta
ZigLLVMDIType *union_member_di_type = ZigLLVMCreateDebugMemberType(g->dbuilder,
ZigLLVMTypeToScope(union_type->llvm_di_type), "payload",
import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
most_aligned_union_member->type_entry->size_in_bits,
8*most_aligned_union_member->align,
union_offset_in_bits,
@ -9079,7 +9090,7 @@ static void resolve_llvm_types_union(CodeGen *g, ZigType *union_type, ResolveSta
ZigLLVMDIType *tag_member_di_type = ZigLLVMCreateDebugMemberType(g->dbuilder,
ZigLLVMTypeToScope(union_type->llvm_di_type), "tag",
import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
tag_debug_size_in_bits,
tag_debug_align_in_bits,
tag_offset_in_bits,
@ -9094,7 +9105,7 @@ static void resolve_llvm_types_union(CodeGen *g, ZigType *union_type, ResolveSta
ZigLLVMDIType *replacement_di_type = ZigLLVMCreateDebugStructType(g->dbuilder,
ZigLLVMFileToScope(import->data.structure.root_struct->di_file),
buf_ptr(&union_type->name),
import->data.structure.root_struct->di_file, (unsigned)(decl_node->line + 1),
import->data.structure.root_struct->di_file, node_line_onebased(decl_node),
debug_size_in_bits,
debug_align_in_bits,
ZigLLVM_DIFlags_Zero, nullptr, di_root_members, 2, 0, nullptr, "");
@ -9774,9 +9785,9 @@ void src_assert_impl(bool ok, AstNode *source_node, char const *file, unsigned i
if (source_node == nullptr) {
fprintf(stderr, "when analyzing (unknown source location) ");
} else {
fprintf(stderr, "when analyzing %s:%u:%u ",
buf_ptr(source_node->owner->data.structure.root_struct->path),
(unsigned)source_node->line + 1, (unsigned)source_node->column + 1);
RootStruct *root_struct = source_node->owner->data.structure.root_struct;
fprintf(stderr, "when analyzing %s:%u:%u ", buf_ptr(root_struct->path),
node_line_onebased(source_node), node_column_onebased(source_node));
}
fprintf(stderr, "in compiler source at %s:%u: ", file, line);
const char *msg = "assertion failed. This is a bug in the Zig compiler.";
@ -9842,6 +9853,14 @@ Error analyze_import(CodeGen *g, ZigType *source_import, Buf *import_target_str,
return ErrorNone;
}
void AstNode::src() {
RootStruct *root_struct = this->owner->data.structure.root_struct;
uint32_t line = root_struct->token_locs[this->main_token].line + 1;
uint32_t column = root_struct->token_locs[this->main_token].column + 1;
fprintf(stderr, "%s:%" PRIu32 ":%" PRIu32 "\n",
buf_ptr(root_struct->path),
line, column);
}
void IrExecutableSrc::src() {
if (this->source_node != nullptr) {

6
src/stage1/analyze.hpp
View File

@ -12,7 +12,9 @@
void semantic_analyze(CodeGen *g);
ErrorMsg *add_node_error(CodeGen *g, AstNode *node, Buf *msg);
ErrorMsg *add_token_error(CodeGen *g, ZigType *owner, Token *token, Buf *msg);
ErrorMsg *add_token_error(CodeGen *g, ZigType *owner, TokenIndex token, Buf *msg);
ErrorMsg *add_token_error_offset(CodeGen *g, ZigType *owner, TokenIndex token, Buf *msg,
uint32_t bad_index);
ErrorMsg *add_error_note(CodeGen *g, ErrorMsg *parent_msg, const AstNode *node, Buf *msg);
ZigType *new_type_table_entry(ZigTypeId id);
ZigType *get_fn_frame_type(CodeGen *g, ZigFn *fn);
@ -140,7 +142,7 @@ Scope *create_runtime_scope(CodeGen *g, AstNode *node, Scope *parent, IrInstSrc
Scope *create_typeof_scope(CodeGen *g, AstNode *node, Scope *parent);
ScopeExpr *create_expr_scope(CodeGen *g, AstNode *node, Scope *parent);
void init_const_str_lit(CodeGen *g, ZigValue *const_val, Buf *str);
void init_const_str_lit(CodeGen *g, ZigValue *const_val, Buf *str, bool move_str);
ZigValue *create_const_str_lit(CodeGen *g, Buf *str);
void init_const_bigint(ZigValue *const_val, ZigType *type, const BigInt *bigint);

1241
src/stage1/ast_render.cpp
View File

File diff suppressed because it is too large Load Diff

20
src/stage1/ast_render.hpp
View File

@ -1,20 +0,0 @@
/*
* Copyright (c) 2015 Andrew Kelley
*
* This file is part of zig, which is MIT licensed.
* See http://opensource.org/licenses/MIT
*/
#ifndef ZIG_AST_RENDER_HPP
#define ZIG_AST_RENDER_HPP
#include "all_types.hpp"
#include "parser.hpp"
#include <stdio.h>
void ast_print(FILE *f, AstNode *node, int indent);
void ast_render(FILE *f, AstNode *node, int indent_size);
#endif

8120
src/stage1/astgen.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

43
src/stage1/astgen.hpp Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright (c) 2021 Andrew Kelley
*
* This file is part of zig, which is MIT licensed.
* See http://opensource.org/licenses/MIT
*/
#ifndef ZIG_ASTGEN_HPP
#define ZIG_ASTGEN_HPP
#include "all_types.hpp"
bool ir_gen(CodeGen *g, AstNode *node, Scope *scope, IrExecutableSrc *ir_executable);
bool ir_gen_fn(CodeGen *g, ZigFn *fn_entry);
bool ir_inst_src_has_side_effects(IrInstSrc *inst);
ZigVar *create_local_var(CodeGen *codegen, AstNode *node, Scope *parent_scope,
Buf *name, bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstSrc *is_comptime,
bool skip_name_check);
ResultLoc *no_result_loc(void);
void invalidate_exec(IrExecutableSrc *exec, ErrorMsg *msg);
AstNode *ast_field_to_symbol_node(AstNode *err_set_field_node);
void ir_add_call_stack_errors_gen(CodeGen *codegen, IrExecutableGen *exec, ErrorMsg *err_msg,
int limit);
void destroy_instruction_src(IrInstSrc *inst);
struct IrBuilderSrc {
CodeGen *codegen;
IrExecutableSrc *exec;
IrBasicBlockSrc *current_basic_block;
AstNode *main_block_node;
};
bool ir_should_inline(IrExecutableSrc *exec, Scope *scope);
Buf *get_anon_type_name(CodeGen *codegen, IrExecutableSrc *exec, const char *kind_name,
Scope *scope, AstNode *source_node, Buf *out_bare_name);
#endif

3
src/stage1/bigfloat.cpp
View File

@ -69,7 +69,7 @@ void bigfloat_init_bigint(BigFloat *dest, const BigInt *op) {
}
}
Error bigfloat_init_buf(BigFloat *dest, const uint8_t *buf_ptr, size_t buf_len) {
Error bigfloat_init_buf(BigFloat *dest, const uint8_t *buf_ptr) {
char *str_begin = (char *)buf_ptr;
char *str_end;
@ -79,7 +79,6 @@ Error bigfloat_init_buf(BigFloat *dest, const uint8_t *buf_ptr, size_t buf_len)
return ErrorOverflow;
}
assert(str_end <= ((char*)buf_ptr) + buf_len);
return ErrorNone;
}

2
src/stage1/bigfloat.hpp
View File

@ -28,7 +28,7 @@ void bigfloat_init_64(BigFloat *dest, double x);
void bigfloat_init_128(BigFloat *dest, float128_t x);
void bigfloat_init_bigfloat(BigFloat *dest, const BigFloat *x);
void bigfloat_init_bigint(BigFloat *dest, const BigInt *op);
Error bigfloat_init_buf(BigFloat *dest, const uint8_t *buf_ptr, size_t buf_len);
Error bigfloat_init_buf(BigFloat *dest, const uint8_t *buf_ptr);
float16_t bigfloat_to_f16(const BigFloat *bigfloat);
float bigfloat_to_f32(const BigFloat *bigfloat);

44
src/stage1/codegen.cpp
View File

@ -6,7 +6,6 @@
*/
#include "analyze.hpp"
#include "ast_render.hpp"
#include "codegen.hpp"
#include "errmsg.hpp"
#include "error.hpp"
@ -642,6 +641,18 @@ static LLVMValueRef fn_llvm_value(CodeGen *g, ZigFn *fn) {
return fn->llvm_value;
}
static uint32_t node_line_onebased(AstNode *node) {
RootStruct *root_struct = node->owner->data.structure.root_struct;
assert(node->main_token < root_struct->token_count);
return root_struct->token_locs[node->main_token].line + 1;
}
static uint32_t node_column_onebased(AstNode *node) {
RootStruct *root_struct = node->owner->data.structure.root_struct;
assert(node->main_token < root_struct->token_count);
return root_struct->token_locs[node->main_token].column + 1;
}
static ZigLLVMDIScope *get_di_scope(CodeGen *g, Scope *scope) {
if (scope->di_scope)
return scope->di_scope;
@ -657,8 +668,7 @@ static ZigLLVMDIScope *get_di_scope(CodeGen *g, Scope *scope) {
ZigFn *fn_table_entry = fn_scope->fn_entry;
if (!fn_table_entry->proto_node)
return get_di_scope(g, scope->parent);
unsigned line_number = (unsigned)(fn_table_entry->proto_node->line == 0) ?
0 : (fn_table_entry->proto_node->line + 1);
unsigned line_number = node_line_onebased(fn_table_entry->proto_node);
unsigned scope_line = line_number;
bool is_definition = fn_table_entry->body_node != nullptr;
bool is_optimized = g->build_mode != BuildModeDebug;
@ -696,8 +706,8 @@ static ZigLLVMDIScope *get_di_scope(CodeGen *g, Scope *scope) {
ZigLLVMDILexicalBlock *di_block = ZigLLVMCreateLexicalBlock(g->dbuilder,
get_di_scope(g, scope->parent),
import->data.structure.root_struct->di_file,
(unsigned)scope->source_node->line + 1,
(unsigned)scope->source_node->column + 1);
node_line_onebased(scope->source_node),
node_column_onebased(scope->source_node));
scope->di_scope = ZigLLVMLexicalBlockToScope(di_block);
return scope->di_scope;
}
@ -1798,8 +1808,8 @@ static void gen_var_debug_decl(CodeGen *g, ZigVar *var) {
if (g->strip_debug_symbols) return;
assert(var->di_loc_var != nullptr);
AstNode *source_node = var->decl_node;
ZigLLVMDILocation *debug_loc = ZigLLVMGetDebugLoc((unsigned)source_node->line + 1,
(unsigned)source_node->column + 1, get_di_scope(g, var->parent_scope));
ZigLLVMDILocation *debug_loc = ZigLLVMGetDebugLoc(node_line_onebased(source_node),
node_column_onebased(source_node), get_di_scope(g, var->parent_scope));
ZigLLVMInsertDeclareAtEnd(g->dbuilder, var->value_ref, var->di_loc_var, debug_loc,
LLVMGetInsertBlock(g->builder));
}
@ -2198,7 +2208,7 @@ var_ok:
// arg index + 1 because the 0 index is return value
var->di_loc_var = ZigLLVMCreateParameterVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
var->name, fn_walk->data.vars.import->data.structure.root_struct->di_file,
(unsigned)(var->decl_node->line + 1),
node_line_onebased(var->decl_node),
get_llvm_di_type(g, dest_ty), !g->strip_debug_symbols, 0, di_arg_index + 1);
}
return true;
@ -4126,7 +4136,7 @@ static void render_async_spills(CodeGen *g) {
if (var->decl_node) {
var->di_loc_var = ZigLLVMCreateAutoVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
var->name, import->data.structure.root_struct->di_file,
(unsigned)(var->decl_node->line + 1),
node_line_onebased(var->decl_node),
get_llvm_di_type(g, var->var_type), !g->strip_debug_symbols, 0);
gen_var_debug_decl(g, var);
}
@ -6797,8 +6807,8 @@ static void set_debug_location(CodeGen *g, IrInstGen *instruction) {
assert(source_node);
assert(scope);
ZigLLVMSetCurrentDebugLocation(g->builder, (int)source_node->line + 1,
(int)source_node->column + 1, get_di_scope(g, scope));
ZigLLVMSetCurrentDebugLocation(g->builder, node_line_onebased(source_node),
node_column_onebased(source_node), get_di_scope(g, scope));
}
static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutableGen *executable, IrInstGen *instruction) {
@ -8021,7 +8031,7 @@ static void gen_global_var(CodeGen *g, ZigVar *var, LLVMValueRef init_val,
bool is_local_to_unit = true;
ZigLLVMCreateGlobalVariable(g->dbuilder, get_di_scope(g, var->parent_scope), var->name,
var->name, import->data.structure.root_struct->di_file,
(unsigned)(var->decl_node->line + 1),
node_line_onebased(var->decl_node),
get_llvm_di_type(g, type_entry), is_local_to_unit);
// TODO ^^ make an actual global variable
@ -8296,7 +8306,8 @@ static void do_code_gen(CodeGen *g) {
if (var->src_arg_index == SIZE_MAX) {
var->di_loc_var = ZigLLVMCreateAutoVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
var->name, import->data.structure.root_struct->di_file, (unsigned)(var->decl_node->line + 1),
var->name, import->data.structure.root_struct->di_file,
node_line_onebased(var->decl_node),
get_llvm_di_type(g, var->var_type), !g->strip_debug_symbols, 0);
} else if (is_c_abi) {
@ -8321,7 +8332,7 @@ static void do_code_gen(CodeGen *g) {
if (var->decl_node) {
var->di_loc_var = ZigLLVMCreateParameterVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
var->name, import->data.structure.root_struct->di_file,
(unsigned)(var->decl_node->line + 1),
node_line_onebased(var->decl_node),
get_llvm_di_type(g, gen_type), !g->strip_debug_symbols, 0, (unsigned)(gen_info->gen_index+1));
}
@ -8365,8 +8376,9 @@ static void do_code_gen(CodeGen *g) {
if (!g->strip_debug_symbols) {
AstNode *source_node = fn_table_entry->proto_node;
ZigLLVMSetCurrentDebugLocation(g->builder, (int)source_node->line + 1,
(int)source_node->column + 1, get_di_scope(g, fn_table_entry->child_scope));
ZigLLVMSetCurrentDebugLocation(g->builder,
node_line_onebased(source_node), node_column_onebased(source_node),
get_di_scope(g, fn_table_entry->child_scope));
}
IrExecutableGen *executable = &fn_table_entry->analyzed_executable;
LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");

42
src/stage1/dump_analysis.cpp
View File

@ -1084,19 +1084,43 @@ static void anal_dump_type(AnalDumpCtx *ctx, ZigType *ty) {
jw_end_object(jw);
}
static Buf *collect_doc_comments(RootStruct *root_struct, TokenIndex first_token) {
if (first_token == 0)
return nullptr;
TokenId *token_ids = root_struct->token_ids;
TokenLoc *token_locs = root_struct->token_locs;
Buf *str = buf_alloc();
const char *source = buf_ptr(root_struct->source_code);
TokenIndex doc_token = first_token;
for (;token_ids[doc_token] == TokenIdDocComment; doc_token += 1) {
// chops off '///' but leaves '\n'
uint32_t start_pos = token_locs[doc_token].offset;
uint32_t token_len = 0;
while (source[start_pos + token_len] != '\n' &&
source[start_pos + token_len] != 0)
{
token_len += 1;
}
buf_append_mem(str, source + start_pos + 3, token_len - 3);
}
return str;
}
static void anal_dump_node(AnalDumpCtx *ctx, const AstNode *node) {
JsonWriter *jw = &ctx->jw;
jw_begin_object(jw);
jw_object_field(jw, "file");
anal_dump_file_ref(ctx, node->owner->data.structure.root_struct->path);
RootStruct *root_struct = node->owner->data.structure.root_struct;
anal_dump_file_ref(ctx, root_struct->path);
jw_object_field(jw, "line");
jw_int(jw, node->line);
jw_int(jw, root_struct->token_locs[node->main_token].line);
jw_object_field(jw, "col");
jw_int(jw, node->column);
jw_int(jw, root_struct->token_locs[node->main_token].column);
const Buf *doc_comments_buf = nullptr;
const Buf *name_buf = nullptr;
@ -1107,30 +1131,30 @@ static void anal_dump_node(AnalDumpCtx *ctx, const AstNode *node) {
switch (node->type) {
case NodeTypeParamDecl:
doc_comments_buf = &node->data.param_decl.doc_comments;
doc_comments_buf = collect_doc_comments(root_struct, node->data.param_decl.doc_comments);
name_buf = node->data.param_decl.name;
is_var_args = node->data.param_decl.is_var_args;
is_noalias = node->data.param_decl.is_noalias;
is_comptime = node->data.param_decl.is_comptime;
break;
case NodeTypeFnProto:
doc_comments_buf = &node->data.fn_proto.doc_comments;
doc_comments_buf = collect_doc_comments(root_struct, node->data.fn_proto.doc_comments);
field_nodes = &node->data.fn_proto.params;
is_var_args = node->data.fn_proto.is_var_args;
break;
case NodeTypeVariableDeclaration:
doc_comments_buf = &node->data.variable_declaration.doc_comments;
doc_comments_buf = collect_doc_comments(root_struct, node->data.variable_declaration.doc_comments);
break;
case NodeTypeErrorSetField:
doc_comments_buf = &node->data.err_set_field.doc_comments;
doc_comments_buf = collect_doc_comments(root_struct, node->data.err_set_field.doc_comments);
break;
case NodeTypeStructField:
doc_comments_buf = &node->data.struct_field.doc_comments;
doc_comments_buf = collect_doc_comments(root_struct, node->data.struct_field.doc_comments);
name_buf = node->data.struct_field.name;
break;
case NodeTypeContainerDecl:
field_nodes = &node->data.container_decl.fields;
doc_comments_buf = &node->data.container_decl.doc_comments;
doc_comments_buf = collect_doc_comments(root_struct, node->data.container_decl.doc_comments);
break;
default:
break;

59
src/stage1/errmsg.cpp
View File

@ -96,14 +96,14 @@ void err_msg_add_note(ErrorMsg *parent, ErrorMsg *note) {
parent->notes.append(note);
}
ErrorMsg *err_msg_create_with_offset(Buf *path, size_t line, size_t column, size_t offset,
const char *source, Buf *msg)
ErrorMsg *err_msg_create_with_offset(Buf *path, uint32_t byte_offset, const char *source,
Buf *msg)
{
ErrorMsg *err_msg = heap::c_allocator.create<ErrorMsg>();
err_msg->path = path;
err_msg->line_start = line;
err_msg->column_start = column;
err_msg->msg = msg;
err_msg->line_start = 0;
err_msg->column_start = 0;
if (source == nullptr) {
// Must initialize the buffer anyway
@ -111,46 +111,25 @@ ErrorMsg *err_msg_create_with_offset(Buf *path, size_t line, size_t column, size
return err_msg;
}
size_t line_start_offset = offset;
for (;;) {
if (line_start_offset == 0) {
break;
}
line_start_offset -= 1;
if (source[line_start_offset] == '\n') {
line_start_offset += 1;
break;
size_t line_start = 0;
size_t i = 0;
for (;i < byte_offset; i += 1) {
switch (source[i]) {
case '\n':
err_msg->line_start += 1;
err_msg->column_start = 0;
line_start = i + 1;
continue;
default:
err_msg->column_start += 1;
continue;
}
}
size_t line_end_offset = offset;
while (source[line_end_offset] && source[line_end_offset] != '\n') {
line_end_offset += 1;
while (source[i] != '\n' && source[i] != 0) {
i += 1;
}
buf_init_from_mem(&err_msg->line_buf, source + line_start_offset, line_end_offset - line_start_offset);
return err_msg;
}
ErrorMsg *err_msg_create_with_line(Buf *path, size_t line, size_t column,
Buf *source, ZigList<size_t> *line_offsets, Buf *msg)
{
ErrorMsg *err_msg = heap::c_allocator.create<ErrorMsg>();
err_msg->path = path;
err_msg->line_start = line;
err_msg->column_start = column;
err_msg->msg = msg;
size_t line_start_offset = line_offsets->at(line);
size_t end_line = line + 1;
size_t line_end_offset = (end_line >= line_offsets->length) ? buf_len(source) : line_offsets->at(line + 1);
size_t len = (line_end_offset + 1 > line_start_offset) ? (line_end_offset - line_start_offset - 1) : 0;
if (len == SIZE_MAX) len = 0;
buf_init_from_mem(&err_msg->line_buf, buf_ptr(source) + line_start_offset, len);
buf_init_from_mem(&err_msg->line_buf, source + line_start, i - line_start);
return err_msg;
}

6
src/stage1/errmsg.hpp
View File

@ -25,10 +25,6 @@ struct ErrorMsg {
void print_err_msg(ErrorMsg *msg, ErrColor color);
void err_msg_add_note(ErrorMsg *parent, ErrorMsg *note);
ErrorMsg *err_msg_create_with_offset(Buf *path, size_t line, size_t column, size_t offset,
const char *source, Buf *msg);
ErrorMsg *err_msg_create_with_line(Buf *path, size_t line, size_t column,
Buf *source, ZigList<size_t> *line_offsets, Buf *msg);
ErrorMsg *err_msg_create_with_offset(Buf *path, uint32_t byte_offset, const char *source, Buf *msg);
#endif

2
src/stage1/error.cpp
View File

@ -88,6 +88,8 @@ const char *err_str(Error err) {
case ErrorZigIsTheCCompiler: return "Zig was not provided with libc installation information, and so it does not know where the libc paths are on the system. Zig attempted to use the system C compiler to find out where the libc paths are, but discovered that Zig is being used as the system C compiler.";
case ErrorFileBusy: return "file is busy";
case ErrorLocked: return "file is locked by another process";
case ErrorInvalidCharacter: return "invalid character";
case ErrorUnicodePointTooLarge: return "unicode codepoint too large";
}
return "(invalid error)";
}

8215
src/stage1/ir.cpp
View File

File diff suppressed because it is too large Load Diff

7
src/stage1/ir.hpp
View File

@ -10,9 +10,6 @@
#include "all_types.hpp"
bool ir_gen(CodeGen *g, AstNode *node, Scope *scope, IrExecutableSrc *ir_executable);
bool ir_gen_fn(CodeGen *g, ZigFn *fn_entry);
IrInstGen *ir_create_alloca(CodeGen *g, Scope *scope, AstNode *source_node, ZigFn *fn,
ZigType *var_type, const char *name_hint);
@ -33,8 +30,4 @@ struct IrAnalyze;
ZigValue *const_ptr_pointee(IrAnalyze *ira, CodeGen *codegen, ZigValue *const_val,
AstNode *source_node);
// for debugging purposes
void dbg_ir_break(const char *src_file, uint32_t line);
void dbg_ir_clear(void);
#endif

1503
src/stage1/parser.cpp
View File

File diff suppressed because it is too large Load Diff

17
src/stage1/parser.hpp
View File

@ -12,14 +12,21 @@
#include "tokenizer.hpp"
#include "errmsg.hpp"
ATTRIBUTE_PRINTF(2, 3)
void ast_token_error(Token *token, const char *format, ...);
AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens, ZigType *owner, ErrColor err_color);
AstNode * ast_parse(Buf *buf, ZigType *owner, ErrColor err_color);
void ast_print(AstNode *node, int indent);
void ast_visit_node_children(AstNode *node, void (*visit)(AstNode **, void *context), void *context);
Buf *node_identifier_buf(AstNode *node);
Buf *node_string_literal_buf(AstNode *node);
Buf *token_identifier_buf(RootStruct *root_struct, TokenIndex token);
Buf *token_string_literal_buf(RootStruct *root_struct, TokenIndex token);
void token_number_literal_bigint(RootStruct *root_struct, BigInt *result, TokenIndex token);
Error source_string_literal_buf(const char *source, Buf *out, size_t *bad_index);
Error source_char_literal(const char *source, uint32_t *out, size_t *bad_index);
#endif

2
src/stage1/stage2.h
View File

@ -112,6 +112,8 @@ enum Error {
ErrorZigIsTheCCompiler,
ErrorFileBusy,
ErrorLocked,
ErrorInvalidCharacter,
ErrorUnicodePointTooLarge,
};
// ABI warning

2189
src/stage1/tokenizer.cpp
View File

File diff suppressed because it is too large Load Diff

72
src/stage1/tokenizer.hpp
View File

@ -12,10 +12,9 @@
#include "bigint.hpp"
#include "bigfloat.hpp"
enum TokenId {
enum TokenId : uint8_t {
TokenIdAmpersand,
TokenIdArrow,
TokenIdAtSign,
TokenIdBang,
TokenIdBarBar,
TokenIdBinOr,
@ -27,6 +26,7 @@ enum TokenId {
TokenIdBitShiftRight,
TokenIdBitShiftRightEq,
TokenIdBitXorEq,
TokenIdBuiltin,
TokenIdCharLiteral,
TokenIdCmpEq,
TokenIdCmpGreaterOrEq,
@ -109,9 +109,7 @@ enum TokenId {
TokenIdMinusPercent,
TokenIdMinusPercentEq,
TokenIdModEq,
TokenIdNumberSign,
TokenIdPercent,
TokenIdPercentDot,
TokenIdPlus,
TokenIdPlusEq,
TokenIdPlusPercent,
@ -125,75 +123,35 @@ enum TokenId {
TokenIdStar,
TokenIdStarStar,
TokenIdStringLiteral,
TokenIdMultilineStringLiteral,
TokenIdSymbol,
TokenIdMultilineStringLiteralLine,
TokenIdIdentifier,
TokenIdTilde,
TokenIdTimesEq,
TokenIdTimesPercent,
TokenIdTimesPercentEq,
TokenIdCount,
};
struct TokenFloatLit {
BigFloat bigfloat;
// overflow is true if when parsing the number, we discovered it would not fit
// without losing data
bool overflow;
typedef uint32_t TokenIndex;
struct TokenLoc {
uint32_t offset;
uint32_t line;
uint32_t column;
};
struct TokenIntLit {
BigInt bigint;
};
struct TokenStrLit {
Buf str;
};
struct TokenCharLit {
uint32_t c;
};
struct Token {
TokenId id;
size_t start_pos;
size_t end_pos;
size_t start_line;
size_t start_column;
union {
// TokenIdIntLiteral
TokenIntLit int_lit;
// TokenIdFloatLiteral
TokenFloatLit float_lit;
// TokenIdStringLiteral, TokenIdMultilineStringLiteral or TokenIdSymbol
TokenStrLit str_lit;
// TokenIdCharLiteral
TokenCharLit char_lit;
} data;
};
// work around conflicting name Token which is also found in libclang
typedef Token ZigToken;
struct Tokenization {
ZigList<Token> *tokens;
ZigList<size_t> *line_offsets;
ZigList<TokenId> ids;
ZigList<TokenLoc> locs;
// if an error occurred
Buf *err;
size_t err_line;
size_t err_column;
uint32_t err_byte_offset;
};
void tokenize(Buf *buf, Tokenization *out_tokenization);
void print_tokens(Buf *buf, ZigList<Token> *tokens);
void tokenize(const char *source, Tokenization *out_tokenization);
const char * token_name(TokenId id);
bool valid_symbol_starter(uint8_t c);
bool is_zig_keyword(Buf *buf);
#endif

15
test/behavior/misc.zig
View File

@ -1,6 +1,7 @@
const std = @import("std");
const expect = std.testing.expect;
const expectEqualSlices = std.testing.expectEqualSlices;
const expectEqualStrings = std.testing.expectEqualStrings;
const mem = std.mem;
const builtin = @import("builtin");
@ -147,13 +148,13 @@ test "array mult operator" {
}
test "string escapes" {
try expect(mem.eql(u8, "\"", "\x22"));
try expect(mem.eql(u8, "\'", "\x27"));
try expect(mem.eql(u8, "\n", "\x0a"));
try expect(mem.eql(u8, "\r", "\x0d"));
try expect(mem.eql(u8, "\t", "\x09"));
try expect(mem.eql(u8, "\\", "\x5c"));
try expect(mem.eql(u8, "\u{1234}\u{069}\u{1}", "\xe1\x88\xb4\x69\x01"));
try expectEqualStrings("\"", "\x22");
try expectEqualStrings("\'", "\x27");
try expectEqualStrings("\n", "\x0a");
try expectEqualStrings("\r", "\x0d");
try expectEqualStrings("\t", "\x09");
try expectEqualStrings("\\", "\x5c");
try expectEqualStrings("\u{1234}\u{069}\u{1}", "\xe1\x88\xb4\x69\x01");
}
test "multiline string" {