From 7a99d63c764f3d5d92370c90f932b1bf156269f6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 22 Mar 2018 16:56:03 -0400 Subject: [PATCH 01/47] ability to use async function pointers closes #817 --- doc/langref.html.in | 2 +- src/all_types.hpp | 1 + src/analyze.cpp | 3 +- src/ir.cpp | 22 ++++++- src/parser.cpp | 126 +++++++++++++++++++++----------------- test/cases/coroutines.zig | 20 ++++++ 6 files changed, 113 insertions(+), 61 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index d227936f82..f4aa42eb89 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -5739,7 +5739,7 @@ UseDecl = "use" Expression ";" ExternDecl = "extern" option(String) (FnProto | VariableDeclaration) ";" -FnProto = option("nakedcc" | "stdcallcc" | "extern" | ("async" option("(" Expression ")"))) "fn" option(Symbol) ParamDeclList option("align" "(" Expression ")") option("section" "(" Expression ")") option("!") (TypeExpr | "var") +FnProto = option("nakedcc" | "stdcallcc" | "extern" | ("async" option("<" Expression ">"))) "fn" option(Symbol) ParamDeclList option("align" "(" Expression ")") option("section" "(" Expression ")") option("!") (TypeExpr | "var") FnDef = option("inline" | "export") FnProto Block diff --git a/src/all_types.hpp b/src/all_types.hpp index 6afa7ccf20..c1dec80d7e 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -2673,6 +2673,7 @@ struct IrInstructionFnProto { IrInstruction **param_types; IrInstruction *align_value; IrInstruction *return_type; + IrInstruction *async_allocator_type_value; bool is_var_args; }; diff --git a/src/analyze.cpp b/src/analyze.cpp index 395df229cd..eb1850c8e1 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -985,7 +985,8 @@ TypeTableEntry *get_fn_type(CodeGen *g, FnTypeId *fn_type_id) { // populate the name of the type buf_resize(&fn_type->name, 0); if (fn_type->data.fn.fn_type_id.cc == CallingConventionAsync) { - buf_appendf(&fn_type->name, "async(%s) ", buf_ptr(&fn_type_id->async_allocator_type->name)); + assert(fn_type_id->async_allocator_type != nullptr); + buf_appendf(&fn_type->name, "async<%s> ", buf_ptr(&fn_type_id->async_allocator_type->name)); } else { const char *cc_str = calling_convention_fn_type_str(fn_type->data.fn.fn_type_id.cc); buf_appendf(&fn_type->name, "%s", cc_str); diff --git a/src/ir.cpp b/src/ir.cpp index db9a2b24c3..8f418e2c2f 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -2141,12 +2141,14 @@ static IrInstruction *ir_build_unwrap_err_payload_from(IrBuilder *irb, IrInstruc } static IrInstruction *ir_build_fn_proto(IrBuilder *irb, Scope *scope, AstNode *source_node, - IrInstruction **param_types, IrInstruction *align_value, IrInstruction *return_type, bool is_var_args) + IrInstruction **param_types, IrInstruction *align_value, IrInstruction *return_type, + IrInstruction *async_allocator_type_value, bool is_var_args) { IrInstructionFnProto *instruction = ir_build_instruction(irb, scope, source_node); instruction->param_types = param_types; instruction->align_value = align_value; instruction->return_type = return_type; + instruction->async_allocator_type_value = async_allocator_type_value; instruction->is_var_args = is_var_args; assert(source_node->type == NodeTypeFnProto); @@ -2156,6 +2158,7 @@ static IrInstruction *ir_build_fn_proto(IrBuilder *irb, Scope *scope, AstNode *s if (param_types[i] != nullptr) ir_ref_instruction(param_types[i], irb->current_basic_block); } if (align_value != nullptr) ir_ref_instruction(align_value, irb->current_basic_block); + if (async_allocator_type_value != nullptr) ir_ref_instruction(async_allocator_type_value, irb->current_basic_block); ir_ref_instruction(return_type, irb->current_basic_block); return &instruction->base; @@ -5989,7 +5992,15 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo return_type = nullptr; } - return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, is_var_args); + IrInstruction *async_allocator_type_value = nullptr; + if (node->data.fn_proto.async_allocator_type != nullptr) { + async_allocator_type_value = ir_gen_node(irb, node->data.fn_proto.async_allocator_type, parent_scope); + if (async_allocator_type_value == irb->codegen->invalid_instruction) + return irb->codegen->invalid_instruction; + } + + return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, + async_allocator_type_value, is_var_args); } static IrInstruction *ir_gen_cancel(IrBuilder *irb, Scope *parent_scope, AstNode *node) { @@ -16561,6 +16572,13 @@ static TypeTableEntry *ir_analyze_instruction_fn_proto(IrAnalyze *ira, IrInstruc if (type_is_invalid(fn_type_id.return_type)) return ira->codegen->builtin_types.entry_invalid; + if (fn_type_id.cc == CallingConventionAsync) { + IrInstruction *async_allocator_type_value = instruction->async_allocator_type_value->other; + fn_type_id.async_allocator_type = ir_resolve_type(ira, async_allocator_type_value); + if (type_is_invalid(fn_type_id.async_allocator_type)) + return ira->codegen->builtin_types.entry_invalid; + } + ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); out_val->data.x_type = get_fn_type(ira->codegen, &fn_type_id); return ira->codegen->builtin_types.entry_type; diff --git a/src/parser.cpp b/src/parser.cpp index 0c9b7e326a..666b9da3c3 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -955,6 +955,66 @@ static AstNode *ast_parse_curly_suffix_expr(ParseContext *pc, size_t *token_inde } } +static AstNode *ast_parse_fn_proto_partial(ParseContext *pc, size_t *token_index, Token *fn_token, + AstNode *async_allocator_type_node, CallingConvention cc, bool is_extern, VisibMod visib_mod) +{ + AstNode *node = ast_create_node(pc, NodeTypeFnProto, fn_token); + node->data.fn_proto.visib_mod = visib_mod; + node->data.fn_proto.cc = cc; + node->data.fn_proto.is_extern = is_extern; + node->data.fn_proto.async_allocator_type = async_allocator_type_node; + + Token *fn_name = &pc->tokens->at(*token_index); + + if (fn_name->id == TokenIdSymbol) { + *token_index += 1; + node->data.fn_proto.name = token_buf(fn_name); + } else { + node->data.fn_proto.name = nullptr; + } + + ast_parse_param_decl_list(pc, token_index, &node->data.fn_proto.params, &node->data.fn_proto.is_var_args); + + Token *next_token = &pc->tokens->at(*token_index); + if (next_token->id == TokenIdKeywordAlign) { + *token_index += 1; + ast_eat_token(pc, token_index, TokenIdLParen); + + node->data.fn_proto.align_expr = ast_parse_expression(pc, token_index, true); + ast_eat_token(pc, token_index, TokenIdRParen); + next_token = &pc->tokens->at(*token_index); + } + if (next_token->id == TokenIdKeywordSection) { + *token_index += 1; + ast_eat_token(pc, token_index, TokenIdLParen); + + node->data.fn_proto.section_expr = ast_parse_expression(pc, token_index, true); + ast_eat_token(pc, token_index, TokenIdRParen); + next_token = &pc->tokens->at(*token_index); + } + if (next_token->id == TokenIdKeywordVar) { + node->data.fn_proto.return_var_token = next_token; + *token_index += 1; + next_token = &pc->tokens->at(*token_index); + } else { + if (next_token->id == TokenIdKeywordError) { + Token *maybe_lbrace_tok = &pc->tokens->at(*token_index + 1); + if (maybe_lbrace_tok->id == TokenIdLBrace) { + *token_index += 1; + node->data.fn_proto.return_type = ast_create_node(pc, NodeTypeErrorType, next_token); + return node; + } + } else if (next_token->id == TokenIdBang) { + *token_index += 1; + node->data.fn_proto.auto_err_set = true; + next_token = &pc->tokens->at(*token_index); + } + node->data.fn_proto.return_type = ast_parse_type_expr(pc, token_index, true); + } + + return node; +} + /* SuffixOpExpression = ("async" option("<" SuffixOpExpression ">") SuffixOpExpression FnCallExpression) | PrimaryExpression option(FnCallExpression | ArrayAccessExpression | FieldAccessExpression | SliceExpression) FnCallExpression : token(LParen) list(Expression, token(Comma)) token(RParen) @@ -979,6 +1039,11 @@ static AstNode *ast_parse_suffix_op_expr(ParseContext *pc, size_t *token_index, } Token *fncall_token = &pc->tokens->at(*token_index); + if (fncall_token->id == TokenIdKeywordFn) { + *token_index += 1; + return ast_parse_fn_proto_partial(pc, token_index, fncall_token, allocator_expr_node, CallingConventionAsync, + false, VisibModPrivate); + } AstNode *node = ast_parse_suffix_op_expr(pc, token_index, true); if (node->type != NodeTypeFnCallExpr) { ast_error(pc, fncall_token, "expected function call, found '%s'", token_name(fncall_token->id)); @@ -2434,9 +2499,10 @@ static AstNode *ast_parse_fn_proto(ParseContext *pc, size_t *token_index, bool m } else if (first_token->id == TokenIdKeywordAsync) { *token_index += 1; Token *next_token = &pc->tokens->at(*token_index); - if (next_token->id == TokenIdLParen) { + if (next_token->id == TokenIdCmpLessThan) { + *token_index += 1; async_allocator_type_node = ast_parse_type_expr(pc, token_index, true); - ast_eat_token(pc, token_index, TokenIdRParen); + ast_eat_token(pc, token_index, TokenIdCmpGreaterThan); } fn_token = ast_eat_token(pc, token_index, TokenIdKeywordFn); cc = CallingConventionAsync; @@ -2470,61 +2536,7 @@ static AstNode *ast_parse_fn_proto(ParseContext *pc, size_t *token_index, bool m return nullptr; } - AstNode *node = ast_create_node(pc, NodeTypeFnProto, fn_token); - node->data.fn_proto.visib_mod = visib_mod; - node->data.fn_proto.cc = cc; - node->data.fn_proto.is_extern = is_extern; - node->data.fn_proto.async_allocator_type = async_allocator_type_node; - - Token *fn_name = &pc->tokens->at(*token_index); - - if (fn_name->id == TokenIdSymbol) { - *token_index += 1; - node->data.fn_proto.name = token_buf(fn_name); - } else { - node->data.fn_proto.name = nullptr; - } - - ast_parse_param_decl_list(pc, token_index, &node->data.fn_proto.params, &node->data.fn_proto.is_var_args); - - Token *next_token = &pc->tokens->at(*token_index); - if (next_token->id == TokenIdKeywordAlign) { - *token_index += 1; - ast_eat_token(pc, token_index, TokenIdLParen); - - node->data.fn_proto.align_expr = ast_parse_expression(pc, token_index, true); - ast_eat_token(pc, token_index, TokenIdRParen); - next_token = &pc->tokens->at(*token_index); - } - if (next_token->id == TokenIdKeywordSection) { - *token_index += 1; - ast_eat_token(pc, token_index, TokenIdLParen); - - node->data.fn_proto.section_expr = ast_parse_expression(pc, token_index, true); - ast_eat_token(pc, token_index, TokenIdRParen); - next_token = &pc->tokens->at(*token_index); - } - if (next_token->id == TokenIdKeywordVar) { - node->data.fn_proto.return_var_token = next_token; - *token_index += 1; - next_token = &pc->tokens->at(*token_index); - } else { - if (next_token->id == TokenIdKeywordError) { - Token *maybe_lbrace_tok = &pc->tokens->at(*token_index + 1); - if (maybe_lbrace_tok->id == TokenIdLBrace) { - *token_index += 1; - node->data.fn_proto.return_type = ast_create_node(pc, NodeTypeErrorType, next_token); - return node; - } - } else if (next_token->id == TokenIdBang) { - *token_index += 1; - node->data.fn_proto.auto_err_set = true; - next_token = &pc->tokens->at(*token_index); - } - node->data.fn_proto.return_type = ast_parse_type_expr(pc, token_index, true); - } - - return node; + return ast_parse_fn_proto_partial(pc, token_index, fn_token, async_allocator_type_node, cc, is_extern, visib_mod); } /* diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 9e98276e0c..f8ad58f70d 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -156,3 +156,23 @@ test "async function with dot syntax" { cancel p; assert(S.y == 2); } + +test "async fn pointer in a struct field" { + var data: i32 = 1; + const Foo = struct { + bar: async<&std.mem.Allocator> fn(&i32) void, + }; + var foo = Foo { + .bar = simpleAsyncFn2, + }; + const p = (async foo.bar(&data)) catch unreachable; + assert(data == 2); + cancel p; + assert(data == 4); +} + +async<&std.mem.Allocator> fn simpleAsyncFn2(y: &i32) void { + defer *y += 2; + *y += 1; + suspend; +} From 3d1732ef6c8ebc7edf10485f61b4ec905303cd8a Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Fri, 23 Mar 2018 20:27:11 +1300 Subject: [PATCH 02/47] Fix OpqaueType usage in exported c functions We prefer `struct typename`. If a typedef is required, this must be done manually after generation. --- src/codegen.cpp | 6 +----- test/gen_h.zig | 11 +++++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index a44091bdbf..de8cfa31ed 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6592,6 +6592,7 @@ static void get_c_type(CodeGen *g, GenH *gen_h, TypeTableEntry *type_entry, Buf } } case TypeTableEntryIdStruct: + case TypeTableEntryIdOpaque: { buf_init_from_str(out_buf, "struct "); buf_append_buf(out_buf, &type_entry->name); @@ -6609,11 +6610,6 @@ static void get_c_type(CodeGen *g, GenH *gen_h, TypeTableEntry *type_entry, Buf buf_append_buf(out_buf, &type_entry->name); return; } - case TypeTableEntryIdOpaque: - { - buf_init_from_buf(out_buf, &type_entry->name); - return; - } case TypeTableEntryIdArray: { TypeTableEntryArray *array_data = &type_entry->data.array; diff --git a/test/gen_h.zig b/test/gen_h.zig index 5f28326ff6..30d168cf2c 100644 --- a/test/gen_h.zig +++ b/test/gen_h.zig @@ -51,6 +51,16 @@ pub fn addCases(cases: &tests.GenHContext) void { \\ ); + cases.add("declare opaque type", + \\export const Foo = @OpaqueType(); + \\ + \\export fn entry(foo: ?&Foo) void { } + , + \\struct Foo; + \\ + \\TEST_EXPORT void entry(struct Foo * foo); + ); + cases.add("array field-type", \\const Foo = extern struct { \\ A: [2]i32, @@ -66,4 +76,5 @@ pub fn addCases(cases: &tests.GenHContext) void { \\TEST_EXPORT void entry(struct Foo foo, uint8_t bar[]); \\ ); + } From 7350181a4a778f9d03186e5123beffdf80f58606 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Fri, 23 Mar 2018 22:41:08 +1300 Subject: [PATCH 03/47] Fix os.File.mode function --- std/os/file.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/os/file.zig b/std/os/file.zig index 772fbf7c73..eed3a443b9 100644 --- a/std/os/file.zig +++ b/std/os/file.zig @@ -233,7 +233,7 @@ pub const File = struct { Unexpected, }; - fn mode(self: &File) ModeError!FileMode { + fn mode(self: &File) ModeError!os.FileMode { if (is_posix) { var stat: posix.Stat = undefined; const err = posix.getErrno(posix.fstat(self.handle, &stat)); From 2cff31937f6008769ad1034f9451d136d03c06bb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 24 Mar 2018 15:57:36 -0400 Subject: [PATCH 04/47] std.os.linux exposes syscall functions and syscall numbers --- std/os/linux/index.zig | 158 +++++++++++++++++------------------------ 1 file changed, 67 insertions(+), 91 deletions(-) diff --git a/std/os/linux/index.zig b/std/os/linux/index.zig index 646b1ef300..8fd8bcbe78 100644 --- a/std/os/linux/index.zig +++ b/std/os/linux/index.zig @@ -1,7 +1,7 @@ const std = @import("../../index.zig"); const assert = std.debug.assert; const builtin = @import("builtin"); -const arch = switch (builtin.arch) { +pub use switch (builtin.arch) { builtin.Arch.x86_64 => @import("x86_64.zig"), builtin.Arch.i386 => @import("i386.zig"), else => @compileError("unsupported arch"), @@ -93,27 +93,6 @@ pub const O_RDONLY = 0o0; pub const O_WRONLY = 0o1; pub const O_RDWR = 0o2; -pub const O_CREAT = arch.O_CREAT; -pub const O_EXCL = arch.O_EXCL; -pub const O_NOCTTY = arch.O_NOCTTY; -pub const O_TRUNC = arch.O_TRUNC; -pub const O_APPEND = arch.O_APPEND; -pub const O_NONBLOCK = arch.O_NONBLOCK; -pub const O_DSYNC = arch.O_DSYNC; -pub const O_SYNC = arch.O_SYNC; -pub const O_RSYNC = arch.O_RSYNC; -pub const O_DIRECTORY = arch.O_DIRECTORY; -pub const O_NOFOLLOW = arch.O_NOFOLLOW; -pub const O_CLOEXEC = arch.O_CLOEXEC; - -pub const O_ASYNC = arch.O_ASYNC; -pub const O_DIRECT = arch.O_DIRECT; -pub const O_LARGEFILE = arch.O_LARGEFILE; -pub const O_NOATIME = arch.O_NOATIME; -pub const O_PATH = arch.O_PATH; -pub const O_TMPFILE = arch.O_TMPFILE; -pub const O_NDELAY = arch.O_NDELAY; - pub const SEEK_SET = 0; pub const SEEK_CUR = 1; pub const SEEK_END = 2; @@ -394,65 +373,65 @@ pub fn getErrno(r: usize) usize { } pub fn dup2(old: i32, new: i32) usize { - return arch.syscall2(arch.SYS_dup2, usize(old), usize(new)); + return syscall2(SYS_dup2, usize(old), usize(new)); } pub fn chdir(path: &const u8) usize { - return arch.syscall1(arch.SYS_chdir, @ptrToInt(path)); + return syscall1(SYS_chdir, @ptrToInt(path)); } pub fn execve(path: &const u8, argv: &const ?&const u8, envp: &const ?&const u8) usize { - return arch.syscall3(arch.SYS_execve, @ptrToInt(path), @ptrToInt(argv), @ptrToInt(envp)); + return syscall3(SYS_execve, @ptrToInt(path), @ptrToInt(argv), @ptrToInt(envp)); } pub fn fork() usize { - return arch.syscall0(arch.SYS_fork); + return syscall0(SYS_fork); } pub fn getcwd(buf: &u8, size: usize) usize { - return arch.syscall2(arch.SYS_getcwd, @ptrToInt(buf), size); + return syscall2(SYS_getcwd, @ptrToInt(buf), size); } pub fn getdents(fd: i32, dirp: &u8, count: usize) usize { - return arch.syscall3(arch.SYS_getdents, usize(fd), @ptrToInt(dirp), count); + return syscall3(SYS_getdents, usize(fd), @ptrToInt(dirp), count); } pub fn isatty(fd: i32) bool { var wsz: winsize = undefined; - return arch.syscall3(arch.SYS_ioctl, usize(fd), TIOCGWINSZ, @ptrToInt(&wsz)) == 0; + return syscall3(SYS_ioctl, usize(fd), TIOCGWINSZ, @ptrToInt(&wsz)) == 0; } pub fn readlink(noalias path: &const u8, noalias buf_ptr: &u8, buf_len: usize) usize { - return arch.syscall3(arch.SYS_readlink, @ptrToInt(path), @ptrToInt(buf_ptr), buf_len); + return syscall3(SYS_readlink, @ptrToInt(path), @ptrToInt(buf_ptr), buf_len); } pub fn mkdir(path: &const u8, mode: u32) usize { - return arch.syscall2(arch.SYS_mkdir, @ptrToInt(path), mode); + return syscall2(SYS_mkdir, @ptrToInt(path), mode); } pub fn mmap(address: ?&u8, length: usize, prot: usize, flags: usize, fd: i32, offset: isize) usize { - return arch.syscall6(arch.SYS_mmap, @ptrToInt(address), length, prot, flags, usize(fd), + return syscall6(SYS_mmap, @ptrToInt(address), length, prot, flags, usize(fd), @bitCast(usize, offset)); } pub fn munmap(address: &u8, length: usize) usize { - return arch.syscall2(arch.SYS_munmap, @ptrToInt(address), length); + return syscall2(SYS_munmap, @ptrToInt(address), length); } pub fn read(fd: i32, buf: &u8, count: usize) usize { - return arch.syscall3(arch.SYS_read, usize(fd), @ptrToInt(buf), count); + return syscall3(SYS_read, usize(fd), @ptrToInt(buf), count); } pub fn rmdir(path: &const u8) usize { - return arch.syscall1(arch.SYS_rmdir, @ptrToInt(path)); + return syscall1(SYS_rmdir, @ptrToInt(path)); } pub fn symlink(existing: &const u8, new: &const u8) usize { - return arch.syscall2(arch.SYS_symlink, @ptrToInt(existing), @ptrToInt(new)); + return syscall2(SYS_symlink, @ptrToInt(existing), @ptrToInt(new)); } pub fn pread(fd: i32, buf: &u8, count: usize, offset: usize) usize { - return arch.syscall4(arch.SYS_pread, usize(fd), @ptrToInt(buf), count, offset); + return syscall4(SYS_pread, usize(fd), @ptrToInt(buf), count, offset); } pub fn pipe(fd: &[2]i32) usize { @@ -460,84 +439,84 @@ pub fn pipe(fd: &[2]i32) usize { } pub fn pipe2(fd: &[2]i32, flags: usize) usize { - return arch.syscall2(arch.SYS_pipe2, @ptrToInt(fd), flags); + return syscall2(SYS_pipe2, @ptrToInt(fd), flags); } pub fn write(fd: i32, buf: &const u8, count: usize) usize { - return arch.syscall3(arch.SYS_write, usize(fd), @ptrToInt(buf), count); + return syscall3(SYS_write, usize(fd), @ptrToInt(buf), count); } pub fn pwrite(fd: i32, buf: &const u8, count: usize, offset: usize) usize { - return arch.syscall4(arch.SYS_pwrite, usize(fd), @ptrToInt(buf), count, offset); + return syscall4(SYS_pwrite, usize(fd), @ptrToInt(buf), count, offset); } pub fn rename(old: &const u8, new: &const u8) usize { - return arch.syscall2(arch.SYS_rename, @ptrToInt(old), @ptrToInt(new)); + return syscall2(SYS_rename, @ptrToInt(old), @ptrToInt(new)); } pub fn open(path: &const u8, flags: u32, perm: usize) usize { - return arch.syscall3(arch.SYS_open, @ptrToInt(path), flags, perm); + return syscall3(SYS_open, @ptrToInt(path), flags, perm); } pub fn create(path: &const u8, perm: usize) usize { - return arch.syscall2(arch.SYS_creat, @ptrToInt(path), perm); + return syscall2(SYS_creat, @ptrToInt(path), perm); } pub fn openat(dirfd: i32, path: &const u8, flags: usize, mode: usize) usize { - return arch.syscall4(arch.SYS_openat, usize(dirfd), @ptrToInt(path), flags, mode); + return syscall4(SYS_openat, usize(dirfd), @ptrToInt(path), flags, mode); } pub fn close(fd: i32) usize { - return arch.syscall1(arch.SYS_close, usize(fd)); + return syscall1(SYS_close, usize(fd)); } pub fn lseek(fd: i32, offset: isize, ref_pos: usize) usize { - return arch.syscall3(arch.SYS_lseek, usize(fd), @bitCast(usize, offset), ref_pos); + return syscall3(SYS_lseek, usize(fd), @bitCast(usize, offset), ref_pos); } pub fn exit(status: i32) noreturn { - _ = arch.syscall1(arch.SYS_exit, @bitCast(usize, isize(status))); + _ = syscall1(SYS_exit, @bitCast(usize, isize(status))); unreachable; } pub fn getrandom(buf: &u8, count: usize, flags: u32) usize { - return arch.syscall3(arch.SYS_getrandom, @ptrToInt(buf), count, usize(flags)); + return syscall3(SYS_getrandom, @ptrToInt(buf), count, usize(flags)); } pub fn kill(pid: i32, sig: i32) usize { - return arch.syscall2(arch.SYS_kill, @bitCast(usize, isize(pid)), usize(sig)); + return syscall2(SYS_kill, @bitCast(usize, isize(pid)), usize(sig)); } pub fn unlink(path: &const u8) usize { - return arch.syscall1(arch.SYS_unlink, @ptrToInt(path)); + return syscall1(SYS_unlink, @ptrToInt(path)); } pub fn waitpid(pid: i32, status: &i32, options: i32) usize { - return arch.syscall4(arch.SYS_wait4, @bitCast(usize, isize(pid)), @ptrToInt(status), @bitCast(usize, isize(options)), 0); + return syscall4(SYS_wait4, @bitCast(usize, isize(pid)), @ptrToInt(status), @bitCast(usize, isize(options)), 0); } pub fn nanosleep(req: &const timespec, rem: ?×pec) usize { - return arch.syscall2(arch.SYS_nanosleep, @ptrToInt(req), @ptrToInt(rem)); + return syscall2(SYS_nanosleep, @ptrToInt(req), @ptrToInt(rem)); } pub fn setuid(uid: u32) usize { - return arch.syscall1(arch.SYS_setuid, uid); + return syscall1(SYS_setuid, uid); } pub fn setgid(gid: u32) usize { - return arch.syscall1(arch.SYS_setgid, gid); + return syscall1(SYS_setgid, gid); } pub fn setreuid(ruid: u32, euid: u32) usize { - return arch.syscall2(arch.SYS_setreuid, ruid, euid); + return syscall2(SYS_setreuid, ruid, euid); } pub fn setregid(rgid: u32, egid: u32) usize { - return arch.syscall2(arch.SYS_setregid, rgid, egid); + return syscall2(SYS_setregid, rgid, egid); } pub fn sigprocmask(flags: u32, noalias set: &const sigset_t, noalias oldset: ?&sigset_t) usize { - return arch.syscall4(arch.SYS_rt_sigprocmask, flags, @ptrToInt(set), @ptrToInt(oldset), NSIG/8); + return syscall4(SYS_rt_sigprocmask, flags, @ptrToInt(set), @ptrToInt(oldset), NSIG/8); } pub fn sigaction(sig: u6, noalias act: &const Sigaction, noalias oact: ?&Sigaction) usize { @@ -548,11 +527,11 @@ pub fn sigaction(sig: u6, noalias act: &const Sigaction, noalias oact: ?&Sigacti .handler = act.handler, .flags = act.flags | SA_RESTORER, .mask = undefined, - .restorer = @ptrCast(extern fn()void, arch.restore_rt), + .restorer = @ptrCast(extern fn()void, restore_rt), }; var ksa_old: k_sigaction = undefined; @memcpy(@ptrCast(&u8, &ksa.mask), @ptrCast(&const u8, &act.mask), 8); - const result = arch.syscall4(arch.SYS_rt_sigaction, sig, @ptrToInt(&ksa), @ptrToInt(&ksa_old), @sizeOf(@typeOf(ksa.mask))); + const result = syscall4(SYS_rt_sigaction, sig, @ptrToInt(&ksa), @ptrToInt(&ksa_old), @sizeOf(@typeOf(ksa.mask))); const err = getErrno(result); if (err != 0) { return result; @@ -592,22 +571,22 @@ pub const empty_sigset = []usize{0} ** sigset_t.len; pub fn raise(sig: i32) usize { var set: sigset_t = undefined; blockAppSignals(&set); - const tid = i32(arch.syscall0(arch.SYS_gettid)); - const ret = arch.syscall2(arch.SYS_tkill, usize(tid), usize(sig)); + const tid = i32(syscall0(SYS_gettid)); + const ret = syscall2(SYS_tkill, usize(tid), usize(sig)); restoreSignals(&set); return ret; } fn blockAllSignals(set: &sigset_t) void { - _ = arch.syscall4(arch.SYS_rt_sigprocmask, SIG_BLOCK, @ptrToInt(&all_mask), @ptrToInt(set), NSIG/8); + _ = syscall4(SYS_rt_sigprocmask, SIG_BLOCK, @ptrToInt(&all_mask), @ptrToInt(set), NSIG/8); } fn blockAppSignals(set: &sigset_t) void { - _ = arch.syscall4(arch.SYS_rt_sigprocmask, SIG_BLOCK, @ptrToInt(&app_mask), @ptrToInt(set), NSIG/8); + _ = syscall4(SYS_rt_sigprocmask, SIG_BLOCK, @ptrToInt(&app_mask), @ptrToInt(set), NSIG/8); } fn restoreSignals(set: &sigset_t) void { - _ = arch.syscall4(arch.SYS_rt_sigprocmask, SIG_SETMASK, @ptrToInt(set), 0, NSIG/8); + _ = syscall4(SYS_rt_sigprocmask, SIG_SETMASK, @ptrToInt(set), 0, NSIG/8); } pub fn sigaddset(set: &sigset_t, sig: u6) void { @@ -653,61 +632,61 @@ pub const iovec = extern struct { }; pub fn getsockname(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t) usize { - return arch.syscall3(arch.SYS_getsockname, usize(fd), @ptrToInt(addr), @ptrToInt(len)); + return syscall3(SYS_getsockname, usize(fd), @ptrToInt(addr), @ptrToInt(len)); } pub fn getpeername(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t) usize { - return arch.syscall3(arch.SYS_getpeername, usize(fd), @ptrToInt(addr), @ptrToInt(len)); + return syscall3(SYS_getpeername, usize(fd), @ptrToInt(addr), @ptrToInt(len)); } pub fn socket(domain: i32, socket_type: i32, protocol: i32) usize { - return arch.syscall3(arch.SYS_socket, usize(domain), usize(socket_type), usize(protocol)); + return syscall3(SYS_socket, usize(domain), usize(socket_type), usize(protocol)); } pub fn setsockopt(fd: i32, level: i32, optname: i32, optval: &const u8, optlen: socklen_t) usize { - return arch.syscall5(arch.SYS_setsockopt, usize(fd), usize(level), usize(optname), usize(optval), @ptrToInt(optlen)); + return syscall5(SYS_setsockopt, usize(fd), usize(level), usize(optname), usize(optval), @ptrToInt(optlen)); } pub fn getsockopt(fd: i32, level: i32, optname: i32, noalias optval: &u8, noalias optlen: &socklen_t) usize { - return arch.syscall5(arch.SYS_getsockopt, usize(fd), usize(level), usize(optname), @ptrToInt(optval), @ptrToInt(optlen)); + return syscall5(SYS_getsockopt, usize(fd), usize(level), usize(optname), @ptrToInt(optval), @ptrToInt(optlen)); } -pub fn sendmsg(fd: i32, msg: &const arch.msghdr, flags: u32) usize { - return arch.syscall3(arch.SYS_sendmsg, usize(fd), @ptrToInt(msg), flags); +pub fn sendmsg(fd: i32, msg: &const msghdr, flags: u32) usize { + return syscall3(SYS_sendmsg, usize(fd), @ptrToInt(msg), flags); } pub fn connect(fd: i32, addr: &const sockaddr, len: socklen_t) usize { - return arch.syscall3(arch.SYS_connect, usize(fd), @ptrToInt(addr), usize(len)); + return syscall3(SYS_connect, usize(fd), @ptrToInt(addr), usize(len)); } -pub fn recvmsg(fd: i32, msg: &arch.msghdr, flags: u32) usize { - return arch.syscall3(arch.SYS_recvmsg, usize(fd), @ptrToInt(msg), flags); +pub fn recvmsg(fd: i32, msg: &msghdr, flags: u32) usize { + return syscall3(SYS_recvmsg, usize(fd), @ptrToInt(msg), flags); } pub fn recvfrom(fd: i32, noalias buf: &u8, len: usize, flags: u32, noalias addr: ?&sockaddr, noalias alen: ?&socklen_t) usize { - return arch.syscall6(arch.SYS_recvfrom, usize(fd), @ptrToInt(buf), len, flags, @ptrToInt(addr), @ptrToInt(alen)); + return syscall6(SYS_recvfrom, usize(fd), @ptrToInt(buf), len, flags, @ptrToInt(addr), @ptrToInt(alen)); } pub fn shutdown(fd: i32, how: i32) usize { - return arch.syscall2(arch.SYS_shutdown, usize(fd), usize(how)); + return syscall2(SYS_shutdown, usize(fd), usize(how)); } pub fn bind(fd: i32, addr: &const sockaddr, len: socklen_t) usize { - return arch.syscall3(arch.SYS_bind, usize(fd), @ptrToInt(addr), usize(len)); + return syscall3(SYS_bind, usize(fd), @ptrToInt(addr), usize(len)); } pub fn listen(fd: i32, backlog: i32) usize { - return arch.syscall2(arch.SYS_listen, usize(fd), usize(backlog)); + return syscall2(SYS_listen, usize(fd), usize(backlog)); } pub fn sendto(fd: i32, buf: &const u8, len: usize, flags: u32, addr: ?&const sockaddr, alen: socklen_t) usize { - return arch.syscall6(arch.SYS_sendto, usize(fd), @ptrToInt(buf), len, flags, @ptrToInt(addr), usize(alen)); + return syscall6(SYS_sendto, usize(fd), @ptrToInt(buf), len, flags, @ptrToInt(addr), usize(alen)); } pub fn socketpair(domain: i32, socket_type: i32, protocol: i32, fd: [2]i32) usize { - return arch.syscall4(arch.SYS_socketpair, usize(domain), usize(socket_type), usize(protocol), @ptrToInt(&fd[0])); + return syscall4(SYS_socketpair, usize(domain), usize(socket_type), usize(protocol), @ptrToInt(&fd[0])); } pub fn accept(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t) usize { @@ -715,7 +694,7 @@ pub fn accept(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t) usize { } pub fn accept4(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t, flags: u32) usize { - return arch.syscall4(arch.SYS_accept4, usize(fd), @ptrToInt(addr), @ptrToInt(len), flags); + return syscall4(SYS_accept4, usize(fd), @ptrToInt(addr), @ptrToInt(len), flags); } // error NameTooLong; @@ -746,11 +725,8 @@ pub fn accept4(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t, flags: // return ifr.ifr_ifindex; // } -pub const Stat = arch.Stat; -pub const timespec = arch.timespec; - pub fn fstat(fd: i32, stat_buf: &Stat) usize { - return arch.syscall2(arch.SYS_fstat, usize(fd), @ptrToInt(stat_buf)); + return syscall2(SYS_fstat, usize(fd), @ptrToInt(stat_buf)); } pub const epoll_data = extern union { @@ -770,19 +746,19 @@ pub fn epoll_create() usize { } pub fn epoll_create1(flags: usize) usize { - return arch.syscall1(arch.SYS_epoll_create1, flags); + return syscall1(SYS_epoll_create1, flags); } pub fn epoll_ctl(epoll_fd: i32, op: i32, fd: i32, ev: &epoll_event) usize { - return arch.syscall4(arch.SYS_epoll_ctl, usize(epoll_fd), usize(op), usize(fd), @ptrToInt(ev)); + return syscall4(SYS_epoll_ctl, usize(epoll_fd), usize(op), usize(fd), @ptrToInt(ev)); } pub fn epoll_wait(epoll_fd: i32, events: &epoll_event, maxevents: u32, timeout: i32) usize { - return arch.syscall4(arch.SYS_epoll_wait, usize(epoll_fd), @ptrToInt(events), usize(maxevents), usize(timeout)); + return syscall4(SYS_epoll_wait, usize(epoll_fd), @ptrToInt(events), usize(maxevents), usize(timeout)); } pub fn timerfd_create(clockid: i32, flags: u32) usize { - return arch.syscall2(arch.SYS_timerfd_create, usize(clockid), usize(flags)); + return syscall2(SYS_timerfd_create, usize(clockid), usize(flags)); } pub const itimerspec = extern struct { @@ -791,11 +767,11 @@ pub const itimerspec = extern struct { }; pub fn timerfd_gettime(fd: i32, curr_value: &itimerspec) usize { - return arch.syscall2(arch.SYS_timerfd_gettime, usize(fd), @ptrToInt(curr_value)); + return syscall2(SYS_timerfd_gettime, usize(fd), @ptrToInt(curr_value)); } pub fn timerfd_settime(fd: i32, flags: u32, new_value: &const itimerspec, old_value: ?&itimerspec) usize { - return arch.syscall4(arch.SYS_timerfd_settime, usize(fd), usize(flags), @ptrToInt(new_value), @ptrToInt(old_value)); + return syscall4(SYS_timerfd_settime, usize(fd), usize(flags), @ptrToInt(new_value), @ptrToInt(old_value)); } test "import linux test" { From b1c07c0ea9e351a43c9bc2fe747fc07c0a19e005 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 12 Mar 2018 23:03:47 -0400 Subject: [PATCH 05/47] move error ret tracing codegen to zig ir progress towards #821 --- src/all_types.hpp | 5 +++ src/codegen.cpp | 39 ++++++++++----------- src/ir.cpp | 86 ++++++++++++++++++++++++++++++++++------------- src/ir_print.cpp | 7 ++++ 4 files changed, 91 insertions(+), 46 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index c1dec80d7e..b5428c9599 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -2034,6 +2034,7 @@ enum IrInstructionId { IrInstructionIdAtomicRmw, IrInstructionIdPromiseResultType, IrInstructionIdAwaitBookkeeping, + IrInstructionIdSaveErrRetAddr, }; struct IrInstruction { @@ -2988,6 +2989,10 @@ struct IrInstructionAwaitBookkeeping { IrInstruction *promise_result_type; }; +struct IrInstructionSaveErrRetAddr { + IrInstruction base; +}; + static const size_t slice_ptr_index = 0; static const size_t slice_len_index = 1; diff --git a/src/codegen.cpp b/src/codegen.cpp index de8cfa31ed..eddadb94ea 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1607,32 +1607,25 @@ static LLVMValueRef ir_llvm_value(CodeGen *g, IrInstruction *instruction) { return instruction->llvm_value; } +static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, IrExecutable *executable, + IrInstructionSaveErrRetAddr *save_err_ret_addr_instruction) +{ + assert(g->have_err_ret_tracing); + + LLVMValueRef return_err_fn = get_return_err_fn(g); + LLVMValueRef args[] = { + g->cur_err_ret_trace_val, + }; + LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, return_err_fn, args, 1, + get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); + LLVMSetTailCall(call_instruction, true); + return call_instruction; +} + static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrInstructionReturn *return_instruction) { LLVMValueRef value = ir_llvm_value(g, return_instruction->value); TypeTableEntry *return_type = return_instruction->value->value.type; - if (g->have_err_ret_tracing) { - bool is_err_return = false; - if (return_type->id == TypeTableEntryIdErrorUnion) { - if (return_instruction->value->value.special == ConstValSpecialStatic) { - is_err_return = return_instruction->value->value.data.x_err_union.err != nullptr; - } else if (return_instruction->value->value.special == ConstValSpecialRuntime) { - is_err_return = return_instruction->value->value.data.rh_error_union == RuntimeHintErrorUnionError; - // TODO: emit a branch to check if the return value is an error - } - } else if (return_type->id == TypeTableEntryIdErrorSet) { - is_err_return = true; - } - if (is_err_return) { - LLVMValueRef return_err_fn = get_return_err_fn(g); - LLVMValueRef args[] = { - g->cur_err_ret_trace_val, - }; - LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, return_err_fn, args, 1, - get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); - LLVMSetTailCall(call_instruction, true); - } - } if (handle_is_ptr(return_type)) { if (calling_convention_does_first_arg_return(g->cur_fn->type_entry->data.fn.fn_type_id.cc)) { assert(g->cur_ret_ptr); @@ -4400,6 +4393,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, return ir_render_coro_alloc_helper(g, executable, (IrInstructionCoroAllocHelper *)instruction); case IrInstructionIdAtomicRmw: return ir_render_atomic_rmw(g, executable, (IrInstructionAtomicRmw *)instruction); + case IrInstructionIdSaveErrRetAddr: + return ir_render_save_err_ret_addr(g, executable, (IrInstructionSaveErrRetAddr *)instruction); } zig_unreachable(); } diff --git a/src/ir.cpp b/src/ir.cpp index 8f418e2c2f..a78be66dd6 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -713,6 +713,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionAwaitBookkeeping return IrInstructionIdAwaitBookkeeping; } +static constexpr IrInstructionId ir_instruction_id(IrInstructionSaveErrRetAddr *) { + return IrInstructionIdSaveErrRetAddr; +} + template static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) { T *special_instruction = allocate(1); @@ -2678,6 +2682,11 @@ static IrInstruction *ir_build_await_bookkeeping(IrBuilder *irb, Scope *scope, A return &instruction->base; } +static IrInstruction *ir_build_save_err_ret_addr(IrBuilder *irb, Scope *scope, AstNode *source_node) { + IrInstructionSaveErrRetAddr *instruction = ir_build_instruction(irb, scope, source_node); + return &instruction->base; +} + static void ir_count_defers(IrBuilder *irb, Scope *inner_scope, Scope *outer_scope, size_t *results) { results[ReturnKindUnconditional] = 0; results[ReturnKindError] = 0; @@ -2750,16 +2759,16 @@ static ScopeDeferExpr *get_scope_defer_expr(Scope *scope) { return nullptr; } +static bool exec_is_async(IrExecutable *exec) { + FnTableEntry *fn_entry = exec_fn_entry(exec); + return fn_entry != nullptr && fn_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync; +} + static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode *node, IrInstruction *return_value, bool is_generated_code) { - FnTableEntry *fn_entry = exec_fn_entry(irb->exec); - bool is_async = fn_entry != nullptr && fn_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync; + bool is_async = exec_is_async(irb->exec); if (!is_async) { - //if (irb->codegen->have_err_ret_tracing) { - // IrInstruction *stack_trace_ptr = ir_build_error_return_trace_nonnull(irb, scope, node); - // ir_build_save_err_ret_addr(irb, scope, node, stack_trace_ptr); - //} IrInstruction *return_inst = ir_build_return(irb, scope, node, return_value); return_inst->is_gen = is_generated_code; return return_inst; @@ -2781,21 +2790,33 @@ static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode // the above blocks are rendered by ir_gen after the rest of codegen } -//static void ir_gen_save_err_ret_addr(IrBuilder *irb, Scope *scope, AstNode *node, bool is_async) { -// if (!irb->codegen->have_err_ret_tracing) -// return; -// -// if (is_async) { -// IrInstruction *err_ret_addr_ptr = ir_build_load_ptr(irb, scope, node, irb->exec->coro_err_ret_addr_ptr); -// IrInstruction *return_address_ptr = ir_build_return_address(irb, scope, node); -// IrInstruction *return_address_usize = ir_build_ptr_to_int(irb, scope, node, return_address_ptr); -// ir_build_store_ptr(irb, scope, node, err_ret_addr_ptr, return_address_usize); -// return; -// } -// -// IrInstruction *stack_trace_ptr = ir_build_error_return_trace_nonnull(irb, scope, node); -// ir_build_save_err_ret_addr(irb, scope, node, stack_trace_ptr); -//} +static bool exec_have_err_ret_trace(CodeGen *g, IrExecutable *exec) { + if (!g->have_err_ret_tracing) + return false; + FnTableEntry *fn_entry = exec_fn_entry(exec); + if (fn_entry == nullptr) + return false; + if (exec->is_inline) + return false; + return type_can_fail(fn_entry->type_entry->data.fn.fn_type_id.return_type); +} + +static void ir_gen_save_err_ret_addr(IrBuilder *irb, Scope *scope, AstNode *node) { + if (!exec_have_err_ret_trace(irb->codegen, irb->exec)) + return; + + bool is_async = exec_is_async(irb->exec); + + if (is_async) { + //IrInstruction *err_ret_addr_ptr = ir_build_load_ptr(irb, scope, node, irb->exec->coro_err_ret_addr_ptr); + //IrInstruction *return_address_ptr = ir_build_instr_addr(irb, scope, node); + //IrInstruction *return_address_usize = ir_build_ptr_to_int(irb, scope, node, return_address_ptr); + //ir_build_store_ptr(irb, scope, node, err_ret_addr_ptr, return_address_usize); + return; + } + + ir_build_save_err_ret_addr(irb, scope, node); +} static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, LVal lval) { assert(node->type == NodeTypeReturnExpr); @@ -2856,7 +2877,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, if (have_err_defers) { ir_gen_defers_for_block(irb, scope, outer_scope, true); } - //ir_gen_save_err_ret_addr(irb, scope, node, is_async); + ir_gen_save_err_ret_addr(irb, scope, node); ir_build_br(irb, scope, node, ret_stmt_block, is_comptime); ir_set_cursor_at_end_and_append_block(irb, ok_block); @@ -2895,6 +2916,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, ir_set_cursor_at_end_and_append_block(irb, return_block); ir_gen_defers_for_block(irb, scope, outer_scope, true); IrInstruction *err_val = ir_build_unwrap_err_code(irb, scope, node, err_union_ptr); + ir_gen_save_err_ret_addr(irb, scope, node); ir_gen_async_return(irb, scope, node, err_val, false); ir_set_cursor_at_end_and_append_block(irb, continue_block); @@ -6406,6 +6428,7 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec return false; if (!instr_is_unreachable(result)) { + // no need for save_err_ret_addr because this cannot return error ir_gen_async_return(irb, scope, result->source_node, result, true); } @@ -11464,13 +11487,17 @@ static TypeTableEntry *ir_analyze_instruction_export(IrAnalyze *ira, IrInstructi return ira->codegen->builtin_types.entry_void; } +static bool exec_has_err_ret_trace(CodeGen *g, IrExecutable *exec) { + FnTableEntry *fn_entry = exec_fn_entry(exec); + return fn_entry != nullptr && fn_entry->calls_or_awaits_errorable_fn && g->have_err_ret_tracing; +} + static TypeTableEntry *ir_analyze_instruction_error_return_trace(IrAnalyze *ira, IrInstructionErrorReturnTrace *instruction) { - FnTableEntry *fn_entry = exec_fn_entry(ira->new_irb.exec); TypeTableEntry *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(ira->codegen); TypeTableEntry *nullable_type = get_maybe_type(ira->codegen, ptr_to_stack_trace_type); - if (fn_entry == nullptr || !fn_entry->calls_or_awaits_errorable_fn || !ira->codegen->have_err_ret_tracing) { + if (!exec_has_err_ret_trace(ira->codegen, ira->new_irb.exec)) { ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); out_val->data.x_maybe = nullptr; return nullable_type; @@ -17775,6 +17802,14 @@ static TypeTableEntry *ir_analyze_instruction_await_bookkeeping(IrAnalyze *ira, return out_val->type; } +static TypeTableEntry *ir_analyze_instruction_save_err_ret_addr(IrAnalyze *ira, IrInstructionSaveErrRetAddr *instruction) { + IrInstruction *result = ir_build_save_err_ret_addr(&ira->new_irb, instruction->base.scope, + instruction->base.source_node); + ir_link_new_instruction(result, &instruction->base); + result->value.type = ira->codegen->builtin_types.entry_void; + return result->value.type; +} + static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstruction *instruction) { switch (instruction->id) { case IrInstructionIdInvalid: @@ -18012,6 +18047,8 @@ static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstructi return ir_analyze_instruction_promise_result_type(ira, (IrInstructionPromiseResultType *)instruction); case IrInstructionIdAwaitBookkeeping: return ir_analyze_instruction_await_bookkeeping(ira, (IrInstructionAwaitBookkeeping *)instruction); + case IrInstructionIdSaveErrRetAddr: + return ir_analyze_instruction_save_err_ret_addr(ira, (IrInstructionSaveErrRetAddr *)instruction); } zig_unreachable(); } @@ -18137,6 +18174,7 @@ bool ir_has_side_effects(IrInstruction *instruction) { case IrInstructionIdCoroSave: case IrInstructionIdCoroAllocHelper: case IrInstructionIdAwaitBookkeeping: + case IrInstructionIdSaveErrRetAddr: return true; case IrInstructionIdPhi: diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 167bd20839..709cb43942 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -1161,6 +1161,10 @@ static void ir_print_await_bookkeeping(IrPrint *irp, IrInstructionAwaitBookkeepi fprintf(irp->f, ")"); } +static void ir_print_save_err_ret_addr(IrPrint *irp, IrInstructionSaveErrRetAddr *instruction) { + fprintf(irp->f, "@saveErrRetAddr()"); +} + static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { ir_print_prefix(irp, instruction); switch (instruction->id) { @@ -1532,6 +1536,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { case IrInstructionIdAwaitBookkeeping: ir_print_await_bookkeeping(irp, (IrInstructionAwaitBookkeeping *)instruction); break; + case IrInstructionIdSaveErrRetAddr: + ir_print_save_err_ret_addr(irp, (IrInstructionSaveErrRetAddr *)instruction); + break; } fprintf(irp->f, "\n"); } From 18af2f9a2764cc340571578d58cb2575faeccdc6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 24 Mar 2018 18:21:51 -0400 Subject: [PATCH 06/47] fix async fns with inferred error sets closes #856 --- src/all_types.hpp | 12 ++++++++++- src/analyze.cpp | 10 ++++----- src/ast_render.cpp | 11 +++++++++- src/codegen.cpp | 1 + src/ir.cpp | 45 ++++++++++++++++++++++++++++++++++----- src/ir_print.cpp | 13 +++++++++-- test/cases/coroutines.zig | 11 ++++++++++ 7 files changed, 89 insertions(+), 14 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index b5428c9599..f5afecfbdd 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1251,7 +1251,10 @@ struct FnTableEntry { ScopeBlock *def_scope; // parent is child_scope Buf symbol_name; TypeTableEntry *type_entry; // function type - TypeTableEntry *implicit_return_type; + // in the case of normal functions this is the implicit return type + // in the case of async functions this is the implicit return type according to the + // zig source code, not according to zig ir + TypeTableEntry *src_implicit_return_type; bool is_test; FnInline fn_inline; FnAnalState anal_state; @@ -2035,6 +2038,7 @@ enum IrInstructionId { IrInstructionIdPromiseResultType, IrInstructionIdAwaitBookkeeping, IrInstructionIdSaveErrRetAddr, + IrInstructionIdAddImplicitReturnType, }; struct IrInstruction { @@ -2993,6 +2997,12 @@ struct IrInstructionSaveErrRetAddr { IrInstruction base; }; +struct IrInstructionAddImplicitReturnType { + IrInstruction base; + + IrInstruction *value; +}; + static const size_t slice_ptr_index = 0; static const size_t slice_len_index = 1; diff --git a/src/analyze.cpp b/src/analyze.cpp index eb1850c8e1..5f2162b2cc 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -3865,7 +3865,7 @@ void analyze_fn_ir(CodeGen *g, FnTableEntry *fn_table_entry, AstNode *return_typ TypeTableEntry *block_return_type = ir_analyze(g, &fn_table_entry->ir_executable, &fn_table_entry->analyzed_executable, fn_type_id->return_type, return_type_node); - fn_table_entry->implicit_return_type = block_return_type; + fn_table_entry->src_implicit_return_type = block_return_type; if (type_is_invalid(block_return_type) || fn_table_entry->analyzed_executable.invalid) { assert(g->errors.length > 0); @@ -3877,10 +3877,10 @@ void analyze_fn_ir(CodeGen *g, FnTableEntry *fn_table_entry, AstNode *return_typ TypeTableEntry *return_err_set_type = fn_type_id->return_type->data.error_union.err_set_type; if (return_err_set_type->data.error_set.infer_fn != nullptr) { TypeTableEntry *inferred_err_set_type; - if (fn_table_entry->implicit_return_type->id == TypeTableEntryIdErrorSet) { - inferred_err_set_type = fn_table_entry->implicit_return_type; - } else if (fn_table_entry->implicit_return_type->id == TypeTableEntryIdErrorUnion) { - inferred_err_set_type = fn_table_entry->implicit_return_type->data.error_union.err_set_type; + if (fn_table_entry->src_implicit_return_type->id == TypeTableEntryIdErrorSet) { + inferred_err_set_type = fn_table_entry->src_implicit_return_type; + } else if (fn_table_entry->src_implicit_return_type->id == TypeTableEntryIdErrorUnion) { + inferred_err_set_type = fn_table_entry->src_implicit_return_type->data.error_union.err_set_type; } else { add_node_error(g, return_type_node, buf_sprintf("function with inferred error set must return at least one possible error")); diff --git a/src/ast_render.cpp b/src/ast_render.cpp index f88feee856..432489c4d9 100644 --- a/src/ast_render.cpp +++ b/src/ast_render.cpp @@ -658,6 +658,15 @@ static void render_node_extra(AstRender *ar, AstNode *node, bool grouped) { if (node->data.fn_call_expr.is_builtin) { fprintf(ar->f, "@"); } + if (node->data.fn_call_expr.is_async) { + fprintf(ar->f, "async"); + if (node->data.fn_call_expr.async_allocator != nullptr) { + fprintf(ar->f, "<"); + render_node_extra(ar, node->data.fn_call_expr.async_allocator, true); + fprintf(ar->f, ">"); + } + fprintf(ar->f, " "); + } AstNode *fn_ref_node = node->data.fn_call_expr.fn_ref_expr; bool grouped = (fn_ref_node->type != NodeTypePrefixOpExpr && fn_ref_node->type != NodeTypeAddrOfExpr); render_node_extra(ar, fn_ref_node, grouped); @@ -1023,7 +1032,7 @@ static void render_node_extra(AstRender *ar, AstNode *node, bool grouped) { case NodeTypeUnwrapErrorExpr: { render_node_ungrouped(ar, node->data.unwrap_err_expr.op1); - fprintf(ar->f, " %%%% "); + fprintf(ar->f, " catch "); if (node->data.unwrap_err_expr.symbol) { Buf *var_name = node->data.unwrap_err_expr.symbol->data.symbol_expr.symbol; fprintf(ar->f, "|%s| ", buf_ptr(var_name)); diff --git a/src/codegen.cpp b/src/codegen.cpp index eddadb94ea..d675010531 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4245,6 +4245,7 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, case IrInstructionIdErrorUnion: case IrInstructionIdPromiseResultType: case IrInstructionIdAwaitBookkeeping: + case IrInstructionIdAddImplicitReturnType: zig_unreachable(); case IrInstructionIdReturn: diff --git a/src/ir.cpp b/src/ir.cpp index a78be66dd6..d896153d0b 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -34,7 +34,7 @@ struct IrAnalyze { size_t old_bb_index; size_t instruction_index; TypeTableEntry *explicit_return_type; - ZigList implicit_return_type_list; + ZigList src_implicit_return_type_list; IrBasicBlock *const_predecessor_bb; }; @@ -717,6 +717,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionSaveErrRetAddr * return IrInstructionIdSaveErrRetAddr; } +static constexpr IrInstructionId ir_instruction_id(IrInstructionAddImplicitReturnType *) { + return IrInstructionIdAddImplicitReturnType; +} + template static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) { T *special_instruction = allocate(1); @@ -2687,6 +2691,17 @@ static IrInstruction *ir_build_save_err_ret_addr(IrBuilder *irb, Scope *scope, A return &instruction->base; } +static IrInstruction *ir_build_add_implicit_return_type(IrBuilder *irb, Scope *scope, AstNode *source_node, + IrInstruction *value) +{ + IrInstructionAddImplicitReturnType *instruction = ir_build_instruction(irb, scope, source_node); + instruction->value = value; + + ir_ref_instruction(value, irb->current_basic_block); + + return &instruction->base; +} + static void ir_count_defers(IrBuilder *irb, Scope *inner_scope, Scope *outer_scope, size_t *results) { results[ReturnKindUnconditional] = 0; results[ReturnKindError] = 0; @@ -2767,6 +2782,8 @@ static bool exec_is_async(IrExecutable *exec) { static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode *node, IrInstruction *return_value, bool is_generated_code) { + ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, return_value)); + bool is_async = exec_is_async(irb->exec); if (!is_async) { IrInstruction *return_inst = ir_build_return(irb, scope, node, return_value); @@ -6399,6 +6416,8 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec ir_build_cond_br(irb, scope, node, alloc_result_is_ok, alloc_ok_block, alloc_err_block, const_bool_false); ir_set_cursor_at_end_and_append_block(irb, alloc_err_block); + // we can return undefined here, because the caller passes a pointer to the error struct field + // in the error union result, and we populate it in case of allocation failure. IrInstruction *undef = ir_build_const_undefined(irb, scope, node); ir_build_return(irb, scope, node, undef); @@ -10108,13 +10127,26 @@ static Buf *ir_resolve_str(IrAnalyze *ira, IrInstruction *value) { return result; } +static TypeTableEntry *ir_analyze_instruction_add_implicit_return_type(IrAnalyze *ira, + IrInstructionAddImplicitReturnType *instruction) +{ + IrInstruction *value = instruction->value->other; + if (type_is_invalid(value->value.type)) + return ir_unreach_error(ira); + + ira->src_implicit_return_type_list.append(value); + + ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); + out_val->type = ira->codegen->builtin_types.entry_void; + return out_val->type; +} + static TypeTableEntry *ir_analyze_instruction_return(IrAnalyze *ira, IrInstructionReturn *return_instruction) { IrInstruction *value = return_instruction->value->other; if (type_is_invalid(value->value.type)) return ir_unreach_error(ira); - ira->implicit_return_type_list.append(value); IrInstruction *casted_value = ir_implicit_cast(ira, value, ira->explicit_return_type); if (casted_value == ira->codegen->invalid_instruction) @@ -18049,6 +18081,8 @@ static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstructi return ir_analyze_instruction_await_bookkeeping(ira, (IrInstructionAwaitBookkeeping *)instruction); case IrInstructionIdSaveErrRetAddr: return ir_analyze_instruction_save_err_ret_addr(ira, (IrInstructionSaveErrRetAddr *)instruction); + case IrInstructionIdAddImplicitReturnType: + return ir_analyze_instruction_add_implicit_return_type(ira, (IrInstructionAddImplicitReturnType *)instruction); } zig_unreachable(); } @@ -18122,11 +18156,11 @@ TypeTableEntry *ir_analyze(CodeGen *codegen, IrExecutable *old_exec, IrExecutabl if (new_exec->invalid) { return ira->codegen->builtin_types.entry_invalid; - } else if (ira->implicit_return_type_list.length == 0) { + } else if (ira->src_implicit_return_type_list.length == 0) { return codegen->builtin_types.entry_unreachable; } else { - return ir_resolve_peer_types(ira, expected_type_source_node, ira->implicit_return_type_list.items, - ira->implicit_return_type_list.length); + return ir_resolve_peer_types(ira, expected_type_source_node, ira->src_implicit_return_type_list.items, + ira->src_implicit_return_type_list.length); } } @@ -18175,6 +18209,7 @@ bool ir_has_side_effects(IrInstruction *instruction) { case IrInstructionIdCoroAllocHelper: case IrInstructionIdAwaitBookkeeping: case IrInstructionIdSaveErrRetAddr: + case IrInstructionIdAddImplicitReturnType: return true; case IrInstructionIdPhi: diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 709cb43942..5008d3564d 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -201,9 +201,9 @@ static void ir_print_call(IrPrint *irp, IrInstructionCall *call_instruction) { if (call_instruction->is_async) { fprintf(irp->f, "async"); if (call_instruction->async_allocator != nullptr) { - fprintf(irp->f, "("); + fprintf(irp->f, "<"); ir_print_other_instruction(irp, call_instruction->async_allocator); - fprintf(irp->f, ")"); + fprintf(irp->f, ">"); } fprintf(irp->f, " "); } @@ -1165,6 +1165,12 @@ static void ir_print_save_err_ret_addr(IrPrint *irp, IrInstructionSaveErrRetAddr fprintf(irp->f, "@saveErrRetAddr()"); } +static void ir_print_add_implicit_return_type(IrPrint *irp, IrInstructionAddImplicitReturnType *instruction) { + fprintf(irp->f, "@addImplicitReturnType("); + ir_print_other_instruction(irp, instruction->value); + fprintf(irp->f, ")"); +} + static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { ir_print_prefix(irp, instruction); switch (instruction->id) { @@ -1539,6 +1545,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { case IrInstructionIdSaveErrRetAddr: ir_print_save_err_ret_addr(irp, (IrInstructionSaveErrRetAddr *)instruction); break; + case IrInstructionIdAddImplicitReturnType: + ir_print_add_implicit_return_type(irp, (IrInstructionAddImplicitReturnType *)instruction); + break; } fprintf(irp->f, "\n"); } diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index f8ad58f70d..087d561713 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -176,3 +176,14 @@ async<&std.mem.Allocator> fn simpleAsyncFn2(y: &i32) void { *y += 1; suspend; } + +test "async fn with inferred error set" { + const p = (async failing()) catch unreachable; + resume p; + cancel p; +} + +async fn failing() !void { + suspend; + return error.Fail; +} From 897e783763d60449ad1b9514cb5ba86a38f7ae4a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 24 Mar 2018 19:25:53 -0400 Subject: [PATCH 07/47] add promise->T syntax parsing closes #857 --- doc/langref.html.in | 4 ++- src/all_types.hpp | 13 ++++++++++ src/analyze.cpp | 1 + src/ast_render.cpp | 11 ++++++++ src/codegen.cpp | 1 + src/ir.cpp | 54 +++++++++++++++++++++++++++++++++++++++ src/ir_print.cpp | 11 ++++++++ src/parser.cpp | 14 +++++++++- src/tokenizer.cpp | 2 ++ src/tokenizer.hpp | 1 + test/cases/coroutines.zig | 1 + 11 files changed, 111 insertions(+), 2 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index f4aa42eb89..b2e14ac195 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -5863,7 +5863,9 @@ StructLiteralField = "." Symbol "=" Expression PrefixOp = "!" | "-" | "~" | "*" | ("&" option("align" "(" Expression option(":" Integer ":" Integer) ")" ) option("const") option("volatile")) | "?" | "??" | "-%" | "try" | "await" -PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl +PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType + +PromiseType = "promise" option("->" TypeExpr) ArrayType : "[" option(Expression) "]" option("align" "(" Expression option(":" Integer ":" Integer) ")")) option("const") option("volatile") TypeExpr diff --git a/src/all_types.hpp b/src/all_types.hpp index f5afecfbdd..64b9b31662 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -409,6 +409,7 @@ enum NodeType { NodeTypeResume, NodeTypeAwaitExpr, NodeTypeSuspend, + NodeTypePromiseType, }; struct AstNodeRoot { @@ -879,6 +880,10 @@ struct AstNodeSuspend { AstNode *promise_symbol; }; +struct AstNodePromiseType { + AstNode *payload_type; // can be NULL +}; + struct AstNode { enum NodeType type; size_t line; @@ -939,6 +944,7 @@ struct AstNode { AstNodeResumeExpr resume_expr; AstNodeAwaitExpr await_expr; AstNodeSuspend suspend; + AstNodePromiseType promise_type; } data; }; @@ -1947,6 +1953,7 @@ enum IrInstructionId { IrInstructionIdSetRuntimeSafety, IrInstructionIdSetFloatMode, IrInstructionIdArrayType, + IrInstructionIdPromiseType, IrInstructionIdSliceType, IrInstructionIdAsm, IrInstructionIdSizeOf, @@ -2365,6 +2372,12 @@ struct IrInstructionArrayType { IrInstruction *child_type; }; +struct IrInstructionPromiseType { + IrInstruction base; + + IrInstruction *payload_type; +}; + struct IrInstructionSliceType { IrInstruction base; diff --git a/src/analyze.cpp b/src/analyze.cpp index 5f2162b2cc..0f4728f822 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -3254,6 +3254,7 @@ void scan_decls(CodeGen *g, ScopeDecls *decls_scope, AstNode *node) { case NodeTypeResume: case NodeTypeAwaitExpr: case NodeTypeSuspend: + case NodeTypePromiseType: zig_unreachable(); } } diff --git a/src/ast_render.cpp b/src/ast_render.cpp index 432489c4d9..7b5fc03ea8 100644 --- a/src/ast_render.cpp +++ b/src/ast_render.cpp @@ -250,6 +250,8 @@ static const char *node_type_str(NodeType node_type) { return "AwaitExpr"; case NodeTypeSuspend: return "Suspend"; + case NodeTypePromiseType: + return "PromiseType"; } zig_unreachable(); } @@ -781,6 +783,15 @@ static void render_node_extra(AstRender *ar, AstNode *node, bool grouped) { render_node_ungrouped(ar, node->data.array_type.child_type); break; } + case NodeTypePromiseType: + { + fprintf(ar->f, "promise"); + if (node->data.promise_type.payload_type != nullptr) { + fprintf(ar->f, "->"); + render_node_grouped(ar, node->data.promise_type.payload_type); + } + break; + } case NodeTypeErrorType: fprintf(ar->f, "error"); break; diff --git a/src/codegen.cpp b/src/codegen.cpp index d675010531..786bd03985 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4205,6 +4205,7 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, case IrInstructionIdSetRuntimeSafety: case IrInstructionIdSetFloatMode: case IrInstructionIdArrayType: + case IrInstructionIdPromiseType: case IrInstructionIdSliceType: case IrInstructionIdSizeOf: case IrInstructionIdSwitchTarget: diff --git a/src/ir.cpp b/src/ir.cpp index d896153d0b..95142c88de 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -349,6 +349,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionArrayType *) { return IrInstructionIdArrayType; } +static constexpr IrInstructionId ir_instruction_id(IrInstructionPromiseType *) { + return IrInstructionIdPromiseType; +} + static constexpr IrInstructionId ir_instruction_id(IrInstructionSliceType *) { return IrInstructionIdSliceType; } @@ -1469,6 +1473,17 @@ static IrInstruction *ir_build_array_type(IrBuilder *irb, Scope *scope, AstNode return &instruction->base; } +static IrInstruction *ir_build_promise_type(IrBuilder *irb, Scope *scope, AstNode *source_node, + IrInstruction *payload_type) +{ + IrInstructionPromiseType *instruction = ir_build_instruction(irb, scope, source_node); + instruction->payload_type = payload_type; + + if (payload_type != nullptr) ir_ref_instruction(payload_type, irb->current_basic_block); + + return &instruction->base; +} + static IrInstruction *ir_build_slice_type(IrBuilder *irb, Scope *scope, AstNode *source_node, IrInstruction *child_type, bool is_const, bool is_volatile, IrInstruction *align_value) { @@ -5074,6 +5089,22 @@ static IrInstruction *ir_gen_array_type(IrBuilder *irb, Scope *scope, AstNode *n } } +static IrInstruction *ir_gen_promise_type(IrBuilder *irb, Scope *scope, AstNode *node) { + assert(node->type == NodeTypePromiseType); + + AstNode *payload_type_node = node->data.promise_type.payload_type; + IrInstruction *payload_type_value = nullptr; + + if (payload_type_node != nullptr) { + payload_type_value = ir_gen_node(irb, payload_type_node, scope); + if (payload_type_value == irb->codegen->invalid_instruction) + return payload_type_value; + + } + + return ir_build_promise_type(irb, scope, node, payload_type_value); +} + static IrInstruction *ir_gen_undefined_literal(IrBuilder *irb, Scope *scope, AstNode *node) { assert(node->type == NodeTypeUndefinedLiteral); return ir_build_const_undefined(irb, scope, node); @@ -6282,6 +6313,8 @@ static IrInstruction *ir_gen_node_raw(IrBuilder *irb, AstNode *node, Scope *scop return ir_lval_wrap(irb, scope, ir_gen_bool_literal(irb, scope, node), lval); case NodeTypeArrayType: return ir_lval_wrap(irb, scope, ir_gen_array_type(irb, scope, node), lval); + case NodeTypePromiseType: + return ir_lval_wrap(irb, scope, ir_gen_promise_type(irb, scope, node), lval); case NodeTypeStringLiteral: return ir_lval_wrap(irb, scope, ir_gen_string_literal(irb, scope, node), lval); case NodeTypeUndefinedLiteral: @@ -14069,6 +14102,24 @@ static TypeTableEntry *ir_analyze_instruction_array_type(IrAnalyze *ira, zig_unreachable(); } +static TypeTableEntry *ir_analyze_instruction_promise_type(IrAnalyze *ira, IrInstructionPromiseType *instruction) { + TypeTableEntry *promise_type; + + if (instruction->payload_type == nullptr) { + promise_type = ira->codegen->builtin_types.entry_promise; + } else { + TypeTableEntry *payload_type = ir_resolve_type(ira, instruction->payload_type->other); + if (type_is_invalid(payload_type)) + return ira->codegen->builtin_types.entry_invalid; + + promise_type = get_promise_type(ira->codegen, payload_type); + } + + ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); + out_val->data.x_type = promise_type; + return ira->codegen->builtin_types.entry_type; +} + static TypeTableEntry *ir_analyze_instruction_size_of(IrAnalyze *ira, IrInstructionSizeOf *size_of_instruction) { @@ -17907,6 +17958,8 @@ static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstructi return ir_analyze_instruction_asm(ira, (IrInstructionAsm *)instruction); case IrInstructionIdArrayType: return ir_analyze_instruction_array_type(ira, (IrInstructionArrayType *)instruction); + case IrInstructionIdPromiseType: + return ir_analyze_instruction_promise_type(ira, (IrInstructionPromiseType *)instruction); case IrInstructionIdSizeOf: return ir_analyze_instruction_size_of(ira, (IrInstructionSizeOf *)instruction); case IrInstructionIdTestNonNull: @@ -18232,6 +18285,7 @@ bool ir_has_side_effects(IrInstruction *instruction) { case IrInstructionIdStructFieldPtr: case IrInstructionIdUnionFieldPtr: case IrInstructionIdArrayType: + case IrInstructionIdPromiseType: case IrInstructionIdSliceType: case IrInstructionIdSizeOf: case IrInstructionIdTestNonNull: diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 5008d3564d..b14d49a4ca 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -404,6 +404,14 @@ static void ir_print_array_type(IrPrint *irp, IrInstructionArrayType *instructio ir_print_other_instruction(irp, instruction->child_type); } +static void ir_print_promise_type(IrPrint *irp, IrInstructionPromiseType *instruction) { + fprintf(irp->f, "promise"); + if (instruction->payload_type != nullptr) { + fprintf(irp->f, "->"); + ir_print_other_instruction(irp, instruction->payload_type); + } +} + static void ir_print_slice_type(IrPrint *irp, IrInstructionSliceType *instruction) { const char *const_kw = instruction->is_const ? "const " : ""; fprintf(irp->f, "[]%s", const_kw); @@ -1263,6 +1271,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { case IrInstructionIdArrayType: ir_print_array_type(irp, (IrInstructionArrayType *)instruction); break; + case IrInstructionIdPromiseType: + ir_print_promise_type(irp, (IrInstructionPromiseType *)instruction); + break; case IrInstructionIdSliceType: ir_print_slice_type(irp, (IrInstructionSliceType *)instruction); break; diff --git a/src/parser.cpp b/src/parser.cpp index 666b9da3c3..d6faf4c984 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -705,7 +705,7 @@ static AstNode *ast_parse_comptime_expr(ParseContext *pc, size_t *token_index, b } /* -PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl +PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType KeywordLiteral = "true" | "false" | "null" | "undefined" | "error" | "this" | "unreachable" | "suspend" ErrorSetDecl = "error" "{" list(Symbol, ",") "}" */ @@ -774,6 +774,15 @@ static AstNode *ast_parse_primary_expr(ParseContext *pc, size_t *token_index, bo AstNode *node = ast_create_node(pc, NodeTypeSuspend, token); *token_index += 1; return node; + } else if (token->id == TokenIdKeywordPromise) { + AstNode *node = ast_create_node(pc, NodeTypePromiseType, token); + *token_index += 1; + Token *arrow_tok = &pc->tokens->at(*token_index); + if (arrow_tok->id == TokenIdArrow) { + *token_index += 1; + node->data.promise_type.payload_type = ast_parse_type_expr(pc, token_index, true); + } + return node; } else if (token->id == TokenIdKeywordError) { Token *next_token = &pc->tokens->at(*token_index + 1); if (next_token->id == TokenIdLBrace) { @@ -3081,6 +3090,9 @@ void ast_visit_node_children(AstNode *node, void (*visit)(AstNode **, void *cont visit_field(&node->data.array_type.child_type, visit, context); visit_field(&node->data.array_type.align_expr, visit, context); break; + case NodeTypePromiseType: + visit_field(&node->data.promise_type.payload_type, visit, context); + break; case NodeTypeErrorType: // none break; diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index dc17829c0f..365b35cdfd 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -135,6 +135,7 @@ static const struct ZigKeyword zig_keywords[] = { {"null", TokenIdKeywordNull}, {"or", TokenIdKeywordOr}, {"packed", TokenIdKeywordPacked}, + {"promise", TokenIdKeywordPromise}, {"pub", TokenIdKeywordPub}, {"resume", TokenIdKeywordResume}, {"return", TokenIdKeywordReturn}, @@ -1558,6 +1559,7 @@ const char * token_name(TokenId id) { case TokenIdKeywordNull: return "null"; case TokenIdKeywordOr: return "or"; case TokenIdKeywordPacked: return "packed"; + case TokenIdKeywordPromise: return "promise"; case TokenIdKeywordPub: return "pub"; case TokenIdKeywordReturn: return "return"; case TokenIdKeywordSection: return "section"; diff --git a/src/tokenizer.hpp b/src/tokenizer.hpp index 2d71427997..b719293704 100644 --- a/src/tokenizer.hpp +++ b/src/tokenizer.hpp @@ -76,6 +76,7 @@ enum TokenId { TokenIdKeywordNull, TokenIdKeywordOr, TokenIdKeywordPacked, + TokenIdKeywordPromise, TokenIdKeywordPub, TokenIdKeywordResume, TokenIdKeywordReturn, diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 087d561713..25a75dca5c 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -5,6 +5,7 @@ var x: i32 = 1; test "create a coroutine and cancel it" { const p = try async simpleAsyncFn(); + comptime assert(@typeOf(p) == promise->void); cancel p; assert(x == 2); } From a43c7af3d1e41ccee73678f114bb3844febcaad6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 24 Mar 2018 19:31:00 -0400 Subject: [PATCH 08/47] add comptime test for the type of suspend promise --- test/cases/coroutines.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 25a75dca5c..922c1a7e58 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -56,6 +56,7 @@ var result = false; async fn testSuspendBlock() void { suspend |p| { + comptime assert(@typeOf(p) == promise->void); a_promise = p; } result = true; From aa2995ee395b2c1329a61513debcac6225fcb8a8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 24 Mar 2018 22:05:29 -0400 Subject: [PATCH 09/47] fix invalid codegen for error return traces across suspend points See #821 Now the code works correctly, but error return traces are missing the frames from coroutines. --- src/all_types.hpp | 10 +++++- src/analyze.cpp | 7 +++++ src/analyze.hpp | 1 + src/codegen.cpp | 65 +++++++++++++++++++++++++++++--------- src/ir.cpp | 69 +++++++++++++++++++++-------------------- test/runtime_safety.zig | 30 ++++++++++++++++++ 6 files changed, 132 insertions(+), 50 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index 64b9b31662..6951230aa4 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1621,7 +1621,8 @@ struct CodeGen { FnTableEntry *panic_fn; LLVMValueRef cur_ret_ptr; LLVMValueRef cur_fn_val; - LLVMValueRef cur_err_ret_trace_val; + LLVMValueRef cur_err_ret_trace_val_arg; + LLVMValueRef cur_err_ret_trace_val_stack; bool c_want_stdint; bool c_want_stdbool; AstNode *root_export_decl; @@ -1760,6 +1761,7 @@ enum ScopeId { ScopeIdLoop, ScopeIdFnDef, ScopeIdCompTime, + ScopeIdCoroPrelude, }; struct Scope { @@ -1867,6 +1869,12 @@ struct ScopeFnDef { FnTableEntry *fn_entry; }; +// This scope is created to indicate that the code in the scope +// is auto-generated coroutine prelude stuff. +struct ScopeCoroPrelude { + Scope base; +}; + // synchronized with code in define_builtin_compile_vars enum AtomicOrder { AtomicOrderUnordered, diff --git a/src/analyze.cpp b/src/analyze.cpp index 0f4728f822..7ee1de78a2 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -170,6 +170,12 @@ Scope *create_comptime_scope(AstNode *node, Scope *parent) { return &scope->base; } +Scope *create_coro_prelude_scope(AstNode *node, Scope *parent) { + ScopeCoroPrelude *scope = allocate(1); + init_scope(&scope->base, ScopeIdCoroPrelude, node, parent); + return &scope->base; +} + ImportTableEntry *get_scope_import(Scope *scope) { while (scope) { if (scope->id == ScopeIdDecls) { @@ -3592,6 +3598,7 @@ FnTableEntry *scope_get_fn_if_root(Scope *scope) { case ScopeIdCImport: case ScopeIdLoop: case ScopeIdCompTime: + case ScopeIdCoroPrelude: scope = scope->parent; continue; case ScopeIdFnDef: diff --git a/src/analyze.hpp b/src/analyze.hpp index 936134030d..aa4557666b 100644 --- a/src/analyze.hpp +++ b/src/analyze.hpp @@ -107,6 +107,7 @@ ScopeLoop *create_loop_scope(AstNode *node, Scope *parent); ScopeFnDef *create_fndef_scope(AstNode *node, Scope *parent, FnTableEntry *fn_entry); ScopeDecls *create_decls_scope(AstNode *node, Scope *parent, TypeTableEntry *container_type, ImportTableEntry *import); Scope *create_comptime_scope(AstNode *node, Scope *parent); +Scope *create_coro_prelude_scope(AstNode *node, Scope *parent); void init_const_str_lit(CodeGen *g, ConstExprValue *const_val, Buf *str); ConstExprValue *create_const_str_lit(CodeGen *g, Buf *str); diff --git a/src/codegen.cpp b/src/codegen.cpp index 786bd03985..25b2ffbf16 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -653,6 +653,7 @@ static ZigLLVMDIScope *get_di_scope(CodeGen *g, Scope *scope) { case ScopeIdDeferExpr: case ScopeIdLoop: case ScopeIdCompTime: + case ScopeIdCoroPrelude: return get_di_scope(g, scope->parent); } zig_unreachable(); @@ -1318,9 +1319,34 @@ static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) { return fn_val; } -static void gen_safety_crash_for_err(CodeGen *g, LLVMValueRef err_val) { +static bool is_coro_prelude_scope(Scope *scope) { + while (scope != nullptr) { + if (scope->id == ScopeIdCoroPrelude) { + return true; + } else if (scope->id == ScopeIdFnDef) { + break; + } + scope = scope->parent; + } + return false; +} + +static LLVMValueRef get_cur_err_ret_trace_val(CodeGen *g, Scope *scope) { + if (!g->have_err_ret_tracing) { + return nullptr; + } + if (g->cur_fn->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync) { + return is_coro_prelude_scope(scope) ? g->cur_err_ret_trace_val_arg : g->cur_err_ret_trace_val_stack; + } + if (g->cur_err_ret_trace_val_stack != nullptr) { + return g->cur_err_ret_trace_val_stack; + } + return g->cur_err_ret_trace_val_arg; +} + +static void gen_safety_crash_for_err(CodeGen *g, LLVMValueRef err_val, Scope *scope) { LLVMValueRef safety_crash_err_fn = get_safety_crash_err_fn(g); - LLVMValueRef err_ret_trace_val = g->cur_err_ret_trace_val; + LLVMValueRef err_ret_trace_val = get_cur_err_ret_trace_val(g, scope); if (err_ret_trace_val == nullptr) { TypeTableEntry *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(g); err_ret_trace_val = LLVMConstNull(ptr_to_stack_trace_type->type_ref); @@ -1614,7 +1640,7 @@ static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, IrExecutable *execut LLVMValueRef return_err_fn = get_return_err_fn(g); LLVMValueRef args[] = { - g->cur_err_ret_trace_val, + get_cur_err_ret_trace_val(g, save_err_ret_addr_instruction->base.scope), }; LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, return_err_fn, args, 1, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); @@ -2725,7 +2751,7 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr gen_param_index += 1; } if (prefix_arg_err_ret_stack) { - gen_param_values[gen_param_index] = g->cur_err_ret_trace_val; + gen_param_values[gen_param_index] = get_cur_err_ret_trace_val(g, instruction->base.scope); gen_param_index += 1; } if (instruction->is_async) { @@ -3292,11 +3318,12 @@ static LLVMValueRef ir_render_align_cast(CodeGen *g, IrExecutable *executable, I static LLVMValueRef ir_render_error_return_trace(CodeGen *g, IrExecutable *executable, IrInstructionErrorReturnTrace *instruction) { - if (g->cur_err_ret_trace_val == nullptr) { + LLVMValueRef cur_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope); + if (cur_err_ret_trace_val == nullptr) { TypeTableEntry *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(g); return LLVMConstNull(ptr_to_stack_trace_type->type_ref); } - return g->cur_err_ret_trace_val; + return cur_err_ret_trace_val; } static LLVMValueRef ir_render_cancel(CodeGen *g, IrExecutable *executable, IrInstructionCancel *instruction) { @@ -3726,7 +3753,7 @@ static LLVMValueRef ir_render_unwrap_err_payload(CodeGen *g, IrExecutable *execu LLVMBuildCondBr(g->builder, cond_val, ok_block, err_block); LLVMPositionBuilderAtEnd(g->builder, err_block); - gen_safety_crash_for_err(g, err_val); + gen_safety_crash_for_err(g, err_val, instruction->base.scope); LLVMPositionBuilderAtEnd(g->builder, ok_block); } @@ -3918,7 +3945,7 @@ static LLVMValueRef ir_render_container_init_list(CodeGen *g, IrExecutable *exec } static LLVMValueRef ir_render_panic(CodeGen *g, IrExecutable *executable, IrInstructionPanic *instruction) { - gen_panic(g, ir_llvm_value(g, instruction->msg), g->cur_err_ret_trace_val); + gen_panic(g, ir_llvm_value(g, instruction->msg), get_cur_err_ret_trace_val(g, instruction->base.scope)); return nullptr; } @@ -5279,9 +5306,17 @@ static void do_code_gen(CodeGen *g) { clear_debug_source_node(g); uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn_table_entry); - if (err_ret_trace_arg_index != UINT32_MAX) { - g->cur_err_ret_trace_val = LLVMGetParam(fn, err_ret_trace_arg_index); - } else if (g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn) { + bool have_err_ret_trace_arg = err_ret_trace_arg_index != UINT32_MAX; + if (have_err_ret_trace_arg) { + g->cur_err_ret_trace_val_arg = LLVMGetParam(fn, err_ret_trace_arg_index); + } else { + g->cur_err_ret_trace_val_arg = nullptr; + } + + bool is_async = fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync; + bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && + (is_async || !have_err_ret_trace_arg); + if (have_err_ret_trace_stack) { // TODO call graph analysis to find out what this number needs to be for every function static const size_t stack_trace_ptr_count = 30; @@ -5289,13 +5324,13 @@ static void do_code_gen(CodeGen *g) { TypeTableEntry *array_type = get_array_type(g, usize, stack_trace_ptr_count); LLVMValueRef err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type)); - g->cur_err_ret_trace_val = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type)); + g->cur_err_ret_trace_val_stack = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type)); size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; - LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val, (unsigned)index_field_index, ""); + LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)index_field_index, ""); gen_store_untyped(g, LLVMConstNull(usize->type_ref), index_field_ptr, 0, false); size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; - LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val, (unsigned)addresses_field_index, ""); + LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)addresses_field_index, ""); TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; @@ -5311,7 +5346,7 @@ static void do_code_gen(CodeGen *g) { LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); gen_store(g, LLVMConstInt(usize->type_ref, stack_trace_ptr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); } else { - g->cur_err_ret_trace_val = nullptr; + g->cur_err_ret_trace_val_stack = nullptr; } // allocate temporary stack data diff --git a/src/ir.cpp b/src/ir.cpp index 95142c88de..d86b7f5b21 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -6412,60 +6412,61 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec VariableTableEntry *coro_size_var; if (is_async) { // create the coro promise - const_bool_false = ir_build_const_bool(irb, scope, node, false); - VariableTableEntry *promise_var = ir_create_var(irb, node, scope, nullptr, false, false, true, const_bool_false); + Scope *coro_scope = create_coro_prelude_scope(node, scope); + const_bool_false = ir_build_const_bool(irb, coro_scope, node, false); + VariableTableEntry *promise_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false); return_type = fn_entry->type_entry->data.fn.fn_type_id.return_type; - IrInstruction *promise_init = ir_build_const_promise_init(irb, scope, node, return_type); - ir_build_var_decl(irb, scope, node, promise_var, nullptr, nullptr, promise_init); - IrInstruction *coro_promise_ptr = ir_build_var_ptr(irb, scope, node, promise_var, false, false); + IrInstruction *promise_init = ir_build_const_promise_init(irb, coro_scope, node, return_type); + ir_build_var_decl(irb, coro_scope, node, promise_var, nullptr, nullptr, promise_init); + IrInstruction *coro_promise_ptr = ir_build_var_ptr(irb, coro_scope, node, promise_var, false, false); - VariableTableEntry *await_handle_var = ir_create_var(irb, node, scope, nullptr, false, false, true, const_bool_false); - IrInstruction *null_value = ir_build_const_null(irb, scope, node); - IrInstruction *await_handle_type_val = ir_build_const_type(irb, scope, node, + VariableTableEntry *await_handle_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false); + IrInstruction *null_value = ir_build_const_null(irb, coro_scope, node); + IrInstruction *await_handle_type_val = ir_build_const_type(irb, coro_scope, node, get_maybe_type(irb->codegen, irb->codegen->builtin_types.entry_promise)); - ir_build_var_decl(irb, scope, node, await_handle_var, await_handle_type_val, nullptr, null_value); - irb->exec->await_handle_var_ptr = ir_build_var_ptr(irb, scope, node, + ir_build_var_decl(irb, coro_scope, node, await_handle_var, await_handle_type_val, nullptr, null_value); + irb->exec->await_handle_var_ptr = ir_build_var_ptr(irb, coro_scope, node, await_handle_var, false, false); - u8_ptr_type = ir_build_const_type(irb, scope, node, + u8_ptr_type = ir_build_const_type(irb, coro_scope, node, get_pointer_to_type(irb->codegen, irb->codegen->builtin_types.entry_u8, false)); - IrInstruction *promise_as_u8_ptr = ir_build_ptr_cast(irb, scope, node, u8_ptr_type, coro_promise_ptr); - coro_id = ir_build_coro_id(irb, scope, node, promise_as_u8_ptr); - coro_size_var = ir_create_var(irb, node, scope, nullptr, false, false, true, const_bool_false); - IrInstruction *coro_size = ir_build_coro_size(irb, scope, node); - ir_build_var_decl(irb, scope, node, coro_size_var, nullptr, nullptr, coro_size); - IrInstruction *implicit_allocator_ptr = ir_build_get_implicit_allocator(irb, scope, node, + IrInstruction *promise_as_u8_ptr = ir_build_ptr_cast(irb, coro_scope, node, u8_ptr_type, coro_promise_ptr); + coro_id = ir_build_coro_id(irb, coro_scope, node, promise_as_u8_ptr); + coro_size_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false); + IrInstruction *coro_size = ir_build_coro_size(irb, coro_scope, node); + ir_build_var_decl(irb, coro_scope, node, coro_size_var, nullptr, nullptr, coro_size); + IrInstruction *implicit_allocator_ptr = ir_build_get_implicit_allocator(irb, coro_scope, node, ImplicitAllocatorIdArg); - irb->exec->coro_allocator_var = ir_create_var(irb, node, scope, nullptr, true, true, true, const_bool_false); - ir_build_var_decl(irb, scope, node, irb->exec->coro_allocator_var, nullptr, nullptr, implicit_allocator_ptr); + irb->exec->coro_allocator_var = ir_create_var(irb, node, coro_scope, nullptr, true, true, true, const_bool_false); + ir_build_var_decl(irb, coro_scope, node, irb->exec->coro_allocator_var, nullptr, nullptr, implicit_allocator_ptr); Buf *alloc_field_name = buf_create_from_str(ASYNC_ALLOC_FIELD_NAME); - IrInstruction *alloc_fn_ptr = ir_build_field_ptr(irb, scope, node, implicit_allocator_ptr, alloc_field_name); - IrInstruction *alloc_fn = ir_build_load_ptr(irb, scope, node, alloc_fn_ptr); - IrInstruction *maybe_coro_mem_ptr = ir_build_coro_alloc_helper(irb, scope, node, alloc_fn, coro_size); - IrInstruction *alloc_result_is_ok = ir_build_test_nonnull(irb, scope, node, maybe_coro_mem_ptr); - IrBasicBlock *alloc_err_block = ir_create_basic_block(irb, scope, "AllocError"); - IrBasicBlock *alloc_ok_block = ir_create_basic_block(irb, scope, "AllocOk"); - ir_build_cond_br(irb, scope, node, alloc_result_is_ok, alloc_ok_block, alloc_err_block, const_bool_false); + IrInstruction *alloc_fn_ptr = ir_build_field_ptr(irb, coro_scope, node, implicit_allocator_ptr, alloc_field_name); + IrInstruction *alloc_fn = ir_build_load_ptr(irb, coro_scope, node, alloc_fn_ptr); + IrInstruction *maybe_coro_mem_ptr = ir_build_coro_alloc_helper(irb, coro_scope, node, alloc_fn, coro_size); + IrInstruction *alloc_result_is_ok = ir_build_test_nonnull(irb, coro_scope, node, maybe_coro_mem_ptr); + IrBasicBlock *alloc_err_block = ir_create_basic_block(irb, coro_scope, "AllocError"); + IrBasicBlock *alloc_ok_block = ir_create_basic_block(irb, coro_scope, "AllocOk"); + ir_build_cond_br(irb, coro_scope, node, alloc_result_is_ok, alloc_ok_block, alloc_err_block, const_bool_false); ir_set_cursor_at_end_and_append_block(irb, alloc_err_block); // we can return undefined here, because the caller passes a pointer to the error struct field // in the error union result, and we populate it in case of allocation failure. - IrInstruction *undef = ir_build_const_undefined(irb, scope, node); - ir_build_return(irb, scope, node, undef); + IrInstruction *undef = ir_build_const_undefined(irb, coro_scope, node); + ir_build_return(irb, coro_scope, node, undef); ir_set_cursor_at_end_and_append_block(irb, alloc_ok_block); - IrInstruction *coro_mem_ptr = ir_build_ptr_cast(irb, scope, node, u8_ptr_type, maybe_coro_mem_ptr); - irb->exec->coro_handle = ir_build_coro_begin(irb, scope, node, coro_id, coro_mem_ptr); + IrInstruction *coro_mem_ptr = ir_build_ptr_cast(irb, coro_scope, node, u8_ptr_type, maybe_coro_mem_ptr); + irb->exec->coro_handle = ir_build_coro_begin(irb, coro_scope, node, coro_id, coro_mem_ptr); Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME); - irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, + irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, coro_scope, node, coro_promise_ptr, awaiter_handle_field_name); Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); - irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name); + irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, coro_scope, node, coro_promise_ptr, result_field_name); result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME); - irb->exec->coro_result_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_ptr_field_name); - ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_ptr_field_ptr, irb->exec->coro_result_field_ptr); + irb->exec->coro_result_ptr_field_ptr = ir_build_field_ptr(irb, coro_scope, node, coro_promise_ptr, result_ptr_field_name); + ir_build_store_ptr(irb, coro_scope, node, irb->exec->coro_result_ptr_field_ptr, irb->exec->coro_result_field_ptr); irb->exec->coro_early_final = ir_create_basic_block(irb, scope, "CoroEarlyFinal"); diff --git a/test/runtime_safety.zig b/test/runtime_safety.zig index 8b8f612056..1fea6347ab 100644 --- a/test/runtime_safety.zig +++ b/test/runtime_safety.zig @@ -281,4 +281,34 @@ pub fn addCases(cases: &tests.CompareOutputContext) void { \\ f.float = 12.34; \\} ); + + // This case makes sure that the code compiles and runs. There is not actually a special + // runtime safety check having to do specifically with error return traces across suspend points. + cases.addRuntimeSafety("error return trace across suspend points", + \\const std = @import("std"); + \\ + \\pub fn panic(message: []const u8, stack_trace: ?&@import("builtin").StackTrace) noreturn { + \\ std.os.exit(126); + \\} + \\ + \\pub fn main() void { + \\ const p = nonFailing(); + \\ resume p; + \\ const p2 = async printTrace(p) catch unreachable; + \\ cancel p2; + \\} + \\ + \\fn nonFailing() promise->error!void { + \\ return async failing() catch unreachable; + \\} + \\ + \\async fn failing() error!void { + \\ suspend; + \\ return error.Fail; + \\} + \\ + \\async fn printTrace(p: promise->error!void) void { + \\ (await p) catch unreachable; + \\} + ); } From 3e836f5516c48bcfa16e0b630e53822af80d4b3b Mon Sep 17 00:00:00 2001 From: Jay Weisskopf Date: Sun, 25 Mar 2018 18:48:07 -0400 Subject: [PATCH 10/47] doc: fix typo and tighten wording in error sections Changes: - Removed superfluous "when possible" - Fixed typo in "documentationt" - Added missing comma - Moved definition of error union type up to first sentence --- doc/langref.html.in | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index b2e14ac195..04ca5314ca 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -2864,18 +2864,18 @@ const err = (error {FileNotFound}).FileNotFound; assert to make sure the error value is in fact in the destination error set.

- The global error set should generally be avoided when possible, because it prevents - the compiler from knowing what errors are possible at compile-time. Knowing - the error set at compile-time is better for generated documentationt and for - helpful error messages such as forgetting a possible error value in a {#link|switch#}. + The global error set should generally be avoided because it prevents the + compiler from knowing what errors are possible at compile-time. Knowing + the error set at compile-time is better for generated documentation and + helpful error messages, such as forgetting a possible error value in a {#link|switch#}.

{#header_close#} {#header_close#} {#header_open|Error Union Type#}

- Most of the time you will not find yourself using an error set type. Instead, - likely you will be using the error union type. This is when you take an error set - and a normal type, and create an error union with the ! binary operator. + An error set type and normal type can be combined with the ! + binary operator to form an error union type. You are likely to use an + error union type more often than an error set type by itself.

Here is a function to parse a string into a 64-bit integer: @@ -6033,4 +6033,3 @@ hljs.registerLanguage("zig", function(t) { - From 0b7b3190fd1121aa4e349740cff1faf213c94411 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 27 Mar 2018 10:43:51 -0400 Subject: [PATCH 11/47] fix bitrotted code in unexpected error tracing --- std/os/index.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/std/os/index.zig b/std/os/index.zig index e472908c68..09409d6d36 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -1690,7 +1690,7 @@ const unexpected_error_tracing = false; pub fn unexpectedErrorPosix(errno: usize) (error{Unexpected}) { if (unexpected_error_tracing) { debug.warn("unexpected errno: {}\n", errno); - debug.dumpStackTrace(); + debug.dumpCurrentStackTrace(null); } return error.Unexpected; } @@ -1700,7 +1700,7 @@ pub fn unexpectedErrorPosix(errno: usize) (error{Unexpected}) { pub fn unexpectedErrorWindows(err: windows.DWORD) (error{Unexpected}) { if (unexpected_error_tracing) { debug.warn("unexpected GetLastError(): {}\n", err); - debug.dumpStackTrace(); + debug.dumpCurrentStackTrace(null); } return error.Unexpected; } From 6cb99fdac3fb02d7e4e5a363295bae0dd3641a45 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 27 Mar 2018 15:07:45 -0400 Subject: [PATCH 12/47] fix crash when compile error in analyzing @panic call --- src/ir.cpp | 6 +++--- test/compile_errors.zig | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index d86b7f5b21..ea001adb93 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -16918,18 +16918,18 @@ static TypeTableEntry *ir_analyze_instruction_can_implicit_cast(IrAnalyze *ira, static TypeTableEntry *ir_analyze_instruction_panic(IrAnalyze *ira, IrInstructionPanic *instruction) { IrInstruction *msg = instruction->msg->other; if (type_is_invalid(msg->value.type)) - return ira->codegen->builtin_types.entry_invalid; + return ir_unreach_error(ira); if (ir_should_inline(ira->new_irb.exec, instruction->base.scope)) { ir_add_error(ira, &instruction->base, buf_sprintf("encountered @panic at compile-time")); - return ira->codegen->builtin_types.entry_invalid; + return ir_unreach_error(ira); } TypeTableEntry *u8_ptr_type = get_pointer_to_type(ira->codegen, ira->codegen->builtin_types.entry_u8, true); TypeTableEntry *str_type = get_slice_type(ira->codegen, u8_ptr_type); IrInstruction *casted_msg = ir_implicit_cast(ira, msg, str_type); if (type_is_invalid(casted_msg->value.type)) - return ira->codegen->builtin_types.entry_invalid; + return ir_unreach_error(ira); IrInstruction *new_instruction = ir_build_panic(&ira->new_irb, instruction->base.scope, instruction->base.source_node, casted_msg); diff --git a/test/compile_errors.zig b/test/compile_errors.zig index ddf5286335..bed5aa1b63 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -1,6 +1,15 @@ const tests = @import("tests.zig"); pub fn addCases(cases: &tests.CompileErrorContext) void { + cases.add("wrong type passed to @panic", + \\export fn entry() void { + \\ var e = error.Foo; + \\ @panic(e); + \\} + , + ".tmp_source.zig:3:12: error: expected type '[]const u8', found 'error{Foo}'"); + + cases.add("@tagName used on union with no associated enum tag", \\const FloatInt = extern union { \\ Float: f32, From 5b00dee0c27ae5a8547d6a4063331302cdc7b0be Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 27 Mar 2018 15:20:07 -0400 Subject: [PATCH 13/47] std.math.cast handles signed integers --- std/math/index.zig | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/std/math/index.zig b/std/math/index.zig index f8668cc00d..477dafcbcc 100644 --- a/std/math/index.zig +++ b/std/math/index.zig @@ -515,15 +515,28 @@ test "math.negateCast" { /// Cast an integer to a different integer type. If the value doesn't fit, /// return an error. -pub fn cast(comptime T: type, x: var) !T { +pub fn cast(comptime T: type, x: var) (error{Overflow}!T) { comptime assert(@typeId(T) == builtin.TypeId.Int); // must pass an integer - if (x > @maxValue(T)) { + comptime assert(@typeId(@typeOf(x)) == builtin.TypeId.Int); // must pass an integer + if (@maxValue(@typeOf(x)) > @maxValue(T) and x > @maxValue(T)) { + return error.Overflow; + } else if (@minValue(@typeOf(x)) < @minValue(T) and x < @minValue(T)) { return error.Overflow; } else { return T(x); } } +test "math.cast" { + if (cast(u8, u32(300))) |_| @panic("fail") else |err| assert(err == error.Overflow); + if (cast(i8, i32(-200))) |_| @panic("fail") else |err| assert(err == error.Overflow); + if (cast(u8, i8(-1))) |_| @panic("fail") else |err| assert(err == error.Overflow); + if (cast(u64, i8(-1))) |_| @panic("fail") else |err| assert(err == error.Overflow); + + assert((try cast(u8, u32(255))) == u8(255)); + assert(@typeOf(try cast(u8, u32(255))) == u8); +} + pub fn floorPowerOfTwo(comptime T: type, value: T) T { var x = value; From f5b43ada469c94ba4968898a7102d27f5c8188f2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 5 Mar 2018 00:57:02 -0500 Subject: [PATCH 14/47] std/os: getting dir entries works on OS X --- std/c/darwin.zig | 10 ++++ std/c/index.zig | 1 + std/os/darwin.zig | 32 +++++++++++++ std/os/index.zig | 104 +++++++++++++++++++++++++++++++++++----- std/os/linux/x86_64.zig | 8 ++++ std/os/test.zig | 12 +++++ 6 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 std/os/test.zig diff --git a/std/c/darwin.zig b/std/c/darwin.zig index f0890d4ec0..aa49dfa3df 100644 --- a/std/c/darwin.zig +++ b/std/c/darwin.zig @@ -1,6 +1,7 @@ extern "c" fn __error() &c_int; pub extern "c" fn _NSGetExecutablePath(buf: &u8, bufsize: &u32) c_int; +pub extern "c" fn __getdirentries64(fd: c_int, buf_ptr: &u8, buf_len: usize, basep: &i64) usize; pub use @import("../os/darwin_errno.zig"); @@ -45,3 +46,12 @@ pub const Sigaction = extern struct { sa_mask: sigset_t, sa_flags: c_int, }; + +pub const dirent = extern struct { + d_ino: usize, + d_seekoff: usize, + d_reclen: u16, + d_namlen: u16, + d_type: u8, + d_name: u8, // field address is address of first byte of name +}; diff --git a/std/c/index.zig b/std/c/index.zig index ce9f3c473a..369ea2b358 100644 --- a/std/c/index.zig +++ b/std/c/index.zig @@ -44,6 +44,7 @@ pub extern "c" fn sigaction(sig: c_int, noalias act: &const Sigaction, noalias o pub extern "c" fn nanosleep(rqtp: &const timespec, rmtp: ?×pec) c_int; pub extern "c" fn setreuid(ruid: c_uint, euid: c_uint) c_int; pub extern "c" fn setregid(rgid: c_uint, egid: c_uint) c_int; +pub extern "c" fn rmdir(path: &const u8) c_int; pub extern "c" fn aligned_alloc(alignment: usize, size: usize) ?&c_void; pub extern "c" fn malloc(usize) ?&c_void; diff --git a/std/os/darwin.zig b/std/os/darwin.zig index ebc6f65f55..f8b1fbed3b 100644 --- a/std/os/darwin.zig +++ b/std/os/darwin.zig @@ -56,10 +56,32 @@ pub const O_SYMLINK = 0x200000; /// allow open of symlinks pub const O_EVTONLY = 0x8000; /// descriptor requested for event notifications only pub const O_CLOEXEC = 0x1000000; /// mark as close-on-exec +pub const O_ACCMODE = 3; +pub const O_ALERT = 536870912; +pub const O_ASYNC = 64; +pub const O_DIRECTORY = 1048576; +pub const O_DP_GETRAWENCRYPTED = 1; +pub const O_DP_GETRAWUNENCRYPTED = 2; +pub const O_DSYNC = 4194304; +pub const O_FSYNC = O_SYNC; +pub const O_NOCTTY = 131072; +pub const O_POPUP = 2147483648; +pub const O_SYNC = 128; + pub const SEEK_SET = 0x0; pub const SEEK_CUR = 0x1; pub const SEEK_END = 0x2; +pub const DT_UNKNOWN = 0; +pub const DT_FIFO = 1; +pub const DT_CHR = 2; +pub const DT_DIR = 4; +pub const DT_BLK = 6; +pub const DT_REG = 8; +pub const DT_LNK = 10; +pub const DT_SOCK = 12; +pub const DT_WHT = 14; + pub const SIG_BLOCK = 1; /// block specified signal set pub const SIG_UNBLOCK = 2; /// unblock specified signal set pub const SIG_SETMASK = 3; /// set specified signal set @@ -192,6 +214,11 @@ pub fn pipe(fds: &[2]i32) usize { return errnoWrap(c.pipe(@ptrCast(&c_int, fds))); } + +pub fn getdirentries64(fd: i32, buf_ptr: &u8, buf_len: usize, basep: &i64) usize { + return errnoWrap(@bitCast(isize, c.__getdirentries64(fd, buf_ptr, buf_len, basep))); +} + pub fn mkdir(path: &const u8, mode: u32) usize { return errnoWrap(c.mkdir(path, mode)); } @@ -204,6 +231,10 @@ pub fn rename(old: &const u8, new: &const u8) usize { return errnoWrap(c.rename(old, new)); } +pub fn rmdir(path: &const u8) usize { + return errnoWrap(c.rmdir(path)); +} + pub fn chdir(path: &const u8) usize { return errnoWrap(c.chdir(path)); } @@ -268,6 +299,7 @@ pub const empty_sigset = sigset_t(0); pub const timespec = c.timespec; pub const Stat = c.Stat; +pub const dirent = c.dirent; /// Renamed from `sigaction` to `Sigaction` to avoid conflict with the syscall. pub const Sigaction = struct { diff --git a/std/os/index.zig b/std/os/index.zig index 09409d6d36..eb753db9b7 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -1055,10 +1055,11 @@ pub fn deleteTree(allocator: &Allocator, full_path: []const u8) DeleteTreeError! return; } else |err| switch (err) { error.FileNotFound => return, + + error.AccessDenied, error.IsDir => {}, error.OutOfMemory, - error.AccessDenied, error.SymLinkLoop, error.NameTooLong, error.SystemResources, @@ -1109,18 +1110,16 @@ pub fn deleteTree(allocator: &Allocator, full_path: []const u8) DeleteTreeError! } pub const Dir = struct { - // See man getdents fd: i32, + darwin_seek: darwin_seek_t, allocator: &Allocator, buf: []u8, index: usize, end_index: usize, - const LinuxEntry = extern struct { - d_ino: usize, - d_off: usize, - d_reclen: u16, - d_name: u8, // field address is the address of first byte of name + const darwin_seek_t = switch (builtin.os) { + Os.macosx, Os.ios => i64, + else => void, }; pub const Entry = struct { @@ -1135,15 +1134,26 @@ pub const Dir = struct { SymLink, File, UnixDomainSocket, + Wht, // TODO wtf is this Unknown, }; }; pub fn open(allocator: &Allocator, dir_path: []const u8) !Dir { - const fd = try posixOpen(allocator, dir_path, posix.O_RDONLY|posix.O_DIRECTORY|posix.O_CLOEXEC, 0); + const fd = switch (builtin.os) { + Os.windows => @compileError("TODO support Dir.open for windows"), + Os.linux => try posixOpen(allocator, dir_path, posix.O_RDONLY|posix.O_DIRECTORY|posix.O_CLOEXEC, 0), + Os.macosx, Os.ios => try posixOpen(allocator, dir_path, posix.O_RDONLY|posix.O_NONBLOCK|posix.O_DIRECTORY|posix.O_CLOEXEC, 0), + else => @compileError("Dir.open is not supported for this platform"), + }; + const darwin_seek_init = switch (builtin.os) { + Os.macosx, Os.ios => 0, + else => {}, + }; return Dir { .allocator = allocator, .fd = fd, + .darwin_seek = darwin_seek_init, .index = 0, .end_index = 0, .buf = []u8{}, @@ -1158,6 +1168,15 @@ pub const Dir = struct { /// Memory such as file names referenced in this returned entry becomes invalid /// with subsequent calls to next, as well as when this ::Dir is deinitialized. pub fn next(self: &Dir) !?Entry { + switch (builtin.os) { + Os.linux => return self.nextLinux(), + Os.macosx, Os.ios => return self.nextDarwin(), + Os.windows => return self.nextWindows(), + else => @compileError("Dir.next not supported on " ++ @tagName(builtin.os)), + } + } + + fn nextDarwin(self: &Dir) !?Entry { start_over: while (true) { if (self.index >= self.end_index) { if (self.buf.len == 0) { @@ -1165,8 +1184,9 @@ pub const Dir = struct { } while (true) { - const result = posix.getdents(self.fd, self.buf.ptr, self.buf.len); - const err = linux.getErrno(result); + const result = posix.getdirentries64(self.fd, self.buf.ptr, self.buf.len, + &self.darwin_seek); + const err = posix.getErrno(result); if (err > 0) { switch (err) { posix.EBADF, posix.EFAULT, posix.ENOTDIR => unreachable, @@ -1184,7 +1204,67 @@ pub const Dir = struct { break; } } - const linux_entry = @ptrCast(& align(1) LinuxEntry, &self.buf[self.index]); + const darwin_entry = @ptrCast(& align(1) posix.dirent, &self.buf[self.index]); + const next_index = self.index + darwin_entry.d_reclen; + self.index = next_index; + + const name = (&darwin_entry.d_name)[0..darwin_entry.d_namlen]; + + // skip . and .. entries + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (darwin_entry.d_type) { + posix.DT_BLK => Entry.Kind.BlockDevice, + posix.DT_CHR => Entry.Kind.CharacterDevice, + posix.DT_DIR => Entry.Kind.Directory, + posix.DT_FIFO => Entry.Kind.NamedPipe, + posix.DT_LNK => Entry.Kind.SymLink, + posix.DT_REG => Entry.Kind.File, + posix.DT_SOCK => Entry.Kind.UnixDomainSocket, + posix.DT_WHT => Entry.Kind.Wht, + else => Entry.Kind.Unknown, + }; + return Entry { + .name = name, + .kind = entry_kind, + }; + } + } + + fn nextWindows(self: &Dir) !?Entry { + @compileError("TODO support Dir.next for windows"); + } + + fn nextLinux(self: &Dir) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + if (self.buf.len == 0) { + self.buf = try self.allocator.alloc(u8, page_size); + } + + while (true) { + const result = posix.getdents(self.fd, self.buf.ptr, self.buf.len); + const err = posix.getErrno(result); + if (err > 0) { + switch (err) { + posix.EBADF, posix.EFAULT, posix.ENOTDIR => unreachable, + posix.EINVAL => { + self.buf = try self.allocator.realloc(u8, self.buf, self.buf.len * 2); + continue; + }, + else => return unexpectedErrorPosix(err), + } + } + if (result == 0) + return null; + self.index = 0; + self.end_index = result; + break; + } + } + const linux_entry = @ptrCast(& align(1) posix.dirent, &self.buf[self.index]); const next_index = self.index + linux_entry.d_reclen; self.index = next_index; @@ -1683,7 +1763,7 @@ test "std.os" { // TODO make this a build variable that you can set -const unexpected_error_tracing = false; +const unexpected_error_tracing = true; /// Call this when you made a syscall or something that sets errno /// and you get an unexpected error. diff --git a/std/os/linux/x86_64.zig b/std/os/linux/x86_64.zig index 3a76ca4f87..cfb2231df9 100644 --- a/std/os/linux/x86_64.zig +++ b/std/os/linux/x86_64.zig @@ -488,3 +488,11 @@ pub const timespec = extern struct { tv_sec: isize, tv_nsec: isize, }; + +pub const dirent = extern struct { + d_ino: usize, + d_off: usize, + d_reclen: u16, + d_name: u8, // field address is the address of first byte of name +}; + diff --git a/std/os/test.zig b/std/os/test.zig new file mode 100644 index 0000000000..2606dc1c58 --- /dev/null +++ b/std/os/test.zig @@ -0,0 +1,12 @@ +const std = @import("../index.zig"); +const os = std.os; +const io = std.io; + +const a = std.debug.global_allocator; + +test "makePath, put some files in it, deleteTree" { + try os.makePath(a, "os_test_tmp/b/c"); + try io.writeFile(a, "os_test_tmp/b/c/file.txt", "nonsense"); + try io.writeFile(a, "os_test_tmp/b/file2.txt", "blah"); + try os.deleteTree(a, "os_test_tmp"); +} From db70b909a0c29ff50984ce63535a6ba9e2ca4e07 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Wed, 28 Mar 2018 18:30:41 +0200 Subject: [PATCH 15/47] non-zero exit when build.zig cannot be created Make the stage 1 compiler exit with a non-zero status code when `zig build --init` cannot create a new build.zig file. --- src/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index eab7f29b10..791cb3b1b5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -452,10 +452,11 @@ int main(int argc, char **argv) { if ((err = os_copy_file(build_template_path, &build_file_abs))) { fprintf(stderr, "Unable to write build.zig template: %s\n", err_str(err)); + return EXIT_FAILURE; } else { fprintf(stderr, "Wrote build.zig template\n"); + return EXIT_SUCCESS; } - return EXIT_SUCCESS; } fprintf(stderr, From b60b01ce9780684dd861c54099204c2578c0c898 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Wed, 28 Mar 2018 18:30:41 +0200 Subject: [PATCH 16/47] skeleton stage 2 support for 'zig build' Initial port of the 'zig build' logic from the stage 1 compiler to the stage 2 compiler sans code generation and BUILD_INFO support. --- src-self-hosted/main.zig | 159 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 742bd03915..a04faaec49 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -41,6 +41,10 @@ pub fn main() !void { const args = try os.argsAlloc(allocator); defer os.argsFree(allocator, args); + if (args.len >= 2 and mem.eql(u8, args[1], "build")) { + return buildMain(allocator, args[2..]); + } + if (args.len >= 2 and mem.eql(u8, args[1], "fmt")) { return fmtMain(allocator, args[2..]); } @@ -560,6 +564,161 @@ fn printZen() !void { ); } +fn buildMain(allocator: &mem.Allocator, argv: []const []const u8) !void { + var build_file: [] const u8 = "build.zig"; + var cache_dir: ?[] const u8 = null; + var zig_install_prefix: ?[] const u8 = null; + var asked_for_help = false; + var asked_for_init = false; + + var args = ArrayList([] const u8).init(allocator); + defer args.deinit(); + + var zig_exe_path = try os.selfExePath(allocator); + defer allocator.free(zig_exe_path); + + try args.append(""); // Placeholder for zig-cache/build + try args.append(""); // Placeholder for zig_exe_path + try args.append(""); // Placeholder for build_file_dirname + try args.append(""); // Placeholder for full_cache_dir + + var i: usize = 0; + while (i < argv.len) : (i += 1) { + var arg = argv[i]; + if (mem.eql(u8, arg, "--help")) { + asked_for_help = true; + try args.append(argv[i]); + } else if (mem.eql(u8, arg, "--init")) { + asked_for_init = true; + try args.append(argv[i]); + } else if (i + 1 < argv.len and mem.eql(u8, arg, "--build-file")) { + build_file = argv[i + 1]; + i += 1; + } else if (i + 1 < argv.len and mem.eql(u8, arg, "--cache-dir")) { + cache_dir = argv[i + 1]; + i += 1; + } else if (i + 1 < argv.len and mem.eql(u8, arg, "--zig-install-prefix")) { + try args.append(arg); + i += 1; + zig_install_prefix = argv[i]; + try args.append(argv[i]); + } else { + try args.append(arg); + } + } + + const zig_lib_dir = try resolveZigLibDir(allocator, zig_install_prefix); + defer allocator.free(zig_lib_dir); + + const zig_std_dir = try os.path.join(allocator, zig_lib_dir, "std"); + defer allocator.free(zig_std_dir); + + const special_dir = try os.path.join(allocator, zig_std_dir, "special"); + defer allocator.free(special_dir); + + const build_runner_path = try os.path.join(allocator, special_dir, "build_runner.zig"); + defer allocator.free(build_runner_path); + + // g = codegen_create(build_runner_path, ...) + // codegen_set_out_name(g, "build") + + const build_file_abs = try os.path.resolve(allocator, ".", build_file); + defer allocator.free(build_file_abs); + + const build_file_basename = os.path.basename(build_file_abs); + const build_file_dirname = os.path.dirname(build_file_abs); + + var full_cache_dir: []u8 = undefined; + if (cache_dir == null) { + full_cache_dir = try os.path.join(allocator, build_file_dirname, "zig-cache"); + } else { + full_cache_dir = try os.path.resolve(allocator, ".", ??cache_dir, full_cache_dir); + } + defer allocator.free(full_cache_dir); + + const path_to_build_exe = try os.path.join(allocator, full_cache_dir, "build"); + defer allocator.free(path_to_build_exe); + // codegen_set_cache_dir(g, full_cache_dir) + + args.items[0] = path_to_build_exe; + args.items[1] = zig_exe_path; + args.items[2] = build_file_dirname; + args.items[3] = full_cache_dir; + + var build_file_exists: bool = undefined; + if (os.File.openRead(allocator, build_file_abs)) |*file| { + file.close(); + build_file_exists = true; + } else |_| { + build_file_exists = false; + } + + if (!build_file_exists and asked_for_help) { + // TODO(bnoordhuis) Print help message from std/special/build_runner.zig + return; + } + + if (!build_file_exists and asked_for_init) { + const build_template_path = try os.path.join(allocator, special_dir, "build_file_template.zig"); + defer allocator.free(build_template_path); + + var srcfile = try os.File.openRead(allocator, build_template_path); + defer srcfile.close(); + + var dstfile = try os.File.openWrite(allocator, build_file_abs); + defer dstfile.close(); + + while (true) { + var buffer: [4096]u8 = undefined; + const n = try srcfile.read(buffer[0..]); + if (n == 0) break; + try dstfile.write(buffer[0..n]); + } + + return; + } + + if (!build_file_exists) { + warn( + \\No 'build.zig' file found. + \\Initialize a 'build.zig' template file with `zig build --init`, + \\or build an executable directly with `zig build-exe $FILENAME.zig`. + \\See: `zig build --help` or `zig help` for more options. + \\ + ); + os.exit(1); + } + + // codegen_build(g) + // codegen_link(g, path_to_build_exe) + // codegen_destroy(g) + + var proc = try os.ChildProcess.init(args.toSliceConst(), allocator); + defer proc.deinit(); + + var term = try proc.spawnAndWait(); + switch (term) { + os.ChildProcess.Term.Exited => |status| { + if (status != 0) { + warn("{} exited with status {}\n", args.at(0), status); + os.exit(1); + } + }, + os.ChildProcess.Term.Signal => |signal| { + warn("{} killed by signal {}\n", args.at(0), signal); + os.exit(1); + }, + os.ChildProcess.Term.Stopped => |signal| { + warn("{} stopped by signal {}\n", args.at(0), signal); + os.exit(1); + }, + os.ChildProcess.Term.Unknown => |status| { + warn("{} encountered unknown failure {}\n", args.at(0), status); + os.exit(1); + }, + } +} + fn fmtMain(allocator: &mem.Allocator, file_paths: []const []const u8) !void { for (file_paths) |file_path| { var file = try os.File.openRead(allocator, file_path); From 72ce1462938c5821936e7b98f95c70c572fda47f Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Thu, 29 Mar 2018 00:53:06 +0200 Subject: [PATCH 17/47] Fixed looking for windows sdk when targeting linux --- src/analyze.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/analyze.cpp b/src/analyze.cpp index 7ee1de78a2..2128339726 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -4287,17 +4287,15 @@ static ZigWindowsSDK *get_windows_sdk(CodeGen *g) { void find_libc_include_path(CodeGen *g) { if (!g->libc_include_dir || buf_len(g->libc_include_dir) == 0) { - ZigWindowsSDK *sdk = get_windows_sdk(g); if (g->zig_target.os == OsWindows) { + ZigWindowsSDK *sdk = get_windows_sdk(g); if (os_get_win32_ucrt_include_path(sdk, g->libc_include_dir)) { zig_panic("Unable to determine libc include path."); } } - } - // TODO find libc at runtime for other operating systems - if(!g->libc_include_dir || buf_len(g->libc_include_dir) == 0) { + // TODO find libc at runtime for other operating systems zig_panic("Unable to determine libc include path."); } } From 032fccf6151ff201ce4b8c7ab28ca460fed794c0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 28 Mar 2018 23:25:12 -0400 Subject: [PATCH 18/47] fix compile time array concatenation for slices closes #866 --- src/ir.cpp | 18 ++++++++++++++++++ test/cases/eval.zig | 12 +++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/ir.cpp b/src/ir.cpp index ea001adb93..18fd02c297 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -11058,6 +11058,24 @@ static TypeTableEntry *ir_analyze_array_cat(IrAnalyze *ira, IrInstructionBinOp * result_type = get_array_type(ira->codegen, child_type, new_len); out_array_val = out_val; + } else if (is_slice(op1_type) || is_slice(op2_type)) { + TypeTableEntry *ptr_type = get_pointer_to_type(ira->codegen, child_type, true); + result_type = get_slice_type(ira->codegen, ptr_type); + out_array_val = create_const_vals(1); + out_array_val->special = ConstValSpecialStatic; + out_array_val->type = get_array_type(ira->codegen, child_type, new_len); + + out_val->data.x_struct.fields = create_const_vals(2); + + out_val->data.x_struct.fields[slice_ptr_index].type = ptr_type; + out_val->data.x_struct.fields[slice_ptr_index].special = ConstValSpecialStatic; + out_val->data.x_struct.fields[slice_ptr_index].data.x_ptr.special = ConstPtrSpecialBaseArray; + out_val->data.x_struct.fields[slice_ptr_index].data.x_ptr.data.base_array.array_val = out_array_val; + out_val->data.x_struct.fields[slice_ptr_index].data.x_ptr.data.base_array.elem_index = 0; + + out_val->data.x_struct.fields[slice_len_index].type = ira->codegen->builtin_types.entry_usize; + out_val->data.x_struct.fields[slice_len_index].special = ConstValSpecialStatic; + bigint_init_unsigned(&out_val->data.x_struct.fields[slice_len_index].data.x_bigint, new_len); } else { new_len += 1; // null byte diff --git a/test/cases/eval.zig b/test/cases/eval.zig index a5b41275bb..d6f7afe864 100644 --- a/test/cases/eval.zig +++ b/test/cases/eval.zig @@ -1,4 +1,5 @@ -const assert = @import("std").debug.assert; +const std = @import("std"); +const assert = std.debug.assert; const builtin = @import("builtin"); test "compile time recursion" { @@ -503,3 +504,12 @@ test "const ptr to comptime mutable data is not memoized" { assert(foo.read_x() == 2); } } + +test "array concat of slices gives slice" { + comptime { + var a: []const u8 = "aoeu"; + var b: []const u8 = "asdf"; + const c = a ++ b; + assert(std.mem.eql(u8, c, "aoeuasdf")); + } +} From 7e951e504302dc2a7b0efa3e2ea0dcde5524ac60 Mon Sep 17 00:00:00 2001 From: hellerve Date: Thu, 29 Mar 2018 10:23:44 +0200 Subject: [PATCH 19/47] st/os: address @andrewrk concerns --- std/os/index.zig | 7 ++++--- std/os/test.zig | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/std/os/index.zig b/std/os/index.zig index eb753db9b7..d8e2fd7009 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -1134,7 +1134,7 @@ pub const Dir = struct { SymLink, File, UnixDomainSocket, - Wht, // TODO wtf is this + Whiteout, Unknown, }; }; @@ -1223,7 +1223,7 @@ pub const Dir = struct { posix.DT_LNK => Entry.Kind.SymLink, posix.DT_REG => Entry.Kind.File, posix.DT_SOCK => Entry.Kind.UnixDomainSocket, - posix.DT_WHT => Entry.Kind.Wht, + posix.DT_WHT => Entry.Kind.Whiteout, else => Entry.Kind.Unknown, }; return Entry { @@ -1759,11 +1759,12 @@ test "std.os" { _ = @import("linux/index.zig"); _ = @import("path.zig"); _ = @import("windows/index.zig"); + _ = @import("test.zig"); } // TODO make this a build variable that you can set -const unexpected_error_tracing = true; +const unexpected_error_tracing = false; /// Call this when you made a syscall or something that sets errno /// and you get an unexpected error. diff --git a/std/os/test.zig b/std/os/test.zig index 2606dc1c58..20d439cc48 100644 --- a/std/os/test.zig +++ b/std/os/test.zig @@ -1,5 +1,6 @@ const std = @import("../index.zig"); const os = std.os; +const debug = std.debug; const io = std.io; const a = std.debug.global_allocator; @@ -9,4 +10,9 @@ test "makePath, put some files in it, deleteTree" { try io.writeFile(a, "os_test_tmp/b/c/file.txt", "nonsense"); try io.writeFile(a, "os_test_tmp/b/file2.txt", "blah"); try os.deleteTree(a, "os_test_tmp"); + if (os.Dir.open(a, "os_test_tmp")) |dir| { + debug.assert(false); // this should not happen! + } else |err| { + debug.assert(err == error.PathNotFound); + } } From 9df2a6a5027be17cba7f11ac9d7c106c3497647b Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Thu, 29 Mar 2018 13:43:17 +0200 Subject: [PATCH 20/47] std.zig.parser can now parse top level test declarations --- std/zig/ast.zig | 29 +++++++++++++++++++++++++ std/zig/parser.zig | 53 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 903dc051e2..180a0a9308 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -22,6 +22,7 @@ pub const Node = struct { StringLiteral, BuiltinCall, LineComment, + TestDecl, }; pub fn iterate(base: &Node, index: usize) ?&Node { @@ -39,6 +40,7 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), + Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), }; } @@ -57,6 +59,7 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), + Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), }; } @@ -75,6 +78,7 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), + Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), }; } }; @@ -476,3 +480,28 @@ pub const NodeLineComment = struct { return self.lines.at(self.lines.len - 1); } }; + +pub const NodeTestDecl = struct { + base: Node, + test_token: Token, + name_token: Token, + body_node: &Node, + + pub fn iterate(self: &NodeTestDecl, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.body_node; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeTestDecl) Token { + return self.test_token; + } + + pub fn lastToken(self: &NodeTestDecl) Token { + return self.body_node.lastToken(); + } +}; + diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 533ad754ac..06b7d35a48 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -171,6 +171,22 @@ pub const Parser = struct { stack.append(State { .TopLevelExtern = token }) catch unreachable; continue; }, + Token.Id.Keyword_test => { + stack.append(State.TopLevel) catch unreachable; + + const name_token = self.getNextToken(); + if (name_token.id != Token.Id.StringLiteral) + return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.StringLiteral), @tagName(name_token.id)); + + const lbrace = self.getNextToken(); + if (lbrace.id != Token.Id.LBrace) + return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); + + const block = try self.createBlock(arena, token); + const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, name_token, block); + try stack.append(State { .Block = block }); + continue; + }, Token.Id.Eof => { root_node.eof_token = token; return Tree {.root_node = root_node, .arena_allocator = arena_allocator}; @@ -733,6 +749,20 @@ pub const Parser = struct { return node; } + fn createTestDecl(self: &Parser, arena: &mem.Allocator, test_token: &const Token, name_token: &const Token, + block: &ast.NodeBlock) !&ast.NodeTestDecl + { + const node = try arena.create(ast.NodeTestDecl); + + *node = ast.NodeTestDecl { + .base = self.initNode(ast.Node.Id.TestDecl), + .test_token = *test_token, + .name_token = *name_token, + .body_node = &block.base, + }; + return node; + } + fn createFnProto(self: &Parser, arena: &mem.Allocator, fn_token: &const Token, extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto { @@ -867,6 +897,14 @@ pub const Parser = struct { return node; } + fn createAttachTestDecl(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), + test_token: &const Token, name_token: &const Token, block: &ast.NodeBlock) !&ast.NodeTestDecl + { + const node = try self.createTestDecl(arena, test_token, name_token, block); + try list.append(&node.base); + return node; + } + fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) { const loc = self.tokenizer.getTokenLocation(token); warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args); @@ -1032,7 +1070,11 @@ pub const Parser = struct { ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); try stack.append(RenderState { .VarDecl = var_decl}); - + }, + ast.Node.Id.TestDecl => { + const test_decl = @fieldParentPtr(ast.NodeTestDecl, "base", decl); + try stream.print("test {} ", self.tokenizer.getTokenSlice(test_decl.name_token)); + try stack.append(RenderState { .Expression = test_decl.body_node }); }, else => unreachable, } @@ -1201,6 +1243,7 @@ pub const Parser = struct { ast.Node.Id.Root, ast.Node.Id.VarDecl, + ast.Node.Id.TestDecl, ast.Node.Id.ParamDecl => unreachable, }, RenderState.FnProtoRParen => |fn_proto| { @@ -1422,4 +1465,12 @@ test "zig fmt" { \\} \\ ); + + try testCanonical( + \\test "test name" { + \\ const a = 1; + \\ var b = 1; + \\} + \\ + ); } From 0fd0f6fd1f4b9e02cc33eac304b9f6db242cbb67 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Fri, 30 Mar 2018 01:36:04 +1300 Subject: [PATCH 21/47] Rewrite Rand functions We now use a generic Rand structure which abstracts the core functions from the backing engine. The old Mersenne Twister engine is removed and replaced instead with three alternatives: - Pcg32 - Xoroshiro128+ - Isaac64 These should provide sufficient coverage for most purposes, including a CSPRNG using Isaac64. Consumers of the library that do not care about the actual engine implementation should use DefaultPrng and DefaultCsprng. --- CMakeLists.txt | 2 +- std/index.zig | 4 +- std/io_test.zig | 6 +- std/rand.zig | 240 ----------------- std/rand/index.zig | 652 +++++++++++++++++++++++++++++++++++++++++++++ std/rand_test.zig | 507 ----------------------------------- std/sort.zig | 154 +++++------ 7 files changed, 735 insertions(+), 830 deletions(-) delete mode 100644 std/rand.zig create mode 100644 std/rand/index.zig delete mode 100644 std/rand_test.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 979d771cda..fbaf929e85 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -509,7 +509,7 @@ set(ZIG_STD_FILES "os/windows/index.zig" "os/windows/util.zig" "os/zen.zig" - "rand.zig" + "rand/index.zig" "sort.zig" "special/bootstrap.zig" "special/bootstrap_lib.zig" diff --git a/std/index.zig b/std/index.zig index 179eae159e..4bc1444ac9 100644 --- a/std/index.zig +++ b/std/index.zig @@ -26,7 +26,7 @@ pub const math = @import("math/index.zig"); pub const mem = @import("mem.zig"); pub const net = @import("net.zig"); pub const os = @import("os/index.zig"); -pub const rand = @import("rand.zig"); +pub const rand = @import("rand/index.zig"); pub const sort = @import("sort.zig"); pub const unicode = @import("unicode.zig"); pub const zig = @import("zig/index.zig"); @@ -58,7 +58,7 @@ test "std" { _ = @import("heap.zig"); _ = @import("net.zig"); _ = @import("os/index.zig"); - _ = @import("rand.zig"); + _ = @import("rand/index.zig"); _ = @import("sort.zig"); _ = @import("unicode.zig"); _ = @import("zig/index.zig"); diff --git a/std/io_test.zig b/std/io_test.zig index 993ec84d20..89959b7b54 100644 --- a/std/io_test.zig +++ b/std/io_test.zig @@ -1,7 +1,7 @@ const std = @import("index.zig"); const io = std.io; const allocator = std.debug.global_allocator; -const Rand = std.rand.Rand; +const DefaultPrng = std.rand.DefaultPrng; const assert = std.debug.assert; const mem = std.mem; const os = std.os; @@ -9,8 +9,8 @@ const builtin = @import("builtin"); test "write a file, read it, then delete it" { var data: [1024]u8 = undefined; - var rng = Rand.init(1234); - rng.fillBytes(data[0..]); + var prng = DefaultPrng.init(1234); + prng.random.bytes(data[0..]); const tmp_file_name = "temp_test_file.txt"; { var file = try os.File.openWrite(allocator, tmp_file_name); diff --git a/std/rand.zig b/std/rand.zig deleted file mode 100644 index 96f0a385a4..0000000000 --- a/std/rand.zig +++ /dev/null @@ -1,240 +0,0 @@ -const std = @import("index.zig"); -const builtin = @import("builtin"); -const assert = std.debug.assert; -const rand_test = @import("rand_test.zig"); -const mem = std.mem; -const math = std.math; - -pub const MT19937_32 = MersenneTwister( - u32, 624, 397, 31, - 0x9908B0DF, - 11, 0xFFFFFFFF, - 7, 0x9D2C5680, - 15, 0xEFC60000, - 18, 1812433253); - -pub const MT19937_64 = MersenneTwister( - u64, 312, 156, 31, - 0xB5026F5AA96619E9, - 29, 0x5555555555555555, - 17, 0x71D67FFFEDA60000, - 37, 0xFFF7EEE000000000, - 43, 6364136223846793005); - -/// Use `init` to initialize this state. -pub const Rand = struct { - const Rng = if (@sizeOf(usize) >= 8) MT19937_64 else MT19937_32; - - rng: Rng, - - /// Initialize random state with the given seed. - pub fn init(seed: usize) Rand { - return Rand { - .rng = Rng.init(seed), - }; - } - - /// Get an integer or boolean with random bits. - pub fn scalar(r: &Rand, comptime T: type) T { - if (T == usize) { - return r.rng.get(); - } else if (T == bool) { - return (r.rng.get() & 0b1) == 0; - } else { - var result: [@sizeOf(T)]u8 = undefined; - r.fillBytes(result[0..]); - return mem.readInt(result, T, builtin.Endian.Little); - } - } - - /// Fill `buf` with randomness. - pub fn fillBytes(r: &Rand, buf: []u8) void { - var bytes_left = buf.len; - while (bytes_left >= @sizeOf(usize)) { - mem.writeInt(buf[buf.len - bytes_left..], r.rng.get(), builtin.Endian.Little); - bytes_left -= @sizeOf(usize); - } - if (bytes_left > 0) { - var rand_val_array: [@sizeOf(usize)]u8 = undefined; - mem.writeInt(rand_val_array[0..], r.rng.get(), builtin.Endian.Little); - while (bytes_left > 0) { - buf[buf.len - bytes_left] = rand_val_array[@sizeOf(usize) - bytes_left]; - bytes_left -= 1; - } - } - } - - /// Get a random unsigned integer with even distribution between `start` - /// inclusive and `end` exclusive. - pub fn range(r: &Rand, comptime T: type, start: T, end: T) T { - assert(start <= end); - if (T.is_signed) { - const uint = @IntType(false, T.bit_count); - if (start >= 0 and end >= 0) { - return T(r.range(uint, uint(start), uint(end))); - } else if (start < 0 and end < 0) { - // Can't overflow because the range is over signed ints - return math.negateCast(r.range(uint, math.absCast(end), math.absCast(start)) + 1) catch unreachable; - } else if (start < 0 and end >= 0) { - const end_uint = uint(end); - const total_range = math.absCast(start) + end_uint; - const value = r.range(uint, 0, total_range); - const result = if (value < end_uint) x: { - break :x T(value); - } else if (value == end_uint) x: { - break :x start; - } else x: { - // Can't overflow because the range is over signed ints - break :x math.negateCast(value - end_uint) catch unreachable; - }; - return result; - } else { - unreachable; - } - } else { - const total_range = end - start; - const leftover = @maxValue(T) % total_range; - const upper_bound = @maxValue(T) - leftover; - var rand_val_array: [@sizeOf(T)]u8 = undefined; - - while (true) { - r.fillBytes(rand_val_array[0..]); - const rand_val = mem.readInt(rand_val_array, T, builtin.Endian.Little); - if (rand_val < upper_bound) { - return start + (rand_val % total_range); - } - } - } - } - - /// Get a floating point value in the range 0.0..1.0. - pub fn float(r: &Rand, comptime T: type) T { - // TODO Implement this way instead: - // const int = @int_type(false, @sizeOf(T) * 8); - // const mask = ((1 << @float_mantissa_bit_count(T)) - 1); - // const rand_bits = r.rng.scalar(int) & mask; - // return @float_compose(T, false, 0, rand_bits) - 1.0 - const int_type = @IntType(false, @sizeOf(T) * 8); - const precision = if (T == f32) - 16777216 - else if (T == f64) - 9007199254740992 - else - @compileError("unknown floating point type") - ; - return T(r.range(int_type, 0, precision)) / T(precision); - } -}; - -fn MersenneTwister( - comptime int: type, comptime n: usize, comptime m: usize, comptime r: int, - comptime a: int, - comptime u: math.Log2Int(int), comptime d: int, - comptime s: math.Log2Int(int), comptime b: int, - comptime t: math.Log2Int(int), comptime c: int, - comptime l: math.Log2Int(int), comptime f: int) type -{ - return struct { - const Self = this; - - array: [n]int, - index: usize, - - pub fn init(seed: int) Self { - var mt = Self { - .array = undefined, - .index = n, - }; - - var prev_value = seed; - mt.array[0] = prev_value; - var i: usize = 1; - while (i < n) : (i += 1) { - prev_value = int(i) +% f *% (prev_value ^ (prev_value >> (int.bit_count - 2))); - mt.array[i] = prev_value; - } - return mt; - } - - pub fn get(mt: &Self) int { - const mag01 = []int{0, a}; - const LM: int = (1 << r) - 1; - const UM = ~LM; - - if (mt.index >= n) { - var i: usize = 0; - - while (i < n - m) : (i += 1) { - const x = (mt.array[i] & UM) | (mt.array[i + 1] & LM); - mt.array[i] = mt.array[i + m] ^ (x >> 1) ^ mag01[usize(x & 0x1)]; - } - - while (i < n - 1) : (i += 1) { - const x = (mt.array[i] & UM) | (mt.array[i + 1] & LM); - mt.array[i] = mt.array[i + m - n] ^ (x >> 1) ^ mag01[usize(x & 0x1)]; - - } - const x = (mt.array[i] & UM) | (mt.array[0] & LM); - mt.array[i] = mt.array[m - 1] ^ (x >> 1) ^ mag01[usize(x & 0x1)]; - - mt.index = 0; - } - - var x = mt.array[mt.index]; - mt.index += 1; - - x ^= ((x >> u) & d); - x ^= ((x << s) & b); - x ^= ((x << t) & c); - x ^= (x >> l); - - return x; - } - }; -} - -test "rand float 32" { - var r = Rand.init(42); - var i: usize = 0; - while (i < 1000) : (i += 1) { - const val = r.float(f32); - assert(val >= 0.0); - assert(val < 1.0); - } -} - -test "rand.MT19937_64" { - var rng = MT19937_64.init(rand_test.mt64_seed); - for (rand_test.mt64_data) |value| { - assert(value == rng.get()); - } -} - -test "rand.MT19937_32" { - var rng = MT19937_32.init(rand_test.mt32_seed); - for (rand_test.mt32_data) |value| { - assert(value == rng.get()); - } -} - -test "rand.Rand.range" { - var r = Rand.init(42); - testRange(&r, -4, 3); - testRange(&r, -4, -1); - testRange(&r, 10, 14); -} - -fn testRange(r: &Rand, start: i32, end: i32) void { - const count = usize(end - start); - var values_buffer = []bool{false} ** 20; - const values = values_buffer[0..count]; - var i: usize = 0; - while (i < count) { - const value = r.range(i32, start, end); - const index = usize(value - start); - if (!values[index]) { - i += 1; - values[index] = true; - } - } -} diff --git a/std/rand/index.zig b/std/rand/index.zig new file mode 100644 index 0000000000..ce69565473 --- /dev/null +++ b/std/rand/index.zig @@ -0,0 +1,652 @@ +// The engines provided here should be initialized from an external source. For now, getRandomBytes +// from the os package is the most suitable. Be sure to use a CSPRNG when required, otherwise using +// a normal PRNG will be faster and use substantially less stack space. +// +// ``` +// var buf: [8]u8 = undefined; +// try std.os.getRandomBytes(buf[0..]); +// const seed = mem.readInt(buf[0..8], u64, builtin.Endian.Little); +// +// var r = DefaultPrng.init(seed); +// +// const s = r.random.scalar(u64); +// ``` +// +// TODO(tiehuis): Benchmark these against other reference implementations. + +const std = @import("../index.zig"); +const builtin = @import("builtin"); +const assert = std.debug.assert; +const mem = std.mem; +const math = std.math; + +// When you need fast unbiased random numbers +pub const DefaultPrng = Xoroshiro128; + +// When you need cryptographically secure random numbers +pub const DefaultCsprng = Isaac64; + +pub const Rand = struct { + fillFn: fn(r: &Rand, buf: []u8) void, + + /// Read random bytes into the specified buffer until fill. + pub fn bytes(r: &Rand, buf: []u8) void { + r.fillFn(r, buf); + } + + /// Return a random integer/boolean type. + pub fn scalar(r: &Rand, comptime T: type) T { + var rand_bytes: [@sizeOf(T)]u8 = undefined; + r.bytes(rand_bytes[0..]); + + if (T == bool) { + return rand_bytes[0] & 0b1 == 0; + } else { + // NOTE: Cannot @bitCast array to integer type. + return mem.readInt(rand_bytes, T, builtin.Endian.Little); + } + } + + /// Get a random unsigned integer with even distribution between `start` + /// inclusive and `end` exclusive. + pub fn range(r: &Rand, comptime T: type, start: T, end: T) T { + assert(start <= end); + if (T.is_signed) { + const uint = @IntType(false, T.bit_count); + if (start >= 0 and end >= 0) { + return T(r.range(uint, uint(start), uint(end))); + } else if (start < 0 and end < 0) { + // Can't overflow because the range is over signed ints + return math.negateCast(r.range(uint, math.absCast(end), math.absCast(start)) + 1) catch unreachable; + } else if (start < 0 and end >= 0) { + const end_uint = uint(end); + const total_range = math.absCast(start) + end_uint; + const value = r.range(uint, 0, total_range); + const result = if (value < end_uint) x: { + break :x T(value); + } else if (value == end_uint) x: { + break :x start; + } else x: { + // Can't overflow because the range is over signed ints + break :x math.negateCast(value - end_uint) catch unreachable; + }; + return result; + } else { + unreachable; + } + } else { + const total_range = end - start; + const leftover = @maxValue(T) % total_range; + const upper_bound = @maxValue(T) - leftover; + var rand_val_array: [@sizeOf(T)]u8 = undefined; + + while (true) { + r.bytes(rand_val_array[0..]); + const rand_val = mem.readInt(rand_val_array, T, builtin.Endian.Little); + if (rand_val < upper_bound) { + return start + (rand_val % total_range); + } + } + } + } + + /// Return a floating point value evenly distributed in the range [0, 1). + pub fn float(r: &Rand, comptime T: type) T { + // Generate a uniform value between [1, 2) and scale down to [0, 1). + // Note: The lowest mantissa bit is always set to 0 so we only use half the available range. + switch (T) { + f32 => { + const s = r.scalar(u32); + const repr = (0x7f << 23) | (s >> 9); + return @bitCast(f32, repr) - 1.0; + }, + f64 => { + const s = r.scalar(u64); + const repr = (0x3ff << 52) | (s >> 12); + return @bitCast(f64, repr) - 1.0; + }, + else => @compileError("unknown floating point type"), + } + } + + /// Return a floating point value normally distributed in the range [0, 1]. + pub fn floatNorm(r: &Rand, comptime T: type) T { + // TODO(tiehuis): See https://www.doornik.com/research/ziggurat.pdf + @compileError("floatNorm is unimplemented"); + } + + /// Return a exponentially distributed float between (0, @maxValue(f64)) + pub fn floatExp(r: &Rand, comptime T: type) T { + @compileError("floatExp is unimplemented"); + } + + /// Shuffle a slice into a random order. + pub fn shuffle(r: &Rand, comptime T: type, buf: []T) void { + if (buf.len < 2) { + return; + } + + var i: usize = 0; + while (i < buf.len - 1) : (i += 1) { + const j = r.range(usize, i, buf.len); + mem.swap(T, &buf[i], &buf[j]); + } + } +}; + +// Generator to extend 64-bit seed values into longer sequences. +// +// The number of cycles is thus limited to 64-bits regardless of the engine, but this +// is still plenty for practical purposes. +const SplitMix64 = struct { + s: u64, + + pub fn init(seed: u64) SplitMix64 { + return SplitMix64 { .s = seed }; + } + + pub fn next(self: &SplitMix64) u64 { + self.s +%= 0x9e3779b97f4a7c15; + + var z = self.s; + z = (z ^ (z >> 30)) *% 0xbf58476d1ce4e5b9; + z = (z ^ (z >> 27)) *% 0x94d049bb133111eb; + return z ^ (z >> 31); + } +}; + +test "splitmix64 sequence" { + var r = SplitMix64.init(0xaeecf86f7878dd75); + + const seq = []const u64 { + 0x5dbd39db0178eb44, + 0xa9900fb66b397da3, + 0x5c1a28b1aeebcf5c, + 0x64a963238f776912, + 0xc6d4177b21d1c0ab, + 0xb2cbdbdb5ea35394, + }; + + for (seq) |s| { + std.debug.assert(s == r.next()); + } +} + +// PCG32 - http://www.pcg-random.org/ +// +// PRNG +pub const Pcg = struct { + const default_multiplier = 6364136223846793005; + + random: Rand, + + s: u64, + i: u64, + + pub fn init(init_s: u64) Pcg { + var pcg = Pcg { + .random = Rand { .fillFn = fill }, + .s = undefined, + .i = undefined, + }; + + pcg.seed(init_s); + return pcg; + } + + fn next(self: &Pcg) u32 { + const l = self.s; + self.s = l *% default_multiplier +% (self.i | 1); + + const xor_s = @truncate(u32, ((l >> 18) ^ l) >> 27); + const rot = u32(l >> 59); + + return (xor_s >> u5(rot)) | (xor_s << u5((0 -% rot) & 31)); + } + + fn seed(self: &Pcg, init_s: u64) void { + // Pcg requires 128-bits of seed. + var gen = SplitMix64.init(init_s); + self.seedTwo(gen.next(), gen.next()); + } + + fn seedTwo(self: &Pcg, init_s: u64, init_i: u64) void { + self.s = 0; + self.i = (init_s << 1) | 1; + self.s = self.s *% default_multiplier +% self.i; + self.s +%= init_i; + self.s = self.s *% default_multiplier +% self.i; + } + + fn fill(r: &Rand, buf: []u8) void { + const self = @fieldParentPtr(Pcg, "random", r); + + var i: usize = 0; + const aligned_len = buf.len - (buf.len & 7); + + // Complete 4 byte segments. + while (i < aligned_len) : (i += 4) { + var n = self.next(); + comptime var j: usize = 0; + inline while (j < 4) : (j += 1) { + buf[i + j] = @truncate(u8, n); + n >>= 8; + } + } + + // Remaining. (cuts the stream) + if (i != buf.len) { + var n = self.next(); + while (i < buf.len) : (i += 1) { + buf[i] = @truncate(u8, n); + n >>= 4; + } + } + } +}; + +test "pcg sequence" { + var r = Pcg.init(0); + const s0: u64 = 0x9394bf54ce5d79de; + const s1: u64 = 0x84e9c579ef59bbf7; + r.seedTwo(s0, s1); + + const seq = []const u32 { + 2881561918, + 3063928540, + 1199791034, + 2487695858, + 1479648952, + 3247963454, + }; + + for (seq) |s| { + std.debug.assert(s == r.next()); + } +} + +// Xoroshiro128+ - http://xoroshiro.di.unimi.it/ +// +// PRNG +pub const Xoroshiro128 = struct { + random: Rand, + + s: [2]u64, + + pub fn init(init_s: u64) Xoroshiro128 { + var x = Xoroshiro128 { + .random = Rand { .fillFn = fill }, + .s = undefined, + }; + + x.seed(init_s); + return x; + } + + fn next(self: &Xoroshiro128) u64 { + const s0 = self.s[0]; + var s1 = self.s[1]; + const r = s0 +% s1; + + s1 ^= s0; + self.s[0] = math.rotl(u64, s0, u8(55)) ^ s1 ^ (s1 << 14); + self.s[1] = math.rotl(u64, s1, u8(36)); + + return r; + } + + // Skip 2^64 places ahead in the sequence + fn jump(self: &Xoroshiro128) void { + var s0: u64 = 0; + var s1: u64 = 0; + + const table = []const u64 { + 0xbeac0467eba5facb, + 0xd86b048b86aa9922 + }; + + inline for (table) |entry| { + var b: usize = 0; + while (b < 64) : (b += 1) { + if ((entry & (u64(1) << u6(b))) != 0) { + s0 ^= self.s[0]; + s1 ^= self.s[1]; + } + _ = self.next(); + } + } + + self.s[0] = s0; + self.s[1] = s1; + } + + fn seed(self: &Xoroshiro128, init_s: u64) void { + // Xoroshiro requires 128-bits of seed. + var gen = SplitMix64.init(init_s); + + self.s[0] = gen.next(); + self.s[1] = gen.next(); + } + + fn fill(r: &Rand, buf: []u8) void { + const self = @fieldParentPtr(Xoroshiro128, "random", r); + + var i: usize = 0; + const aligned_len = buf.len - (buf.len & 7); + + // Complete 8 byte segments. + while (i < aligned_len) : (i += 8) { + var n = self.next(); + comptime var j: usize = 0; + inline while (j < 8) : (j += 1) { + buf[i + j] = @truncate(u8, n); + n >>= 8; + } + } + + // Remaining. (cuts the stream) + if (i != buf.len) { + var n = self.next(); + while (i < buf.len) : (i += 1) { + buf[i] = @truncate(u8, n); + n >>= 8; + } + } + } +}; + +test "xoroshiro sequence" { + var r = Xoroshiro128.init(0); + r.s[0] = 0xaeecf86f7878dd75; + r.s[1] = 0x01cd153642e72622; + + const seq1 = []const u64 { + 0xb0ba0da5bb600397, + 0x18a08afde614dccc, + 0xa2635b956a31b929, + 0xabe633c971efa045, + 0x9ac19f9706ca3cac, + 0xf62b426578c1e3fb, + }; + + for (seq1) |s| { + std.debug.assert(s == r.next()); + } + + + r.jump(); + + const seq2 = []const u64 { + 0x95344a13556d3e22, + 0xb4fb32dafa4d00df, + 0xb2011d9ccdcfe2dd, + 0x05679a9b2119b908, + 0xa860a1da7c9cd8a0, + 0x658a96efe3f86550, + }; + + for (seq2) |s| { + std.debug.assert(s == r.next()); + } +} + +// ISAAC64 - http://www.burtleburtle.net/bob/rand/isaacafa.html +// +// CSPRNG +// +// Follows the general idea of the implementation from here with a few shortcuts. +// https://doc.rust-lang.org/rand/src/rand/prng/isaac64.rs.html +pub const Isaac64 = struct { + random: Rand, + + r: [256]u64, + m: [256]u64, + a: u64, + b: u64, + c: u64, + i: usize, + + pub fn init(init_s: u64) Isaac64 { + var isaac = Isaac64 { + .random = Rand { .fillFn = fill }, + .r = undefined, + .m = undefined, + .a = undefined, + .b = undefined, + .c = undefined, + .i = undefined, + }; + + // seed == 0 => same result as the unseeded reference implementation + isaac.seed(init_s, 1); + return isaac; + } + + fn step(self: &Isaac64, mix: u64, base: usize, comptime m1: usize, comptime m2: usize) void { + const x = self.m[base + m1]; + self.a = mix +% self.m[base + m2]; + + const y = self.a +% self.b +% self.m[(x >> 3) % self.m.len]; + self.m[base + m1] = y; + + self.b = x +% self.m[(y >> 11) % self.m.len]; + self.r[self.r.len - 1 - base - m1] = self.b; + } + + fn refill(self: &Isaac64) void { + const midpoint = self.r.len / 2; + + self.c +%= 1; + self.b +%= self.c; + + { + var i: usize = 0; + while (i < midpoint) : (i += 4) { + self.step( ~(self.a ^ (self.a << 21)), i + 0, 0, midpoint); + self.step( self.a ^ (self.a >> 5) , i + 1, 0, midpoint); + self.step( self.a ^ (self.a << 12) , i + 2, 0, midpoint); + self.step( self.a ^ (self.a >> 33) , i + 3, 0, midpoint); + } + } + + { + var i: usize = 0; + while (i < midpoint) : (i += 4) { + self.step( ~(self.a ^ (self.a << 21)), i + 0, midpoint, 0); + self.step( self.a ^ (self.a >> 5) , i + 1, midpoint, 0); + self.step( self.a ^ (self.a << 12) , i + 2, midpoint, 0); + self.step( self.a ^ (self.a >> 33) , i + 3, midpoint, 0); + } + } + + self.i = 0; + } + + fn next(self: &Isaac64) u64 { + if (self.i >= self.r.len) { + self.refill(); + } + + const value = self.r[self.i]; + self.i += 1; + return value; + } + + fn seed(self: &Isaac64, init_s: u64, comptime rounds: usize) void { + // We ignore the multi-pass requirement since we don't currently expose full access to + // seeding the self.m array completely. + mem.set(u64, self.m[0..], 0); + self.m[0] = init_s; + + // prescrambled golden ratio constants + var a = []const u64 { + 0x647c4677a2884b7c, + 0xb9f8b322c73ac862, + 0x8c0ea5053d4712a0, + 0xb29b2e824a595524, + 0x82f053db8355e0ce, + 0x48fe4a0fa5a09315, + 0xae985bf2cbfc89ed, + 0x98f5704f6c44c0ab, + }; + + comptime var i: usize = 0; + inline while (i < rounds) : (i += 1) { + var j: usize = 0; + while (j < self.m.len) : (j += 8) { + comptime var x1: usize = 0; + inline while (x1 < 8) : (x1 += 1) { + a[x1] +%= self.m[j + x1]; + } + + a[0] -%= a[4]; a[5] ^= a[7] >> 9; a[7] +%= a[0]; + a[1] -%= a[5]; a[6] ^= a[0] << 9; a[0] +%= a[1]; + a[2] -%= a[6]; a[7] ^= a[1] >> 23; a[1] +%= a[2]; + a[3] -%= a[7]; a[0] ^= a[2] << 15; a[2] +%= a[3]; + a[4] -%= a[0]; a[1] ^= a[3] >> 14; a[3] +%= a[4]; + a[5] -%= a[1]; a[2] ^= a[4] << 20; a[4] +%= a[5]; + a[6] -%= a[2]; a[3] ^= a[5] >> 17; a[5] +%= a[6]; + a[7] -%= a[3]; a[4] ^= a[6] << 14; a[6] +%= a[7]; + + comptime var x2: usize = 0; + inline while (x2 < 8) : (x2 += 1) { + self.m[j + x2] = a[x2]; + } + } + } + + mem.set(u64, self.r[0..], 0); + self.a = 0; + self.b = 0; + self.c = 0; + self.i = self.r.len; // trigger refill on first value + } + + fn fill(r: &Rand, buf: []u8) void { + const self = @fieldParentPtr(Isaac64, "random", r); + + var i: usize = 0; + const aligned_len = buf.len - (buf.len & 7); + + // Fill complete 64-byte segments + while (i < aligned_len) : (i += 8) { + var n = self.next(); + comptime var j: usize = 0; + inline while (j < 8) : (j += 1) { + buf[i + j] = @truncate(u8, n); + n >>= 8; + } + } + + // Fill trailing, ignoring excess (cut the stream). + if (i != buf.len) { + var n = self.next(); + while (i < buf.len) : (i += 1) { + buf[i] = @truncate(u8, n); + n >>= 8; + } + } + } +}; + +test "isaac64 sequence" { + var r = Isaac64.init(0); + + // from reference implementation + const seq = []const u64 { + 0xf67dfba498e4937c, + 0x84a5066a9204f380, + 0xfee34bd5f5514dbb, + 0x4d1664739b8f80d6, + 0x8607459ab52a14aa, + 0x0e78bc5a98529e49, + 0xfe5332822ad13777, + 0x556c27525e33d01a, + 0x08643ca615f3149f, + 0xd0771faf3cb04714, + 0x30e86f68a37b008d, + 0x3074ebc0488a3adf, + 0x270645ea7a2790bc, + 0x5601a0a8d3763c6a, + 0x2f83071f53f325dd, + 0xb9090f3d42d2d2ea, + }; + + for (seq) |s| { + std.debug.assert(s == r.next()); + } +} + +// Actual Rand helper function tests, pcg engine is assumed correct. +test "Rand float" { + var prng = DefaultPrng.init(0); + + var i: usize = 0; + while (i < 1000) : (i += 1) { + const val1 = prng.random.float(f32); + std.debug.assert(val1 >= 0.0); + std.debug.assert(val1 < 1.0); + + const val2 = prng.random.float(f64); + std.debug.assert(val2 >= 0.0); + std.debug.assert(val2 < 1.0); + } +} + +test "Rand scalar" { + var prng = DefaultPrng.init(0); + const s = prng .random.scalar(u64); +} + +test "Rand bytes" { + var prng = DefaultPrng.init(0); + var buf: [2048]u8 = undefined; + prng.random.bytes(buf[0..]); +} + +test "Rand shuffle" { + var prng = DefaultPrng.init(0); + + var seq = []const u8 { 0, 1, 2, 3, 4 }; + var seen = []bool {false} ** 5; + + var i: usize = 0; + while (i < 1000) : (i += 1) { + prng.random.shuffle(u8, seq[0..]); + seen[seq[0]] = true; + std.debug.assert(sumArray(seq[0..]) == 10); + } + + // we should see every entry at the head at least once + for (seen) |e| { + std.debug.assert(e == true); + } +} + +fn sumArray(s: []const u8) u32 { + var r: u32 = 0; + for (s) |e| r += e; + return r; +} + +test "Rand range" { + var prng = DefaultPrng.init(0); + testRange(&prng.random, -4, 3); + testRange(&prng.random, -4, -1); + testRange(&prng.random, 10, 14); +} + +fn testRange(r: &Rand, start: i32, end: i32) void { + const count = usize(end - start); + var values_buffer = []bool{false} ** 20; + const values = values_buffer[0..count]; + var i: usize = 0; + while (i < count) { + const value = r.range(i32, start, end); + const index = usize(value - start); + if (!values[index]) { + i += 1; + values[index] = true; + } + } +} diff --git a/std/rand_test.zig b/std/rand_test.zig deleted file mode 100644 index 200b50a5b6..0000000000 --- a/std/rand_test.zig +++ /dev/null @@ -1,507 +0,0 @@ -pub const mt64_seed = 0xb334e49d0977c37e; -pub const mt64_data = []u64 { - 0x2ad1a63fab6d25a9, 0xb7143aba12569814, 0xc1b60d8d49e53e8, 0x5652adfc8da656dc, - 0x43e3beb6d9e484a9, 0x17b09b71e9418ff7, 0x541646292686cfa4, 0x260457071268ecfc, - 0x1627af31774e1dc1, 0x362a49b34ed75bb3, 0x7acf72002fe0f733, 0x3aaaf2e7b9409452, - 0x9cfc2d9908115c2, 0x6e81a7f16ae613e9, 0xfc4da89c04acf3c7, 0x6984b6adb4feb9ae, - 0x6a128b334e27b03d, 0xcc45a2b02937871a, 0xe585b229e00b2283, 0x7a92c0664a6f678a, - 0x972735011bdc0744, 0xb494e743d658a084, 0x1eda3c4e7b1b2d0c, 0x4c7adb3831d87332, - 0x12f8c7355f5ec631, 0xfc2bcb6be7d60eba, 0x74b95b47895f8687, 0x171de6fe92b97f5a, - 0x86383730f52719ac, 0xe4e43ce0f61274f6, 0x514f7e072d96f19c, 0xabef324fbc6cb7fe, - 0x7534b945b742f14f, 0x47f9efe33265adbe, 0x7bcab027a0abf16b, 0x3312a2b34225bff7, - 0xb61455ce8c2e3e0b, 0x2b81008deeee4d94, 0x743b0b2b4974c7b6, 0xfc219101fd665f7d, - 0x863d78891cbfe5e5, 0x7531bb1839181778, 0xf614359a65356e72, 0xfcdd1f6e3f250bdd, - 0x528e10bd536eed5b, 0x9f69386ac60cd618, 0xbf5f242706817f01, 0x8e7da8070072cf64, - 0xaa318b622da0667f, 0xc9580540fb7efd66, 0xb5d996deccd02e0e, 0x81c6a799ea3f5a41, - 0xe6b896f4e21a8550, 0xde5206f177a24ceb, 0x53343e81639ec0b6, 0x5a6edb63d08be9f6, - 0x3602c2892f7da1b9, 0xda84f4259841bdea, 0x5880e169a7746e45, 0x57cddb5ffd3c2423, - 0x28fe1166fe7b8595, 0x92136c9decb42243, 0xa8c4199818ca7d62, 0x5042cd96f62854dd, - 0x22b2d38c3f21f8d0, 0x73a2bfccf1b5f7bb, 0xaba3718f40b6984e, 0x9c1f5dc3e399f5f0, - 0x9cf463f95369c149, 0xa7546d69e4232e18, 0x9ea57317d19ab7fc, 0xfb13c83d830731fb, - 0x635a123eaa099259, 0x9a2fe7d0ba6e3c5c, 0x40b903cd0d0d3b4e, 0xc8210eb2d2e941cb, - 0xd2582d4b1e016484, 0x1d048030875af39c, 0xb51c31a6c193d76f, 0x5ce9b801b8d61626, - 0x2bae30455cbb0022, 0xba54df5998b2443f, 0x927abb9342c9a90a, 0xc431eb7df3e06727, - 0x726f885d3da88d5, 0x7d85ff1ca4260280, 0xaf3fecf8f019817, 0x31d39105d6fc4fe8, - 0x262d9842dbafd4bd, 0x54c28a2876e62e39, 0x95a986e24e214dde, 0xbf677a1abd2e553, - 0x48ac890ff787b2b6, 0x2890ec1c67c539f4, 0x7ce88bf3975882c3, 0x88ef340414a29c88, - 0xe30de9c88a00805b, 0xe772225e3ee6c68a, 0xa3a7d0921c5d5816, 0x8354957227b1663f, - 0xb5b65ded7c747cbc, 0x93b4a12ff2e8fcea, 0x6359579c3c438b3c, 0x45b2c12e9722f2bd, - 0x659a604414f19e1, 0x4ec9a149d4219ca5, 0xd830290dd6aebe2b, 0xabc7874a6b4827f8, - 0xc91be5dd875847e7, 0x5b761d39f3f96aee, 0x5749dffad692b6c8, 0x86c94840cbd249d2, - 0x411a466e886ad7, 0x27dca1f51aebb9a0, 0x1cceb093fbab7a42, 0x7140c2d6706e927c, - 0x6881fdb87299a92f, 0xa81a28de171f3c47, 0x8fa9a1b3bb5dfb2a, 0xae076853e3e0abde, - 0xe76572308ecd6b54, 0x6cd926c2e2760d8a, 0xdf080266cfbe3dc3, 0xb99b961999765d7b, - 0xadb5d4e2b896ddf1, 0x8d3aaf4c83c83c56, 0x9b66e4f6eb65bef7, 0x7a81c3bf785eb1df, - 0xc53f02b3e8c38647, 0xcdfeb25ee787759d, 0xead5e734d64ab5f6, 0x7930d87af1072499, - 0xf30690a71d88ad6e, 0x73c347923c84728a, 0x3f2b588221003fe4, 0xe747052d0b453af2, - 0xabe6fa70539b5edf, 0x4db6d1530d628c2, 0x4ec929af434eb1b0, 0x15afbe39886181ff, - 0xa9141b9c89a07b80, 0x7d33f966c6232057, 0x8ddcb412f34a491d, 0xa74472b8ecc1e2f2, - 0x34d745de1cb7de2e, 0x6cf67091309e5e93, 0xfb25004efa59450a, 0x3355947066522286, - 0x5a8cffdf079dac21, 0x419445d6e6825887, 0x6e9c064f84381dcf, 0xbbcaf462a3a8ad76, - 0x75836c68d6c1a13e, 0xf38141565d5c3759, 0x8c65989142ffa802, 0x106067ec26e6463, - 0xadfac5e3a80de9c2, 0xc48b16e2df25b9f2, 0xa3257889c33669e6, 0x5bc760d4d65a1745, - 0x303cd31fead81139, 0xfb97f78cade31e1d, 0xb888b8e05820a469, 0x7ebb8e44d47f54d9, - 0xce76cdbb5ecdc529, 0xb1eb29949a099d52, 0xb6affc1b240a7eb3, 0x22977eac542906f1, - 0x9b105b391ff729df, 0x83371186b2834968, 0xd5b893f382ea9e90, 0x5d17aa80a1fd4854, - 0xe8ed8525eb29210c, 0xc789f3cd36c4dae3, 0x556e50a5f46b73fb, 0xef1d129b523dff77, - 0x851de0f53f6707f9, 0x8deeeadfb1fa8bfc, 0x3d8c89c0e08c4f2e, 0xecfaaea537123333, - 0x5bc9053d2dfd7669, 0x408c5bb2e880a9a2, 0x495726b3f3248219, 0x2b23cca4a6ea1ccc, - 0x1df3663045092d61, 0xaf977a46e965e45b, 0x43a2facfff7f97e, 0x9b7714344c7b51e, - 0x35643b24efb0559a, 0x502820785dc1af13, 0xbf82d2775b46433d, 0x1db626f2e16ca66, - 0x744b031447c1e27d, 0x99e79898612f4606, 0xda02a728d234821f, 0xcf00c6fbb637a6e9, - 0x242f2963196fd8b, 0x7aed8efc2dd562bb, 0x6204fb5d3dc6208a, 0x3e84861182fc7f6, - 0xd14c4ee5aeef5c7a, 0x3749fbef94378dc1, 0x8fe710ec5cfc8566, 0xf43d7e495d5384d7, - 0x8ff6396f1f1ce7c4, 0xf1252a6b6f86b42c, 0xddbbd098d6dca83f, 0x4e228724a227232a, - 0x92a5a52ba2b24fa8, 0xdfc172b03fde669c, 0x34ee55adf7f0711c, 0x21d181e79b8000bc, - 0xc788b1f48b37b693, 0x544fc4cfed0e0f92, 0xafca0c6de41789cc, 0xbb37bb5107ef97f8, - 0xb9d62bf1dc0f6c95, 0xc78b5a36110dfb1, 0x1d615b658f39657e, 0x2bb2cd04cabcc360, - 0xe563488ece6362f0, 0x213b56ce006fecc6, 0xc38207089fed0270, 0xa33199ff4a51d095, - 0x1802ad28fb1896b1, 0xbead8f18c164a332, 0xceb5149101aa450f, 0x39ad89851ee8b62a, - 0x317229aafabf37c4, 0xee68b8b9bf3520b3, 0xe4db499288350f04, 0xf8ea27feddb0ae7a, - 0xa17235067b489c42, 0xbf4a570245f95d78, 0x8065c67e1d1537ab, 0xbd9357fb1b30aee5, - 0xc224166ebeb24c42, 0xf9baf8ccd01b53bf, 0x5c13775c3fea8038, 0x4ea66f6d650ce62d, - 0x470592ed81c140f2, 0xc2d0eb6f7999321f, 0x85d762f20290dc0c, 0x9f7d0d13936f6e78, - 0x41f1fd2d20f2d62d, 0x891cb19ce1af2c2f, 0xe7ff34c3b29c3719, 0x246743f43126c69c, - 0xea4b2da3195ebab9, 0x4831e4de995187dc, 0x7fb8969bbee45ce5, 0xe35b483da73c44ed, - 0xf89158ca9af36227, 0x9fc7f34a35469a7a, 0xd02483ebca6564e7, 0xca00da156aaeda03, - 0x303d1514646822f1, 0x226832ae582b8eac, 0xf772d719e413504e, 0x87603b928c068ab1, - 0x4dc1552230e9b883, 0xef8c5e9db946fc87, 0x935581290bf7a4ee, 0xca632d2c7674bf2, - 0xd8a3933b80d39efd, 0xf026574d0ffee6fb, 0xe4412d0dcd2fe94f, 0x668916490a2983ec, - 0x73b1fe84a995718, 0x729bedefe21cc0e7, 0xd3a770f1c683b98f, 0x5d597a96323a10c2, - 0xfbb7834bbf5fed23, 0xf3546c805a42ccdd, 0x9ef3e2164bb31a0c, 0x388363ce6c6c2253, - 0x8120f4a949f017cb, 0x925a61942bbd3d10, 0xa03182d8599c0521, 0x2412e23004b40ebb, - 0x35010a126bf2aecc, 0x21147869a1a84ca7, 0x53cba503b6127b98, 0x10c89dd62ab3591c, - 0xf4c7f84faaf9f5f1, 0x8b4a37a2e844b97b, 0x23ddeb236a0bd9af, 0x4fd51d7207f49e62, - 0x6cdab447c27706b2, 0x9e8f54b9a2d1a790, 0x191aed85d4d77087, 0xf74ecf5015265af6, - 0x45925e25404922a1, 0xcf5467a0f5b42b98, 0x73590809c85c728c, 0xc16beeda74a1a1b8, - 0xc3bbe7999803dd6a, 0x1a368bb32eec184, 0xafad2d86b7bb574f, 0xdc7c7b8960dc921a, - 0xd9b68d854f5e0ae1, 0xe9e1a6a16efe0bea, 0x304a15bd6ca1cb14, 0x713ce3144e0af4b9, - 0xc50eb410981be1d2, 0xb0fba6119bf7a300, 0x7a107296731fd314, 0xdd764898a90042b3, - 0x8a69973262da3bc8, 0x29c6b9d048596c44, 0x62581bd20da76f1b, 0x4d1a3941d6d3e4bb, - 0x19c447306245055, 0xb2978afcd04ba357, 0x7c01cdefcfe24432, 0xc4b268314411deae, - 0x5ba56d49da714765, 0x33299186ac6dfd09, 0xede087aec096ef0d, 0xf758da2c7bcf9ddb, - 0x5ea6c40d56824cdd, 0x121ff879d6ba905b, 0xb5fed0c42b616f5c, 0x21029cdc347de152, - 0xb251d93f4cd7bf4a, 0xaedcb2dc6402cf13, 0x840e5e0d96e89407, 0x6a92fd328efcc6ef, - 0xc63f5d8f6fcadcbd, 0x405bd64d1621128e, 0xe1318888172f58ed, 0x1009c9764d49da2b, - 0x4bc0592cbfdf9f91, 0x972f0e080dfecd02, 0xa1cb961958eeb6aa, 0x6ee6467ad8c20aca, - 0xb3f1c738390d6a83, 0x6504eb7ac498650d, 0xdf7dda67f198f59f, 0x72615652e56a82c3, - 0x85e0fa2dfb51755a, 0x98b1f92a3d2ad940, 0xce81d51d875c0045, 0x437004d0be0a4d69, - 0x64065895526f896c, 0xe1e1fea920785d49, 0x7d507ffd56fda19a, 0x17309b625cecb42, - 0x67b6d83f0fd0f572, 0x5178665a5bcd38f4, 0x3c49fda2d35a8606, 0x7f058d2cb0ad351c, - 0xfb95691559245416, 0xc991b857662b1b9f, 0x9e6d0f4e19774f96, 0x26cc7502212ca578, - 0x3466110f03225e49, 0x2ae4958375eab9f3, 0x939a8d94c8871191, 0xa27356548ac6b28d, - 0xacf86d43ec3aa030, 0xe0d16c7fa0b13a8c, 0x408ec2b2f8da531b, 0xde72494115ee4e83, - 0xd26d7f79a02c5b1e, 0x2b6f835520c97f6a, 0x1f30f008b109ae7d, 0x698dbf9acaa222f1, - 0xbd55de40838376d5, 0xebe53822cec7eb80, 0x7ce900793008d2bc, 0x494fc7d10e8331bb, - 0x53509b90bdb7d588, 0x62aa920e9554b2f2, 0xe103098542011b6f, 0xeb722b9523d68af8, - 0xb71b1a6ea2c6591b, 0x97cd7da55c940270, 0x8ab70184427e45dc, 0x6cb6907427808465, - 0xf69232a42dbe7475, 0xbc9816d429fa8909, 0xef1e74244539d41b, 0x5569b6d10440d5c, - 0x7dda6817985c8ef5, 0xb270ed19cc8161b3, 0x87d80b66c8a15db0, 0x96966684c3ba47fb, - 0x996669bc87aff3bf, 0x17c015383c793f2, 0xa4b5de41fc69f61a, 0x14a0eb4e3055742f, - 0x5f0da5a7a2b8bc79, 0x5fe0353728aac023, 0x1554daf4abe92eed, 0x545722dac774b6a4, - 0x733fc54174f2e0d1, 0x3478ef85dd994316, 0x58ba2ac090ef7575, 0xa66b9b0cbc77b6e2, - 0xdc78cab5c708a3ee, 0x97a30c27be510f61, 0xb95d0b06fc0910b6, 0xdc80bdc42f79a25f, - 0x5cadc438e65b3070, 0x6263df49ce691b9c, 0xa7ce160daa64416b, 0xa4f8bfedb57288c1, - 0xa51714e187bbcfe9, 0xe25df4e9fc44644c, 0xeaf1854b3116ce11, 0xde1b8f810991a604, - 0xd4fc2e365e99be4b, 0x8d1b0d2799527e06, 0x7cac59eaf46baba, 0x4fa63c74d2dabaf1, - 0x4f6c5d5e676733a6, 0x7ab7ddb9c1b789b7, 0x5d9beb4877a37034, 0x5e96bc9de3985bcb, - 0x72bab4dcb75b3228, 0xfa40f33c4d799e1f, 0x73e6f61a69984a6b, 0x7499c9af466cf22f, - 0x42fab9136bfa64dd, 0xd5e8e39513b6fffd, 0x8eda1fd5ad8cd51d, 0x95338744859dff44, - 0x4f0c5e5ae768c729, 0x5bc92c60495ae348, 0xcbb48c170ac21168, 0x8374aa2440eeb138, - 0x70b663d6f6d70ca9, 0x11264ca6dd79e5a0, 0xf058a2c156974514, 0x36820eefc435ba63, - 0xd7b69f3d0c0c27a1, 0xe1a2eddf3b41d205, 0x80508ec93b038bc8, 0xd7e0429bd511ff37, - 0x7bf55a4e87e183b8, 0x4cb370ce7edb4bea, 0x26fbd0b31dcef45b, 0xf7acbd6781419fa6, - 0xf7849659f05c90c, 0xb686271ea57a47c6, 0x16f3f839dbfb4e1b, 0x906872b08b2c61a, - 0xc30c86d0a0203c15, 0xdaf238a6aa4fc9f7, 0x2399aa09ad2c069a, 0xf133c3aca703f545, - 0x868a10304a1c98ba, 0x60ef0607f46f7e90, 0xe4e69f26931e11a5, 0x487b8f6bd6d92941, - 0xd10cb2971798c0c7, 0x7126d81aa4bd0106, 0xcb620311dc84ab82, 0x26f8734a7a356bb2, - 0xcf9b9eeb02ee978c, 0x8a9ff0285d4d6b30, 0xe30e3b957e2cc7f8, 0x7c15a09e4d275809, - 0xdf723ae1dda5d167, 0xac212e4f6264bba2, 0xe3d60920ed983308, 0x91b403cbc91e290c, - 0xcdb905b012aff7c7, 0xb5ee73d45f900897, 0xafacc7cd7f5d52e7, 0xa8653272621165d6, - 0x90e2efc485ddd0d1, 0x56ef1ca9097b1a96, 0xc7a20f85777eb0a, 0xf4cec0271a50eae9, - 0x21acd76442024973, 0x19a46be82a4abdbe, 0x1bc12e0b8bf41fbb, 0x3766fe3d8e5119f2, - 0x7fea355c4c18e8ad, 0xd08b496a24fb8017, 0xe1a7cfaa0877aa2, 0xd37ad9e2a2a3fa96, - 0xaa362ba0e696f679, 0x7e7de89142c3aca5, 0x2dedb0842a3575b8, 0xb74c1e1d9082fe5b, - 0xb1ae74323699140f, 0x73623f80c727a6ea, 0x132ed204b0f10441, 0xc3e6ebe8ffc252bf, - 0x5f15cffb8286dce0, 0x66dab32df780fa8f, 0xeb00da7b25ea99e4, 0x113ad2448fafb671, - 0xee065c10a8f1924f, 0x2fcb7367cc01fe5, 0x484338f5c2d0aacd, 0xecfacd785e42d7a6, - 0x11513f845de8af3a, 0xd11be6b29054de0d, 0x2536e5d2856af9b7, 0x60ab519760acd4c4, - 0x6bfe010250a831ac, 0xb28a93e44b53b21b, 0x281cf9b233858583, 0x4ca6139abc79a710, - 0x5717d33616d77a95, 0x9ba52d2b7dfe71b5, 0x32e1c543476aa17d, 0xae242cc75806b7fa, - 0xa1415cb8fde770da, 0x3956c67542dc004d, 0x3a6f51518fdd20ce, 0x448c848f6c936d93, - 0x8fec38ff51bb5fab, 0x7463816cfc0754ac, 0x83b38e531ba39e73, 0xf9fc84dcf4f8c93e, - 0xdf20dbe8b91c3d6f, 0xec65939ac9516f9a, 0x888346f6c1aaa94a, 0xe42cabb108f60d95, - 0x39bb2e46b0599fa9, 0x529335ed75acba9e, 0x6a8767c5d00776c4, 0x8243346104fe61a5, - 0x7a2bd0339b3e9bac, 0xb68ccdf14473f4ba, 0xa06f389531ab553a, 0xc7c6f074fc2882d3, - 0x50a5fd6e6d0df962, 0xd7c0000d194139b7, 0x5ae27ef4033f873d, 0x4e7abe8a6d3570f8, - 0x27011ccd3885e709, 0x3dae53f7b7a8924a, 0xa9086c9b2b86fb71, 0xa3f9e534a399e62c, - 0x9f2f0379f9a33ec6, 0xceb51af95d4472bd, 0x15aa534182f8465, 0x96373b9cd28a627b, - 0x9fdce0ad99d41907, 0x2755bb0f52b8c239, 0x2f2e241b6aa7d243, 0xf040afbacb2f6001, - 0x552267c5f8b4c1b0, 0x22bfb3f0b58f9e48, 0x6bff8de368dbee3a, 0x652025c63d4069ce, - 0x743eb697b25f9c90, 0x8a3742c9dbf67b1c, 0xaf3bcfc260d7e69, 0x491facaa59b7e1d, - 0xd07d0761e6535fcd, 0x79ef09d9d3859232, 0xe0e0a013317d9207, 0x4b94baecf6fe4b4e, - 0x574576ed054cfc38, 0x90e90edd1a26f0aa, 0x616d32a371af78c6, 0x392cea9c34ffb0a8, - 0x692a6e730c33ac6f, 0x9e4b92ef425b78a6, 0x291d4c962d2f3e7c, 0x5f0f8ebb67f308fc, - 0xc1a0faf70ec747a3, 0x641da9550cd89392, 0x6adbe38115f09648, 0x3a51980fb324da0d, - 0xee894f2b5c17a380, 0x58788fa693f767c9, 0xc9f490ff5d88c4c0, 0x2ca6e8b204aa3070, - 0x7693837d7910c40c, 0x9240e04f16051720, 0xab773072d922faa2, 0xacb6892db8362872, - 0x5ac8b4d130613c0e, 0x65cd5ac259c653cb, 0x9647e0276864f3f8, 0xe207c57ee9237a53, - 0xe41f2663482a8fdb, 0xf605931fce8b95bd, 0x5b0247a9009255d2, 0xae949df3ab5a4ab5, - 0xc0e912dacbb72c42, 0x4226971e6351f33a, 0xd98d41f9f0fdb6c3, 0x64bf6a0af66782d5, - 0xb4ca689b3959bf46, 0xb4cbf621b4970922, 0xe9f27469c2412fe6, 0xb529d406d27c785e, - 0xf33be21f2672ec78, 0x34b3c4c4603656cd, 0x4a7efe099a0ae9d1, 0xa6ad5b909667945b, - 0xe20850e47ab440f7, 0x68739e1b4fa069e5, 0x4c191300a9207cf4, 0x9b613a1a38160c98, - 0x2cc631eb015081c9, 0x52af80aad7b676f4, 0x904b38943300ca8f, 0x50a515c0d302620f, - 0x52ae95b8a0c1f36c, 0x6a62ab87786774e2, 0x17ea45367ad58985, 0x92959c57166aa21b, - 0x2a9d91bd1a9716b4, 0x552c2f8528220174, 0xd665856c96b47d17, 0x14c6cba13b6dca3b, - 0x865f1c93ec9000d0, 0x23dda8b5e161910e, 0xc191ac4342717953, 0x6783fdb95e098f1d, - 0x1a4c26a5cad1e8ee, 0xde96e2a1e33e3046, 0xa57e65bedcf9047e, 0xa1cdef163fe7f80c, - 0xeb0abcc13feeb7b2, 0xaa158c61b0e44470, 0x98dc901679caf85b, 0x954046758f8d2e96, - 0x56db5e99c5d8c68, 0x7bb8d36962a4ac81, 0x2e301b4ecb03821, 0x121c1b2df70f0e1c, - 0x3fbdad1e8faa0543, 0x6efb222398a2f88b, 0x65760de4d96338e0, 0x15b2c58a67fb43fa, - 0x22f1532c89367d77, 0x1f726ffdad411d9e, 0x42572b54dcedf3ba, 0x3f8e0f6e9f0bbb7b, - 0x4b0e705c86571a1c, 0xf8c9b04f8bd75117, 0x67ed2e9e4545557, 0x57e8853f681bbf8c, - 0x6f99d1fd5fdbf582, 0xd2aa9bd48ad692e2, 0x2efc88bddecfe616, 0x8e9779a1e119abf4, - 0x52dfee4722a20b75, 0x79465be3aea146d6, 0x588997dbcbd5f005, 0x79bda8bcc5d4c650, - 0x2384e131ed3b5330, 0x229cb1d89738aa26, 0x1526d1a020a96507, 0xc7b6ac961a740cb9, - 0xe78cec14478c4f71, 0xaba61cfd8f1e2a71, 0x24123c8a37ae66f3, 0xbad8b07709aa215d, - 0x7896fa53fd2418e7, 0x72265842e8c4f955, 0x9f6331fd80527661, 0x7c649eeddb382c9d, - 0xd3b0708dd6b5ae84, 0xeda51f244551e15, 0xb7822c860b93dd44, 0xdf9afcc3c9cac88f, - 0x9244b9816573be70, 0xf3d103887cc4ce2d, 0xb3295c2e5bcb0218, 0x85243b7a2e0af441, - 0xbffd3df508d06098, 0x4908b967f6765c12, 0x8886ac94c0dabb, 0x9c7865af133eb6c0, - 0x5946fee66e64d7b2, 0x7737bc5af713087f, 0xbec815a80782dd5d, 0xe7672cfe0f7945fb, - 0xe1f80b6df5beece1, 0x5b749d3a22450fd6, 0xaf34a567bf838668, 0xa72a177d20943ceb, - 0x257c45c1601c4922, 0x3d7c68f6fe36ce59, 0x1b8ef47186e06c96, 0x8040426656154c17, - 0xc15f74f980647bda, 0x36389393336a78be, 0x15d174ed5536afe2, 0x51e702e8adb61f63, - 0xc9c95ca3f1c08f30, 0x6653094c8531c93a, 0xf7be5dfc2eb3b5b, 0xb5e21ec5c63850b0, - 0x9ab5f082e01021a5, 0x75c4ca38d7678fb1, 0xb20bd13e05e5bd67, 0x3378133e631fbaee, - 0x17cb3686511aeb6e, 0x4c0dea4c80ec7bc, 0x21fe874bf5509300, 0xece4f84e34d52e54, - 0x3f2649803ca9918b, 0xdb493adeac5c60f5, 0x7c02a1d153afedd, 0x2d08ceda0fef7967, - 0x6a32e018e432b0ea, 0x86479f3ea38ad57e, 0x84e3e04f1e5877be, 0x41b898cb02772049, - 0xd6dded3714d71084, 0x9df5b2f3a0ab275d, 0xd5ab652dfa73ee0a, 0x4633e03fa37d6edb, - 0x226314b8c4b937ff, 0x45d66a00ab031188, 0xb93ce1cfbcd1eaa0, 0xf88e0756f7e7c1c1, - 0xf3611966fed51e03, 0x81235048c662d9f7, 0xcc8275f866147b4f, 0xd3b3ca0f5033a863, - 0x970212eb8c2b3429, 0xec848dd58f3449d6, 0xa1d527af824d09f2, 0x60bd5e91448b9cf4, - 0x1210ddfac603aa88, 0xcbf4270f3407e25a, 0x212955fec55466a0, 0x3afeaef4c9ccc793, - 0xdd114286ec304817, 0x849e6ae3c2cf794f, 0x71c08228d6a05310, 0x177e77779d155b11, - 0xdc59148f219a9c04, 0xb0702a7802d5276d, 0x56085d6761aee015, 0x3f79ce06bcfb4f3, - 0x459a1f917f8f3e0, 0xe4ea5635e8bcd512, 0xa88e99b63f135a39, 0x95bd628d77d39446, - 0xe6432158ef4c7d98, 0xb349cd3d1c74369e, 0x25b1a32db58efb0a, 0xc2add3a44cecc0b5, - 0x5d676629f23010b5, 0x9890b3a62599408e, 0xef68ea8144d97805, 0x429e0fda34046a85, - 0x7723d9043053bf52, 0xbb78842d9b67ae91, 0x9155ef932192e6a3, 0xdb523ea403d39f6b, - 0xdb8fa1eee23ea58, 0x7c735492524a3448, 0xc580cb82e81505, 0xb0be6a006414841c, - 0xdec2e763b5cedaa5, 0x8da58bb23638af5f, 0xb0e6b33f6736e7d0, 0x8146bbcd3dd4df61, - 0x978080148a8989bb, 0x7f8119caa3308095, 0x4e88c318ea0604f3, 0x6ee5262f16b3cf83, - 0xc44395c7a578ace9, 0x92016ee635de27e1, 0xb8dc5ceb36e67fd2, 0x95c851d65bd35f8c, - 0xbe393620503c49fa, 0x42af183b92eac923, 0xfef0e660435135ce, 0x262d67d480451ed1, - 0x590f92e1ee0502b3, 0x18825c97d49e3700, 0x2cd8c30a848c9acf, 0x1025c7747fda0115, - 0x54109f23e82590aa, 0x93917bf1f8325981, 0xda674b183fa0e3cf, 0x2a0b2467eb8aefc5, - 0xb0085eb83468793c, 0x607cabb2c9d3a81b, 0xe7a6b6013804d665, 0x67629a769c1efede, - 0x2830ab6ef6d10166, 0xffd02b0655332bd4, 0x19bd056c3117568f, 0x385a834785662c6f, - 0x938b5d56bd5f7248, 0x969afe82dd4829c8, 0xf455d4ace41c797d, 0x23d9cd67eff27512, - 0x2b0c7037ecb322e2, 0x73df193328258bda, 0x5d7ee05cf2054f93, 0xdabfbd5b46b61cea, - 0xcea02d82546b96de, 0x2245d5e74d5f60ae, 0x842ca45f8ef2a44, 0x7505cc4e1c3060d8, - 0x869146ac8e68565e, 0x22ea711fb30e73e3, 0x53cd64736898a0c0, 0xfa88458b920684df, - 0xc3ae23f451e0616f, 0xe2dd69393141ff32, 0x98863ab129bcd866, 0x8c9756a40dd5b834, - 0x2eeeef78c36fede5, 0xe84d2eb23e22153b, 0xb0ccc2f7ac541d78, 0x151faba0f513acfa, - 0x4300e3cea0260717, 0xaba308c6d857d2d0, 0x5eb25dd325256c6a, 0x3342627b68038da4, - 0x70d7d75526da35, 0x80ff3f5ea2ac3cf1, 0xe7434f2f026394b0, 0xa7c5ff17f7d2cb07, - 0xdfed0bb33a06ff78, 0x485cc64d38ce2596, 0x88db8580147aa8fc, 0xf52a5111d693b973, - 0xeeaa031c02b370a8, 0xbb3b1678222d0e81, 0x27b569f2a6630939, 0x2b301fa3e50efeb8, - 0x4dbc1f85bf3f8972, 0xb37e4f2cb75a825b, 0x3c8848e0f1777cc7, 0x8fae2e6938ba00aa, - 0xfb4674b50884992c, 0x2b765005b85b7388, 0xaaf0007fb9662ce5, 0x684bd59009a4ad65, - 0x221ffead3d73ab35, 0xccfe6d02d46856b5, 0x54d3323359e1b114, 0xcb412202ed42f097, - 0x15d63df422771b9b, 0x71852bca1581d14f, 0xf30dcbb6cf891e63, 0x478fb1eed3cc5a10, - 0x849a3b52bc5bb196, 0x4c1a98dbc546dd81, 0x846dc8c2258ec4f8, 0xbd4da447c7340bd0, - 0x8f1ee1d6a85b9db0, 0x123ebfa8aaec07f2, 0xae34948e375d4477, 0xd466a4177842d8e4, - 0xd5108efeb19cac6, 0x3266f7db8f133bdb, 0xe69af4e5d8d767e4, 0xea0efc0331df64a2, - 0xb879052746ed72ed, 0x2c8233cc84de4144, 0xdcd5dda825186731, 0xb9e3b679d268f34c, - 0x7ce12e2fa95bda8b, 0xa34afd12e611a4c1, 0x6043d06ee7619f90, 0xca3a3d4813c0addf, - 0x6e97a61d1e4b3c4f, 0x8cdeae467a4bb292, 0xf08a6c69e70076b5, 0x97aa5c3180d3edbe, - 0xc39813e1573904d5, 0x42577549d026e8c8, 0xaf5827ffe259b62a, 0x9e1d48596c4f0b24, - 0xab9dd230ba8efb64, 0x81769493b85868d4, 0x715b45c5e3952245, 0x84735e138d228f35, - 0xf4f987da7d19c74d, 0x1bdc77979baf29b, 0x785b3640158e0278, 0x77fc9d00681bcdd, - 0xb5980cc490dcab93, 0x9062c158196b8244, 0xc16af5418cc97c4a, 0xdd4a4e9e8e00e524, - 0x870b554b629277f1, 0xf90ff72bb54322c0, 0xd4c273bc2199823, 0xeebc75970d438466, - 0xe4cad67413074a53, 0x6eccde71bd09dc0c, 0x14278b0ca6b910b4, 0x6e8895f17cdb933c, - 0xa0b3987821416e11, 0x71b7d24b81fa769f, 0x1d3b1a805b885a58, 0x1bc737b1719736a, - 0xea4d1dbb8823037, 0xe50ce48c8469adbd, 0x34c2d5e6c41a888e, 0x446a756eb06dc3a4, - 0xbac5ed8a8f90262, 0x7f1b76e0c707ab9d, 0xb31323309b94a12e, 0xf58269b9852f986e, - 0x3b74c2b338c244fc, 0x879b46f23a4deae4, 0x3f2591e34cdce1c9, 0x73c6f81eb560ed5c, - 0xd2aa923c7a5c18a9, 0x7170c1f7621cace9, 0xa18b327b11b951a, 0x2b36315510f56370, - 0x5cfed2f703dfc0bb, 0x1e43b99175054c07, 0x392dfa3210e8013b, 0x65e0c5c0454ee693, - 0x2a795f6f493349db, 0x17995ebdfe848db0, 0xf23174b823a52cf7, 0xaac6ef104bfd396e, - 0xfa0f5b29156eeccd, 0x9ccb3590a6e88c1a, 0x2edae0b1b80aafab, 0x1e1e92baae39aa30, - 0x24475af89cec7f33, 0xacdbeceaadb9936c, 0x7725948da8586e93, 0xcc05595e17947215, - 0x1f3cbde17a508faa, 0x2795f58ff5c8919b, 0x309658d1748f30d3, 0xe90c11962e5ed4bb, - 0xc22b876286d32e27, 0x495c8b667c6ea3dd, 0x84263045f7d6eab5, 0xf8d3a9ab1494a315, - 0xabc42cf769a21d9d, 0x3fef7fb40bdf3a81, 0xfd35336e188925a3, 0xd472bdaf277ab26d, - 0x4949e8ff51da2307, 0xbec86dce960ff3b, 0x1aa1ce4f70256b10, 0xd52986ff8cd700ea, - 0x364d8efc0f5afad6, 0xca3d1958f57a9050, 0x17a0a2ec122dc677, 0x7992695be3363fd6, - 0x5c66a265e0607da8, 0x7250114e050bb917, 0x30cd3a70bc7b723d, 0x7b77433392b3fbf8, - 0x295bb7bf46318b38, 0xdb15025af2a71c65, 0x26a82b21ef67f50c, 0x14a5573d4fc798c1, - 0xd8cad4642ec68e5e, 0x9276d7f60142822c, 0xf7b8efc27522e52f, 0xd0a3f36f6d340bed, - 0x341260fb11765c02, 0xcd1d394d796702cc, 0x7d483ef031eb3346, 0x74eaab49374576c1, - 0xa073bea32a71a273, 0xd65ce2552993b5cf, 0x8670afe77caaeb16, 0xb607f1455d072a47, - 0x30ed96be92809559, 0xa58380a503c23c9c, 0x916aa68fb957a30d, 0x30c5a675bf19738c, - 0xd4cbad34e4e4b886, 0xd6cb83061f2b0ebf, 0xceddb4040f535fa9, 0x778c586927b1e247, - 0xe4bb5c4b6e0f3c3d, 0x3e857d671db80667, 0x2b909dd8725f1fa2, 0x558ffd0772db7841, - 0x710f9638d3edb2c4, 0x21a4ccee53d46556, 0xf76e8e4737b9628b, 0x71cd157f23581c71, - 0x68d8fded9b66efd6, 0x7d9f5e182c0b9457, 0x2140757748a217ff, 0xdd1e5365520b77a4, - 0x644d8e4b2f30dcfa, 0xa1de42f4e9791564, 0x70e148ababfe9f86, 0xb97f463e0ac7daec, - 0x82844f729d9fa554, 0x5c8475e84470c924, 0x3a83de748eac32fd, 0x68725fbc9c202c5b, -}; - -pub const mt32_seed = 0x7dc0d160; -pub const mt32_data = []u32 { - 0x59327332, 0x200858fa, 0xab53c028, 0x5c442427, - 0xd8be0287, 0x3b69d304, 0x15fdf62f, 0x59b8ecd, - 0x6d7ab30c, 0x3a3dd6f1, 0xc1b9773e, 0xa12fb017, - 0xa805b5c1, 0x4a313ba5, 0xd82c790c, 0x8de311f2, - 0xe7cb23dd, 0x784b2efb, 0x9743487c, 0x73e2f2fb, - 0x1a7ac286, 0xaef90d, 0x6c0a4514, 0xae1d83aa, - 0x412fcca1, 0x3acd2d28, 0xde78292f, 0x13237756, - 0xdd6cdeba, 0x44ae4df9, 0x3e9902eb, 0x39e1cf20, - 0x62f561b9, 0x6cbdf531, 0x4a000673, 0xb1c82daa, - 0x896156ca, 0x75e410f2, 0x9c69e72c, 0x396b42bb, - 0x25c97ec0, 0xe12173f1, 0x8dcd42e5, 0x82aac3e3, - 0xcdc1c84d, 0x13509c0c, 0x46a696d2, 0xb89ad987, - 0x92da5e7f, 0xaa87d8a9, 0xe433ff57, 0x80a7ee49, - 0xd387cfcc, 0x7dc47d92, 0x21516140, 0x989ca465, - 0xf2a8e002, 0x73b99ddb, 0x2204108f, 0x27e84890, - 0x371c81c0, 0x9f581854, 0xc0841c35, 0x770f6804, - 0x87c55f4e, 0xd29516bb, 0x2b6d6bde, 0x5541f6ee, - 0x1ec9b182, 0x7d599729, 0x4f4b9a14, 0x6f8c8562, - 0x2d5151aa, 0xb54f5bc, 0xa252452b, 0x2a4266da, - 0x25a6b75d, 0x2d11106e, 0xc5d77943, 0xb10b6e0b, - 0xeb5cae4a, 0x43a0dd53, 0xa40bea1f, 0x63e632c2, - 0xf420b6ce, 0x8b080233, 0x7f70ae87, 0xf460f0d6, - 0x147c7e74, 0x710692ea, 0xa0a7d8fb, 0xe7f05808, - 0xa6173aaf, 0xae608de0, 0x8702036, 0xbf1bfc7b, - 0xf14cd548, 0xbbc7553d, 0x5358dd1d, 0xcc0c1fe5, - 0xfab6f78d, 0x9365c118, 0xf64216a3, 0xb4bdcf1b, - 0xc90b8a7a, 0x8b7b78a, 0x4c7b6854, 0xba7b5628, - 0xdd728c15, 0xcb1f8905, 0xa63e2342, 0xa78822, - 0xbda61b18, 0x160a59fa, 0xeccf473b, 0xc5a445b5, - 0x7aa86430, 0x362e0d7c, 0x8006a0cb, 0x8b11586f, - 0x6677bba9, 0x6208cf27, 0xeec9b5, 0x3dfedfc9, - 0x886cc0e8, 0x32ed77ca, 0x43525faf, 0x9786354a, - 0x1a2eb378, 0xf0e6b168, 0x49064b09, 0x8ab39681, - 0x7b6655fc, 0x35adb168, 0xc417d430, 0x2784288a, - 0xea17836, 0xc85006e7, 0x673dfdc3, 0x42765688, - 0xc2b9251, 0x840a45b, 0xcac98e2f, 0x1a6f9777, - 0x34959b23, 0xf0dcec81, 0xcaa2c6c8, 0x1cf93061, - 0x787e598d, 0xd5d9e31e, 0x14e08791, 0xd9d9d782, - 0xef162f23, 0x238f4113, 0x23f42107, 0x6ed5cc3f, - 0xa55e5a7c, 0x4650595, 0x5217da8b, 0x6eeaacdc, - 0xb453d7b1, 0xfa1ff004, 0xb9d17f74, 0x2bd6a53e, - 0xe4c2d9dd, 0xed66375e, 0xf8215568, 0x9bcadbb3, - 0x9c4f9d51, 0xff68312, 0x82308422, 0x83e990b0, - 0x38b6135b, 0x70e2aa13, 0xa30065d2, 0x6396a00, - 0x77d423bc, 0xa93abf0a, 0xc7bb8c31, 0x5d7bd3d3, - 0x6a374f2e, 0xe4b5bc88, 0x39f6e512, 0xd6aea995, - 0x878c1bfa, 0x4636014d, 0x9caa2c09, 0x7ac4758b, - 0xbd3b957e, 0x518c2fd6, 0xea009a2e, 0x542bf419, - 0x59090006, 0xb1d94703, 0xe0d9eefc, 0xe7fccb17, - 0x40111951, 0xf2560485, 0xb50ce9e1, 0xd7a1ee51, - 0x28dffa99, 0x41d12275, 0xdd89a365, 0xf22eda29, - 0x104f94ee, 0xe669983b, 0x6346a250, 0x86326fc5, - 0xb7f347df, 0x3849a39f, 0xf433929a, 0xeea5155, - 0x4cf9b778, 0x6bd7926a, 0xcda9496, 0xf430d7a2, - 0x41637670, 0xaf3bbad6, 0xeb66e44e, 0x2499605d, - 0x9988920d, 0xf9d652ef, 0x67aa80c0, 0x505073c9, - 0x85cd418f, 0x9f83fb65, 0xf50b3eac, 0x812ba6bd, - 0x74d61788, 0x86d64f3b, 0xb1f8fc1c, 0x3e2af667, - 0x4d118a2, 0xd028ffa9, 0x32e88a44, 0x4ed9ba35, - 0xea3c7030, 0xffe44aaf, 0x5e39c467, 0xeabcfebb, - 0x53e656ec, 0xced701d0, 0x31020b02, 0x4b4c1dc5, - 0x8744885c, 0xa8e93656, 0x3ef457e5, 0x272bde23, - 0xe541477c, 0x3ad3ac04, 0x63eaa692, 0x81055cf9, - 0x3ff5f782, 0xa8efe6bc, 0x15f37656, 0xaaaebf1d, - 0xf73d461a, 0xe8b2c0b5, 0x5035ff48, 0x3a95e34b, - 0x6f21d94f, 0x6f6d1f96, 0xdaf79f37, 0x826f69f3, - 0x209a00b8, 0x2ad1b2f2, 0x2c64fb45, 0xcf8bf26e, - 0x9befcff2, 0xc08f6951, 0x96d98205, 0xa267dcb5, - 0xbc43ec5, 0xee6a7e1c, 0x49224eae, 0x14e820e, - 0xbb340212, 0x68ed572c, 0x45e9e623, 0x1297f3af, - 0x49a98ed2, 0xddd34ae8, 0x211838ab, 0x47e7652d, - 0xb40430c6, 0xc8d3bd7, 0x4352356e, 0xf0e5cac9, - 0x21880df4, 0xc16b343a, 0xd9ed7350, 0x17fe1f65, - 0x6637192e, 0xd81c93aa, 0x7d6e17d2, 0xd407b13f, - 0x425da072, 0x380d423d, 0x6ce57b22, 0x7b17ed17, - 0x95fbf626, 0x768303d6, 0x76ab6b3e, 0x591491e3, - 0x259f79ab, 0xd4babeaf, 0x9c7de2f8, 0x4fe6cb58, - 0xf43680a9, 0x651a1266, 0x730ea3c8, 0x9188d4c5, - 0x12d01e34, 0x47afb2e9, 0xb4b76d35, 0x5e5164bc, - 0xc864fc46, 0x5d018aa7, 0x17fac975, 0x5a775fbd, - 0x40e6fa14, 0x7a00b683, 0x99e4e102, 0x2f933b90, - 0x474e14ba, 0xde1b0754, 0xe84aba2b, 0xb386cd43, - 0x17ca77c9, 0x7b4f38ef, 0x803ea1a8, 0x93553947, - 0x806c8224, 0x2608451e, 0x63157fe3, 0xaf53930e, - 0x5dfe8c16, 0x65592bda, 0x7086eb3f, 0x838e6a50, - 0xa27836d9, 0xf2f16d92, 0xdc0a981, 0xfbf8f915, - 0x2caea00d, 0x86bb3e18, 0x6d94c209, 0x3bbbeb6c, - 0x114d68f4, 0xc271e48f, 0xa3350dc1, 0xb8d55eb4, - 0x68be5ee1, 0xbf22ef29, 0xd6e0aa54, 0x48f7219, - 0x21aca253, 0xfbf07910, 0xfcdd61a8, 0x118a09b, - 0x3f2bbde6, 0x46eea63f, 0xdb51ed16, 0xf8a9fc36, - 0x31614dc0, 0xdd84f54d, 0xd2b66065, 0xdae0af99, - 0x6d071a51, 0xbdbac46c, 0x15deee25, 0xf792e64c, - 0x910194e8, 0xfc989a8f, 0x919727fd, 0x6f93a56c, - 0x2df36e9a, 0xd395b948, 0xb026b54a, 0xf0938a5, - 0xe9c64399, 0xb5cda15b, 0xb7b8dd41, 0x7146f944, - 0x8d41ce2f, 0x47c74099, 0x2e5a8e5f, 0x28f7a19c, - 0xef7a8ef9, 0x6a763eb9, 0xf13a3ec4, 0x9f352360, - 0x42317561, 0x6c6a0ca5, 0x5e40b472, 0x3ddaadd4, - 0x2f5d14eb, 0x5dd49aeb, 0xc89edb24, 0xa2da269b, - 0x5cf0a38b, 0x8e2f435c, 0x40970e54, 0xa2cb730e, - 0xf9d8c301, 0x8ef29fb1, 0xf08b1840, 0x7d45e4a2, - 0xa0fe4ce1, 0x939a21c4, 0xfdeebfea, 0x4c661550, - 0xdd304d1c, 0x3cdb078d, 0x94ae8db2, 0x4f6b4287, - 0xffe64fa8, 0x50384bb0, 0x16cf5ed3, 0xa91a8fec, - 0xdb8ebb1, 0x59c2898b, 0xd587edc9, 0xdec2e75a, - 0x496ccdd2, 0x897db91d, 0xf8ea5149, 0x6bed4bad, - 0xce76e472, 0x43c7f976, 0xb055dc01, 0x7ffd5671, - 0xe193b86a, 0xe288ce11, 0x514d531e, 0xa42fa47e, - 0xe7c0e194, 0xffc059ba, 0x26548e36, 0xe1f10d92, - 0x3ef5d95e, 0xa6e69282, 0xffacb09e, 0xf4a16ff5, - 0x9b7f03bd, 0x588c54b4, 0xc2b6eaa1, 0x2d83acdc, - 0x7fdbb606, 0x2b160650, 0x9923e57e, 0x32bd23bd, - 0x50cd6d4c, 0x205d901f, 0x810a9935, 0x27ce6e7a, - 0xe0c6c66, 0xac06c99c, 0x4326aa9b, 0xe1af1e90, - 0xe358c8b1, 0x2f601c2a, 0xefca77e7, 0x1a7ed2f8, - 0x8ad2e191, 0xe5520809, 0x27084438, 0xe4d8e782, - 0x5e8a4038, 0x87bba694, 0x65f07eba, 0x616f8f07, - 0xc5565d9, 0x555955e4, 0xf41c2caa, 0xb085fbf5, - 0xa5f9d9ff, 0x418fa0df, 0xec5a576d, 0x7fc332ab, - 0x7683ed33, 0x968ef54b, 0x834d598d, 0x6833f356, - 0x59dc7e7f, 0x779661dc, 0x58942dd4, 0x80387aab, - 0xf6dac9e5, 0xe043be04, 0x2ae4f872, 0x881f8d01, - 0x82cfd69d, 0x931f4648, 0x2a76ab31, 0xa3f1dd7c, - 0xd7f4826a, 0xe74918da, 0xe4c98636, 0x441164f, - 0x15a0e9aa, 0xce7480ad, 0xba39076b, 0x233aa8d, - 0x6c32f0e6, 0x169c62bf, 0xa2cd17f6, 0xb5590084, - 0xb2036f00, 0x18315935, 0x11e9c9c7, 0x25c77861, - 0x41596cda, 0x635e5e02, 0x8f396cc, 0x4cd00d8d, - 0xd665597e, 0x90f891ef, 0x547b93ee, 0x376959c1, - 0xdc5fa80, 0x9b4797a6, 0x53673041, 0x25ab117a, - 0x7b8b8292, 0xf4e99584, 0x5139da98, 0x30e2afeb, - 0xff2664b9, 0x591eb6f0, 0x9e87e602, 0xf5e26193, - 0x61831f07, 0xabc139f9, 0x984eda0a, 0xaea1b8da, - 0x65c7410d, 0x2b84800d, 0x1d3cfec3, 0xd05cb8a1, - 0x4529641b, 0x7d6712e6, 0xc38cbde7, 0xacad7787, - 0xd8482f3a, 0xa5662eaa, 0x24836ee9, 0xf3b5cc97, - 0x50a581ae, 0xff6004b6, 0x650fc547, 0x161898b1, - 0xa7593447, 0x325827dd, 0xf1844a1a, 0x7eb56de2, - 0x89882452, 0xfebb49a, 0xfe86ae9c, 0x7dba98b1, - 0x1d65adb5, 0xb71acffa, 0x861215af, 0xc0f1496, - 0x70967c72, 0x3803d127, 0x6c8fdd84, 0xe40991f1, - 0x1343e3a, 0xf57b4e73, 0x25f34f76, 0xaebcdee8, - 0x8752d71f, 0xfc710e54, 0x34f3af44, 0xfa7dea4e, - 0x477d4d83, 0x42640ff1, 0x2c5c31ce, 0xa82de5e4, - 0xcc813271, 0x4d40bf86, 0x4e416095, 0xb5ac332c, - 0xd2d44703, 0xe4c5ef57, 0xde193a29, 0xbf3e7974, - 0xbb313d75, 0x8dc973d5, 0x301b2657, 0x44dc5064, - 0x8c58c633, 0x83424c74, 0xb7cbf7ac, 0xa04238c2, - 0x6ceabd59, 0xd25e6fd0, 0x3409167, 0x42d6ef80, - 0x1f47c437, 0xdb21e45f, 0x2fd48e29, 0x9498cfb7, - 0xc9e4cb12, 0xc6dcf0df, 0xa1633c39, 0x1b349670, - 0xf76d4a64, 0x15ecd8dd, 0x777bb76d, 0xc46008e7, - 0x23d94e44, 0x78aa07de, 0x2eeac782, 0x3757b114, - 0x2b22de2a, 0x37726519, 0xf107546d, 0xe9847f74, - 0x449a4ea6, 0x2e31fa5a, 0xd719ea88, 0xb2115c87, - 0xfa6b7231, 0xf72fc9ff, 0xcd22bc37, 0x9080778a, - 0x93430a21, 0x97c24360, 0x6e5b1a76, 0x5e8baa7c, - 0x300c94f8, 0x2843d9da, 0xdceac0ae, 0xeeed885, - 0x1898ffd0, 0xa3bbee3c, 0xc16f8fd7, 0x82992b68, - 0x39c153b6, 0x1b3ba4c8, 0x41e7c3ac, 0xcdf8f06a, - 0xd40b8ae6, 0x4982b6c2, 0xb32f7437, 0x22ed3691, - 0x16579a2a, 0xff9de457, 0xc421e8e4, 0x17c8f6cb, - 0xa5c4a8da, 0x49bd8afa, 0xe2be081c, 0x95170f28, - 0xd679fbdf, 0xcf39d563, 0x4e2d2ee9, 0x39471096, - 0x3918bef0, 0x279b7679, 0xa5281a0f, 0x49481d6f, - 0x11f95ee1, 0xd9df649f, 0x2993eb27, 0x48ad815f, - 0x99cf306d, 0xca9457e4, 0xc27c51d2, 0xc2a838ec, - 0x537faf4c, 0x55dccddf, 0x8df5aeb8, 0xabb317ca, - 0xfc1bcf6b, 0x669c2b1b, 0x719b62d5, 0x6b9325cf, - 0xc123d0d3, 0x2ddc6ace, 0x27fdc30a, 0xd3f93cd8, - 0x704f5486, 0xd3f448ec, 0xbbd1e32c, 0x3bcd4c0b, - 0x86f8166, 0x957db888, 0x899b6a5e, 0x270dc8b7, - 0xff16222e, 0x51e139a8, 0x3d8b4b9f, 0x68d20818, - 0xa639ad00, 0x4c2e0fd2, 0xb4949cdc, 0x2ab6eb32, - 0xdd0c67ad, 0xd2208cbe, 0xcd17a0bc, 0xacc541f7, - 0xfa9e714f, 0x316d31a7, 0xed79fa91, 0xb5c0e980, - 0x412ecc9b, 0x9815753, 0xd0df1f43, 0x8e37dbb9, - 0xe640df75, 0x379c2fb6, 0xc7ed26a4, 0xc5190400, - 0x1cc81b53, 0xcb0b0cd5, 0x360f061b, 0x6d90284e, - 0x83c05bd0, 0xbd80bae9, 0xd584ef12, 0x228a46ec, - 0x657c4fbe, 0x5ca1043c, 0x852aca0f, 0x31ce950, - 0x33ee2cd8, 0x3cdecbf7, 0x787ef08c, 0xea610ee, - 0x47c1db89, 0x90eeda11, 0x74f8d429, 0x51d3a4c5, - 0x3135b401, 0x2e14783c, 0xb9af855c, 0xb66348d9, - 0xa3a47387, 0x6eb72af1, 0x7bb56088, 0xc664542d, - 0x7ed96b8, 0x995870a8, 0x385b1fd6, 0xa430680d, - 0x98a883ec, 0x2497a389, 0x7a880627, 0x8350ba9d, - 0x4cb35c33, 0x30bf6b14, 0x8695a469, 0x9a81e44b, - 0x8bb27c9a, 0xbfb6a4dd, 0xbae7cf6e, 0x4ebccc87, - 0xb712ed3d, 0x31e90365, 0xcc1fa63f, 0x32b93df6, - 0xbad4c7bc, 0xb2570e17, 0x73fa21be, 0x5c02a8d2, - 0x94446d75, 0x7265f3ad, 0xd58487a2, 0x919b7a07, - 0xbe2d0e05, 0xd36ccf4f, 0x6d5c66d7, 0x8448522f, - 0x8409c294, 0x6f1c7af7, 0x173a13bc, 0x1b3e4a0b, - 0x705b941b, 0x77eb584f, 0x85b68458, 0x8e3ad1ac, - 0x4aa99702, 0x7ae1b24c, 0x899ba29c, 0x860a3711, - 0xabe53a4f, 0x37870133, 0x1ed7cb89, 0xea539762, - 0x4ba64130, 0x48517a2d, 0xce0a869d, 0x937ba48, - 0xd0f234c4, 0xf9b2cf26, 0xc3c311f0, 0x153d09a9, - 0x404d3af9, 0x9f7edbc1, 0xbdcecded, 0x97969ba8, - 0x3379437, 0xadd3c893, 0x7c024639, 0x459390b, - 0xcb7c7320, 0xa5c63725, 0x65907e3e, 0xbf70583b, - 0xcebb601b, 0x4edfb286, 0x9350336f, 0xdfb4be76, - 0x88b56f39, 0x9937d7f9, 0xa12a286d, 0x34f141c, - 0xa2e75c15, 0xd69a7060, 0x931340c3, 0x22447f25, - 0xe8aed82c, 0xd76a9ae7, 0xc967288, 0xe572facd, - 0xbe82b0ee, 0x10f5dce1, 0x4f03ee35, 0x2340b923, - 0xf4fb6bd0, 0x64adbf01, 0x3d277a0a, 0x39e76f2c, - 0xe3c024d9, 0x57869c82, 0x743b7826, 0xf66f1574, - 0xc93965c, 0xf86a552, 0x13557069, 0x9e0845de, - 0xaee084f9, 0x5eafaedb, 0xc06f5f3, 0x9051f6ea, - 0x98fceda2, 0x2af2f8cc, 0x6c41b5a8, 0xc1af74de, - 0x57302276, 0x253923c9, 0xd79996b3, 0x8ecb3141, - 0x641387cc, 0xf87a4101, 0x96a50c76, 0xbcf24a11, - 0x87b6bb6, 0x58ee501b, 0xaa859695, 0xb2eed107, - 0x554173f0, 0xb12ec0e6, 0x57785c1b, 0x53685c9a, - 0x114c3163, 0x9383cf19, 0x31fd7cdf, 0xaeb8225c, - 0x58774fe7, 0x54700ad4, 0xad418726, 0xf055b71c, - 0x7d31237f, 0xdd97cad5, 0xcdd5325e, 0x42f2acf4, - 0x4bed262b, 0x7a8faaf2, 0x2b1eafdd, 0xa1b806ac, - 0x26965c6e, 0xb41b7168, 0x15e2e70b, 0x7daa8e13, - 0x6198aa5a, 0xc9b8b94c, 0x339b5754, 0xcd3b285c, - 0xffd1486c, 0xf224979a, 0xafb89ec5, 0x222058c, - 0xcb4814d0, 0x2b0e7c7d, 0x9eb25b84, 0x271564b0, - 0xbb72e076, 0x48251020, 0x18008023, 0x48d10005, - 0x4a452eaa, 0xb2365308, 0x19cdb632, 0x1fd56d04, - 0xffa5ff2a, 0xaba89e42, 0x388fc17d, 0xea61c00f, - 0x5156273d, 0x776f1a56, 0x8d539d28, 0x289c01cb, - 0x857aa71f, 0x348e411f, 0xc9eb3c91, 0x67a61079, - 0xe4276a0f, 0x45bdc15f, 0x8e0a698e, 0xbdefc310, - 0x82377ba6, 0x3bfbf404, 0xcbf22c79, 0x35f501bc, - 0xb16044a7, 0xeffdb8, 0xdbac383d, 0x7816663f, - 0x18f5a318, 0x3d04f1cb, 0x735da9b4, 0x75e339a1, - 0x5b6c55f, 0x1c18887e, 0xf698e14f, 0x338a6da1, - 0xdac85699, 0x1aca7768, 0x8eb0fa7a, 0xc98fa71d, - 0x3b794408, 0x92913041, 0xf8dc8827, 0x1cf706e9, - 0x3aeee292, 0x321dbaa8, 0xee1eb8d1, 0x23554be9, - 0x811c7804, 0xf0f4de6b, 0xd457e382, 0xeda56795, - 0xeffdfc71, 0xf2a52829, 0xa7460732, 0x2c1321c0, - 0x2f734db0, 0xf04ecb0b, 0xec7d777e, 0x43c54317, - 0xccaa74cd, 0xfe49dd9d, 0x4c509829, 0x278f9bd7, - 0x581dc500, 0x4ad38c2e, 0xcbee1047, 0x13302c1c, - 0xbc0cb734, 0xc1c8f234, 0x1df52b35, 0xd8815548, - 0x319edefb, 0x437cebe5, 0x3dcb6026, 0xe9d4f93f, - 0xb2661154, 0xeb8c15a0, 0xb008505, 0x5f869981, - 0xf5588ca4, 0xd6929c5b, 0xa3dd13d1, 0xdc863314, - 0x891a454f, 0x91737e49, 0x5064d4d8, 0x2fd32675, - 0xadefe9b1, 0xdde32b11, 0x741bbd6, 0x3b4363a9, - 0xb121d9e8, 0x916ca61d, 0x38c0af15, 0x5e3dfd72, -}; diff --git a/std/sort.zig b/std/sort.zig index c13e99feda..a3ebd293cf 100644 --- a/std/sort.zig +++ b/std/sort.zig @@ -67,7 +67,7 @@ const Iterator = struct { self.numerator -= self.denominator; self.decimal += 1; } - + return Range {.start = start, .end = self.decimal}; } @@ -82,7 +82,7 @@ const Iterator = struct { self.numerator_step -= self.denominator; self.decimal_step += 1; } - + return (self.decimal_step < self.size); } @@ -219,7 +219,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons var B1 = iterator.nextRange(); var A2 = iterator.nextRange(); var B2 = iterator.nextRange(); - + if (lessThan(items[B1.end - 1], items[A1.start])) { // the two ranges are in reverse order, so copy them in reverse order into the cache mem.copy(T, cache[B1.length()..], items[A1.start..A1.end]); @@ -230,13 +230,13 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } else { // if A1, B1, A2, and B2 are all in order, skip doing anything else if (!lessThan(items[B2.start], items[A2.end - 1]) and !lessThan(items[A2.start], items[B1.end - 1])) continue; - + // copy A1 and B1 into the cache in the same order mem.copy(T, cache[0..], items[A1.start..A1.end]); mem.copy(T, cache[A1.length()..], items[B1.start..B1.end]); } A1 = Range.init(A1.start, B1.end); - + // merge A2 and B2 into the cache if (lessThan(items[B2.end - 1], items[A2.start])) { // the two ranges are in reverse order, so copy them in reverse order into the cache @@ -251,11 +251,11 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons mem.copy(T, cache[A1.length() + A2.length()..], items[B2.start..B2.end]); } A2 = Range.init(A2.start, B2.end); - + // merge A1 and A2 from the cache into the items const A3 = Range.init(0, A1.length()); const B3 = Range.init(A1.length(), A1.length() + A2.length()); - + if (lessThan(cache[B3.end - 1], cache[A3.start])) { // the two ranges are in reverse order, so copy them in reverse order into the items mem.copy(T, items[A1.start + A2.length()..], cache[A3.start..A3.end]); @@ -269,17 +269,17 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons mem.copy(T, items[A1.start + A1.length()..], cache[B3.start..B3.end]); } } - + // we merged two levels at the same time, so we're done with this level already // (iterator.nextLevel() is called again at the bottom of this outer merge loop) _ = iterator.nextLevel(); - + } else { iterator.begin(); while (!iterator.finished()) { var A = iterator.nextRange(); var B = iterator.nextRange(); - + if (lessThan(items[B.end - 1], items[A.start])) { // the two ranges are in reverse order, so a simple rotation should fix it mem.rotate(T, items[A.start..B.end], A.length()); @@ -301,10 +301,10 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons // 6. merge each A block with any B values that follow, using the cache or the second internal buffer // 7. sort the second internal buffer if it exists // 8. redistribute the two internal buffers back into the items - + var block_size: usize = math.sqrt(iterator.length()); var buffer_size = iterator.length()/block_size + 1; - + // as an optimization, we really only need to pull out the internal buffers once for each level of merges // after that we can reuse the same buffers over and over, then redistribute it when we're finished with this level var A: Range = undefined; @@ -322,11 +322,11 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons var buffer1 = Range.init(0, 0); var buffer2 = Range.init(0, 0); - + // find two internal buffers of size 'buffer_size' each find = buffer_size + buffer_size; var find_separately = false; - + if (block_size <= cache.len) { // if every A block fits into the cache then we won't need the second internal buffer, // so we really only need to find 'buffer_size' unique values @@ -336,21 +336,21 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons find = buffer_size; find_separately = true; } - + // we need to find either a single contiguous space containing 2√A unique values (which will be split up into two buffers of size √A each), // or we need to find one buffer of < 2√A unique values, and a second buffer of √A unique values, // OR if we couldn't find that many unique values, we need the largest possible buffer we can get - + // in the case where it couldn't find a single buffer of at least √A unique values, // all of the Merge steps must be replaced by a different merge algorithm (MergeInPlace) iterator.begin(); while (!iterator.finished()) { A = iterator.nextRange(); B = iterator.nextRange(); - + // just store information about where the values will be pulled from and to, // as well as how many values there are, to create the two internal buffers - + // check A for the number of unique values we need to fill an internal buffer // these values will be pulled out to the start of A last = A.start; @@ -360,7 +360,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons if (index == A.end) break; } index = last; - + if (count >= buffer_size) { // keep track of the range within the items where we'll need to "pull out" these values to create the internal buffer pull[pull_index] = Pull { @@ -370,7 +370,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons .to = A.start, }; pull_index = 1; - + if (count == buffer_size + buffer_size) { // we were able to find a single contiguous section containing 2√A unique values, // so this section can be used to contain both of the internal buffers we'll need @@ -405,7 +405,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons .to = A.start, }; } - + // check B for the number of unique values we need to fill an internal buffer // these values will be pulled out to the end of B last = B.end - 1; @@ -415,7 +415,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons if (index == B.start) break; } index = last; - + if (count >= buffer_size) { // keep track of the range within the items where we'll need to "pull out" these values to create the internal buffe pull[pull_index] = Pull { @@ -425,7 +425,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons .to = B.end, }; pull_index = 1; - + if (count == buffer_size + buffer_size) { // we were able to find a single contiguous section containing 2√A unique values, // so this section can be used to contain both of the internal buffers we'll need @@ -449,7 +449,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons // buffer2 will be pulled out from a 'B' subarray, so if the first buffer was pulled out from the corresponding 'A' subarray, // we need to adjust the end point for that A subarray so it knows to stop redistributing its values before reaching buffer2 if (pull[0].range.start == A.start) pull[0].range.end -= pull[1].count; - + // we found a second buffer in an 'B' subarray containing √A unique values, so we're done! buffer2 = Range.init(B.end - count, B.end); break; @@ -465,12 +465,12 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons }; } } - + // pull out the two ranges so we can use them as internal buffers pull_index = 0; while (pull_index < 2) : (pull_index += 1) { const length = pull[pull_index].count; - + if (pull[pull_index].to < pull[pull_index].from) { // we're pulling the values out to the left, which means the start of an A subarray index = pull[pull_index].from; @@ -493,27 +493,27 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } } } - + // adjust block_size and buffer_size based on the values we were able to pull out buffer_size = buffer1.length(); block_size = iterator.length()/buffer_size + 1; - + // the first buffer NEEDS to be large enough to tag each of the evenly sized A blocks, // so this was originally here to test the math for adjusting block_size above // assert((iterator.length() + 1)/block_size <= buffer_size); - + // now that the two internal buffers have been created, it's time to merge each A+B combination at this level of the merge sort! iterator.begin(); while (!iterator.finished()) { A = iterator.nextRange(); B = iterator.nextRange(); - + // remove any parts of A or B that are being used by the internal buffers start = A.start; if (start == pull[0].range.start) { if (pull[0].from > pull[0].to) { A.start += pull[0].count; - + // if the internal buffer takes up the entire A or B subarray, then there's nothing to merge // this only happens for very small subarrays, like √4 = 2, 2 * (2 internal buffers) = 4, // which also only happens when cache.len is small or 0 since it'd otherwise use MergeExternal @@ -532,25 +532,25 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons if (B.length() == 0) continue; } } - + if (lessThan(items[B.end - 1], items[A.start])) { // the two ranges are in reverse order, so a simple rotation should fix it mem.rotate(T, items[A.start..B.end], A.length()); } else if (lessThan(items[A.end], items[A.end - 1])) { // these two ranges weren't already in order, so we'll need to merge them! var findA: usize = undefined; - + // break the remainder of A into blocks. firstA is the uneven-sized first A block var blockA = Range.init(A.start, A.end); var firstA = Range.init(A.start, A.start + blockA.length() % block_size); - + // swap the first value of each A block with the value in buffer1 var indexA = buffer1.start; index = firstA.end; while (index < blockA.end) : ({indexA += 1; index += block_size;}) { mem.swap(T, &items[indexA], &items[index]); } - + // start rolling the A blocks through the B blocks! // whenever we leave an A block behind, we'll need to merge the previous A block with any B blocks that follow it, so track that information as well var lastA = firstA; @@ -558,7 +558,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons var blockB = Range.init(B.start, B.start + math.min(block_size, B.length())); blockA.start += firstA.length(); indexA = buffer1.start; - + // if the first unevenly sized A block fits into the cache, copy it there for when we go to Merge it // otherwise, if the second buffer is available, block swap the contents into that if (lastA.length() <= cache.len) { @@ -566,7 +566,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } else if (buffer2.length() > 0) { blockSwap(T, items, lastA.start, buffer2.start, lastA.length()); } - + if (blockA.length() > 0) { while (true) { // if there's a previous B block and the first value of the minimum A block is <= the last value of the previous B block, @@ -575,7 +575,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons // figure out where to split the previous B block, and rotate it at the split const B_split = binaryFirst(T, items, items[indexA], lastB, lessThan); const B_remaining = lastB.end - B_split; - + // swap the minimum A block to the beginning of the rolling A blocks var minA = blockA.start; findA = minA + block_size; @@ -585,16 +585,16 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } } blockSwap(T, items, blockA.start, minA, block_size); - + // swap the first item of the previous A block back with its original value, which is stored in buffer1 mem.swap(T, &items[blockA.start], &items[indexA]); indexA += 1; - + // locally merge the previous A block with the B values that follow it // if lastA fits into the external cache we'll use that (with MergeExternal), // or if the second internal buffer exists we'll use that (with MergeInternal), // or failing that we'll use a strictly in-place merge algorithm (MergeInPlace) - + if (lastA.length() <= cache.len) { mergeExternal(T, items, lastA, Range.init(lastA.end, B_split), lessThan, cache[0..]); } else if (buffer2.length() > 0) { @@ -602,7 +602,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } else { mergeInPlace(T, items, lastA, Range.init(lastA.end, B_split), lessThan); } - + if (buffer2.length() > 0 or block_size <= cache.len) { // copy the previous A block into the cache or buffer2, since that's where we need it to be when we go to merge it anyway if (block_size <= cache.len) { @@ -610,7 +610,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } else { blockSwap(T, items, blockA.start, buffer2.start, block_size); } - + // this is equivalent to rotating, but faster // the area normally taken up by the A block is either the contents of buffer2, or data we don't need anymore since we memcopied it // either way, we don't need to retain the order of those items, so instead of rotating we can just block swap B to where it belongs @@ -619,21 +619,21 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons // we are unable to use the 'buffer2' trick to speed up the rotation operation since buffer2 doesn't exist, so perform a normal rotation mem.rotate(T, items[B_split..blockA.start + block_size], blockA.start - B_split); } - + // update the range for the remaining A blocks, and the range remaining from the B block after it was split lastA = Range.init(blockA.start - B_remaining, blockA.start - B_remaining + block_size); lastB = Range.init(lastA.end, lastA.end + B_remaining); - + // if there are no more A blocks remaining, this step is finished! blockA.start += block_size; if (blockA.length() == 0) break; - + } else if (blockB.length() < block_size) { // move the last B block, which is unevenly sized, to before the remaining A blocks, by using a rotation // the cache is disabled here since it might contain the contents of the previous A block mem.rotate(T, items[blockA.start..blockB.end], blockB.start - blockA.start); - + lastB = Range.init(blockA.start, blockA.start + blockB.length()); blockA.start += blockB.length(); blockA.end += blockB.length(); @@ -642,11 +642,11 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons // roll the leftmost A block to the end by swapping it with the next B block blockSwap(T, items, blockA.start, blockB.start, block_size); lastB = Range.init(blockA.start, blockA.start + block_size); - + blockA.start += block_size; blockA.end += block_size; blockB.start += block_size; - + if (blockB.end > B.end - block_size) { blockB.end = B.end; } else { @@ -655,7 +655,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } } } - + // merge the last A block with the remaining B values if (lastA.length() <= cache.len) { mergeExternal(T, items, lastA, Range.init(lastA.end, B.end), lessThan, cache[0..]); @@ -666,14 +666,14 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } } } - + // when we're finished with this merge step we should have the one or two internal buffers left over, where the second buffer is all jumbled up // insertion sort the second buffer, then redistribute the buffers back into the items using the opposite process used for creating the buffer - + // while an unstable sort like quicksort could be applied here, in benchmarks it was consistently slightly slower than a simple insertion sort, // even for tens of millions of items. this may be because insertion sort is quite fast when the data is already somewhat sorted, like it is here insertionSort(T, items[buffer2.start..buffer2.end], lessThan); - + pull_index = 0; while (pull_index < 2) : (pull_index += 1) { var unique = pull[pull_index].count * 2; @@ -702,7 +702,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } } } - + // double the size of each A and B subarray that will be merged in the next level if (!iterator.nextLevel()) break; } @@ -711,37 +711,37 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons // merge operation without a buffer fn mergeInPlace(comptime T: type, items: []T, A_arg: &const Range, B_arg: &const Range, lessThan: fn(&const T,&const T)bool) void { if (A_arg.length() == 0 or B_arg.length() == 0) return; - + // this just repeatedly binary searches into B and rotates A into position. // the paper suggests using the 'rotation-based Hwang and Lin algorithm' here, // but I decided to stick with this because it had better situational performance - // + // // (Hwang and Lin is designed for merging subarrays of very different sizes, // but WikiSort almost always uses subarrays that are roughly the same size) - // + // // normally this is incredibly suboptimal, but this function is only called // when none of the A or B blocks in any subarray contained 2√A unique values, // which places a hard limit on the number of times this will ACTUALLY need // to binary search and rotate. - // + // // according to my analysis the worst case is √A rotations performed on √A items // once the constant factors are removed, which ends up being O(n) - // + // // again, this is NOT a general-purpose solution – it only works well in this case! // kind of like how the O(n^2) insertion sort is used in some places var A = *A_arg; var B = *B_arg; - + while (true) { // find the first place in B where the first item in A needs to be inserted const mid = binaryFirst(T, items, items[A.start], B, lessThan); - + // rotate A into place const amount = mid - A.end; mem.rotate(T, items[A.start..mid], A.length()); if (B.end == mid) break; - + // calculate the new A and B ranges B.start = mid; A = Range.init(A.start + amount, B.start); @@ -757,7 +757,7 @@ fn mergeInternal(comptime T: type, items: []T, A: &const Range, B: &const Range, var A_count: usize = 0; var B_count: usize = 0; var insert: usize = 0; - + if (B.length() > 0 and A.length() > 0) { while (true) { if (!lessThan(items[B.start + B_count], items[buffer.start + A_count])) { @@ -773,7 +773,7 @@ fn mergeInternal(comptime T: type, items: []T, A: &const Range, B: &const Range, } } } - + // swap the remainder of A into the final array blockSwap(T, items, buffer.start + A_count, A.start + insert, A.length() - A_count); } @@ -790,56 +790,56 @@ fn blockSwap(comptime T: type, items: []T, start1: usize, start2: usize, block_s fn findFirstForward(comptime T: type, items: []T, value: &const T, range: &const Range, lessThan: fn(&const T,&const T)bool, unique: usize) usize { if (range.length() == 0) return range.start; const skip = math.max(range.length()/unique, usize(1)); - + var index = range.start + skip; while (lessThan(items[index - 1], value)) : (index += skip) { if (index >= range.end - skip) { return binaryFirst(T, items, value, Range.init(index, range.end), lessThan); } } - + return binaryFirst(T, items, value, Range.init(index - skip, index), lessThan); } fn findFirstBackward(comptime T: type, items: []T, value: &const T, range: &const Range, lessThan: fn(&const T,&const T)bool, unique: usize) usize { if (range.length() == 0) return range.start; const skip = math.max(range.length()/unique, usize(1)); - + var index = range.end - skip; while (index > range.start and !lessThan(items[index - 1], value)) : (index -= skip) { if (index < range.start + skip) { return binaryFirst(T, items, value, Range.init(range.start, index), lessThan); } } - + return binaryFirst(T, items, value, Range.init(index, index + skip), lessThan); } fn findLastForward(comptime T: type, items: []T, value: &const T, range: &const Range, lessThan: fn(&const T,&const T)bool, unique: usize) usize { if (range.length() == 0) return range.start; const skip = math.max(range.length()/unique, usize(1)); - + var index = range.start + skip; while (!lessThan(value, items[index - 1])) : (index += skip) { if (index >= range.end - skip) { return binaryLast(T, items, value, Range.init(index, range.end), lessThan); } } - + return binaryLast(T, items, value, Range.init(index - skip, index), lessThan); } fn findLastBackward(comptime T: type, items: []T, value: &const T, range: &const Range, lessThan: fn(&const T,&const T)bool, unique: usize) usize { if (range.length() == 0) return range.start; const skip = math.max(range.length()/unique, usize(1)); - + var index = range.end - skip; while (index > range.start and lessThan(value, items[index - 1])) : (index -= skip) { if (index < range.start + skip) { return binaryLast(T, items, value, Range.init(range.start, index), lessThan); } } - + return binaryLast(T, items, value, Range.init(index, index + skip), lessThan); } @@ -885,7 +885,7 @@ fn mergeInto(comptime T: type, from: []T, A: &const Range, B: &const Range, less const A_last = A.end; const B_last = B.end; var insert_index: usize = 0; - + while (true) { if (!lessThan(from[B_index], from[A_index])) { into[insert_index] = from[A_index]; @@ -916,7 +916,7 @@ fn mergeExternal(comptime T: type, items: []T, A: &const Range, B: &const Range, var insert_index: usize = A.start; const A_last = A.length(); const B_last = B.end; - + if (B.length() > 0 and A.length() > 0) { while (true) { if (!lessThan(items[B_index], cache[A_index])) { @@ -932,7 +932,7 @@ fn mergeExternal(comptime T: type, items: []T, A: &const Range, B: &const Range, } } } - + // copy the remainder of A into the final array mem.copy(T, items[insert_index..], cache[A_index..A_last]); } @@ -1081,11 +1081,11 @@ test "another sort case" { } test "sort fuzz testing" { - var rng = std.rand.Rand.init(0x12345678); + var prng = std.rand.DefaultPrng.init(0x12345678); const test_case_count = 10; var i: usize = 0; while (i < test_case_count) : (i += 1) { - fuzzTest(&rng); + fuzzTest(&prng.random); } } From ccadcbc715c084f78dc093811d74f93b80536a0a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 29 Mar 2018 12:33:29 -0400 Subject: [PATCH 22/47] fix examples and rename std.rand.Rand to std.rand.Random --- example/guess_number/main.zig | 9 +++---- src/parser.hpp | 1 - std/rand/index.zig | 50 +++++++++++++++++------------------ std/sort.zig | 2 +- 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/example/guess_number/main.zig b/example/guess_number/main.zig index d6a7b94b6c..d26518a1ed 100644 --- a/example/guess_number/main.zig +++ b/example/guess_number/main.zig @@ -2,7 +2,6 @@ const builtin = @import("builtin"); const std = @import("std"); const io = std.io; const fmt = std.fmt; -const Rand = std.rand.Rand; const os = std.os; pub fn main() !void { @@ -14,15 +13,15 @@ pub fn main() !void { try stdout.print("Welcome to the Guess Number Game in Zig.\n"); - var seed_bytes: [@sizeOf(usize)]u8 = undefined; + var seed_bytes: [@sizeOf(u64)]u8 = undefined; os.getRandomBytes(seed_bytes[0..]) catch |err| { std.debug.warn("unable to seed random number generator: {}", err); return err; }; - const seed = std.mem.readInt(seed_bytes, usize, builtin.Endian.Big); - var rand = Rand.init(seed); + const seed = std.mem.readInt(seed_bytes, u64, builtin.Endian.Big); + var prng = std.rand.DefaultPrng.init(seed); - const answer = rand.range(u8, 0, 100) + 1; + const answer = prng.random.range(u8, 0, 100) + 1; while (true) { try stdout.print("\nGuess a number between 1 and 100: "); diff --git a/src/parser.hpp b/src/parser.hpp index 21922db4ed..224e289bf5 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -16,7 +16,6 @@ ATTRIBUTE_PRINTF(2, 3) void ast_token_error(Token *token, const char *format, ...); -// This function is provided by generated code, generated by parsergen.cpp AstNode * ast_parse(Buf *buf, ZigList *tokens, ImportTableEntry *owner, ErrColor err_color); void ast_print(AstNode *node, int indent); diff --git a/std/rand/index.zig b/std/rand/index.zig index ce69565473..6a746fce92 100644 --- a/std/rand/index.zig +++ b/std/rand/index.zig @@ -26,16 +26,16 @@ pub const DefaultPrng = Xoroshiro128; // When you need cryptographically secure random numbers pub const DefaultCsprng = Isaac64; -pub const Rand = struct { - fillFn: fn(r: &Rand, buf: []u8) void, +pub const Random = struct { + fillFn: fn(r: &Random, buf: []u8) void, /// Read random bytes into the specified buffer until fill. - pub fn bytes(r: &Rand, buf: []u8) void { + pub fn bytes(r: &Random, buf: []u8) void { r.fillFn(r, buf); } /// Return a random integer/boolean type. - pub fn scalar(r: &Rand, comptime T: type) T { + pub fn scalar(r: &Random, comptime T: type) T { var rand_bytes: [@sizeOf(T)]u8 = undefined; r.bytes(rand_bytes[0..]); @@ -49,7 +49,7 @@ pub const Rand = struct { /// Get a random unsigned integer with even distribution between `start` /// inclusive and `end` exclusive. - pub fn range(r: &Rand, comptime T: type, start: T, end: T) T { + pub fn range(r: &Random, comptime T: type, start: T, end: T) T { assert(start <= end); if (T.is_signed) { const uint = @IntType(false, T.bit_count); @@ -91,7 +91,7 @@ pub const Rand = struct { } /// Return a floating point value evenly distributed in the range [0, 1). - pub fn float(r: &Rand, comptime T: type) T { + pub fn float(r: &Random, comptime T: type) T { // Generate a uniform value between [1, 2) and scale down to [0, 1). // Note: The lowest mantissa bit is always set to 0 so we only use half the available range. switch (T) { @@ -110,18 +110,18 @@ pub const Rand = struct { } /// Return a floating point value normally distributed in the range [0, 1]. - pub fn floatNorm(r: &Rand, comptime T: type) T { + pub fn floatNorm(r: &Random, comptime T: type) T { // TODO(tiehuis): See https://www.doornik.com/research/ziggurat.pdf @compileError("floatNorm is unimplemented"); } /// Return a exponentially distributed float between (0, @maxValue(f64)) - pub fn floatExp(r: &Rand, comptime T: type) T { + pub fn floatExp(r: &Random, comptime T: type) T { @compileError("floatExp is unimplemented"); } /// Shuffle a slice into a random order. - pub fn shuffle(r: &Rand, comptime T: type, buf: []T) void { + pub fn shuffle(r: &Random, comptime T: type, buf: []T) void { if (buf.len < 2) { return; } @@ -178,14 +178,14 @@ test "splitmix64 sequence" { pub const Pcg = struct { const default_multiplier = 6364136223846793005; - random: Rand, + random: Random, s: u64, i: u64, pub fn init(init_s: u64) Pcg { var pcg = Pcg { - .random = Rand { .fillFn = fill }, + .random = Random { .fillFn = fill }, .s = undefined, .i = undefined, }; @@ -218,7 +218,7 @@ pub const Pcg = struct { self.s = self.s *% default_multiplier +% self.i; } - fn fill(r: &Rand, buf: []u8) void { + fn fill(r: &Random, buf: []u8) void { const self = @fieldParentPtr(Pcg, "random", r); var i: usize = 0; @@ -269,13 +269,13 @@ test "pcg sequence" { // // PRNG pub const Xoroshiro128 = struct { - random: Rand, + random: Random, s: [2]u64, pub fn init(init_s: u64) Xoroshiro128 { var x = Xoroshiro128 { - .random = Rand { .fillFn = fill }, + .random = Random { .fillFn = fill }, .s = undefined, }; @@ -328,7 +328,7 @@ pub const Xoroshiro128 = struct { self.s[1] = gen.next(); } - fn fill(r: &Rand, buf: []u8) void { + fn fill(r: &Random, buf: []u8) void { const self = @fieldParentPtr(Xoroshiro128, "random", r); var i: usize = 0; @@ -397,7 +397,7 @@ test "xoroshiro sequence" { // Follows the general idea of the implementation from here with a few shortcuts. // https://doc.rust-lang.org/rand/src/rand/prng/isaac64.rs.html pub const Isaac64 = struct { - random: Rand, + random: Random, r: [256]u64, m: [256]u64, @@ -408,7 +408,7 @@ pub const Isaac64 = struct { pub fn init(init_s: u64) Isaac64 { var isaac = Isaac64 { - .random = Rand { .fillFn = fill }, + .random = Random { .fillFn = fill }, .r = undefined, .m = undefined, .a = undefined, @@ -522,7 +522,7 @@ pub const Isaac64 = struct { self.i = self.r.len; // trigger refill on first value } - fn fill(r: &Rand, buf: []u8) void { + fn fill(r: &Random, buf: []u8) void { const self = @fieldParentPtr(Isaac64, "random", r); var i: usize = 0; @@ -577,8 +577,8 @@ test "isaac64 sequence" { } } -// Actual Rand helper function tests, pcg engine is assumed correct. -test "Rand float" { +// Actual Random helper function tests, pcg engine is assumed correct. +test "Random float" { var prng = DefaultPrng.init(0); var i: usize = 0; @@ -593,18 +593,18 @@ test "Rand float" { } } -test "Rand scalar" { +test "Random scalar" { var prng = DefaultPrng.init(0); const s = prng .random.scalar(u64); } -test "Rand bytes" { +test "Random bytes" { var prng = DefaultPrng.init(0); var buf: [2048]u8 = undefined; prng.random.bytes(buf[0..]); } -test "Rand shuffle" { +test "Random shuffle" { var prng = DefaultPrng.init(0); var seq = []const u8 { 0, 1, 2, 3, 4 }; @@ -629,14 +629,14 @@ fn sumArray(s: []const u8) u32 { return r; } -test "Rand range" { +test "Random range" { var prng = DefaultPrng.init(0); testRange(&prng.random, -4, 3); testRange(&prng.random, -4, -1); testRange(&prng.random, 10, 14); } -fn testRange(r: &Rand, start: i32, end: i32) void { +fn testRange(r: &Random, start: i32, end: i32) void { const count = usize(end - start); var values_buffer = []bool{false} ** 20; const values = values_buffer[0..count]; diff --git a/std/sort.zig b/std/sort.zig index a3ebd293cf..0f83df7bb4 100644 --- a/std/sort.zig +++ b/std/sort.zig @@ -1091,7 +1091,7 @@ test "sort fuzz testing" { var fixed_buffer_mem: [100 * 1024]u8 = undefined; -fn fuzzTest(rng: &std.rand.Rand) void { +fn fuzzTest(rng: &std.rand.Random) void { const array_size = rng.range(usize, 0, 1000); var fixed_allocator = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]); var array = fixed_allocator.allocator.alloc(IdAndValue, array_size) catch unreachable; From 530f79576923989928afeeb8b50b7e40b95a529f Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Thu, 29 Mar 2018 22:31:17 +0200 Subject: [PATCH 23/47] std.zig.parser now supports all infix operators --- std/zig/ast.zig | 61 ++++++++- std/zig/parser.zig | 249 +++++++++++++++++++++++++++--------- std/zig/tokenizer.zig | 287 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 535 insertions(+), 62 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 180a0a9308..59ba0d79ca 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,6 +20,7 @@ pub const Node = struct { IntegerLiteral, FloatLiteral, StringLiteral, + UndefinedLiteral, BuiltinCall, LineComment, TestDecl, @@ -38,6 +39,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), + Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), @@ -57,6 +59,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), + Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), @@ -76,6 +79,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), + Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), @@ -309,9 +313,47 @@ pub const NodeInfixOp = struct { rhs: &Node, const InfixOp = enum { - EqualEqual, + Add, + AddWrap, + ArrayCat, + ArrayMult, + Assign, + AssignBitAnd, + AssignBitOr, + AssignBitShiftLeft, + AssignBitShiftRight, + AssignBitXor, + AssignDiv, + AssignMinus, + AssignMinusWrap, + AssignMod, + AssignPlus, + AssignPlusWrap, + AssignTimes, + AssignTimesWarp, BangEqual, + BitAnd, + BitOr, + BitShiftLeft, + BitShiftRight, + BitXor, + BoolAnd, + BoolOr, + Div, + EqualEqual, + ErrorUnion, + GreaterOrEqual, + GreaterThan, + LessOrEqual, + LessThan, + MergeErrorSets, + Mod, + Mult, + MultWrap, Period, + Sub, + SubWrap, + UnwrapMaybe, }; pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node { @@ -464,6 +506,23 @@ pub const NodeStringLiteral = struct { } }; +pub const NodeUndefinedLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeUndefinedLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeUndefinedLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeUndefinedLiteral) Token { + return self.token; + } +}; + pub const NodeLineComment = struct { base: Node, lines: ArrayList(Token), diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 06b7d35a48..a96277df04 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -371,6 +371,13 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_undefined => { + try stack.append(State { + .Operand = &(try self.createUndefined(arena, token)).base + }); + try stack.append(State.AfterOperand); + continue; + }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -414,56 +421,41 @@ pub const Parser = struct { // or a postfix operator (like () or {}), // otherwise this expression is done (like on a ; or else). var token = self.getNextToken(); - switch (token.id) { - Token.Id.EqualEqual => { + if (tokenIdToInfixOp(token.id)) |infix_id| { try stack.append(State { - .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.EqualEqual) + .InfixOp = try self.createInfixOp(arena, token, infix_id) }); try stack.append(State.ExpectOperand); continue; - }, - Token.Id.BangEqual => { - try stack.append(State { - .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BangEqual) - }); - try stack.append(State.ExpectOperand); - continue; - }, - Token.Id.Period => { - try stack.append(State { - .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period) - }); - try stack.append(State.ExpectOperand); - continue; - }, - else => { - // no postfix/infix operator after this operand. - self.putBackToken(token); - // reduce the stack - var expression: &ast.Node = stack.pop().Operand; - while (true) { - switch (stack.pop()) { - State.Expression => |dest_ptr| { - // we're done - try dest_ptr.store(expression); - break; - }, - State.InfixOp => |infix_op| { - infix_op.rhs = expression; - infix_op.lhs = stack.pop().Operand; - expression = &infix_op.base; - continue; - }, - State.PrefixOp => |prefix_op| { - prefix_op.rhs = expression; - expression = &prefix_op.base; - continue; - }, - else => unreachable, - } + + // TODO: Parse postfix operator + } else { + // no postfix/infix operator after this operand. + self.putBackToken(token); + // reduce the stack + var expression: &ast.Node = stack.pop().Operand; + while (true) { + switch (stack.pop()) { + State.Expression => |dest_ptr| { + // we're done + try dest_ptr.store(expression); + break; + }, + State.InfixOp => |infix_op| { + infix_op.rhs = expression; + infix_op.lhs = stack.pop().Operand; + expression = &infix_op.base; + continue; + }, + State.PrefixOp => |prefix_op| { + prefix_op.rhs = expression; + expression = &prefix_op.base; + continue; + }, + else => unreachable, } - continue; - }, + } + continue; } }, @@ -706,6 +698,53 @@ pub const Parser = struct { } } + fn tokenIdToInfixOp(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + return switch (*id) { + Token.Id.Ampersand => ast.NodeInfixOp.InfixOp.BitAnd, + Token.Id.AmpersandEqual => ast.NodeInfixOp.InfixOp.AssignBitAnd, + Token.Id.AngleBracketAngleBracketLeft => ast.NodeInfixOp.InfixOp.BitShiftLeft, + Token.Id.AngleBracketAngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftLeft, + Token.Id.AngleBracketAngleBracketRight => ast.NodeInfixOp.InfixOp.BitShiftRight, + Token.Id.AngleBracketAngleBracketRightEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftRight, + Token.Id.AngleBracketLeft => ast.NodeInfixOp.InfixOp.LessThan, + Token.Id.AngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.LessOrEqual, + Token.Id.AngleBracketRight => ast.NodeInfixOp.InfixOp.GreaterThan, + Token.Id.AngleBracketRightEqual => ast.NodeInfixOp.InfixOp.GreaterOrEqual, + Token.Id.Asterisk => ast.NodeInfixOp.InfixOp.Mult, + Token.Id.AsteriskAsterisk => ast.NodeInfixOp.InfixOp.ArrayMult, + Token.Id.AsteriskEqual => ast.NodeInfixOp.InfixOp.AssignTimes, + Token.Id.AsteriskPercent => ast.NodeInfixOp.InfixOp.MultWrap, + Token.Id.AsteriskPercentEqual => ast.NodeInfixOp.InfixOp.AssignTimesWarp, + Token.Id.Bang => ast.NodeInfixOp.InfixOp.ErrorUnion, + Token.Id.BangEqual => ast.NodeInfixOp.InfixOp.BangEqual, + Token.Id.Caret => ast.NodeInfixOp.InfixOp.BitXor, + Token.Id.CaretEqual => ast.NodeInfixOp.InfixOp.AssignBitXor, + Token.Id.Equal => ast.NodeInfixOp.InfixOp.Assign, + Token.Id.EqualEqual => ast.NodeInfixOp.InfixOp.EqualEqual, + Token.Id.Keyword_and => ast.NodeInfixOp.InfixOp.BoolAnd, + Token.Id.Keyword_or => ast.NodeInfixOp.InfixOp.BoolOr, + Token.Id.Minus => ast.NodeInfixOp.InfixOp.Sub, + Token.Id.MinusEqual => ast.NodeInfixOp.InfixOp.AssignMinus, + Token.Id.MinusPercent => ast.NodeInfixOp.InfixOp.SubWrap, + Token.Id.MinusPercentEqual => ast.NodeInfixOp.InfixOp.AssignMinusWrap, + Token.Id.Percent => ast.NodeInfixOp.InfixOp.Mod, + Token.Id.PercentEqual => ast.NodeInfixOp.InfixOp.AssignMod, + Token.Id.Period => ast.NodeInfixOp.InfixOp.Period, + Token.Id.Pipe => ast.NodeInfixOp.InfixOp.BitOr, + Token.Id.PipeEqual => ast.NodeInfixOp.InfixOp.AssignBitOr, + Token.Id.PipePipe => ast.NodeInfixOp.InfixOp.MergeErrorSets, + Token.Id.Plus => ast.NodeInfixOp.InfixOp.Add, + Token.Id.PlusEqual => ast.NodeInfixOp.InfixOp.AssignPlus, + Token.Id.PlusPercent => ast.NodeInfixOp.InfixOp.AddWrap, + Token.Id.PlusPercentEqual => ast.NodeInfixOp.InfixOp.AssignPlusWrap, + Token.Id.PlusPlus => ast.NodeInfixOp.InfixOp.ArrayCat, + Token.Id.QuestionMarkQuestionMark => ast.NodeInfixOp.InfixOp.UnwrapMaybe, + Token.Id.Slash => ast.NodeInfixOp.InfixOp.Div, + Token.Id.SlashEqual => ast.NodeInfixOp.InfixOp.AssignDiv, + else => null, + }; + } + fn initNode(self: &Parser, id: ast.Node.Id) ast.Node { if (self.pending_line_comment_node) |comment_node| { self.pending_line_comment_node = null; @@ -867,6 +906,16 @@ pub const Parser = struct { return node; } + fn createUndefined(self: &Parser, arena: &mem.Allocator, token: &const Token) !&ast.NodeUndefinedLiteral { + const node = try arena.create(ast.NodeUndefinedLiteral); + + *node = ast.NodeUndefinedLiteral { + .base = self.initNode(ast.Node.Id.UndefinedLiteral), + .token = *token, + }; + return node; + } + fn createAttachIdentifier(self: &Parser, arena: &mem.Allocator, dest_ptr: &const DestPtr, name_token: &const Token) !&ast.NodeIdentifier { const node = try self.createIdentifier(arena, name_token); try dest_ptr.store(&node.base); @@ -1173,17 +1222,51 @@ pub const Parser = struct { ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); - switch (prefix_op_node.op) { - ast.NodeInfixOp.InfixOp.EqualEqual => { - try stack.append(RenderState { .Text = " == "}); - }, - ast.NodeInfixOp.InfixOp.BangEqual => { - try stack.append(RenderState { .Text = " != "}); - }, - ast.NodeInfixOp.InfixOp.Period => { - try stack.append(RenderState { .Text = "."}); - }, - } + const text = switch (prefix_op_node.op) { + ast.NodeInfixOp.InfixOp.Add => " + ", + ast.NodeInfixOp.InfixOp.AddWrap => " +% ", + ast.NodeInfixOp.InfixOp.ArrayCat => " ++ ", + ast.NodeInfixOp.InfixOp.ArrayMult => " ** ", + ast.NodeInfixOp.InfixOp.Assign => " = ", + ast.NodeInfixOp.InfixOp.AssignBitAnd => " &= ", + ast.NodeInfixOp.InfixOp.AssignBitOr => " |= ", + ast.NodeInfixOp.InfixOp.AssignBitShiftLeft => " <<= ", + ast.NodeInfixOp.InfixOp.AssignBitShiftRight => " >>= ", + ast.NodeInfixOp.InfixOp.AssignBitXor => " ^= ", + ast.NodeInfixOp.InfixOp.AssignDiv => " /= ", + ast.NodeInfixOp.InfixOp.AssignMinus => " -= ", + ast.NodeInfixOp.InfixOp.AssignMinusWrap => " -%= ", + ast.NodeInfixOp.InfixOp.AssignMod => " %= ", + ast.NodeInfixOp.InfixOp.AssignPlus => " += ", + ast.NodeInfixOp.InfixOp.AssignPlusWrap => " +%= ", + ast.NodeInfixOp.InfixOp.AssignTimes => " *= ", + ast.NodeInfixOp.InfixOp.AssignTimesWarp => " *%= ", + ast.NodeInfixOp.InfixOp.BangEqual => " != ", + ast.NodeInfixOp.InfixOp.BitAnd => " & ", + ast.NodeInfixOp.InfixOp.BitOr => " | ", + ast.NodeInfixOp.InfixOp.BitShiftLeft => " << ", + ast.NodeInfixOp.InfixOp.BitShiftRight => " >> ", + ast.NodeInfixOp.InfixOp.BitXor => " ^ ", + ast.NodeInfixOp.InfixOp.BoolAnd => " and ", + ast.NodeInfixOp.InfixOp.BoolOr => " or ", + ast.NodeInfixOp.InfixOp.Div => " / ", + ast.NodeInfixOp.InfixOp.EqualEqual => " == ", + ast.NodeInfixOp.InfixOp.ErrorUnion => "!", + ast.NodeInfixOp.InfixOp.GreaterOrEqual => " >= ", + ast.NodeInfixOp.InfixOp.GreaterThan => " > ", + ast.NodeInfixOp.InfixOp.LessOrEqual => " <= ", + ast.NodeInfixOp.InfixOp.LessThan => " < ", + ast.NodeInfixOp.InfixOp.MergeErrorSets => " || ", + ast.NodeInfixOp.InfixOp.Mod => " % ", + ast.NodeInfixOp.InfixOp.Mult => " * ", + ast.NodeInfixOp.InfixOp.MultWrap => " *% ", + ast.NodeInfixOp.InfixOp.Period => ".", + ast.NodeInfixOp.InfixOp.Sub => " - ", + ast.NodeInfixOp.InfixOp.SubWrap => " -% ", + ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ", + }; + + try stack.append(RenderState { .Text = text }); try stack.append(RenderState { .Expression = prefix_op_node.lhs }); }, ast.Node.Id.PrefixOp => { @@ -1224,6 +1307,10 @@ pub const Parser = struct { const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token)); }, + ast.Node.Id.UndefinedLiteral => { + const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(undefined_literal.token)); + }, ast.Node.Id.BuiltinCall => { const builtin_call = @fieldParentPtr(ast.NodeBuiltinCall, "base", base); try stream.print("{}(", self.tokenizer.getTokenSlice(builtin_call.builtin_token)); @@ -1473,4 +1560,54 @@ test "zig fmt" { \\} \\ ); + + try testCanonical( + \\test "operators" { + \\ var i = undefined; + \\ i = 2; + \\ i *= 2; + \\ i |= 2; + \\ i ^= 2; + \\ i <<= 2; + \\ i >>= 2; + \\ i &= 2; + \\ i *= 2; + \\ i *%= 2; + \\ i -= 2; + \\ i -%= 2; + \\ i += 2; + \\ i +%= 2; + \\ i /= 2; + \\ i %= 2; + \\ _ = i == i; + \\ _ = i != i; + \\ _ = i != i; + \\ _ = i.i; + \\ _ = i || i; + \\ _ = i!i; + \\ _ = i ** i; + \\ _ = i ++ i; + \\ _ = i ?? i; + \\ _ = i % i; + \\ _ = i / i; + \\ _ = i *% i; + \\ _ = i * i; + \\ _ = i -% i; + \\ _ = i - i; + \\ _ = i +% i; + \\ _ = i + i; + \\ _ = i << i; + \\ _ = i >> i; + \\ _ = i & i; + \\ _ = i ^ i; + \\ _ = i | i; + \\ _ = i >= i; + \\ _ = i <= i; + \\ _ = i > i; + \\ _ = i < i; + \\ _ = i and i; + \\ _ = i or i; + \\} + \\ + ); } diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 4af6c20cad..4b1e35f62f 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -77,6 +77,7 @@ pub const Token = struct { Builtin, Bang, Pipe, + PipePipe, PipeEqual, Equal, EqualEqual, @@ -85,18 +86,45 @@ pub const Token = struct { RParen, Semicolon, Percent, + PercentEqual, LBrace, RBrace, Period, Ellipsis2, Ellipsis3, + Caret, + CaretEqual, + Plus, + PlusPlus, + PlusEqual, + PlusPercent, + PlusPercentEqual, Minus, + MinusEqual, + MinusPercent, + MinusPercentEqual, + Asterisk, + AsteriskEqual, + AsteriskAsterisk, + AsteriskPercent, + AsteriskPercentEqual, Arrow, Colon, Slash, + SlashEqual, Comma, Ampersand, AmpersandEqual, + QuestionMark, + QuestionMarkQuestionMark, + AngleBracketLeft, + AngleBracketLeftEqual, + AngleBracketAngleBracketLeft, + AngleBracketAngleBracketLeftEqual, + AngleBracketRight, + AngleBracketRightEqual, + AngleBracketAngleBracketRight, + AngleBracketAngleBracketRightEqual, IntegerLiteral, FloatLiteral, LineComment, @@ -200,6 +228,9 @@ pub const Tokenizer = struct { Bang, Pipe, Minus, + MinusPercent, + Asterisk, + AsteriskPercent, Slash, LineComment, Zero, @@ -210,6 +241,15 @@ pub const Tokenizer = struct { FloatExponentUnsigned, FloatExponentNumber, Ampersand, + Caret, + Percent, + QuestionMark, + Plus, + PlusPercent, + AngleBracketLeft, + AngleBracketAngleBracketLeft, + AngleBracketRight, + AngleBracketAngleBracketRight, Period, Period2, SawAtSign, @@ -291,9 +331,25 @@ pub const Tokenizer = struct { break; }, '%' => { - result.id = Token.Id.Percent; - self.index += 1; - break; + state = State.Percent; + }, + '*' => { + state = State.Asterisk; + }, + '+' => { + state = State.Plus; + }, + '?' => { + state = State.QuestionMark; + }, + '<' => { + state = State.AngleBracketLeft; + }, + '>' => { + state = State.AngleBracketRight; + }, + '^' => { + state = State.Caret; }, '{' => { result.id = Token.Id.LBrace; @@ -356,6 +412,107 @@ pub const Tokenizer = struct { break; }, }, + + State.Asterisk => switch (c) { + '=' => { + result.id = Token.Id.AsteriskEqual; + self.index += 1; + break; + }, + '*' => { + result.id = Token.Id.AsteriskAsterisk; + self.index += 1; + break; + }, + '%' => { + state = State.AsteriskPercent; + }, + else => { + result.id = Token.Id.Asterisk; + break; + } + }, + + State.AsteriskPercent => switch (c) { + '=' => { + result.id = Token.Id.AsteriskPercentEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.AsteriskPercent; + break; + } + }, + + State.QuestionMark => switch (c) { + '?' => { + result.id = Token.Id.QuestionMarkQuestionMark; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.QuestionMark; + break; + }, + }, + + State.Percent => switch (c) { + '=' => { + result.id = Token.Id.PercentEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.Percent; + break; + }, + }, + + State.Plus => switch (c) { + '=' => { + result.id = Token.Id.PlusEqual; + self.index += 1; + break; + }, + '+' => { + result.id = Token.Id.PlusPlus; + self.index += 1; + break; + }, + '%' => { + state = State.PlusPercent; + }, + else => { + result.id = Token.Id.Plus; + break; + }, + }, + + State.PlusPercent => switch (c) { + '=' => { + result.id = Token.Id.PlusPercentEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.PlusPercent; + break; + }, + }, + + State.Caret => switch (c) { + '=' => { + result.id = Token.Id.CaretEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.Caret; + break; + } + }, + State.Identifier => switch (c) { 'a'...'z', 'A'...'Z', '_', '0'...'9' => {}, else => { @@ -417,6 +574,11 @@ pub const Tokenizer = struct { self.index += 1; break; }, + '|' => { + result.id = Token.Id.PipePipe; + self.index += 1; + break; + }, else => { result.id = Token.Id.Pipe; break; @@ -441,12 +603,86 @@ pub const Tokenizer = struct { self.index += 1; break; }, + '=' => { + result.id = Token.Id.MinusEqual; + self.index += 1; + break; + }, + '%' => { + state = State.MinusPercent; + }, else => { result.id = Token.Id.Minus; break; }, }, + State.MinusPercent => switch (c) { + '=' => { + result.id = Token.Id.MinusPercentEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.MinusPercent; + break; + } + }, + + State.AngleBracketLeft => switch (c) { + '<' => { + state = State.AngleBracketAngleBracketLeft; + }, + '=' => { + result.id = Token.Id.AngleBracketLeftEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.AngleBracketLeft; + break; + }, + }, + + State.AngleBracketAngleBracketLeft => switch (c) { + '=' => { + result.id = Token.Id.AngleBracketAngleBracketLeftEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.AngleBracketAngleBracketLeft; + break; + }, + }, + + State.AngleBracketRight => switch (c) { + '>' => { + state = State.AngleBracketAngleBracketRight; + }, + '=' => { + result.id = Token.Id.AngleBracketRightEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.AngleBracketRight; + break; + }, + }, + + State.AngleBracketAngleBracketRight => switch (c) { + '=' => { + result.id = Token.Id.AngleBracketAngleBracketRightEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.AngleBracketAngleBracketRight; + break; + }, + }, + State.Period => switch (c) { '.' => { state = State.Period2; @@ -474,6 +710,11 @@ pub const Tokenizer = struct { result.id = Token.Id.LineComment; state = State.LineComment; }, + '=' => { + result.id = Token.Id.SlashEqual; + self.index += 1; + break; + }, else => { result.id = Token.Id.Slash; break; @@ -609,6 +850,42 @@ pub const Tokenizer = struct { State.Pipe => { result.id = Token.Id.Pipe; }, + State.AngleBracketAngleBracketRight => { + result.id = Token.Id.AngleBracketAngleBracketRight; + }, + State.AngleBracketRight => { + result.id = Token.Id.AngleBracketRight; + }, + State.AngleBracketAngleBracketLeft => { + result.id = Token.Id.AngleBracketAngleBracketLeft; + }, + State.AngleBracketLeft => { + result.id = Token.Id.AngleBracketLeft; + }, + State.PlusPercent => { + result.id = Token.Id.PlusPercent; + }, + State.Plus => { + result.id = Token.Id.Plus; + }, + State.QuestionMark => { + result.id = Token.Id.QuestionMark; + }, + State.Percent => { + result.id = Token.Id.Percent; + }, + State.Caret => { + result.id = Token.Id.Caret; + }, + State.AsteriskPercent => { + result.id = Token.Id.AsteriskPercent; + }, + State.Asterisk => { + result.id = Token.Id.Asterisk; + }, + State.MinusPercent => { + result.id = Token.Id.MinusPercent; + }, } } if (result.id == Token.Id.Eof) { @@ -752,8 +1029,8 @@ test "tokenizer - string identifier and builtin fns" { test "tokenizer - pipe and then invalid" { testTokenize("||=", []Token.Id{ - Token.Id.Pipe, - Token.Id.PipeEqual, + Token.Id.PipePipe, + Token.Id.Equal, }); } From 8ada03097117cd70800f9e5e7cce97226107f79e Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Thu, 29 Mar 2018 22:37:54 +0200 Subject: [PATCH 24/47] Fixed self hosted compiler compiler error from prev commit --- std/zig/ast.zig | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 59ba0d79ca..36952ec3c3 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -363,9 +363,47 @@ pub const NodeInfixOp = struct { i -= 1; switch (self.op) { - InfixOp.EqualEqual, + InfixOp.Add, + InfixOp.AddWrap, + InfixOp.ArrayCat, + InfixOp.ArrayMult, + InfixOp.Assign, + InfixOp.AssignBitAnd, + InfixOp.AssignBitOr, + InfixOp.AssignBitShiftLeft, + InfixOp.AssignBitShiftRight, + InfixOp.AssignBitXor, + InfixOp.AssignDiv, + InfixOp.AssignMinus, + InfixOp.AssignMinusWrap, + InfixOp.AssignMod, + InfixOp.AssignPlus, + InfixOp.AssignPlusWrap, + InfixOp.AssignTimes, + InfixOp.AssignTimesWarp, InfixOp.BangEqual, - InfixOp.Period => {}, + InfixOp.BitAnd, + InfixOp.BitOr, + InfixOp.BitShiftLeft, + InfixOp.BitShiftRight, + InfixOp.BitXor, + InfixOp.BoolAnd, + InfixOp.BoolOr, + InfixOp.Div, + InfixOp.EqualEqual, + InfixOp.ErrorUnion, + InfixOp.GreaterOrEqual, + InfixOp.GreaterThan, + InfixOp.LessOrEqual, + InfixOp.LessThan, + InfixOp.MergeErrorSets, + InfixOp.Mod, + InfixOp.Mult, + InfixOp.MultWrap, + InfixOp.Period, + InfixOp.Sub, + InfixOp.SubWrap, + InfixOp.UnwrapMaybe => {}, } if (i < 1) return self.rhs; From edca1739975f69a695d1ee12ba88ebca1778fefb Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Thu, 29 Mar 2018 23:40:46 +0200 Subject: [PATCH 25/47] std.zig.parser now parses call expr --- std/zig/ast.zig | 31 ++++++++++++++++++++++++++++++ std/zig/parser.zig | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 36952ec3c3..0e41836f07 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -22,6 +22,7 @@ pub const Node = struct { StringLiteral, UndefinedLiteral, BuiltinCall, + Call, LineComment, TestDecl, }; @@ -41,6 +42,7 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), + Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), }; @@ -61,6 +63,7 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), + Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), }; @@ -81,6 +84,7 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), + Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), }; @@ -527,6 +531,33 @@ pub const NodeBuiltinCall = struct { } }; +pub const NodeCall = struct { + base: Node, + callee: &Node, + params: ArrayList(&Node), + rparen_token: Token, + + pub fn iterate(self: &NodeCall, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.callee; + i -= 1; + + if (i < self.params.len) return self.params.at(i); + i -= self.params.len; + + return null; + } + + pub fn firstToken(self: &NodeCall) Token { + return self.callee.firstToken(); + } + + pub fn lastToken(self: &NodeCall) Token { + return self.rparen_token; + } +}; + pub const NodeStringLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index a96277df04..33bcbaeda0 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -428,6 +428,29 @@ pub const Parser = struct { try stack.append(State.ExpectOperand); continue; + } else if (token.id == Token.Id.LParen) { + self.putBackToken(token); + + const node = try arena.create(ast.NodeCall); + *node = ast.NodeCall { + .base = self.initNode(ast.Node.Id.Call), + .callee = stack.pop().Operand, + .params = ArrayList(&ast.Node).init(arena), + .rparen_token = undefined, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + try stack.append(State {.ExprListItemOrEnd = &node.params }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.LParen, + .ptr = &node.rparen_token, + }, + }); + continue; + // TODO: Parse postfix operator } else { // no postfix/infix operator after this operand. @@ -1325,6 +1348,21 @@ pub const Parser = struct { } } }, + ast.Node.Id.Call => { + const call = @fieldParentPtr(ast.NodeCall, "base", base); + try stack.append(RenderState { .Text = ")"}); + var i = call.params.len; + while (i != 0) { + i -= 1; + const param_node = call.params.at(i); + try stack.append(RenderState { .Expression = param_node}); + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + try stack.append(RenderState { .Text = "("}); + try stack.append(RenderState { .Expression = call.callee }); + }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), @@ -1610,4 +1648,14 @@ test "zig fmt" { \\} \\ ); + + try testCanonical( + \\test "test calls" { + \\ a(); + \\ a(1); + \\ a(1, 2); + \\ a(1, 2) + a(1, 2); + \\} + \\ + ); } From f586acabdcc4adaeda7bc278463c20e4033e8cc9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 30 Mar 2018 13:20:13 -0400 Subject: [PATCH 26/47] add ZIG_STATIC cmake option it's not compatible with glibc but it works with musl --- CMakeLists.txt | 3 +++ cmake/Findllvm.cmake | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fbaf929e85..766a787562 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,7 @@ set(ZIG_LIBC_STATIC_LIB_DIR "" CACHE STRING "Default native target libc director set(ZIG_LIBC_INCLUDE_DIR "/usr/include" CACHE STRING "Default native target libc include directory") set(ZIG_DYNAMIC_LINKER "" CACHE STRING "Override dynamic linker for native target") set(ZIG_EACH_LIB_RPATH off CACHE BOOL "Add each dynamic library to rpath for native target") +set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)") string(REGEX REPLACE "\\\\" "\\\\\\\\" ZIG_LIBC_LIB_DIR_ESCAPED "${ZIG_LIBC_LIB_DIR}") string(REGEX REPLACE "\\\\" "\\\\\\\\" ZIG_LIBC_STATIC_LIB_DIR_ESCAPED "${ZIG_LIBC_STATIC_LIB_DIR}") @@ -698,6 +699,8 @@ if(MINGW) set(EXE_LDFLAGS "-static -static-libgcc -static-libstdc++") elseif(MSVC) set(EXE_LDFLAGS "/STACK:16777216") +elseif(ZIG_STATIC) + set(EXE_LDFLAGS "-static") else() set(EXE_LDFLAGS " ") endif() diff --git a/cmake/Findllvm.cmake b/cmake/Findllvm.cmake index 296e0a91f7..021c3c6f27 100644 --- a/cmake/Findllvm.cmake +++ b/cmake/Findllvm.cmake @@ -15,7 +15,7 @@ find_program(LLVM_CONFIG_EXE "c:/msys64/mingw64/bin" "C:/Libraries/llvm-6.0.0/bin") -if(NOT(CMAKE_BUILD_TYPE STREQUAL "Debug")) +if(NOT(CMAKE_BUILD_TYPE STREQUAL "Debug") OR ZIG_STATIC) execute_process( COMMAND ${LLVM_CONFIG_EXE} --libfiles --link-static OUTPUT_VARIABLE LLVM_LIBRARIES_SPACES From 1dfa927a67566a043e0c1d2982333904d18c3b3a Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 30 Mar 2018 20:47:09 +0200 Subject: [PATCH 27/47] std.zig.parser now treats call expr as a suffix operator --- std/zig/parser.zig | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 33bcbaeda0..3a01f03987 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -86,6 +86,7 @@ pub const Parser = struct { AfterOperand, InfixOp: &ast.NodeInfixOp, PrefixOp: &ast.NodePrefixOp, + SuffixOp: &ast.Node, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, TypeExpr: DestPtr, VarDecl: &ast.NodeVarDecl, @@ -434,13 +435,11 @@ pub const Parser = struct { const node = try arena.create(ast.NodeCall); *node = ast.NodeCall { .base = self.initNode(ast.Node.Id.Call), - .callee = stack.pop().Operand, + .callee = undefined, .params = ArrayList(&ast.Node).init(arena), .rparen_token = undefined, }; - try stack.append(State { - .Operand = &node.base - }); + try stack.append(State { .SuffixOp = &node.base }); try stack.append(State.AfterOperand); try stack.append(State {.ExprListItemOrEnd = &node.params }); try stack.append(State { @@ -455,8 +454,8 @@ pub const Parser = struct { } else { // no postfix/infix operator after this operand. self.putBackToken(token); - // reduce the stack - var expression: &ast.Node = stack.pop().Operand; + + var expression = popSuffixOp(&stack); while (true) { switch (stack.pop()) { State.Expression => |dest_ptr| { @@ -466,7 +465,7 @@ pub const Parser = struct { }, State.InfixOp => |infix_op| { infix_op.rhs = expression; - infix_op.lhs = stack.pop().Operand; + infix_op.lhs = popSuffixOp(&stack); expression = &infix_op.base; continue; }, @@ -716,11 +715,39 @@ pub const Parser = struct { // These are data, not control flow. State.InfixOp => unreachable, State.PrefixOp => unreachable, + State.SuffixOp => unreachable, State.Operand => unreachable, } } } + fn popSuffixOp(stack: &ArrayList(State)) &ast.Node { + var expression: &ast.Node = undefined; + var left_leaf_ptr: &&ast.Node = &expression; + while (true) { + switch (stack.pop()) { + State.SuffixOp => |suffix_op| { + switch (suffix_op.id) { + ast.Node.Id.Call => { + const call = @fieldParentPtr(ast.NodeCall, "base", suffix_op); + *left_leaf_ptr = &call.base; + left_leaf_ptr = &call.callee; + continue; + }, + else => unreachable, + } + }, + State.Operand => |operand| { + *left_leaf_ptr = operand; + break; + }, + else => unreachable, + } + } + + return expression; + } + fn tokenIdToInfixOp(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { return switch (*id) { Token.Id.Ampersand => ast.NodeInfixOp.InfixOp.BitAnd, From 24071c6803ccc0f11b2c4d7c8f22fa8933a98a7c Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 30 Mar 2018 21:45:42 +0200 Subject: [PATCH 28/47] std.zig.parser parses all prefix operators --- std/zig/ast.zig | 18 ++++++++++--- std/zig/parser.zig | 59 ++++++++++++++++++++++++++++++++++++++----- std/zig/tokenizer.zig | 6 +++++ 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 0e41836f07..715a333c0f 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -432,9 +432,15 @@ pub const NodePrefixOp = struct { rhs: &Node, const PrefixOp = union(enum) { + AddrOf: AddrOfInfo, + BitNot, + BoolNot, + Deref, + Negation, + NegationWrap, Return, Try, - AddrOf: AddrOfInfo, + UnwrapMaybe, }; const AddrOfInfo = struct { align_expr: ?&Node, @@ -448,14 +454,20 @@ pub const NodePrefixOp = struct { var i = index; switch (self.op) { - PrefixOp.Return, - PrefixOp.Try => {}, PrefixOp.AddrOf => |addr_of_info| { if (addr_of_info.align_expr) |align_expr| { if (i < 1) return align_expr; i -= 1; } }, + PrefixOp.BitNot, + PrefixOp.BoolNot, + PrefixOp.Deref, + PrefixOp.Negation, + PrefixOp.NegationWrap, + PrefixOp.Return, + PrefixOp.Try, + PrefixOp.UnwrapMaybe => {}, } if (i < 1) return self.rhs; diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3a01f03987..62c62ed185 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -336,6 +336,42 @@ pub const Parser = struct { try stack.append(State.ExpectOperand); continue; }, + Token.Id.Minus => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.Negation) }); + try stack.append(State.ExpectOperand); + continue; + }, + Token.Id.MinusPercent => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.NegationWrap) }); + try stack.append(State.ExpectOperand); + continue; + }, + Token.Id.Tilde => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.BitNot) }); + try stack.append(State.ExpectOperand); + continue; + }, + Token.Id.QuestionMarkQuestionMark => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.UnwrapMaybe) }); + try stack.append(State.ExpectOperand); + continue; + }, + Token.Id.Bang => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.BoolNot) }); + try stack.append(State.ExpectOperand); + continue; + }, + Token.Id.Asterisk => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.Deref) }); + try stack.append(State.ExpectOperand); + continue; + }, Token.Id.Ampersand => { const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ .AddrOf = ast.NodePrefixOp.AddrOfInfo { @@ -1323,12 +1359,6 @@ pub const Parser = struct { const prefix_op_node = @fieldParentPtr(ast.NodePrefixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); switch (prefix_op_node.op) { - ast.NodePrefixOp.PrefixOp.Return => { - try stream.write("return "); - }, - ast.NodePrefixOp.PrefixOp.Try => { - try stream.write("try "); - }, ast.NodePrefixOp.PrefixOp.AddrOf => |addr_of_info| { try stream.write("&"); if (addr_of_info.volatile_token != null) { @@ -1343,6 +1373,14 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = align_expr}); } }, + ast.NodePrefixOp.PrefixOp.BitNot => try stream.write("~"), + ast.NodePrefixOp.PrefixOp.BoolNot => try stream.write("!"), + ast.NodePrefixOp.PrefixOp.Deref => try stream.write("*"), + ast.NodePrefixOp.PrefixOp.Negation => try stream.write("-"), + ast.NodePrefixOp.PrefixOp.NegationWrap => try stream.write("-%"), + ast.NodePrefixOp.PrefixOp.Return => try stream.write("return "), + ast.NodePrefixOp.PrefixOp.Try => try stream.write("try "), + ast.NodePrefixOp.PrefixOp.UnwrapMaybe => try stream.write("??"), } }, ast.Node.Id.IntegerLiteral => { @@ -1627,7 +1665,7 @@ test "zig fmt" { ); try testCanonical( - \\test "operators" { + \\test "infix operators" { \\ var i = undefined; \\ i = 2; \\ i *= 2; @@ -1676,6 +1714,13 @@ test "zig fmt" { \\ ); + try testCanonical( + \\test "prefix operators" { + \\ --%~??!*&0; + \\} + \\ + ); + try testCanonical( \\test "test calls" { \\ a(); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 4b1e35f62f..7a13d89975 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -125,6 +125,7 @@ pub const Token = struct { AngleBracketRightEqual, AngleBracketAngleBracketRight, AngleBracketAngleBracketRightEqual, + Tilde, IntegerLiteral, FloatLiteral, LineComment, @@ -361,6 +362,11 @@ pub const Tokenizer = struct { self.index += 1; break; }, + '~' => { + result.id = Token.Id.Tilde; + self.index += 1; + break; + }, '.' => { state = State.Period; }, From b01c50d6fae581050affdc438942b979ae54a8da Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 30 Mar 2018 17:10:54 -0400 Subject: [PATCH 29/47] find libc and zig std lib at runtime this removes the following configure options: * ZIG_LIBC_LIB_DIR * ZIG_LIBC_STATIC_LIB_DIR * ZIG_LIBC_INCLUDE_DIR * ZIG_DYNAMIC_LINKER * ZIG_EACH_LIB_RPATH * zig's reliance on CMAKE_INSTALL_PREFIX these options are still available as command line options, however, the default will attempt to execute the system's C compiler to collect system defaults for these values. closes #870 --- CMakeLists.txt | 5 --- README.md | 8 +--- src/analyze.cpp | 116 ++++++++++++++++++++++++++++++++++++++++++++---- src/codegen.cpp | 19 ++++---- src/config.h.in | 8 ---- src/link.cpp | 44 +++++++++++++++--- src/main.cpp | 7 --- src/os.cpp | 17 ++++--- src/target.cpp | 4 ++ 9 files changed, 171 insertions(+), 57 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 766a787562..1acc789de9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,11 +30,6 @@ if(GIT_EXE) endif() message("Configuring zig version ${ZIG_VERSION}") -set(ZIG_LIBC_LIB_DIR "" CACHE STRING "Default native target libc directory where crt1.o can be found") -set(ZIG_LIBC_STATIC_LIB_DIR "" CACHE STRING "Default native target libc directory where crtbeginT.o can be found") -set(ZIG_LIBC_INCLUDE_DIR "/usr/include" CACHE STRING "Default native target libc include directory") -set(ZIG_DYNAMIC_LINKER "" CACHE STRING "Override dynamic linker for native target") -set(ZIG_EACH_LIB_RPATH off CACHE BOOL "Add each dynamic library to rpath for native target") set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)") string(REGEX REPLACE "\\\\" "\\\\\\\\" ZIG_LIBC_LIB_DIR_ESCAPED "${ZIG_LIBC_LIB_DIR}") diff --git a/README.md b/README.md index acc7e891e7..d1e7e8f6d0 100644 --- a/README.md +++ b/README.md @@ -138,14 +138,10 @@ libc. Create demo games using Zig. ##### POSIX -If you have gcc or clang installed, you can find out what `ZIG_LIBC_LIB_DIR`, -`ZIG_LIBC_STATIC_LIB_DIR`, and `ZIG_LIBC_INCLUDE_DIR` should be set to -(example below). - ``` mkdir build cd build -cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) -DZIG_LIBC_LIB_DIR=$(dirname $(cc -print-file-name=crt1.o)) -DZIG_LIBC_INCLUDE_DIR=$(echo -n | cc -E -x c - -v 2>&1 | grep -B1 "End of search list." | head -n1 | cut -c 2- | sed "s/ .*//") -DZIG_LIBC_STATIC_LIB_DIR=$(dirname $(cc -print-file-name=crtbegin.o)) +cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) make make install ./zig build --build-file ../build.zig test @@ -153,8 +149,6 @@ make install ##### MacOS -`ZIG_LIBC_LIB_DIR` and `ZIG_LIBC_STATIC_LIB_DIR` are unused. - ``` brew install cmake llvm@6 brew outdated llvm@6 || brew upgrade llvm@6 diff --git a/src/analyze.cpp b/src/analyze.cpp index 2128339726..5dd15a6943 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -4285,24 +4285,117 @@ static ZigWindowsSDK *get_windows_sdk(CodeGen *g) { return g->win_sdk; } + +Buf *get_linux_libc_lib_path(const char *o_file) { + const char *cc_exe = getenv("CC"); + cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe; + ZigList args = {}; + args.append(buf_ptr(buf_sprintf("-print-file-name=%s", o_file))); + Termination term; + Buf *out_stderr = buf_alloc(); + Buf *out_stdout = buf_alloc(); + int err; + if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) { + zig_panic("unable to determine libc lib path: executing C compiler: %s", err_str(err)); + } + if (term.how != TerminationIdClean || term.code != 0) { + zig_panic("unable to determine libc lib path: executing C compiler command failed"); + } + if (buf_ends_with_str(out_stdout, "\n")) { + buf_resize(out_stdout, buf_len(out_stdout) - 1); + } + if (buf_len(out_stdout) == 0 || buf_eql_str(out_stdout, o_file)) { + zig_panic("unable to determine libc lib path: C compiler could not find %s", o_file); + } + Buf *result = buf_alloc(); + os_path_dirname(out_stdout, result); + return result; +} + +Buf *get_linux_libc_include_path(void) { + const char *cc_exe = getenv("CC"); + cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe; + ZigList args = {}; + args.append("-E"); + args.append("-Wp,-v"); + args.append("-xc"); + args.append("/dev/null"); + Termination term; + Buf *out_stderr = buf_alloc(); + Buf *out_stdout = buf_alloc(); + int err; + if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) { + zig_panic("unable to determine libc include path: executing C compiler: %s", err_str(err)); + } + if (term.how != TerminationIdClean || term.code != 0) { + zig_panic("unable to determine libc include path: executing C compiler command failed"); + } + char *prev_newline = buf_ptr(out_stderr); + ZigList search_paths = {}; + bool found_search_paths = false; + for (;;) { + char *newline = strchr(prev_newline, '\n'); + if (newline == nullptr) { + zig_panic("unable to determine libc include path: bad output from C compiler command"); + } + *newline = 0; + if (found_search_paths) { + if (strcmp(prev_newline, "End of search list.") == 0) { + break; + } + search_paths.append(prev_newline); + } else { + if (strcmp(prev_newline, "#include <...> search starts here:") == 0) { + found_search_paths = true; + } + } + prev_newline = newline + 1; + } + if (search_paths.length == 0) { + zig_panic("unable to determine libc include path: even C compiler does not know where libc headers are"); + } + for (size_t i = 0; i < search_paths.length; i += 1) { + // search in reverse order + const char *search_path = search_paths.items[search_paths.length - i - 1]; + // cut off spaces + while (*search_path == ' ') { + search_path += 1; + } + Buf *stdlib_path = buf_sprintf("%s/stdlib.h", search_path); + bool exists; + if ((err = os_file_exists(stdlib_path, &exists))) { + exists = false; + } + if (exists) { + return buf_create_from_str(search_path); + } + } + zig_panic("unable to determine libc include path: stdlib.h not found in C compiler search paths"); +} + void find_libc_include_path(CodeGen *g) { - if (!g->libc_include_dir || buf_len(g->libc_include_dir) == 0) { + if (g->libc_include_dir == nullptr) { if (g->zig_target.os == OsWindows) { ZigWindowsSDK *sdk = get_windows_sdk(g); if (os_get_win32_ucrt_include_path(sdk, g->libc_include_dir)) { zig_panic("Unable to determine libc include path."); } + } else if (g->zig_target.os == OsLinux) { + g->libc_include_dir = get_linux_libc_include_path(); + } else if (g->zig_target.os == OsMacOSX) { + g->libc_include_dir = buf_create_from_str("/usr/include"); + } else { + // TODO find libc at runtime for other operating systems + zig_panic("Unable to determine libc include path."); } - - // TODO find libc at runtime for other operating systems - zig_panic("Unable to determine libc include path."); } + assert(buf_len(g->libc_include_dir) != 0); } void find_libc_lib_path(CodeGen *g) { // later we can handle this better by reporting an error via the normal mechanism - if (!g->libc_lib_dir || buf_len(g->libc_lib_dir) == 0 || + if (g->libc_lib_dir == nullptr || (g->zig_target.os == OsWindows && (g->msvc_lib_dir == nullptr || g->kernel32_lib_dir == nullptr))) { if (g->zig_target.os == OsWindows) { @@ -4326,18 +4419,25 @@ void find_libc_lib_path(CodeGen *g) { g->msvc_lib_dir = vc_lib_dir; g->libc_lib_dir = ucrt_lib_path; g->kernel32_lib_dir = kern_lib_path; + } else if (g->zig_target.os == OsLinux) { + g->libc_lib_dir = get_linux_libc_lib_path("crt1.o"); } else { zig_panic("Unable to determine libc lib path."); } + } else { + assert(buf_len(g->libc_lib_dir) != 0); } - if (!g->libc_static_lib_dir || buf_len(g->libc_static_lib_dir) == 0) { + if (g->libc_static_lib_dir == nullptr) { if ((g->zig_target.os == OsWindows) && (g->msvc_lib_dir != NULL)) { return; - } - else { + } else if (g->zig_target.os == OsLinux) { + g->libc_static_lib_dir = get_linux_libc_lib_path("crtbegin.o"); + } else { zig_panic("Unable to determine libc static lib path."); } + } else { + assert(buf_len(g->libc_static_lib_dir) != 0); } } diff --git a/src/codegen.cpp b/src/codegen.cpp index 25b2ffbf16..0ebdf7fa3d 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -112,10 +112,10 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out // that's for native compilation g->zig_target = *target; resolve_target_object_format(&g->zig_target); - g->dynamic_linker = buf_create_from_str(""); - g->libc_lib_dir = buf_create_from_str(""); - g->libc_static_lib_dir = buf_create_from_str(""); - g->libc_include_dir = buf_create_from_str(""); + g->dynamic_linker = nullptr; + g->libc_lib_dir = nullptr; + g->libc_static_lib_dir = nullptr; + g->libc_include_dir = nullptr; g->msvc_lib_dir = nullptr; g->kernel32_lib_dir = nullptr; g->each_lib_rpath = false; @@ -123,16 +123,13 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out // native compilation, we can rely on the configuration stuff g->is_native_target = true; get_native_target(&g->zig_target); - g->dynamic_linker = buf_create_from_str(ZIG_DYNAMIC_LINKER); - g->libc_lib_dir = buf_create_from_str(ZIG_LIBC_LIB_DIR); - g->libc_static_lib_dir = buf_create_from_str(ZIG_LIBC_STATIC_LIB_DIR); - g->libc_include_dir = buf_create_from_str(ZIG_LIBC_INCLUDE_DIR); + g->dynamic_linker = nullptr; // find it at runtime + g->libc_lib_dir = nullptr; // find it at runtime + g->libc_static_lib_dir = nullptr; // find it at runtime + g->libc_include_dir = nullptr; // find it at runtime g->msvc_lib_dir = nullptr; // find it at runtime g->kernel32_lib_dir = nullptr; // find it at runtime - -#ifdef ZIG_EACH_LIB_RPATH g->each_lib_rpath = true; -#endif if (g->zig_target.os == OsMacOSX || g->zig_target.os == OsIOS) diff --git a/src/config.h.in b/src/config.h.in index 1fcc3fe12c..16896273b3 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -13,14 +13,6 @@ #define ZIG_VERSION_PATCH @ZIG_VERSION_PATCH@ #define ZIG_VERSION_STRING "@ZIG_VERSION@" -#define ZIG_INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@" -#define ZIG_LIBC_INCLUDE_DIR "@ZIG_LIBC_INCLUDE_DIR_ESCAPED@" -#define ZIG_LIBC_LIB_DIR "@ZIG_LIBC_LIB_DIR_ESCAPED@" -#define ZIG_LIBC_STATIC_LIB_DIR "@ZIG_LIBC_STATIC_LIB_DIR_ESCAPED@" -#define ZIG_DYNAMIC_LINKER "@ZIG_DYNAMIC_LINKER@" - -#cmakedefine ZIG_EACH_LIB_RPATH - // Only used for running tests before installing. #define ZIG_TEST_DIR "@CMAKE_SOURCE_DIR@/test" diff --git a/src/link.cpp b/src/link.cpp index f0537ffa0f..57fa59b675 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -164,6 +164,34 @@ static void add_rpath(LinkJob *lj, Buf *rpath) { lj->rpath_table.put(rpath, true); } +static Buf *get_dynamic_linker_path(CodeGen *g) { + if (g->is_native_target && g->zig_target.arch.arch == ZigLLVM_x86_64) { + const char *cc_exe = getenv("CC"); + cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe; + ZigList args = {}; + args.append("-print-file-name=ld-linux-x86-64.so.2"); + Termination term; + Buf *out_stderr = buf_alloc(); + Buf *out_stdout = buf_alloc(); + int err; + if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) { + return target_dynamic_linker(&g->zig_target); + } + if (term.how != TerminationIdClean || term.code != 0) { + return target_dynamic_linker(&g->zig_target); + } + if (buf_ends_with_str(out_stdout, "\n")) { + buf_resize(out_stdout, buf_len(out_stdout) - 1); + } + if (buf_len(out_stdout) == 0 || buf_eql_str(out_stdout, "ld-linux-x86-64.so.2")) { + return target_dynamic_linker(&g->zig_target); + } + return out_stdout; + } else { + return target_dynamic_linker(&g->zig_target); + } +} + static void construct_linker_job_elf(LinkJob *lj) { CodeGen *g = lj->codegen; @@ -259,12 +287,16 @@ static void construct_linker_job_elf(LinkJob *lj) { lj->args.append(buf_ptr(g->libc_static_lib_dir)); } - if (g->dynamic_linker && buf_len(g->dynamic_linker) > 0) { - lj->args.append("-dynamic-linker"); - lj->args.append(buf_ptr(g->dynamic_linker)); - } else { - lj->args.append("-dynamic-linker"); - lj->args.append(buf_ptr(target_dynamic_linker(&g->zig_target))); + if (!g->is_static) { + if (g->dynamic_linker != nullptr) { + assert(buf_len(g->dynamic_linker) != 0); + lj->args.append("-dynamic-linker"); + lj->args.append(buf_ptr(g->dynamic_linker)); + } else { + Buf *resolved_dynamic_linker = get_dynamic_linker_path(g); + lj->args.append("-dynamic-linker"); + lj->args.append(buf_ptr(resolved_dynamic_linker)); + } } if (shared) { diff --git a/src/main.cpp b/src/main.cpp index 791cb3b1b5..f2f4cc970e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -195,13 +195,6 @@ static int find_zig_lib_dir(Buf *out_path) { } } - if (ZIG_INSTALL_PREFIX != nullptr) { - if (test_zig_install_prefix(buf_create_from_str(ZIG_INSTALL_PREFIX), out_path)) { - return 0; - } - } - - return ErrorFileNotFound; } diff --git a/src/os.cpp b/src/os.cpp index e312854612..90eecef3cf 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -57,10 +57,6 @@ static clock_serv_t cclock; #include #include -// these implementations are lazy. But who cares, we'll make a robust -// implementation in the zig standard library and then this code all gets -// deleted when we self-host. it works for now. - #if defined(ZIG_OS_POSIX) static void populate_termination(Termination *term, int status) { if (WIFEXITED(status)) { @@ -929,7 +925,18 @@ int os_self_exe_path(Buf *out_path) { #elif defined(ZIG_OS_DARWIN) return ErrorFileNotFound; #elif defined(ZIG_OS_LINUX) - return ErrorFileNotFound; + buf_resize(out_path, 256); + for (;;) { + ssize_t amt = readlink("/proc/self/exe", buf_ptr(out_path), buf_len(out_path)); + if (amt == -1) { + return ErrorUnexpected; + } + if (amt == (ssize_t)buf_len(out_path)) { + buf_resize(out_path, buf_len(out_path) * 2); + continue; + } + return 0; + } #endif return ErrorFileNotFound; } diff --git a/src/target.cpp b/src/target.cpp index 8e7c5ce578..5008b51a09 100644 --- a/src/target.cpp +++ b/src/target.cpp @@ -863,6 +863,10 @@ Buf *target_dynamic_linker(ZigTarget *target) { env == ZigLLVM_GNUX32) { return buf_create_from_str("/libx32/ld-linux-x32.so.2"); + } else if (arch == ZigLLVM_x86_64 && + (env == ZigLLVM_Musl || env == ZigLLVM_MuslEABI || env == ZigLLVM_MuslEABIHF)) + { + return buf_create_from_str("/lib/ld-musl-x86_64.so.1"); } else { return buf_create_from_str("/lib64/ld-linux-x86-64.so.2"); } From 5d5feb11deb797bbecb8f3676457a74056f09496 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 30 Mar 2018 17:24:56 -0400 Subject: [PATCH 30/47] appveyor and travis ci: stop passing unused configure args --- ci/appveyor/build_script.bat | 4 +--- ci/travis_linux_script | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/appveyor/build_script.bat b/ci/appveyor/build_script.bat index 5ae47df5ec..9aee7a7bf0 100644 --- a/ci/appveyor/build_script.bat +++ b/ci/appveyor/build_script.bat @@ -20,9 +20,7 @@ call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_ mkdir %ZIGBUILDDIR% cd %ZIGBUILDDIR% -cmake.exe .. -Thost=x64 -G"Visual Studio 14 2015 Win64" "-DCMAKE_INSTALL_PREFIX=%ZIGBUILDDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release "-DZIG_LIBC_INCLUDE_DIR=C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt" "-DZIG_LIBC_LIB_DIR=C:\Program Files (x86)\Windows Kits\10\bin\x64\ucrt" "-DZIG_LIBC_STATIC_LIB_DIR=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64" || exit /b +cmake.exe .. -Thost=x64 -G"Visual Studio 14 2015 Win64" "-DCMAKE_INSTALL_PREFIX=%ZIGBUILDDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release || exit /b msbuild /p:Configuration=Release INSTALL.vcxproj || exit /b bin\zig.exe build --build-file ..\build.zig test || exit /b - -@echo "MSVC build succeeded" diff --git a/ci/travis_linux_script b/ci/travis_linux_script index d6b9eb9230..a709539bc7 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -8,7 +8,7 @@ export CXX=clang++-6.0 echo $PATH mkdir build cd build -cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) -DZIG_LIBC_LIB_DIR=$(dirname $($CC -print-file-name=crt1.o)) -DZIG_LIBC_INCLUDE_DIR=$(echo -n | $CC -E -x c - -v 2>&1 | grep -B1 "End of search list." | head -n1 | cut -c 2- | sed "s/ .*//") -DZIG_LIBC_STATIC_LIB_DIR=$(dirname $($CC -print-file-name=crtbegin.o)) +cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) make VERBOSE=1 make install ./zig build --build-file ../build.zig test From c3724ec506a9e3a4b23a145a733050c17321da9d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 31 Mar 2018 02:12:44 -0400 Subject: [PATCH 31/47] implement os_self_exe_path in the c++ compiler for darwin ported from the zig std lib this fixes looking for zig std lib at runtime on darwin --- src/os.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/os.cpp b/src/os.cpp index 90eecef3cf..de1a1f21d2 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -45,6 +45,7 @@ typedef SSIZE_T ssize_t; #if defined(__MACH__) #include #include +#include #endif #if defined(ZIG_OS_WINDOWS) @@ -923,7 +924,13 @@ int os_self_exe_path(Buf *out_path) { } #elif defined(ZIG_OS_DARWIN) - return ErrorFileNotFound; + uint32_t u32_len = 0; + int ret1 = _NSGetExecutablePath(nullptr, &u32_len); + assert(ret1 != 0); + buf_resize(out_path, u32_len); + int ret2 = _NSGetExecutablePath(buf_ptr(out_path), &u32_len); + assert(ret2 == 0); + return 0; #elif defined(ZIG_OS_LINUX) buf_resize(out_path, 256); for (;;) { From 7d66908f294eed1138802c060185721a2e265f3b Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Sat, 31 Mar 2018 23:17:02 +1300 Subject: [PATCH 32/47] docs: fix unclosed code tag --- doc/langref.html.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 04ca5314ca..7f837186b5 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -2873,7 +2873,7 @@ const err = (error {FileNotFound}).FileNotFound; {#header_close#} {#header_open|Error Union Type#}

- An error set type and normal type can be combined with the ! + An error set type and normal type can be combined with the ! binary operator to form an error union type. You are likely to use an error union type more often than an error set type by itself.

From 8f962a957a3645342fba8219cf8f33d0ac42e16d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 31 Mar 2018 11:26:02 -0400 Subject: [PATCH 33/47] fix regressions on windows --- src/analyze.cpp | 1 + src/link.cpp | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/analyze.cpp b/src/analyze.cpp index 5dd15a6943..291e7e7644 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -4378,6 +4378,7 @@ void find_libc_include_path(CodeGen *g) { if (g->zig_target.os == OsWindows) { ZigWindowsSDK *sdk = get_windows_sdk(g); + g->libc_include_dir = buf_alloc(); if (os_get_win32_ucrt_include_path(sdk, g->libc_include_dir)) { zig_panic("Unable to determine libc include path."); } diff --git a/src/link.cpp b/src/link.cpp index 57fa59b675..c089d69611 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -455,7 +455,9 @@ static void construct_linker_job_coff(LinkJob *lj) { lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->kernel32_lib_dir)))); lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->libc_lib_dir)))); - lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->libc_static_lib_dir)))); + if (g->libc_static_lib_dir != nullptr) { + lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->libc_static_lib_dir)))); + } } if (lj->link_in_crt) { From eb6ff796c17b50969a8f0171ba089f31667b7278 Mon Sep 17 00:00:00 2001 From: Raul Leal Date: Sat, 31 Mar 2018 17:21:19 +0100 Subject: [PATCH 34/47] Fix undeclared identifier error in readUntilDelimiterBuffer and incorrect number of parameters in readUntilDelimiterAlloc (#877) --- std/io.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/std/io.zig b/std/io.zig index 94685c4d03..952d9081f2 100644 --- a/std/io.zig +++ b/std/io.zig @@ -144,7 +144,7 @@ pub fn InStream(comptime ReadError: type) type { /// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and the contents /// read from the stream so far are lost. pub fn readUntilDelimiterBuffer(self: &Self, buffer: &Buffer, delimiter: u8, max_size: usize) !void { - try buf.resize(0); + try buffer.resize(0); while (true) { var byte: u8 = try self.readByte(); @@ -153,11 +153,11 @@ pub fn InStream(comptime ReadError: type) type { return; } - if (buf.len() == max_size) { + if (buffer.len() == max_size) { return error.StreamTooLong; } - try buf.appendByte(byte); + try buffer.appendByte(byte); } } @@ -171,7 +171,7 @@ pub fn InStream(comptime ReadError: type) type { var buf = Buffer.initNull(allocator); defer buf.deinit(); - try self.readUntilDelimiterBuffer(self, &buf, delimiter, max_size); + try self.readUntilDelimiterBuffer(&buf, delimiter, max_size); return buf.toOwnedSlice(); } From 67f11190d10caac1d08b76d60ec28eb2f5173946 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 31 Mar 2018 16:34:55 -0400 Subject: [PATCH 35/47] musl-friendly dynamic linking --- build.zig | 5 +++++ src/link.cpp | 57 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/build.zig b/build.zig index 88775498ca..b72641a2ef 100644 --- a/build.zig +++ b/build.zig @@ -45,6 +45,11 @@ pub fn build(b: &Builder) !void { var exe = b.addExecutable("zig", "src-self-hosted/main.zig"); exe.setBuildMode(mode); + + // This is for finding /lib/libz.a on alpine linux. + // TODO turn this into -Dextra-lib-path=/lib option + exe.addLibPath("/lib"); + exe.addIncludeDir("src"); exe.addIncludeDir(cmake_binary_dir); addCppLib(b, exe, cmake_binary_dir, "zig_cpp"); diff --git a/src/link.cpp b/src/link.cpp index c089d69611..3c6e27e331 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -164,32 +164,45 @@ static void add_rpath(LinkJob *lj, Buf *rpath) { lj->rpath_table.put(rpath, true); } +static Buf *try_dynamic_linker_path(const char *ld_name) { + const char *cc_exe = getenv("CC"); + cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe; + ZigList args = {}; + args.append(buf_ptr(buf_sprintf("-print-file-name=%s", ld_name))); + Termination term; + Buf *out_stderr = buf_alloc(); + Buf *out_stdout = buf_alloc(); + int err; + if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) { + return nullptr; + } + if (term.how != TerminationIdClean || term.code != 0) { + return nullptr; + } + if (buf_ends_with_str(out_stdout, "\n")) { + buf_resize(out_stdout, buf_len(out_stdout) - 1); + } + if (buf_len(out_stdout) == 0 || buf_eql_str(out_stdout, ld_name)) { + return nullptr; + } + return out_stdout; +} + static Buf *get_dynamic_linker_path(CodeGen *g) { if (g->is_native_target && g->zig_target.arch.arch == ZigLLVM_x86_64) { - const char *cc_exe = getenv("CC"); - cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe; - ZigList args = {}; - args.append("-print-file-name=ld-linux-x86-64.so.2"); - Termination term; - Buf *out_stderr = buf_alloc(); - Buf *out_stdout = buf_alloc(); - int err; - if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) { - return target_dynamic_linker(&g->zig_target); + static const char *ld_names[] = { + "ld-linux-x86-64.so.2", + "ld-musl-x86_64.so.1", + }; + for (size_t i = 0; i < array_length(ld_names); i += 1) { + const char *ld_name = ld_names[i]; + Buf *result = try_dynamic_linker_path(ld_name); + if (result != nullptr) { + return result; + } } - if (term.how != TerminationIdClean || term.code != 0) { - return target_dynamic_linker(&g->zig_target); - } - if (buf_ends_with_str(out_stdout, "\n")) { - buf_resize(out_stdout, buf_len(out_stdout) - 1); - } - if (buf_len(out_stdout) == 0 || buf_eql_str(out_stdout, "ld-linux-x86-64.so.2")) { - return target_dynamic_linker(&g->zig_target); - } - return out_stdout; - } else { - return target_dynamic_linker(&g->zig_target); } + return target_dynamic_linker(&g->zig_target); } static void construct_linker_job_elf(LinkJob *lj) { From 2e5115b0687ee9b7078dbf70da7d7070a7c80399 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Sat, 31 Mar 2018 19:04:01 +1300 Subject: [PATCH 36/47] Add run compiler command 'zig run file.zig' builds a file and stores the artifacts in the global cache. On successful compilation the binary is executed. 'zig run file.zig -- a b c' does the same, but passes the arguments a, b and c as runtime arguments to the program. Everything after an '--' are treated as runtime arguments. On a posix system, a shebang can be used to run a zig file directly. An example shebang would be '#!/usr/bin/zig run'. You may not be able pass extra compile arguments currently as part of the shebang. Linux for example treats all arguments after the first as a single argument which will result in an 'invalid command'. Currently there is no customisability for the cache path as a compile argument. For a posix system you can use `TMPDIR=. zig run file.zig` to override, in this case using the current directory for the run cache. The input file is always recompiled, even if it has changed. This is intended to be cached but further discussion/thought needs to go into this. Closes #466. --- src/codegen.cpp | 6 ++-- src/ir.cpp | 4 +-- src/main.cpp | 58 ++++++++++++++++++++++++++++++++------ src/os.cpp | 75 +++++++++++++++++++++++++++++++++++++++++++++---- src/os.hpp | 6 ++-- 5 files changed, 129 insertions(+), 20 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 0ebdf7fa3d..bdd28b86fd 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6286,7 +6286,7 @@ static ImportTableEntry *add_special_code(CodeGen *g, PackageTableEntry *package zig_panic("unable to open '%s': %s", buf_ptr(&path_to_code_src), err_str(err)); } Buf *import_code = buf_alloc(); - if ((err = os_fetch_file_path(abs_full_path, import_code))) { + if ((err = os_fetch_file_path(abs_full_path, import_code, false))) { zig_panic("unable to open '%s': %s", buf_ptr(&path_to_code_src), err_str(err)); } @@ -6374,7 +6374,7 @@ static void gen_root_source(CodeGen *g) { } Buf *source_code = buf_alloc(); - if ((err = os_fetch_file_path(rel_full_path, source_code))) { + if ((err = os_fetch_file_path(rel_full_path, source_code, true))) { zig_panic("unable to open '%s': %s", buf_ptr(rel_full_path), err_str(err)); } @@ -6439,7 +6439,7 @@ static void gen_global_asm(CodeGen *g) { int err; for (size_t i = 0; i < g->assembly_files.length; i += 1) { Buf *asm_file = g->assembly_files.at(i); - if ((err = os_fetch_file_path(asm_file, &contents))) { + if ((err = os_fetch_file_path(asm_file, &contents, false))) { zig_panic("Unable to read %s: %s", buf_ptr(asm_file), err_str(err)); } buf_append_buf(&g->global_asm, &contents); diff --git a/src/ir.cpp b/src/ir.cpp index 18fd02c297..4fe6769f78 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -14709,7 +14709,7 @@ static TypeTableEntry *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructi return ira->codegen->builtin_types.entry_namespace; } - if ((err = os_fetch_file_path(abs_full_path, import_code))) { + if ((err = os_fetch_file_path(abs_full_path, import_code, true))) { if (err == ErrorFileNotFound) { ir_add_error_node(ira, source_node, buf_sprintf("unable to find '%s'", buf_ptr(import_target_path))); @@ -15570,7 +15570,7 @@ static TypeTableEntry *ir_analyze_instruction_embed_file(IrAnalyze *ira, IrInstr // load from file system into const expr Buf *file_contents = buf_alloc(); int err; - if ((err = os_fetch_file_path(&file_path, file_contents))) { + if ((err = os_fetch_file_path(&file_path, file_contents, false))) { if (err == ErrorFileNotFound) { ir_add_error(ira, instruction->name, buf_sprintf("unable to find '%s'", buf_ptr(&file_path))); return ira->codegen->builtin_types.entry_invalid; diff --git a/src/main.cpp b/src/main.cpp index f2f4cc970e..63b077e833 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -23,6 +23,7 @@ static int usage(const char *arg0) { " build-exe [source] create executable from source or object files\n" " build-lib [source] create library from source or object files\n" " build-obj [source] create object from source or assembly\n" + " run [source] create executable and run immediately\n" " translate-c [source] convert c code to zig code\n" " targets list available compilation targets\n" " test [source] create and run a test build\n" @@ -220,6 +221,7 @@ static Buf *resolve_zig_lib_dir(const char *zig_install_prefix_arg) { enum Cmd { CmdInvalid, CmdBuild, + CmdRun, CmdTest, CmdVersion, CmdZen, @@ -329,6 +331,8 @@ int main(int argc, char **argv) { CliPkg *cur_pkg = allocate(1); BuildMode build_mode = BuildModeDebug; ZigList test_exec_args = {0}; + int comptime_args_end = 0; + int runtime_args_start = argc; if (argc >= 2 && strcmp(argv[1], "build") == 0) { const char *zig_exe_path = arg0; @@ -481,11 +485,15 @@ int main(int argc, char **argv) { return (term.how == TerminationIdClean) ? term.code : -1; } - for (int i = 1; i < argc; i += 1) { + for (int i = 1; i < argc; i += 1, comptime_args_end += 1) { char *arg = argv[i]; if (arg[0] == '-') { - if (strcmp(arg, "--release-fast") == 0) { + if (strcmp(arg, "--") == 0) { + // ignore -- from both compile and runtime arg sets + runtime_args_start = i + 1; + break; + } else if (strcmp(arg, "--release-fast") == 0) { build_mode = BuildModeFastRelease; } else if (strcmp(arg, "--release-safe") == 0) { build_mode = BuildModeSafeRelease; @@ -652,6 +660,9 @@ int main(int argc, char **argv) { } else if (strcmp(arg, "build-lib") == 0) { cmd = CmdBuild; out_type = OutTypeLib; + } else if (strcmp(arg, "run") == 0) { + cmd = CmdRun; + out_type = OutTypeExe; } else if (strcmp(arg, "version") == 0) { cmd = CmdVersion; } else if (strcmp(arg, "zen") == 0) { @@ -670,6 +681,7 @@ int main(int argc, char **argv) { } else { switch (cmd) { case CmdBuild: + case CmdRun: case CmdTranslateC: case CmdTest: if (!in_file) { @@ -724,8 +736,8 @@ int main(int argc, char **argv) { } } - switch (cmd) { + case CmdRun: case CmdBuild: case CmdTranslateC: case CmdTest: @@ -733,7 +745,7 @@ int main(int argc, char **argv) { if (cmd == CmdBuild && !in_file && objects.length == 0 && asm_files.length == 0) { fprintf(stderr, "Expected source file argument or at least one --object or --assembly argument.\n"); return usage(arg0); - } else if ((cmd == CmdTranslateC || cmd == CmdTest) && !in_file) { + } else if ((cmd == CmdTranslateC || cmd == CmdTest || cmd == CmdRun) && !in_file) { fprintf(stderr, "Expected source file argument.\n"); return usage(arg0); } else if (cmd == CmdBuild && out_type == OutTypeObj && objects.length != 0) { @@ -745,6 +757,10 @@ int main(int argc, char **argv) { bool need_name = (cmd == CmdBuild || cmd == CmdTranslateC); + if (cmd == CmdRun) { + out_name = "run"; + } + Buf *in_file_buf = nullptr; Buf *buf_out_name = (cmd == CmdTest) ? buf_create_from_str("test") : @@ -769,9 +785,23 @@ int main(int argc, char **argv) { Buf *zig_root_source_file = (cmd == CmdTranslateC) ? nullptr : in_file_buf; Buf *full_cache_dir = buf_alloc(); - os_path_resolve(buf_create_from_str("."), - buf_create_from_str((cache_dir == nullptr) ? default_zig_cache_name : cache_dir), - full_cache_dir); + Buf *run_exec_path = buf_alloc(); + if (cmd == CmdRun) { + if (buf_out_name == nullptr) { + buf_out_name = buf_create_from_str("run"); + } + + Buf *global_cache_dir = buf_alloc(); + os_get_global_cache_directory(global_cache_dir); + os_path_join(global_cache_dir, buf_out_name, run_exec_path); + os_path_resolve(buf_create_from_str("."), global_cache_dir, full_cache_dir); + + out_file = buf_ptr(run_exec_path); + } else { + os_path_resolve(buf_create_from_str("."), + buf_create_from_str((cache_dir == nullptr) ? default_zig_cache_name : cache_dir), + full_cache_dir); + } Buf *zig_lib_dir_buf = resolve_zig_lib_dir(zig_install_prefix); @@ -855,7 +885,7 @@ int main(int argc, char **argv) { add_package(g, cur_pkg, g->root_package); - if (cmd == CmdBuild) { + if (cmd == CmdBuild || cmd == CmdRun) { codegen_set_emit_file_type(g, emit_file_type); for (size_t i = 0; i < objects.length; i += 1) { @@ -868,6 +898,18 @@ int main(int argc, char **argv) { codegen_link(g, out_file); if (timing_info) codegen_print_timing_report(g, stdout); + + if (cmd == CmdRun) { + ZigList args = {0}; + for (int i = runtime_args_start; i < argc; ++i) { + args.append(argv[i]); + } + + Termination term; + os_spawn_process(buf_ptr(run_exec_path), args, &term); + return term.code; + } + return EXIT_SUCCESS; } else if (cmd == CmdTranslateC) { codegen_translate_c(g, in_file_buf); diff --git a/src/os.cpp b/src/os.cpp index de1a1f21d2..e0491b21de 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -288,13 +288,39 @@ void os_path_resolve(Buf *ref_path, Buf *target_path, Buf *out_abs_path) { return; } -int os_fetch_file(FILE *f, Buf *out_buf) { +int os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { static const ssize_t buf_size = 0x2000; buf_resize(out_buf, buf_size); ssize_t actual_buf_len = 0; + + bool first_read = true; + for (;;) { size_t amt_read = fread(buf_ptr(out_buf) + actual_buf_len, 1, buf_size, f); actual_buf_len += amt_read; + + if (skip_shebang && first_read && buf_starts_with_str(out_buf, "#!")) { + size_t i = 0; + while (true) { + if (i > buf_len(out_buf)) { + zig_panic("shebang line exceeded %zd characters", buf_size); + } + + size_t current_pos = i; + i += 1; + + if (out_buf->list.at(current_pos) == '\n') { + break; + } + } + + ZigList *list = &out_buf->list; + memmove(list->items, list->items + i, list->length - i); + list->length -= i; + + actual_buf_len -= i; + } + if (amt_read != buf_size) { if (feof(f)) { buf_resize(out_buf, actual_buf_len); @@ -305,6 +331,7 @@ int os_fetch_file(FILE *f, Buf *out_buf) { } buf_resize(out_buf, actual_buf_len + buf_size); + first_read = false; } zig_unreachable(); } @@ -374,8 +401,8 @@ static int os_exec_process_posix(const char *exe, ZigList &args, FILE *stdout_f = fdopen(stdout_pipe[0], "rb"); FILE *stderr_f = fdopen(stderr_pipe[0], "rb"); - os_fetch_file(stdout_f, out_stdout); - os_fetch_file(stderr_f, out_stderr); + os_fetch_file(stdout_f, out_stdout, false); + os_fetch_file(stderr_f, out_stderr, false); fclose(stdout_f); fclose(stderr_f); @@ -588,7 +615,7 @@ int os_copy_file(Buf *src_path, Buf *dest_path) { } } -int os_fetch_file_path(Buf *full_path, Buf *out_contents) { +int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang) { FILE *f = fopen(buf_ptr(full_path), "rb"); if (!f) { switch (errno) { @@ -607,7 +634,7 @@ int os_fetch_file_path(Buf *full_path, Buf *out_contents) { return ErrorFileSystem; } } - int result = os_fetch_file(f, out_contents); + int result = os_fetch_file(f, out_contents, skip_shebang); fclose(f); return result; } @@ -780,6 +807,44 @@ int os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path) { #endif } +#if defined(ZIG_OS_POSIX) +int os_get_global_cache_directory(Buf *out_tmp_path) { + const char *tmp_dir = getenv("TMPDIR"); + if (!tmp_dir) { + tmp_dir = P_tmpdir; + } + + Buf *tmp_dir_buf = buf_create_from_str(tmp_dir); + Buf *cache_dirname_buf = buf_create_from_str("zig-cache"); + + buf_resize(out_tmp_path, 0); + os_path_join(tmp_dir_buf, cache_dirname_buf, out_tmp_path); + + buf_deinit(tmp_dir_buf); + buf_deinit(cache_dirname_buf); + return 0; +} +#endif + +#if defined(ZIG_OS_WINDOWS) +int os_get_global_cache_directory(Buf *out_tmp_path) { + char tmp_dir[MAX_PATH + 1]; + if (GetTempPath(MAX_PATH, tmp_dir) == 0) { + zig_panic("GetTempPath failed"); + } + + Buf *tmp_dir_buf = buf_create_from_str(tmp_dir); + Buf *cache_dirname_buf = buf_create_from_str("zig-cache"); + + buf_resize(out_tmp_path, 0); + os_path_join(tmp_dir_buf, cache_dirname_buf, out_tmp_path); + + buf_deinit(tmp_dir_buf); + buf_deinit(cache_dirname_buf); + return 0; +} +#endif + int os_delete_file(Buf *path) { if (remove(buf_ptr(path))) { return ErrorFileSystem; diff --git a/src/os.hpp b/src/os.hpp index 5d29db0d07..b94e98ec3d 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -51,14 +51,16 @@ int os_path_real(Buf *rel_path, Buf *out_abs_path); void os_path_resolve(Buf *ref_path, Buf *target_path, Buf *out_abs_path); bool os_path_is_absolute(Buf *path); +int os_get_global_cache_directory(Buf *out_tmp_path); + int os_make_path(Buf *path); int os_make_dir(Buf *path); void os_write_file(Buf *full_path, Buf *contents); int os_copy_file(Buf *src_path, Buf *dest_path); -int os_fetch_file(FILE *file, Buf *out_contents); -int os_fetch_file_path(Buf *full_path, Buf *out_contents); +int os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); +int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); int os_get_cwd(Buf *out_cwd); From 4eb68987d81cc4a1a92ebb4b5e83be73669801ad Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 2 Apr 2018 11:34:31 -0400 Subject: [PATCH 37/47] std.io.readLine function this provides a better input for guess number example. see #882 --- example/guess_number/main.zig | 13 +++++++------ std/io.zig | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/example/guess_number/main.zig b/example/guess_number/main.zig index d26518a1ed..7178c5274a 100644 --- a/example/guess_number/main.zig +++ b/example/guess_number/main.zig @@ -9,8 +9,6 @@ pub fn main() !void { var stdout_file_stream = io.FileOutStream.init(&stdout_file); const stdout = &stdout_file_stream.stream; - var stdin_file = try io.getStdIn(); - try stdout.print("Welcome to the Guess Number Game in Zig.\n"); var seed_bytes: [@sizeOf(u64)]u8 = undefined; @@ -27,12 +25,15 @@ pub fn main() !void { try stdout.print("\nGuess a number between 1 and 100: "); var line_buf : [20]u8 = undefined; - const line_len = stdin_file.read(line_buf[0..]) catch |err| { - try stdout.print("Unable to read from stdin: {}\n", @errorName(err)); - return err; + const line_len = io.readLine(line_buf[0..]) catch |err| switch (err) { + error.InputTooLong => { + try stdout.print("Input too long.\n"); + continue; + }, + error.EndOfFile, error.StdInUnavailable => return err, }; - const guess = fmt.parseUnsigned(u8, line_buf[0..line_len - 1], 10) catch { + const guess = fmt.parseUnsigned(u8, line_buf[0..line_len], 10) catch { try stdout.print("Invalid number.\n"); continue; }; diff --git a/std/io.zig b/std/io.zig index 952d9081f2..93d50e6709 100644 --- a/std/io.zig +++ b/std/io.zig @@ -478,3 +478,20 @@ test "import io tests" { } } +pub fn readLine(buf: []u8) !usize { + var stdin = getStdIn() catch return error.StdInUnavailable; + var adapter = FileInStream.init(&stdin); + var stream = &adapter.stream; + var index: usize = 0; + while (true) { + const byte = stream.readByte() catch return error.EndOfFile; + switch (byte) { + '\n' => return index, + else => { + if (index == buf.len) return error.InputTooLong; + buf[index] = byte; + index += 1; + }, + } + } +} From aadc14fd78c4137ffcc02e3ba67c0c60100b4d69 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 10:17:57 -0400 Subject: [PATCH 38/47] upload static linux artifacts on successful travis build --- .travis.yml | 8 ++++++++ ci/travis_linux_script | 22 +++------------------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index 271b6069b5..df6f60a48e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,11 @@ +sudo: required +services: + - docker +addons: + artifacts: + working_dir: artifacts + target_paths: + - /builds os: - linux - osx diff --git a/ci/travis_linux_script b/ci/travis_linux_script index a709539bc7..55e188d55a 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -9,24 +9,8 @@ echo $PATH mkdir build cd build cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) -make VERBOSE=1 -make install +make -j2 install ./zig build --build-file ../build.zig test -./zig test ../test/behavior.zig --target-os windows --target-arch i386 --target-environ msvc -wine zig-cache/test.exe - -./zig test ../test/behavior.zig --target-os windows --target-arch i386 --target-environ msvc --release-fast -wine zig-cache/test.exe - -./zig test ../test/behavior.zig --target-os windows --target-arch i386 --target-environ msvc --release-safe -wine zig-cache/test.exe - -./zig test ../test/behavior.zig --target-os windows --target-arch x86_64 --target-environ msvc -wine64 zig-cache/test.exe - -#./zig test ../test/behavior.zig --target-os windows --target-arch x86_64 --target-environ msvc --release-fast -#wine64 test.exe -# -#./zig test ../test/behavior.zig --target-os windows --target-arch x86_64 --target-environ msvc --release-safe -#wine64 test.exe +mkdir $TRAVIS_BUILD_DIR/artifacts +docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR/artifacts",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT From 65e4bb149e2226e3d796c6378c74d9af3af81828 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 12:04:06 -0400 Subject: [PATCH 39/47] travis artifacts: don't upload extra stuff --- ci/travis_linux_script | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/travis_linux_script b/ci/travis_linux_script index 55e188d55a..b88c29c602 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -12,5 +12,6 @@ cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) make -j2 install ./zig build --build-file ../build.zig test -mkdir $TRAVIS_BUILD_DIR/artifacts -docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR/artifacts",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT +cd $TRAVIS_BUILD_DIR +git clean -fd +docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT From 21b47b34d85f6b974c371b6e5b9421a87650344a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 12:59:28 -0400 Subject: [PATCH 40/47] travis: don't upload build/ folder as artifacts --- ci/travis_linux_script | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/travis_linux_script b/ci/travis_linux_script index b88c29c602..8259477019 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -14,4 +14,5 @@ make -j2 install cd $TRAVIS_BUILD_DIR git clean -fd +rm -rf build docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT From d1f8e722b5e48666b4d7d833c0cd6b36f45bd9bb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 14:23:56 -0400 Subject: [PATCH 41/47] travis: don't upload other files as artifacts --- ci/travis_linux_script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/travis_linux_script b/ci/travis_linux_script index 8259477019..f80ac33c24 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -13,6 +13,6 @@ make -j2 install ./zig build --build-file ../build.zig test cd $TRAVIS_BUILD_DIR +rm .gitignore git clean -fd -rm -rf build docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT From 9dfd1a7c8a79cd5213878c56695f2fcb8aa25580 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 18:26:49 -0400 Subject: [PATCH 42/47] remove more signal handling stuff from std.os.ChildProcess 439621e44a68b436f958a84fcdb0bdac83613aea failed to remove everything. this finishes the job --- std/os/child_process.zig | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/std/os/child_process.zig b/std/os/child_process.zig index 06802e657c..8bb8b2d7e7 100644 --- a/std/os/child_process.zig +++ b/std/os/child_process.zig @@ -13,8 +13,6 @@ const builtin = @import("builtin"); const Os = builtin.Os; const LinkedList = std.LinkedList; -var children_nodes = LinkedList(&ChildProcess).init(); - const is_windows = builtin.os == Os.windows; pub const ChildProcess = struct { @@ -296,8 +294,6 @@ pub const ChildProcess = struct { } fn cleanupAfterWait(self: &ChildProcess, status: i32) !Term { - children_nodes.remove(&self.llnode); - defer { os.close(self.err_pipe[0]); os.close(self.err_pipe[1]); @@ -427,9 +423,6 @@ pub const ChildProcess = struct { self.llnode = LinkedList(&ChildProcess).Node.init(self); self.term = null; - // TODO make this atomic so it works even with threads - children_nodes.prepend(&self.llnode); - if (self.stdin_behavior == StdIo.Pipe) { os.close(stdin_pipe[0]); } if (self.stdout_behavior == StdIo.Pipe) { os.close(stdout_pipe[1]); } if (self.stderr_behavior == StdIo.Pipe) { os.close(stderr_pipe[1]); } @@ -773,31 +766,3 @@ fn readIntFd(fd: i32) !ErrInt { os.posixRead(fd, bytes[0..]) catch return error.SystemResources; return mem.readInt(bytes[0..], ErrInt, builtin.endian); } - -extern fn sigchld_handler(_: i32) void { - while (true) { - var status: i32 = undefined; - const pid_result = posix.waitpid(-1, &status, posix.WNOHANG); - if (pid_result == 0) { - return; - } - const err = posix.getErrno(pid_result); - if (err > 0) { - if (err == posix.ECHILD) { - return; - } - unreachable; - } - handleTerm(i32(pid_result), status); - } -} - -fn handleTerm(pid: i32, status: i32) void { - var it = children_nodes.first; - while (it) |node| : (it = node.next) { - if (node.data.pid == pid) { - node.data.handleWaitResult(status); - return; - } - } -} From 2676da61a6ce0808509872a267c8bbd307410966 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 20:33:33 -0400 Subject: [PATCH 43/47] travis: better s3 artifacts --- .travis.yml | 6 ------ ci/travis_linux_install | 2 +- ci/travis_linux_script | 19 ++++++++++++------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index df6f60a48e..c5299e914e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,11 @@ sudo: required services: - docker -addons: - artifacts: - working_dir: artifacts - target_paths: - - /builds os: - linux - osx dist: trusty osx_image: xcode8.3 -sudo: required language: cpp before_install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ci/travis_linux_before_install; fi diff --git a/ci/travis_linux_install b/ci/travis_linux_install index 02d0d6d2e8..40322c58cb 100755 --- a/ci/travis_linux_install +++ b/ci/travis_linux_install @@ -4,4 +4,4 @@ set -x sudo apt-get remove -y llvm-* sudo rm -rf /usr/local/* -sudo apt-get install -y clang-6.0 libclang-6.0 libclang-6.0-dev llvm-6.0 llvm-6.0-dev liblld-6.0 liblld-6.0-dev cmake wine1.6-amd64 +sudo apt-get install -y clang-6.0 libclang-6.0 libclang-6.0-dev llvm-6.0 llvm-6.0-dev liblld-6.0 liblld-6.0-dev cmake wine1.6-amd64 s3cmd diff --git a/ci/travis_linux_script b/ci/travis_linux_script index f80ac33c24..2bb4323066 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -8,11 +8,16 @@ export CXX=clang++-6.0 echo $PATH mkdir build cd build -cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) -make -j2 install -./zig build --build-file ../build.zig test +# cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) +# make -j2 install +# ./zig build --build-file ../build.zig test -cd $TRAVIS_BUILD_DIR -rm .gitignore -git clean -fd -docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT +if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then + mkdir $TRAVIS_BUILD_DIR/artifacts + docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR/artifacts",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT + echo "access_key = $AWS_ACCESS_KEY_ID" >> ~/.s3cfg + echo "secret_key = $AWS_SECRET_ACCESS_KEY" >> ~/.s3cfg + s3cmd put -P $TRAVIS_BUILD_DIR/artifacts/* s3://ziglang.org/builds/ + touch empty + s3cmd put -P empty s3://ziglang.org/builds/zig-linux-x86_64-$TRAVIS_BRANCH.tar.xz --add-header=x-amz-website-redirect-location:/builds/$(ls $TRAVIS_BUILD_DIR/artifacts) +fi From 6050b9d8359d1ae6b808c332aa27142b0064a6be Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 21:40:36 -0400 Subject: [PATCH 44/47] travis: don't skip tests fix broken previous commit --- ci/travis_linux_script | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/travis_linux_script b/ci/travis_linux_script index 2bb4323066..9b43dd20fb 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -8,9 +8,9 @@ export CXX=clang++-6.0 echo $PATH mkdir build cd build -# cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) -# make -j2 install -# ./zig build --build-file ../build.zig test +cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) +make -j2 install +./zig build --build-file ../build.zig test if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then mkdir $TRAVIS_BUILD_DIR/artifacts From abd389209b2b25ac3d3567797bff580c2ce7d9ad Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 4 Apr 2018 00:08:10 -0400 Subject: [PATCH 45/47] fix up logic for macos std.os.deleteTree --- doc/docgen.zig | 2 +- std/os/index.zig | 11 ++++++++--- std/os/test.zig | 13 ++++++++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index 5332a62ac7..56d9a04412 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -55,7 +55,7 @@ pub fn main() !void { // TODO issue #709 // disabled to pass CI tests, but obviously we want to implement this // and then remove this workaround - if (builtin.os == builtin.Os.linux) { + if (builtin.os != builtin.Os.windows) { os.deleteTree(allocator, tmp_dir_name) catch {}; } } diff --git a/std/os/index.zig b/std/os/index.zig index d8e2fd7009..4b74af035e 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -1050,14 +1050,14 @@ const DeleteTreeError = error { }; pub fn deleteTree(allocator: &Allocator, full_path: []const u8) DeleteTreeError!void { start_over: while (true) { + var got_access_denied = false; // First, try deleting the item as a file. This way we don't follow sym links. if (deleteFile(allocator, full_path)) { return; } else |err| switch (err) { error.FileNotFound => return, - - error.AccessDenied, error.IsDir => {}, + error.AccessDenied => got_access_denied = true, error.OutOfMemory, error.SymLinkLoop, @@ -1072,7 +1072,12 @@ pub fn deleteTree(allocator: &Allocator, full_path: []const u8) DeleteTreeError! } { var dir = Dir.open(allocator, full_path) catch |err| switch (err) { - error.NotDir => continue :start_over, + error.NotDir => { + if (got_access_denied) { + return error.AccessDenied; + } + continue :start_over; + }, error.OutOfMemory, error.AccessDenied, diff --git a/std/os/test.zig b/std/os/test.zig index 20d439cc48..9c718d5b6b 100644 --- a/std/os/test.zig +++ b/std/os/test.zig @@ -1,18 +1,25 @@ const std = @import("../index.zig"); const os = std.os; -const debug = std.debug; +const assert = std.debug.assert; const io = std.io; const a = std.debug.global_allocator; +const builtin = @import("builtin"); + test "makePath, put some files in it, deleteTree" { + if (builtin.os == builtin.Os.windows) { + // TODO implement os.Dir for windows + // https://github.com/zig-lang/zig/issues/709 + return; + } try os.makePath(a, "os_test_tmp/b/c"); try io.writeFile(a, "os_test_tmp/b/c/file.txt", "nonsense"); try io.writeFile(a, "os_test_tmp/b/file2.txt", "blah"); try os.deleteTree(a, "os_test_tmp"); if (os.Dir.open(a, "os_test_tmp")) |dir| { - debug.assert(false); // this should not happen! + @panic("expected error"); } else |err| { - debug.assert(err == error.PathNotFound); + assert(err == error.PathNotFound); } } From f68c2e0a14c7d9997db0305981ce3e5998d88a36 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Wed, 4 Apr 2018 21:32:23 +1200 Subject: [PATCH 46/47] Fix off-by-one error in all crypto functions --- std/crypto/blake2.zig | 22 ++++++++++++++++++++-- std/crypto/md5.zig | 11 ++++++++++- std/crypto/sha1.zig | 11 ++++++++++- std/crypto/sha2.zig | 22 ++++++++++++++++++++-- std/crypto/sha3.zig | 18 ++++++++++++++++++ 5 files changed, 78 insertions(+), 6 deletions(-) diff --git a/std/crypto/blake2.zig b/std/crypto/blake2.zig index ea0a68b184..99f0e629cd 100644 --- a/std/crypto/blake2.zig +++ b/std/crypto/blake2.zig @@ -84,7 +84,7 @@ fn Blake2s(comptime out_len: usize) type { return struct { } // Full middle blocks. - while (off + 64 < b.len) : (off += 64) { + while (off + 64 <= b.len) : (off += 64) { d.t += 64; d.round(b[off..off + 64], false); } @@ -229,6 +229,15 @@ test "blake2s256 streaming" { htest.assertEqual(h2, out[0..]); } +test "blake2s256 aligned final" { + var block = []u8 {0} ** Blake2s256.block_size; + var out: [Blake2s256.digest_size]u8 = undefined; + + var h = Blake2s256.init(); + h.update(block); + h.final(out[0..]); +} + ///////////////////// // Blake2b @@ -305,7 +314,7 @@ fn Blake2b(comptime out_len: usize) type { return struct { } // Full middle blocks. - while (off + 128 < b.len) : (off += 128) { + while (off + 128 <= b.len) : (off += 128) { d.t += 128; d.round(b[off..off + 128], false); } @@ -447,3 +456,12 @@ test "blake2b512 streaming" { h.final(out[0..]); htest.assertEqual(h2, out[0..]); } + +test "blake2b512 aligned final" { + var block = []u8 {0} ** Blake2b512.block_size; + var out: [Blake2b512.digest_size]u8 = undefined; + + var h = Blake2b512.init(); + h.update(block); + h.final(out[0..]); +} diff --git a/std/crypto/md5.zig b/std/crypto/md5.zig index 26700cd65b..705b2428a7 100644 --- a/std/crypto/md5.zig +++ b/std/crypto/md5.zig @@ -59,7 +59,7 @@ pub const Md5 = struct { } // Full middle blocks. - while (off + 64 < b.len) : (off += 64) { + while (off + 64 <= b.len) : (off += 64) { d.round(b[off..off + 64]); } @@ -253,3 +253,12 @@ test "md5 streaming" { htest.assertEqual("900150983cd24fb0d6963f7d28e17f72", out[0..]); } + +test "md5 aligned final" { + var block = []u8 {0} ** Md5.block_size; + var out: [Md5.digest_size]u8 = undefined; + + var h = Md5.init(); + h.update(block); + h.final(out[0..]); +} diff --git a/std/crypto/sha1.zig b/std/crypto/sha1.zig index f0dd3c3377..333597b12d 100644 --- a/std/crypto/sha1.zig +++ b/std/crypto/sha1.zig @@ -60,7 +60,7 @@ pub const Sha1 = struct { } // Full middle blocks. - while (off + 64 < b.len) : (off += 64) { + while (off + 64 <= b.len) : (off += 64) { d.round(b[off..off + 64]); } @@ -284,3 +284,12 @@ test "sha1 streaming" { h.final(out[0..]); htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]); } + +test "sha1 aligned final" { + var block = []u8 {0} ** Sha1.block_size; + var out: [Sha1.digest_size]u8 = undefined; + + var h = Sha1.init(); + h.update(block); + h.final(out[0..]); +} diff --git a/std/crypto/sha2.zig b/std/crypto/sha2.zig index 113bab926b..b70450c0ad 100644 --- a/std/crypto/sha2.zig +++ b/std/crypto/sha2.zig @@ -105,7 +105,7 @@ fn Sha2_32(comptime params: Sha2Params32) type { return struct { } // Full middle blocks. - while (off + 64 < b.len) : (off += 64) { + while (off + 64 <= b.len) : (off += 64) { d.round(b[off..off + 64]); } @@ -319,6 +319,15 @@ test "sha256 streaming" { htest.assertEqual("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", out[0..]); } +test "sha256 aligned final" { + var block = []u8 {0} ** Sha256.block_size; + var out: [Sha256.digest_size]u8 = undefined; + + var h = Sha256.init(); + h.update(block); + h.final(out[0..]); +} + ///////////////////// // Sha384 + Sha512 @@ -420,7 +429,7 @@ fn Sha2_64(comptime params: Sha2Params64) type { return struct { } // Full middle blocks. - while (off + 128 < b.len) : (off += 128) { + while (off + 128 <= b.len) : (off += 128) { d.round(b[off..off + 128]); } @@ -669,3 +678,12 @@ test "sha512 streaming" { h.final(out[0..]); htest.assertEqual(h2, out[0..]); } + +test "sha512 aligned final" { + var block = []u8 {0} ** Sha512.block_size; + var out: [Sha512.digest_size]u8 = undefined; + + var h = Sha512.init(); + h.update(block); + h.final(out[0..]); +} diff --git a/std/crypto/sha3.zig b/std/crypto/sha3.zig index 6e6a86b3d5..f92f56d68f 100644 --- a/std/crypto/sha3.zig +++ b/std/crypto/sha3.zig @@ -217,6 +217,15 @@ test "sha3-256 streaming" { htest.assertEqual("3a985da74fe225b2045c172d6bd390bd855f086e3e9d525b46bfe24511431532", out[0..]); } +test "sha3-256 aligned final" { + var block = []u8 {0} ** Sha3_256.block_size; + var out: [Sha3_256.digest_size]u8 = undefined; + + var h = Sha3_256.init(); + h.update(block); + h.final(out[0..]); +} + test "sha3-384 single" { const h1 = "0c63a75b845e4f7d01107d852e4c2485c51a50aaaa94fc61995e71bbee983a2ac3713831264adb47fb6bd1e058d5f004"; htest.assertEqualHash(Sha3_384, h1 , ""); @@ -278,3 +287,12 @@ test "sha3-512 streaming" { h.final(out[0..]); htest.assertEqual(h2, out[0..]); } + +test "sha3-512 aligned final" { + var block = []u8 {0} ** Sha3_512.block_size; + var out: [Sha3_512.digest_size]u8 = undefined; + + var h = Sha3_512.init(); + h.update(block); + h.final(out[0..]); +} From 8938429ea12ff2857ace5380932a7cd68d3b4ab1 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Thu, 5 Apr 2018 02:31:10 +1200 Subject: [PATCH 47/47] Add Hmac function (#890) --- CMakeLists.txt | 1 + std/crypto/hmac.zig | 81 ++++++++++++++++++++++++++++++++++++++++++++ std/crypto/index.zig | 6 ++++ 3 files changed, 88 insertions(+) create mode 100644 std/crypto/hmac.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 1acc789de9..cea302cb06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -425,6 +425,7 @@ set(ZIG_STD_FILES "crypto/sha2.zig" "crypto/sha3.zig" "crypto/blake2.zig" + "crypto/hmac.zig" "cstr.zig" "debug/failing_allocator.zig" "debug/index.zig" diff --git a/std/crypto/hmac.zig b/std/crypto/hmac.zig new file mode 100644 index 0000000000..2a36f15b71 --- /dev/null +++ b/std/crypto/hmac.zig @@ -0,0 +1,81 @@ +const std = @import("../index.zig"); +const crypto = std.crypto; +const debug = std.debug; +const mem = std.mem; + +pub const HmacMd5 = Hmac(crypto.Md5); +pub const HmacSha1 = Hmac(crypto.Sha1); +pub const HmacSha256 = Hmac(crypto.Sha256); + +pub fn Hmac(comptime H: type) type { + return struct { + const digest_size = H.digest_size; + + pub fn hash(output: []u8, key: []const u8, message: []const u8) void { + debug.assert(output.len >= H.digest_size); + debug.assert(H.digest_size <= H.block_size); // HMAC makes this assumption + var scratch: [H.block_size]u8 = undefined; + + // Normalize key length to block size of hash + if (key.len > H.block_size) { + H.hash(key, scratch[0..H.digest_size]); + mem.set(u8, scratch[H.digest_size..H.block_size], 0); + } else if (key.len < H.block_size) { + mem.copy(u8, scratch[0..key.len], key); + mem.set(u8, scratch[key.len..H.block_size], 0); + } else { + mem.copy(u8, scratch[0..], key); + } + + var o_key_pad: [H.block_size]u8 = undefined; + for (o_key_pad) |*b, i| { + *b = scratch[i] ^ 0x5c; + } + + var i_key_pad: [H.block_size]u8 = undefined; + for (i_key_pad) |*b, i| { + *b = scratch[i] ^ 0x36; + } + + // HMAC(k, m) = H(o_key_pad | H(i_key_pad | message)) where | is concatenation + var hmac = H.init(); + hmac.update(i_key_pad[0..]); + hmac.update(message); + hmac.final(scratch[0..H.digest_size]); + + hmac.reset(); + hmac.update(o_key_pad[0..]); + hmac.update(scratch[0..H.digest_size]); + hmac.final(output[0..H.digest_size]); + } + }; +} + +const htest = @import("test.zig"); + +test "hmac md5" { + var out: [crypto.Md5.digest_size]u8 = undefined; + HmacMd5.hash(out[0..], "", ""); + htest.assertEqual("74e6f7298a9c2d168935f58c001bad88", out[0..]); + + HmacMd5.hash(out[0..], "key", "The quick brown fox jumps over the lazy dog"); + htest.assertEqual("80070713463e7749b90c2dc24911e275", out[0..]); +} + +test "hmac sha1" { + var out: [crypto.Sha1.digest_size]u8 = undefined; + HmacSha1.hash(out[0..], "", ""); + htest.assertEqual("fbdb1d1b18aa6c08324b7d64b71fb76370690e1d", out[0..]); + + HmacSha1.hash(out[0..], "key", "The quick brown fox jumps over the lazy dog"); + htest.assertEqual("de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9", out[0..]); +} + +test "hmac sha256" { + var out: [crypto.Sha256.digest_size]u8 = undefined; + HmacSha256.hash(out[0..], "", ""); + htest.assertEqual("b613679a0814d9ec772f95d778c35fc5ff1697c493715653c6c712144292c5ad", out[0..]); + + HmacSha256.hash(out[0..], "key", "The quick brown fox jumps over the lazy dog"); + htest.assertEqual("f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8", out[0..]); +} diff --git a/std/crypto/index.zig b/std/crypto/index.zig index ee7dc2aa0e..2f39020228 100644 --- a/std/crypto/index.zig +++ b/std/crypto/index.zig @@ -19,10 +19,16 @@ pub const Blake2s256 = blake2.Blake2s256; pub const Blake2b384 = blake2.Blake2b384; pub const Blake2b512 = blake2.Blake2b512; +const hmac = @import("hmac.zig"); +pub const HmacMd5 = hmac.HmacMd5; +pub const HmacSha1 = hmac.Sha1; +pub const HmacSha256 = hmac.Sha256; + test "crypto" { _ = @import("md5.zig"); _ = @import("sha1.zig"); _ = @import("sha2.zig"); _ = @import("sha3.zig"); _ = @import("blake2.zig"); + _ = @import("hmac.zig"); }