From 283d441c19d5bafa01a7df24db277a6b08a86c00 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Sun, 30 Aug 2020 10:35:18 +1000 Subject: [PATCH 01/37] zig fmt: fix #3978, fix #2748 --- lib/std/zig/parser_test.zig | 53 +++++++++++++++++++++++++++++++++++++ lib/std/zig/render.zig | 15 ++++++++--- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 36ceb400dc..1aec1c3567 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3321,6 +3321,59 @@ test "zig fmt: Don't add extra newline after if" { ); } +test "zig fmt: comments in ternary ifs" { + try testCanonical( + \\const x = if (true) { + \\ 1; + \\} else if (false) + \\ // Comment + \\ 0; + \\const y = if (true) + \\ // Comment + \\ 1 + \\else + \\ 0; + \\ + \\pub extern "c" fn printf(format: [*:0]const u8, ...) c_int; + \\ + ); +} + +test "zig fmt: test comments in field access chain" { + try testCanonical( + \\pub const str = struct { + \\ pub const Thing = more.more // + \\ .more() // + \\ .more().more() // + \\ .more() // + \\ // .more() // + \\ .more() // + \\ .more(); + \\ data: Data, + \\}; + \\ + \\pub const str = struct { + \\ pub const Thing = more.more // + \\ .more() // + \\ // .more() // + \\ // .more() // + \\ // .more() // + \\ .more() // + \\ .more(); + \\ data: Data, + \\}; + \\ + \\pub const str = struct { + \\ pub const Thing = more // + \\ .more // + \\ .more() // + \\ .more(); + \\ data: Data, + \\}; + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 237ca07d2b..4432d08787 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -522,8 +522,12 @@ fn renderExpression( break :blk if (loc.line == 0) op_space else Space.Newline; }; - try renderToken(tree, ais, infix_op_node.op_token, after_op_space); - ais.pushIndentOneShot(); + { + try ais.pushIndent(); + defer ais.popIndent(); + try renderToken(tree, ais, infix_op_node.op_token, after_op_space); + } + try ais.pushIndentOneShot(); return renderExpression(allocator, ais, tree, infix_op_node.rhs, space); }, @@ -1873,7 +1877,12 @@ fn renderExpression( if (src_has_newline) { const after_rparen_space = if (if_node.payload == null) Space.Newline else Space.Space; - try renderToken(tree, ais, rparen, after_rparen_space); // ) + + { + try ais.pushIndent(); + defer ais.popIndent(); + try renderToken(tree, ais, rparen, after_rparen_space); // ) + } if (if_node.payload) |payload| { try renderExpression(allocator, ais, tree, payload, Space.Newline); From 601331833a148ff3a1ab5cb4bba8bc63f4850e13 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Sun, 30 Aug 2020 10:34:44 +1000 Subject: [PATCH 02/37] Add passing test. close #5343 --- lib/std/zig/parser_test.zig | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 1aec1c3567..6b8734c9d4 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3374,6 +3374,24 @@ test "zig fmt: test comments in field access chain" { ); } +test "zig fmt: Indent comma correctly after multiline string literals in arg list (trailing comma)" { + try testCanonical( + \\fn foo() void { + \\ z.display_message_dialog( + \\ *const [323:0]u8, + \\ \\Message Text + \\ \\------------ + \\ \\xxxxxxxxxxxx + \\ \\xxxxxxxxxxxx + \\ , + \\ g.GtkMessageType.GTK_MESSAGE_WARNING, + \\ null, + \\ ); + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; From ea6181aaf6c3e22ced8b8ac202c5fc93a8e90674 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Fri, 11 Sep 2020 13:02:06 +1000 Subject: [PATCH 03/37] zig fmt: Add test for nesting if expressions --- lib/std/zig/parser_test.zig | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 6b8734c9d4..fe32a371e9 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3392,6 +3392,43 @@ test "zig fmt: Indent comma correctly after multiline string literals in arg lis ); } +test "zig fmt: Control flow statement as body of blockless if" { + try testCanonical( + \\pub fn main() void { + \\ const zoom_node = if (focused_node == layout_first) + \\ if (it.next()) { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ } else null + \\ else + \\ focused_node; + \\ + \\ const zoom_node = if (focused_node == layout_first) while (it.next()) |node| { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ } else null else + \\ focused_node; + \\ + \\ const zoom_node = if (focused_node == layout_first) + \\ if (it.next()) { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ } else null; + \\ + \\ const zoom_node = if (focused_node == layout_first) while (it.next()) |node| { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ }; + \\ + \\ const zoom_node = if (focused_node == layout_first) for (nodes) |node| { + \\ break node; + \\ }; + \\ + \\ const zoom_node = if (focused_node == layout_first) switch (nodes) { + \\ 0 => 0, + \\ } else + \\ focused_node; + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; From 9f0821e68836a495c726e0aae13e62c5235c5446 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Sun, 30 Aug 2020 15:02:05 +1000 Subject: [PATCH 04/37] zig fmt: Fix erroneously commented out code, add passing test case to close #5722 --- lib/std/zig/parser_test.zig | 15 +++++++++++++++ lib/std/zig/render.zig | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index fe32a371e9..50208582dc 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3429,6 +3429,21 @@ test "zig fmt: Control flow statement as body of blockless if" { ); } +test "zig fmt: " { + try testCanonical( + \\pub fn sendViewTags(self: Self) void { + \\ var it = ViewStack(View).iterator(self.output.views.first, std.math.maxInt(u32)); + \\ while (it.next()) |node| + \\ view_tags.append(node.view.current_tags) catch { + \\ c.wl_resource_post_no_memory(self.wl_resource); + \\ log.crit(.river_status, "out of memory", .{}); + \\ return; + \\ }; + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 4432d08787..522be107b0 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -1762,7 +1762,7 @@ fn renderExpression( } if (while_node.payload) |payload| { - const payload_space = Space.Space; //if (while_node.continue_expr != null) Space.Space else block_start_space; + const payload_space = if (while_node.continue_expr != null) Space.Space else block_start_space; try renderExpression(allocator, ais, tree, payload, payload_space); } From e1bd27119220c59211509f65c39fbb89c69b939b Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Sun, 30 Aug 2020 18:25:04 +1000 Subject: [PATCH 05/37] zig fmt: Allow trailing comments to do manual array formatting. close #5948 --- lib/std/zig/parser_test.zig | 44 ++++++- lib/std/zig/render.zig | 229 ++++++++++++++++++++++-------------- 2 files changed, 182 insertions(+), 91 deletions(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 50208582dc..20cafed5d3 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -1301,8 +1301,10 @@ test "zig fmt: array literal with hint" { \\const a = []u8{ \\ 1, 2, \\ 3, 4, - \\ 5, 6, // blah - \\ 7, 8, + \\ 5, + \\ 6, // blah + \\ 7, + \\ 8, \\}; \\const a = []u8{ \\ 1, 2, @@ -3444,6 +3446,44 @@ test "zig fmt: " { ); } +test "zig fmt: allow trailing line comments to do manual array formatting" { + try testCanonical( + \\fn foo() void { + \\ self.code.appendSliceAssumeCapacity(&[_]u8{ + \\ 0x55, // push rbp + \\ 0x48, 0x89, 0xe5, // mov rbp, rsp + \\ 0x48, 0x81, 0xec, // sub rsp, imm32 (with reloc) + \\ }); + \\ + \\ di_buf.appendAssumeCapacity(&[_]u8{ + \\ 1, DW.TAG_compile_unit, DW.CHILDREN_no, // header + \\ DW.AT_stmt_list, DW_FORM_data4, // form value pairs + \\ DW.AT_low_pc, DW_FORM_addr, + \\ DW.AT_high_pc, DW_FORM_addr, + \\ DW.AT_name, DW_FORM_strp, + \\ DW.AT_comp_dir, DW_FORM_strp, + \\ DW.AT_producer, DW_FORM_strp, + \\ DW.AT_language, DW_FORM_data2, + \\ 0, 0, // sentinel + \\ }); + \\ + \\ self.code.appendSliceAssumeCapacity(&[_]u8{ + \\ 0x55, // push rbp + \\ 0x48, 0x89, 0xe5, // mov rbp, rsp + \\ // How do we handle this? + \\ //0x48, 0x81, 0xec, // sub rsp, imm32 (with reloc) + \\ // Here's a blank line, should that be allowed? + \\ + \\ 0x48, 0x89, 0xe5, + \\ 0x33, 0x45, + \\ // Now the comment breaks a single line -- how do we handle this? + \\ 0x88, + \\ }); + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 522be107b0..3594cd5ca9 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -523,11 +523,11 @@ fn renderExpression( }; { - try ais.pushIndent(); + ais.pushIndent(); defer ais.popIndent(); try renderToken(tree, ais, infix_op_node.op_token, after_op_space); } - try ais.pushIndentOneShot(); + ais.pushIndentOneShot(); return renderExpression(allocator, ais, tree, infix_op_node.rhs, space); }, @@ -746,109 +746,130 @@ fn renderExpression( } // scan to find row size - const maybe_row_size: ?usize = blk: { - var count: usize = 1; - for (exprs) |expr, i| { - if (i + 1 < exprs.len) { - const expr_last_token = expr.lastToken() + 1; - const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, exprs[i + 1].firstToken()); - if (loc.line != 0) break :blk count; - count += 1; - } else { - const expr_last_token = expr.lastToken(); - const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, rtoken); - if (loc.line == 0) { - // all on one line - const src_has_trailing_comma = trailblk: { - const maybe_comma = tree.prevToken(rtoken); - break :trailblk tree.token_ids[maybe_comma] == .Comma; - }; - if (src_has_trailing_comma) { - break :blk 1; // force row size 1 - } else { - break :blk null; // no newlines - } - } - break :blk count; - } - } - unreachable; - }; - - if (maybe_row_size) |row_size| { - // A place to store the width of each expression and its column's maximum - var widths = try allocator.alloc(usize, exprs.len + row_size); - defer allocator.free(widths); - mem.set(usize, widths, 0); - - var expr_widths = widths[0 .. widths.len - row_size]; - var column_widths = widths[widths.len - row_size ..]; - - // Null ais for counting the printed length of each expression - var counting_stream = std.io.countingOutStream(std.io.null_out_stream); - var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, counting_stream.writer()); - - for (exprs) |expr, i| { - counting_stream.bytes_written = 0; - try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); - const width = @intCast(usize, counting_stream.bytes_written); - const col = i % row_size; - column_widths[col] = std.math.max(column_widths[col], width); - expr_widths[i] = width; - } - + if (rowSize(tree, exprs, rtoken, false) != null) { { ais.pushIndentNextLine(); defer ais.popIndent(); try renderToken(tree, ais, lbrace, Space.Newline); - var col: usize = 1; - for (exprs) |expr, i| { - if (i + 1 < exprs.len) { - const next_expr = exprs[i + 1]; - try renderExpression(allocator, ais, tree, expr, Space.None); + var expr_index: usize = 0; + while (rowSize(tree, exprs[expr_index..], rtoken, true)) |row_size| { + const row_exprs = exprs[expr_index..]; + // A place to store the width of each expression and its column's maximum + var widths = try allocator.alloc(usize, row_exprs.len + row_size); + defer allocator.free(widths); + mem.set(usize, widths, 0); - const comma = tree.nextToken(expr.*.lastToken()); + var expr_widths = widths[0 .. widths.len - row_size]; + var column_widths = widths[widths.len - row_size ..]; - if (col != row_size) { - try renderToken(tree, ais, comma, Space.Space); // , + // Null stream for counting the printed length of each expression + var counting_stream = std.io.countingOutStream(std.io.null_out_stream); + var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, counting_stream.writer()); - const padding = column_widths[i % row_size] - expr_widths[i]; - try ais.writer().writeByteNTimes(' ', padding); - - col += 1; - continue; + // Find next row with trailing comment (if any) to end the current section then + var section_end = sec_end: { + var this_line_first_expr: usize = 0; + var this_line_size = rowSize(tree, row_exprs, rtoken, true); + for (row_exprs) |expr, i| { + // Ignore comment on first line of this section + if (i == 0 or tree.tokensOnSameLine(row_exprs[0].firstToken(), expr.lastToken())) continue; + // Track start of line containing comment + if (!tree.tokensOnSameLine(row_exprs[this_line_first_expr].firstToken(), expr.lastToken())) { + this_line_first_expr = i; + this_line_size = rowSize(tree, row_exprs[this_line_first_expr..], rtoken, true); + } + if (expr.lastToken() + 2 < tree.token_ids.len) { + if (tree.token_ids[expr.lastToken() + 1] == .Comma and + tree.token_ids[expr.lastToken() + 2] == .LineComment and + tree.tokensOnSameLine(expr.lastToken(), expr.lastToken() + 2)) + { + var comment_token_loc = tree.token_locs[expr.lastToken() + 2]; + const comment_is_empty = mem.trimRight(u8, tree.tokenSliceLoc(comment_token_loc), " ").len == 2; + if (!comment_is_empty) { + // Found row ending in comment + break :sec_end i - this_line_size.? + 1; + } + } + } } - col = 1; + break :sec_end row_exprs.len; + }; + expr_index += section_end; - if (tree.token_ids[tree.nextToken(comma)] != .MultilineStringLiteralLine) { - try renderToken(tree, ais, comma, Space.Newline); // , - } else { - try renderToken(tree, ais, comma, Space.None); // , - } + const section_exprs = row_exprs[0..section_end]; - try renderExtraNewline(tree, ais, next_expr); - } else { - try renderExpression(allocator, ais, tree, expr, Space.Comma); // , + // Calculate size of columns in current section + for (section_exprs) |expr, i| { + counting_stream.bytes_written = 0; + try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); + const width = @intCast(usize, counting_stream.bytes_written); + const col = i % row_size; + column_widths[col] = std.math.max(column_widths[col], width); + expr_widths[i] = width; + } + + // Render exprs in current section + var col: usize = 1; + for (section_exprs) |expr, i| { + if (i + 1 < section_exprs.len) { + const next_expr = section_exprs[i + 1]; + try renderExpression(allocator, ais, tree, expr, Space.None); + + const comma = tree.nextToken(expr.*.lastToken()); + + if (col != row_size) { + try renderToken(tree, ais, comma, Space.Space); // , + + const padding = column_widths[i % row_size] - expr_widths[i]; + try ais.writer().writeByteNTimes(' ', padding); + + col += 1; + continue; + } + col = 1; + + if (tree.token_ids[tree.nextToken(comma)] != .MultilineStringLiteralLine) { + try renderToken(tree, ais, comma, Space.Newline); // , + } else { + try renderToken(tree, ais, comma, Space.None); // , + } + + try renderExtraNewline(tree, ais, next_expr); + } else { + const maybe_comma = tree.nextToken(expr.*.lastToken()); + if (tree.token_ids[maybe_comma] == .Comma) { + try renderExpression(allocator, ais, tree, expr, Space.None); // , + try renderToken(tree, ais, maybe_comma, Space.Newline); // , + } else { + try renderExpression(allocator, ais, tree, expr, Space.Comma); // , + } + } + } + + if (expr_index == exprs.len) { + break; } - } - } - return renderToken(tree, ais, rtoken, space); - } else { - try renderToken(tree, ais, lbrace, Space.Space); - for (exprs) |expr, i| { - if (i + 1 < exprs.len) { - const next_expr = exprs[i + 1]; - try renderExpression(allocator, ais, tree, expr, Space.None); - const comma = tree.nextToken(expr.*.lastToken()); - try renderToken(tree, ais, comma, Space.Space); // , - } else { - try renderExpression(allocator, ais, tree, expr, Space.Space); } } return renderToken(tree, ais, rtoken, space); } + + // Single line + try renderToken(tree, ais, lbrace, Space.Space); + for (exprs) |expr, i| { + if (i + 1 < exprs.len) { + const next_expr = exprs[i + 1]; + try renderExpression(allocator, ais, tree, expr, Space.None); + const comma = tree.nextToken(expr.*.lastToken()); + try renderToken(tree, ais, comma, Space.Space); // , + } else { + try renderExpression(allocator, ais, tree, expr, Space.Space); + } + } + + return renderToken(tree, ais, rtoken, space); }, .StructInitializer, .StructInitializerDot => { @@ -1879,7 +1900,7 @@ fn renderExpression( const after_rparen_space = if (if_node.payload == null) Space.Newline else Space.Space; { - try ais.pushIndent(); + ais.pushIndent(); defer ais.popIndent(); try renderToken(tree, ais, rparen, after_rparen_space); // ) } @@ -2567,3 +2588,33 @@ fn copyFixingWhitespace(ais: anytype, slice: []const u8) @TypeOf(ais.*).Error!vo else => try ais.writer().writeByte(byte), }; } + +fn rowSize(tree: *ast.Tree, exprs: []*ast.Node, rtoken: ast.TokenIndex, force: bool) ?usize { + var count: usize = 1; + for (exprs) |expr, i| { + if (i + 1 < exprs.len) { + const expr_last_token = expr.lastToken() + 1; + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, exprs[i + 1].firstToken()); + if (loc.line != 0) return count; + count += 1; + } else { + if (force) return count; + const expr_last_token = expr.lastToken(); + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, rtoken); + if (loc.line == 0) { + // all on one line + const src_has_trailing_comma = trailblk: { + const maybe_comma = tree.prevToken(rtoken); + break :trailblk tree.token_ids[maybe_comma] == .Comma; + }; + if (src_has_trailing_comma) { + return 1; // force row size 1 + } else { + return null; // no newlines + } + } + return count; + } + } + unreachable; +} From 291482a0310312fa3d84b3a967fa3f2d5b71b165 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Mon, 9 Mar 2020 14:04:31 +1100 Subject: [PATCH 06/37] zig fmt: Don't consider width of expressions containing multiline string literals when calculating padding for array initializers. fixes #3739 Changes some of the special casing for multiline string literals. --- lib/std/zig/ast.zig | 9 +++ lib/std/zig/parser_test.zig | 59 +++++++++++++++ lib/std/zig/render.zig | 145 +++++++++++++++++++++--------------- 3 files changed, 155 insertions(+), 58 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 404e8c413a..d8943adde0 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -823,6 +823,15 @@ pub const Node = struct { } } + pub fn findFirstWithId(self: *Node, id: Id) ?*Node { + if (self.id == id) return self; + var child_i: usize = 0; + while (self.iterate(child_i)) |child| : (child_i += 1) { + if (child.findFirstWithId(id)) |result| return result; + } + return null; + } + pub fn dump(self: *Node, indent: usize) void { { var i: usize = 0; diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 20cafed5d3..78443afe7a 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3484,6 +3484,65 @@ test "zig fmt: allow trailing line comments to do manual array formatting" { ); } +test "zig fmt: multiline string literals should play nice with array initializers" { + try testCanonical( + \\fn main() void { + \\ var a = .{.{.{.{.{.{.{.{ + \\ 0, + \\ }}}}}}}}; + \\ myFunc(.{ + \\ "aaaaaaa", "bbbbbb", "ccccc", + \\ "dddd", ("eee"), ("fff"), + \\ ("gggg"), + \\ // Line comment + \\ \\Multiline String Literals can be quite long + \\ , + \\ \\Multiline String Literals can be quite long + \\ \\Multiline String Literals can be quite long + \\ , + \\ \\Multiline String Literals can be quite long + \\ \\Multiline String Literals can be quite long + \\ \\Multiline String Literals can be quite long + \\ \\Multiline String Literals can be quite long + \\ , + \\ ( + \\ \\Multiline String Literals can be quite long + \\ ), + \\ .{ + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ }, + \\ .{( + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ )}, + \\ .{ "xxxxxxx", "xxx", ( + \\ \\ xxx + \\ ), "xxx", "xxx" }, + \\ .{ "xxxxxxx", "xxx", "xxx", "xxx" }, .{ "xxxxxxx", "xxx", "xxx", "xxx" }, + \\ "aaaaaaa", "bbbbbb", "ccccc", // - + \\ "dddd", ("eee"), ("fff"), + \\ .{ + \\ "xxx", "xxx", + \\ ( + \\ \\ xxx + \\ ), + \\ "xxxxxxxxxxxxxx", "xxx", + \\ }, + \\ .{ + \\ ( + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ ), + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ }, + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ }); + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 3594cd5ca9..b2687ada98 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -714,37 +714,24 @@ fn renderExpression( .node => |node| tree.nextToken(node.lastToken()), }; - if (exprs.len == 0) { - switch (lhs) { - .dot => |dot| try renderToken(tree, ais, dot, Space.None), - .node => |node| try renderExpression(allocator, ais, tree, node, Space.None), - } - - { - ais.pushIndent(); - defer ais.popIndent(); - try renderToken(tree, ais, lbrace, Space.None); - } - - return renderToken(tree, ais, rtoken, space); - } - if (exprs.len == 1 and tree.token_ids[exprs[0].*.lastToken() + 1] == .RBrace) { - const expr = exprs[0]; - - switch (lhs) { - .dot => |dot| try renderToken(tree, ais, dot, Space.None), - .node => |node| try renderExpression(allocator, ais, tree, node, Space.None), - } - try renderToken(tree, ais, lbrace, Space.None); - try renderExpression(allocator, ais, tree, expr, Space.None); - return renderToken(tree, ais, rtoken, space); - } - switch (lhs) { .dot => |dot| try renderToken(tree, ais, dot, Space.None), .node => |node| try renderExpression(allocator, ais, tree, node, Space.None), } + if (exprs.len == 0) { + try renderToken(tree, ais, lbrace, Space.None); + return renderToken(tree, ais, rtoken, space); + } + + if (exprs.len == 1 and exprs[0].tag != .MultilineStringLiteral and tree.token_ids[exprs[0].*.lastToken() + 1] == .RBrace) { + const expr = exprs[0]; + + try renderToken(tree, ais, lbrace, Space.None); + try renderExpression(allocator, ais, tree, expr, Space.None); + return renderToken(tree, ais, rtoken, space); + } + // scan to find row size if (rowSize(tree, exprs, rtoken, false) != null) { { @@ -763,11 +750,7 @@ fn renderExpression( var expr_widths = widths[0 .. widths.len - row_size]; var column_widths = widths[widths.len - row_size ..]; - // Null stream for counting the printed length of each expression - var counting_stream = std.io.countingOutStream(std.io.null_out_stream); - var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, counting_stream.writer()); - - // Find next row with trailing comment (if any) to end the current section then + // Find next row with trailing comment (if any) to end the current section var section_end = sec_end: { var this_line_first_expr: usize = 0; var this_line_size = rowSize(tree, row_exprs, rtoken, true); @@ -779,12 +762,15 @@ fn renderExpression( this_line_first_expr = i; this_line_size = rowSize(tree, row_exprs[this_line_first_expr..], rtoken, true); } - if (expr.lastToken() + 2 < tree.token_ids.len) { - if (tree.token_ids[expr.lastToken() + 1] == .Comma and - tree.token_ids[expr.lastToken() + 2] == .LineComment and - tree.tokensOnSameLine(expr.lastToken(), expr.lastToken() + 2)) + + const maybe_comma = expr.lastToken() + 1; + const maybe_comment = expr.lastToken() + 2; + if (maybe_comment < tree.token_ids.len) { + if (tree.token_ids[maybe_comma] == .Comma and + tree.token_ids[maybe_comment] == .LineComment and + tree.tokensOnSameLine(expr.lastToken(), maybe_comment)) { - var comment_token_loc = tree.token_locs[expr.lastToken() + 2]; + var comment_token_loc = tree.token_locs[maybe_comment]; const comment_is_empty = mem.trimRight(u8, tree.tokenSliceLoc(comment_token_loc), " ").len == 2; if (!comment_is_empty) { // Found row ending in comment @@ -799,18 +785,56 @@ fn renderExpression( const section_exprs = row_exprs[0..section_end]; + // Null stream for counting the printed length of each expression + var line_find_stream = std.io.findByteOutStream('\n', std.io.null_out_stream); + var counting_stream = std.io.countingOutStream(line_find_stream.writer()); + var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, counting_stream.writer()); + // Calculate size of columns in current section + var c: usize = 0; + var single_line = true; for (section_exprs) |expr, i| { - counting_stream.bytes_written = 0; - try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); - const width = @intCast(usize, counting_stream.bytes_written); - const col = i % row_size; - column_widths[col] = std.math.max(column_widths[col], width); - expr_widths[i] = width; + if (i + 1 < section_exprs.len) { + counting_stream.bytes_written = 0; + line_find_stream.byte_found = false; + try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); + const width = @intCast(usize, counting_stream.bytes_written); + expr_widths[i] = width; + + if (!line_find_stream.byte_found) { + const col = c % row_size; + column_widths[col] = std.math.max(column_widths[col], width); + + const expr_last_token = expr.*.lastToken() + 1; + const next_expr = section_exprs[i + 1]; + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].start, next_expr.*.firstToken()); + if (loc.line == 0) { + c += 1; + } else { + single_line = false; + c = 0; + } + } else { + single_line = false; + c = 0; + } + } else { + counting_stream.bytes_written = 0; + try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); + const width = @intCast(usize, counting_stream.bytes_written); + expr_widths[i] = width; + + if (!line_find_stream.byte_found) { + const col = c % row_size; + column_widths[col] = std.math.max(column_widths[col], width); + } + break; + } } // Render exprs in current section - var col: usize = 1; + c = 0; + var last_col_index: usize = row_size - 1; for (section_exprs) |expr, i| { if (i + 1 < section_exprs.len) { const next_expr = section_exprs[i + 1]; @@ -818,23 +842,28 @@ fn renderExpression( const comma = tree.nextToken(expr.*.lastToken()); - if (col != row_size) { + if (c != last_col_index) { + line_find_stream.byte_found = false; + try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); + try renderExpression(allocator, &auto_indenting_stream, tree, next_expr, Space.None); + if (!line_find_stream.byte_found) { + // Neither the current or next expression is multiline + try renderToken(tree, ais, comma, Space.Space); // , + assert(column_widths[c % row_size] >= expr_widths[i]); + const padding = column_widths[c % row_size] - expr_widths[i]; + try ais.writer().writeByteNTimes(' ', padding); + + c += 1; + continue; + } + } + if (single_line) { try renderToken(tree, ais, comma, Space.Space); // , - - const padding = column_widths[i % row_size] - expr_widths[i]; - try ais.writer().writeByteNTimes(' ', padding); - - col += 1; continue; } - col = 1; - - if (tree.token_ids[tree.nextToken(comma)] != .MultilineStringLiteralLine) { - try renderToken(tree, ais, comma, Space.Newline); // , - } else { - try renderToken(tree, ais, comma, Space.None); // , - } + c = 0; + try renderToken(tree, ais, comma, Space.Newline); // , try renderExtraNewline(tree, ais, next_expr); } else { const maybe_comma = tree.nextToken(expr.*.lastToken()); @@ -2594,13 +2623,13 @@ fn rowSize(tree: *ast.Tree, exprs: []*ast.Node, rtoken: ast.TokenIndex, force: b for (exprs) |expr, i| { if (i + 1 < exprs.len) { const expr_last_token = expr.lastToken() + 1; - const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, exprs[i + 1].firstToken()); + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].start, exprs[i + 1].firstToken()); if (loc.line != 0) return count; count += 1; } else { if (force) return count; const expr_last_token = expr.lastToken(); - const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, rtoken); + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].start, rtoken); if (loc.line == 0) { // all on one line const src_has_trailing_comma = trailblk: { From 206a8cf6709df51213252b03bf3ba4a9b8b52b6f Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Wed, 9 Sep 2020 21:45:05 +1000 Subject: [PATCH 07/37] zig fmt: fix comments and multiline literals in function args --- lib/std/zig/parser_test.zig | 40 +++++++++++++++++++++++ lib/std/zig/render.zig | 63 +++++++++++++++++++++---------------- 2 files changed, 76 insertions(+), 27 deletions(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 78443afe7a..d6dd9c1a73 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3543,6 +3543,46 @@ test "zig fmt: multiline string literals should play nice with array initializer ); } +test "zig fmt: use of comments and Multiline string literals may force the parameters over multiple lines" { + try testCanonical( + \\pub fn makeMemUndefined(qzz: []u8) i1 { + \\ cases.add( // fixed bug #2032 + \\ "compile diagnostic string for top level decl type", + \\ \\export fn entry() void { + \\ \\ var foo: u32 = @This(){}; + \\ \\} + \\ , &[_][]const u8{ + \\ "tmp.zig:2:27: error: type 'u32' does not support array initialization", + \\ }); + \\ @compileError( + \\ \\ unknown-length pointers and C pointers cannot be hashed deeply. + \\ \\ Consider providing your own hash function. + \\ \\ unknown-length pointers and C pointers cannot be hashed deeply. + \\ \\ Consider providing your own hash function. + \\ ); + \\ return @intCast(i1, doMemCheckClientRequestExpr(0, // default return + \\ .MakeMemUndefined, @ptrToInt(qzz.ptr), qzz.len, 0, 0, 0)); + \\} + \\ + \\// This looks like garbage don't do this + \\const rparen = tree.prevToken( + \\// the first token for the annotation expressions is the left + \\// parenthesis, hence the need for two prevToken + \\ if (fn_proto.getAlignExpr()) |align_expr| + \\ tree.prevToken(tree.prevToken(align_expr.firstToken())) + \\else if (fn_proto.getSectionExpr()) |section_expr| + \\ tree.prevToken(tree.prevToken(section_expr.firstToken())) + \\else if (fn_proto.getCallconvExpr()) |callconv_expr| + \\ tree.prevToken(tree.prevToken(callconv_expr.firstToken())) + \\else switch (fn_proto.return_type) { + \\ .Explicit => |node| node.firstToken(), + \\ .InferErrorSet => |node| tree.prevToken(node.firstToken()), + \\ .Invalid => unreachable, + \\}); + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index b2687ada98..319788546e 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -1058,21 +1058,22 @@ fn renderExpression( }; if (src_has_trailing_comma) { - try renderToken(tree, ais, lparen, Space.Newline); - - const params = call.params(); - for (params) |param_node, i| { + { ais.pushIndent(); defer ais.popIndent(); - if (i + 1 < params.len) { - const next_node = params[i + 1]; - try renderExpression(allocator, ais, tree, param_node, Space.None); - const comma = tree.nextToken(param_node.lastToken()); - try renderToken(tree, ais, comma, Space.Newline); // , - try renderExtraNewline(tree, ais, next_node); - } else { - try renderExpression(allocator, ais, tree, param_node, Space.Comma); + try renderToken(tree, ais, lparen, Space.Newline); // ( + const params = call.params(); + for (params) |param_node, i| { + if (i + 1 < params.len) { + const next_node = params[i + 1]; + try renderExpression(allocator, ais, tree, param_node, Space.None); + const comma = tree.nextToken(param_node.lastToken()); + try renderToken(tree, ais, comma, Space.Newline); // , + try renderExtraNewline(tree, ais, next_node); + } else { + try renderExpression(allocator, ais, tree, param_node, Space.Comma); + } } } return renderToken(tree, ais, call.rtoken, space); @@ -1082,7 +1083,10 @@ fn renderExpression( const params = call.params(); for (params) |param_node, i| { - if (param_node.*.tag == .MultilineStringLiteral) ais.pushIndentOneShot(); + const maybe_comment = param_node.firstToken() - 1; + if (param_node.*.tag == .MultilineStringLiteral or tree.token_ids[maybe_comment] == .LineComment) { + ais.pushIndentOneShot(); + } try renderExpression(allocator, ais, tree, param_node, Space.None); @@ -1092,7 +1096,7 @@ fn renderExpression( try renderToken(tree, ais, comma, Space.Space); } } - return renderToken(tree, ais, call.rtoken, space); + return renderToken(tree, ais, call.rtoken, space); // ) }, .ArrayAccess => { @@ -1497,6 +1501,10 @@ fn renderExpression( // render all on one line, no trailing comma const params = builtin_call.params(); for (params) |param_node, i| { + const maybe_comment = param_node.firstToken() - 1; + if (param_node.*.tag == .MultilineStringLiteral or tree.token_ids[maybe_comment] == .LineComment) { + ais.pushIndentOneShot(); + } try renderExpression(allocator, ais, tree, param_node, Space.None); if (i + 1 < params.len) { @@ -1548,19 +1556,20 @@ fn renderExpression( assert(tree.token_ids[lparen] == .LParen); const rparen = tree.prevToken( - // the first token for the annotation expressions is the left - // parenthesis, hence the need for two prevToken - if (fn_proto.getAlignExpr()) |align_expr| - tree.prevToken(tree.prevToken(align_expr.firstToken())) - else if (fn_proto.getSectionExpr()) |section_expr| - tree.prevToken(tree.prevToken(section_expr.firstToken())) - else if (fn_proto.getCallconvExpr()) |callconv_expr| - tree.prevToken(tree.prevToken(callconv_expr.firstToken())) - else switch (fn_proto.return_type) { - .Explicit => |node| node.firstToken(), - .InferErrorSet => |node| tree.prevToken(node.firstToken()), - .Invalid => unreachable, - }); + // the first token for the annotation expressions is the left + // parenthesis, hence the need for two prevToken + if (fn_proto.getAlignExpr()) |align_expr| + tree.prevToken(tree.prevToken(align_expr.firstToken())) + else if (fn_proto.getSectionExpr()) |section_expr| + tree.prevToken(tree.prevToken(section_expr.firstToken())) + else if (fn_proto.getCallconvExpr()) |callconv_expr| + tree.prevToken(tree.prevToken(callconv_expr.firstToken())) + else switch (fn_proto.return_type) { + .Explicit => |node| node.firstToken(), + .InferErrorSet => |node| tree.prevToken(node.firstToken()), + .Invalid => unreachable, + }, + ); assert(tree.token_ids[rparen] == .RParen); const src_params_trailing_comma = blk: { From c06674e701fdd74165778dd46b72cc469ba29140 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Wed, 9 Sep 2020 22:58:13 +1000 Subject: [PATCH 08/37] zig fmt: Small cleanup --- lib/std/zig/render.zig | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 319788546e..78f8ddb022 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -747,6 +747,10 @@ fn renderExpression( defer allocator.free(widths); mem.set(usize, widths, 0); + var expr_newlines = try allocator.alloc(bool, row_exprs.len); + defer allocator.free(expr_newlines); + mem.set(bool, expr_newlines, false); + var expr_widths = widths[0 .. widths.len - row_size]; var column_widths = widths[widths.len - row_size ..]; @@ -791,7 +795,7 @@ fn renderExpression( var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, counting_stream.writer()); // Calculate size of columns in current section - var c: usize = 0; + var column_counter: usize = 0; var single_line = true; for (section_exprs) |expr, i| { if (i + 1 < section_exprs.len) { @@ -800,40 +804,42 @@ fn renderExpression( try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); const width = @intCast(usize, counting_stream.bytes_written); expr_widths[i] = width; + expr_newlines[i] = line_find_stream.byte_found; if (!line_find_stream.byte_found) { - const col = c % row_size; - column_widths[col] = std.math.max(column_widths[col], width); + const column = column_counter % row_size; + column_widths[column] = std.math.max(column_widths[column], width); const expr_last_token = expr.*.lastToken() + 1; const next_expr = section_exprs[i + 1]; const loc = tree.tokenLocation(tree.token_locs[expr_last_token].start, next_expr.*.firstToken()); if (loc.line == 0) { - c += 1; + column_counter += 1; } else { single_line = false; - c = 0; + column_counter = 0; } } else { single_line = false; - c = 0; + column_counter = 0; } } else { counting_stream.bytes_written = 0; try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); const width = @intCast(usize, counting_stream.bytes_written); expr_widths[i] = width; + expr_newlines[i] = line_find_stream.byte_found; if (!line_find_stream.byte_found) { - const col = c % row_size; - column_widths[col] = std.math.max(column_widths[col], width); + const column = column_counter % row_size; + column_widths[column] = std.math.max(column_widths[column], width); } break; } } // Render exprs in current section - c = 0; + column_counter = 0; var last_col_index: usize = row_size - 1; for (section_exprs) |expr, i| { if (i + 1 < section_exprs.len) { @@ -842,18 +848,15 @@ fn renderExpression( const comma = tree.nextToken(expr.*.lastToken()); - if (c != last_col_index) { - line_find_stream.byte_found = false; - try renderExpression(allocator, &auto_indenting_stream, tree, expr, Space.None); - try renderExpression(allocator, &auto_indenting_stream, tree, next_expr, Space.None); - if (!line_find_stream.byte_found) { + if (column_counter != last_col_index) { + if (!expr_newlines[i] and !expr_newlines[i + 1]) { // Neither the current or next expression is multiline try renderToken(tree, ais, comma, Space.Space); // , - assert(column_widths[c % row_size] >= expr_widths[i]); - const padding = column_widths[c % row_size] - expr_widths[i]; + assert(column_widths[column_counter % row_size] >= expr_widths[i]); + const padding = column_widths[column_counter % row_size] - expr_widths[i]; try ais.writer().writeByteNTimes(' ', padding); - c += 1; + column_counter += 1; continue; } } @@ -862,7 +865,7 @@ fn renderExpression( continue; } - c = 0; + column_counter = 0; try renderToken(tree, ais, comma, Space.Newline); // , try renderExtraNewline(tree, ais, next_expr); } else { @@ -1091,7 +1094,6 @@ fn renderExpression( try renderExpression(allocator, ais, tree, param_node, Space.None); if (i + 1 < params.len) { - const next_param = params[i + 1]; const comma = tree.nextToken(param_node.lastToken()); try renderToken(tree, ais, comma, Space.Space); } From 40b6e86a999ce80b9f71c7a88df6186400f151ac Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Thu, 10 Sep 2020 20:32:40 +1000 Subject: [PATCH 09/37] zig fmt: fix #6171 --- lib/std/zig/parser_test.zig | 18 ++++++++++++++++++ lib/std/zig/render.zig | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index d6dd9c1a73..f3cfe811a4 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3583,6 +3583,24 @@ test "zig fmt: use of comments and Multiline string literals may force the param ); } +test "zig fmt: single argument trailing commas in @builtins()" { + try testCanonical( + \\pub fn foo(qzz: []u8) i1 { + \\ @panic( + \\ foo, + \\ ); + \\ panic( + \\ foo, + \\ ); + \\ @panic( + \\ foo, + \\ bar, + \\ ); + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 78f8ddb022..fbf2139b42 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -1489,7 +1489,7 @@ fn renderExpression( try renderToken(tree, ais, builtin_call.builtin_token, Space.None); // @name const src_params_trailing_comma = blk: { - if (builtin_call.params_len < 2) break :blk false; + if (builtin_call.params_len == 0) break :blk false; const last_node = builtin_call.params()[builtin_call.params_len - 1]; const maybe_comma = tree.nextToken(last_node.lastToken()); break :blk tree.token_ids[maybe_comma] == .Comma; From 1aacedf6e197ea212025dccad622894a44eb5461 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Thu, 10 Sep 2020 23:35:18 +1000 Subject: [PATCH 10/37] zig fmt: Fix regression in ArrayInitializers --- lib/std/zig/parser_test.zig | 41 ++++++++++++++++++++++++++++++++++--- lib/std/zig/render.zig | 39 +++++++++++++++-------------------- 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index f3cfe811a4..c7d64bc513 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3516,9 +3516,13 @@ test "zig fmt: multiline string literals should play nice with array initializer \\ .{( \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx \\ )}, - \\ .{ "xxxxxxx", "xxx", ( - \\ \\ xxx - \\ ), "xxx", "xxx" }, + \\ .{ + \\ "xxxxxxx", "xxx", + \\ ( + \\ \\ xxx + \\ ), + \\ "xxx", "xxx", + \\ }, \\ .{ "xxxxxxx", "xxx", "xxx", "xxx" }, .{ "xxxxxxx", "xxx", "xxx", "xxx" }, \\ "aaaaaaa", "bbbbbb", "ccccc", // - \\ "dddd", ("eee"), ("fff"), @@ -3601,6 +3605,37 @@ test "zig fmt: single argument trailing commas in @builtins()" { ); } +test "zig fmt: trailing comma should force multiline 1 column" { + try testTransform( + \\pub const UUID_NULL: uuid_t = [16]u8{0,0,0,0,}; + \\ + , + \\pub const UUID_NULL: uuid_t = [16]u8{ + \\ 0, + \\ 0, + \\ 0, + \\ 0, + \\}; + \\ + ); +} + +test "zig fmt: function params should align nicely" { + try testCanonical( + \\pub fn foo() void { + \\ cases.addRuntimeSafety("slicing operator with sentinel", + \\ \\const std = @import("std"); + \\ ++ check_panic_msg ++ + \\ \\pub fn main() void { + \\ \\ var buf = [4]u8{'a','b','c',0}; + \\ \\ const slice = buf[0..:0]; + \\ \\} + \\ ); + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index fbf2139b42..30f739aaef 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -733,14 +733,14 @@ fn renderExpression( } // scan to find row size - if (rowSize(tree, exprs, rtoken, false) != null) { + if (rowSize(tree, exprs, rtoken) != null) { { ais.pushIndentNextLine(); defer ais.popIndent(); try renderToken(tree, ais, lbrace, Space.Newline); var expr_index: usize = 0; - while (rowSize(tree, exprs[expr_index..], rtoken, true)) |row_size| { + while (rowSize(tree, exprs[expr_index..], rtoken)) |row_size| { const row_exprs = exprs[expr_index..]; // A place to store the width of each expression and its column's maximum var widths = try allocator.alloc(usize, row_exprs.len + row_size); @@ -757,14 +757,14 @@ fn renderExpression( // Find next row with trailing comment (if any) to end the current section var section_end = sec_end: { var this_line_first_expr: usize = 0; - var this_line_size = rowSize(tree, row_exprs, rtoken, true); + var this_line_size = rowSize(tree, row_exprs, rtoken); for (row_exprs) |expr, i| { // Ignore comment on first line of this section if (i == 0 or tree.tokensOnSameLine(row_exprs[0].firstToken(), expr.lastToken())) continue; // Track start of line containing comment if (!tree.tokensOnSameLine(row_exprs[this_line_first_expr].firstToken(), expr.lastToken())) { this_line_first_expr = i; - this_line_size = rowSize(tree, row_exprs[this_line_first_expr..], rtoken, true); + this_line_size = rowSize(tree, row_exprs[this_line_first_expr..], rtoken); } const maybe_comma = expr.lastToken() + 1; @@ -860,7 +860,7 @@ fn renderExpression( continue; } } - if (single_line) { + if (single_line and row_size != 1) { try renderToken(tree, ais, comma, Space.Space); // , continue; } @@ -1087,7 +1087,8 @@ fn renderExpression( const params = call.params(); for (params) |param_node, i| { const maybe_comment = param_node.firstToken() - 1; - if (param_node.*.tag == .MultilineStringLiteral or tree.token_ids[maybe_comment] == .LineComment) { + const maybe_multiline_string = param_node.firstToken(); + if (tree.token_ids[maybe_multiline_string] == .MultilineStringLiteralLine or tree.token_ids[maybe_comment] == .LineComment) { ais.pushIndentOneShot(); } @@ -2629,7 +2630,16 @@ fn copyFixingWhitespace(ais: anytype, slice: []const u8) @TypeOf(ais.*).Error!vo }; } -fn rowSize(tree: *ast.Tree, exprs: []*ast.Node, rtoken: ast.TokenIndex, force: bool) ?usize { +fn rowSize(tree: *ast.Tree, exprs: []*ast.Node, rtoken: ast.TokenIndex) ?usize { + const first_token = exprs[0].firstToken(); + const first_loc = tree.tokenLocation(tree.token_locs[first_token].start, rtoken); + if (first_loc.line == 0) { + const maybe_comma = tree.prevToken(rtoken); + if (tree.token_ids[maybe_comma] == .Comma) + return 1; + return null; // no newlines + } + var count: usize = 1; for (exprs) |expr, i| { if (i + 1 < exprs.len) { @@ -2638,21 +2648,6 @@ fn rowSize(tree: *ast.Tree, exprs: []*ast.Node, rtoken: ast.TokenIndex, force: b if (loc.line != 0) return count; count += 1; } else { - if (force) return count; - const expr_last_token = expr.lastToken(); - const loc = tree.tokenLocation(tree.token_locs[expr_last_token].start, rtoken); - if (loc.line == 0) { - // all on one line - const src_has_trailing_comma = trailblk: { - const maybe_comma = tree.prevToken(rtoken); - break :trailblk tree.token_ids[maybe_comma] == .Comma; - }; - if (src_has_trailing_comma) { - return 1; // force row size 1 - } else { - return null; // no newlines - } - } return count; } } From 4496a6c9cccbd6a9c82b5d4ca7f533b18ebeab32 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Tue, 15 Sep 2020 18:49:59 +1000 Subject: [PATCH 11/37] zig fmt: Special case un-indent comma after multiline string in param list --- lib/std/zig/parser_test.zig | 11 +++++++++-- lib/std/zig/render.zig | 7 +++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index c7d64bc513..994ad6d5d1 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -1374,7 +1374,7 @@ test "zig fmt: multiline string parameter in fn call with trailing comma" { \\ \\ZIG_C_HEADER_FILES {} \\ \\ZIG_DIA_GUIDS_LIB {} \\ \\ - \\ , + \\ , \\ std.cstr.toSliceConst(c.ZIG_CMAKE_BINARY_DIR), \\ std.cstr.toSliceConst(c.ZIG_CXX_COMPILER), \\ std.cstr.toSliceConst(c.ZIG_DIA_GUIDS_LIB), @@ -3385,10 +3385,17 @@ test "zig fmt: Indent comma correctly after multiline string literals in arg lis \\ \\------------ \\ \\xxxxxxxxxxxx \\ \\xxxxxxxxxxxx - \\ , + \\ , \\ g.GtkMessageType.GTK_MESSAGE_WARNING, \\ null, \\ ); + \\ + \\ z.display_message_dialog(*const [323:0]u8, + \\ \\Message Text + \\ \\------------ + \\ \\xxxxxxxxxxxx + \\ \\xxxxxxxxxxxx + \\ , g.GtkMessageType.GTK_MESSAGE_WARNING, null); \\} \\ ); diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 30f739aaef..67afbb77d9 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -1071,6 +1071,13 @@ fn renderExpression( if (i + 1 < params.len) { const next_node = params[i + 1]; try renderExpression(allocator, ais, tree, param_node, Space.None); + + // Unindent the comma for multiline string literals + const maybe_multiline_string = param_node.firstToken(); + const is_multiline_string = tree.token_ids[maybe_multiline_string] == .MultilineStringLiteralLine; + if (is_multiline_string) ais.popIndent(); + defer if (is_multiline_string) ais.pushIndent(); + const comma = tree.nextToken(param_node.lastToken()); try renderToken(tree, ais, comma, Space.Newline); // , try renderExtraNewline(tree, ais, next_node); From e85c89630e78ccc0e4bab44064779a07a029cecd Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Tue, 16 Jun 2020 22:38:51 +0200 Subject: [PATCH 12/37] accept Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 34 ++++++++++++++++++++++++++++++++++ lib/std/net.zig | 15 ++++++++++----- lib/std/os.zig | 7 +------ 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index c547f50365..f44b2f06e4 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -720,6 +720,40 @@ pub const Loop = struct { } } + /// ------- I/0 APIs ------- + pub fn accept( + self: *Loop, + /// This argument is a socket that has been created with `socket`, bound to a local address + /// with `bind`, and is listening for connections after a `listen`. + sockfd: os.fd_t, + /// This argument is a pointer to a sockaddr structure. This structure is filled in with the + /// address of the peer socket, as known to the communications layer. The exact format of the + /// address returned addr is determined by the socket's address family (see `socket` and the + /// respective protocol man pages). + addr: *os.sockaddr, + /// This argument is a value-result argument: the caller must initialize it to contain the + /// size (in bytes) of the structure pointed to by addr; on return it will contain the actual size + /// of the peer address. + /// + /// The returned address is truncated if the buffer provided is too small; in this case, `addr_size` + /// will return a value greater than was supplied to the call. + addr_size: *os.socklen_t, + /// The following values can be bitwise ORed in flags to obtain different behavior: + /// * `SOCK_CLOEXEC` - Set the close-on-exec (`FD_CLOEXEC`) flag on the new file descriptor. See the + /// description of the `O_CLOEXEC` flag in `open` for reasons why this may be useful. + flags: u32, + ) os.AcceptError!os.fd_t { + while (true) { + return os.accept(sockfd, addr, addr_size, flags | os.SOCK_NONBLOCK) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdReadable(sockfd); + continue; + }, + else => return err, + }; + } + } + /// Performs an async `os.open` using a separate thread. pub fn openZ(self: *Loop, file_path: [*:0]const u8, flags: u32, mode: os.mode_t) os.OpenError!os.fd_t { var req_node = Request.Node{ diff --git a/lib/std/net.zig b/lib/std/net.zig index 45d8f07f04..6b6d234843 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -1661,18 +1661,23 @@ pub const StreamServer = struct { /// If this function succeeds, the returned `Connection` is a caller-managed resource. pub fn accept(self: *StreamServer) AcceptError!Connection { - const nonblock = if (std.io.is_async) os.SOCK_NONBLOCK else 0; - const accept_flags = nonblock | os.SOCK_CLOEXEC; var accepted_addr: Address = undefined; var adr_len: os.socklen_t = @sizeOf(Address); - if (os.accept(self.sockfd.?, &accepted_addr.any, &adr_len, accept_flags)) |fd| { + const accept_result = blk: { + if (std.io.is_async) { + const loop = std.event.Loop.instance orelse return error.UnexpectedError; + break :blk loop.accept(self.sockfd.?, &accepted_addr.any, &adr_len, os.SOCK_CLOEXEC); + } else { + break :blk os.accept(self.sockfd.?, &accepted_addr.any, &adr_len, os.SOCK_CLOEXEC); + } + }; + + if (accept_result) |fd| { return Connection{ .file = fs.File{ .handle = fd }, .address = accepted_addr, }; } else |err| switch (err) { - // We only give SOCK_NONBLOCK when I/O mode is async, in which case this error - // is handled by os.accept4. error.WouldBlock => unreachable, else => |e| return e, } diff --git a/lib/std/os.zig b/lib/std/os.zig index 0b09b1f82a..c5c34d4f40 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -2890,12 +2890,7 @@ pub fn accept( return fd; }, EINTR => continue, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdReadable(sockfd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => unreachable, // always a race condition ECONNABORTED => return error.ConnectionAborted, EFAULT => unreachable, From 730428bfd615cab415b2942fc9b781428a0ff692 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Tue, 16 Jun 2020 22:39:09 +0200 Subject: [PATCH 13/37] connect Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 10 ++++++++++ lib/std/net.zig | 18 ++++++++++++------ lib/std/os.zig | 6 +----- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index f44b2f06e4..2dc1d5659e 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -754,6 +754,16 @@ pub const Loop = struct { } } + pub fn connect(self: *Loop, sockfd: os.socket_t, sock_addr: *const os.sockaddr, len: os.socklen_t) os.ConnectError!void { + os.connect(sockfd, sock_addr, len) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdWritable(sockfd); + return os.getsockoptError(sockfd); + }, + else => return err, + }; + } + /// Performs an async `os.open` using a separate thread. pub fn openZ(self: *Loop, file_path: [*:0]const u8, flags: u32, mode: os.mode_t) os.OpenError!os.fd_t { var req_node = Request.Node{ diff --git a/lib/std/net.zig b/lib/std/net.zig index 6b6d234843..928ebbbce5 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -614,11 +614,11 @@ pub fn connectUnixSocket(path: []const u8) !fs.File { var addr = try std.net.Address.initUnix(path); - try os.connect( - sockfd, - &addr.any, - addr.getOsSockLen(), - ); + if (std.io.is_async) { + try loop.connect(sockfd, &addr.any, addr.getOsSockLen()); + } else { + try os.connect(sockfd, &addr.any, addr.getOsSockLen()); + } return fs.File{ .handle = sockfd, @@ -677,7 +677,13 @@ pub fn tcpConnectToAddress(address: Address) !fs.File { (if (builtin.os.tag == .windows) 0 else os.SOCK_CLOEXEC); const sockfd = try os.socket(address.any.family, sock_flags, os.IPPROTO_TCP); errdefer os.close(sockfd); - try os.connect(sockfd, &address.any, address.getOsSockLen()); + + if (std.io.is_async) { + const loop = std.event.Loop.instance orelse return error.WouldBlock; + try loop.connect(sockfd, &address.any, address.getOsSockLen()); + } else { + try os.connect(sockfd, &address.any, address.getOsSockLen()); + } return fs.File{ .handle = sockfd }; } diff --git a/lib/std/os.zig b/lib/std/os.zig index c5c34d4f40..023c1d5971 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -3108,11 +3108,7 @@ pub fn connect(sockfd: socket_t, sock_addr: *const sockaddr, len: socklen_t) Con EADDRINUSE => return error.AddressInUse, EADDRNOTAVAIL => return error.AddressNotAvailable, EAFNOSUPPORT => return error.AddressFamilyNotSupported, - EAGAIN, EINPROGRESS => { - const loop = std.event.Loop.instance orelse return error.WouldBlock; - loop.waitUntilFdWritable(sockfd); - return getsockoptError(sockfd); - }, + EAGAIN, EINPROGRESS => return error.WouldBlock, EALREADY => unreachable, // The socket is nonblocking and a previous connection attempt has not yet been completed. EBADF => unreachable, // sockfd is not a valid open file descriptor. ECONNREFUSED => return error.ConnectionRefused, From 08364ac773bdc95b9407974b5c761dbdab863f4d Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Tue, 16 Jun 2020 23:14:05 +0200 Subject: [PATCH 14/37] read Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 40 ++++++++++++++++++++++++++-------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index 2dc1d5659e..7adc6e3a8f 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -822,23 +822,35 @@ pub const Loop = struct { /// Performs an async `os.read` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn read(self: *Loop, fd: os.fd_t, buf: []u8) os.ReadError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .read = .{ - .fd = fd, - .buf = buf, - .result = undefined, + pub fn read(self: *Loop, fd: os.fd_t, buf: []u8, simulate_evented: bool) os.ReadError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .read = .{ + .fd = fd, + .buf = buf, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.read.result; + } else { + while (true) { + return os.read(fd, buf) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdReadable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.read.result; } /// Performs an async `os.readv` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 73babf5fa2..bd0b8a2943 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -414,10 +414,12 @@ pub const File = struct { pub fn read(self: File, buffer: []u8) ReadError!usize { if (is_windows) { return windows.ReadFile(self.handle, buffer, null, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.read(self.handle, buffer); - } else { + } + + if (self.intended_io_mode == .blocking or !std.io.is_async) { return os.read(self.handle, buffer); + } else { + return std.event.Loop.instance.?.read(self.handle, buffer, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 023c1d5971..7db545a17c 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -366,12 +366,7 @@ pub fn read(fd: fd_t, buf: []u8) ReadError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdReadable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForReading, // Can be a race condition. EIO => return error.InputOutput, EISDIR => return error.IsDir, From bc35435ca6e51d0e120538398e3c708ada57f6de Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 19 Jun 2020 23:00:17 +0200 Subject: [PATCH 15/37] readv Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 40 ++++++++++++++++++++++++++-------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index 7adc6e3a8f..96774e3f11 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -855,23 +855,35 @@ pub const Loop = struct { /// Performs an async `os.readv` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn readv(self: *Loop, fd: os.fd_t, iov: []const os.iovec) os.ReadError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .readv = .{ - .fd = fd, - .iov = iov, - .result = undefined, + pub fn readv(self: *Loop, fd: os.fd_t, iov: []const os.iovec, simulate_evented: bool) os.ReadError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .readv = .{ + .fd = fd, + .iov = iov, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.readv.result; + } else { + while (true) { + return os.readv(fd, iov) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdReadable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.readv.result; } /// Performs an async `os.pread` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index bd0b8a2943..51ad931504 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -463,10 +463,12 @@ pub const File = struct { if (iovecs.len == 0) return @as(usize, 0); const first = iovecs[0]; return windows.ReadFile(self.handle, first.iov_base[0..first.iov_len], null, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.readv(self.handle, iovecs); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.readv(self.handle, iovecs); + } else { + return std.event.Loop.instance.?.readv(self.handle, iovecs, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 7db545a17c..e5a06965d8 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -423,12 +423,7 @@ pub fn readv(fd: fd_t, iov: []const iovec) ReadError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdReadable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForReading, // can be a race condition EIO => return error.InputOutput, EISDIR => return error.IsDir, From bd9f2369d5c4e5fcd38342c877f9ae6531f78909 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 19 Jun 2020 23:08:34 +0200 Subject: [PATCH 16/37] pread Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 42 +++++++++++++++++++++++++++--------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index 96774e3f11..27c00b9ab3 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -888,24 +888,36 @@ pub const Loop = struct { /// Performs an async `os.pread` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn pread(self: *Loop, fd: os.fd_t, buf: []u8, offset: u64) os.PReadError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .pread = .{ - .fd = fd, - .buf = buf, - .offset = offset, - .result = undefined, + pub fn pread(self: *Loop, fd: os.fd_t, buf: []u8, offset: u64, simulate_evented: bool) os.PReadError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .pread = .{ + .fd = fd, + .buf = buf, + .offset = offset, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.pread.result; + } else { + while (true) { + return os.pread(fd, buf, offset) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdReadable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.pread.result; } /// Performs an async `os.preadv` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 51ad931504..cb3b41ffdf 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -438,10 +438,12 @@ pub const File = struct { pub fn pread(self: File, buffer: []u8, offset: u64) PReadError!usize { if (is_windows) { return windows.ReadFile(self.handle, buffer, offset, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.pread(self.handle, buffer, offset); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.pread(self.handle, buffer, offset); + } else { + return std.event.Loop.instance.?.pread(self.handle, buffer, offset, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index e5a06965d8..6ed555011e 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -482,12 +482,7 @@ pub fn pread(fd: fd_t, buf: []u8, offset: u64) PReadError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdReadable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForReading, // Can be a race condition. EIO => return error.InputOutput, EISDIR => return error.IsDir, From 59ecdaea127cb680d295ad02319dacba75ac1e73 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 19 Jun 2020 23:13:54 +0200 Subject: [PATCH 17/37] preadv Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 42 +++++++++++++++++++++++++++--------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index 27c00b9ab3..a14e798c50 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -922,24 +922,36 @@ pub const Loop = struct { /// Performs an async `os.preadv` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn preadv(self: *Loop, fd: os.fd_t, iov: []const os.iovec, offset: u64) os.ReadError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .preadv = .{ - .fd = fd, - .iov = iov, - .offset = offset, - .result = undefined, + pub fn preadv(self: *Loop, fd: os.fd_t, iov: []const os.iovec, offset: u64, simulate_evented: bool) os.ReadError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .preadv = .{ + .fd = fd, + .iov = iov, + .offset = offset, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.preadv.result; + } else { + while (true) { + return os.preadv(fd, iov, offset) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdReadable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.preadv.result; } /// Performs an async `os.write` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index cb3b41ffdf..4aa33fc5b9 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -506,10 +506,12 @@ pub const File = struct { if (iovecs.len == 0) return @as(usize, 0); const first = iovecs[0]; return windows.ReadFile(self.handle, first.iov_base[0..first.iov_len], offset, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.preadv(self.handle, iovecs, offset); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.preadv(self.handle, iovecs, offset); + } else { + return std.event.Loop.instance.?.preadv(self.handle, iovecs, offset, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 6ed555011e..6fb9d4388b 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -622,12 +622,7 @@ pub fn preadv(fd: fd_t, iov: []const iovec, offset: u64) PReadError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdReadable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForReading, // can be a race condition EIO => return error.InputOutput, EISDIR => return error.IsDir, From 9075f8e5a1a788770cc8193d6f54118434e72a4b Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 19 Jun 2020 23:29:11 +0200 Subject: [PATCH 18/37] write Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 40 ++++++++++++++++++++++++++-------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index a14e798c50..ec45e85630 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -956,23 +956,35 @@ pub const Loop = struct { /// Performs an async `os.write` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn write(self: *Loop, fd: os.fd_t, bytes: []const u8) os.WriteError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .write = .{ - .fd = fd, - .bytes = bytes, - .result = undefined, + pub fn write(self: *Loop, fd: os.fd_t, bytes: []const u8, simulate_evented: bool) os.WriteError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .write = .{ + .fd = fd, + .bytes = bytes, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.write.result; + } else { + while (true) { + return os.write(fd, bytes) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdWritable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.write.result; } /// Performs an async `os.writev` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 4aa33fc5b9..428b758e61 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -547,10 +547,12 @@ pub const File = struct { pub fn write(self: File, bytes: []const u8) WriteError!usize { if (is_windows) { return windows.WriteFile(self.handle, bytes, null, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.write(self.handle, bytes); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.write(self.handle, bytes); + } else { + return std.event.Loop.instance.?.write(self.handle, bytes, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 6fb9d4388b..da5ce13508 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -721,12 +721,7 @@ pub fn write(fd: fd_t, bytes: []const u8) WriteError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdWritable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForWriting, // can be a race condition. EDESTADDRREQ => unreachable, // `connect` was never called. EDQUOT => return error.DiskQuota, From 18f6629bd8ad8bd13108ddce82c32baf6f53628e Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 19 Jun 2020 23:34:02 +0200 Subject: [PATCH 19/37] writev Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 40 ++++++++++++++++++++++++++-------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index ec45e85630..e5fdbd2e3a 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -989,23 +989,35 @@ pub const Loop = struct { /// Performs an async `os.writev` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn writev(self: *Loop, fd: os.fd_t, iov: []const os.iovec_const) os.WriteError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .writev = .{ - .fd = fd, - .iov = iov, - .result = undefined, + pub fn writev(self: *Loop, fd: os.fd_t, iov: []const os.iovec_const, simulate_evented: bool) os.WriteError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .writev = .{ + .fd = fd, + .iov = iov, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.writev.result; + } else { + while (true) { + return os.writev(fd, iov) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdWritable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.writev.result; } /// Performs an async `os.pwritev` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 428b758e61..ab7ea2f579 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -586,10 +586,12 @@ pub const File = struct { if (iovecs.len == 0) return @as(usize, 0); const first = iovecs[0]; return windows.WriteFile(self.handle, first.iov_base[0..first.iov_len], null, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.writev(self.handle, iovecs); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.writev(self.handle, iovecs); + } else { + return std.event.Loop.instance.?.writev(self.handle, iovecs, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index da5ce13508..8f761b48be 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -789,12 +789,7 @@ pub fn writev(fd: fd_t, iov: []const iovec_const) WriteError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdWritable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForWriting, // Can be a race condition. EDESTADDRREQ => unreachable, // `connect` was never called. EDQUOT => return error.DiskQuota, From 7a07c62a075ada32c5eca298e4ed6a4c8a14cf89 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Sat, 20 Jun 2020 00:17:10 +0200 Subject: [PATCH 20/37] pwrite Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 49 +++++++++++++++++++++++++++++++++++++++++- lib/std/fs/file.zig | 8 ++++--- lib/std/os.zig | 7 +----- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index e5fdbd2e3a..b3b62bbe8e 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -1020,9 +1020,43 @@ pub const Loop = struct { } } + /// Performs an async `os.pwrite` using a separate thread. + /// `fd` must block and not return EAGAIN. + pub fn pwrite(self: *Loop, fd: os.fd_t, bytes: []const u8, offset: u64, simulate_evented: bool) os.PWriteError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .pwrite = .{ + .fd = fd, + .bytes = bytes, + .offset = offset, + .result = undefined, + }, + }, + .finish = .{ .TickNode = .{ .data = @frame() } }, + }, + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.pwrite.result; + } else { + while (true) { + return os.pwrite(fd, bytes, offset) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdWritable(fd); + continue; + }, + else => return err, + }; + } + } + } + /// Performs an async `os.pwritev` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn pwritev(self: *Loop, fd: os.fd_t, iov: []const os.iovec_const, offset: u64) os.WriteError!usize { + pub fn pwritev(self: *Loop, fd: os.fd_t, iov: []const os.iovec_const, offset: u64) os.PWriteError!usize { var req_node = Request.Node{ .data = .{ .msg = .{ @@ -1194,6 +1228,9 @@ pub const Loop = struct { .writev => |*msg| { msg.result = os.writev(msg.fd, msg.iov); }, + .pwrite => |*msg| { + msg.result = os.pwrite(msg.fd, msg.bytes, msg.offset); + }, .pwritev => |*msg| { msg.result = os.pwritev(msg.fd, msg.iov, msg.offset); }, @@ -1263,6 +1300,7 @@ pub const Loop = struct { readv: ReadV, write: Write, writev: WriteV, + pwrite: PWrite, pwritev: PWriteV, pread: PRead, preadv: PReadV, @@ -1306,6 +1344,15 @@ pub const Loop = struct { pub const Error = os.WriteError; }; + pub const PWrite = struct { + fd: os.fd_t, + bytes: []const u8, + offset: usize, + result: Error!usize, + + pub const Error = os.PWriteError; + }; + pub const PWriteV = struct { fd: os.fd_t, iov: []const os.iovec_const, diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index ab7ea2f579..60efe28ec2 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -566,10 +566,12 @@ pub const File = struct { pub fn pwrite(self: File, bytes: []const u8, offset: u64) PWriteError!usize { if (is_windows) { return windows.WriteFile(self.handle, bytes, offset, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.pwrite(self.handle, bytes, offset); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.pwrite(self.handle, bytes, offset); + } else { + return std.event.Loop.instance.?.pwrite(self.handle, bytes, offset, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 8f761b48be..595815a1e7 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -875,12 +875,7 @@ pub fn pwrite(fd: fd_t, bytes: []const u8, offset: u64) PWriteError!usize { EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdWritable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForWriting, // Can be a race condition. EDESTADDRREQ => unreachable, // `connect` was never called. EDQUOT => return error.DiskQuota, From 7fec5b3def36bc73e9e48a777bcca838c4b86770 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Sat, 20 Jun 2020 00:19:38 +0200 Subject: [PATCH 21/37] pwritev Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 44 +++++++++++++++++++++++++++--------------- lib/std/fs/file.zig | 8 +++++--- lib/std/os.zig | 7 +------ 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index b3b62bbe8e..ae2d2f1499 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -1022,7 +1022,7 @@ pub const Loop = struct { /// Performs an async `os.pwrite` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn pwrite(self: *Loop, fd: os.fd_t, bytes: []const u8, offset: u64, simulate_evented: bool) os.PWriteError!usize { + pub fn pwrite(self: *Loop, fd: os.fd_t, bytes: []const u8, offset: u64, simulate_evented: bool) os.PerformsWriteError!usize { if (simulate_evented) { var req_node = Request.Node{ .data = .{ @@ -1056,24 +1056,36 @@ pub const Loop = struct { /// Performs an async `os.pwritev` using a separate thread. /// `fd` must block and not return EAGAIN. - pub fn pwritev(self: *Loop, fd: os.fd_t, iov: []const os.iovec_const, offset: u64) os.PWriteError!usize { - var req_node = Request.Node{ - .data = .{ - .msg = .{ - .pwritev = .{ - .fd = fd, - .iov = iov, - .offset = offset, - .result = undefined, + pub fn pwritev(self: *Loop, fd: os.fd_t, iov: []const os.iovec_const, offset: u64, simulate_evented: bool) os.PWriteError!usize { + if (simulate_evented) { + var req_node = Request.Node{ + .data = .{ + .msg = .{ + .pwritev = .{ + .fd = fd, + .iov = iov, + .offset = offset, + .result = undefined, + }, }, + .finish = .{ .TickNode = .{ .data = @frame() } }, }, - .finish = .{ .TickNode = .{ .data = @frame() } }, - }, - }; - suspend { - self.posixFsRequest(&req_node); + }; + suspend { + self.posixFsRequest(&req_node); + } + return req_node.data.msg.pwritev.result; + } else { + while (true) { + return os.pwritev(fd, iov, offset) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdWritable(fd); + continue; + }, + else => return err, + }; + } } - return req_node.data.msg.pwritev.result; } /// Performs an async `os.faccessatZ` using a separate thread. diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 60efe28ec2..f3e980b9f0 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -621,10 +621,12 @@ pub const File = struct { if (iovecs.len == 0) return @as(usize, 0); const first = iovecs[0]; return windows.WriteFile(self.handle, first.iov_base[0..first.iov_len], offset, self.intended_io_mode); - } else if (self.capable_io_mode != self.intended_io_mode) { - return std.event.Loop.instance.?.pwritev(self.handle, iovecs, offset); - } else { + } + + if (self.intended_io_mode == .blocking) { return os.pwritev(self.handle, iovecs, offset); + } else { + return std.event.Loop.instance.?.pwritev(self.handle, iovecs, offset, self.capable_io_mode != self.intended_io_mode); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 595815a1e7..1138013e8c 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -958,12 +958,7 @@ pub fn pwritev(fd: fd_t, iov: []const iovec_const, offset: u64) PWriteError!usiz EINTR => continue, EINVAL => unreachable, EFAULT => unreachable, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdWritable(fd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EBADF => return error.NotOpenForWriting, // Can be a race condition. EDESTADDRREQ => unreachable, // `connect` was never called. EDQUOT => return error.DiskQuota, From 419aea54cb30b394191778fcc70effaf5181bf33 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Sat, 20 Jun 2020 00:35:08 +0200 Subject: [PATCH 22/37] sendto Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 21 +++++++++++++++++++++ lib/std/net.zig | 12 ++++++++++-- lib/std/os.zig | 8 +------- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index ae2d2f1499..3a79d36a10 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -1088,6 +1088,27 @@ pub const Loop = struct { } } + pub fn sendto( + self: *Loop, + /// The file descriptor of the sending socket. + sockfd: os.fd_t, + /// Message to send. + buf: []const u8, + flags: u32, + dest_addr: ?*const os.sockaddr, + addrlen: os.socklen_t, + ) os.SendError!usize { + while (true) { + return os.sendto(sockfd, buf, flags, dest_addr, addrlen) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdWritable(sockfd); + continue; + }, + else => return err, + }; + } + } + /// Performs an async `os.faccessatZ` using a separate thread. /// `fd` must block and not return EAGAIN. pub fn faccessatZ( diff --git a/lib/std/net.zig b/lib/std/net.zig index 928ebbbce5..8fe19f955d 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -1435,7 +1435,11 @@ fn resMSendRc( if (answers[i].len == 0) { var j: usize = 0; while (j < ns.len) : (j += 1) { - _ = os.sendto(fd, queries[i], os.MSG_NOSIGNAL, &ns[j].any, sl) catch undefined; + if (std.io.is_async) { + _ = std.event.Loop.instance.?.sendto(fd, queries[i], os.MSG_NOSIGNAL, &ns[j].any, sl) catch undefined; + } else { + _ = os.sendto(fd, queries[i], os.MSG_NOSIGNAL, &ns[j].any, sl) catch undefined; + } } } } @@ -1476,7 +1480,11 @@ fn resMSendRc( 0, 3 => {}, 2 => if (servfail_retry != 0) { servfail_retry -= 1; - _ = os.sendto(fd, queries[i], os.MSG_NOSIGNAL, &ns[j].any, sl) catch undefined; + if (std.io.is_async) { + _ = std.event.Loop.instance.?.sendto(fd, queries[i], os.MSG_NOSIGNAL, &ns[j].any, sl) catch undefined; + } else { + _ = os.sendto(fd, queries[i], os.MSG_NOSIGNAL, &ns[j].any, sl) catch undefined; + } }, else => continue, } diff --git a/lib/std/os.zig b/lib/std/os.zig index 1138013e8c..2f354e33d6 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -4571,14 +4571,8 @@ pub fn sendto( const rc = system.sendto(sockfd, buf.ptr, buf.len, flags, dest_addr, addrlen); switch (errno(rc)) { 0 => return @intCast(usize, rc), - EACCES => return error.AccessDenied, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdWritable(sockfd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, EALREADY => return error.FastOpenAlreadyInProgress, EBADF => unreachable, // always a race condition ECONNRESET => return error.ConnectionResetByPeer, From c196c27af86f0f10b2f53240a68477391c4b9820 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Sat, 20 Jun 2020 00:45:51 +0200 Subject: [PATCH 23/37] recvfrom Signed-off-by: Loris Cro --- lib/std/event/loop.zig | 18 ++++++++++++++++++ lib/std/net.zig | 5 ++++- lib/std/os.zig | 7 +------ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index 3a79d36a10..c3bf2495ff 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -1109,6 +1109,24 @@ pub const Loop = struct { } } + pub fn recvfrom( + sockfd: os.fd_t, + buf: []u8, + flags: u32, + src_addr: ?*os.sockaddr, + addrlen: ?*os.socklen_t, + ) os.RecvFromError!usize { + while (true) { + return os.recvfrom(sockfd, buf, flags, src_addr, addrlen) catch |err| switch (err) { + error.WouldBlock => { + self.waitUntilFdReadable(sockfd); + continue; + }, + else => return err, + }; + } + } + /// Performs an async `os.faccessatZ` using a separate thread. /// `fd` must block and not return EAGAIN. pub fn faccessatZ( diff --git a/lib/std/net.zig b/lib/std/net.zig index 8fe19f955d..fe7d0fafe6 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -1454,7 +1454,10 @@ fn resMSendRc( while (true) { var sl_copy = sl; - const rlen = os.recvfrom(fd, answer_bufs[next], 0, &sa.any, &sl_copy) catch break; + const rlen = if (std.io.is_async) + std.event.Loop.instance.?.recvfrom(fd, answer_bufs[next], 0, &sa.any, &sl_copy) catch break + else + os.recvfrom(fd, answer_bufs[next], 0, &sa.any, &sl_copy) catch break; // Ignore non-identifiable packets if (rlen < 4) continue; diff --git a/lib/std/os.zig b/lib/std/os.zig index 2f354e33d6..2c5f3065b2 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -5068,12 +5068,7 @@ pub fn recvfrom( ENOTCONN => unreachable, ENOTSOCK => unreachable, EINTR => continue, - EAGAIN => if (std.event.Loop.instance) |loop| { - loop.waitUntilFdReadable(sockfd); - continue; - } else { - return error.WouldBlock; - }, + EAGAIN => return error.WouldBlock, ENOMEM => return error.SystemResources, ECONNREFUSED => return error.ConnectionRefused, else => |err| return unexpectedErrno(err), From 7f68b14377ffe33ded2866f6bf53f213a9c5f620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20=28xq=29=20Quei=C3=9Fner?= Date: Fri, 25 Sep 2020 09:27:00 +0200 Subject: [PATCH 24/37] Implements std.meta.Tuple(), implements #4607 in userland. --- lib/std/meta.zig | 64 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/lib/std/meta.zig b/lib/std/meta.zig index 1507aa9de8..c92637250f 100644 --- a/lib/std/meta.zig +++ b/lib/std/meta.zig @@ -807,7 +807,7 @@ pub fn sizeof(target: anytype) usize { // TODO to get the correct result we have to translate // `1073741824 * 4` as `int(1073741824) *% int(4)` since // sizeof(1073741824 * 4) != sizeof(4294967296). - + // TODO test if target fits in int, long or long long return @sizeOf(c_int); }, @@ -826,3 +826,65 @@ test "sizeof" { testing.expect(sizeof(E.One) == @sizeOf(c_int)); testing.expect(sizeof(S) == 4); } + +/// For a given anonymous list of types, returns a new tuple type +/// with those types as fields. +/// +/// Examples: +/// - `Tuple(.{})` ⇒ `tuple { }` +/// - `Tuple(.{f32})` ⇒ `tuple { f32 }` +/// - `Tuple(.{f32,u32})` ⇒ `tuple { f32, u32 }` +pub fn Tuple(comptime types: anytype) type { + var tuple_fields: [types.len]std.builtin.TypeInfo.StructField = undefined; + inline for (types) |T, i| { + @setEvalBranchQuota(10_000); + var num_buf: [128]u8 = undefined; + tuple_fields[i] = std.builtin.TypeInfo.StructField{ + .name = std.fmt.bufPrint(&num_buf, "{d}", .{i}) catch unreachable, + .field_type = T, + .default_value = @as(?T, null), + .is_comptime = false, + }; + } + + return @Type(std.builtin.TypeInfo{ + .Struct = std.builtin.TypeInfo.Struct{ + .is_tuple = true, + .layout = .Auto, + .decls = &[_]std.builtin.TypeInfo.Declaration{}, + .fields = &tuple_fields, + }, + }); +} + +comptime { + const T = struct { + fn assertTypeEqual(comptime Expected: type, comptime Actual: type) void { + if (Expected != Actual) + @compileError("Expected type " ++ @typeName(Expected) ++ ", but got type " ++ @typeName(Actual)); + } + + fn assertTuple(comptime expected: anytype, comptime Actual: type) void { + const info = @typeInfo(Actual); + if (info != .Struct) + @compileError("Expected struct type"); + if (!info.Struct.is_tuple) + @compileError("Struct type must be a tuple type"); + + const fields_list = std.meta.fields(Actual); + if (expected.len != fields_list.len) + @compileError("Argument count mismatch"); + + inline for (fields_list) |fld, i| { + if (expected[i] != fld.field_type) { + @compileError("Field " ++ fld.name ++ " expected to be type " ++ @typeName(expected[i]) ++ ", but was type " ++ @typeName(fld.field_type)); + } + } + } + }; + + T.assertTuple(.{}, Tuple(.{})); + T.assertTuple(.{u32}, Tuple(.{u32})); + T.assertTuple(.{ u32, f16 }, Tuple(.{ u32, f16 })); + T.assertTuple(.{ u32, f16, []const u8 }, Tuple(.{ u32, f16, []const u8 })); +} From bd9003ed5b16a6e187999fb1190d89eb80bd587b Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 25 Sep 2020 00:21:57 +0200 Subject: [PATCH 25/37] std: ArenaAllocator tries to resize before allocating Closes #5116 --- lib/std/heap.zig | 7 +++++++ lib/std/heap/arena_allocator.zig | 21 +++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 16de215cc2..cf32cff645 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -919,6 +919,13 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void { const zero_bit_ptr = try allocator.create(u0); zero_bit_ptr.* = 0; allocator.destroy(zero_bit_ptr); + + const oversize = try allocator.allocAdvanced(u32, null, 5, .at_least); + testing.expect(oversize.len >= 5); + for (oversize) |*item| { + item.* = 0xDEADBEEF; + } + allocator.free(oversize); } pub fn testAllocatorAligned(base_allocator: *mem.Allocator, comptime alignment: u29) !void { diff --git a/lib/std/heap/arena_allocator.zig b/lib/std/heap/arena_allocator.zig index 0737cb2ef8..b7ee1d54c1 100644 --- a/lib/std/heap/arena_allocator.zig +++ b/lib/std/heap/arena_allocator.zig @@ -75,13 +75,22 @@ pub const ArenaAllocator = struct { const adjusted_addr = mem.alignForward(addr, ptr_align); const adjusted_index = self.state.end_index + (adjusted_addr - addr); const new_end_index = adjusted_index + n; - if (new_end_index > cur_buf.len) { - cur_node = try self.createNode(cur_buf.len, n + ptr_align); - continue; + + if (new_end_index <= cur_buf.len) { + const result = cur_buf[adjusted_index..new_end_index]; + self.state.end_index = new_end_index; + return result; } - const result = cur_buf[adjusted_index..new_end_index]; - self.state.end_index = new_end_index; - return result; + + const bigger_buf_size = @sizeOf(BufNode) + new_end_index; + // Try to grow the buffer in-place + cur_node.data = self.child_allocator.resize(cur_node.data, bigger_buf_size) catch |err| switch (err) { + error.OutOfMemory => { + // Allocate a new node if that's not possible + cur_node = try self.createNode(cur_buf.len, n + ptr_align); + continue; + }, + }; } } From 8d01133bd04423f896dee81b89c56372aa1ee310 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 25 Sep 2020 18:42:24 +0200 Subject: [PATCH 26/37] update doc comments Signed-off-by: Loris Cro --- lib/std/os.zig | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 2c5f3065b2..c06ce4ed00 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -314,8 +314,8 @@ pub const ReadError = error{ /// Returns the number of bytes that were read, which can be less than /// buf.len. If 0 bytes were read, that means EOF. -/// If the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in error.WouldBlock. +/// If `fd` is opened in non blocking mode, the function will return error.WouldBlock +/// when EAGAIN is received. /// /// Linux has a limit on how many bytes may be transferred in one `read` call, which is `0x7ffff000` /// on both 64-bit and 32-bit systems. This is due to using a signed C int as the return value, as @@ -382,8 +382,8 @@ pub fn read(fd: fd_t, buf: []u8) ReadError!usize { /// Number of bytes read is returned. Upon reading end-of-file, zero is returned. /// -/// For POSIX systems, if the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// For POSIX systems, if `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. /// On Windows, if the application has a global event loop enabled, I/O Completion Ports are /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. /// @@ -440,8 +440,8 @@ pub const PReadError = ReadError || error{Unseekable}; /// /// Retries when interrupted by a signal. /// -/// For POSIX systems, if the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// For POSIX systems, if `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. /// On Windows, if the application has a global event loop enabled, I/O Completion Ports are /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. pub fn pread(fd: fd_t, buf: []u8, offset: u64) PReadError!usize { @@ -571,8 +571,8 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void { /// /// Retries when interrupted by a signal. /// -/// For POSIX systems, if the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// For POSIX systems, if `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. /// On Windows, if the application has a global event loop enabled, I/O Completion Ports are /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. /// @@ -667,8 +667,8 @@ pub const WriteError = error{ /// another write() call to transfer the remaining bytes. The subsequent call will either /// transfer further bytes or may result in an error (e.g., if the disk is now full). /// -/// For POSIX systems, if the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// For POSIX systems, if `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. /// On Windows, if the application has a global event loop enabled, I/O Completion Ports are /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. /// @@ -747,8 +747,8 @@ pub fn write(fd: fd_t, bytes: []const u8) WriteError!usize { /// another write() call to transfer the remaining bytes. The subsequent call will either /// transfer further bytes or may result in an error (e.g., if the disk is now full). /// -/// For POSIX systems, if the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// For POSIX systems, if `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received.k`. /// On Windows, if the application has a global event loop enabled, I/O Completion Ports are /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. /// @@ -817,8 +817,8 @@ pub const PWriteError = WriteError || error{Unseekable}; /// another write() call to transfer the remaining bytes. The subsequent call will either /// transfer further bytes or may result in an error (e.g., if the disk is now full). /// -/// For POSIX systems, if the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// For POSIX systems, if `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. /// On Windows, if the application has a global event loop enabled, I/O Completion Ports are /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. /// @@ -904,8 +904,8 @@ pub fn pwrite(fd: fd_t, bytes: []const u8, offset: u64) PWriteError!usize { /// another write() call to transfer the remaining bytes. The subsequent call will either /// transfer further bytes or may result in an error (e.g., if the disk is now full). /// -/// If the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in `error.WouldBlock`. +/// If `fd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. /// /// The following systems do not have this syscall, and will return partial writes if more than one /// vector is provided: @@ -2806,8 +2806,8 @@ pub const AcceptError = error{ } || UnexpectedError; /// Accept a connection on a socket. -/// If the application has a global event loop enabled, EAGAIN is handled -/// via the event loop. Otherwise EAGAIN results in error.WouldBlock. +/// If `sockfd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. pub fn accept( /// This argument is a socket that has been created with `socket`, bound to a local address /// with `bind`, and is listening for connections after a `listen`. @@ -3036,6 +3036,8 @@ pub const ConnectError = error{ } || UnexpectedError; /// Initiate a connection on a socket. +/// If `sockfd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN or EINPROGRESS is received. pub fn connect(sockfd: socket_t, sock_addr: *const sockaddr, len: socklen_t) ConnectError!void { if (builtin.os.tag == .windows) { const rc = windows.ws2_32.connect(sockfd, sock_addr, len); @@ -5051,6 +5053,8 @@ pub const RecvFromError = error{ SystemResources, } || UnexpectedError; +/// If `sockfd` is opened in non blocking mode, the function will +/// return error.WouldBlock when EAGAIN is received. pub fn recvfrom( sockfd: fd_t, buf: []u8, From dc01ef738828a4eba08e95eaaf89442ca2f3e2f8 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 25 Sep 2020 23:21:20 +0200 Subject: [PATCH 27/37] Remove noop check Co-authored-by: Andrew Kelley --- lib/std/fs/file.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index f3e980b9f0..8d4f5df2e8 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -416,7 +416,7 @@ pub const File = struct { return windows.ReadFile(self.handle, buffer, null, self.intended_io_mode); } - if (self.intended_io_mode == .blocking or !std.io.is_async) { + if (self.intended_io_mode == .blocking) { return os.read(self.handle, buffer); } else { return std.event.Loop.instance.?.read(self.handle, buffer, self.capable_io_mode != self.intended_io_mode); From f78652484a2bf5ee967d0ed7ca1db495932fad48 Mon Sep 17 00:00:00 2001 From: Suirad Date: Tue, 22 Sep 2020 20:05:12 -0500 Subject: [PATCH 28/37] Stdlib fix for os.windows.deleteFile to fail with a proper error when attempting to delete a directory that isnt empty --- lib/std/fs/test.zig | 23 +++++++++++++++++++++++ lib/std/os/windows.zig | 16 +++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index b3cc1fe569..8d7ef5172e 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -813,3 +813,26 @@ fn run_lock_file_test(contexts: []FileLockTestContext) !void { try threads.append(try std.Thread.spawn(ctx, FileLockTestContext.run)); } } + +test "deleteDir" { + var tmp_dir = tmpDir(.{}); + defer tmp_dir.cleanup(); + + // deleting a non-existent directory + testing.expectError(error.FileNotFound, tmp_dir.dir.deleteDir("test_dir")); + + var dir = try tmp_dir.dir.makeOpenPath("test_dir", .{}); + var file = try dir.createFile("test_file", .{}); + file.close(); + dir.close(); + + // deleting a non-empty directory + testing.expectError(error.DirNotEmpty, tmp_dir.dir.deleteDir("test_dir")); + + dir = try tmp_dir.dir.openDir("test_dir", .{}); + try dir.deleteFile("test_file"); + dir.close(); + + // deleting an empty directory + try tmp_dir.dir.deleteDir("test_dir"); +} diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index de0d0ea45f..c4037ccf0a 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -764,6 +764,7 @@ pub const DeleteFileError = error{ Unexpected, NotDir, IsDir, + DirNotEmpty, }; pub const DeleteFileOptions = struct { @@ -818,7 +819,7 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil 0, ); switch (rc) { - .SUCCESS => return CloseHandle(tmp_handle), + .SUCCESS => CloseHandle(tmp_handle), .OBJECT_NAME_INVALID => unreachable, .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, .INVALID_PARAMETER => unreachable, @@ -826,6 +827,19 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil .NOT_A_DIRECTORY => return error.NotDir, else => return unexpectedStatus(rc), } + + if (options.remove_dir){ + var basic_info: FILE_BASIC_INFORMATION = undefined; + switch (ntdll.NtQueryAttributesFile(&attr, &basic_info)) { + .SUCCESS => return error.DirNotEmpty, + .OBJECT_NAME_NOT_FOUND => return, + .OBJECT_PATH_NOT_FOUND => return, + .INVALID_PARAMETER => unreachable, + .ACCESS_DENIED => return error.AccessDenied, + .OBJECT_PATH_SYNTAX_BAD => unreachable, + else => |urc| return unexpectedStatus(urc), + } + } } pub const MoveFileError = error{ FileNotFound, Unexpected }; From 43cd9eb110f6803f4e19d92347ebf263e6e644af Mon Sep 17 00:00:00 2001 From: Suirad Date: Fri, 25 Sep 2020 18:11:31 -0500 Subject: [PATCH 29/37] Add clarification comment --- lib/std/os/windows.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index c4037ccf0a..2aa222414f 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -828,7 +828,9 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil else => return unexpectedStatus(rc), } - if (options.remove_dir){ + // If a directory fails to be deleted, CloseHandle will still report success + // Check if the directory still exists and return error.DirNotEmpty if true + if (options.remove_dir) { var basic_info: FILE_BASIC_INFORMATION = undefined; switch (ntdll.NtQueryAttributesFile(&attr, &basic_info)) { .SUCCESS => return error.DirNotEmpty, From ed357f9897a6c96a1744307b1bc75a370dd85461 Mon Sep 17 00:00:00 2001 From: Woze Parrrot Date: Sat, 26 Sep 2020 20:44:49 -0400 Subject: [PATCH 30/37] uefi system_table --- lib/std/os/uefi/tables/system_table.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/os/uefi/tables/system_table.zig b/lib/std/os/uefi/tables/system_table.zig index cbe66fbb68..3f0624d2ce 100644 --- a/lib/std/os/uefi/tables/system_table.zig +++ b/lib/std/os/uefi/tables/system_table.zig @@ -35,7 +35,7 @@ pub const SystemTable = extern struct { runtime_services: *RuntimeServices, boot_services: ?*BootServices, number_of_table_entries: usize, - configuration_table: *ConfigurationTable, + configuration_table: [*]ConfigurationTable, pub const signature: u64 = 0x5453595320494249; pub const revision_1_02: u32 = (1 << 16) | 2; From eab51b7785ce0989f90a5cdde4b1110f40875ddb Mon Sep 17 00:00:00 2001 From: Ian Simonson Date: Sun, 27 Sep 2020 15:07:50 +1000 Subject: [PATCH 31/37] Make LinearFifo not crash when discarding from empty buffer Previously if a LinearFifo was empty and discard was called an unsigned overflow would occur. However it is safe to perform this overflow as a bitwise & operation with 0xFFFFFFFFFFFFFF is a noop --- lib/std/fifo.zig | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/std/fifo.zig b/lib/std/fifo.zig index c92da615f9..91d4c0330b 100644 --- a/lib/std/fifo.zig +++ b/lib/std/fifo.zig @@ -186,7 +186,9 @@ pub fn LinearFifo( } else { var head = self.head + count; if (powers_of_two) { - head &= self.buf.len - 1; + // Note it is safe to do a wrapping subtract as + // bitwise & with all 1s is a noop + head &= self.buf.len -% 1; } else { head %= self.buf.len; } @@ -376,6 +378,14 @@ pub fn LinearFifo( }; } +test "LinearFifo(u8, .Dynamic) discard(0) from empty buffer should not error on overflow" { + var fifo = LinearFifo(u8, .Dynamic).init(testing.allocator); + defer fifo.deinit(); + + // If overflow is not explicitly allowed this will crash in debug / safe mode + fifo.discard(0); +} + test "LinearFifo(u8, .Dynamic)" { var fifo = LinearFifo(u8, .Dynamic).init(testing.allocator); defer fifo.deinit(); From a31d9f92f282a878836a3ecac5a48d5f4037868c Mon Sep 17 00:00:00 2001 From: kprotty <45520026+kprotty@users.noreply.github.com> Date: Sun, 27 Sep 2020 14:05:38 -0500 Subject: [PATCH 32/37] new std.event.Lock implementation --- lib/std/event/lock.zig | 194 ++++++++++++++++------------------------- 1 file changed, 76 insertions(+), 118 deletions(-) diff --git a/lib/std/event/lock.zig b/lib/std/event/lock.zig index d27a12aef8..452420b9cd 100644 --- a/lib/std/event/lock.zig +++ b/lib/std/event/lock.zig @@ -16,107 +16,90 @@ const Loop = std.event.Loop; /// Allows only one actor to hold the lock. /// TODO: make this API also work in blocking I/O mode. pub const Lock = struct { - shared: bool, - queue: Queue, - queue_empty: bool, + mutex: std.Mutex = std.Mutex{}, + head: usize = UNLOCKED, - const Queue = std.atomic.Queue(anyframe); + const UNLOCKED = 0; + const LOCKED = 69; const global_event_loop = Loop.instance orelse @compileError("std.event.Lock currently only works with event-based I/O"); - pub const Held = struct { - lock: *Lock, - - pub fn release(self: Held) void { - // Resume the next item from the queue. - if (self.lock.queue.get()) |node| { - global_event_loop.onNextTick(node); - return; - } - - // We need to release the lock. - @atomicStore(bool, &self.lock.queue_empty, true, .SeqCst); - @atomicStore(bool, &self.lock.shared, false, .SeqCst); - - // There might be a queue item. If we know the queue is empty, we can be done, - // because the other actor will try to obtain the lock. - // But if there's a queue item, we are the actor which must loop and attempt - // to grab the lock again. - if (@atomicLoad(bool, &self.lock.queue_empty, .SeqCst)) { - return; - } - - while (true) { - if (@atomicRmw(bool, &self.lock.shared, .Xchg, true, .SeqCst)) { - // We did not obtain the lock. Great, the queue is someone else's problem. - return; - } - - // Resume the next item from the queue. - if (self.lock.queue.get()) |node| { - global_event_loop.onNextTick(node); - return; - } - - // Release the lock again. - @atomicStore(bool, &self.lock.queue_empty, true, .SeqCst); - @atomicStore(bool, &self.lock.shared, false, .SeqCst); - - // Find out if we can be done. - if (@atomicLoad(bool, &self.lock.queue_empty, .SeqCst)) { - return; - } - } - } + const Waiter = struct { + next: ?*Waiter, + tail: *Waiter, + node: Loop.NextTickNode, }; - pub fn init() Lock { - return Lock{ - .shared = false, - .queue = Queue.init(), - .queue_empty = true, + pub fn acquire(self: *Lock) Held { + const held = self.mutex.acquire(); + + if (self.head == UNLOCKED) { + self.head = LOCKED; + held.release(); + return Held{ .lock = self }; + } + + var waiter: Waiter = undefined; + waiter.next = null; + waiter.tail = &waiter; + + const head = switch (self.head) { + UNLOCKED => unreachable, + LOCKED => null, + else => @intToPtr(?*Waiter, self.head), }; - } - pub fn initLocked() Lock { - return Lock{ - .shared = true, - .queue = Queue.init(), - .queue_empty = true, - }; - } + if (head) |h| { + h.tail.next = &waiter; + h.tail = &waiter; + } else { + self.head = @ptrToInt(&waiter); + } - /// Must be called when not locked. Not thread safe. - /// All calls to acquire() and release() must complete before calling deinit(). - pub fn deinit(self: *Lock) void { - assert(!self.shared); - while (self.queue.get()) |node| resume node.data; - } - - pub fn acquire(self: *Lock) callconv(.Async) Held { - var my_tick_node = Loop.NextTickNode.init(@frame()); - - errdefer _ = self.queue.remove(&my_tick_node); // TODO test canceling an acquire suspend { - self.queue.put(&my_tick_node); - - // At this point, we are in the queue, so we might have already been resumed. - - // We set this bit so that later we can rely on the fact, that if queue_empty == true, some actor - // will attempt to grab the lock. - @atomicStore(bool, &self.queue_empty, false, .SeqCst); - - if (!@atomicRmw(bool, &self.shared, .Xchg, true, .SeqCst)) { - if (self.queue.get()) |node| { - // Whether this node is us or someone else, we tail resume it. - resume node.data; - } - } + waiter.node = Loop.NextTickNode{ + .prev = undefined, + .next = undefined, + .data = @frame(), + }; + held.release(); } return Held{ .lock = self }; } + + pub const Held = struct { + lock: *Lock, + + pub fn release(self: Held) void { + const waiter = blk: { + const held = self.lock.mutex.acquire(); + defer held.release(); + + switch (self.lock.head) { + UNLOCKED => { + std.debug.panic("Lock unlocked when already unlocked", .{}); + }, + LOCKED => { + self.lock.head = UNLOCKED; + break :blk null; + }, + else => { + const waiter = @intToPtr(*Waiter, self.lock.head); + self.lock.head = if (waiter.next == null) LOCKED else @ptrToInt(waiter.next); + if (waiter.next) |next| + next.tail = waiter.tail; + break :blk waiter; + }, + } + }; + + if (waiter) |w| { + global_event_loop.onNextTick(&w.node); + } + } + }; }; test "std.event.Lock" { @@ -128,41 +111,16 @@ test "std.event.Lock" { // TODO https://github.com/ziglang/zig/issues/3251 if (builtin.os.tag == .freebsd) return error.SkipZigTest; - // TODO this file has bit-rotted. repair it - if (true) return error.SkipZigTest; - - var lock = Lock.init(); - defer lock.deinit(); - - _ = async testLock(&lock); + var lock = Lock{}; + testLock(&lock); const expected_result = [1]i32{3 * @intCast(i32, shared_test_data.len)} ** shared_test_data.len; testing.expectEqualSlices(i32, &expected_result, &shared_test_data); } -fn testLock(lock: *Lock) callconv(.Async) void { +fn testLock(lock: *Lock) void { var handle1 = async lockRunner(lock); - var tick_node1 = Loop.NextTickNode{ - .prev = undefined, - .next = undefined, - .data = &handle1, - }; - Loop.instance.?.onNextTick(&tick_node1); - var handle2 = async lockRunner(lock); - var tick_node2 = Loop.NextTickNode{ - .prev = undefined, - .next = undefined, - .data = &handle2, - }; - Loop.instance.?.onNextTick(&tick_node2); - var handle3 = async lockRunner(lock); - var tick_node3 = Loop.NextTickNode{ - .prev = undefined, - .next = undefined, - .data = &handle3, - }; - Loop.instance.?.onNextTick(&tick_node3); await handle1; await handle2; @@ -171,13 +129,13 @@ fn testLock(lock: *Lock) callconv(.Async) void { var shared_test_data = [1]i32{0} ** 10; var shared_test_index: usize = 0; -fn lockRunner(lock: *Lock) callconv(.Async) void { - suspend; // resumed by onNextTick + +fn lockRunner(lock: *Lock) void { + Lock.global_event_loop.yield(); var i: usize = 0; while (i < shared_test_data.len) : (i += 1) { - var lock_frame = async lock.acquire(); - const handle = await lock_frame; + const handle = lock.acquire(); defer handle.release(); shared_test_index = 0; From 933146699806fc25f8182f53cd8c217654d51664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20=28xq=29=20Quei=C3=9Fner?= Date: Mon, 28 Sep 2020 11:42:39 +0200 Subject: [PATCH 33/37] Changes comptime block to test. --- lib/std/meta.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/meta.zig b/lib/std/meta.zig index c92637250f..5b4920157a 100644 --- a/lib/std/meta.zig +++ b/lib/std/meta.zig @@ -857,7 +857,7 @@ pub fn Tuple(comptime types: anytype) type { }); } -comptime { +test "Tuple" { const T = struct { fn assertTypeEqual(comptime Expected: type, comptime Actual: type) void { if (Expected != Actual) From c2d60bc5b5dd9ac39760588c777cc6809732af19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20=28xq=29=20Quei=C3=9Fner?= Date: Mon, 28 Sep 2020 12:24:22 +0200 Subject: [PATCH 34/37] Follows @tadeokondrak remark about taking `[]const type`. --- lib/std/meta.zig | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/std/meta.zig b/lib/std/meta.zig index 5b4920157a..13d3c574e9 100644 --- a/lib/std/meta.zig +++ b/lib/std/meta.zig @@ -831,10 +831,10 @@ test "sizeof" { /// with those types as fields. /// /// Examples: -/// - `Tuple(.{})` ⇒ `tuple { }` -/// - `Tuple(.{f32})` ⇒ `tuple { f32 }` -/// - `Tuple(.{f32,u32})` ⇒ `tuple { f32, u32 }` -pub fn Tuple(comptime types: anytype) type { +/// - `Tuple(&[_]type {})` ⇒ `tuple { }` +/// - `Tuple(&[_]type {f32})` ⇒ `tuple { f32 }` +/// - `Tuple(&[_]type {f32,u32})` ⇒ `tuple { f32, u32 }` +pub fn Tuple(comptime types: []const type) type { var tuple_fields: [types.len]std.builtin.TypeInfo.StructField = undefined; inline for (types) |T, i| { @setEvalBranchQuota(10_000); @@ -883,8 +883,8 @@ test "Tuple" { } }; - T.assertTuple(.{}, Tuple(.{})); - T.assertTuple(.{u32}, Tuple(.{u32})); - T.assertTuple(.{ u32, f16 }, Tuple(.{ u32, f16 })); - T.assertTuple(.{ u32, f16, []const u8 }, Tuple(.{ u32, f16, []const u8 })); + T.assertTuple(.{}, Tuple(&[_]type{})); + T.assertTuple(.{u32}, Tuple(&[_]type{u32})); + T.assertTuple(.{ u32, f16 }, Tuple(&[_]type{ u32, f16 })); + T.assertTuple(.{ u32, f16, []const u8 }, Tuple(&[_]type{ u32, f16, []const u8 })); } From 468a4bf0b443067b9d4a0bf68ea7e675a9bca727 Mon Sep 17 00:00:00 2001 From: kprotty <45520026+kprotty@users.noreply.github.com> Date: Mon, 28 Sep 2020 07:25:51 -0500 Subject: [PATCH 35/37] address some review changes --- lib/std/event/lock.zig | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/lib/std/event/lock.zig b/lib/std/event/lock.zig index 452420b9cd..a83395d7d0 100644 --- a/lib/std/event/lock.zig +++ b/lib/std/event/lock.zig @@ -20,13 +20,14 @@ pub const Lock = struct { head: usize = UNLOCKED, const UNLOCKED = 0; - const LOCKED = 69; + const LOCKED = 1; const global_event_loop = Loop.instance orelse @compileError("std.event.Lock currently only works with event-based I/O"); const Waiter = struct { - next: ?*Waiter, + // forced Waiter alignment to ensure it doesn't clash with LOCKED + next: ?*Waiter align(2), tail: *Waiter, node: Loop.NextTickNode, }; @@ -34,6 +35,14 @@ pub const Lock = struct { pub fn acquire(self: *Lock) Held { const held = self.mutex.acquire(); + // self.head transitions from multiple stages depending on the value: + // UNLOCKED -> LOCKED: + // acquire Lock ownership when theres no waiters + // LOCKED -> : + // Lock is already owned, enqueue first Waiter + // -> : + // Lock is owned with pending waiters. Push our waiter to the queue. + if (self.head == UNLOCKED) { self.head = LOCKED; held.release(); @@ -47,7 +56,7 @@ pub const Lock = struct { const head = switch (self.head) { UNLOCKED => unreachable, LOCKED => null, - else => @intToPtr(?*Waiter, self.head), + else => @intToPtr(*Waiter, self.head), }; if (head) |h| { @@ -77,9 +86,17 @@ pub const Lock = struct { const held = self.lock.mutex.acquire(); defer held.release(); + // self.head goes through the reverse transition from acquire(): + // -> : + // pop a waiter from the queue to give Lock ownership when theres still others pending + // -> LOCKED: + // pop the laster waiter from the queue, while also giving it lock ownership when awaken + // LOCKED -> UNLOCKED: + // last lock owner releases lock while no one else is waiting for it + switch (self.lock.head) { UNLOCKED => { - std.debug.panic("Lock unlocked when already unlocked", .{}); + unreachable; // Lock unlocked while unlocking }, LOCKED => { self.lock.head = UNLOCKED; From 5c6cd5e2c9e8b2d0feb0026bad7c201035a175b4 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sun, 27 Sep 2020 17:17:27 +0200 Subject: [PATCH 36/37] stage{1,2}: Fix parsing of range literals stage1 was unable to parse ranges whose starting point was written in binary/octal as the first dot in '...' was incorrectly interpreted as decimal point. stage2 forgot to reset the literal type to IntegerLiteral when it discovered the dot was not a decimal point. I've only stumbled across this bug because zig fmt keeps formatting the ranges without any space around the ... --- lib/std/zig/tokenizer.zig | 9 +++++++++ src/tokenizer.cpp | 6 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 86968c73b2..e40483c022 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -1195,6 +1195,7 @@ pub const Tokenizer = struct { }, .num_dot_hex => switch (c) { '.' => { + result.id = .IntegerLiteral; self.index -= 1; state = .start; break; @@ -1758,6 +1759,14 @@ test "correctly parse pointer assignment" { }); } +test "tokenizer - range literals" { + testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral }); + testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral }); + testTokenize("0x00...0x09", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral }); + testTokenize("0b00...0b11", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral }); + testTokenize("0o00...0o11", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral }); +} + test "tokenizer - number literals decimal" { testTokenize("0", &[_]Token.Id{.IntegerLiteral}); testTokenize("1", &[_]Token.Id{.IntegerLiteral}); diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 4415bdf431..fa14dd40fa 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -1225,9 +1225,6 @@ void tokenize(Buf *buf, Tokenization *out) { invalid_char_error(&t, c); break; } - if (t.radix != 16 && t.radix != 10) { - invalid_char_error(&t, c); - } t.state = TokenizeStateNumberDot; break; } @@ -1281,6 +1278,9 @@ void tokenize(Buf *buf, Tokenization *out) { t.state = TokenizeStateStart; continue; } + if (t.radix != 16 && t.radix != 10) { + invalid_char_error(&t, c); + } t.pos -= 1; t.state = TokenizeStateFloatFractionNoUnderscore; assert(t.cur_tok->id == TokenIdIntLiteral); From a45a4230bc2ef67dca4bd3267863695d6b04adfa Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Tue, 29 Sep 2020 11:18:35 +0200 Subject: [PATCH 37/37] Fix std.event.Future Signed-off-by: Loris Cro --- lib/std/event/future.zig | 2 +- lib/std/event/lock.zig | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/std/event/future.zig b/lib/std/event/future.zig index c9777288e4..40c7845d53 100644 --- a/lib/std/event/future.zig +++ b/lib/std/event/future.zig @@ -95,7 +95,7 @@ test "std.event.Future" { // TODO provide a way to run tests in evented I/O mode if (!std.io.is_async) return error.SkipZigTest; - const handle = async testFuture(); + testFuture(); } fn testFuture() void { diff --git a/lib/std/event/lock.zig b/lib/std/event/lock.zig index a83395d7d0..6819e413d2 100644 --- a/lib/std/event/lock.zig +++ b/lib/std/event/lock.zig @@ -27,20 +27,24 @@ pub const Lock = struct { const Waiter = struct { // forced Waiter alignment to ensure it doesn't clash with LOCKED - next: ?*Waiter align(2), + next: ?*Waiter align(2), tail: *Waiter, node: Loop.NextTickNode, }; + pub fn initLocked() Lock { + return Lock{ .head = LOCKED }; + } + pub fn acquire(self: *Lock) Held { const held = self.mutex.acquire(); // self.head transitions from multiple stages depending on the value: - // UNLOCKED -> LOCKED: + // UNLOCKED -> LOCKED: // acquire Lock ownership when theres no waiters // LOCKED -> : // Lock is already owned, enqueue first Waiter - // -> : + // -> : // Lock is owned with pending waiters. Push our waiter to the queue. if (self.head == UNLOCKED) { @@ -51,7 +55,7 @@ pub const Lock = struct { var waiter: Waiter = undefined; waiter.next = null; - waiter.tail = &waiter; + waiter.tail = &waiter; const head = switch (self.head) { UNLOCKED => unreachable, @@ -79,15 +83,15 @@ pub const Lock = struct { } pub const Held = struct { - lock: *Lock, - + lock: *Lock, + pub fn release(self: Held) void { const waiter = blk: { const held = self.lock.mutex.acquire(); defer held.release(); // self.head goes through the reverse transition from acquire(): - // -> : + // -> : // pop a waiter from the queue to give Lock ownership when theres still others pending // -> LOCKED: // pop the laster waiter from the queue, while also giving it lock ownership when awaken