From c0a0193c0a90f633d7a02bb8631f974ab5e2481e Mon Sep 17 00:00:00 2001 From: Jesse Rudolph Date: Tue, 2 Jun 2020 11:12:26 -0500 Subject: [PATCH 001/104] add replaceRange() function to ArrayList generalizes functionality of ArrayList.insertSlice() to overwrite a range of elements in the list and to grow or shrink the list as needed to accommodate size difference of the replacing slice and the range of existing elements. --- lib/std/array_list.zig | 65 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig index e096a65491..07a6ab011f 100644 --- a/lib/std/array_list.zig +++ b/lib/std/array_list.zig @@ -108,6 +108,33 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type { mem.copy(T, self.items[i .. i + items.len], items); } + /// Replace range of elements `list[start..start+len]` with `new_items` + /// grows list if `len < new_items.len`. may allocate + /// shrinks list if `len > new_items.len` + pub fn replaceRange(self: *Self, start: usize, len: usize, new_items: SliceConst) !void { + const after_range = start + len; + const range = self.items[start..after_range]; + + if (range.len == new_items.len) + mem.copy(T, range, new_items) + else if (range.len < new_items.len) { + const first = new_items[0..range.len]; + const rest = new_items[range.len..]; + + mem.copy(T, range, first); + try self.insertSlice(after_range, rest); + } else { + mem.copy(T, range, new_items); + const after_subrange = start + new_items.len; + + for (self.items[after_range..]) |item, i| { + self.items[after_subrange..][i] = item; + } + + self.items.len -= len - new_items.len; + } + } + /// Extend the list by 1 element. Allocates more memory as necessary. pub fn append(self: *Self, item: T) !void { const new_item_ptr = try self.addOne(); @@ -335,6 +362,15 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ mem.copy(T, self.items[i .. i + items.len], items); } + /// Replace range of elements `list[start..start+len]` with `new_items` + /// grows list if `len < new_items.len`. may allocate + /// shrinks list if `len > new_items.len` + pub fn replaceRange(self: *Self, start: usize, len: usize, new_items: SliceConst) !void { + var managed = self.toManaged(allocator); + try managed.replaceRange(start, len, new_items); + self.* = managed.toUnmanaged(); + } + /// Extend the list by 1 element. Allocates more memory as necessary. pub fn append(self: *Self, allocator: *Allocator, item: T) !void { const new_item_ptr = try self.addOne(allocator); @@ -657,6 +693,35 @@ test "std.ArrayList.insertSlice" { testing.expect(list.items[0] == 1); } +test "std.ArrayList.replaceRange" { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const alloc = &arena.allocator; + const init = [_]i32{ 1, 2, 3, 4, 5 }; + const new = [_]i32{ 0, 0, 0 }; + + var list_zero = ArrayList(i32).init(alloc); + var list_eq = ArrayList(i32).init(alloc); + var list_lt = ArrayList(i32).init(alloc); + var list_gt = ArrayList(i32).init(alloc); + + try list_zero.appendSlice(&init); + try list_eq.appendSlice(&init); + try list_lt.appendSlice(&init); + try list_gt.appendSlice(&init); + + try list_zero.replaceRange(1, 0, &new); + try list_eq.replaceRange(1, 3, &new); + try list_lt.replaceRange(1, 2, &new); + try list_gt.replaceRange(1, 4, &new); + + testing.expectEqualSlices(i32, list_zero.items, &[_]i32{ 1, 0, 0, 0, 2, 3, 4, 5 }); + testing.expectEqualSlices(i32, list_eq.items, &[_]i32{ 1, 0, 0, 0, 5 }); + testing.expectEqualSlices(i32, list_lt.items, &[_]i32{ 1, 0, 0, 0, 4, 5 }); + testing.expectEqualSlices(i32, list_gt.items, &[_]i32{ 1, 0, 0, 0 }); +} + const Item = struct { integer: i32, sub_items: ArrayList(Item), From 3fba076f920c35ad26b462cf11f9b16b550e97ab Mon Sep 17 00:00:00 2001 From: Jesse Rudolph Date: Wed, 3 Jun 2020 06:29:50 -0500 Subject: [PATCH 002/104] demonstrate start + len > new_items.len --- lib/std/array_list.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig index 07a6ab011f..dda124031f 100644 --- a/lib/std/array_list.zig +++ b/lib/std/array_list.zig @@ -714,6 +714,9 @@ test "std.ArrayList.replaceRange" { try list_zero.replaceRange(1, 0, &new); try list_eq.replaceRange(1, 3, &new); try list_lt.replaceRange(1, 2, &new); + + // after_range > new_items.len in function body + testing.expect(1 + 4 > new.len); try list_gt.replaceRange(1, 4, &new); testing.expectEqualSlices(i32, list_zero.items, &[_]i32{ 1, 0, 0, 0, 2, 3, 4, 5 }); From 2fc2355fc3d72cbf1ffdc982a42090ada901d21f Mon Sep 17 00:00:00 2001 From: antlilja Date: Tue, 23 Jun 2020 18:05:32 +0200 Subject: [PATCH 003/104] Add expectWithinMargin and test --- lib/std/testing.zig | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 2d136d56c9..9dca969f45 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -168,6 +168,32 @@ test "expectEqual.union(enum)" { expectEqual(a10, a10); } +/// This function is intended to be used only in tests. When the actual value is not +/// within the margin of the expected value, +/// prints diagnostics to stderr to show exactly how they are not equal, then aborts. +/// The types must be floating point +pub fn expectWithinMargin(expected: var, actual: @TypeOf(expected), margin: @TypeOf(expected)) void { + std.debug.assert(margin >= 0.0); + + switch (@typeInfo(@TypeOf(actual))) { + .Float, + .ComptimeFloat, + => { + if (@fabs(expected - actual) > margin) { + std.debug.panic("actual {}, not within margin {} of expected {}", .{ actual, margin, expected }); + } + }, + else => @compileError("Unable to compare non floating point values"), + } +} + +test "expectWithinMargin.f32" { + const x: f32 = 12.0; + const y: f32 = 12.06; + + expectWithinMargin(x, y, 0.1); +} + /// This function is intended to be used only in tests. When the two slices are not /// equal, prints diagnostics to stderr to show exactly how they are not equal, /// then aborts. From 46106b018cc5c870648cf26d4e7413e8a60ad1e4 Mon Sep 17 00:00:00 2001 From: antlilja Date: Tue, 23 Jun 2020 18:08:15 +0200 Subject: [PATCH 004/104] Add expectWithinEpsilon + test --- lib/std/testing.zig | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 9dca969f45..9f7601b19e 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -194,6 +194,33 @@ test "expectWithinMargin.f32" { expectWithinMargin(x, y, 0.1); } +/// This function is intended to be used only in tests. When the actual value is not +/// within the epsilon of the expected value, +/// prints diagnostics to stderr to show exactly how they are not equal, then aborts. +/// The types must be floating point +pub fn expectWithinEpsilon(expected: var, actual: @TypeOf(expected), epsilon: @TypeOf(expected)) void { + std.debug.assert(epsilon >= 0.0 and epsilon <= 1.0); + + const margin = epsilon * expected; + switch (@typeInfo(@TypeOf(actual))) { + .Float, + .ComptimeFloat, + => { + if (@fabs(expected - actual) > margin) { + std.debug.panic("actual {}, not within epsilon {}, of expected {}", .{ actual, epsilon, expected }); + } + }, + else => @compileError("Unable to compare non floating point values"), + } +} + +test "expectWithinEpsilon.f32" { + const x: f32 = 12.0; + const y: f32 = 13.2; + + expectWithinEpsilon(x, y, 0.1); +} + /// This function is intended to be used only in tests. When the two slices are not /// equal, prints diagnostics to stderr to show exactly how they are not equal, /// then aborts. From fd50696359ec86abd238993930e979433a31db9a Mon Sep 17 00:00:00 2001 From: antlilja Date: Tue, 23 Jun 2020 15:18:28 +0200 Subject: [PATCH 005/104] Store else node in IrInstSrcCheckSwitchProngs * Remove have_else_prong (bool) * Add else_prong (AstNode*) --- src/all_types.hpp | 2 +- src/ir.cpp | 16 ++++++++-------- src/ir_print.cpp | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index 88c7e96943..ef60a05fab 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -4095,7 +4095,7 @@ struct IrInstSrcCheckSwitchProngs { IrInstSrc *target_value; IrInstSrcCheckSwitchProngsRange *ranges; size_t range_count; - bool have_else_prong; + AstNode* else_prong; bool have_underscore_prong; }; diff --git a/src/ir.cpp b/src/ir.cpp index 635af397c4..44df69aeed 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -4274,14 +4274,14 @@ static IrInstGen *ir_build_err_to_int_gen(IrAnalyze *ira, Scope *scope, AstNode static IrInstSrc *ir_build_check_switch_prongs(IrBuilderSrc *irb, Scope *scope, AstNode *source_node, IrInstSrc *target_value, IrInstSrcCheckSwitchProngsRange *ranges, size_t range_count, - bool have_else_prong, bool have_underscore_prong) + AstNode* else_prong, bool have_underscore_prong) { IrInstSrcCheckSwitchProngs *instruction = ir_build_instruction( irb, scope, source_node); instruction->target_value = target_value; instruction->ranges = ranges; instruction->range_count = range_count; - instruction->have_else_prong = have_else_prong; + instruction->else_prong = else_prong; instruction->have_underscore_prong = have_underscore_prong; ir_ref_instruction(target_value, irb->current_basic_block); @@ -9305,7 +9305,7 @@ static IrInstSrc *ir_gen_switch_expr(IrBuilderSrc *irb, Scope *scope, AstNode *n } IrInstSrc *switch_prongs_void = ir_build_check_switch_prongs(irb, scope, node, target_value, - check_ranges.items, check_ranges.length, else_prong != nullptr, underscore_prong != nullptr); + check_ranges.items, check_ranges.length, else_prong, underscore_prong != nullptr); IrInstSrc *br_instruction; if (cases.length == 0) { @@ -28685,7 +28685,7 @@ static IrInstGen *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, buf_ptr(enum_field->name))); } } - } else if (!instruction->have_else_prong) { + } else if (instruction->else_prong == nullptr) { if (switch_type->data.enumeration.non_exhaustive) { ir_add_error(ira, &instruction->base.base, buf_sprintf("switch on non-exhaustive enum must include `else` or `_` prong")); @@ -28746,7 +28746,7 @@ static IrInstGen *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, } field_prev_uses[start_index] = start_value->base.source_node; } - if (!instruction->have_else_prong) { + if (instruction->else_prong == nullptr) { if (type_is_global_error_set(switch_type)) { ir_add_error(ira, &instruction->base.base, buf_sprintf("else prong required when switching on type 'anyerror'")); @@ -28808,7 +28808,7 @@ static IrInstGen *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, return ira->codegen->invalid_inst_gen; } } - if (!instruction->have_else_prong) { + if (instruction->else_prong == nullptr) { BigInt min_val; eval_min_max_value_int(ira->codegen, switch_type, &min_val, false); BigInt max_val; @@ -28847,11 +28847,11 @@ static IrInstGen *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, return ira->codegen->invalid_inst_gen; } } - if (((seenTrue < 1) || (seenFalse < 1)) && !instruction->have_else_prong) { + if (((seenTrue < 1) || (seenFalse < 1)) && instruction->else_prong == nullptr) { ir_add_error(ira, &instruction->base.base, buf_sprintf("switch must handle all possibilities")); return ira->codegen->invalid_inst_gen; } - } else if (!instruction->have_else_prong) { + } else if (instruction->else_prong == nullptr) { ir_add_error(ira, &instruction->base.base, buf_sprintf("else prong required when switching on type '%s'", buf_ptr(&switch_type->name))); return ira->codegen->invalid_inst_gen; diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 27bedff47f..e9f029c930 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -2177,7 +2177,7 @@ static void ir_print_check_switch_prongs(IrPrintSrc *irp, IrInstSrcCheckSwitchPr fprintf(irp->f, "..."); ir_print_other_inst_src(irp, instruction->ranges[i].end); } - const char *have_else_str = instruction->have_else_prong ? "yes" : "no"; + const char *have_else_str = instruction->else_prong != nullptr ? "yes" : "no"; fprintf(irp->f, ")else:%s", have_else_str); } From e60be3082499ff19078699675044b993f6921ad0 Mon Sep 17 00:00:00 2001 From: antlilja Date: Wed, 24 Jun 2020 19:02:31 +0200 Subject: [PATCH 006/104] Remove unreachable else prongs --- lib/std/zig/string_literal.zig | 1 - src-self-hosted/translate_c.zig | 1 - test/stage1/behavior/bugs/1111.zig | 1 - 3 files changed, 3 deletions(-) diff --git a/lib/std/zig/string_literal.zig b/lib/std/zig/string_literal.zig index cc6030ad15..9def0bdefc 100644 --- a/lib/std/zig/string_literal.zig +++ b/lib/std/zig/string_literal.zig @@ -104,7 +104,6 @@ pub fn parse( return error.InvalidCharacter; }, }, - else => unreachable, } } unreachable; diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 3b1a91b28c..dad3b14a29 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -2791,7 +2791,6 @@ fn transCharLiteral( "TODO: support character literal kind {}", .{kind}, ), - else => unreachable, }; if (suppress_as == .no_as) { return maybeSuppressResult(rp, scope, result_used, int_lit_node); diff --git a/test/stage1/behavior/bugs/1111.zig b/test/stage1/behavior/bugs/1111.zig index f62107f9a3..607bc33666 100644 --- a/test/stage1/behavior/bugs/1111.zig +++ b/test/stage1/behavior/bugs/1111.zig @@ -7,6 +7,5 @@ test "issue 1111 fixed" { switch (v) { Foo.Bar => return, - else => return, } } From dcc406deff0fb4ee3c2cd1f4ff8614e972a8ea7a Mon Sep 17 00:00:00 2001 From: antlilja Date: Wed, 24 Jun 2020 19:12:42 +0200 Subject: [PATCH 007/104] Add new error message for unreachable else prongs * Adds error message for types: enum, int and bool * Adds compile error tests --- src/ir.cpp | 20 +++++++-- test/compile_errors.zig | 96 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 3 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index 44df69aeed..8395cf0a3e 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -28700,6 +28700,10 @@ static IrInstGen *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, buf_ptr(enum_field->name))); } } + } else if(!switch_type->data.enumeration.non_exhaustive && switch_type->data.enumeration.src_field_count == instruction->range_count) { + ir_add_error_node(ira, instruction->else_prong, + buf_sprintf("unreachable else prong, all cases already handled")); + return ira->codegen->invalid_inst_gen; } } else if (switch_type->id == ZigTypeIdErrorSet) { if (!resolve_inferred_error_set(ira->codegen, switch_type, target_value->base.source_node)) { @@ -28808,16 +28812,20 @@ static IrInstGen *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, return ira->codegen->invalid_inst_gen; } } - if (instruction->else_prong == nullptr) { + BigInt min_val; eval_min_max_value_int(ira->codegen, switch_type, &min_val, false); BigInt max_val; eval_min_max_value_int(ira->codegen, switch_type, &max_val, true); - if (!rangeset_spans(&rs, &min_val, &max_val)) { + bool handles_all_cases = rangeset_spans(&rs, &min_val, &max_val); + if (!handles_all_cases && instruction->else_prong == nullptr) { ir_add_error(ira, &instruction->base.base, buf_sprintf("switch must handle all possibilities")); return ira->codegen->invalid_inst_gen; + } else if(handles_all_cases && instruction->else_prong != nullptr) { + ir_add_error_node(ira, instruction->else_prong, + buf_sprintf("unreachable else prong, all cases already handled")); + return ira->codegen->invalid_inst_gen; } - } } else if (switch_type->id == ZigTypeIdBool) { int seenTrue = 0; int seenFalse = 0; @@ -28851,6 +28859,12 @@ static IrInstGen *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, ir_add_error(ira, &instruction->base.base, buf_sprintf("switch must handle all possibilities")); return ira->codegen->invalid_inst_gen; } + + if(seenTrue == 1 && seenFalse == 1 && instruction->else_prong != nullptr) { + ir_add_error_node(ira, instruction->else_prong, + buf_sprintf("unreachable else prong, all cases already handled")); + return ira->codegen->invalid_inst_gen; + } } else if (instruction->else_prong == nullptr) { ir_add_error(ira, &instruction->base.base, buf_sprintf("else prong required when switching on type '%s'", buf_ptr(&switch_type->name))); diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 3f898cc337..29ff994cd3 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -468,6 +468,102 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { "tmp.zig:12:5: error: switch on non-exhaustive enum must include `else` or `_` prong", }); + cases.add("switch expression - unreachable else prong (bool)", + \\fn foo(x: bool) void { + \\ switch (x) { + \\ true => {}, + \\ false => {}, + \\ else => {}, + \\ } + \\} + \\export fn entry() usize { return @sizeOf(@TypeOf(foo)); } + , &[_][]const u8{ + "tmp.zig:5:9: error: unreachable else prong, all cases already handled", + }); + + cases.add("switch expression - unreachable else prong (u1)", + \\fn foo(x: u1) void { + \\ switch (x) { + \\ 0 => {}, + \\ 1 => {}, + \\ else => {}, + \\ } + \\} + \\export fn entry() usize { return @sizeOf(@TypeOf(foo)); } + , &[_][]const u8{ + "tmp.zig:5:9: error: unreachable else prong, all cases already handled", + }); + + cases.add("switch expression - unreachable else prong (u2)", + \\fn foo(x: u2) void { + \\ switch (x) { + \\ 0 => {}, + \\ 1 => {}, + \\ 2 => {}, + \\ 3 => {}, + \\ else => {}, + \\ } + \\} + \\export fn entry() usize { return @sizeOf(@TypeOf(foo)); } + , &[_][]const u8{ + "tmp.zig:7:9: error: unreachable else prong, all cases already handled", + }); + + cases.add("switch expression - unreachable else prong (range u8)", + \\fn foo(x: u8) void { + \\ switch (x) { + \\ 0 => {}, + \\ 1 => {}, + \\ 2 => {}, + \\ 3 => {}, + \\ 4...255 => {}, + \\ else => {}, + \\ } + \\} + \\export fn entry() usize { return @sizeOf(@TypeOf(foo)); } + , &[_][]const u8{ + "tmp.zig:8:9: error: unreachable else prong, all cases already handled", + }); + + cases.add("switch expression - unreachable else prong (range i8)", + \\fn foo(x: i8) void { + \\ switch (x) { + \\ -128...0 => {}, + \\ 1 => {}, + \\ 2 => {}, + \\ 3 => {}, + \\ 4...127 => {}, + \\ else => {}, + \\ } + \\} + \\export fn entry() usize { return @sizeOf(@TypeOf(foo)); } + , &[_][]const u8{ + "tmp.zig:8:9: error: unreachable else prong, all cases already handled", + }); + + cases.add("switch expression - unreachable else prong (enum)", + \\const TestEnum = enum{ T1, T2 }; + \\ + \\fn err(x: u8) TestEnum { + \\ switch (x) { + \\ 0 => return TestEnum.T1, + \\ else => return TestEnum.T2, + \\ } + \\} + \\ + \\fn foo(x: u8) void { + \\ switch (err(x)) { + \\ TestEnum.T1 => {}, + \\ TestEnum.T2 => {}, + \\ else => {}, + \\ } + \\} + \\ + \\export fn entry() usize { return @sizeOf(@TypeOf(foo)); } + , &[_][]const u8{ + "tmp.zig:14:9: error: unreachable else prong, all cases already handled", + }); + cases.addTest("@export with empty name string", \\pub export fn entry() void { } \\comptime { From 9b75091fd4e2e6f406515fce645e1b50e978fded Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 25 Jul 2020 22:21:52 -0700 Subject: [PATCH 008/104] ci: update msys2 installer --- ci/azure/pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/pipelines.yml b/ci/azure/pipelines.yml index 388d6db044..7cd1d596b4 100644 --- a/ci/azure/pipelines.yml +++ b/ci/azure/pipelines.yml @@ -41,7 +41,7 @@ jobs: steps: - powershell: | - (New-Object Net.WebClient).DownloadFile("https://github.com/msys2/msys2-installer/releases/download/2020-06-02/msys2-base-x86_64-20200602.sfx.exe", "sfx.exe") + (New-Object Net.WebClient).DownloadFile("https://github.com/msys2/msys2-installer/releases/download/2020-07-20/msys2-base-x86_64-20200720.sfx.exe", "sfx.exe") .\sfx.exe -y -o\ del sfx.exe displayName: Download/Extract/Install MSYS2 From f050150ffaef7bc5e72e23c48d46617f640117e2 Mon Sep 17 00:00:00 2001 From: meme Date: Sat, 25 Jul 2020 19:38:26 -0400 Subject: [PATCH 009/104] Add memory replacement routines * Added `replace` for simple pre-allocated search-and-replace * Added `replacementSize` for calculating ahead-of-time buffer sizes for performing a safe search-and-replace * Added `replaceOwned` for automatically allocating and performing replacement --- lib/std/mem.zig | 73 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/lib/std/mem.zig b/lib/std/mem.zig index ac7bde47c1..0d6d7c118a 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -2030,6 +2030,79 @@ test "rotate" { testing.expect(eql(i32, &arr, &[_]i32{ 1, 2, 4, 5, 3 })); } +/// Replace needle with replacement as many times as possible, writing to an output buffer which is assumed to be of +/// appropriate size. Use replacementSize to calculate an appropriate buffer size. +pub fn replace(comptime T: type, input: []const T, needle: []const T, replacement: []const T, output: []T) usize { + var i: usize = 0; + var slide: usize = 0; + var replacements: usize = 0; + while (slide < input.len) { + if (mem.indexOf(T, input[slide..], needle) == @as(usize, 0)) { + mem.copy(T, output[i..i + replacement.len], replacement); + i += replacement.len; + slide += needle.len; + replacements += 1; + } else { + output[i] = input[slide]; + i += 1; + slide += 1; + } + } + + return replacements; +} + +test "replace" { + var output: [29]u8 = undefined; + var replacements = replace(u8, "All your base are belong to us", "base", "Zig", output[0..]); + testing.expect(replacements == 1); + testing.expect(eql(u8, output[0..], "All your Zig are belong to us")); + + replacements = replace(u8, "Favor reading code over writing code.", "code", "", output[0..]); + testing.expect(replacements == 2); + testing.expect(eql(u8, output[0..], "Favor reading over writing .")); +} + +/// Calculate the size needed in an output buffer to perform a replacement. +pub fn replacementSize(comptime T: type, input: []const T, needle: []const T, replacement: []const T) usize { + var i: usize = 0; + var size: usize = input.len; + while (i < input.len) : (i += 1) { + if (mem.indexOf(T, input[i..], needle) == @as(usize, 0)) { + size = size - needle.len + replacement.len; + i += needle.len; + } + } + + return size; +} + +test "replacementSize" { + testing.expect(replacementSize(u8, "All your base are belong to us", "base", "Zig") == 29); + testing.expect(replacementSize(u8, "", "", "") == 0); + testing.expect(replacementSize(u8, "Favor reading code over writing code.", "code", "") == 29); + testing.expect(replacementSize(u8, "Only one obvious way to do things.", "things.", "things in Zig.") == 41); +} + +/// Perform a replacement on an allocated buffer of pre-determined size. Caller must free returned memory. +pub fn replaceOwned(comptime T: type, allocator: *Allocator, input: []const T, needle: []const T, replacement: []const T) Allocator.Error![]T { + var output = try allocator.alloc(T, replacementSize(T, input, needle, replacement)); + _ = replace(T, input, needle, replacement, output); + return output; +} + +test "replaceOwned" { + const allocator = std.heap.page_allocator; + + const base_replace = replaceOwned(u8, allocator, "All your base are belong to us", "base", "Zig") catch unreachable; + defer allocator.free(base_replace); + testing.expect(eql(u8, base_replace, "All your Zig are belong to us")); + + const zen_replace = replaceOwned(u8, allocator, "Favor reading code over writing code.", " code", "") catch unreachable; + defer allocator.free(zen_replace); + testing.expect(eql(u8, zen_replace, "Favor reading over writing.")); +} + /// Converts a little-endian integer to host endianness. pub fn littleToNative(comptime T: type, x: T) T { return switch (builtin.endian) { From 09b0ad494bd9e3531ababb25e2531c18861b2148 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 25 Jul 2020 23:32:30 -0700 Subject: [PATCH 010/104] stage2: remove superfluous else => unreachable --- src-self-hosted/link.zig | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 1e5c8d5cd7..cde91cdc01 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -125,14 +125,13 @@ pub const File = struct { switch (base.tag) { .Elf => return @fieldParentPtr(Elf, "base", base).makeWritable(dir, sub_path), .C => {}, - else => unreachable, } } pub fn makeExecutable(base: *File) !void { switch (base.tag) { .Elf => return @fieldParentPtr(Elf, "base", base).makeExecutable(), - else => unreachable, + .C => unreachable, } } @@ -140,7 +139,6 @@ pub const File = struct { switch (base.tag) { .Elf => return @fieldParentPtr(Elf, "base", base).updateDecl(module, decl), .C => return @fieldParentPtr(C, "base", base).updateDecl(module, decl), - else => unreachable, } } @@ -148,7 +146,6 @@ pub const File = struct { switch (base.tag) { .Elf => return @fieldParentPtr(Elf, "base", base).allocateDeclIndexes(decl), .C => {}, - else => unreachable, } } @@ -156,7 +153,6 @@ pub const File = struct { switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).deinit(), .C => @fieldParentPtr(C, "base", base).deinit(), - else => unreachable, } } @@ -172,7 +168,6 @@ pub const File = struct { parent.deinit(); parent.allocator.destroy(parent); }, - else => unreachable, } } @@ -180,14 +175,13 @@ pub const File = struct { try switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).flush(), .C => @fieldParentPtr(C, "base", base).flush(), - else => unreachable, }; } pub fn freeDecl(base: *File, decl: *Module.Decl) void { switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).freeDecl(decl), - else => unreachable, + .C => unreachable, } } @@ -195,7 +189,6 @@ pub const File = struct { return switch (base.tag) { .Elf => @fieldParentPtr(Elf, "base", base).error_flags, .C => return .{ .no_entry_point_found = false }, - else => unreachable, }; } From c95091e5a50f0bfa120a83292ee546b8c3e11910 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 25 Jul 2020 23:33:15 -0700 Subject: [PATCH 011/104] run zig fmt on std.testing --- lib/std/testing.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 423f537f74..8f16b50cd2 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -175,7 +175,7 @@ test "expectEqual.union(enum)" { /// within the margin of the expected value, /// prints diagnostics to stderr to show exactly how they are not equal, then aborts. /// The types must be floating point -pub fn expectWithinMargin(expected: var, actual: @TypeOf(expected), margin: @TypeOf(expected)) void { +pub fn expectWithinMargin(expected: anytype, actual: @TypeOf(expected), margin: @TypeOf(expected)) void { std.debug.assert(margin >= 0.0); switch (@typeInfo(@TypeOf(actual))) { @@ -201,7 +201,7 @@ test "expectWithinMargin.f32" { /// within the epsilon of the expected value, /// prints diagnostics to stderr to show exactly how they are not equal, then aborts. /// The types must be floating point -pub fn expectWithinEpsilon(expected: var, actual: @TypeOf(expected), epsilon: @TypeOf(expected)) void { +pub fn expectWithinEpsilon(expected: anytype, actual: @TypeOf(expected), epsilon: @TypeOf(expected)) void { std.debug.assert(epsilon >= 0.0 and epsilon <= 1.0); const margin = epsilon * expected; From 7ae1b3a6b348a24f07bacb6e19f1aa6d7b3ba32d Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Sun, 26 Jul 2020 22:01:33 +0200 Subject: [PATCH 012/104] add trait hasUniqueRepresentation --- lib/std/meta/trait.zig | 68 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/lib/std/meta/trait.zig b/lib/std/meta/trait.zig index 5cea0ecb9a..7414edac4f 100644 --- a/lib/std/meta/trait.zig +++ b/lib/std/meta/trait.zig @@ -416,3 +416,71 @@ test "std.meta.trait.hasFunctions" { testing.expect(!hasFunctions(TestStruct2, .{ "a", "b", "c" })); testing.expect(!hasFunctions(TestStruct2, tuple)); } + +/// True if every value of the type `T` has a unique bit pattern representing it. +/// In other words, `T` has no unused bits and no padding. +pub fn hasUniqueRepresentation(comptime T: type) bool { + switch (@typeInfo(T)) { + else => return false, // TODO can we know if it's true for some of these types ? + + .AnyFrame, + .Bool, + .BoundFn, + .Enum, + .ErrorSet, + .Fn, + .Int, // TODO check that it is still true + .Pointer, + => return true, + + .Array => |info| return comptime hasUniqueRepresentation(info.child), + + .Struct => |info| { + var sum_size = @as(usize, 0); + + inline for (info.fields) |field| { + const FieldType = field.field_type; + if (comptime !hasUniqueRepresentation(FieldType)) return false; + sum_size += @sizeOf(FieldType); + } + + return @sizeOf(T) == sum_size; + }, + + .Vector => |info| return comptime hasUniqueRepresentation(info.child), + } +} + +test "std.meta.trait.hasUniqueRepresentation" { + const TestStruct1 = struct { + a: u32, + b: u32, + }; + + testing.expect(hasUniqueRepresentation(TestStruct1)); + + const TestStruct2 = struct { + a: u32, + b: u16, + }; + + testing.expect(!hasUniqueRepresentation(TestStruct2)); + + const TestStruct3 = struct { + a: u32, + b: u32, + }; + + testing.expect(hasUniqueRepresentation(TestStruct3)); + + testing.expect(hasUniqueRepresentation(i1)); + testing.expect(hasUniqueRepresentation(u2)); + testing.expect(hasUniqueRepresentation(i3)); + testing.expect(hasUniqueRepresentation(u4)); + testing.expect(hasUniqueRepresentation(i5)); + testing.expect(hasUniqueRepresentation(u6)); + testing.expect(hasUniqueRepresentation(i7)); + testing.expect(hasUniqueRepresentation(u8)); + testing.expect(hasUniqueRepresentation(i9)); + testing.expect(hasUniqueRepresentation(u10)); +} From 345cb3200c353d6fb7aeb0e058986d8ca59ced1e Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Sun, 26 Jul 2020 22:04:10 +0200 Subject: [PATCH 013/104] improve autoHash type switch floats shouldn't be autoHash'd as they have multiple representations for some values, preventing it by default is safer --- lib/std/hash/auto_hash.zig | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/std/hash/auto_hash.zig b/lib/std/hash/auto_hash.zig index a3e1a390c2..aaa68ca8fd 100644 --- a/lib/std/hash/auto_hash.zig +++ b/lib/std/hash/auto_hash.zig @@ -81,24 +81,22 @@ pub fn hash(hasher: anytype, key: anytype, comptime strat: HashStrategy) void { .Undefined, .Void, .Null, - .BoundFn, .ComptimeFloat, .ComptimeInt, .Type, .EnumLiteral, .Frame, + .Float, => @compileError("cannot hash this type"), // Help the optimizer see that hashing an int is easy by inlining! // TODO Check if the situation is better after #561 is resolved. .Int => @call(.{ .modifier = .always_inline }, hasher.update, .{std.mem.asBytes(&key)}), - .Float => |info| hash(hasher, @bitCast(std.meta.Int(false, info.bits), key), strat), - .Bool => hash(hasher, @boolToInt(key), strat), .Enum => hash(hasher, @enumToInt(key), strat), .ErrorSet => hash(hasher, @errorToInt(key), strat), - .AnyFrame, .Fn => hash(hasher, @ptrToInt(key), strat), + .AnyFrame, .BoundFn, .Fn => hash(hasher, @ptrToInt(key), strat), .Pointer => @call(.{ .modifier = .always_inline }, hashPointer, .{ hasher, key, strat }), @@ -266,12 +264,12 @@ test "hash slice deep" { test "hash struct deep" { const Foo = struct { a: u32, - b: f64, + b: u16, c: *bool, const Self = @This(); - pub fn init(allocator: *mem.Allocator, a_: u32, b_: f64, c_: bool) !Self { + pub fn init(allocator: *mem.Allocator, a_: u32, b_: u16, c_: bool) !Self { const ptr = try allocator.create(bool); ptr.* = c_; return Self{ .a = a_, .b = b_, .c = ptr }; @@ -279,9 +277,9 @@ test "hash struct deep" { }; const allocator = std.testing.allocator; - const foo = try Foo.init(allocator, 123, 1.0, true); - const bar = try Foo.init(allocator, 123, 1.0, true); - const baz = try Foo.init(allocator, 123, 1.0, false); + const foo = try Foo.init(allocator, 123, 10, true); + const bar = try Foo.init(allocator, 123, 10, true); + const baz = try Foo.init(allocator, 123, 10, false); defer allocator.destroy(foo.c); defer allocator.destroy(bar.c); defer allocator.destroy(baz.c); @@ -338,12 +336,12 @@ test "testHash struct" { test "testHash union" { const Foo = union(enum) { A: u32, - B: f32, + B: bool, C: u32, }; const a = Foo{ .A = 18 }; - var b = Foo{ .B = 12.34 }; + var b = Foo{ .B = true }; const c = Foo{ .C = 18 }; testing.expect(testHash(a) == testHash(a)); testing.expect(testHash(a) != testHash(b)); From f67ce1e35fe3ecf19b50f64b9fe2d85747f7934d Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Sun, 26 Jul 2020 22:08:48 +0200 Subject: [PATCH 014/104] make use of hasUniqueRepresentation to speed up hashing facilities, fastpath in getAutoHashFn is particularly important for hashmap performance gives a 1.18x speedup on gotta-go-fast hashmap bench --- lib/std/hash/auto_hash.zig | 12 ++++++------ lib/std/hash_map.zig | 11 ++++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/std/hash/auto_hash.zig b/lib/std/hash/auto_hash.zig index aaa68ca8fd..85f8e4b0d2 100644 --- a/lib/std/hash/auto_hash.zig +++ b/lib/std/hash/auto_hash.zig @@ -56,9 +56,6 @@ pub fn hashPointer(hasher: anytype, key: anytype, comptime strat: HashStrategy) pub fn hashArray(hasher: anytype, key: anytype, comptime strat: HashStrategy) void { switch (strat) { .Shallow => { - // TODO detect via a trait when Key has no padding bits to - // hash it as an array of bytes. - // Otherwise, hash every element. for (key) |element| { hash(hasher, element, .Shallow); } @@ -75,6 +72,12 @@ pub fn hashArray(hasher: anytype, key: anytype, comptime strat: HashStrategy) vo /// Strategy is provided to determine if pointers should be followed or not. pub fn hash(hasher: anytype, key: anytype, comptime strat: HashStrategy) void { const Key = @TypeOf(key); + + if (strat == .Shallow and comptime meta.trait.hasUniqueRepresentation(Key)) { + @call(.{ .modifier = .always_inline }, hasher.update, .{mem.asBytes(&key)}); + return; + } + switch (@typeInfo(Key)) { .NoReturn, .Opaque, @@ -119,9 +122,6 @@ pub fn hash(hasher: anytype, key: anytype, comptime strat: HashStrategy) void { }, .Struct => |info| { - // TODO detect via a trait when Key has no padding bits to - // hash it as an array of bytes. - // Otherwise, hash every field. inline for (info.fields) |field| { // We reuse the hash of the previous field as the seed for the // next one so that they're dependant. diff --git a/lib/std/hash_map.zig b/lib/std/hash_map.zig index 3952ecb4b2..c81def3a00 100644 --- a/lib/std/hash_map.zig +++ b/lib/std/hash_map.zig @@ -5,6 +5,7 @@ const testing = std.testing; const math = std.math; const mem = std.mem; const meta = std.meta; +const trait = meta.trait; const autoHash = std.hash.autoHash; const Wyhash = std.hash.Wyhash; const Allocator = mem.Allocator; @@ -1023,9 +1024,13 @@ pub fn getTrivialEqlFn(comptime K: type) (fn (K, K) bool) { pub fn getAutoHashFn(comptime K: type) (fn (K) u32) { return struct { fn hash(key: K) u32 { - var hasher = Wyhash.init(0); - autoHash(&hasher, key); - return @truncate(u32, hasher.final()); + if (comptime trait.hasUniqueRepresentation(K)) { + return @truncate(u32, Wyhash.hash(0, std.mem.asBytes(&key))); + } else { + var hasher = Wyhash.init(0); + autoHash(&hasher, key); + return @truncate(u32, hasher.final()); + } } }.hash; } From 6cc72af03df86796251f2bca49fa45a006e67be5 Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Sat, 25 Jul 2020 23:29:02 -0600 Subject: [PATCH 015/104] Provide Ip4Address and Ip6Address in addition to Address --- lib/std/net.zig | 481 ++++++++++++++++++++++++++++-------------------- 1 file changed, 282 insertions(+), 199 deletions(-) diff --git a/lib/std/net.zig b/lib/std/net.zig index 71bab383fa..514e48e0a7 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -14,8 +14,8 @@ const has_unix_sockets = @hasDecl(os, "sockaddr_un"); pub const Address = extern union { any: os.sockaddr, - in: os.sockaddr_in, - in6: os.sockaddr_in6, + in: Ip4Address, + in6: Ip6Address, un: if (has_unix_sockets) os.sockaddr_un else void, // TODO this crashed the compiler. https://github.com/ziglang/zig/issues/3512 @@ -76,19 +76,227 @@ pub const Address = extern union { } } + pub fn parseIp6(buf: []const u8, port: u16) !Address { + return Address{.in6 = try Ip6Address.parse(buf, port) }; + } + + pub fn resolveIp6(buf: []const u8, port: u16) !Address { + return Address{.in6 = try Ip6Address.resolve(buf, port) }; + } + + pub fn parseIp4(buf: []const u8, port: u16) !Address { + return Address {.in = try Ip4Address.parse(buf, port) }; + } + + pub fn initIp4(addr: [4]u8, port: u16) Address { + return Address{.in = Ip4Address.init(addr, port) }; + } + + pub fn initIp6(addr: [16]u8, port: u16, flowinfo: u32, scope_id: u32) Address { + return Address{.in6 = Ip6Address.init(addr, port, flowinfo, scope_id) }; + } + + pub fn initUnix(path: []const u8) !Address { + var sock_addr = os.sockaddr_un{ + .family = os.AF_UNIX, + .path = undefined, + }; + + // this enables us to have the proper length of the socket in getOsSockLen + mem.set(u8, &sock_addr.path, 0); + + if (path.len > sock_addr.path.len) return error.NameTooLong; + mem.copy(u8, &sock_addr.path, path); + + return Address{ .un = sock_addr }; + } + + /// Returns the port in native endian. + /// Asserts that the address is ip4 or ip6. + pub fn getPort(self: Address) u16 { + return switch (self.any.family) { + os.AF_INET => self.in.getPort(), + os.AF_INET6 => self.in6.getPort(), + else => unreachable, + }; + } + + /// `port` is native-endian. + /// Asserts that the address is ip4 or ip6. + pub fn setPort(self: *Address, port: u16) void { + switch (self.any.family) { + os.AF_INET => self.in.setPort(port), + os.AF_INET6 => self.in6.setPort(port), + else => unreachable, + } + } + + /// Asserts that `addr` is an IP address. + /// This function will read past the end of the pointer, with a size depending + /// on the address family. + pub fn initPosix(addr: *align(4) const os.sockaddr) Address { + switch (addr.family) { + os.AF_INET => return Address{ .in = Ip4Address{ .sa = @ptrCast(*const os.sockaddr_in, addr).*} }, + os.AF_INET6 => return Address{ .in6 = Ip6Address{ .sa = @ptrCast(*const os.sockaddr_in6, addr).*} }, + else => unreachable, + } + } + + pub fn format( + self: Address, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + out_stream: anytype, + ) !void { + switch (self.any.family) { + os.AF_INET => try self.in.format(fmt, options, out_stream), + os.AF_INET6 => try self.in6.format(fmt, options, out_stream), + os.AF_UNIX => { + if (!has_unix_sockets) { + unreachable; + } + + try std.fmt.format(out_stream, "{}", .{&self.un.path}); + }, + else => unreachable, + } + } + + pub fn eql(a: Address, b: Address) bool { + const a_bytes = @ptrCast([*]const u8, &a.any)[0..a.getOsSockLen()]; + const b_bytes = @ptrCast([*]const u8, &b.any)[0..b.getOsSockLen()]; + return mem.eql(u8, a_bytes, b_bytes); + } + + pub fn getOsSockLen(self: Address) os.socklen_t { + switch (self.any.family) { + os.AF_INET => return self.in.getOsSockLen(), + os.AF_INET6 => return self.in6.getOsSockLen(), + os.AF_UNIX => { + if (!has_unix_sockets) { + unreachable; + } + + const path_len = std.mem.len(@ptrCast([*:0]const u8, &self.un.path)); + return @intCast(os.socklen_t, @sizeOf(os.sockaddr_un) - self.un.path.len + path_len); + }, + else => unreachable, + } + } +}; + +pub const Ip4Address = extern struct { + sa: os.sockaddr_in, + + pub fn parse(buf: []const u8, port: u16) !Ip4Address { + var result = Ip4Address{ + .sa = .{ + .port = mem.nativeToBig(u16, port), + .addr = undefined, + } + }; + const out_ptr = mem.sliceAsBytes(@as(*[1]u32, &result.sa.addr)[0..]); + + var x: u8 = 0; + var index: u8 = 0; + var saw_any_digits = false; + for (buf) |c| { + if (c == '.') { + if (!saw_any_digits) { + return error.InvalidCharacter; + } + if (index == 3) { + return error.InvalidEnd; + } + out_ptr[index] = x; + index += 1; + x = 0; + saw_any_digits = false; + } else if (c >= '0' and c <= '9') { + saw_any_digits = true; + x = try std.math.mul(u8, x, 10); + x = try std.math.add(u8, x, c - '0'); + } else { + return error.InvalidCharacter; + } + } + if (index == 3 and saw_any_digits) { + out_ptr[index] = x; + return result; + } + + return error.Incomplete; + } + + pub fn resolveIp(name: []const u8, port: u16) !Ip4Address { + if (parse(name, port)) |ip4| return ip4 else |err| switch (err) { + error.Overflow, + error.InvalidEnd, + error.InvalidCharacter, + error.Incomplete, + => {}, + } + return error.InvalidIPAddressFormat; + } + + pub fn init(addr: [4]u8, port: u16) Ip4Address { + return Ip4Address { + .sa = os.sockaddr_in{ + .port = mem.nativeToBig(u16, port), + .addr = @ptrCast(*align(1) const u32, &addr).*, + }, + }; + } + + /// Returns the port in native endian. + /// Asserts that the address is ip4 or ip6. + pub fn getPort(self: Ip4Address) u16 { + return mem.bigToNative(u16, self.sa.port); + } + + /// `port` is native-endian. + /// Asserts that the address is ip4 or ip6. + pub fn setPort(self: *Ip4Address, port: u16) void { + self.sa.port = mem.nativeToBig(u16, port); + } + + pub fn format( + self: Ip4Address, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + out_stream: anytype, + ) !void { + const bytes = @ptrCast(*const [4]u8, &self.sa.addr); + try std.fmt.format(out_stream, "{}.{}.{}.{}:{}", .{ + bytes[0], + bytes[1], + bytes[2], + bytes[3], + self.getPort(), + }); + } + + pub fn getOsSockLen(self: Ip4Address) os.socklen_t { + return @sizeOf(os.sockaddr_in); + } +}; + +pub const Ip6Address = extern struct { + sa: os.sockaddr_in6, + /// Parse a given IPv6 address string into an Address. /// Assumes the Scope ID of the address is fully numeric. /// For non-numeric addresses, see `resolveIp6`. - pub fn parseIp6(buf: []const u8, port: u16) !Address { - var result = Address{ - .in6 = os.sockaddr_in6{ + pub fn parse(buf: []const u8, port: u16) !Ip6Address { + var result = Ip6Address{ + .sa = os.sockaddr_in6{ .scope_id = 0, .port = mem.nativeToBig(u16, port), .flowinfo = 0, .addr = undefined, }, }; - var ip_slice = result.in6.addr[0..]; + var ip_slice = result.sa.addr[0..]; var tail: [16]u8 = undefined; @@ -101,10 +309,10 @@ pub const Address = extern union { if (scope_id) { if (c >= '0' and c <= '9') { const digit = c - '0'; - if (@mulWithOverflow(u32, result.in6.scope_id, 10, &result.in6.scope_id)) { + if (@mulWithOverflow(u32, result.sa.scope_id, 10, &result.sa.scope_id)) { return error.Overflow; } - if (@addWithOverflow(u32, result.in6.scope_id, digit, &result.in6.scope_id)) { + if (@addWithOverflow(u32, result.sa.scope_id, digit, &result.sa.scope_id)) { return error.Overflow; } } else { @@ -141,10 +349,10 @@ pub const Address = extern union { return error.InvalidIpv4Mapping; } const start_index = mem.lastIndexOfScalar(u8, buf[0..i], ':').? + 1; - const addr = (parseIp4(buf[start_index..], 0) catch { + const addr = (Ip4Address.parse(buf[start_index..], 0) catch { return error.InvalidIpv4Mapping; - }).in.addr; - ip_slice = result.in6.addr[0..]; + }).sa.addr; + ip_slice = result.sa.addr[0..]; ip_slice[10] = 0xff; ip_slice[11] = 0xff; @@ -180,22 +388,22 @@ pub const Address = extern union { index += 1; ip_slice[index] = @truncate(u8, x); index += 1; - mem.copy(u8, result.in6.addr[16 - index ..], ip_slice[0..index]); + mem.copy(u8, result.sa.addr[16 - index ..], ip_slice[0..index]); return result; } } - pub fn resolveIp6(buf: []const u8, port: u16) !Address { + pub fn resolve(buf: []const u8, port: u16) !Ip6Address { // TODO: Unify the implementations of resolveIp6 and parseIp6. - var result = Address{ - .in6 = os.sockaddr_in6{ + var result = Ip6Address{ + .sa = os.sockaddr_in6{ .scope_id = 0, .port = mem.nativeToBig(u16, port), .flowinfo = 0, .addr = undefined, }, }; - var ip_slice = result.in6.addr[0..]; + var ip_slice = result.sa.addr[0..]; var tail: [16]u8 = undefined; @@ -256,10 +464,10 @@ pub const Address = extern union { return error.InvalidIpv4Mapping; } const start_index = mem.lastIndexOfScalar(u8, buf[0..i], ':').? + 1; - const addr = (parseIp4(buf[start_index..], 0) catch { + const addr = (Ip4Address.parse(buf[start_index..], 0) catch { return error.InvalidIpv4Mapping; - }).in.addr; - ip_slice = result.in6.addr[0..]; + }).sa.addr; + ip_slice = result.sa.addr[0..]; ip_slice[10] = 0xff; ip_slice[11] = 0xff; @@ -299,7 +507,7 @@ pub const Address = extern union { }; } - result.in6.scope_id = resolved_scope_id; + result.sa.scope_id = resolved_scope_id; if (index == 14) { ip_slice[14] = @truncate(u8, x >> 8); @@ -310,63 +518,14 @@ pub const Address = extern union { index += 1; ip_slice[index] = @truncate(u8, x); index += 1; - mem.copy(u8, result.in6.addr[16 - index ..], ip_slice[0..index]); + mem.copy(u8, result.sa.addr[16 - index ..], ip_slice[0..index]); return result; } } - pub fn parseIp4(buf: []const u8, port: u16) !Address { - var result = Address{ - .in = os.sockaddr_in{ - .port = mem.nativeToBig(u16, port), - .addr = undefined, - }, - }; - const out_ptr = mem.sliceAsBytes(@as(*[1]u32, &result.in.addr)[0..]); - - var x: u8 = 0; - var index: u8 = 0; - var saw_any_digits = false; - for (buf) |c| { - if (c == '.') { - if (!saw_any_digits) { - return error.InvalidCharacter; - } - if (index == 3) { - return error.InvalidEnd; - } - out_ptr[index] = x; - index += 1; - x = 0; - saw_any_digits = false; - } else if (c >= '0' and c <= '9') { - saw_any_digits = true; - x = try std.math.mul(u8, x, 10); - x = try std.math.add(u8, x, c - '0'); - } else { - return error.InvalidCharacter; - } - } - if (index == 3 and saw_any_digits) { - out_ptr[index] = x; - return result; - } - - return error.Incomplete; - } - - pub fn initIp4(addr: [4]u8, port: u16) Address { - return Address{ - .in = os.sockaddr_in{ - .port = mem.nativeToBig(u16, port), - .addr = @ptrCast(*align(1) const u32, &addr).*, - }, - }; - } - - pub fn initIp6(addr: [16]u8, port: u16, flowinfo: u32, scope_id: u32) Address { - return Address{ - .in6 = os.sockaddr_in6{ + pub fn init(addr: [16]u8, port: u16, flowinfo: u32, scope_id: u32) Ip6Address { + return Ip6Address{ + .sa = os.sockaddr_in6{ .addr = addr, .port = mem.nativeToBig(u16, port), .flowinfo = flowinfo, @@ -375,147 +534,71 @@ pub const Address = extern union { }; } - pub fn initUnix(path: []const u8) !Address { - var sock_addr = os.sockaddr_un{ - .family = os.AF_UNIX, - .path = undefined, - }; - - // this enables us to have the proper length of the socket in getOsSockLen - mem.set(u8, &sock_addr.path, 0); - - if (path.len > sock_addr.path.len) return error.NameTooLong; - mem.copy(u8, &sock_addr.path, path); - - return Address{ .un = sock_addr }; - } - /// Returns the port in native endian. /// Asserts that the address is ip4 or ip6. - pub fn getPort(self: Address) u16 { - const big_endian_port = switch (self.any.family) { - os.AF_INET => self.in.port, - os.AF_INET6 => self.in6.port, - else => unreachable, - }; - return mem.bigToNative(u16, big_endian_port); + pub fn getPort(self: Ip6Address) u16 { + return mem.bigToNative(u16, self.sa.port); } /// `port` is native-endian. /// Asserts that the address is ip4 or ip6. - pub fn setPort(self: *Address, port: u16) void { - const ptr = switch (self.any.family) { - os.AF_INET => &self.in.port, - os.AF_INET6 => &self.in6.port, - else => unreachable, - }; - ptr.* = mem.nativeToBig(u16, port); - } - - /// Asserts that `addr` is an IP address. - /// This function will read past the end of the pointer, with a size depending - /// on the address family. - pub fn initPosix(addr: *align(4) const os.sockaddr) Address { - switch (addr.family) { - os.AF_INET => return Address{ .in = @ptrCast(*const os.sockaddr_in, addr).* }, - os.AF_INET6 => return Address{ .in6 = @ptrCast(*const os.sockaddr_in6, addr).* }, - else => unreachable, - } + pub fn setPort(self: *Ip6Address, port: u16) void { + self.sa.port = mem.nativeToBig(u16, port); } pub fn format( - self: Address, + self: Ip6Address, comptime fmt: []const u8, options: std.fmt.FormatOptions, out_stream: anytype, ) !void { - switch (self.any.family) { - os.AF_INET => { - const port = mem.bigToNative(u16, self.in.port); - const bytes = @ptrCast(*const [4]u8, &self.in.addr); - try std.fmt.format(out_stream, "{}.{}.{}.{}:{}", .{ - bytes[0], - bytes[1], - bytes[2], - bytes[3], - port, - }); - }, - os.AF_INET6 => { - const port = mem.bigToNative(u16, self.in6.port); - if (mem.eql(u8, self.in6.addr[0..12], &[_]u8{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff })) { - try std.fmt.format(out_stream, "[::ffff:{}.{}.{}.{}]:{}", .{ - self.in6.addr[12], - self.in6.addr[13], - self.in6.addr[14], - self.in6.addr[15], - port, - }); - return; - } - const big_endian_parts = @ptrCast(*align(1) const [8]u16, &self.in6.addr); - const native_endian_parts = switch (builtin.endian) { - .Big => big_endian_parts.*, - .Little => blk: { - var buf: [8]u16 = undefined; - for (big_endian_parts) |part, i| { - buf[i] = mem.bigToNative(u16, part); - } - break :blk buf; - }, - }; - try out_stream.writeAll("["); - var i: usize = 0; - var abbrv = false; - while (i < native_endian_parts.len) : (i += 1) { - if (native_endian_parts[i] == 0) { - if (!abbrv) { - try out_stream.writeAll(if (i == 0) "::" else ":"); - abbrv = true; - } - continue; - } - try std.fmt.format(out_stream, "{x}", .{native_endian_parts[i]}); - if (i != native_endian_parts.len - 1) { - try out_stream.writeAll(":"); - } - } - try std.fmt.format(out_stream, "]:{}", .{port}); - }, - os.AF_UNIX => { - if (!has_unix_sockets) { - unreachable; - } - - try std.fmt.format(out_stream, "{}", .{&self.un.path}); - }, - else => unreachable, + const port = mem.bigToNative(u16, self.sa.port); + if (mem.eql(u8, self.sa.addr[0..12], &[_]u8{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff })) { + try std.fmt.format(out_stream, "[::ffff:{}.{}.{}.{}]:{}", .{ + self.sa.addr[12], + self.sa.addr[13], + self.sa.addr[14], + self.sa.addr[15], + port, + }); + return; } + const big_endian_parts = @ptrCast(*align(1) const [8]u16, &self.sa.addr); + const native_endian_parts = switch (builtin.endian) { + .Big => big_endian_parts.*, + .Little => blk: { + var buf: [8]u16 = undefined; + for (big_endian_parts) |part, i| { + buf[i] = mem.bigToNative(u16, part); + } + break :blk buf; + }, + }; + try out_stream.writeAll("["); + var i: usize = 0; + var abbrv = false; + while (i < native_endian_parts.len) : (i += 1) { + if (native_endian_parts[i] == 0) { + if (!abbrv) { + try out_stream.writeAll(if (i == 0) "::" else ":"); + abbrv = true; + } + continue; + } + try std.fmt.format(out_stream, "{x}", .{native_endian_parts[i]}); + if (i != native_endian_parts.len - 1) { + try out_stream.writeAll(":"); + } + } + try std.fmt.format(out_stream, "]:{}", .{port}); } - pub fn eql(a: Address, b: Address) bool { - const a_bytes = @ptrCast([*]const u8, &a.any)[0..a.getOsSockLen()]; - const b_bytes = @ptrCast([*]const u8, &b.any)[0..b.getOsSockLen()]; - return mem.eql(u8, a_bytes, b_bytes); - } - - pub fn getOsSockLen(self: Address) os.socklen_t { - switch (self.any.family) { - os.AF_INET => return @sizeOf(os.sockaddr_in), - os.AF_INET6 => return @sizeOf(os.sockaddr_in6), - os.AF_UNIX => { - if (!has_unix_sockets) { - unreachable; - } - - const path_len = std.mem.len(@ptrCast([*:0]const u8, &self.un.path)); - return @intCast(os.socklen_t, @sizeOf(os.sockaddr_un) - self.un.path.len + path_len); - }, - else => unreachable, - } + pub fn getOsSockLen(self: Ip6Address) os.socklen_t { + return @sizeOf(os.sockaddr_in6); } }; + pub fn connectUnixSocket(path: []const u8) !fs.File { const opt_non_block = if (std.io.is_async) os.SOCK_NONBLOCK else 0; const sockfd = try os.socket( @@ -777,7 +860,7 @@ fn linuxLookupName( @memset(@ptrCast([*]u8, &sa6), 0, @sizeOf(os.sockaddr_in6)); var da6 = os.sockaddr_in6{ .family = os.AF_INET6, - .scope_id = addr.addr.in6.scope_id, + .scope_id = addr.addr.in6.sa.scope_id, .port = 65535, .flowinfo = 0, .addr = [1]u8{0} ** 16, @@ -795,7 +878,7 @@ fn linuxLookupName( var salen: os.socklen_t = undefined; var dalen: os.socklen_t = undefined; if (addr.addr.any.family == os.AF_INET6) { - mem.copy(u8, &da6.addr, &addr.addr.in6.addr); + mem.copy(u8, &da6.addr, &addr.addr.in6.sa.addr); da = @ptrCast(*os.sockaddr, &da6); dalen = @sizeOf(os.sockaddr_in6); sa = @ptrCast(*os.sockaddr, &sa6); @@ -803,8 +886,8 @@ fn linuxLookupName( } else { mem.copy(u8, &sa6.addr, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff"); mem.copy(u8, &da6.addr, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff"); - mem.writeIntNative(u32, da6.addr[12..], addr.addr.in.addr); - da4.addr = addr.addr.in.addr; + mem.writeIntNative(u32, da6.addr[12..], addr.addr.in.sa.addr); + da4.addr = addr.addr.in.sa.addr; da = @ptrCast(*os.sockaddr, &da4); dalen = @sizeOf(os.sockaddr_in); sa = @ptrCast(*os.sockaddr, &sa4); From 616355807fc8b0a2e7ccc2e54cd4453776c0e187 Mon Sep 17 00:00:00 2001 From: joachimschmidt557 Date: Sun, 26 Jul 2020 18:03:03 +0200 Subject: [PATCH 016/104] Fix bug in big.int.Mutable.toManaged() and add tests Fixes #5918 --- lib/std/math/big/int.zig | 2 +- lib/std/math/big/int_test.zig | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/std/math/big/int.zig b/lib/std/math/big/int.zig index b6d7731f1a..92b3f80429 100644 --- a/lib/std/math/big/int.zig +++ b/lib/std/math/big/int.zig @@ -99,7 +99,7 @@ pub const Mutable = struct { pub fn toManaged(self: Mutable, allocator: *Allocator) Managed { return .{ .allocator = allocator, - .limbs = limbs, + .limbs = self.limbs, .metadata = if (self.positive) self.len & ~Managed.sign_bit else diff --git a/lib/std/math/big/int_test.zig b/lib/std/math/big/int_test.zig index d7e354879e..a86b55d2de 100644 --- a/lib/std/math/big/int_test.zig +++ b/lib/std/math/big/int_test.zig @@ -2,6 +2,7 @@ const std = @import("../../std.zig"); const mem = std.mem; const testing = std.testing; const Managed = std.math.big.int.Managed; +const Mutable = std.math.big.int.Mutable; const Limb = std.math.big.Limb; const DoubleLimb = std.math.big.DoubleLimb; const maxInt = std.math.maxInt; @@ -1453,3 +1454,24 @@ test "big.int gcd one large" { testing.expect((try r.to(u64)) == 1); } + +test "big.int mutable to managed" { + const allocator = testing.allocator; + var limbs_buf = try allocator.alloc(Limb, 8); + defer allocator.free(limbs_buf); + + var a = Mutable.init(limbs_buf, 0xdeadbeef); + var a_managed = a.toManaged(allocator); + + testing.expect(a.toConst().eq(a_managed.toConst())); +} + +test "big.int const to managed" { + var a = try Managed.initSet(testing.allocator, 123423453456); + defer a.deinit(); + + var b = try a.toConst().toManaged(testing.allocator); + defer b.deinit(); + + testing.expect(a.toConst().eq(b.toConst())); +} From 442025481c40625f8010355d2790935e5add0db5 Mon Sep 17 00:00:00 2001 From: Henrik Laxhuber Date: Sun, 26 Jul 2020 16:27:47 +0200 Subject: [PATCH 017/104] Fix parsing of `unsigned` in translate-c. Previously, `unsigned` was parsed as the shorthand for `unsigned int`. This commit introduces code to parse `unsigned short`, `unsigned int`, `unsigned long`, and `unsigned long long`. There is a comment in the code about std.c.parse` - Im not familiar with zig internals, but it seems like this is a separate C parsing implementation. In the long run, it probably makes sense to merge both implementations, so this commit should be regarded as a quick fix that doesn't address an apparently underlying issue. --- src-self-hosted/translate_c.zig | 17 ++++++++++++++++- test/translate_c.zig | 10 ++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 30cf0bc071..9de3dff94f 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -5764,7 +5764,22 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, .Keyword_float => return transCreateNodeIdentifierUnchecked(c, "f32"), .Keyword_short => return transCreateNodeIdentifierUnchecked(c, "c_short"), .Keyword_char => return transCreateNodeIdentifierUnchecked(c, "c_char"), - .Keyword_unsigned => return transCreateNodeIdentifierUnchecked(c, "c_uint"), + .Keyword_unsigned => if (it.next()) |t| { + switch (t.id) { + .Keyword_short => return transCreateNodeIdentifierUnchecked(c, "c_ushort"), + .Keyword_int => return transCreateNodeIdentifierUnchecked(c, "c_uint"), + .Keyword_long => if (it.peek() != null and it.peek().?.id == .Keyword_long) { + _ = it.next(); + return transCreateNodeIdentifierUnchecked(c, "c_ulonglong"); + } else return transCreateNodeIdentifierUnchecked(c, "c_ulong"), + else => { + _ = it.prev(); + return transCreateNodeIdentifierUnchecked(c, "c_uint"); + }, + } + } else { + return transCreateNodeIdentifierUnchecked(c, "c_uint"); + }, .Identifier => { const mangled_name = scope.getAlias(source[tok.start..tok.end]); return transCreateNodeIdentifier(c, mangled_name); diff --git a/test/translate_c.zig b/test/translate_c.zig index d12b2ab860..c1c4ef07fa 100644 --- a/test/translate_c.zig +++ b/test/translate_c.zig @@ -2715,6 +2715,16 @@ pub fn addCases(cases: *tests.TranslateCContext) void { \\pub const BAR = (@import("std").meta.cast(?*c_void, a)); }); + cases.add("macro with cast to unsigned short, long, and long long", + \\#define CURLAUTH_BASIC_BUT_USHORT ((unsigned short) 1) + \\#define CURLAUTH_BASIC ((unsigned long) 1) + \\#define CURLAUTH_BASIC_BUT_ULONGLONG ((unsigned long long) 1) + , &[_][]const u8{ + \\pub const CURLAUTH_BASIC_BUT_USHORT = (@import("std").meta.cast(c_ushort, 1)); + \\pub const CURLAUTH_BASIC = (@import("std").meta.cast(c_ulong, 1)); + \\pub const CURLAUTH_BASIC_BUT_ULONGLONG = (@import("std").meta.cast(c_ulonglong, 1)); + }); + cases.add("macro conditional operator", \\#define FOO a ? b : c , &[_][]const u8{ From 9f6401c69275f5c80b92c2c4c18c9b73321b82aa Mon Sep 17 00:00:00 2001 From: Henrik Laxhuber Date: Sun, 26 Jul 2020 18:29:45 +0200 Subject: [PATCH 018/104] Fix a use of appendToken to appendIdentifier in translate-c --- src-self-hosted/translate_c.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 9de3dff94f..aee9c979e1 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -4514,7 +4514,7 @@ const CtrlFlow = struct { const ltoken = try appendToken(c, kw, kw_text); const label_token = if (label) |l| blk: { _ = try appendToken(c, .Colon, ":"); - break :blk try appendToken(c, .Identifier, l); + break :blk try appendIdentifier(c, l); } else null; return CtrlFlow{ .c = c, From e7007fa7bd94b3290982b1c39933f700e4ee133f Mon Sep 17 00:00:00 2001 From: Vexu Date: Mon, 27 Jul 2020 15:19:07 +0300 Subject: [PATCH 019/104] translate-c: use ArrayList for macro tokens --- lib/std/c.zig | 4 + lib/std/c/tokenizer.zig | 191 ++++++-------- src-self-hosted/translate_c.zig | 430 +++++++++++++++++--------------- 3 files changed, 302 insertions(+), 323 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index e483b5b50f..e836a49c30 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -8,6 +8,10 @@ pub const Tokenizer = tokenizer.Tokenizer; pub const parse = @import("c/parse.zig").parse; pub const ast = @import("c/ast.zig"); +test "" { + _ = tokenizer; +} + pub usingnamespace @import("os/bits.zig"); pub usingnamespace switch (std.Target.current.os.tag) { diff --git a/lib/std/c/tokenizer.zig b/lib/std/c/tokenizer.zig index f3fc8b8107..a3e6710e2f 100644 --- a/lib/std/c/tokenizer.zig +++ b/lib/std/c/tokenizer.zig @@ -1,19 +1,10 @@ const std = @import("std"); const mem = std.mem; -pub const Source = struct { - buffer: []const u8, - file_name: []const u8, - tokens: TokenList, - - pub const TokenList = std.SegmentedList(Token, 64); -}; - pub const Token = struct { id: Id, start: usize, end: usize, - source: *Source, pub const Id = union(enum) { Invalid, @@ -251,31 +242,6 @@ pub const Token = struct { } }; - pub fn eql(a: Token, b: Token) bool { - // do we really need this cast here - if (@as(@TagType(Id), a.id) != b.id) return false; - return mem.eql(u8, a.slice(), b.slice()); - } - - pub fn slice(tok: Token) []const u8 { - return tok.source.buffer[tok.start..tok.end]; - } - - pub const Keyword = struct { - bytes: []const u8, - id: Id, - hash: u32, - - fn init(bytes: []const u8, id: Id) Keyword { - @setEvalBranchQuota(2000); - return .{ - .bytes = bytes, - .id = id, - .hash = std.hash_map.hashString(bytes), - }; - } - }; - // TODO extensions pub const keywords = std.ComptimeStringMap(Id, .{ .{ "auto", .Keyword_auto }, @@ -355,26 +321,26 @@ pub const Token = struct { } pub const NumSuffix = enum { - None, - F, - L, - U, - LU, - LL, - LLU, + none, + f, + l, + u, + lu, + ll, + llu, }; pub const StrKind = enum { - None, - Wide, - Utf8, - Utf16, - Utf32, + none, + wide, + utf_8, + utf_16, + utf_32, }; }; pub const Tokenizer = struct { - source: *Source, + buffer: []const u8, index: usize = 0, prev_tok_id: @TagType(Token.Id) = .Invalid, pp_directive: bool = false, @@ -385,7 +351,6 @@ pub const Tokenizer = struct { .id = .Eof, .start = self.index, .end = undefined, - .source = self.source, }; var state: enum { Start, @@ -446,8 +411,8 @@ pub const Tokenizer = struct { } = .Start; var string = false; var counter: u32 = 0; - while (self.index < self.source.buffer.len) : (self.index += 1) { - const c = self.source.buffer[self.index]; + while (self.index < self.buffer.len) : (self.index += 1) { + const c = self.buffer[self.index]; switch (state) { .Start => switch (c) { '\n' => { @@ -460,11 +425,11 @@ pub const Tokenizer = struct { state = .Cr; }, '"' => { - result.id = .{ .StringLiteral = .None }; + result.id = .{ .StringLiteral = .none }; state = .StringLiteral; }, '\'' => { - result.id = .{ .CharLiteral = .None }; + result.id = .{ .CharLiteral = .none }; state = .CharLiteralStart; }, 'u' => { @@ -641,11 +606,11 @@ pub const Tokenizer = struct { state = .u8; }, '\'' => { - result.id = .{ .CharLiteral = .Utf16 }; + result.id = .{ .CharLiteral = .utf_16 }; state = .CharLiteralStart; }, '\"' => { - result.id = .{ .StringLiteral = .Utf16 }; + result.id = .{ .StringLiteral = .utf_16 }; state = .StringLiteral; }, else => { @@ -655,7 +620,7 @@ pub const Tokenizer = struct { }, .u8 => switch (c) { '\"' => { - result.id = .{ .StringLiteral = .Utf8 }; + result.id = .{ .StringLiteral = .utf_8 }; state = .StringLiteral; }, else => { @@ -665,11 +630,11 @@ pub const Tokenizer = struct { }, .U => switch (c) { '\'' => { - result.id = .{ .CharLiteral = .Utf32 }; + result.id = .{ .CharLiteral = .utf_32 }; state = .CharLiteralStart; }, '\"' => { - result.id = .{ .StringLiteral = .Utf32 }; + result.id = .{ .StringLiteral = .utf_32 }; state = .StringLiteral; }, else => { @@ -679,11 +644,11 @@ pub const Tokenizer = struct { }, .L => switch (c) { '\'' => { - result.id = .{ .CharLiteral = .Wide }; + result.id = .{ .CharLiteral = .wide }; state = .CharLiteralStart; }, '\"' => { - result.id = .{ .StringLiteral = .Wide }; + result.id = .{ .StringLiteral = .wide }; state = .StringLiteral; }, else => { @@ -808,7 +773,7 @@ pub const Tokenizer = struct { .Identifier => switch (c) { 'a'...'z', 'A'...'Z', '_', '0'...'9' => {}, else => { - result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier; + result.id = Token.getKeyword(self.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier; if (self.prev_tok_id == .Hash) self.pp_directive = true; break; @@ -1137,7 +1102,7 @@ pub const Tokenizer = struct { state = .IntegerSuffixL; }, else => { - result.id = .{ .IntegerLiteral = .None }; + result.id = .{ .IntegerLiteral = .none }; break; }, }, @@ -1146,7 +1111,7 @@ pub const Tokenizer = struct { state = .IntegerSuffixUL; }, else => { - result.id = .{ .IntegerLiteral = .U }; + result.id = .{ .IntegerLiteral = .u }; break; }, }, @@ -1155,34 +1120,34 @@ pub const Tokenizer = struct { state = .IntegerSuffixLL; }, 'u', 'U' => { - result.id = .{ .IntegerLiteral = .LU }; + result.id = .{ .IntegerLiteral = .lu }; self.index += 1; break; }, else => { - result.id = .{ .IntegerLiteral = .L }; + result.id = .{ .IntegerLiteral = .l }; break; }, }, .IntegerSuffixLL => switch (c) { 'u', 'U' => { - result.id = .{ .IntegerLiteral = .LLU }; + result.id = .{ .IntegerLiteral = .llu }; self.index += 1; break; }, else => { - result.id = .{ .IntegerLiteral = .LL }; + result.id = .{ .IntegerLiteral = .ll }; break; }, }, .IntegerSuffixUL => switch (c) { 'l', 'L' => { - result.id = .{ .IntegerLiteral = .LLU }; + result.id = .{ .IntegerLiteral = .llu }; self.index += 1; break; }, else => { - result.id = .{ .IntegerLiteral = .LU }; + result.id = .{ .IntegerLiteral = .lu }; break; }, }, @@ -1230,26 +1195,26 @@ pub const Tokenizer = struct { }, .FloatSuffix => switch (c) { 'l', 'L' => { - result.id = .{ .FloatLiteral = .L }; + result.id = .{ .FloatLiteral = .l }; self.index += 1; break; }, 'f', 'F' => { - result.id = .{ .FloatLiteral = .F }; + result.id = .{ .FloatLiteral = .f }; self.index += 1; break; }, else => { - result.id = .{ .FloatLiteral = .None }; + result.id = .{ .FloatLiteral = .none }; break; }, }, } - } else if (self.index == self.source.buffer.len) { + } else if (self.index == self.buffer.len) { switch (state) { .Start => {}, .u, .u8, .U, .L, .Identifier => { - result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier; + result.id = Token.getKeyword(self.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier; }, .Cr, @@ -1270,11 +1235,11 @@ pub const Tokenizer = struct { .MacroString, => result.id = .Invalid, - .FloatExponentDigits => result.id = if (counter == 0) .Invalid else .{ .FloatLiteral = .None }, + .FloatExponentDigits => result.id = if (counter == 0) .Invalid else .{ .FloatLiteral = .none }, .FloatFraction, .FloatFractionHex, - => result.id = .{ .FloatLiteral = .None }, + => result.id = .{ .FloatLiteral = .none }, .IntegerLiteralOct, .IntegerLiteralBinary, @@ -1282,13 +1247,13 @@ pub const Tokenizer = struct { .IntegerLiteral, .IntegerSuffix, .Zero, - => result.id = .{ .IntegerLiteral = .None }, - .IntegerSuffixU => result.id = .{ .IntegerLiteral = .U }, - .IntegerSuffixL => result.id = .{ .IntegerLiteral = .L }, - .IntegerSuffixLL => result.id = .{ .IntegerLiteral = .LL }, - .IntegerSuffixUL => result.id = .{ .IntegerLiteral = .LU }, + => result.id = .{ .IntegerLiteral = .none }, + .IntegerSuffixU => result.id = .{ .IntegerLiteral = .u }, + .IntegerSuffixL => result.id = .{ .IntegerLiteral = .l }, + .IntegerSuffixLL => result.id = .{ .IntegerLiteral = .ll }, + .IntegerSuffixUL => result.id = .{ .IntegerLiteral = .lu }, - .FloatSuffix => result.id = .{ .FloatLiteral = .None }, + .FloatSuffix => result.id = .{ .FloatLiteral = .none }, .Equal => result.id = .Equal, .Bang => result.id = .Bang, .Minus => result.id = .Minus, @@ -1466,7 +1431,7 @@ test "preprocessor keywords" { .Hash, .Identifier, .AngleBracketLeft, - .{ .IntegerLiteral = .None }, + .{ .IntegerLiteral = .none }, .Nl, .Hash, .Keyword_ifdef, @@ -1499,18 +1464,18 @@ test "line continuation" { .Identifier, .Identifier, .Nl, - .{ .StringLiteral = .None }, + .{ .StringLiteral = .none }, .Nl, .Hash, .Keyword_define, - .{ .StringLiteral = .None }, + .{ .StringLiteral = .none }, .Nl, - .{ .StringLiteral = .None }, + .{ .StringLiteral = .none }, .Nl, .Hash, .Keyword_define, - .{ .StringLiteral = .None }, - .{ .StringLiteral = .None }, + .{ .StringLiteral = .none }, + .{ .StringLiteral = .none }, }); } @@ -1527,23 +1492,23 @@ test "string prefix" { \\L'foo' \\ , &[_]Token.Id{ - .{ .StringLiteral = .None }, + .{ .StringLiteral = .none }, .Nl, - .{ .StringLiteral = .Utf16 }, + .{ .StringLiteral = .utf_16 }, .Nl, - .{ .StringLiteral = .Utf8 }, + .{ .StringLiteral = .utf_8 }, .Nl, - .{ .StringLiteral = .Utf32 }, + .{ .StringLiteral = .utf_32 }, .Nl, - .{ .StringLiteral = .Wide }, + .{ .StringLiteral = .wide }, .Nl, - .{ .CharLiteral = .None }, + .{ .CharLiteral = .none }, .Nl, - .{ .CharLiteral = .Utf16 }, + .{ .CharLiteral = .utf_16 }, .Nl, - .{ .CharLiteral = .Utf32 }, + .{ .CharLiteral = .utf_32 }, .Nl, - .{ .CharLiteral = .Wide }, + .{ .CharLiteral = .wide }, .Nl, }); } @@ -1555,33 +1520,29 @@ test "num suffixes" { \\ 1u 1ul 1ull 1 \\ , &[_]Token.Id{ - .{ .FloatLiteral = .F }, - .{ .FloatLiteral = .L }, - .{ .FloatLiteral = .None }, - .{ .FloatLiteral = .None }, - .{ .FloatLiteral = .None }, + .{ .FloatLiteral = .f }, + .{ .FloatLiteral = .l }, + .{ .FloatLiteral = .none }, + .{ .FloatLiteral = .none }, + .{ .FloatLiteral = .none }, .Nl, - .{ .IntegerLiteral = .L }, - .{ .IntegerLiteral = .LU }, - .{ .IntegerLiteral = .LL }, - .{ .IntegerLiteral = .LLU }, - .{ .IntegerLiteral = .None }, + .{ .IntegerLiteral = .l }, + .{ .IntegerLiteral = .lu }, + .{ .IntegerLiteral = .ll }, + .{ .IntegerLiteral = .llu }, + .{ .IntegerLiteral = .none }, .Nl, - .{ .IntegerLiteral = .U }, - .{ .IntegerLiteral = .LU }, - .{ .IntegerLiteral = .LLU }, - .{ .IntegerLiteral = .None }, + .{ .IntegerLiteral = .u }, + .{ .IntegerLiteral = .lu }, + .{ .IntegerLiteral = .llu }, + .{ .IntegerLiteral = .none }, .Nl, }); } fn expectTokens(source: []const u8, expected_tokens: []const Token.Id) void { var tokenizer = Tokenizer{ - .source = &Source{ - .buffer = source, - .file_name = undefined, - .tokens = undefined, - }, + .buffer = source, }; for (expected_tokens) |expected_token_id| { const token = tokenizer.next(); diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index aee9c979e1..023ac78459 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -8,7 +8,6 @@ const Token = std.zig.Token; usingnamespace @import("clang.zig"); const ctok = std.c.tokenizer; const CToken = std.c.Token; -const CTokenList = std.c.tokenizer.Source.TokenList; const mem = std.mem; const math = std.math; @@ -5196,16 +5195,39 @@ pub fn freeErrors(errors: []ClangErrMsg) void { ZigClangErrorMsg_delete(errors.ptr, errors.len); } +const CTokIterator = struct { + source: []const u8, + list: []const CToken, + i: usize = 0, + + fn peek(self: *CTokIterator) ?CToken.Id { + if (self.i >= self.list.len) return null; + return self.list[self.i + 1].id; + } + + fn next(self: *CTokIterator) ?CToken.Id { + if (self.i >= self.list.len) return null; + self.i += 1; + return self.list[self.i].id; + } + + fn slice(self: *CTokIterator, index: usize) []const u8 { + const tok = self.list[index]; + return self.source[tok.start..tok.end]; + } +}; + fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void { // TODO if we see #undef, delete it from the table var it = ZigClangASTUnit_getLocalPreprocessingEntities_begin(unit); const it_end = ZigClangASTUnit_getLocalPreprocessingEntities_end(unit); - var tok_list = CTokenList.init(c.arena); + var tok_list = std.ArrayList(CToken).init(c.gpa); + defer tok_list.deinit(); const scope = c.global_scope; while (it.I != it_end.I) : (it.I += 1) { const entity = ZigClangPreprocessingRecord_iterator_deref(it); - tok_list.shrink(0); + tok_list.items.len = 0; switch (ZigClangPreprocessedEntity_getKind(entity)) { .MacroDefinitionKind => { const macro = @ptrCast(*ZigClangMacroDefinitionRecord, entity); @@ -5223,38 +5245,34 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void { const begin_c = ZigClangSourceManager_getCharacterData(c.source_manager, begin_loc); const slice = begin_c[0..mem.len(begin_c)]; - tok_list.shrink(0); var tokenizer = std.c.Tokenizer{ - .source = &std.c.tokenizer.Source{ - .buffer = slice, - .file_name = undefined, - .tokens = undefined, - }, + .buffer = slice, }; while (true) { const tok = tokenizer.next(); switch (tok.id) { .Nl, .Eof => { - try tok_list.push(tok); + try tok_list.append(tok); break; }, .LineComment, .MultiLineComment => continue, else => {}, } - try tok_list.push(tok); + try tok_list.append(tok); } - var tok_it = tok_list.iterator(0); - const first_tok = tok_it.next().?; - assert(mem.eql(u8, slice[first_tok.start..first_tok.end], name)); + var tok_it = CTokIterator{ + .source = slice, + .list = tok_list.items, + }; + assert(mem.eql(u8, tok_it.slice(0), name)); var macro_fn = false; - const next = tok_it.peek().?; - switch (next.id) { + switch (tok_it.peek().?) { .Identifier => { // if it equals itself, ignore. for example, from stdio.h: // #define stdin stdin - if (mem.eql(u8, name, slice[next.start..next.end])) { + if (mem.eql(u8, name, tok_it.slice(1))) { continue; } }, @@ -5265,15 +5283,15 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void { }, .LParen => { // if the name is immediately followed by a '(' then it is a function - macro_fn = first_tok.end == next.start; + macro_fn = tok_it.list[0].end == tok_it.list[1].start; }, else => {}, } (if (macro_fn) - transMacroFnDefine(c, &tok_it, slice, mangled_name, begin_loc) + transMacroFnDefine(c, &tok_it, mangled_name, begin_loc) else - transMacroDefine(c, &tok_it, slice, mangled_name, begin_loc)) catch |err| switch (err) { + transMacroDefine(c, &tok_it, mangled_name, begin_loc)) catch |err| switch (err) { error.ParseError => continue, error.OutOfMemory => |e| return e, }; @@ -5283,7 +5301,7 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void { } } -fn transMacroDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void { +fn transMacroDefine(c: *Context, it: *CTokIterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void { const scope = &c.global_scope.base; const visib_tok = try appendToken(c, .Keyword_pub, "pub"); @@ -5291,15 +5309,15 @@ fn transMacroDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, n const name_tok = try appendIdentifier(c, name); const eq_token = try appendToken(c, .Equal, "="); - const init_node = try parseCExpr(c, it, source, source_loc, scope); + const init_node = try parseCExpr(c, it, source_loc, scope); const last = it.next().?; - if (last.id != .Eof and last.id != .Nl) + if (last != .Eof and last != .Nl) return failDecl( c, source_loc, name, "unable to translate C expr: unexpected token .{}", - .{@tagName(last.id)}, + .{@tagName(last)}, ); const semicolon_token = try appendToken(c, .Semicolon, ";"); @@ -5315,7 +5333,7 @@ fn transMacroDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, n _ = try c.global_scope.macro_table.put(name, &node.base); } -fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void { +fn transMacroFnDefine(c: *Context, it: *CTokIterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void { var block_scope = try Scope.Block.init(c, &c.global_scope.base, null); defer block_scope.deinit(); const scope = &block_scope.base; @@ -5326,7 +5344,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, const name_tok = try appendIdentifier(c, name); _ = try appendToken(c, .LParen, "("); - if (it.next().?.id != .LParen) { + if (it.next().? != .LParen) { return failDecl( c, source_loc, @@ -5340,8 +5358,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, defer fn_params.deinit(); while (true) { - const param_tok = it.next().?; - if (param_tok.id != .Identifier) { + if (it.next().? != .Identifier) { return failDecl( c, source_loc, @@ -5351,7 +5368,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, ); } - const mangled_name = try block_scope.makeMangledName(c, source[param_tok.start..param_tok.end]); + const mangled_name = try block_scope.makeMangledName(c, it.slice(it.i)); const param_name_tok = try appendIdentifier(c, mangled_name); _ = try appendToken(c, .Colon, ":"); @@ -5369,13 +5386,13 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, .param_type = .{ .any_type = &any_type.base }, }; - if (it.peek().?.id != .Comma) + if (it.peek().? != .Comma) break; _ = it.next(); _ = try appendToken(c, .Comma, ","); } - if (it.next().?.id != .RParen) { + if (it.next().? != .RParen) { return failDecl( c, source_loc, @@ -5390,15 +5407,15 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, const type_of = try c.createBuiltinCall("@TypeOf", 1); const return_kw = try appendToken(c, .Keyword_return, "return"); - const expr = try parseCExpr(c, it, source, source_loc, scope); + const expr = try parseCExpr(c, it, source_loc, scope); const last = it.next().?; - if (last.id != .Eof and last.id != .Nl) + if (last != .Eof and last != .Nl) return failDecl( c, source_loc, name, "unable to translate C expr: unexpected token .{}", - .{@tagName(last.id)}, + .{@tagName(last)}, ); _ = try appendToken(c, .Semicolon, ";"); const type_of_arg = if (expr.tag != .Block) expr else blk: { @@ -5435,28 +5452,27 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, const ParseError = Error || error{ParseError}; -fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node { - const node = try parseCPrefixOpExpr(c, it, source, source_loc, scope); - switch (it.next().?.id) { +fn parseCExpr(c: *Context, it: *CTokIterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node { + const node = try parseCPrefixOpExpr(c, it, source_loc, scope); + switch (it.next().?) { .QuestionMark => { // must come immediately after expr _ = try appendToken(c, .RParen, ")"); const if_node = try transCreateNodeIf(c); if_node.condition = node; - if_node.body = try parseCPrimaryExpr(c, it, source, source_loc, scope); - if (it.next().?.id != .Colon) { - const first_tok = it.list.at(0); + if_node.body = try parseCPrimaryExpr(c, it, source_loc, scope); + if (it.next().? != .Colon) { try failDecl( c, source_loc, - source[first_tok.start..first_tok.end], + it.slice(0), "unable to translate C expr: expected ':'", .{}, ); return error.ParseError; } if_node.@"else" = try transCreateNodeElse(c); - if_node.@"else".?.body = try parseCPrimaryExpr(c, it, source, source_loc, scope); + if_node.@"else".?.body = try parseCPrimaryExpr(c, it, source_loc, scope); return &if_node.base; }, .Comma => { @@ -5479,10 +5495,10 @@ fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_ }; try block_scope.statements.append(&op_node.base); - last = try parseCPrefixOpExpr(c, it, source, source_loc, scope); + last = try parseCPrefixOpExpr(c, it, source_loc, scope); _ = try appendToken(c, .Semicolon, ";"); - if (it.next().?.id != .Comma) { - _ = it.prev(); + if (it.next().? != .Comma) { + it.i -= 1; break; } } @@ -5493,70 +5509,74 @@ fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_ return &block_node.base; }, else => { - _ = it.prev(); + it.i -= 1; return node; }, } } -fn parseCNumLit(c: *Context, tok: *CToken, source: []const u8, source_loc: ZigClangSourceLocation) ParseError!*ast.Node { - var lit_bytes = source[tok.start..tok.end]; +fn parseCNumLit(c: *Context, it: *CTokIterator, source_loc: ZigClangSourceLocation) ParseError!*ast.Node { + var lit_bytes = it.slice(it.i); - if (tok.id == .IntegerLiteral) { - if (lit_bytes.len > 2 and lit_bytes[0] == '0') { - switch (lit_bytes[1]) { - '0'...'7' => { - // Octal - lit_bytes = try std.fmt.allocPrint(c.arena, "0o{}", .{lit_bytes}); - }, - 'X' => { - // Hexadecimal with capital X, valid in C but not in Zig - lit_bytes = try std.fmt.allocPrint(c.arena, "0x{}", .{lit_bytes[2..]}); - }, - else => {}, + switch (it.list[it.i].id) { + .IntegerLiteral => |suffix| { + if (lit_bytes.len > 2 and lit_bytes[0] == '0') { + switch (lit_bytes[1]) { + '0'...'7' => { + // Octal + lit_bytes = try std.fmt.allocPrint(c.arena, "0o{}", .{lit_bytes}); + }, + 'X' => { + // Hexadecimal with capital X, valid in C but not in Zig + lit_bytes = try std.fmt.allocPrint(c.arena, "0x{}", .{lit_bytes[2..]}); + }, + else => {}, + } } - } - if (tok.id.IntegerLiteral == .None) { - return transCreateNodeInt(c, lit_bytes); - } + if (suffix == .none) { + return transCreateNodeInt(c, lit_bytes); + } - const cast_node = try c.createBuiltinCall("@as", 2); - cast_node.params()[0] = try transCreateNodeIdentifier(c, switch (tok.id.IntegerLiteral) { - .U => "c_uint", - .L => "c_long", - .LU => "c_ulong", - .LL => "c_longlong", - .LLU => "c_ulonglong", - else => unreachable, - }); - lit_bytes = lit_bytes[0 .. lit_bytes.len - switch (tok.id.IntegerLiteral) { - .U, .L => @as(u8, 1), - .LU, .LL => 2, - .LLU => 3, - else => unreachable, - }]; - _ = try appendToken(c, .Comma, ","); - cast_node.params()[1] = try transCreateNodeInt(c, lit_bytes); - cast_node.rparen_token = try appendToken(c, .RParen, ")"); - return &cast_node.base; - } else if (tok.id == .FloatLiteral) { - if (lit_bytes[0] == '.') - lit_bytes = try std.fmt.allocPrint(c.arena, "0{}", .{lit_bytes}); - if (tok.id.FloatLiteral == .None) { - return transCreateNodeFloat(c, lit_bytes); - } - const cast_node = try c.createBuiltinCall("@as", 2); - cast_node.params()[0] = try transCreateNodeIdentifier(c, switch (tok.id.FloatLiteral) { - .F => "f32", - .L => "c_longdouble", - else => unreachable, - }); - _ = try appendToken(c, .Comma, ","); - cast_node.params()[1] = try transCreateNodeFloat(c, lit_bytes[0 .. lit_bytes.len - 1]); - cast_node.rparen_token = try appendToken(c, .RParen, ")"); - return &cast_node.base; - } else unreachable; + const cast_node = try c.createBuiltinCall("@as", 2); + cast_node.params()[0] = try transCreateNodeIdentifier(c, switch (suffix) { + .u => "c_uint", + .l => "c_long", + .lu => "c_ulong", + .ll => "c_longlong", + .llu => "c_ulonglong", + else => unreachable, + }); + lit_bytes = lit_bytes[0 .. lit_bytes.len - switch (suffix) { + .u, .l => @as(u8, 1), + .lu, .ll => 2, + .llu => 3, + else => unreachable, + }]; + _ = try appendToken(c, .Comma, ","); + cast_node.params()[1] = try transCreateNodeInt(c, lit_bytes); + cast_node.rparen_token = try appendToken(c, .RParen, ")"); + return &cast_node.base; + }, + .FloatLiteral => |suffix| { + if (lit_bytes[0] == '.') + lit_bytes = try std.fmt.allocPrint(c.arena, "0{}", .{lit_bytes}); + if (suffix == .none) { + return transCreateNodeFloat(c, lit_bytes); + } + const cast_node = try c.createBuiltinCall("@as", 2); + cast_node.params()[0] = try transCreateNodeIdentifier(c, switch (suffix) { + .f => "f32", + .l => "c_longdouble", + else => unreachable, + }); + _ = try appendToken(c, .Comma, ","); + cast_node.params()[1] = try transCreateNodeFloat(c, lit_bytes[0 .. lit_bytes.len - 1]); + cast_node.rparen_token = try appendToken(c, .RParen, ")"); + return &cast_node.base; + }, + else => unreachable, + } } fn zigifyEscapeSequences(ctx: *Context, source_bytes: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ![]const u8 { @@ -5719,13 +5739,13 @@ fn zigifyEscapeSequences(ctx: *Context, source_bytes: []const u8, name: []const return bytes[0..i]; } -fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node { +fn parseCPrimaryExpr(c: *Context, it: *CTokIterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node { const tok = it.next().?; - switch (tok.id) { + const slice = it.slice(it.i); + switch (tok) { .CharLiteral => { - const first_tok = it.list.at(0); - if (source[tok.start] != '\'' or source[tok.start + 1] == '\\' or tok.end - tok.start == 3) { - const token = try appendToken(c, .CharLiteral, try zigifyEscapeSequences(c, source[tok.start..tok.end], source[first_tok.start..first_tok.end], source_loc)); + if (slice[0] != '\'' or slice[1] == '\\' or slice.len == 3) { + const token = try appendToken(c, .CharLiteral, try zigifyEscapeSequences(c, slice, it.slice(0), source_loc)); const node = try c.arena.create(ast.Node.OneToken); node.* = .{ .base = .{ .tag = .CharLiteral }, @@ -5733,7 +5753,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, }; return &node.base; } else { - const token = try appendTokenFmt(c, .IntegerLiteral, "0x{x}", .{source[tok.start + 1 .. tok.end - 1]}); + const token = try appendTokenFmt(c, .IntegerLiteral, "0x{x}", .{slice[1 .. slice.len - 1]}); const node = try c.arena.create(ast.Node.OneToken); node.* = .{ .base = .{ .tag = .IntegerLiteral }, @@ -5743,8 +5763,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, } }, .StringLiteral => { - const first_tok = it.list.at(0); - const token = try appendToken(c, .StringLiteral, try zigifyEscapeSequences(c, source[tok.start..tok.end], source[first_tok.start..first_tok.end], source_loc)); + const token = try appendToken(c, .StringLiteral, try zigifyEscapeSequences(c, slice, it.slice(0), source_loc)); const node = try c.arena.create(ast.Node.OneToken); node.* = .{ .base = .{ .tag = .StringLiteral }, @@ -5753,7 +5772,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, return &node.base; }, .IntegerLiteral, .FloatLiteral => { - return parseCNumLit(c, tok, source, source_loc); + return parseCNumLit(c, it, source_loc); }, // eventually this will be replaced by std.c.parse which will handle these correctly .Keyword_void => return transCreateNodeIdentifierUnchecked(c, "c_void"), @@ -5763,37 +5782,50 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, .Keyword_int => return transCreateNodeIdentifierUnchecked(c, "c_int"), .Keyword_float => return transCreateNodeIdentifierUnchecked(c, "f32"), .Keyword_short => return transCreateNodeIdentifierUnchecked(c, "c_short"), - .Keyword_char => return transCreateNodeIdentifierUnchecked(c, "c_char"), - .Keyword_unsigned => if (it.next()) |t| { - switch (t.id) { - .Keyword_short => return transCreateNodeIdentifierUnchecked(c, "c_ushort"), - .Keyword_int => return transCreateNodeIdentifierUnchecked(c, "c_uint"), - .Keyword_long => if (it.peek() != null and it.peek().?.id == .Keyword_long) { - _ = it.next(); - return transCreateNodeIdentifierUnchecked(c, "c_ulonglong"); - } else return transCreateNodeIdentifierUnchecked(c, "c_ulong"), - else => { - _ = it.prev(); - return transCreateNodeIdentifierUnchecked(c, "c_uint"); - }, - } + .Keyword_char => return transCreateNodeIdentifierUnchecked(c, "u8"), + .Keyword_unsigned => if (it.next()) |t| switch (t) { + .Keyword_char => return transCreateNodeIdentifierUnchecked(c, "u8"), + .Keyword_short => return transCreateNodeIdentifierUnchecked(c, "c_ushort"), + .Keyword_int => return transCreateNodeIdentifierUnchecked(c, "c_uint"), + .Keyword_long => if (it.peek() != null and it.peek().? == .Keyword_long) { + _ = it.next(); + return transCreateNodeIdentifierUnchecked(c, "c_ulonglong"); + } else return transCreateNodeIdentifierUnchecked(c, "c_ulong"), + else => { + it.i -= 1; + return transCreateNodeIdentifierUnchecked(c, "c_uint"); + }, } else { return transCreateNodeIdentifierUnchecked(c, "c_uint"); }, + .Keyword_signed => if (it.next()) |t| switch (t) { + .Keyword_char => return transCreateNodeIdentifierUnchecked(c, "i8"), + .Keyword_short => return transCreateNodeIdentifierUnchecked(c, "c_short"), + .Keyword_int => return transCreateNodeIdentifierUnchecked(c, "c_int"), + .Keyword_long => if (it.peek() != null and it.peek().? == .Keyword_long) { + _ = it.next(); + return transCreateNodeIdentifierUnchecked(c, "c_longlong"); + } else return transCreateNodeIdentifierUnchecked(c, "c_long"), + else => { + it.i -= 1; + return transCreateNodeIdentifierUnchecked(c, "c_int"); + }, + } else { + return transCreateNodeIdentifierUnchecked(c, "c_int"); + }, .Identifier => { - const mangled_name = scope.getAlias(source[tok.start..tok.end]); + const mangled_name = scope.getAlias(it.slice(it.i)); return transCreateNodeIdentifier(c, mangled_name); }, .LParen => { - const inner_node = try parseCExpr(c, it, source, source_loc, scope); + const inner_node = try parseCExpr(c, it, source_loc, scope); - const next_id = it.next().?.id; + const next_id = it.next().?; if (next_id != .RParen) { - const first_tok = it.list.at(0); try failDecl( c, source_loc, - source[first_tok.start..first_tok.end], + it.slice(0), "unable to translate C expr: expected ')'' instead got: {}", .{@tagName(next_id)}, ); @@ -5801,7 +5833,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, } var saw_l_paren = false; var saw_integer_literal = false; - switch (it.peek().?.id) { + switch (it.peek().?) { // (type)(to_cast) .LParen => { saw_l_paren = true; @@ -5819,14 +5851,13 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, // hack to get zig fmt to render a comma in builtin calls _ = try appendToken(c, .Comma, ","); - const node_to_cast = try parseCExpr(c, it, source, source_loc, scope); + const node_to_cast = try parseCExpr(c, it, source_loc, scope); - if (saw_l_paren and it.next().?.id != .RParen) { - const first_tok = it.list.at(0); + if (saw_l_paren and it.next().? != .RParen) { try failDecl( c, source_loc, - source[first_tok.start..first_tok.end], + it.slice(0), "unable to translate C expr: expected ')''", .{}, ); @@ -5857,13 +5888,12 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, return &group_node.base; }, else => { - const first_tok = it.list.at(0); try failDecl( c, source_loc, - source[first_tok.start..first_tok.end], + it.slice(0), "unable to translate C expr: unexpected token .{}", - .{@tagName(tok.id)}, + .{@tagName(tok)}, ); return error.ParseError; }, @@ -5971,61 +6001,52 @@ fn macroIntToBool(c: *Context, node: *ast.Node) !*ast.Node { return &group_node.base; } -fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node { - var node = try parseCPrimaryExpr(c, it, source, source_loc, scope); +fn parseCSuffixOpExpr(c: *Context, it: *CTokIterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node { + var node = try parseCPrimaryExpr(c, it, source_loc, scope); while (true) { - const tok = it.next().?; var op_token: ast.TokenIndex = undefined; var op_id: ast.Node.Tag = undefined; var bool_op = false; - switch (tok.id) { + switch (it.next().?) { .Period => { - const name_tok = it.next().?; - if (name_tok.id != .Identifier) { - const first_tok = it.list.at(0); + if (it.next().? != .Identifier) { try failDecl( c, source_loc, - source[first_tok.start..first_tok.end], + it.slice(0), "unable to translate C expr: expected identifier", .{}, ); return error.ParseError; } - node = try transCreateNodeFieldAccess(c, node, source[name_tok.start..name_tok.end]); + node = try transCreateNodeFieldAccess(c, node, it.slice(it.i)); continue; }, .Arrow => { - const name_tok = it.next().?; - if (name_tok.id != .Identifier) { - const first_tok = it.list.at(0); + if (it.next().? != .Identifier) { try failDecl( c, source_loc, - source[first_tok.start..first_tok.end], + it.slice(0), "unable to translate C expr: expected identifier", .{}, ); return error.ParseError; } const deref = try transCreateNodePtrDeref(c, node); - node = try transCreateNodeFieldAccess(c, deref, source[name_tok.start..name_tok.end]); + node = try transCreateNodeFieldAccess(c, deref, it.slice(it.i)); continue; }, .Asterisk => { - if (it.peek().?.id == .RParen) { + if (it.peek().? == .RParen) { // type *) // hack to get zig fmt to render a comma in builtin calls _ = try appendToken(c, .Comma, ","); - // * token - _ = it.prev(); // last token of `node` - const prev_id = it.prev().?.id; - _ = it.next(); - _ = it.next(); + const prev_id = it.list[it.i - 1].id; if (prev_id == .Keyword_void) { const ptr = try transCreateNodePtrType(c, false, false, .Asterisk); @@ -6096,15 +6117,14 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, }, .LBracket => { const arr_node = try transCreateNodeArrayAccess(c, node); - arr_node.index_expr = try parseCPrefixOpExpr(c, it, source, source_loc, scope); + arr_node.index_expr = try parseCPrefixOpExpr(c, it, source_loc, scope); arr_node.rtoken = try appendToken(c, .RBracket, "]"); node = &arr_node.base; - if (it.next().?.id != .RBracket) { - const first_tok = it.list.at(0); + if (it.next().? != .RBracket) { try failDecl( c, source_loc, - source[first_tok.start..first_tok.end], + it.slice(0), "unable to translate C expr: expected ']'", .{}, ); @@ -6117,23 +6137,21 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, var call_params = std.ArrayList(*ast.Node).init(c.gpa); defer call_params.deinit(); while (true) { - const arg = try parseCPrefixOpExpr(c, it, source, source_loc, scope); + const arg = try parseCPrefixOpExpr(c, it, source_loc, scope); try call_params.append(arg); - const next = it.next().?; - if (next.id == .Comma) - _ = try appendToken(c, .Comma, ",") - else if (next.id == .RParen) - break - else { - const first_tok = it.list.at(0); - try failDecl( - c, - source_loc, - source[first_tok.start..first_tok.end], - "unable to translate C expr: expected ',' or ')'", - .{}, - ); - return error.ParseError; + switch (it.next().?) { + .Comma => _ = try appendToken(c, .Comma, ","), + .RParen => break, + else => { + try failDecl( + c, + source_loc, + it.slice(0), + "unable to translate C expr: expected ',' or ')'", + .{}, + ); + return error.ParseError; + }, } } const call_node = try ast.Node.Call.alloc(c.arena, call_params.items.len); @@ -6158,23 +6176,21 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, defer init_vals.deinit(); while (true) { - const val = try parseCPrefixOpExpr(c, it, source, source_loc, scope); + const val = try parseCPrefixOpExpr(c, it, source_loc, scope); try init_vals.append(val); - const next = it.next().?; - if (next.id == .Comma) - _ = try appendToken(c, .Comma, ",") - else if (next.id == .RBrace) - break - else { - const first_tok = it.list.at(0); - try failDecl( - c, - source_loc, - source[first_tok.start..first_tok.end], - "unable to translate C expr: expected ',' or '}}'", - .{}, - ); - return error.ParseError; + switch (it.next().?) { + .Comma => _ = try appendToken(c, .Comma, ","), + .RBrace => break, + else => { + try failDecl( + c, + source_loc, + it.slice(0), + "unable to translate C expr: expected ',' or '}}'", + .{}, + ); + return error.ParseError; + }, } } const tuple_node = try ast.Node.StructInitializerDot.alloc(c.arena, init_vals.items.len); @@ -6221,22 +6237,22 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, op_id = .ArrayCat; op_token = try appendToken(c, .PlusPlus, "++"); - _ = it.prev(); + it.i -= 1; }, .Identifier => { op_id = .ArrayCat; op_token = try appendToken(c, .PlusPlus, "++"); - _ = it.prev(); + it.i -= 1; }, else => { - _ = it.prev(); + it.i -= 1; return node; }, } const cast_fn = if (bool_op) macroIntToBool else macroBoolToInt; const lhs_node = try cast_fn(c, node); - const rhs_node = try parseCPrefixOpExpr(c, it, source, source_loc, scope); + const rhs_node = try parseCPrefixOpExpr(c, it, source_loc, scope); const op_node = try c.arena.create(ast.Node.SimpleInfixOp); op_node.* = .{ .base = .{ .tag = op_id }, @@ -6248,38 +6264,36 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, } } -fn parseCPrefixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node { - const op_tok = it.next().?; - - switch (op_tok.id) { +fn parseCPrefixOpExpr(c: *Context, it: *CTokIterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node { + switch (it.next().?) { .Bang => { const node = try transCreateNodeSimplePrefixOp(c, .BoolNot, .Bang, "!"); - node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope); + node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope); return &node.base; }, .Minus => { const node = try transCreateNodeSimplePrefixOp(c, .Negation, .Minus, "-"); - node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope); + node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope); return &node.base; }, - .Plus => return try parseCPrefixOpExpr(c, it, source, source_loc, scope), + .Plus => return try parseCPrefixOpExpr(c, it, source_loc, scope), .Tilde => { const node = try transCreateNodeSimplePrefixOp(c, .BitNot, .Tilde, "~"); - node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope); + node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope); return &node.base; }, .Asterisk => { - const node = try parseCPrefixOpExpr(c, it, source, source_loc, scope); + const node = try parseCPrefixOpExpr(c, it, source_loc, scope); return try transCreateNodePtrDeref(c, node); }, .Ampersand => { const node = try transCreateNodeSimplePrefixOp(c, .AddressOf, .Ampersand, "&"); - node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope); + node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope); return &node.base; }, else => { - _ = it.prev(); - return try parseCSuffixOpExpr(c, it, source, source_loc, scope); + it.i -= 1; + return try parseCSuffixOpExpr(c, it, source_loc, scope); }, } } From fa4a9ab51fd41ccd2a43be52cb684e4aa64b50c6 Mon Sep 17 00:00:00 2001 From: frmdstryr Date: Sat, 25 Jul 2020 20:47:50 -0400 Subject: [PATCH 020/104] Use writer for LinearFifo instead of deprecated outStream --- lib/std/fifo.zig | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/std/fifo.zig b/lib/std/fifo.zig index 2dc1b23665..1edb413f47 100644 --- a/lib/std/fifo.zig +++ b/lib/std/fifo.zig @@ -224,6 +224,7 @@ pub fn LinearFifo( pub fn reader(self: *Self) std.io.Reader(*Self, error{}, readFn) { return .{ .context = self }; } + /// Deprecated: `use reader` pub fn inStream(self: *Self) std.io.InStream(*Self, error{}, readFn) { return .{ .context = self }; @@ -315,6 +316,11 @@ pub fn LinearFifo( return bytes.len; } + pub fn writer(self: *Self) std.io.Writer(*Self, error{OutOfMemory}, appendWrite) { + return .{ .context = self }; + } + + /// Deprecated: `use writer` pub fn outStream(self: *Self) std.io.OutStream(*Self, error{OutOfMemory}, appendWrite) { return .{ .context = self }; } @@ -426,14 +432,14 @@ test "LinearFifo(u8, .Dynamic)" { fifo.shrink(0); { - try fifo.outStream().print("{}, {}!", .{ "Hello", "World" }); + try fifo.writer().print("{}, {}!", .{ "Hello", "World" }); var result: [30]u8 = undefined; testing.expectEqualSlices(u8, "Hello, World!", result[0..fifo.read(&result)]); testing.expectEqual(@as(usize, 0), fifo.readableLength()); } { - try fifo.outStream().writeAll("This is a test"); + try fifo.writer().writeAll("This is a test"); var result: [30]u8 = undefined; testing.expectEqualSlices(u8, "This", (try fifo.reader().readUntilDelimiterOrEof(&result, ' ')).?); testing.expectEqualSlices(u8, "is", (try fifo.reader().readUntilDelimiterOrEof(&result, ' ')).?); From d1755e7f16879d678b0accb6a1ccfbe56654c3ab Mon Sep 17 00:00:00 2001 From: Ashok Gautham Date: Mon, 27 Jul 2020 19:40:55 +0530 Subject: [PATCH 021/104] Add meta viewport to fix mobile rendering, add missing block Co-authored-by: Vexu --- doc/langref.html.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 6180ab592f..941c2cc1a6 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -2,6 +2,7 @@ + Documentation - The Zig Programming Language