diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 6afe7750d3..a7442b8b25 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -1007,7 +1007,7 @@ pub const Tokenizer = struct { break; }, '|' => { - result.tag = .angle_bracket_angle_bracket_left_pipe; + state = .angle_bracket_angle_bracket_left_pipe; }, else => { result.tag = .angle_bracket_angle_bracket_left; @@ -2015,6 +2015,12 @@ test "tokenizer - invalid token with unfinished escape right before eof" { try testTokenize("'\\u", &.{.invalid}); } +test "tokenizer - saturating" { + try testTokenize("<<", &.{.angle_bracket_angle_bracket_left}); + try testTokenize("<<|", &.{.angle_bracket_angle_bracket_left_pipe}); + try testTokenize("<<|=", &.{.angle_bracket_angle_bracket_left_pipe_equal}); +} + fn testTokenize(source: [:0]const u8, expected_tokens: []const Token.Tag) !void { var tokenizer = Tokenizer.init(source); for (expected_tokens) |expected_token_id| { diff --git a/src/Air.zig b/src/Air.zig index b7d3938352..00f223ad21 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -44,7 +44,7 @@ pub const Inst = struct { /// is the same as both operands. /// Uses the `bin_op` field. addwrap, - /// Saturating integer addition. + /// Saturating integer addition. /// Both operands are guaranteed to be the same type, and the result type /// is the same as both operands. /// Uses the `bin_op` field. @@ -59,7 +59,7 @@ pub const Inst = struct { /// is the same as both operands. /// Uses the `bin_op` field. subwrap, - /// Saturating integer subtraction. + /// Saturating integer subtraction. /// Both operands are guaranteed to be the same type, and the result type /// is the same as both operands. /// Uses the `bin_op` field. @@ -74,7 +74,7 @@ pub const Inst = struct { /// is the same as both operands. /// Uses the `bin_op` field. mulwrap, - /// Saturating integer multiplication. + /// Saturating integer multiplication. /// Both operands are guaranteed to be the same type, and the result type /// is the same as both operands. /// Uses the `bin_op` field. diff --git a/src/stage1/tokenizer.cpp b/src/stage1/tokenizer.cpp index 3560193927..47e324c933 100644 --- a/src/stage1/tokenizer.cpp +++ b/src/stage1/tokenizer.cpp @@ -995,7 +995,6 @@ void tokenize(const char *source, Tokenization *out) { t.state = TokenizeState_start; break; case '|': - // t.out->ids.last() = TokenIdBitShiftLeftPipe; t.state = TokenizeState_angle_bracket_angle_bracket_left_pipe; break; default: