mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2026-01-21 23:05:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

sat-arithmetic: correctly tokenize <<|, <<|=

Browse Source 
- set state rather than result.tag in tokenizer.zig
- add test to tokenizer.zig for <<, <<|, <<|=
This commit is contained in:
Travis Staloch 2021-09-08 15:58:37 -07:00 committed by Andrew Kelley
parent b9a95f2dd9
commit dcbc52ec85
3 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lib/std/zig/tokenizer.zig
View File

@ -1007,7 +1007,7 @@ pub const Tokenizer = struct {
break;
},
'|' => {
result.tag = .angle_bracket_angle_bracket_left_pipe;
state = .angle_bracket_angle_bracket_left_pipe;
},
else => {
result.tag = .angle_bracket_angle_bracket_left;
@ -2015,6 +2015,12 @@ test "tokenizer - invalid token with unfinished escape right before eof" {
try testTokenize("'\\u", &.{.invalid});
}
test "tokenizer - saturating" {
try testTokenize("<<", &.{.angle_bracket_angle_bracket_left});
try testTokenize("<<|", &.{.angle_bracket_angle_bracket_left_pipe});
try testTokenize("<<|=", &.{.angle_bracket_angle_bracket_left_pipe_equal});
}
fn testTokenize(source: [:0]const u8, expected_tokens: []const Token.Tag) !void {
var tokenizer = Tokenizer.init(source);
for (expected_tokens) |expected_token_id| {

6
src/Air.zig
View File

@ -44,7 +44,7 @@ pub const Inst = struct {
/// is the same as both operands.
/// Uses the `bin_op` field.
addwrap,
/// Saturating integer addition.
/// Saturating integer addition.
/// Both operands are guaranteed to be the same type, and the result type
/// is the same as both operands.
/// Uses the `bin_op` field.
@ -59,7 +59,7 @@ pub const Inst = struct {
/// is the same as both operands.
/// Uses the `bin_op` field.
subwrap,
/// Saturating integer subtraction.
/// Saturating integer subtraction.
/// Both operands are guaranteed to be the same type, and the result type
/// is the same as both operands.
/// Uses the `bin_op` field.
@ -74,7 +74,7 @@ pub const Inst = struct {
/// is the same as both operands.
/// Uses the `bin_op` field.
mulwrap,
/// Saturating integer multiplication.
/// Saturating integer multiplication.
/// Both operands are guaranteed to be the same type, and the result type
/// is the same as both operands.
/// Uses the `bin_op` field.

1
src/stage1/tokenizer.cpp
View File

@ -995,7 +995,6 @@ void tokenize(const char *source, Tokenization *out) {
t.state = TokenizeState_start;
break;
case '|':
// t.out->ids.last() = TokenIdBitShiftLeftPipe;
t.state = TokenizeState_angle_bracket_angle_bracket_left_pipe;
break;
default: