Big rework - Now use global ziql parser - still buggy, need to debug the tests
This commit is contained in:
@@ -0,0 +1,177 @@
|
||||
// From https://github.com/ziglang/zig/blob/master/lib/std/zig/tokenizer.zig
|
||||
const std = @import("std");
|
||||
|
||||
pub const Token = struct {
|
||||
tag: Tag,
|
||||
loc: Loc,
|
||||
|
||||
pub const Loc = struct {
|
||||
start: usize,
|
||||
end: usize,
|
||||
};
|
||||
|
||||
pub const keywords = std.StaticStringMap(Tag).initComptime(.{
|
||||
.{ "run", .keyword_run },
|
||||
.{ "help", .keyword_help },
|
||||
.{ "describe", .keyword_describe },
|
||||
.{ "build", .keyword_build },
|
||||
.{ "schema", .keyword_schema },
|
||||
.{ "quit", .keyword_quit },
|
||||
});
|
||||
|
||||
pub fn getKeyword(bytes: []const u8) ?Tag {
|
||||
return keywords.get(bytes);
|
||||
}
|
||||
|
||||
pub const Tag = enum {
|
||||
eof,
|
||||
invalid,
|
||||
|
||||
keyword_run,
|
||||
keyword_help,
|
||||
keyword_describe,
|
||||
keyword_schema,
|
||||
keyword_build,
|
||||
keyword_quit,
|
||||
|
||||
string_literal,
|
||||
identifier,
|
||||
};
|
||||
};
|
||||
|
||||
pub const Tokenizer = struct {
|
||||
buffer: [:0]const u8,
|
||||
index: usize,
|
||||
|
||||
pub fn init(buffer: [:0]const u8) Tokenizer {
|
||||
// Skip the UTF-8 BOM if present.
|
||||
return .{
|
||||
.buffer = buffer,
|
||||
.index = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0,
|
||||
};
|
||||
}
|
||||
|
||||
const State = enum {
|
||||
start,
|
||||
invalid,
|
||||
identifier,
|
||||
string_literal,
|
||||
string_literal_backslash,
|
||||
};
|
||||
|
||||
pub fn next(self: *Tokenizer) Token {
|
||||
var state: State = .start;
|
||||
var result: Token = .{
|
||||
.tag = undefined,
|
||||
.loc = .{
|
||||
.start = self.index,
|
||||
.end = undefined,
|
||||
},
|
||||
};
|
||||
while (true) : (self.index += 1) {
|
||||
const c = self.buffer[self.index];
|
||||
switch (state) {
|
||||
.start => switch (c) {
|
||||
0 => {
|
||||
if (self.index == self.buffer.len) return .{
|
||||
.tag = .eof,
|
||||
.loc = .{
|
||||
.start = self.index,
|
||||
.end = self.index,
|
||||
},
|
||||
};
|
||||
state = .invalid;
|
||||
},
|
||||
' ', '\n', '\t', '\r' => {
|
||||
result.loc.start = self.index + 1;
|
||||
},
|
||||
'a'...'z', 'A'...'Z', '_' => {
|
||||
state = .identifier;
|
||||
result.tag = .identifier;
|
||||
},
|
||||
'"' => {
|
||||
state = .string_literal;
|
||||
result.tag = .string_literal;
|
||||
},
|
||||
else => {
|
||||
state = .invalid;
|
||||
},
|
||||
},
|
||||
|
||||
.invalid => {
|
||||
result.tag = .invalid;
|
||||
break;
|
||||
},
|
||||
|
||||
.identifier => switch (c) {
|
||||
'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,
|
||||
else => {
|
||||
if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |tag| {
|
||||
result.tag = tag;
|
||||
}
|
||||
break;
|
||||
},
|
||||
},
|
||||
|
||||
.string_literal => switch (c) {
|
||||
0 => {
|
||||
if (self.index != self.buffer.len) {
|
||||
state = .invalid;
|
||||
continue;
|
||||
}
|
||||
result.tag = .invalid;
|
||||
break;
|
||||
},
|
||||
'\n' => {
|
||||
result.tag = .invalid;
|
||||
break;
|
||||
},
|
||||
'\\' => {
|
||||
state = .string_literal_backslash;
|
||||
},
|
||||
'"' => {
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
0x01...0x09, 0x0b...0x1f, 0x7f => {
|
||||
state = .invalid;
|
||||
},
|
||||
else => continue,
|
||||
},
|
||||
|
||||
.string_literal_backslash => switch (c) {
|
||||
0, '\n' => {
|
||||
result.tag = .invalid;
|
||||
break;
|
||||
},
|
||||
else => {
|
||||
state = .string_literal;
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
result.loc.end = self.index;
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
test "Basics" {
|
||||
try testTokenize("help", &.{.keyword_help});
|
||||
try testTokenize("run \"Hello world\"", &.{ .keyword_run, .string_literal });
|
||||
}
|
||||
|
||||
fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {
|
||||
var tokenizer = Tokenizer.init(source);
|
||||
for (expected_token_tags) |expected_token_tag| {
|
||||
const token = tokenizer.next();
|
||||
try std.testing.expectEqual(expected_token_tag, token.tag);
|
||||
}
|
||||
// Last token should always be eof, even when the last token was invalid,
|
||||
// in which case the tokenizer is in an invalid state, which can only be
|
||||
// recovered by opinionated means outside the scope of this implementation.
|
||||
const last_token = tokenizer.next();
|
||||
try std.testing.expectEqual(Token.Tag.eof, last_token.tag);
|
||||
try std.testing.expectEqual(source.len, last_token.loc.start);
|
||||
try std.testing.expectEqual(source.len, last_token.loc.end);
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
// From https://github.com/ziglang/zig/blob/master/lib/std/zig/tokenizer.zig
|
||||
const std = @import("std");
|
||||
|
||||
pub const Token = struct {
|
||||
tag: Tag,
|
||||
loc: Loc,
|
||||
|
||||
pub const Loc = struct {
|
||||
start: usize,
|
||||
end: usize,
|
||||
};
|
||||
|
||||
pub const types = std.StaticStringMap(Tag).initComptime(.{
|
||||
.{ "int", .type_int },
|
||||
.{ "float", .type_float },
|
||||
.{ "str", .type_str },
|
||||
.{ "date", .type_date },
|
||||
});
|
||||
|
||||
pub fn getType(bytes: []const u8) ?Tag {
|
||||
return types.get(bytes);
|
||||
}
|
||||
|
||||
pub const Tag = enum {
|
||||
eof,
|
||||
invalid,
|
||||
|
||||
type_int,
|
||||
type_float,
|
||||
type_str,
|
||||
type_date,
|
||||
|
||||
identifier,
|
||||
l_paren,
|
||||
r_paren,
|
||||
lr_bracket,
|
||||
comma,
|
||||
period,
|
||||
two_dot,
|
||||
};
|
||||
};
|
||||
|
||||
pub const Tokenizer = struct {
|
||||
buffer: [:0]const u8,
|
||||
index: usize,
|
||||
|
||||
pub fn init(buffer: [:0]const u8) Tokenizer {
|
||||
// Skip the UTF-8 BOM if present.
|
||||
return .{
|
||||
.buffer = buffer,
|
||||
.index = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0,
|
||||
};
|
||||
}
|
||||
|
||||
const State = enum {
|
||||
start,
|
||||
invalid,
|
||||
identifier,
|
||||
l_bracket,
|
||||
};
|
||||
|
||||
pub fn next(self: *Tokenizer) Token {
|
||||
var state: State = .start;
|
||||
var result: Token = .{
|
||||
.tag = undefined,
|
||||
.loc = .{
|
||||
.start = self.index,
|
||||
.end = undefined,
|
||||
},
|
||||
};
|
||||
while (true) : (self.index += 1) {
|
||||
const c = self.buffer[self.index];
|
||||
switch (state) {
|
||||
.start => switch (c) {
|
||||
0 => {
|
||||
if (self.index == self.buffer.len) return .{
|
||||
.tag = .eof,
|
||||
.loc = .{
|
||||
.start = self.index,
|
||||
.end = self.index,
|
||||
},
|
||||
};
|
||||
state = .invalid;
|
||||
},
|
||||
' ', '\n', '\t', '\r' => {
|
||||
result.loc.start = self.index + 1;
|
||||
},
|
||||
'a'...'z', 'A'...'Z', '_' => {
|
||||
state = .identifier;
|
||||
result.tag = .identifier;
|
||||
},
|
||||
'(' => {
|
||||
result.tag = .l_paren;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
')' => {
|
||||
result.tag = .r_paren;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'[' => {
|
||||
state = .l_bracket;
|
||||
},
|
||||
',' => {
|
||||
result.tag = .comma;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'.' => {
|
||||
result.tag = .period;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
':' => {
|
||||
result.tag = .two_dot;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
else => {
|
||||
state = .invalid;
|
||||
},
|
||||
},
|
||||
|
||||
.invalid => {
|
||||
// TODO make a better invalid handler
|
||||
@panic("Unknow char!!!");
|
||||
},
|
||||
|
||||
.identifier => switch (c) {
|
||||
'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,
|
||||
else => {
|
||||
if (Token.getType(self.buffer[result.loc.start..self.index])) |tag| {
|
||||
result.tag = tag;
|
||||
}
|
||||
break;
|
||||
},
|
||||
},
|
||||
|
||||
.l_bracket => switch (c) {
|
||||
']' => {
|
||||
result.tag = .lr_bracket;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
else => {
|
||||
state = .invalid;
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
result.loc.end = self.index;
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
test "keywords" {
|
||||
try testTokenize("int float str date", &.{ .type_int, .type_float, .type_str, .type_date });
|
||||
}
|
||||
|
||||
test "basic query" {
|
||||
try testTokenize("User ()", &.{ .identifier, .l_paren, .r_paren });
|
||||
try testTokenize("User (name:str)", &.{ .identifier, .l_paren, .identifier, .two_dot, .type_str, .r_paren });
|
||||
try testTokenize("User (name:str, email: str, messages:[]Message.from)", &.{
|
||||
.identifier,
|
||||
.l_paren,
|
||||
.identifier,
|
||||
.two_dot,
|
||||
.type_str,
|
||||
.comma,
|
||||
.identifier,
|
||||
.two_dot,
|
||||
.type_str,
|
||||
.comma,
|
||||
.identifier,
|
||||
.two_dot,
|
||||
.lr_bracket,
|
||||
.identifier,
|
||||
.period,
|
||||
.identifier,
|
||||
.r_paren,
|
||||
});
|
||||
}
|
||||
|
||||
fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {
|
||||
var tokenizer = Tokenizer.init(source);
|
||||
for (expected_token_tags) |expected_token_tag| {
|
||||
const token = tokenizer.next();
|
||||
try std.testing.expectEqual(expected_token_tag, token.tag);
|
||||
}
|
||||
// Last token should always be eof, even when the last token was invalid,
|
||||
// in which case the tokenizer is in an invalid state, which can only be
|
||||
// recovered by opinionated means outside the scope of this implementation.
|
||||
const last_token = tokenizer.next();
|
||||
try std.testing.expectEqual(Token.Tag.eof, last_token.tag);
|
||||
try std.testing.expectEqual(source.len, last_token.loc.start);
|
||||
try std.testing.expectEqual(source.len, last_token.loc.end);
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
// From https://github.com/ziglang/zig/blob/master/lib/std/zig/tokenizer.zig
|
||||
const std = @import("std");
|
||||
|
||||
pub const Token = struct {
|
||||
tag: Tag,
|
||||
loc: Loc,
|
||||
|
||||
pub const Loc = struct {
|
||||
start: usize,
|
||||
end: usize,
|
||||
};
|
||||
|
||||
pub const keywords = std.StaticStringMap(Tag).initComptime(.{
|
||||
.{ "GRAB", .keyword_grab },
|
||||
.{ "UPDATE", .keyword_update },
|
||||
.{ "DELETE", .keyword_delete },
|
||||
.{ "ADD", .keyword_add },
|
||||
.{ "IN", .keyword_in },
|
||||
.{ "null", .keyword_null },
|
||||
.{ "__DESCRIBE__", .keyword__describe__ },
|
||||
.{ "__INIT__", .keyword__init__ },
|
||||
.{ "true", .bool_literal_true },
|
||||
.{ "false", .bool_literal_false },
|
||||
.{ "AND", .keyword_and },
|
||||
.{ "OR", .keyword_or },
|
||||
});
|
||||
|
||||
pub fn getKeyword(bytes: []const u8) ?Tag {
|
||||
return keywords.get(bytes);
|
||||
}
|
||||
|
||||
pub const Tag = enum {
|
||||
eof,
|
||||
invalid,
|
||||
|
||||
keyword_grab,
|
||||
keyword_update,
|
||||
keyword_delete,
|
||||
keyword_add,
|
||||
keyword_in,
|
||||
keyword_null,
|
||||
keyword_and,
|
||||
keyword_or,
|
||||
keyword__describe__,
|
||||
keyword__init__,
|
||||
|
||||
string_literal,
|
||||
int_literal,
|
||||
float_literal,
|
||||
bool_literal_true,
|
||||
bool_literal_false,
|
||||
identifier,
|
||||
equal,
|
||||
bang, // !
|
||||
pipe, // |
|
||||
l_paren, // (
|
||||
r_paren, // )
|
||||
l_bracket, // [
|
||||
r_bracket, // ]
|
||||
l_brace, // {
|
||||
r_brace, // }
|
||||
semicolon, // ;
|
||||
comma, // ,
|
||||
angle_bracket_left, // <
|
||||
angle_bracket_right, // >
|
||||
angle_bracket_left_equal, // <=
|
||||
angle_bracket_right_equal, // >=
|
||||
equal_angle_bracket_right, // =>
|
||||
period, // .
|
||||
bang_equal, // !=
|
||||
};
|
||||
};
|
||||
|
||||
pub const Tokenizer = struct {
|
||||
buffer: [:0]const u8,
|
||||
index: usize,
|
||||
|
||||
pub fn getTokenSlice(self: *Tokenizer, token: Token) []const u8 {
|
||||
return self.buffer[token.loc.start..token.loc.end];
|
||||
}
|
||||
|
||||
pub fn init(buffer: [:0]const u8) Tokenizer {
|
||||
// Skip the UTF-8 BOM if present.
|
||||
return .{
|
||||
.buffer = buffer,
|
||||
.index = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0,
|
||||
};
|
||||
}
|
||||
|
||||
const State = enum {
|
||||
start,
|
||||
invalid,
|
||||
string_literal,
|
||||
identifier,
|
||||
equal,
|
||||
bang,
|
||||
angle_bracket_left,
|
||||
angle_bracket_right,
|
||||
string_literal_backslash,
|
||||
int_exponent,
|
||||
float,
|
||||
float_exponent,
|
||||
int,
|
||||
};
|
||||
|
||||
pub fn next(self: *Tokenizer) Token {
|
||||
var state: State = .start;
|
||||
var result: Token = .{
|
||||
.tag = undefined,
|
||||
.loc = .{
|
||||
.start = self.index,
|
||||
.end = undefined,
|
||||
},
|
||||
};
|
||||
while (true) : (self.index += 1) {
|
||||
const c = self.buffer[self.index];
|
||||
switch (state) {
|
||||
.start => switch (c) {
|
||||
0 => {
|
||||
if (self.index == self.buffer.len) return .{
|
||||
.tag = .eof,
|
||||
.loc = .{
|
||||
.start = self.index,
|
||||
.end = self.index,
|
||||
},
|
||||
};
|
||||
state = .invalid;
|
||||
},
|
||||
' ', '\n', '\t', '\r' => {
|
||||
result.loc.start = self.index + 1;
|
||||
},
|
||||
'\'' => {
|
||||
state = .string_literal;
|
||||
result.tag = .string_literal;
|
||||
},
|
||||
'a'...'z', 'A'...'Z', '_' => {
|
||||
state = .identifier;
|
||||
result.tag = .identifier;
|
||||
},
|
||||
'=' => {
|
||||
state = .equal;
|
||||
},
|
||||
'!' => {
|
||||
state = .bang;
|
||||
},
|
||||
'|' => {
|
||||
result.tag = .pipe;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'(' => {
|
||||
result.tag = .l_paren;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
')' => {
|
||||
result.tag = .r_paren;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'[' => {
|
||||
result.tag = .l_bracket;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
']' => {
|
||||
result.tag = .r_bracket;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
';' => {
|
||||
result.tag = .semicolon;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
',' => {
|
||||
result.tag = .comma;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'<' => {
|
||||
state = .angle_bracket_left;
|
||||
},
|
||||
'>' => {
|
||||
state = .angle_bracket_right;
|
||||
},
|
||||
'{' => {
|
||||
result.tag = .l_brace;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'}' => {
|
||||
result.tag = .r_brace;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'.' => {
|
||||
state = .float;
|
||||
result.tag = .float_literal;
|
||||
},
|
||||
'0'...'9' => {
|
||||
state = .int;
|
||||
result.tag = .int_literal;
|
||||
},
|
||||
else => {
|
||||
state = .invalid;
|
||||
},
|
||||
},
|
||||
|
||||
.invalid => {
|
||||
// TODO make a better invalid handler
|
||||
@panic("Unknow char!!!");
|
||||
},
|
||||
|
||||
.identifier => switch (c) {
|
||||
'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,
|
||||
else => {
|
||||
if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |tag| {
|
||||
result.tag = tag;
|
||||
} else {
|
||||
result.tag = .identifier;
|
||||
}
|
||||
break;
|
||||
},
|
||||
},
|
||||
|
||||
.string_literal => switch (c) {
|
||||
0 => {
|
||||
if (self.index != self.buffer.len) {
|
||||
state = .invalid;
|
||||
continue;
|
||||
}
|
||||
result.tag = .invalid;
|
||||
break;
|
||||
},
|
||||
'\n' => {
|
||||
result.tag = .invalid;
|
||||
break;
|
||||
},
|
||||
'\\' => {
|
||||
state = .string_literal_backslash;
|
||||
},
|
||||
'\'' => {
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
0x01...0x09, 0x0b...0x1f, 0x7f => {
|
||||
state = .invalid;
|
||||
},
|
||||
else => continue,
|
||||
},
|
||||
|
||||
.string_literal_backslash => switch (c) {
|
||||
0, '\n' => {
|
||||
result.tag = .invalid;
|
||||
break;
|
||||
},
|
||||
else => {
|
||||
state = .string_literal;
|
||||
},
|
||||
},
|
||||
|
||||
.bang => switch (c) {
|
||||
'=' => {
|
||||
result.tag = .bang_equal;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
//TODO Add the !IN
|
||||
else => {
|
||||
result.tag = .bang;
|
||||
break;
|
||||
},
|
||||
},
|
||||
|
||||
.equal => switch (c) {
|
||||
'>' => {
|
||||
result.tag = .equal_angle_bracket_right;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
else => {
|
||||
result.tag = .equal;
|
||||
break;
|
||||
},
|
||||
},
|
||||
|
||||
.angle_bracket_left => switch (c) {
|
||||
'=' => {
|
||||
result.tag = .angle_bracket_left_equal;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
else => {
|
||||
result.tag = .angle_bracket_left;
|
||||
break;
|
||||
},
|
||||
},
|
||||
|
||||
.angle_bracket_right => switch (c) {
|
||||
'=' => {
|
||||
result.tag = .angle_bracket_right_equal;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
else => {
|
||||
result.tag = .angle_bracket_right;
|
||||
break;
|
||||
},
|
||||
},
|
||||
|
||||
.int => switch (c) {
|
||||
'.' => {
|
||||
state = .float;
|
||||
result.tag = .float_literal;
|
||||
},
|
||||
'e', 'E' => {
|
||||
state = .int_exponent;
|
||||
result.tag = .float_literal;
|
||||
},
|
||||
'_', '0'...'9' => continue,
|
||||
else => break,
|
||||
},
|
||||
.int_exponent => switch (c) {
|
||||
'+', '-', '0'...'9' => {
|
||||
state = .float;
|
||||
},
|
||||
else => {
|
||||
self.index -= 1;
|
||||
break;
|
||||
},
|
||||
},
|
||||
.float => switch (c) {
|
||||
'e', 'E' => {
|
||||
state = .float_exponent;
|
||||
},
|
||||
'_', '0'...'9' => {
|
||||
continue;
|
||||
},
|
||||
else => {
|
||||
break;
|
||||
},
|
||||
},
|
||||
.float_exponent => switch (c) {
|
||||
'+', '-', '0'...'9' => {
|
||||
continue;
|
||||
},
|
||||
else => {
|
||||
self.index -= 1;
|
||||
break;
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
result.loc.end = self.index;
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
test "keywords" {
|
||||
try testTokenize("GRAB UPDATE ADD DELETE IN", &.{ .keyword_grab, .keyword_update, .keyword_add, .keyword_delete, .keyword_in });
|
||||
}
|
||||
|
||||
test "basic query" {
|
||||
try testTokenize("GRAB User {}", &.{ .keyword_grab, .identifier, .l_brace, .r_brace });
|
||||
try testTokenize("GRAB User { name = 'Adrien'}", &.{ .keyword_grab, .identifier, .l_brace, .identifier, .equal, .string_literal, .r_brace });
|
||||
try testTokenize("GRAB User { age = 1.5}", &.{ .keyword_grab, .identifier, .l_brace, .identifier, .equal, .float_literal, .r_brace });
|
||||
try testTokenize("GRAB User { admin = true}", &.{ .keyword_grab, .identifier, .l_brace, .identifier, .equal, .bool_literal_true, .r_brace });
|
||||
try testTokenize("GRAB User [1; name] {}", &.{ .keyword_grab, .identifier, .l_bracket, .int_literal, .semicolon, .identifier, .r_bracket, .l_brace, .r_brace });
|
||||
try testTokenize("GRAB User{}|ASCENDING name|", &.{ .keyword_grab, .identifier, .l_brace, .r_brace, .pipe, .identifier, .identifier, .pipe });
|
||||
try testTokenize("DELETE User[1]{name='Adrien'}|ASCENDING name, age|", &.{ .keyword_delete, .identifier, .l_bracket, .int_literal, .r_bracket, .l_brace, .identifier, .equal, .string_literal, .r_brace, .pipe, .identifier, .identifier, .comma, .identifier, .pipe });
|
||||
}
|
||||
|
||||
fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {
|
||||
var tokenizer = Tokenizer.init(source);
|
||||
for (expected_token_tags) |expected_token_tag| {
|
||||
const token = tokenizer.next();
|
||||
try std.testing.expectEqual(expected_token_tag, token.tag);
|
||||
}
|
||||
// Last token should always be eof, even when the last token was invalid,
|
||||
// in which case the tokenizer is in an invalid state, which can only be
|
||||
// recovered by opinionated means outside the scope of this implementation.
|
||||
const last_token = tokenizer.next();
|
||||
try std.testing.expectEqual(Token.Tag.eof, last_token.tag);
|
||||
try std.testing.expectEqual(source.len, last_token.loc.start);
|
||||
try std.testing.expectEqual(source.len, last_token.loc.end);
|
||||
}
|
||||
Reference in New Issue
Block a user