mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2025-12-06 06:13:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

validate number literals in AstGen

Browse Source
This commit is contained in:
Veikka Tuominen 2022-08-31 13:36:48 +03:00 committed by Andrew Kelley
parent 716d9237cb
commit 349d78a443
34 changed files with 592 additions and 652 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/docgen.zig
View File

@ -1057,9 +1057,7 @@ fn tokenizeAndPrintRaw(
}
},
.integer_literal,
.float_literal,
=> {
.number_literal => {
try out.writeAll("<span class=\"tok-number\">");
try writeEscaped(out, src[token.loc.start..token.loc.end]);
try out.writeAll("</span>");

2
lib/std/zig.zig
View File

@ -10,6 +10,7 @@ pub const fmtEscapes = fmt.fmtEscapes;
pub const isValidId = fmt.isValidId;
pub const parse = @import("zig/parse.zig").parse;
pub const string_literal = @import("zig/string_literal.zig");
pub const number_literal = @import("zig/number_literal.zig");
pub const Ast = @import("zig/Ast.zig");
pub const system = @import("zig/system.zig");
pub const CrossTarget = @import("zig/CrossTarget.zig");
@ -17,6 +18,7 @@ pub const CrossTarget = @import("zig/CrossTarget.zig");
// Character literal parsing
pub const ParsedCharLiteral = string_literal.ParsedCharLiteral;
pub const parseCharLiteral = string_literal.parseCharLiteral;
pub const parseNumberLiteral = number_literal.parseNumberLiteral;
// Files needed by translate-c.
pub const c_builtins = @import("zig/c_builtins.zig");

10
lib/std/zig/Ast.zig
View File

@ -406,8 +406,7 @@ pub fn firstToken(tree: Ast, node: Node.Index) TokenIndex {
.identifier,
.anyframe_literal,
.char_literal,
.integer_literal,
.float_literal,
.number_literal,
.unreachable_literal,
.string_literal,
.multiline_string_literal,
@ -781,8 +780,7 @@ pub fn lastToken(tree: Ast, node: Node.Index) TokenIndex {
.anyframe_literal,
.char_literal,
.integer_literal,
.float_literal,
.number_literal,
.unreachable_literal,
.identifier,
.deref,
@ -2919,9 +2917,7 @@ pub const Node = struct {
/// Both lhs and rhs unused.
char_literal,
/// Both lhs and rhs unused.
integer_literal,
/// Both lhs and rhs unused.
float_literal,
number_literal,
/// Both lhs and rhs unused.
unreachable_literal,
/// Both lhs and rhs unused.

167
lib/std/zig/number_literal.zig Normal file
View File

@ -0,0 +1,167 @@
const std = @import("../std.zig");
const assert = std.debug.assert;
const utf8Decode = std.unicode.utf8Decode;
const utf8Encode = std.unicode.utf8Encode;
pub const ParseError = error{
OutOfMemory,
InvalidLiteral,
};
pub const Base = enum(u8) { decimal = 10, hex = 16, binary = 2, octal = 8 };
pub const FloatBase = enum(u8) { decimal = 10, hex = 16 };
pub const Result = union(enum) {
/// Result fits if it fits in u64
int: u64,
/// Result is an int that doesn't fit in u64. Payload is the base, if it is
/// not `.decimal` then the slice has a two character prefix.
big_int: Base,
/// Result is a float. Payload is the base, if it is not `.decimal` then
/// the slice has a two character prefix.
float: FloatBase,
failure: Error,
};
pub const Error = union(enum) {
/// The number has leading zeroes.
leading_zero,
/// Expected a digit after base prefix.
digit_after_base,
/// The base prefix is in uppercase.
upper_case_base: usize,
/// Float literal has an invalid base prefix.
invalid_float_base: usize,
/// Repeated '_' digit separator.
repeated_underscore: usize,
/// '_' digit separator after special character (+-.)
invalid_underscore_after_special: usize,
/// Invalid digit for the specified base.
invalid_digit: struct { i: usize, base: Base },
/// Invalid digit for an exponent.
invalid_digit_exponent: usize,
/// Float literal has multiple periods.
duplicate_period,
/// Float literal has multiple exponents.
duplicate_exponent: usize,
/// Decimal float has hexadecimal exponent.
invalid_hex_exponent: usize,
/// Exponent comes directly after '_' digit separator.
exponent_after_underscore: usize,
/// Special character (+-.) comes directly after exponent.
special_after_underscore: usize,
/// Number ends in special character (+-.)
trailing_special: usize,
/// Number ends in '_' digit separator.
trailing_underscore: usize,
/// Character not in [0-9a-zA-Z.+-_]
invalid_character: usize,
/// [+-] not immediately after [pPeE]
invalid_exponent_sign: usize,
};
/// Parse Zig number literal accepted by fmt.parseInt, fmt.parseFloat and big_int.setString.
/// Valid for any input.
pub fn parseNumberLiteral(bytes: []const u8) Result {
var i: usize = 0;
var base: u8 = 10;
if (bytes.len >= 2 and bytes[0] == '0') switch (bytes[1]) {
'b' => {
base = 2;
i = 2;
},
'o' => {
base = 8;
i = 2;
},
'x' => {
base = 16;
i = 2;
},
'B', 'O', 'X' => return .{ .failure = .{ .upper_case_base = 1 } },
'.', 'e', 'E' => {},
else => return .{ .failure = .leading_zero },
};
if (bytes.len == 2 and base != 10) return .{ .failure = .digit_after_base };
var x: u64 = 0;
var overflow = false;
var underscore = false;
var period = false;
var special: u8 = 0;
var exponent = false;
var float = false;
while (i < bytes.len) : (i += 1) {
const c = bytes[i];
switch (c) {
'_' => {
if (i == 2 and base != 10) return .{ .failure = .{ .invalid_underscore_after_special = i } };
if (special != 0) return .{ .failure = .{ .invalid_underscore_after_special = i } };
if (underscore) return .{ .failure = .{ .repeated_underscore = i } };
underscore = true;
continue;
},
'e', 'E' => if (base == 10) {
float = true;
if (base != 10 and base != 16) return .{ .failure = .{ .invalid_float_base = 2 } };
if (exponent) return .{ .failure = .{ .duplicate_exponent = i } };
if (underscore) return .{ .failure = .{ .exponent_after_underscore = i } };
special = c;
exponent = true;
continue;
},
'p', 'P' => if (base == 16) {
float = true;
if (base != 10 and base != 16) return .{ .failure = .{ .invalid_float_base = 2 } };
if (exponent) return .{ .failure = .{ .duplicate_exponent = i } };
if (underscore) return .{ .failure = .{ .exponent_after_underscore = i } };
if (base != 16) return .{ .failure = .{ .invalid_hex_exponent = i } };
special = c;
exponent = true;
continue;
},
'.' => {
float = true;
if (base != 10 and base != 16) return .{ .failure = .{ .invalid_float_base = 2 } };
if (period) return .{ .failure = .{ .duplicate_exponent = i } };
period = true;
if (underscore) return .{ .failure = .{ .special_after_underscore = i } };
special = c;
continue;
},
'+', '-' => {
switch (special) {
'p', 'P', 'e', 'E' => {},
else => return .{ .failure = .{ .invalid_exponent_sign = i } },
}
special = c;
continue;
},
else => {},
}
const digit = switch (c) {
'0'...'9' => c - '0',
'A'...'Z' => c - 'A' + 10,
'a'...'z' => c - 'a' + 10,
else => return .{ .failure = .{ .invalid_character = i } },
};
if (digit >= base) return .{ .failure = .{ .invalid_digit = .{ .i = i, .base = @intToEnum(Base, base) } } };
if (exponent and digit >= 10) return .{ .failure = .{ .invalid_digit_exponent = i } };
underscore = false;
special = 0;
if (float) continue;
if (x != 0) if (@mulWithOverflow(u64, x, base, &x)) {
overflow = true;
};
if (@addWithOverflow(u64, x, digit, &x)) {
overflow = true;
}
}
if (underscore) return .{ .failure = .{ .trailing_underscore = bytes.len - 1 } };
if (special != 0) return .{ .failure = .{ .trailing_special = bytes.len - 1 } };
if (float) return .{ .float = @intToEnum(FloatBase, base) };
if (overflow) return .{ .big_int = @intToEnum(Base, base) };
return .{ .int = x };
}

12
lib/std/zig/parse.zig
View File

@ -2401,16 +2401,8 @@ const Parser = struct {
.rhs = undefined,
},
}),
.integer_literal => return p.addNode(.{
.tag = .integer_literal,
.main_token = p.nextToken(),
.data = .{
.lhs = undefined,
.rhs = undefined,
},
}),
.float_literal => return p.addNode(.{
.tag = .float_literal,
.number_literal => return p.addNode(.{
.tag = .number_literal,
.main_token = p.nextToken(),
.data = .{
.lhs = undefined,

3
lib/std/zig/render.zig
View File

@ -199,8 +199,7 @@ fn renderExpression(gpa: Allocator, ais: *Ais, tree: Ast, node: Ast.Node.Index,
return renderSpace(ais, tree, token_index, lexeme.len, space);
},
.integer_literal,
.float_literal,
.number_literal,
.char_literal,
.unreachable_literal,
.anyframe_literal,

716
lib/std/zig/tokenizer.zig
View File

@ -136,8 +136,7 @@ pub const Token = struct {
angle_bracket_angle_bracket_right,
angle_bracket_angle_bracket_right_equal,
tilde,
integer_literal,
float_literal,
number_literal,
doc_comment,
container_doc_comment,
keyword_addrspace,
@ -199,8 +198,7 @@ pub const Token = struct {
.char_literal,
.eof,
.builtin,
.integer_literal,
.float_literal,
.number_literal,
.doc_comment,
.container_doc_comment,
=> null,
@ -328,8 +326,7 @@ pub const Token = struct {
.char_literal => "a character literal",
.eof => "EOF",
.builtin => "a builtin function",
.integer_literal => "an integer literal",
.float_literal => "a floating point literal",
.number_literal => "a number literal",
.doc_comment, .container_doc_comment => "a document comment",
else => unreachable,
};
@ -387,24 +384,11 @@ pub const Tokenizer = struct {
line_comment,
doc_comment_start,
doc_comment,
zero,
int_literal_dec,
int_literal_dec_no_underscore,
int_literal_bin,
int_literal_bin_no_underscore,
int_literal_oct,
int_literal_oct_no_underscore,
int_literal_hex,
int_literal_hex_no_underscore,
num_dot_dec,
num_dot_hex,
float_fraction_dec,
float_fraction_dec_no_underscore,
float_fraction_hex,
float_fraction_hex_no_underscore,
float_exponent_unsigned,
float_exponent_num,
float_exponent_num_no_underscore,
int,
int_exponent,
int_period,
float,
float_exponent,
ampersand,
caret,
percent,
@ -557,13 +541,9 @@ pub const Tokenizer = struct {
'&' => {
state = .ampersand;
},
'0' => {
state = .zero;
result.tag = .integer_literal;
},
'1'...'9' => {
state = .int_literal_dec;
result.tag = .integer_literal;
'0'...'9' => {
state = .int;
result.tag = .number_literal;
},
else => {
result.tag = .invalid;
@ -1175,232 +1155,42 @@ pub const Tokenizer = struct {
'\t', '\r' => {},
else => self.checkLiteralCharacter(),
},
.zero => switch (c) {
'b' => {
state = .int_literal_bin_no_underscore;
.int => switch (c) {
'.' => state = .int_period,
'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {},
'e', 'E', 'p', 'P' => state = .int_exponent,
else => break,
},
.int_exponent => switch (c) {
'-', '+' => {
state = .float;
},
'o' => {
state = .int_literal_oct_no_underscore;
},
'x' => {
state = .int_literal_hex_no_underscore;
},
'0'...'9', '_', '.', 'e', 'E' => {
// reinterpret as a decimal number
else => {
self.index -= 1;
state = .int_literal_dec;
state = .int;
},
'a', 'c', 'd', 'f'...'n', 'p'...'w', 'y', 'z', 'A'...'D', 'F'...'Z' => {
result.tag = .invalid;
break;
},
else => break,
},
.int_literal_bin_no_underscore => switch (c) {
'0'...'1' => {
state = .int_literal_bin;
.int_period => switch (c) {
'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {
state = .float;
},
'e', 'E', 'p', 'P' => state = .float_exponent,
else => {
result.tag = .invalid;
break;
},
},
.int_literal_bin => switch (c) {
'_' => {
state = .int_literal_bin_no_underscore;
},
'0'...'1' => {},
'2'...'9', 'a'...'z', 'A'...'Z' => {
result.tag = .invalid;
break;
},
else => break,
},
.int_literal_oct_no_underscore => switch (c) {
'0'...'7' => {
state = .int_literal_oct;
},
else => {
result.tag = .invalid;
break;
},
},
.int_literal_oct => switch (c) {
'_' => {
state = .int_literal_oct_no_underscore;
},
'0'...'7' => {},
'8', '9', 'a'...'z', 'A'...'Z' => {
result.tag = .invalid;
break;
},
else => break,
},
.int_literal_dec_no_underscore => switch (c) {
'0'...'9' => {
state = .int_literal_dec;
},
else => {
result.tag = .invalid;
break;
},
},
.int_literal_dec => switch (c) {
'_' => {
state = .int_literal_dec_no_underscore;
},
'.' => {
state = .num_dot_dec;
result.tag = .invalid;
},
'e', 'E' => {
state = .float_exponent_unsigned;
result.tag = .float_literal;
},
'0'...'9' => {},
'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {
result.tag = .invalid;
break;
},
else => break,
},
.int_literal_hex_no_underscore => switch (c) {
'0'...'9', 'a'...'f', 'A'...'F' => {
state = .int_literal_hex;
},
else => {
result.tag = .invalid;
break;
},
},
.int_literal_hex => switch (c) {
'_' => {
state = .int_literal_hex_no_underscore;
},
'.' => {
state = .num_dot_hex;
result.tag = .invalid;
},
'p', 'P' => {
state = .float_exponent_unsigned;
result.tag = .float_literal;
},
'0'...'9', 'a'...'f', 'A'...'F' => {},
'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {
result.tag = .invalid;
break;
},
else => break,
},
.num_dot_dec => switch (c) {
'.' => {
result.tag = .integer_literal;
self.index -= 1;
state = .start;
break;
},
'0'...'9' => {
result.tag = .float_literal;
state = .float_fraction_dec;
},
'_', 'a'...'z', 'A'...'Z' => {
result.tag = .invalid;
break;
},
},
.float => switch (c) {
'_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {},
'e', 'E', 'p', 'P' => state = .float_exponent,
else => break,
},
.num_dot_hex => switch (c) {
'.' => {
result.tag = .integer_literal;
.float_exponent => switch (c) {
'-', '+' => state = .float,
else => {
self.index -= 1;
state = .start;
break;
state = .float;
},
'0'...'9', 'a'...'f', 'A'...'F' => {
result.tag = .float_literal;
state = .float_fraction_hex;
},
'_', 'g'...'z', 'G'...'Z' => {
result.tag = .invalid;
break;
},
else => break,
},
.float_fraction_dec_no_underscore => switch (c) {
'0'...'9' => {
state = .float_fraction_dec;
},
else => {
result.tag = .invalid;
break;
},
},
.float_fraction_dec => switch (c) {
'_' => {
state = .float_fraction_dec_no_underscore;
},
'e', 'E' => {
state = .float_exponent_unsigned;
},
'0'...'9' => {},
'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {
result.tag = .invalid;
break;
},
else => break,
},
.float_fraction_hex_no_underscore => switch (c) {
'0'...'9', 'a'...'f', 'A'...'F' => {
state = .float_fraction_hex;
},
else => {
result.tag = .invalid;
break;
},
},
.float_fraction_hex => switch (c) {
'_' => {
state = .float_fraction_hex_no_underscore;
},
'p', 'P' => {
state = .float_exponent_unsigned;
},
'0'...'9', 'a'...'f', 'A'...'F' => {},
'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {
result.tag = .invalid;
break;
},
else => break,
},
.float_exponent_unsigned => switch (c) {
'+', '-' => {
state = .float_exponent_num_no_underscore;
},
else => {
// reinterpret as a normal exponent number
self.index -= 1;
state = .float_exponent_num_no_underscore;
},
},
.float_exponent_num_no_underscore => switch (c) {
'0'...'9' => {
state = .float_exponent_num;
},
else => {
result.tag = .invalid;
break;
},
},
.float_exponent_num => switch (c) {
'_' => {
state = .float_exponent_num_no_underscore;
},
'0'...'9' => {},
'a'...'z', 'A'...'Z' => {
result.tag = .invalid;
break;
},
else => break,
},
}
}
@ -1571,7 +1361,7 @@ test "code point literal with unicode escapes" {
, &.{ .invalid, .invalid });
try testTokenize(
\\'\U0333'
, &.{ .invalid, .integer_literal, .invalid });
, &.{ .invalid, .number_literal, .invalid });
}
test "code point literal with unicode code point" {
@ -1584,7 +1374,7 @@ test "float literal e exponent" {
try testTokenize("a = 4.94065645841246544177e-324;\n", &.{
.identifier,
.equal,
.float_literal,
.number_literal,
.semicolon,
});
}
@ -1593,7 +1383,7 @@ test "float literal p exponent" {
try testTokenize("a = 0x1.a827999fcef32p+1022;\n", &.{
.identifier,
.equal,
.float_literal,
.number_literal,
.semicolon,
});
}
@ -1757,7 +1547,7 @@ test "correctly parse pointer assignment" {
.identifier,
.period_asterisk,
.equal,
.integer_literal,
.number_literal,
.semicolon,
});
}
@ -1767,7 +1557,7 @@ test "correctly parse pointer dereference followed by asterisk" {
.string_literal,
.period_asterisk,
.asterisk_asterisk,
.integer_literal,
.number_literal,
});
try testTokenize("(\"b\".*)** 10", &.{
@ -1776,256 +1566,256 @@ test "correctly parse pointer dereference followed by asterisk" {
.period_asterisk,
.r_paren,
.asterisk_asterisk,
.integer_literal,
.number_literal,
});
try testTokenize("\"b\".*** 10", &.{
.string_literal,
.invalid_periodasterisks,
.asterisk_asterisk,
.integer_literal,
.number_literal,
});
}
test "range literals" {
try testTokenize("0...9", &.{ .integer_literal, .ellipsis3, .integer_literal });
try testTokenize("0...9", &.{ .number_literal, .ellipsis3, .number_literal });
try testTokenize("'0'...'9'", &.{ .char_literal, .ellipsis3, .char_literal });
try testTokenize("0x00...0x09", &.{ .integer_literal, .ellipsis3, .integer_literal });
try testTokenize("0b00...0b11", &.{ .integer_literal, .ellipsis3, .integer_literal });
try testTokenize("0o00...0o11", &.{ .integer_literal, .ellipsis3, .integer_literal });
try testTokenize("0x00...0x09", &.{ .number_literal, .ellipsis3, .number_literal });
try testTokenize("0b00...0b11", &.{ .number_literal, .ellipsis3, .number_literal });
try testTokenize("0o00...0o11", &.{ .number_literal, .ellipsis3, .number_literal });
}
test "number literals decimal" {
try testTokenize("0", &.{.integer_literal});
try testTokenize("1", &.{.integer_literal});
try testTokenize("2", &.{.integer_literal});
try testTokenize("3", &.{.integer_literal});
try testTokenize("4", &.{.integer_literal});
try testTokenize("5", &.{.integer_literal});
try testTokenize("6", &.{.integer_literal});
try testTokenize("7", &.{.integer_literal});
try testTokenize("8", &.{.integer_literal});
try testTokenize("9", &.{.integer_literal});
try testTokenize("1..", &.{ .integer_literal, .ellipsis2 });
try testTokenize("0a", &.{ .invalid, .identifier });
try testTokenize("9b", &.{ .invalid, .identifier });
try testTokenize("1z", &.{ .invalid, .identifier });
try testTokenize("1z_1", &.{ .invalid, .identifier });
try testTokenize("9z3", &.{ .invalid, .identifier });
try testTokenize("0", &.{.number_literal});
try testTokenize("1", &.{.number_literal});
try testTokenize("2", &.{.number_literal});
try testTokenize("3", &.{.number_literal});
try testTokenize("4", &.{.number_literal});
try testTokenize("5", &.{.number_literal});
try testTokenize("6", &.{.number_literal});
try testTokenize("7", &.{.number_literal});
try testTokenize("8", &.{.number_literal});
try testTokenize("9", &.{.number_literal});
try testTokenize("1..", &.{ .number_literal, .ellipsis2 });
try testTokenize("0a", &.{.number_literal});
try testTokenize("9b", &.{.number_literal});
try testTokenize("1z", &.{.number_literal});
try testTokenize("1z_1", &.{.number_literal});
try testTokenize("9z3", &.{.number_literal});
try testTokenize("0_0", &.{.integer_literal});
try testTokenize("0001", &.{.integer_literal});
try testTokenize("01234567890", &.{.integer_literal});
try testTokenize("012_345_6789_0", &.{.integer_literal});
try testTokenize("0_1_2_3_4_5_6_7_8_9_0", &.{.integer_literal});
try testTokenize("0_0", &.{.number_literal});
try testTokenize("0001", &.{.number_literal});
try testTokenize("01234567890", &.{.number_literal});
try testTokenize("012_345_6789_0", &.{.number_literal});
try testTokenize("0_1_2_3_4_5_6_7_8_9_0", &.{.number_literal});
try testTokenize("00_", &.{.invalid});
try testTokenize("0_0_", &.{.invalid});
try testTokenize("0__0", &.{ .invalid, .identifier });
try testTokenize("0_0f", &.{ .invalid, .identifier });
try testTokenize("0_0_f", &.{ .invalid, .identifier });
try testTokenize("0_0_f_00", &.{ .invalid, .identifier });
try testTokenize("1_,", &.{ .invalid, .comma });
try testTokenize("00_", &.{.number_literal});
try testTokenize("0_0_", &.{.number_literal});
try testTokenize("0__0", &.{.number_literal});
try testTokenize("0_0f", &.{.number_literal});
try testTokenize("0_0_f", &.{.number_literal});
try testTokenize("0_0_f_00", &.{.number_literal});
try testTokenize("1_,", &.{ .number_literal, .comma });
try testTokenize("0.0", &.{.float_literal});
try testTokenize("1.0", &.{.float_literal});
try testTokenize("10.0", &.{.float_literal});
try testTokenize("0e0", &.{.float_literal});
try testTokenize("1e0", &.{.float_literal});
try testTokenize("1e100", &.{.float_literal});
try testTokenize("1.0e100", &.{.float_literal});
try testTokenize("1.0e+100", &.{.float_literal});
try testTokenize("1.0e-100", &.{.float_literal});
try testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &.{.float_literal});
try testTokenize("0.0", &.{.number_literal});
try testTokenize("1.0", &.{.number_literal});
try testTokenize("10.0", &.{.number_literal});
try testTokenize("0e0", &.{.number_literal});
try testTokenize("1e0", &.{.number_literal});
try testTokenize("1e100", &.{.number_literal});
try testTokenize("1.0e100", &.{.number_literal});
try testTokenize("1.0e+100", &.{.number_literal});
try testTokenize("1.0e-100", &.{.number_literal});
try testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &.{.number_literal});
try testTokenize("1.", &.{.invalid});
try testTokenize("1e", &.{.invalid});
try testTokenize("1.e100", &.{ .invalid, .identifier });
try testTokenize("1.0e1f0", &.{ .invalid, .identifier });
try testTokenize("1.0p100", &.{ .invalid, .identifier });
try testTokenize("1.0p-100", &.{ .invalid, .identifier, .minus, .integer_literal });
try testTokenize("1.0p1f0", &.{ .invalid, .identifier });
try testTokenize("1.0_,", &.{ .invalid, .comma });
try testTokenize("1_.0", &.{ .invalid, .period, .integer_literal });
try testTokenize("1._", &.{ .invalid, .identifier });
try testTokenize("1.a", &.{ .invalid, .identifier });
try testTokenize("1.z", &.{ .invalid, .identifier });
try testTokenize("1._0", &.{ .invalid, .identifier });
try testTokenize("1.+", &.{ .invalid, .plus });
try testTokenize("1._+", &.{ .invalid, .identifier, .plus });
try testTokenize("1._e", &.{ .invalid, .identifier });
try testTokenize("1.0e", &.{.invalid});
try testTokenize("1.0e,", &.{ .invalid, .comma });
try testTokenize("1.0e_", &.{ .invalid, .identifier });
try testTokenize("1.0e+_", &.{ .invalid, .identifier });
try testTokenize("1.0e-_", &.{ .invalid, .identifier });
try testTokenize("1.0e0_+", &.{ .invalid, .plus });
try testTokenize("1.", &.{ .number_literal, .period });
try testTokenize("1e", &.{.number_literal});
try testTokenize("1.e100", &.{.number_literal});
try testTokenize("1.0e1f0", &.{.number_literal});
try testTokenize("1.0p100", &.{.number_literal});
try testTokenize("1.0p-100", &.{.number_literal});
try testTokenize("1.0p1f0", &.{.number_literal});
try testTokenize("1.0_,", &.{ .number_literal, .comma });
try testTokenize("1_.0", &.{.number_literal});
try testTokenize("1._", &.{.number_literal});
try testTokenize("1.a", &.{.number_literal});
try testTokenize("1.z", &.{.number_literal});
try testTokenize("1._0", &.{.number_literal});
try testTokenize("1.+", &.{ .number_literal, .period, .plus });
try testTokenize("1._+", &.{ .number_literal, .plus });
try testTokenize("1._e", &.{.number_literal});
try testTokenize("1.0e", &.{.number_literal});
try testTokenize("1.0e,", &.{ .number_literal, .comma });
try testTokenize("1.0e_", &.{.number_literal});
try testTokenize("1.0e+_", &.{.number_literal});
try testTokenize("1.0e-_", &.{.number_literal});
try testTokenize("1.0e0_+", &.{ .number_literal, .plus });
}
test "number literals binary" {
try testTokenize("0b0", &.{.integer_literal});
try testTokenize("0b1", &.{.integer_literal});
try testTokenize("0b2", &.{ .invalid, .integer_literal });
try testTokenize("0b3", &.{ .invalid, .integer_literal });
try testTokenize("0b4", &.{ .invalid, .integer_literal });
try testTokenize("0b5", &.{ .invalid, .integer_literal });
try testTokenize("0b6", &.{ .invalid, .integer_literal });
try testTokenize("0b7", &.{ .invalid, .integer_literal });
try testTokenize("0b8", &.{ .invalid, .integer_literal });
try testTokenize("0b9", &.{ .invalid, .integer_literal });
try testTokenize("0ba", &.{ .invalid, .identifier });
try testTokenize("0bb", &.{ .invalid, .identifier });
try testTokenize("0bc", &.{ .invalid, .identifier });
try testTokenize("0bd", &.{ .invalid, .identifier });
try testTokenize("0be", &.{ .invalid, .identifier });
try testTokenize("0bf", &.{ .invalid, .identifier });
try testTokenize("0bz", &.{ .invalid, .identifier });
try testTokenize("0b0", &.{.number_literal});
try testTokenize("0b1", &.{.number_literal});
try testTokenize("0b2", &.{.number_literal});
try testTokenize("0b3", &.{.number_literal});
try testTokenize("0b4", &.{.number_literal});
try testTokenize("0b5", &.{.number_literal});
try testTokenize("0b6", &.{.number_literal});
try testTokenize("0b7", &.{.number_literal});
try testTokenize("0b8", &.{.number_literal});
try testTokenize("0b9", &.{.number_literal});
try testTokenize("0ba", &.{.number_literal});
try testTokenize("0bb", &.{.number_literal});
try testTokenize("0bc", &.{.number_literal});
try testTokenize("0bd", &.{.number_literal});
try testTokenize("0be", &.{.number_literal});
try testTokenize("0bf", &.{.number_literal});
try testTokenize("0bz", &.{.number_literal});
try testTokenize("0b0000_0000", &.{.integer_literal});
try testTokenize("0b1111_1111", &.{.integer_literal});
try testTokenize("0b10_10_10_10", &.{.integer_literal});
try testTokenize("0b0_1_0_1_0_1_0_1", &.{.integer_literal});
try testTokenize("0b1.", &.{ .integer_literal, .period });
try testTokenize("0b1.0", &.{ .integer_literal, .period, .integer_literal });
try testTokenize("0b0000_0000", &.{.number_literal});
try testTokenize("0b1111_1111", &.{.number_literal});
try testTokenize("0b10_10_10_10", &.{.number_literal});
try testTokenize("0b0_1_0_1_0_1_0_1", &.{.number_literal});
try testTokenize("0b1.", &.{ .number_literal, .period });
try testTokenize("0b1.0", &.{.number_literal});
try testTokenize("0B0", &.{ .invalid, .identifier });
try testTokenize("0b_", &.{ .invalid, .identifier });
try testTokenize("0b_0", &.{ .invalid, .identifier });
try testTokenize("0b1_", &.{.invalid});
try testTokenize("0b0__1", &.{ .invalid, .identifier });
try testTokenize("0b0_1_", &.{.invalid});
try testTokenize("0b1e", &.{ .invalid, .identifier });
try testTokenize("0b1p", &.{ .invalid, .identifier });
try testTokenize("0b1e0", &.{ .invalid, .identifier });
try testTokenize("0b1p0", &.{ .invalid, .identifier });
try testTokenize("0b1_,", &.{ .invalid, .comma });
try testTokenize("0B0", &.{.number_literal});
try testTokenize("0b_", &.{.number_literal});
try testTokenize("0b_0", &.{.number_literal});
try testTokenize("0b1_", &.{.number_literal});
try testTokenize("0b0__1", &.{.number_literal});
try testTokenize("0b0_1_", &.{.number_literal});
try testTokenize("0b1e", &.{.number_literal});
try testTokenize("0b1p", &.{.number_literal});
try testTokenize("0b1e0", &.{.number_literal});
try testTokenize("0b1p0", &.{.number_literal});
try testTokenize("0b1_,", &.{ .number_literal, .comma });
}
test "number literals octal" {
try testTokenize("0o0", &.{.integer_literal});
try testTokenize("0o1", &.{.integer_literal});
try testTokenize("0o2", &.{.integer_literal});
try testTokenize("0o3", &.{.integer_literal});
try testTokenize("0o4", &.{.integer_literal});
try testTokenize("0o5", &.{.integer_literal});
try testTokenize("0o6", &.{.integer_literal});
try testTokenize("0o7", &.{.integer_literal});
try testTokenize("0o8", &.{ .invalid, .integer_literal });
try testTokenize("0o9", &.{ .invalid, .integer_literal });
try testTokenize("0oa", &.{ .invalid, .identifier });
try testTokenize("0ob", &.{ .invalid, .identifier });
try testTokenize("0oc", &.{ .invalid, .identifier });
try testTokenize("0od", &.{ .invalid, .identifier });
try testTokenize("0oe", &.{ .invalid, .identifier });
try testTokenize("0of", &.{ .invalid, .identifier });
try testTokenize("0oz", &.{ .invalid, .identifier });
try testTokenize("0o0", &.{.number_literal});
try testTokenize("0o1", &.{.number_literal});
try testTokenize("0o2", &.{.number_literal});
try testTokenize("0o3", &.{.number_literal});
try testTokenize("0o4", &.{.number_literal});
try testTokenize("0o5", &.{.number_literal});
try testTokenize("0o6", &.{.number_literal});
try testTokenize("0o7", &.{.number_literal});
try testTokenize("0o8", &.{.number_literal});
try testTokenize("0o9", &.{.number_literal});
try testTokenize("0oa", &.{.number_literal});
try testTokenize("0ob", &.{.number_literal});
try testTokenize("0oc", &.{.number_literal});
try testTokenize("0od", &.{.number_literal});
try testTokenize("0oe", &.{.number_literal});
try testTokenize("0of", &.{.number_literal});
try testTokenize("0oz", &.{.number_literal});
try testTokenize("0o01234567", &.{.integer_literal});
try testTokenize("0o0123_4567", &.{.integer_literal});
try testTokenize("0o01_23_45_67", &.{.integer_literal});
try testTokenize("0o0_1_2_3_4_5_6_7", &.{.integer_literal});
try testTokenize("0o7.", &.{ .integer_literal, .period });
try testTokenize("0o7.0", &.{ .integer_literal, .period, .integer_literal });
try testTokenize("0o01234567", &.{.number_literal});
try testTokenize("0o0123_4567", &.{.number_literal});
try testTokenize("0o01_23_45_67", &.{.number_literal});
try testTokenize("0o0_1_2_3_4_5_6_7", &.{.number_literal});
try testTokenize("0o7.", &.{ .number_literal, .period });
try testTokenize("0o7.0", &.{.number_literal});
try testTokenize("0O0", &.{ .invalid, .identifier });
try testTokenize("0o_", &.{ .invalid, .identifier });
try testTokenize("0o_0", &.{ .invalid, .identifier });
try testTokenize("0o1_", &.{.invalid});
try testTokenize("0o0__1", &.{ .invalid, .identifier });
try testTokenize("0o0_1_", &.{.invalid});
try testTokenize("0o1e", &.{ .invalid, .identifier });
try testTokenize("0o1p", &.{ .invalid, .identifier });
try testTokenize("0o1e0", &.{ .invalid, .identifier });
try testTokenize("0o1p0", &.{ .invalid, .identifier });
try testTokenize("0o_,", &.{ .invalid, .identifier, .comma });
try testTokenize("0O0", &.{.number_literal});
try testTokenize("0o_", &.{.number_literal});
try testTokenize("0o_0", &.{.number_literal});
try testTokenize("0o1_", &.{.number_literal});
try testTokenize("0o0__1", &.{.number_literal});
try testTokenize("0o0_1_", &.{.number_literal});
try testTokenize("0o1e", &.{.number_literal});
try testTokenize("0o1p", &.{.number_literal});
try testTokenize("0o1e0", &.{.number_literal});
try testTokenize("0o1p0", &.{.number_literal});
try testTokenize("0o_,", &.{ .number_literal, .comma });
}
test "number literals hexadecimal" {
try testTokenize("0x0", &.{.integer_literal});
try testTokenize("0x1", &.{.integer_literal});
try testTokenize("0x2", &.{.integer_literal});
try testTokenize("0x3", &.{.integer_literal});
try testTokenize("0x4", &.{.integer_literal});
try testTokenize("0x5", &.{.integer_literal});
try testTokenize("0x6", &.{.integer_literal});
try testTokenize("0x7", &.{.integer_literal});
try testTokenize("0x8", &.{.integer_literal});
try testTokenize("0x9", &.{.integer_literal});
try testTokenize("0xa", &.{.integer_literal});
try testTokenize("0xb", &.{.integer_literal});
try testTokenize("0xc", &.{.integer_literal});
try testTokenize("0xd", &.{.integer_literal});
try testTokenize("0xe", &.{.integer_literal});
try testTokenize("0xf", &.{.integer_literal});
try testTokenize("0xA", &.{.integer_literal});
try testTokenize("0xB", &.{.integer_literal});
try testTokenize("0xC", &.{.integer_literal});
try testTokenize("0xD", &.{.integer_literal});
try testTokenize("0xE", &.{.integer_literal});
try testTokenize("0xF", &.{.integer_literal});
try testTokenize("0x0z", &.{ .invalid, .identifier });
try testTokenize("0xz", &.{ .invalid, .identifier });
try testTokenize("0x0", &.{.number_literal});
try testTokenize("0x1", &.{.number_literal});
try testTokenize("0x2", &.{.number_literal});
try testTokenize("0x3", &.{.number_literal});
try testTokenize("0x4", &.{.number_literal});
try testTokenize("0x5", &.{.number_literal});
try testTokenize("0x6", &.{.number_literal});
try testTokenize("0x7", &.{.number_literal});
try testTokenize("0x8", &.{.number_literal});
try testTokenize("0x9", &.{.number_literal});
try testTokenize("0xa", &.{.number_literal});
try testTokenize("0xb", &.{.number_literal});
try testTokenize("0xc", &.{.number_literal});
try testTokenize("0xd", &.{.number_literal});
try testTokenize("0xe", &.{.number_literal});
try testTokenize("0xf", &.{.number_literal});
try testTokenize("0xA", &.{.number_literal});
try testTokenize("0xB", &.{.number_literal});
try testTokenize("0xC", &.{.number_literal});
try testTokenize("0xD", &.{.number_literal});
try testTokenize("0xE", &.{.number_literal});
try testTokenize("0xF", &.{.number_literal});
try testTokenize("0x0z", &.{.number_literal});
try testTokenize("0xz", &.{.number_literal});
try testTokenize("0x0123456789ABCDEF", &.{.integer_literal});
try testTokenize("0x0123_4567_89AB_CDEF", &.{.integer_literal});
try testTokenize("0x01_23_45_67_89AB_CDE_F", &.{.integer_literal});
try testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &.{.integer_literal});
try testTokenize("0x0123456789ABCDEF", &.{.number_literal});
try testTokenize("0x0123_4567_89AB_CDEF", &.{.number_literal});
try testTokenize("0x01_23_45_67_89AB_CDE_F", &.{.number_literal});
try testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &.{.number_literal});
try testTokenize("0X0", &.{ .invalid, .identifier });
try testTokenize("0x_", &.{ .invalid, .identifier });
try testTokenize("0x_1", &.{ .invalid, .identifier });
try testTokenize("0x1_", &.{.invalid});
try testTokenize("0x0__1", &.{ .invalid, .identifier });
try testTokenize("0x0_1_", &.{.invalid});
try testTokenize("0x_,", &.{ .invalid, .identifier, .comma });
try testTokenize("0X0", &.{.number_literal});
try testTokenize("0x_", &.{.number_literal});
try testTokenize("0x_1", &.{.number_literal});
try testTokenize("0x1_", &.{.number_literal});
try testTokenize("0x0__1", &.{.number_literal});
try testTokenize("0x0_1_", &.{.number_literal});
try testTokenize("0x_,", &.{ .number_literal, .comma });
try testTokenize("0x1.0", &.{.float_literal});
try testTokenize("0xF.0", &.{.float_literal});
try testTokenize("0xF.F", &.{.float_literal});
try testTokenize("0xF.Fp0", &.{.float_literal});
try testTokenize("0xF.FP0", &.{.float_literal});
try testTokenize("0x1p0", &.{.float_literal});
try testTokenize("0xfp0", &.{.float_literal});
try testTokenize("0x1.0+0xF.0", &.{ .float_literal, .plus, .float_literal });
try testTokenize("0x1.0", &.{.number_literal});
try testTokenize("0xF.0", &.{.number_literal});
try testTokenize("0xF.F", &.{.number_literal});
try testTokenize("0xF.Fp0", &.{.number_literal});
try testTokenize("0xF.FP0", &.{.number_literal});
try testTokenize("0x1p0", &.{.number_literal});
try testTokenize("0xfp0", &.{.number_literal});
try testTokenize("0x1.0+0xF.0", &.{ .number_literal, .plus, .number_literal });
try testTokenize("0x1.", &.{.invalid});
try testTokenize("0xF.", &.{.invalid});
try testTokenize("0x1.+0xF.", &.{ .invalid, .plus, .invalid });
try testTokenize("0xff.p10", &.{ .invalid, .identifier });
try testTokenize("0x1.", &.{ .number_literal, .period });
try testTokenize("0xF.", &.{ .number_literal, .period });
try testTokenize("0x1.+0xF.", &.{ .number_literal, .period, .plus, .number_literal, .period });
try testTokenize("0xff.p10", &.{.number_literal});
try testTokenize("0x0123456.789ABCDEF", &.{.float_literal});
try testTokenize("0x0_123_456.789_ABC_DEF", &.{.float_literal});
try testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &.{.float_literal});
try testTokenize("0x0p0", &.{.float_literal});
try testTokenize("0x0.0p0", &.{.float_literal});
try testTokenize("0xff.ffp10", &.{.float_literal});
try testTokenize("0xff.ffP10", &.{.float_literal});
try testTokenize("0xffp10", &.{.float_literal});
try testTokenize("0xff_ff.ff_ffp1_0_0_0", &.{.float_literal});
try testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &.{.float_literal});
try testTokenize("0xf_f_f_f.f_f_f_fp-1_00_0", &.{.float_literal});
try testTokenize("0x0123456.789ABCDEF", &.{.number_literal});
try testTokenize("0x0_123_456.789_ABC_DEF", &.{.number_literal});
try testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &.{.number_literal});
try testTokenize("0x0p0", &.{.number_literal});
try testTokenize("0x0.0p0", &.{.number_literal});
try testTokenize("0xff.ffp10", &.{.number_literal});
try testTokenize("0xff.ffP10", &.{.number_literal});
try testTokenize("0xffp10", &.{.number_literal});
try testTokenize("0xff_ff.ff_ffp1_0_0_0", &.{.number_literal});
try testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &.{.number_literal});
try testTokenize("0xf_f_f_f.f_f_f_fp-1_00_0", &.{.number_literal});
try testTokenize("0x1e", &.{.integer_literal});
try testTokenize("0x1e0", &.{.integer_literal});
try testTokenize("0x1p", &.{.invalid});
try testTokenize("0xfp0z1", &.{ .invalid, .identifier });
try testTokenize("0xff.ffpff", &.{ .invalid, .identifier });
try testTokenize("0x0.p", &.{ .invalid, .identifier });
try testTokenize("0x0.z", &.{ .invalid, .identifier });
try testTokenize("0x0._", &.{ .invalid, .identifier });
try testTokenize("0x0_.0", &.{ .invalid, .period, .integer_literal });
try testTokenize("0x0_.0.0", &.{ .invalid, .period, .float_literal });
try testTokenize("0x0._0", &.{ .invalid, .identifier });
try testTokenize("0x0.0_", &.{.invalid});
try testTokenize("0x0_p0", &.{ .invalid, .identifier });
try testTokenize("0x0_.p0", &.{ .invalid, .period, .identifier });
try testTokenize("0x0._p0", &.{ .invalid, .identifier });
try testTokenize("0x0.0_p0", &.{ .invalid, .identifier });
try testTokenize("0x0._0p0", &.{ .invalid, .identifier });
try testTokenize("0x0.0p_0", &.{ .invalid, .identifier });
try testTokenize("0x0.0p+_0", &.{ .invalid, .identifier });
try testTokenize("0x0.0p-_0", &.{ .invalid, .identifier });
try testTokenize("0x0.0p0_", &.{ .invalid, .eof });
try testTokenize("0x1e", &.{.number_literal});
try testTokenize("0x1e0", &.{.number_literal});
try testTokenize("0x1p", &.{.number_literal});
try testTokenize("0xfp0z1", &.{.number_literal});
try testTokenize("0xff.ffpff", &.{.number_literal});
try testTokenize("0x0.p", &.{.number_literal});
try testTokenize("0x0.z", &.{.number_literal});
try testTokenize("0x0._", &.{.number_literal});
try testTokenize("0x0_.0", &.{.number_literal});
try testTokenize("0x0_.0.0", &.{ .number_literal, .period, .number_literal });
try testTokenize("0x0._0", &.{.number_literal});
try testTokenize("0x0.0_", &.{.number_literal});
try testTokenize("0x0_p0", &.{.number_literal});
try testTokenize("0x0_.p0", &.{.number_literal});
try testTokenize("0x0._p0", &.{.number_literal});
try testTokenize("0x0.0_p0", &.{.number_literal});
try testTokenize("0x0._0p0", &.{.number_literal});
try testTokenize("0x0.0p_0", &.{.number_literal});
try testTokenize("0x0.0p+_0", &.{.number_literal});
try testTokenize("0x0.0p-_0", &.{.number_literal});
try testTokenize("0x0.0p0_", &.{.number_literal});
}
test "multi line string literal with only 1 backslash" {
@ -2034,7 +1824,7 @@ test "multi line string literal with only 1 backslash" {
test "invalid builtin identifiers" {
try testTokenize("@()", &.{ .invalid, .l_paren, .r_paren });
try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren });
try testTokenize("@0()", &.{ .invalid, .number_literal, .l_paren, .r_paren });
}
test "invalid token with unfinished escape right before eof" {

188
src/AstGen.zig
View File

@ -441,7 +441,7 @@ fn lvalExpr(gz: *GenZir, scope: *Scope, node: Ast.Node.Index) InnerError!Zir.Ins
.@"asm",
.asm_simple,
.string_literal,
.integer_literal,
.number_literal,
.call,
.call_comma,
.async_call,
@ -459,7 +459,6 @@ fn lvalExpr(gz: *GenZir, scope: *Scope, node: Ast.Node.Index) InnerError!Zir.Ins
.while_cont,
.bool_not,
.address_of,
.float_literal,
.optional_type,
.block,
.block_semicolon,
@ -732,7 +731,7 @@ fn expr(gz: *GenZir, scope: *Scope, rl: ResultLoc, node: Ast.Node.Index) InnerEr
.string_literal => return stringLiteral(gz, rl, node),
.multiline_string_literal => return multilineStringLiteral(gz, rl, node),
.integer_literal => return integerLiteral(gz, rl, node),
.number_literal => return numberLiteral(gz, rl, node, node, .positive),
// zig fmt: on
.builtin_call_two, .builtin_call_two_comma => {
@ -773,7 +772,6 @@ fn expr(gz: *GenZir, scope: *Scope, rl: ResultLoc, node: Ast.Node.Index) InnerEr
},
.@"return" => return ret(gz, scope, node),
.field_access => return fieldAccess(gz, scope, rl, node),
.float_literal => return floatLiteral(gz, rl, node, .positive),
.if_simple => return ifExpr(gz, scope, rl.br(), node, tree.ifSimple(node)),
.@"if" => return ifExpr(gz, scope, rl.br(), node, tree.ifFull(node)),
@ -7052,93 +7050,101 @@ fn charLiteral(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index) InnerError!Zir.
}
}
fn integerLiteral(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
const astgen = gz.astgen;
const tree = astgen.tree;
const main_tokens = tree.nodes.items(.main_token);
const int_token = main_tokens[node];
const prefixed_bytes = tree.tokenSlice(int_token);
var base: u8 = 10;
var non_prefixed: []const u8 = prefixed_bytes;
if (mem.startsWith(u8, prefixed_bytes, "0x")) {
base = 16;
non_prefixed = prefixed_bytes[2..];
} else if (mem.startsWith(u8, prefixed_bytes, "0o")) {
base = 8;
non_prefixed = prefixed_bytes[2..];
} else if (mem.startsWith(u8, prefixed_bytes, "0b")) {
base = 2;
non_prefixed = prefixed_bytes[2..];
}
if (base == 10 and prefixed_bytes.len >= 2 and prefixed_bytes[0] == '0') {
return astgen.failNodeNotes(node, "integer literal '{s}' has leading zero", .{prefixed_bytes}, &.{
try astgen.errNoteNode(node, "use '0o' prefix for octal literals", .{}),
});
}
if (std.fmt.parseUnsigned(u64, non_prefixed, base)) |small_int| {
const result: Zir.Inst.Ref = switch (small_int) {
0 => .zero,
1 => .one,
else => try gz.addInt(small_int),
};
return rvalue(gz, rl, result, node);
} else |err| switch (err) {
error.InvalidCharacter => unreachable, // Caught by the parser.
error.Overflow => {},
}
const gpa = astgen.gpa;
var big_int = try std.math.big.int.Managed.init(gpa);
defer big_int.deinit();
big_int.setString(base, non_prefixed) catch |err| switch (err) {
error.InvalidCharacter => unreachable, // caught by parser
error.InvalidBase => unreachable, // we only pass 16, 8, 2, see above
error.OutOfMemory => return error.OutOfMemory,
};
const limbs = big_int.limbs[0..big_int.len()];
assert(big_int.isPositive());
const result = try gz.addIntBig(limbs);
return rvalue(gz, rl, result, node);
}
const Sign = enum { negative, positive };
fn floatLiteral(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index, sign: Sign) InnerError!Zir.Inst.Ref {
fn numberLiteral(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index, source_node: Ast.Node.Index, sign: Sign) InnerError!Zir.Inst.Ref {
const astgen = gz.astgen;
const tree = astgen.tree;
const main_tokens = tree.nodes.items(.main_token);
const num_token = main_tokens[node];
const bytes = tree.tokenSlice(num_token);
const main_token = main_tokens[node];
const bytes = tree.tokenSlice(main_token);
const unsigned_float_number = std.fmt.parseFloat(f128, bytes) catch |err| switch (err) {
error.InvalidCharacter => unreachable, // validated by tokenizer
const result: Zir.Inst.Ref = switch (std.zig.parseNumberLiteral(bytes)) {
.int => |num| switch (num) {
0 => .zero,
1 => .one,
else => try gz.addInt(num),
},
.big_int => |base| big: {
const gpa = astgen.gpa;
var big_int = try std.math.big.int.Managed.init(gpa);
defer big_int.deinit();
const prefix_offset = @as(u8, 2) * @boolToInt(base != .decimal);
big_int.setString(@enumToInt(base), bytes[prefix_offset..]) catch |err| switch (err) {
error.InvalidCharacter => unreachable, // caught in `parseNumberLiteral`
error.InvalidBase => unreachable, // we only pass 16, 8, 2, see above
error.OutOfMemory => return error.OutOfMemory,
};
const limbs = big_int.limbs[0..big_int.len()];
assert(big_int.isPositive());
break :big try gz.addIntBig(limbs);
},
.float => {
const unsigned_float_number = std.fmt.parseFloat(f128, bytes) catch |err| switch (err) {
error.InvalidCharacter => unreachable, // validated by tokenizer
};
const float_number = switch (sign) {
.negative => -unsigned_float_number,
.positive => unsigned_float_number,
};
// If the value fits into a f64 without losing any precision, store it that way.
@setFloatMode(.Strict);
const smaller_float = @floatCast(f64, float_number);
const bigger_again: f128 = smaller_float;
if (bigger_again == float_number) {
const result = try gz.addFloat(smaller_float);
return rvalue(gz, rl, result, source_node);
}
// We need to use 128 bits. Break the float into 4 u32 values so we can
// put it into the `extra` array.
const int_bits = @bitCast(u128, float_number);
const result = try gz.addPlNode(.float128, node, Zir.Inst.Float128{
.piece0 = @truncate(u32, int_bits),
.piece1 = @truncate(u32, int_bits >> 32),
.piece2 = @truncate(u32, int_bits >> 64),
.piece3 = @truncate(u32, int_bits >> 96),
});
return rvalue(gz, rl, result, source_node);
},
.failure => |err| return astgen.failWithNumberError(err, num_token, bytes),
};
const float_number = switch (sign) {
.negative => -unsigned_float_number,
.positive => unsigned_float_number,
};
// If the value fits into a f64 without losing any precision, store it that way.
@setFloatMode(.Strict);
const smaller_float = @floatCast(f64, float_number);
const bigger_again: f128 = smaller_float;
if (bigger_again == float_number) {
const result = try gz.addFloat(smaller_float);
return rvalue(gz, rl, result, node);
if (sign == .positive) {
return rvalue(gz, rl, result, source_node);
} else {
const negated = try gz.addUnNode(.negate, result, source_node);
return rvalue(gz, rl, negated, source_node);
}
}
fn failWithNumberError(astgen: *AstGen, err: std.zig.number_literal.Error, token: Ast.TokenIndex, bytes: []const u8) InnerError {
const is_float = std.mem.indexOfScalar(u8, bytes, '.') != null;
switch (err) {
.leading_zero => if (is_float) {
return astgen.failTok(token, "number '{s}' has leading zero", .{bytes});
} else {
return astgen.failTokNotes(token, "number '{s}' has leading zero", .{bytes}, &.{
try astgen.errNoteTok(token, "use '0o' prefix for octal literals", .{}),
});
},
.digit_after_base => return astgen.failTok(token, "expected a digit after base prefix", .{}),
.upper_case_base => |i| return astgen.failOff(token, @intCast(u32, i), "base prefix must be lowercase", .{}),
.invalid_float_base => |i| return astgen.failOff(token, @intCast(u32, i), "invalid base for float literal", .{}),
.repeated_underscore => |i| return astgen.failOff(token, @intCast(u32, i), "repeated digit separator", .{}),
.invalid_underscore_after_special => |i| return astgen.failOff(token, @intCast(u32, i), "expected digit before digit separator", .{}),
.invalid_digit => |info| return astgen.failOff(token, @intCast(u32, info.i), "invalid digit '{c}' for {s} base", .{ bytes[info.i], @tagName(info.base) }),
.invalid_digit_exponent => |i| return astgen.failOff(token, @intCast(u32, i), "invalid digit '{c}' in exponent", .{bytes[i]}),
.duplicate_exponent => |i| return astgen.failOff(token, @intCast(u32, i), "duplicate exponent", .{}),
.invalid_hex_exponent => |i| return astgen.failOff(token, @intCast(u32, i), "hex exponent in decimal float", .{}),
.exponent_after_underscore => |i| return astgen.failOff(token, @intCast(u32, i), "expected digit before exponent", .{}),
.special_after_underscore => |i| return astgen.failOff(token, @intCast(u32, i), "expected digit before '{c}'", .{bytes[i]}),
.trailing_special => |i| return astgen.failOff(token, @intCast(u32, i), "expected digit after '{c}'", .{bytes[i - 1]}),
.trailing_underscore => |i| return astgen.failOff(token, @intCast(u32, i), "trailing digit separator", .{}),
.duplicate_period => unreachable, // Validated by tokenizer
.invalid_character => unreachable, // Validated by tokenizer
.invalid_exponent_sign => unreachable, // Validated by tokenizer
}
// We need to use 128 bits. Break the float into 4 u32 values so we can
// put it into the `extra` array.
const int_bits = @bitCast(u128, float_number);
const result = try gz.addPlNode(.float128, node, Zir.Inst.Float128{
.piece0 = @truncate(u32, int_bits),
.piece1 = @truncate(u32, int_bits >> 32),
.piece2 = @truncate(u32, int_bits >> 64),
.piece3 = @truncate(u32, int_bits >> 96),
});
return rvalue(gz, rl, result, node);
}
fn asmExpr(
@ -8088,8 +8094,8 @@ fn negation(
// Check for float literal as the sub-expression because we want to preserve
// its negativity rather than having it go through comptime subtraction.
const operand_node = node_datas[node].lhs;
if (node_tags[operand_node] == .float_literal) {
return floatLiteral(gz, rl, operand_node, .negative);
if (node_tags[operand_node] == .number_literal) {
return numberLiteral(gz, rl, operand_node, node, .negative);
}
const operand = try expr(gz, scope, .none, operand_node);
@ -8497,8 +8503,7 @@ fn nodeMayNeedMemoryLocation(tree: *const Ast, start_node: Ast.Node.Index, have_
.fn_decl,
.anyframe_type,
.anyframe_literal,
.integer_literal,
.float_literal,
.number_literal,
.enum_literal,
.string_literal,
.multiline_string_literal,
@ -8757,8 +8762,7 @@ fn nodeMayEvalToError(tree: *const Ast, start_node: Ast.Node.Index) BuiltinFn.Ev
.fn_decl,
.anyframe_type,
.anyframe_literal,
.integer_literal,
.float_literal,
.number_literal,
.enum_literal,
.string_literal,
.multiline_string_literal,
@ -8931,8 +8935,7 @@ fn nodeImpliesMoreThanOnePossibleValue(tree: *const Ast, start_node: Ast.Node.In
.@"suspend",
.fn_decl,
.anyframe_literal,
.integer_literal,
.float_literal,
.number_literal,
.enum_literal,
.string_literal,
.multiline_string_literal,
@ -9174,8 +9177,7 @@ fn nodeImpliesComptimeOnly(tree: *const Ast, start_node: Ast.Node.Index) bool {
.@"suspend",
.fn_decl,
.anyframe_literal,
.integer_literal,
.float_literal,
.number_literal,
.enum_literal,
.string_literal,
.multiline_string_literal,

4
src/Compilation.zig
View File

@ -610,7 +610,7 @@ pub const AllErrors = struct {
}
const token_starts = file.tree.tokens.items(.start);
const start = token_starts[item.data.token] + item.data.byte_offset;
const end = start + @intCast(u32, file.tree.tokenSlice(item.data.token).len);
const end = start + @intCast(u32, file.tree.tokenSlice(item.data.token).len) - item.data.byte_offset;
break :blk Module.SrcLoc.Span{ .start = start, .end = end, .main = start };
};
const err_loc = std.zig.findLineColumn(file.source, err_span.main);
@ -629,7 +629,7 @@ pub const AllErrors = struct {
}
const token_starts = file.tree.tokens.items(.start);
const start = token_starts[note_item.data.token] + note_item.data.byte_offset;
const end = start + @intCast(u32, file.tree.tokenSlice(note_item.data.token).len);
const end = start + @intCast(u32, file.tree.tokenSlice(note_item.data.token).len) - item.data.byte_offset;
break :blk Module.SrcLoc.Span{ .start = start, .end = end, .main = start };
};
const loc = std.zig.findLineColumn(file.source, span.main);

4
src/autodoc/render_source.zig
View File

@ -316,9 +316,7 @@ pub fn tokenizeAndPrintRaw(
}
},
.integer_literal,
.float_literal,
=> {
.number_literal => {
try out.writeAll("<span class=\"tok-number\">");
try writeEscaped(out, src[token.loc.start..token.loc.end]);
try out.writeAll("</span>");

52
src/translate_c/ast.zig
View File

@ -934,13 +934,13 @@ fn renderNode(c: *Context, node: Node) Allocator.Error!NodeIndex {
.data = undefined,
}),
.zero_literal => return c.addNode(.{
.tag = .integer_literal,
.main_token = try c.addToken(.integer_literal, "0"),
.tag = .number_literal,
.main_token = try c.addToken(.number_literal, "0"),
.data = undefined,
}),
.one_literal => return c.addNode(.{
.tag = .integer_literal,
.main_token = try c.addToken(.integer_literal, "1"),
.tag = .number_literal,
.main_token = try c.addToken(.number_literal, "1"),
.data = undefined,
}),
.void_type => return c.addNode(.{
@ -1074,16 +1074,16 @@ fn renderNode(c: *Context, node: Node) Allocator.Error!NodeIndex {
.float_literal => {
const payload = node.castTag(.float_literal).?.data;
return c.addNode(.{
.tag = .float_literal,
.main_token = try c.addToken(.float_literal, payload),
.tag = .number_literal,
.main_token = try c.addToken(.number_literal, payload),
.data = undefined,
});
},
.integer_literal => {
const payload = node.castTag(.integer_literal).?.data;
return c.addNode(.{
.tag = .integer_literal,
.main_token = try c.addToken(.integer_literal, payload),
.tag = .number_literal,
.main_token = try c.addToken(.number_literal, payload),
.data = undefined,
});
},
@ -1137,14 +1137,14 @@ fn renderNode(c: *Context, node: Node) Allocator.Error!NodeIndex {
const string = try renderNode(c, payload.string);
const l_bracket = try c.addToken(.l_bracket, "[");
const start = try c.addNode(.{
.tag = .integer_literal,
.main_token = try c.addToken(.integer_literal, "0"),
.tag = .number_literal,
.main_token = try c.addToken(.number_literal, "0"),
.data = undefined,
});
_ = try c.addToken(.ellipsis2, "..");
const end = try c.addNode(.{
.tag = .integer_literal,
.main_token = try c.addTokenFmt(.integer_literal, "{d}", .{payload.end}),
.tag = .number_literal,
.main_token = try c.addTokenFmt(.number_literal, "{d}", .{payload.end}),
.data = undefined,
});
_ = try c.addToken(.r_bracket, "]");
@ -1827,8 +1827,8 @@ fn renderNode(c: *Context, node: Node) Allocator.Error!NodeIndex {
.data = .{
.lhs = init,
.rhs = try c.addNode(.{
.tag = .integer_literal,
.main_token = try c.addTokenFmt(.integer_literal, "{d}", .{payload.count}),
.tag = .number_literal,
.main_token = try c.addTokenFmt(.number_literal, "{d}", .{payload.count}),
.data = undefined,
}),
},
@ -2039,8 +2039,8 @@ fn renderRecord(c: *Context, node: Node) !NodeIndex {
_ = try c.addToken(.keyword_align, "align");
_ = try c.addToken(.l_paren, "(");
const align_expr = try c.addNode(.{
.tag = .integer_literal,
.main_token = try c.addTokenFmt(.integer_literal, "{d}", .{alignment}),
.tag = .number_literal,
.main_token = try c.addTokenFmt(.number_literal, "{d}", .{alignment}),
.data = undefined,
});
_ = try c.addToken(.r_paren, ")");
@ -2143,8 +2143,8 @@ fn renderArrayInit(c: *Context, lhs: NodeIndex, inits: []const Node) !NodeIndex
fn renderArrayType(c: *Context, len: usize, elem_type: Node) !NodeIndex {
const l_bracket = try c.addToken(.l_bracket, "[");
const len_expr = try c.addNode(.{
.tag = .integer_literal,
.main_token = try c.addTokenFmt(.integer_literal, "{d}", .{len}),
.tag = .number_literal,
.main_token = try c.addTokenFmt(.number_literal, "{d}", .{len}),
.data = undefined,
});
_ = try c.addToken(.r_bracket, "]");
@ -2162,15 +2162,15 @@ fn renderArrayType(c: *Context, len: usize, elem_type: Node) !NodeIndex {
fn renderNullSentinelArrayType(c: *Context, len: usize, elem_type: Node) !NodeIndex {
const l_bracket = try c.addToken(.l_bracket, "[");
const len_expr = try c.addNode(.{
.tag = .integer_literal,
.main_token = try c.addTokenFmt(.integer_literal, "{d}", .{len}),
.tag = .number_literal,
.main_token = try c.addTokenFmt(.number_literal, "{d}", .{len}),
.data = undefined,
});
_ = try c.addToken(.colon, ":");
const sentinel_expr = try c.addNode(.{
.tag = .integer_literal,
.main_token = try c.addToken(.integer_literal, "0"),
.tag = .number_literal,
.main_token = try c.addToken(.number_literal, "0"),
.data = undefined,
});
@ -2571,8 +2571,8 @@ fn renderVar(c: *Context, node: Node) !NodeIndex {
_ = try c.addToken(.keyword_align, "align");
_ = try c.addToken(.l_paren, "(");
const res = try c.addNode(.{
.tag = .integer_literal,
.main_token = try c.addTokenFmt(.integer_literal, "{d}", .{some}),
.tag = .number_literal,
.main_token = try c.addTokenFmt(.number_literal, "{d}", .{some}),
.data = undefined,
});
_ = try c.addToken(.r_paren, ")");
@ -2655,8 +2655,8 @@ fn renderFunc(c: *Context, node: Node) !NodeIndex {
_ = try c.addToken(.keyword_align, "align");
_ = try c.addToken(.l_paren, "(");
const res = try c.addNode(.{
.tag = .integer_literal,
.main_token = try c.addTokenFmt(.integer_literal, "{d}", .{some}),
.tag = .number_literal,
.main_token = try c.addTokenFmt(.number_literal, "{d}", .{some}),
.data = undefined,
});
_ = try c.addToken(.r_paren, ")");

5
test/cases/compile_errors/cast_negative_integer_literal_to_usize.zig
View File

@ -2,9 +2,14 @@ export fn entry() void {
const x = @as(usize, -10);
_ = x;
}
export fn entry1() void {
const x = @as(usize, -10.0);
_ = x;
}
// error
// backend=stage2
// target=native
//
// :2:26: error: type 'usize' cannot represent integer value '-10'
// :6:26: error: float value '-10' cannot be stored in integer type 'usize'

3
test/cases/compile_errors/invalid_exponent_in_float_literal-1.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:28: note: invalid byte: 'a'
// :2:28: error: invalid digit 'a' in exponent

3
test/cases/compile_errors/invalid_exponent_in_float_literal-2.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:29: note: invalid byte: 'F'
// :2:29: error: invalid digit 'F' in exponent

3
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-1.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:23: note: invalid byte: '_'
// :2:23: error: expected digit before digit separator

3
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-10.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:25: note: invalid byte: '_'
// :2:25: error: repeated digit separator

3
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-11.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:28: note: invalid byte: '_'
// :2:28: error: repeated digit separator

5
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-12.zig
View File

@ -1,5 +1,5 @@
fn main() void {
var bad: f128 = 0_x0.0;
var bad: f128 = 1_x0.0;
_ = bad;
}
@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:23: note: invalid byte: 'x'
// :2:23: error: invalid digit 'x' for decimal base

3
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-13.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:23: note: invalid byte: '_'
// :2:23: error: expected digit before digit separator

3
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-14.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:27: note: invalid byte: 'p'
// :2:27: error: expected digit before exponent

5
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-2.zig
View File

@ -1,5 +1,5 @@
fn main() void {
var bad: f128 = 0_.0;
var bad: f128 = 1_.0;
_ = bad;
}
@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:23: note: invalid byte: '.'
// :2:23: error: expected digit before '.'

3
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-3.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:25: note: invalid byte: ';'
// :2:24: error: trailing digit separator

3
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-4.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:25: note: invalid byte: '_'
// :2:25: error: expected digit before digit separator

3
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-5.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:26: note: invalid byte: '_'
// :2:26: error: expected digit before digit separator

3
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-6.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:26: note: invalid byte: '_'
// :2:26: error: expected digit before digit separator

3
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-7.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:28: note: invalid byte: ';'
// :2:27: error: trailing digit separator

3
test/cases/compile_errors/invalid_underscore_placement_in_float_literal-9.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:23: note: invalid byte: '_'
// :2:23: error: repeated digit separator

3
test/cases/compile_errors/invalid_underscore_placement_in_int_literal-1.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=llvm
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:24: note: invalid byte: ';'
// :2:23: error: trailing digit separator

3
test/cases/compile_errors/invalid_underscore_placement_in_int_literal-2.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:28: note: invalid byte: ';'
// :2:27: error: trailing digit separator

3
test/cases/compile_errors/invalid_underscore_placement_in_int_literal-3.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:28: note: invalid byte: ';'
// :2:27: error: trailing digit separator

3
test/cases/compile_errors/invalid_underscore_placement_in_int_literal-4.zig
View File

@ -7,5 +7,4 @@ fn main() void {
// backend=stage2
// target=native
//
// :2:21: error: expected expression, found 'invalid bytes'
// :2:28: note: invalid byte: ';'
// :2:27: error: trailing digit separator

4
test/cases/compile_errors/leading_zero_in_integer.zig
View File

@ -21,7 +21,7 @@ export fn entry4() void {
//
// :2:15: error: primitive integer type 'u000123' has leading zero
// :8:12: error: primitive integer type 'i01' has leading zero
// :12:9: error: integer literal '000123' has leading zero
// :12:9: error: number '000123' has leading zero
// :12:9: note: use '0o' prefix for octal literals
// :15:9: error: integer literal '01' has leading zero
// :15:9: error: number '01' has leading zero
// :15:9: note: use '0o' prefix for octal literals

10
test/cases/compile_errors/missing_digit_after_base.zig Normal file
View File

@ -0,0 +1,10 @@
export fn entry() void {
const x = @as(usize, -0x);
_ = x;
}
// error
// backend=stage2
// target=native
//
// :2:27: error: expected a digit after base prefix

2
test/compare_output.zig
View File

@ -75,7 +75,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
\\ _ = c.printf("0.0e0: %.013a\n",
\\ @as(f64, 0.0e0));
\\ _ = c.printf("000000000000000000000000000000000000000000000000000000000.0e0: %.013a\n",
\\ @as(f64, 000000000000000000000000000000000000000000000000000000000.0e0));
\\ @as(f64, 0.0e0));
\\ _ = c.printf("0.000000000000000000000000000000000000000000000000000000000e0: %.013a\n",
\\ @as(f64, 0.000000000000000000000000000000000000000000000000000000000e0));
\\ _ = c.printf("0.0e000000000000000000000000000000000000000000000000000000000: %.013a\n",