mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2025-12-18 12:13:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #22522 from squeek502/resinator-sync

Browse Source 
resinator: Sync with upstream
This commit is contained in:
Andrew Kelley 2025-01-20 23:16:47 -05:00 committed by GitHub
commit 160445ef31
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 2151 additions and 955 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
lib/compiler/resinator/ast.zig
View File

@ -1,7 +1,7 @@
const std = @import("std"); const std = @import("std");
const Allocator = std.mem.Allocator; const Allocator = std.mem.Allocator;
const Token = @import("lex.zig").Token; const Token = @import("lex.zig").Token;
const CodePage = @import("code_pages.zig").CodePage; const SupportedCodePage = @import("code_pages.zig").SupportedCodePage;
pub const Tree = struct { pub const Tree = struct {
node: *Node, node: *Node,
@ -28,11 +28,11 @@ pub const Tree = struct {
}; };
pub const CodePageLookup = struct { pub const CodePageLookup = struct {
lookup: std.ArrayListUnmanaged(CodePage) = .empty, lookup: std.ArrayListUnmanaged(SupportedCodePage) = .empty,
allocator: Allocator, allocator: Allocator,
default_code_page: CodePage, default_code_page: SupportedCodePage,
pub fn init(allocator: Allocator, default_code_page: CodePage) CodePageLookup { pub fn init(allocator: Allocator, default_code_page: SupportedCodePage) CodePageLookup {
return .{ return .{
.allocator = allocator, .allocator = allocator,
.default_code_page = default_code_page, .default_code_page = default_code_page,
@ -44,7 +44,7 @@ pub const CodePageLookup = struct {
} }
/// line_num is 1-indexed /// line_num is 1-indexed
pub fn setForLineNum(self: *CodePageLookup, line_num: usize, code_page: CodePage) !void { pub fn setForLineNum(self: *CodePageLookup, line_num: usize, code_page: SupportedCodePage) !void {
const index = line_num - 1; const index = line_num - 1;
if (index >= self.lookup.items.len) { if (index >= self.lookup.items.len) {
const new_size = line_num; const new_size = line_num;
@ -66,16 +66,16 @@ pub const CodePageLookup = struct {
self.lookup.items[index] = code_page; self.lookup.items[index] = code_page;
} }
pub fn setForToken(self: *CodePageLookup, token: Token, code_page: CodePage) !void { pub fn setForToken(self: *CodePageLookup, token: Token, code_page: SupportedCodePage) !void {
return self.setForLineNum(token.line_number, code_page); return self.setForLineNum(token.line_number, code_page);
} }
/// line_num is 1-indexed /// line_num is 1-indexed
pub fn getForLineNum(self: CodePageLookup, line_num: usize) CodePage { pub fn getForLineNum(self: CodePageLookup, line_num: usize) SupportedCodePage {
return self.lookup.items[line_num - 1]; return self.lookup.items[line_num - 1];
} }
pub fn getForToken(self: CodePageLookup, token: Token) CodePage { pub fn getForToken(self: CodePageLookup, token: Token) SupportedCodePage {
return self.getForLineNum(token.line_number); return self.getForLineNum(token.line_number);
} }
}; };
@ -85,21 +85,21 @@ test "CodePageLookup" {
defer lookup.deinit(); defer lookup.deinit();
try lookup.setForLineNum(5, .utf8); try lookup.setForLineNum(5, .utf8);
try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(1)); try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(1));
try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(2)); try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(2));
try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(3)); try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(3));
try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(4)); try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(4));
try std.testing.expectEqual(CodePage.utf8, lookup.getForLineNum(5)); try std.testing.expectEqual(SupportedCodePage.utf8, lookup.getForLineNum(5));
try std.testing.expectEqual(@as(usize, 5), lookup.lookup.items.len); try std.testing.expectEqual(@as(usize, 5), lookup.lookup.items.len);
try lookup.setForLineNum(7, .windows1252); try lookup.setForLineNum(7, .windows1252);
try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(1)); try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(1));
try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(2)); try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(2));
try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(3)); try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(3));
try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(4)); try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(4));
try std.testing.expectEqual(CodePage.utf8, lookup.getForLineNum(5)); try std.testing.expectEqual(SupportedCodePage.utf8, lookup.getForLineNum(5));
try std.testing.expectEqual(CodePage.utf8, lookup.getForLineNum(6)); try std.testing.expectEqual(SupportedCodePage.utf8, lookup.getForLineNum(6));
try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(7)); try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(7));
try std.testing.expectEqual(@as(usize, 7), lookup.lookup.items.len); try std.testing.expectEqual(@as(usize, 7), lookup.lookup.items.len);
} }
@ -734,31 +734,31 @@ pub const Node = struct {
switch (node.id) { switch (node.id) {
.root => { .root => {
try writer.writeAll("\n"); try writer.writeAll("\n");
const root: *Node.Root = @alignCast(@fieldParentPtr("base", node)); const root: *const Node.Root = @alignCast(@fieldParentPtr("base", node));
for (root.body) |body_node| { for (root.body) |body_node| {
try body_node.dump(tree, writer, indent + 1); try body_node.dump(tree, writer, indent + 1);
} }
}, },
.resource_external => { .resource_external => {
const resource: *Node.ResourceExternal = @alignCast(@fieldParentPtr("base", node)); const resource: *const Node.ResourceExternal = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ resource.id.slice(tree.source), resource.type.slice(tree.source), resource.common_resource_attributes.len }); try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ resource.id.slice(tree.source), resource.type.slice(tree.source), resource.common_resource_attributes.len });
try resource.filename.dump(tree, writer, indent + 1); try resource.filename.dump(tree, writer, indent + 1);
}, },
.resource_raw_data => { .resource_raw_data => {
const resource: *Node.ResourceRawData = @alignCast(@fieldParentPtr("base", node)); const resource: *const Node.ResourceRawData = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s} [{d} common_resource_attributes] raw data: {}\n", .{ resource.id.slice(tree.source), resource.type.slice(tree.source), resource.common_resource_attributes.len, resource.raw_data.len }); try writer.print(" {s} {s} [{d} common_resource_attributes] raw data: {}\n", .{ resource.id.slice(tree.source), resource.type.slice(tree.source), resource.common_resource_attributes.len, resource.raw_data.len });
for (resource.raw_data) |data_expression| { for (resource.raw_data) |data_expression| {
try data_expression.dump(tree, writer, indent + 1); try data_expression.dump(tree, writer, indent + 1);
} }
}, },
.literal => { .literal => {
const literal: *Node.Literal = @alignCast(@fieldParentPtr("base", node)); const literal: *const Node.Literal = @alignCast(@fieldParentPtr("base", node));
try writer.writeAll(" "); try writer.writeAll(" ");
try writer.writeAll(literal.token.slice(tree.source)); try writer.writeAll(literal.token.slice(tree.source));
try writer.writeAll("\n"); try writer.writeAll("\n");
}, },
.binary_expression => { .binary_expression => {
const binary: *Node.BinaryExpression = @alignCast(@fieldParentPtr("base", node)); const binary: *const Node.BinaryExpression = @alignCast(@fieldParentPtr("base", node));
try writer.writeAll(" "); try writer.writeAll(" ");
try writer.writeAll(binary.operator.slice(tree.source)); try writer.writeAll(binary.operator.slice(tree.source));
try writer.writeAll("\n"); try writer.writeAll("\n");
@ -766,7 +766,7 @@ pub const Node = struct {
try binary.right.dump(tree, writer, indent + 1); try binary.right.dump(tree, writer, indent + 1);
}, },
.grouped_expression => { .grouped_expression => {
const grouped: *Node.GroupedExpression = @alignCast(@fieldParentPtr("base", node)); const grouped: *const Node.GroupedExpression = @alignCast(@fieldParentPtr("base", node));
try writer.writeAll("\n"); try writer.writeAll("\n");
try writer.writeByteNTimes(' ', indent); try writer.writeByteNTimes(' ', indent);
try writer.writeAll(grouped.open_token.slice(tree.source)); try writer.writeAll(grouped.open_token.slice(tree.source));
@ -777,7 +777,7 @@ pub const Node = struct {
try writer.writeAll("\n"); try writer.writeAll("\n");
}, },
.not_expression => { .not_expression => {
const not: *Node.NotExpression = @alignCast(@fieldParentPtr("base", node)); const not: *const Node.NotExpression = @alignCast(@fieldParentPtr("base", node));
try writer.writeAll(" "); try writer.writeAll(" ");
try writer.writeAll(not.not_token.slice(tree.source)); try writer.writeAll(not.not_token.slice(tree.source));
try writer.writeAll(" "); try writer.writeAll(" ");
@ -785,7 +785,7 @@ pub const Node = struct {
try writer.writeAll("\n"); try writer.writeAll("\n");
}, },
.accelerators => { .accelerators => {
const accelerators: *Node.Accelerators = @alignCast(@fieldParentPtr("base", node)); const accelerators: *const Node.Accelerators = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ accelerators.id.slice(tree.source), accelerators.type.slice(tree.source), accelerators.common_resource_attributes.len }); try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ accelerators.id.slice(tree.source), accelerators.type.slice(tree.source), accelerators.common_resource_attributes.len });
for (accelerators.optional_statements) |statement| { for (accelerators.optional_statements) |statement| {
try statement.dump(tree, writer, indent + 1); try statement.dump(tree, writer, indent + 1);
@ -801,7 +801,7 @@ pub const Node = struct {
try writer.writeAll("\n"); try writer.writeAll("\n");
}, },
.accelerator => { .accelerator => {
const accelerator: *Node.Accelerator = @alignCast(@fieldParentPtr("base", node)); const accelerator: *const Node.Accelerator = @alignCast(@fieldParentPtr("base", node));
for (accelerator.type_and_options, 0..) |option, i| { for (accelerator.type_and_options, 0..) |option, i| {
if (i != 0) try writer.writeAll(","); if (i != 0) try writer.writeAll(",");
try writer.writeByte(' '); try writer.writeByte(' ');
@ -812,7 +812,7 @@ pub const Node = struct {
try accelerator.idvalue.dump(tree, writer, indent + 1); try accelerator.idvalue.dump(tree, writer, indent + 1);
}, },
.dialog => { .dialog => {
const dialog: *Node.Dialog = @alignCast(@fieldParentPtr("base", node)); const dialog: *const Node.Dialog = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ dialog.id.slice(tree.source), dialog.type.slice(tree.source), dialog.common_resource_attributes.len }); try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ dialog.id.slice(tree.source), dialog.type.slice(tree.source), dialog.common_resource_attributes.len });
inline for (.{ "x", "y", "width", "height" }) |arg| { inline for (.{ "x", "y", "width", "height" }) |arg| {
try writer.writeByteNTimes(' ', indent + 1); try writer.writeByteNTimes(' ', indent + 1);
@ -838,7 +838,7 @@ pub const Node = struct {
try writer.writeAll("\n"); try writer.writeAll("\n");
}, },
.control_statement => { .control_statement => {
const control: *Node.ControlStatement = @alignCast(@fieldParentPtr("base", node)); const control: *const Node.ControlStatement = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s}", .{control.type.slice(tree.source)}); try writer.print(" {s}", .{control.type.slice(tree.source)});
if (control.text) |text| { if (control.text) |text| {
try writer.print(" text: {s}", .{text.slice(tree.source)}); try writer.print(" text: {s}", .{text.slice(tree.source)});
@ -874,7 +874,7 @@ pub const Node = struct {
} }
}, },
.toolbar => { .toolbar => {
const toolbar: *Node.Toolbar = @alignCast(@fieldParentPtr("base", node)); const toolbar: *const Node.Toolbar = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ toolbar.id.slice(tree.source), toolbar.type.slice(tree.source), toolbar.common_resource_attributes.len }); try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ toolbar.id.slice(tree.source), toolbar.type.slice(tree.source), toolbar.common_resource_attributes.len });
inline for (.{ "button_width", "button_height" }) |arg| { inline for (.{ "button_width", "button_height" }) |arg| {
try writer.writeByteNTimes(' ', indent + 1); try writer.writeByteNTimes(' ', indent + 1);
@ -892,7 +892,7 @@ pub const Node = struct {
try writer.writeAll("\n"); try writer.writeAll("\n");
}, },
.menu => { .menu => {
const menu: *Node.Menu = @alignCast(@fieldParentPtr("base", node)); const menu: *const Node.Menu = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ menu.id.slice(tree.source), menu.type.slice(tree.source), menu.common_resource_attributes.len }); try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ menu.id.slice(tree.source), menu.type.slice(tree.source), menu.common_resource_attributes.len });
for (menu.optional_statements) |statement| { for (menu.optional_statements) |statement| {
try statement.dump(tree, writer, indent + 1); try statement.dump(tree, writer, indent + 1);
@ -913,16 +913,16 @@ pub const Node = struct {
try writer.writeAll("\n"); try writer.writeAll("\n");
}, },
.menu_item => { .menu_item => {
const menu_item: *Node.MenuItem = @alignCast(@fieldParentPtr("base", node)); const menu_item: *const Node.MenuItem = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s} [{d} options]\n", .{ menu_item.menuitem.slice(tree.source), menu_item.text.slice(tree.source), menu_item.option_list.len }); try writer.print(" {s} {s} [{d} options]\n", .{ menu_item.menuitem.slice(tree.source), menu_item.text.slice(tree.source), menu_item.option_list.len });
try menu_item.result.dump(tree, writer, indent + 1); try menu_item.result.dump(tree, writer, indent + 1);
}, },
.menu_item_separator => { .menu_item_separator => {
const menu_item: *Node.MenuItemSeparator = @alignCast(@fieldParentPtr("base", node)); const menu_item: *const Node.MenuItemSeparator = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s}\n", .{ menu_item.menuitem.slice(tree.source), menu_item.separator.slice(tree.source) }); try writer.print(" {s} {s}\n", .{ menu_item.menuitem.slice(tree.source), menu_item.separator.slice(tree.source) });
}, },
.menu_item_ex => { .menu_item_ex => {
const menu_item: *Node.MenuItemEx = @alignCast(@fieldParentPtr("base", node)); const menu_item: *const Node.MenuItemEx = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s}\n", .{ menu_item.menuitem.slice(tree.source), menu_item.text.slice(tree.source) }); try writer.print(" {s} {s}\n", .{ menu_item.menuitem.slice(tree.source), menu_item.text.slice(tree.source) });
inline for (.{ "id", "type", "state" }) |arg| { inline for (.{ "id", "type", "state" }) |arg| {
if (@field(menu_item, arg)) |val_node| { if (@field(menu_item, arg)) |val_node| {
@ -933,7 +933,7 @@ pub const Node = struct {
} }
}, },
.popup => { .popup => {
const popup: *Node.Popup = @alignCast(@fieldParentPtr("base", node)); const popup: *const Node.Popup = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s} [{d} options]\n", .{ popup.popup.slice(tree.source), popup.text.slice(tree.source), popup.option_list.len }); try writer.print(" {s} {s} [{d} options]\n", .{ popup.popup.slice(tree.source), popup.text.slice(tree.source), popup.option_list.len });
try writer.writeByteNTimes(' ', indent); try writer.writeByteNTimes(' ', indent);
try writer.writeAll(popup.begin_token.slice(tree.source)); try writer.writeAll(popup.begin_token.slice(tree.source));
@ -946,7 +946,7 @@ pub const Node = struct {
try writer.writeAll("\n"); try writer.writeAll("\n");
}, },
.popup_ex => { .popup_ex => {
const popup: *Node.PopupEx = @alignCast(@fieldParentPtr("base", node)); const popup: *const Node.PopupEx = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s}\n", .{ popup.popup.slice(tree.source), popup.text.slice(tree.source) }); try writer.print(" {s} {s}\n", .{ popup.popup.slice(tree.source), popup.text.slice(tree.source) });
inline for (.{ "id", "type", "state", "help_id" }) |arg| { inline for (.{ "id", "type", "state", "help_id" }) |arg| {
if (@field(popup, arg)) |val_node| { if (@field(popup, arg)) |val_node| {
@ -966,7 +966,7 @@ pub const Node = struct {
try writer.writeAll("\n"); try writer.writeAll("\n");
}, },
.version_info => { .version_info => {
const version_info: *Node.VersionInfo = @alignCast(@fieldParentPtr("base", node)); const version_info: *const Node.VersionInfo = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ version_info.id.slice(tree.source), version_info.versioninfo.slice(tree.source), version_info.common_resource_attributes.len }); try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ version_info.id.slice(tree.source), version_info.versioninfo.slice(tree.source), version_info.common_resource_attributes.len });
for (version_info.fixed_info) |fixed_info| { for (version_info.fixed_info) |fixed_info| {
try fixed_info.dump(tree, writer, indent + 1); try fixed_info.dump(tree, writer, indent + 1);
@ -982,14 +982,14 @@ pub const Node = struct {
try writer.writeAll("\n"); try writer.writeAll("\n");
}, },
.version_statement => { .version_statement => {
const version_statement: *Node.VersionStatement = @alignCast(@fieldParentPtr("base", node)); const version_statement: *const Node.VersionStatement = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s}\n", .{version_statement.type.slice(tree.source)}); try writer.print(" {s}\n", .{version_statement.type.slice(tree.source)});
for (version_statement.parts) |part| { for (version_statement.parts) |part| {
try part.dump(tree, writer, indent + 1); try part.dump(tree, writer, indent + 1);
} }
}, },
.block => { .block => {
const block: *Node.Block = @alignCast(@fieldParentPtr("base", node)); const block: *const Node.Block = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s}\n", .{ block.identifier.slice(tree.source), block.key.slice(tree.source) }); try writer.print(" {s} {s}\n", .{ block.identifier.slice(tree.source), block.key.slice(tree.source) });
for (block.values) |value| { for (block.values) |value| {
try value.dump(tree, writer, indent + 1); try value.dump(tree, writer, indent + 1);
@ -1005,14 +1005,14 @@ pub const Node = struct {
try writer.writeAll("\n"); try writer.writeAll("\n");
}, },
.block_value => { .block_value => {
const block_value: *Node.BlockValue = @alignCast(@fieldParentPtr("base", node)); const block_value: *const Node.BlockValue = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} {s}\n", .{ block_value.identifier.slice(tree.source), block_value.key.slice(tree.source) }); try writer.print(" {s} {s}\n", .{ block_value.identifier.slice(tree.source), block_value.key.slice(tree.source) });
for (block_value.values) |value| { for (block_value.values) |value| {
try value.dump(tree, writer, indent + 1); try value.dump(tree, writer, indent + 1);
} }
}, },
.block_value_value => { .block_value_value => {
const block_value: *Node.BlockValueValue = @alignCast(@fieldParentPtr("base", node)); const block_value: *const Node.BlockValueValue = @alignCast(@fieldParentPtr("base", node));
if (block_value.trailing_comma) { if (block_value.trailing_comma) {
try writer.writeAll(" ,"); try writer.writeAll(" ,");
} }
@ -1020,7 +1020,7 @@ pub const Node = struct {
try block_value.expression.dump(tree, writer, indent + 1); try block_value.expression.dump(tree, writer, indent + 1);
}, },
.string_table => { .string_table => {
const string_table: *Node.StringTable = @alignCast(@fieldParentPtr("base", node)); const string_table: *const Node.StringTable = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} [{d} common_resource_attributes]\n", .{ string_table.type.slice(tree.source), string_table.common_resource_attributes.len }); try writer.print(" {s} [{d} common_resource_attributes]\n", .{ string_table.type.slice(tree.source), string_table.common_resource_attributes.len });
for (string_table.optional_statements) |statement| { for (string_table.optional_statements) |statement| {
try statement.dump(tree, writer, indent + 1); try statement.dump(tree, writer, indent + 1);
@ -1037,19 +1037,19 @@ pub const Node = struct {
}, },
.string_table_string => { .string_table_string => {
try writer.writeAll("\n"); try writer.writeAll("\n");
const string: *Node.StringTableString = @alignCast(@fieldParentPtr("base", node)); const string: *const Node.StringTableString = @alignCast(@fieldParentPtr("base", node));
try string.id.dump(tree, writer, indent + 1); try string.id.dump(tree, writer, indent + 1);
try writer.writeByteNTimes(' ', indent + 1); try writer.writeByteNTimes(' ', indent + 1);
try writer.print("{s}\n", .{string.string.slice(tree.source)}); try writer.print("{s}\n", .{string.string.slice(tree.source)});
}, },
.language_statement => { .language_statement => {
const language: *Node.LanguageStatement = @alignCast(@fieldParentPtr("base", node)); const language: *const Node.LanguageStatement = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s}\n", .{language.language_token.slice(tree.source)}); try writer.print(" {s}\n", .{language.language_token.slice(tree.source)});
try language.primary_language_id.dump(tree, writer, indent + 1); try language.primary_language_id.dump(tree, writer, indent + 1);
try language.sublanguage_id.dump(tree, writer, indent + 1); try language.sublanguage_id.dump(tree, writer, indent + 1);
}, },
.font_statement => { .font_statement => {
const font: *Node.FontStatement = @alignCast(@fieldParentPtr("base", node)); const font: *const Node.FontStatement = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s} typeface: {s}\n", .{ font.identifier.slice(tree.source), font.typeface.slice(tree.source) }); try writer.print(" {s} typeface: {s}\n", .{ font.identifier.slice(tree.source), font.typeface.slice(tree.source) });
try writer.writeByteNTimes(' ', indent + 1); try writer.writeByteNTimes(' ', indent + 1);
try writer.writeAll("point_size:\n"); try writer.writeAll("point_size:\n");
@ -1063,12 +1063,12 @@ pub const Node = struct {
} }
}, },
.simple_statement => { .simple_statement => {
const statement: *Node.SimpleStatement = @alignCast(@fieldParentPtr("base", node)); const statement: *const Node.SimpleStatement = @alignCast(@fieldParentPtr("base", node));
try writer.print(" {s}\n", .{statement.identifier.slice(tree.source)}); try writer.print(" {s}\n", .{statement.identifier.slice(tree.source)});
try statement.value.dump(tree, writer, indent + 1); try statement.value.dump(tree, writer, indent + 1);
}, },
.invalid => { .invalid => {
const invalid: *Node.Invalid = @alignCast(@fieldParentPtr("base", node)); const invalid: *const Node.Invalid = @alignCast(@fieldParentPtr("base", node));
try writer.print(" context.len: {}\n", .{invalid.context.len}); try writer.print(" context.len: {}\n", .{invalid.context.len});
for (invalid.context) |context_token| { for (invalid.context) |context_token| {
try writer.writeByteNTimes(' ', indent + 1); try writer.writeByteNTimes(' ', indent + 1);

9
lib/compiler/resinator/bmp.zig
View File

@ -60,10 +60,17 @@ pub const BitmapInfo = struct {
} }
pub fn getBitmasksByteLen(self: *const BitmapInfo) u8 { pub fn getBitmasksByteLen(self: *const BitmapInfo) u8 {
return switch (self.compression) { // Only BITMAPINFOHEADER (3.1) has trailing bytes for the BITFIELDS
// The 2.0 format doesn't have a compression field and 4.0+ has dedicated
// fields for the masks in the header.
const dib_version = BitmapHeader.Version.get(self.dib_header_size);
return switch (dib_version) {
.@"nt3.1" => switch (self.compression) {
.BI_BITFIELDS => 12, .BI_BITFIELDS => 12,
.BI_ALPHABITFIELDS => 16, .BI_ALPHABITFIELDS => 16,
else => 0, else => 0,
},
else => 0,
}; };
} }

115
lib/compiler/resinator/cli.zig
View File

@ -1,5 +1,6 @@
const std = @import("std"); const std = @import("std");
const CodePage = @import("code_pages.zig").CodePage; const code_pages = @import("code_pages.zig");
const SupportedCodePage = code_pages.SupportedCodePage;
const lang = @import("lang.zig"); const lang = @import("lang.zig");
const res = @import("res.zig"); const res = @import("res.zig");
const Allocator = std.mem.Allocator; const Allocator = std.mem.Allocator;
@ -14,6 +15,8 @@ pub const usage_string_after_command_name =
\\The sequence -- can be used to signify when to stop parsing options. \\The sequence -- can be used to signify when to stop parsing options.
\\This is necessary when the input path begins with a forward slash. \\This is necessary when the input path begins with a forward slash.
\\ \\
\\Supported option prefixes are /, -, and --, so e.g. /h, -h, and --h all work.
\\
\\Supported Win32 RC Options: \\Supported Win32 RC Options:
\\ /?, /h Print this help and exit. \\ /?, /h Print this help and exit.
\\ /v Verbose (print progress messages). \\ /v Verbose (print progress messages).
@ -56,8 +59,6 @@ pub const usage_string_after_command_name =
\\ the .rc includes or otherwise depends on. \\ the .rc includes or otherwise depends on.
\\ /:depfile-fmt <value> Output format of the depfile, if /:depfile is set. \\ /:depfile-fmt <value> Output format of the depfile, if /:depfile is set.
\\ json (default) A top-level JSON array of paths \\ json (default) A top-level JSON array of paths
\\ /:mingw-includes <path> Path to a directory containing MinGW include files. If
\\ not specified, bundled MinGW include files will be used.
\\ \\
\\Note: For compatibility reasons, all custom options start with : \\Note: For compatibility reasons, all custom options start with :
\\ \\
@ -136,7 +137,7 @@ pub const Options = struct {
ignore_include_env_var: bool = false, ignore_include_env_var: bool = false,
preprocess: Preprocess = .yes, preprocess: Preprocess = .yes,
default_language_id: ?u16 = null, default_language_id: ?u16 = null,
default_code_page: ?CodePage = null, default_code_page: ?SupportedCodePage = null,
verbose: bool = false, verbose: bool = false,
symbols: std.StringArrayHashMapUnmanaged(SymbolValue) = .empty, symbols: std.StringArrayHashMapUnmanaged(SymbolValue) = .empty,
null_terminate_string_table_strings: bool = false, null_terminate_string_table_strings: bool = false,
@ -148,7 +149,6 @@ pub const Options = struct {
auto_includes: AutoIncludes = .any, auto_includes: AutoIncludes = .any,
depfile_path: ?[]const u8 = null, depfile_path: ?[]const u8 = null,
depfile_fmt: DepfileFormat = .json, depfile_fmt: DepfileFormat = .json,
mingw_includes_dir: ?[]const u8 = null,
pub const AutoIncludes = enum { any, msvc, gnu, none }; pub const AutoIncludes = enum { any, msvc, gnu, none };
pub const DepfileFormat = enum { json }; pub const DepfileFormat = enum { json };
@ -243,9 +243,6 @@ pub const Options = struct {
if (self.depfile_path) |depfile_path| { if (self.depfile_path) |depfile_path| {
self.allocator.free(depfile_path); self.allocator.free(depfile_path);
} }
if (self.mingw_includes_dir) |mingw_includes_dir| {
self.allocator.free(mingw_includes_dir);
}
} }
pub fn dumpVerbose(self: *const Options, writer: anytype) !void { pub fn dumpVerbose(self: *const Options, writer: anytype) !void {
@ -358,6 +355,29 @@ pub const Arg = struct {
}; };
} }
pub fn looksLikeFilepath(self: Arg) bool {
const meets_min_requirements = self.prefix == .slash and isSupportedInputExtension(std.fs.path.extension(self.full));
if (!meets_min_requirements) return false;
const could_be_fo_option = could_be_fo_option: {
var window_it = std.mem.window(u8, self.full[1..], 2, 1);
while (window_it.next()) |window| {
if (std.ascii.eqlIgnoreCase(window, "fo")) break :could_be_fo_option true;
// If we see '/' before "fo", then it's not possible for this to be a valid
// `/fo` option.
if (window[0] == '/') break;
}
break :could_be_fo_option false;
};
if (!could_be_fo_option) return true;
// It's still possible for a file path to look like a /fo option but not actually
// be one, e.g. `/foo/bar.rc`. As a last ditch effort to reduce false negatives,
// check if the file path exists and, if so, then we ignore the 'could be /fo option'-ness
std.fs.accessAbsolute(self.full, .{}) catch return false;
return true;
}
pub const Value = struct { pub const Value = struct {
slice: []const u8, slice: []const u8,
index_increment: u2 = 1, index_increment: u2 = 1,
@ -432,6 +452,16 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
} }
} }
const args_remaining = args.len - arg_i;
if (args_remaining <= 2 and arg.looksLikeFilepath()) {
var err_details = Diagnostics.ErrorDetails{ .type = .note, .print_args = true, .arg_index = arg_i };
var msg_writer = err_details.msg.writer(allocator);
try msg_writer.writeAll("this argument was inferred to be a filepath, so argument parsing was terminated");
try diagnostics.append(err_details);
break;
}
while (arg.name().len > 0) { while (arg.name().len > 0) {
const arg_name = arg.name(); const arg_name = arg.name();
// Note: These cases should be in order from longest to shortest, since // Note: These cases should be in order from longest to shortest, since
@ -440,24 +470,6 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
if (std.ascii.startsWithIgnoreCase(arg_name, ":no-preprocess")) { if (std.ascii.startsWithIgnoreCase(arg_name, ":no-preprocess")) {
options.preprocess = .no; options.preprocess = .no;
arg.name_offset += ":no-preprocess".len; arg.name_offset += ":no-preprocess".len;
} else if (std.ascii.startsWithIgnoreCase(arg_name, ":mingw-includes")) {
const value = arg.value(":mingw-includes".len, arg_i, args) catch {
var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = arg.missingSpan() };
var msg_writer = err_details.msg.writer(allocator);
try msg_writer.print("missing value after {s}{s} option", .{ arg.prefixSlice(), arg.optionWithoutPrefix(":mingw-includes".len) });
try diagnostics.append(err_details);
arg_i += 1;
break :next_arg;
};
if (options.mingw_includes_dir) |overwritten_path| {
allocator.free(overwritten_path);
options.mingw_includes_dir = null;
}
const path = try allocator.dupe(u8, value.slice);
errdefer allocator.free(path);
options.mingw_includes_dir = path;
arg_i += value.index_increment;
continue :next_arg;
} else if (std.ascii.startsWithIgnoreCase(arg_name, ":auto-includes")) { } else if (std.ascii.startsWithIgnoreCase(arg_name, ":auto-includes")) {
const value = arg.value(":auto-includes".len, arg_i, args) catch { const value = arg.value(":auto-includes".len, arg_i, args) catch {
var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = arg.missingSpan() }; var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = arg.missingSpan() };
@ -769,7 +781,7 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
arg_i += value.index_increment; arg_i += value.index_increment;
continue :next_arg; continue :next_arg;
}; };
options.default_code_page = CodePage.getByIdentifierEnsureSupported(code_page_id) catch |err| switch (err) { options.default_code_page = code_pages.getByIdentifierEnsureSupported(code_page_id) catch |err| switch (err) {
error.InvalidCodePage => { error.InvalidCodePage => {
var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = value.argSpan(arg) }; var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = value.argSpan(arg) };
var msg_writer = err_details.msg.writer(allocator); var msg_writer = err_details.msg.writer(allocator);
@ -782,7 +794,7 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = value.argSpan(arg) }; var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = value.argSpan(arg) };
var msg_writer = err_details.msg.writer(allocator); var msg_writer = err_details.msg.writer(allocator);
try msg_writer.print("unsupported code page: {s} (id={})", .{ try msg_writer.print("unsupported code page: {s} (id={})", .{
@tagName(CodePage.getByIdentifier(code_page_id) catch unreachable), @tagName(code_pages.getByIdentifier(code_page_id) catch unreachable),
code_page_id, code_page_id,
}); });
try diagnostics.append(err_details); try diagnostics.append(err_details);
@ -900,12 +912,13 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
const positionals = args[arg_i..]; const positionals = args[arg_i..];
if (positionals.len < 1) { if (positionals.len == 0) {
var err_details = Diagnostics.ErrorDetails{ .print_args = false, .arg_index = arg_i }; var err_details = Diagnostics.ErrorDetails{ .print_args = false, .arg_index = arg_i };
var msg_writer = err_details.msg.writer(allocator); var msg_writer = err_details.msg.writer(allocator);
try msg_writer.writeAll("missing input filename"); try msg_writer.writeAll("missing input filename");
try diagnostics.append(err_details); try diagnostics.append(err_details);
if (args.len > 0) {
const last_arg = args[args.len - 1]; const last_arg = args[args.len - 1];
if (arg_i > 0 and last_arg.len > 0 and last_arg[0] == '/' and std.ascii.endsWithIgnoreCase(last_arg, ".rc")) { if (arg_i > 0 and last_arg.len > 0 and last_arg[0] == '/' and std.ascii.endsWithIgnoreCase(last_arg, ".rc")) {
var note_details = Diagnostics.ErrorDetails{ .type = .note, .print_args = true, .arg_index = arg_i - 1 }; var note_details = Diagnostics.ErrorDetails{ .type = .note, .print_args = true, .arg_index = arg_i - 1 };
@ -913,6 +926,7 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
try note_writer.writeAll("if this argument was intended to be the input filename, then -- should be specified in front of it to exclude it from option parsing"); try note_writer.writeAll("if this argument was intended to be the input filename, then -- should be specified in front of it to exclude it from option parsing");
try diagnostics.append(note_details); try diagnostics.append(note_details);
} }
}
// This is a fatal enough problem to justify an early return, since // This is a fatal enough problem to justify an early return, since
// things after this rely on the value of the input filename. // things after this rely on the value of the input filename.
@ -969,6 +983,12 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
return options; return options;
} }
pub fn isSupportedInputExtension(ext: []const u8) bool {
if (std.ascii.eqlIgnoreCase(ext, ".rc")) return true;
if (std.ascii.eqlIgnoreCase(ext, ".rcpp")) return true;
return false;
}
/// Returns true if the str is a valid C identifier for use in a #define/#undef macro /// Returns true if the str is a valid C identifier for use in a #define/#undef macro
pub fn isValidIdentifier(str: []const u8) bool { pub fn isValidIdentifier(str: []const u8) bool {
for (str, 0..) |c, i| switch (c) { for (str, 0..) |c, i| switch (c) {
@ -1271,6 +1291,43 @@ test "parse errors: basic" {
); );
} }
test "inferred absolute filepaths" {
{
var options = try testParseWarning(&.{ "/fo", "foo.res", "/home/absolute/path.rc" },
\\<cli>: note: this argument was inferred to be a filepath, so argument parsing was terminated
\\ ... /home/absolute/path.rc
\\ ^~~~~~~~~~~~~~~~~~~~~~
\\
);
defer options.deinit();
}
{
var options = try testParseWarning(&.{ "/home/absolute/path.rc", "foo.res" },
\\<cli>: note: this argument was inferred to be a filepath, so argument parsing was terminated
\\ ... /home/absolute/path.rc ...
\\ ^~~~~~~~~~~~~~~~~~~~~~
\\
);
defer options.deinit();
}
{
// Only the last two arguments are checked, so the /h is parsed as an option
var options = try testParse(&.{ "/home/absolute/path.rc", "foo.rc", "foo.res" });
defer options.deinit();
try std.testing.expect(options.print_help_and_exit);
}
{
var options = try testParse(&.{ "/xvFO/some/absolute/path.res", "foo.rc" });
defer options.deinit();
try std.testing.expectEqual(true, options.verbose);
try std.testing.expectEqual(true, options.ignore_include_env_var);
try std.testing.expectEqualStrings("foo.rc", options.input_source.filename);
try std.testing.expectEqualStrings("/some/absolute/path.res", options.output_source.filename);
}
}
test "parse errors: /ln" { test "parse errors: /ln" {
try testParseError(&.{ "/ln", "invalid", "foo.rc" }, try testParseError(&.{ "/ln", "invalid", "foo.rc" },
\\<cli>: error: invalid language tag: invalid \\<cli>: error: invalid language tag: invalid

203
lib/compiler/resinator/code_pages.zig
View File

@ -1,86 +1,30 @@
const std = @import("std"); const std = @import("std");
const windows1252 = @import("windows1252.zig"); const windows1252 = @import("windows1252.zig");
// TODO: Parts of this comment block may be more relevant to string/NameOrOrdinal parsing
// than it is to the stuff in this file.
//
// ‰ representations for context:
// Win-1252 89
// UTF-8 E2 80 B0
// UTF-16 20 30
//
// With code page 65001:
// ‰ RCDATA { "‰" L"‰" }
// File encoded as Windows-1252:
// ‰ => <U+FFFD REPLACEMENT CHARACTER> as u16
// "‰" => 0x3F ('?')
// L"‰" => <U+FFFD REPLACEMENT CHARACTER> as u16
// File encoded as UTF-8:
// ‰ => <U+2030 ‰> as u16
// "‰" => 0x89 ('‰' encoded as Windows-1252)
// L"‰" => <U+2030 ‰> as u16
//
// With code page 1252:
// ‰ RCDATA { "‰" L"‰" }
// File encoded as Windows-1252:
// ‰ => <U+2030 ‰> as u16
// "‰" => 0x89 ('‰' encoded as Windows-1252)
// L"‰" => <U+2030 ‰> as u16
// File encoded as UTF-8:
// ‰ => 0xE2 as u16, 0x20AC as u16, 0xB0 as u16
// ^ first byte of utf8 representation
// ^ second byte of UTF-8 representation (0x80), but interpretted as
// Windows-1252 ('€') and then converted to UTF-16 (<U+20AC>)
// ^ third byte of utf8 representation
// "‰" => 0xE2, 0x80, 0xB0 (the bytes of the UTF-8 representation)
// L"‰" => 0xE2 as u16, 0x20AC as u16, 0xB0 as u16 (see '‰ =>' explanation)
//
// With code page 1252:
// <0x90> RCDATA { "<0x90>" L"<0x90>" }
// File encoded as Windows-1252:
// <0x90> => 0x90 as u16
// "<0x90>" => 0x90
// L"<0x90>" => 0x90 as u16
// File encoded as UTF-8:
// <0x90> => 0xC2 as u16, 0x90 as u16
// "<0x90>" => 0xC2, 0x90 (the bytes of the UTF-8 representation of <U+0090>)
// L"<0x90>" => 0xC2 as u16, 0x90 as u16
//
// Within a raw data block, file encoded as Windows-1252 (Â is <0xC2>):
// "Âa" L"Âa" "\xC2ad" L"\xC2AD"
// With code page 1252:
// C2 61 C2 00 61 00 C2 61 64 AD C2
// Â^ a^ Â~~~^ a~~~^ .^ a^ d^ ^~~~~\xC2AD
// \xC2~`
// With code page 65001:
// 3F 61 FD FF 61 00 C2 61 64 AD C2
// ^. a^ ^~~~. a~~~^ ^. a^ d^ ^~~~~\xC2AD
// `. `. `~\xC2
// `. `.~<0xC2>a is not well-formed UTF-8 (0xC2 expects a continutation byte after it).
// `. Because 'a' is a valid first byte of a UTF-8 sequence, it is not included in the
// `. invalid sequence so only the <0xC2> gets converted to <U+FFFD>.
// `~Same as ^ but converted to '?' instead.
//
// Within a raw data block, file encoded as Windows-1252 (ð is <0xF0>, € is <0x80>):
// "ð€a" L"ð€a"
// With code page 1252:
// F0 80 61 F0 00 AC 20 61 00
// ð^ €^ a^ ð~~~^ €~~~^ a~~~^
// With code page 65001:
// 3F 61 FD FF 61 00
// ^. a^ ^~~~. a~~~^
// `. `.
// `. `.~<0xF0><0x80> is not well-formed UTF-8, and <0x80> is not a valid first byte, so
// `. both bytes are considered an invalid sequence and get converted to '<U+FFFD>'
// `~Same as ^ but converted to '?' instead.
/// https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers /// https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
pub const CodePage = enum(u16) { pub const SupportedCodePage = enum(u16) {
// supported
windows1252 = 1252, // windows-1252 ANSI Latin 1; Western European (Windows) windows1252 = 1252, // windows-1252 ANSI Latin 1; Western European (Windows)
utf8 = 65001, // utf-8 Unicode (UTF-8) utf8 = 65001, // utf-8 Unicode (UTF-8)
// unsupported but valid pub fn codepointAt(code_page: SupportedCodePage, index: usize, bytes: []const u8) ?Codepoint {
if (index >= bytes.len) return null;
switch (code_page) {
.windows1252 => {
// All byte values have a representation, so just convert the byte
return Codepoint{
.value = windows1252.toCodepoint(bytes[index]),
.byte_len = 1,
};
},
.utf8 => {
return Utf8.WellFormedDecoder.decode(bytes[index..]);
},
}
}
};
/// https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
pub const UnsupportedCodePage = enum(u16) {
ibm037 = 37, // IBM037 IBM EBCDIC US-Canada ibm037 = 37, // IBM037 IBM EBCDIC US-Canada
ibm437 = 437, // IBM437 OEM United States ibm437 = 437, // IBM437 OEM United States
ibm500 = 500, // IBM500 IBM EBCDIC International ibm500 = 500, // IBM500 IBM EBCDIC International
@ -231,32 +175,28 @@ pub const CodePage = enum(u16) {
x_iscii_gu = 57010, // x-iscii-gu ISCII Gujarati x_iscii_gu = 57010, // x-iscii-gu ISCII Gujarati
x_iscii_pa = 57011, // x-iscii-pa ISCII Punjabi x_iscii_pa = 57011, // x-iscii-pa ISCII Punjabi
utf7 = 65000, // utf-7 Unicode (UTF-7) utf7 = 65000, // utf-7 Unicode (UTF-7)
};
pub fn codepointAt(code_page: CodePage, index: usize, bytes: []const u8) ?Codepoint { pub const CodePage = blk: {
if (index >= bytes.len) return null; const fields = @typeInfo(SupportedCodePage).@"enum".fields ++ @typeInfo(UnsupportedCodePage).@"enum".fields;
switch (code_page) { break :blk @Type(.{ .@"enum" = .{
.windows1252 => { .tag_type = u16,
// All byte values have a representation, so just convert the byte .decls = &.{},
return Codepoint{ .fields = fields,
.value = windows1252.toCodepoint(bytes[index]), .is_exhaustive = true,
.byte_len = 1, } });
}; };
},
.utf8 => { pub fn isSupported(code_page: CodePage) bool {
return Utf8.WellFormedDecoder.decode(bytes[index..]); inline for (@typeInfo(SupportedCodePage).@"enum".fields) |enumField| {
}, if (@intFromEnum(code_page) == @intFromEnum(@field(SupportedCodePage, enumField.name))) {
else => unreachable, return true;
} }
} }
return false;
}
pub fn isSupported(code_page: CodePage) bool { pub fn getByIdentifier(identifier: u16) !CodePage {
return switch (code_page) {
.windows1252, .utf8 => true,
else => false,
};
}
pub fn getByIdentifier(identifier: u16) !CodePage {
// There's probably a more efficient way to do this (e.g. ComptimeHashMap?) but // There's probably a more efficient way to do this (e.g. ComptimeHashMap?) but
// this should be fine, especially since this function likely won't be called much. // this should be fine, especially since this function likely won't be called much.
inline for (@typeInfo(CodePage).@"enum".fields) |enumField| { inline for (@typeInfo(CodePage).@"enum".fields) |enumField| {
@ -265,16 +205,15 @@ pub const CodePage = enum(u16) {
} }
} }
return error.InvalidCodePage; return error.InvalidCodePage;
} }
pub fn getByIdentifierEnsureSupported(identifier: u16) !CodePage { pub fn getByIdentifierEnsureSupported(identifier: u16) !SupportedCodePage {
const code_page = try getByIdentifier(identifier); const code_page = try getByIdentifier(identifier);
switch (isSupported(code_page)) { return if (isSupported(code_page))
true => return code_page, @enumFromInt(@intFromEnum(code_page))
false => return error.UnsupportedCodePage, else
} error.UnsupportedCodePage;
} }
};
pub const Utf8 = struct { pub const Utf8 = struct {
/// Implements decoding with rejection of ill-formed UTF-8 sequences based on section /// Implements decoding with rejection of ill-formed UTF-8 sequences based on section
@ -378,20 +317,20 @@ test "codepointAt invalid utf8" {
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 1, .byte_len = 1,
}, CodePage.utf8.codepointAt(0, invalid_utf8).?); }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 2, .byte_len = 2,
}, CodePage.utf8.codepointAt(1, invalid_utf8).?); }, SupportedCodePage.utf8.codepointAt(1, invalid_utf8).?);
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 1, .byte_len = 1,
}, CodePage.utf8.codepointAt(3, invalid_utf8).?); }, SupportedCodePage.utf8.codepointAt(3, invalid_utf8).?);
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 1, .byte_len = 1,
}, CodePage.utf8.codepointAt(4, invalid_utf8).?); }, SupportedCodePage.utf8.codepointAt(4, invalid_utf8).?);
try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(5, invalid_utf8)); try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(5, invalid_utf8));
} }
{ {
@ -399,12 +338,12 @@ test "codepointAt invalid utf8" {
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 2, .byte_len = 2,
}, CodePage.utf8.codepointAt(0, invalid_utf8).?); }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 1, .byte_len = 1,
}, CodePage.utf8.codepointAt(2, invalid_utf8).?); }, SupportedCodePage.utf8.codepointAt(2, invalid_utf8).?);
try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(3, invalid_utf8)); try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(3, invalid_utf8));
} }
{ {
@ -412,8 +351,8 @@ test "codepointAt invalid utf8" {
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 1, .byte_len = 1,
}, CodePage.utf8.codepointAt(0, invalid_utf8).?); }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(1, invalid_utf8)); try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(1, invalid_utf8));
} }
{ {
@ -421,8 +360,8 @@ test "codepointAt invalid utf8" {
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 2, .byte_len = 2,
}, CodePage.utf8.codepointAt(0, invalid_utf8).?); }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(2, invalid_utf8)); try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, invalid_utf8));
} }
{ {
@ -430,12 +369,12 @@ test "codepointAt invalid utf8" {
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 1, .byte_len = 1,
}, CodePage.utf8.codepointAt(0, invalid_utf8).?); }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 1, .byte_len = 1,
}, CodePage.utf8.codepointAt(1, invalid_utf8).?); }, SupportedCodePage.utf8.codepointAt(1, invalid_utf8).?);
try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(2, invalid_utf8)); try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, invalid_utf8));
} }
{ {
@ -444,11 +383,11 @@ test "codepointAt invalid utf8" {
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 2, .byte_len = 2,
}, CodePage.utf8.codepointAt(0, invalid_utf8).?); }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 1, .byte_len = 1,
}, CodePage.utf8.codepointAt(2, invalid_utf8).?); }, SupportedCodePage.utf8.codepointAt(2, invalid_utf8).?);
} }
} }
@ -459,19 +398,19 @@ test "codepointAt utf8 encoded" {
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = '²', .value = '²',
.byte_len = 2, .byte_len = 2,
}, CodePage.utf8.codepointAt(0, utf8_encoded).?); }, SupportedCodePage.utf8.codepointAt(0, utf8_encoded).?);
try std.testing.expectEqual(@as(?Codepoint, null), CodePage.utf8.codepointAt(2, utf8_encoded)); try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, utf8_encoded));
// with code page windows1252 // with code page windows1252
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = '\xC2', .value = '\xC2',
.byte_len = 1, .byte_len = 1,
}, CodePage.windows1252.codepointAt(0, utf8_encoded).?); }, SupportedCodePage.windows1252.codepointAt(0, utf8_encoded).?);
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = '\xB2', .value = '\xB2',
.byte_len = 1, .byte_len = 1,
}, CodePage.windows1252.codepointAt(1, utf8_encoded).?); }, SupportedCodePage.windows1252.codepointAt(1, utf8_encoded).?);
try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(2, utf8_encoded)); try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.windows1252.codepointAt(2, utf8_encoded));
} }
test "codepointAt windows1252 encoded" { test "codepointAt windows1252 encoded" {
@ -481,15 +420,15 @@ test "codepointAt windows1252 encoded" {
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = Codepoint.invalid, .value = Codepoint.invalid,
.byte_len = 1, .byte_len = 1,
}, CodePage.utf8.codepointAt(0, windows1252_encoded).?); }, SupportedCodePage.utf8.codepointAt(0, windows1252_encoded).?);
try std.testing.expectEqual(@as(?Codepoint, null), CodePage.utf8.codepointAt(2, windows1252_encoded)); try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, windows1252_encoded));
// with code page windows1252 // with code page windows1252
try std.testing.expectEqual(Codepoint{ try std.testing.expectEqual(Codepoint{
.value = '\xB2', .value = '\xB2',
.byte_len = 1, .byte_len = 1,
}, CodePage.windows1252.codepointAt(0, windows1252_encoded).?); }, SupportedCodePage.windows1252.codepointAt(0, windows1252_encoded).?);
try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(1, windows1252_encoded)); try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.windows1252.codepointAt(1, windows1252_encoded));
} }
pub const Codepoint = struct { pub const Codepoint = struct {

25
lib/compiler/resinator/comments.zig
View File

@ -174,6 +174,21 @@ pub fn removeComments(source: []const u8, buf: []u8, source_mappings: ?*SourceMa
}, },
}, },
} }
} else {
switch (state) {
.start,
.line_comment,
.multiline_comment,
.multiline_comment_end,
.single_quoted,
.single_quoted_escape,
.double_quoted,
.double_quoted_escape,
=> {},
.forward_slash => {
result.writeSlice(source[pending_start.?..index]);
},
}
} }
return result.getWritten(); return result.getWritten();
} }
@ -334,6 +349,16 @@ test "comments appended to a line" {
); );
} }
test "forward slash only" {
try testRemoveComments(
\\ /
\\/
,
\\ /
\\/
);
}
test "remove comments with mappings" { test "remove comments with mappings" {
const allocator = std.testing.allocator; const allocator = std.testing.allocator;
var mut_source = "blah/*\rcommented line*\r/blah".*; var mut_source = "blah/*\rcommented line*\r/blah".*;

296
lib/compiler/resinator/compile.zig
View File

@ -4,7 +4,7 @@ const Allocator = std.mem.Allocator;
const Node = @import("ast.zig").Node; const Node = @import("ast.zig").Node;
const lex = @import("lex.zig"); const lex = @import("lex.zig");
const Parser = @import("parse.zig").Parser; const Parser = @import("parse.zig").Parser;
const Resource = @import("rc.zig").Resource; const ResourceType = @import("rc.zig").ResourceType;
const Token = @import("lex.zig").Token; const Token = @import("lex.zig").Token;
const literals = @import("literals.zig"); const literals = @import("literals.zig");
const Number = literals.Number; const Number = literals.Number;
@ -21,7 +21,7 @@ const WORD = std.os.windows.WORD;
const DWORD = std.os.windows.DWORD; const DWORD = std.os.windows.DWORD;
const utils = @import("utils.zig"); const utils = @import("utils.zig");
const NameOrOrdinal = res.NameOrOrdinal; const NameOrOrdinal = res.NameOrOrdinal;
const CodePage = @import("code_pages.zig").CodePage; const SupportedCodePage = @import("code_pages.zig").SupportedCodePage;
const CodePageLookup = @import("ast.zig").CodePageLookup; const CodePageLookup = @import("ast.zig").CodePageLookup;
const SourceMappings = @import("source_mapping.zig").SourceMappings; const SourceMappings = @import("source_mapping.zig").SourceMappings;
const windows1252 = @import("windows1252.zig"); const windows1252 = @import("windows1252.zig");
@ -39,7 +39,10 @@ pub const CompileOptions = struct {
/// freed by the caller. /// freed by the caller.
/// TODO: Maybe a dedicated struct for this purpose so that it's a bit nicer to work with. /// TODO: Maybe a dedicated struct for this purpose so that it's a bit nicer to work with.
dependencies_list: ?*std.ArrayList([]const u8) = null, dependencies_list: ?*std.ArrayList([]const u8) = null,
default_code_page: CodePage = .windows1252, default_code_page: SupportedCodePage = .windows1252,
/// If true, the first #pragma code_page directive only sets the input code page, but not the output code page.
/// This check must be done before comments are removed from the file.
disjoint_code_page: bool = false,
ignore_include_env_var: bool = false, ignore_include_env_var: bool = false,
extra_include_paths: []const []const u8 = &.{}, extra_include_paths: []const []const u8 = &.{},
/// This is just an API convenience to allow separately passing 'system' (i.e. those /// This is just an API convenience to allow separately passing 'system' (i.e. those
@ -66,6 +69,7 @@ pub fn compile(allocator: Allocator, source: []const u8, writer: anytype, option
}); });
var parser = Parser.init(&lexer, .{ var parser = Parser.init(&lexer, .{
.warn_instead_of_error_on_invalid_code_page = options.warn_instead_of_error_on_invalid_code_page, .warn_instead_of_error_on_invalid_code_page = options.warn_instead_of_error_on_invalid_code_page,
.disjoint_code_page = options.disjoint_code_page,
}); });
var tree = try parser.parse(allocator, options.diagnostics); var tree = try parser.parse(allocator, options.diagnostics);
defer tree.deinit(); defer tree.deinit();
@ -98,6 +102,7 @@ pub fn compile(allocator: Allocator, source: []const u8, writer: anytype, option
.end = 0, .end = 0,
.line_number = 1, .line_number = 1,
}, },
.code_page = .utf8,
.print_source_line = false, .print_source_line = false,
.extra = .{ .file_open_error = .{ .extra = .{ .file_open_error = .{
.err = ErrorDetails.FileOpenError.enumFromError(err), .err = ErrorDetails.FileOpenError.enumFromError(err),
@ -213,7 +218,12 @@ pub const Compiler = struct {
try self.addErrorDetails(.{ try self.addErrorDetails(.{
.err = .result_contains_fontdir, .err = .result_contains_fontdir,
.type = .hint, .type = .hint,
.token = undefined, .token = .{
.id = .invalid,
.start = 0,
.end = 0,
.line_number = 1,
},
}); });
} }
// once we've written every else out, we can write out the finalized STRINGTABLE resources // once we've written every else out, we can write out the finalized STRINGTABLE resources
@ -301,7 +311,10 @@ pub const Compiler = struct {
// UTF-8, we can parse either string type directly to UTF-8. // UTF-8, we can parse either string type directly to UTF-8.
var parser = literals.IterativeStringParser.init(bytes, .{ var parser = literals.IterativeStringParser.init(bytes, .{
.start_column = column, .start_column = column,
.diagnostics = .{ .diagnostics = self.diagnostics, .token = literal_node.token }, .diagnostics = self.errContext(literal_node.token),
// TODO: Re-evaluate this. It's not been tested whether or not using the actual
// output code page would make more sense.
.output_code_page = .windows1252,
}); });
while (try parser.nextUnchecked()) |parsed| { while (try parser.nextUnchecked()) |parsed| {
@ -401,56 +414,55 @@ pub const Compiler = struct {
return first_error orelse error.FileNotFound; return first_error orelse error.FileNotFound;
} }
/// Returns a Windows-1252 encoded string regardless of the current output code page.
/// All codepoints are encoded as a maximum of 2 bytes, where unescaped codepoints
/// >= 0x10000 are encoded as `??` and everything else is encoded as 1 byte.
pub fn parseDlgIncludeString(self: *Compiler, token: Token) ![]u8 { pub fn parseDlgIncludeString(self: *Compiler, token: Token) ![]u8 {
// For the purposes of parsing, we want to strip the L prefix const bytes = self.sourceBytesForToken(token);
// if it exists since we want escaped integers to be limited to const output_code_page = self.output_code_pages.getForToken(token);
// their ascii string range.
//
// We keep track of whether or not there was an L prefix, though,
// since there's more weirdness to come.
var bytes = self.sourceBytesForToken(token);
var was_wide_string = false;
if (bytes.slice[0] == 'L' or bytes.slice[0] == 'l') {
was_wide_string = true;
bytes.slice = bytes.slice[1..];
}
var buf = try std.ArrayList(u8).initCapacity(self.allocator, bytes.slice.len); var buf = try std.ArrayList(u8).initCapacity(self.allocator, bytes.slice.len);
errdefer buf.deinit(); errdefer buf.deinit();
var iterative_parser = literals.IterativeStringParser.init(bytes, .{ var iterative_parser = literals.IterativeStringParser.init(bytes, .{
.start_column = token.calculateColumn(self.source, 8, null), .start_column = token.calculateColumn(self.source, 8, null),
.diagnostics = .{ .diagnostics = self.diagnostics, .token = token }, .diagnostics = self.errContext(token),
// TODO: Potentially re-evaluate this, it's not been tested whether or not
// using the actual output code page would make more sense.
.output_code_page = .windows1252,
}); });
// No real idea what's going on here, but this matches the rc.exe behavior // This is similar to the logic in parseQuotedString, but ends up with everything
// encoded as Windows-1252. This effectively consolidates the two-step process
// of rc.exe into one step, since rc.exe's preprocessor converts to UTF-16 (this
// is when invalid sequences are replaced by the replacement character (U+FFFD)),
// and then that's run through the parser. Our preprocessor keeps things in their
// original encoding, meaning we emulate the <encoding> -> UTF-16 -> Windows-1252
// results all at once.
while (try iterative_parser.next()) |parsed| { while (try iterative_parser.next()) |parsed| {
const c = parsed.codepoint; const c = parsed.codepoint;
switch (was_wide_string) { switch (iterative_parser.declared_string_type) {
true => { .wide => {
switch (c) {
0...0x7F, 0xA0...0xFF => try buf.append(@intCast(c)),
0x80...0x9F => {
if (windows1252.bestFitFromCodepoint(c)) |_| {
try buf.append(@intCast(c));
} else {
try buf.append('?');
}
},
else => {
if (windows1252.bestFitFromCodepoint(c)) |best_fit| { if (windows1252.bestFitFromCodepoint(c)) |best_fit| {
try buf.append(best_fit); try buf.append(best_fit);
} else if (c < 0x10000 or c == code_pages.Codepoint.invalid) { } else if (c < 0x10000 or c == code_pages.Codepoint.invalid or parsed.escaped_surrogate_pair) {
try buf.append('?'); try buf.append('?');
} else { } else {
try buf.appendSlice("??"); try buf.appendSlice("??");
} }
}, },
} .ascii => {
},
false => {
if (parsed.from_escaped_integer) { if (parsed.from_escaped_integer) {
try buf.append(@truncate(c)); const truncated: u8 = @truncate(c);
switch (output_code_page) {
.utf8 => switch (truncated) {
0...0x7F => try buf.append(truncated),
else => try buf.append('?'),
},
.windows1252 => {
try buf.append(truncated);
},
}
} else { } else {
if (windows1252.bestFitFromCodepoint(c)) |best_fit| { if (windows1252.bestFitFromCodepoint(c)) |best_fit| {
try buf.append(best_fit); try buf.append(best_fit);
@ -484,8 +496,12 @@ pub const Compiler = struct {
const parsed_filename_terminated = std.mem.sliceTo(parsed_filename, 0); const parsed_filename_terminated = std.mem.sliceTo(parsed_filename, 0);
header.applyMemoryFlags(node.common_resource_attributes, self.source); header.applyMemoryFlags(node.common_resource_attributes, self.source);
// This is effectively limited by `max_string_literal_codepoints` which is a u15.
// Each codepoint within a DLGINCLUDE string is encoded as a maximum of
// 2 bytes, which means that the maximum byte length of a DLGINCLUDE string is
// (including the NUL terminator): 32,767 * 2 + 1 = 65,535 or exactly the u16 max.
header.data_size = @intCast(parsed_filename_terminated.len + 1); header.data_size = @intCast(parsed_filename_terminated.len + 1);
try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id }); try header.write(writer, self.errContext(node.id));
try writer.writeAll(parsed_filename_terminated); try writer.writeAll(parsed_filename_terminated);
try writer.writeByte(0); try writer.writeByte(0);
try writeDataPadding(writer, header.data_size); try writeDataPadding(writer, header.data_size);
@ -568,7 +584,7 @@ pub const Compiler = struct {
header.applyMemoryFlags(node.common_resource_attributes, self.source); header.applyMemoryFlags(node.common_resource_attributes, self.source);
header.data_size = @intCast(try file.getEndPos()); header.data_size = @intCast(try file.getEndPos());
try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id }); try header.write(writer, self.errContext(node.id));
try file.seekTo(0); try file.seekTo(0);
try writeResourceData(writer, file.reader(), header.data_size); try writeResourceData(writer, file.reader(), header.data_size);
return; return;
@ -644,7 +660,7 @@ pub const Compiler = struct {
.version = self.state.version, .version = self.state.version,
.characteristics = self.state.characteristics, .characteristics = self.state.characteristics,
}; };
try image_header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id }); try image_header.write(writer, self.errContext(node.id));
// From https://learn.microsoft.com/en-us/windows/win32/menurc/localheader: // From https://learn.microsoft.com/en-us/windows/win32/menurc/localheader:
// > The LOCALHEADER structure is the first data written to the RT_CURSOR // > The LOCALHEADER structure is the first data written to the RT_CURSOR
@ -817,12 +833,26 @@ pub const Compiler = struct {
header.data_size = icon_dir.getResDataSize(); header.data_size = icon_dir.getResDataSize();
try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id }); try header.write(writer, self.errContext(node.id));
try icon_dir.writeResData(writer, first_icon_id); try icon_dir.writeResData(writer, first_icon_id);
try writeDataPadding(writer, header.data_size); try writeDataPadding(writer, header.data_size);
return; return;
}, },
.RCDATA, .HTML, .MANIFEST, .MESSAGETABLE, .DLGINIT, .PLUGPLAY => { .RCDATA,
.HTML,
.MESSAGETABLE,
.DLGINIT,
.PLUGPLAY,
.VXD,
// Note: All of the below can only be specified by using a number
// as the resource type.
.MANIFEST,
.CURSOR,
.ICON,
.ANICURSOR,
.ANIICON,
.FONTDIR,
=> {
header.applyMemoryFlags(node.common_resource_attributes, self.source); header.applyMemoryFlags(node.common_resource_attributes, self.source);
}, },
.BITMAP => { .BITMAP => {
@ -855,48 +885,33 @@ pub const Compiler = struct {
} else if (bitmap_info.getActualPaletteByteLen() < bitmap_info.getExpectedPaletteByteLen()) { } else if (bitmap_info.getActualPaletteByteLen() < bitmap_info.getExpectedPaletteByteLen()) {
const num_padding_bytes = bitmap_info.getExpectedPaletteByteLen() - bitmap_info.getActualPaletteByteLen(); const num_padding_bytes = bitmap_info.getExpectedPaletteByteLen() - bitmap_info.getActualPaletteByteLen();
// TODO: Make this configurable (command line option)
const max_missing_bytes = 4096;
if (num_padding_bytes > max_missing_bytes) {
var numbers_as_bytes: [16]u8 = undefined;
std.mem.writeInt(u64, numbers_as_bytes[0..8], num_padding_bytes, native_endian);
std.mem.writeInt(u64, numbers_as_bytes[8..16], max_missing_bytes, native_endian);
const values_string_index = try self.diagnostics.putString(&numbers_as_bytes);
try self.addErrorDetails(.{
.err = .bmp_too_many_missing_palette_bytes,
.token = filename_token,
.extra = .{ .number = values_string_index },
});
return self.addErrorDetailsAndFail(.{
.err = .bmp_too_many_missing_palette_bytes,
.type = .note,
.print_source_line = false,
.token = filename_token,
});
}
var number_as_bytes: [8]u8 = undefined; var number_as_bytes: [8]u8 = undefined;
std.mem.writeInt(u64, &number_as_bytes, num_padding_bytes, native_endian); std.mem.writeInt(u64, &number_as_bytes, num_padding_bytes, native_endian);
const value_string_index = try self.diagnostics.putString(&number_as_bytes); const value_string_index = try self.diagnostics.putString(&number_as_bytes);
try self.addErrorDetails(.{ try self.addErrorDetails(.{
.err = .bmp_missing_palette_bytes, .err = .bmp_missing_palette_bytes,
.type = .warning, .type = .err,
.token = filename_token, .token = filename_token,
.extra = .{ .number = value_string_index }, .extra = .{ .number = value_string_index },
}); });
const pixel_data_len = bitmap_info.getPixelDataLen(file_size); const pixel_data_len = bitmap_info.getPixelDataLen(file_size);
// TODO: This is a hack, but we know we have already added
// at least one entry to the diagnostics strings, so we can
// get away with using 0 to mean 'no string' here.
var miscompiled_bytes_string_index: u32 = 0;
if (pixel_data_len > 0) { if (pixel_data_len > 0) {
const miscompiled_bytes = @min(pixel_data_len, num_padding_bytes); const miscompiled_bytes = @min(pixel_data_len, num_padding_bytes);
std.mem.writeInt(u64, &number_as_bytes, miscompiled_bytes, native_endian); std.mem.writeInt(u64, &number_as_bytes, miscompiled_bytes, native_endian);
const miscompiled_bytes_string_index = try self.diagnostics.putString(&number_as_bytes); miscompiled_bytes_string_index = try self.diagnostics.putString(&number_as_bytes);
try self.addErrorDetails(.{ }
return self.addErrorDetailsAndFail(.{
.err = .rc_would_miscompile_bmp_palette_padding, .err = .rc_would_miscompile_bmp_palette_padding,
.type = .warning, .type = .note,
.print_source_line = false,
.token = filename_token, .token = filename_token,
.extra = .{ .number = miscompiled_bytes_string_index }, .extra = .{ .number = miscompiled_bytes_string_index },
}); });
} }
}
// TODO: It might be possible that the calculation done in this function // TODO: It might be possible that the calculation done in this function
// could underflow if the underlying file is modified while reading // could underflow if the underlying file is modified while reading
@ -905,7 +920,7 @@ pub const Compiler = struct {
const bmp_bytes_to_write: u32 = @intCast(bitmap_info.getExpectedByteLen(file_size)); const bmp_bytes_to_write: u32 = @intCast(bitmap_info.getExpectedByteLen(file_size));
header.data_size = bmp_bytes_to_write; header.data_size = bmp_bytes_to_write;
try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id }); try header.write(writer, self.errContext(node.id));
try file.seekTo(bmp.file_header_len); try file.seekTo(bmp.file_header_len);
const file_reader = file.reader(); const file_reader = file.reader();
try writeResourceDataNoPadding(writer, file_reader, bitmap_info.dib_header_size); try writeResourceDataNoPadding(writer, file_reader, bitmap_info.dib_header_size);
@ -914,12 +929,6 @@ pub const Compiler = struct {
} }
if (bitmap_info.getExpectedPaletteByteLen() > 0) { if (bitmap_info.getExpectedPaletteByteLen() > 0) {
try writeResourceDataNoPadding(writer, file_reader, @intCast(bitmap_info.getActualPaletteByteLen())); try writeResourceDataNoPadding(writer, file_reader, @intCast(bitmap_info.getActualPaletteByteLen()));
// We know that the number of missing palette bytes is <= 4096
// (see `bmp_too_many_missing_palette_bytes` error case above)
const padding_bytes: usize = @intCast(bitmap_info.getMissingPaletteByteLen());
if (padding_bytes > 0) {
try writer.writeByteNTimes(0, padding_bytes);
}
} }
try file.seekTo(bitmap_info.pixel_data_offset); try file.seekTo(bitmap_info.pixel_data_offset);
const pixel_bytes: u32 = @intCast(file_size - bitmap_info.pixel_data_offset); const pixel_bytes: u32 = @intCast(file_size - bitmap_info.pixel_data_offset);
@ -932,13 +941,13 @@ pub const Compiler = struct {
// Add warning and skip this resource // Add warning and skip this resource
// Note: The Win32 compiler prints this as an error but it doesn't fail the compilation // Note: The Win32 compiler prints this as an error but it doesn't fail the compilation
// and the duplicate resource is skipped. // and the duplicate resource is skipped.
try self.addErrorDetails(ErrorDetails{ try self.addErrorDetails(.{
.err = .font_id_already_defined, .err = .font_id_already_defined,
.token = node.id, .token = node.id,
.type = .warning, .type = .warning,
.extra = .{ .number = header.name_value.ordinal }, .extra = .{ .number = header.name_value.ordinal },
}); });
try self.addErrorDetails(ErrorDetails{ try self.addErrorDetails(.{
.err = .font_id_already_defined, .err = .font_id_already_defined,
.token = self.state.font_dir.ids.get(header.name_value.ordinal).?, .token = self.state.font_dir.ids.get(header.name_value.ordinal).?,
.type = .note, .type = .note,
@ -957,7 +966,7 @@ pub const Compiler = struct {
// We now know that the data size will fit in a u32 // We now know that the data size will fit in a u32
header.data_size = @intCast(file_size); header.data_size = @intCast(file_size);
try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id }); try header.write(writer, self.errContext(node.id));
var header_slurping_reader = headerSlurpingReader(148, file.reader()); var header_slurping_reader = headerSlurpingReader(148, file.reader());
try writeResourceData(writer, header_slurping_reader.reader(), header.data_size); try writeResourceData(writer, header_slurping_reader.reader(), header.data_size);
@ -968,19 +977,13 @@ pub const Compiler = struct {
}, node.id); }, node.id);
return; return;
}, },
.ACCELERATOR, .ACCELERATOR, // Cannot use an external file, enforced by the parser
.ANICURSOR, .DIALOG, // Cannot use an external file, enforced by the parser
.ANIICON, .DLGINCLUDE, // Handled specially above
.CURSOR, .MENU, // Cannot use an external file, enforced by the parser
.DIALOG, .STRING, // Parser error if this resource is specified as a number
.DLGINCLUDE, .TOOLBAR, // Cannot use an external file, enforced by the parser
.FONTDIR, .VERSION, // Cannot use an external file, enforced by the parser
.ICON,
.MENU,
.STRING,
.TOOLBAR,
.VERSION,
.VXD,
=> unreachable, => unreachable,
_ => unreachable, _ => unreachable,
} }
@ -998,7 +1001,7 @@ pub const Compiler = struct {
} }
// We now know that the data size will fit in a u32 // We now know that the data size will fit in a u32
header.data_size = @intCast(data_size); header.data_size = @intCast(data_size);
try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id }); try header.write(writer, self.errContext(node.id));
try writeResourceData(writer, file.reader(), header.data_size); try writeResourceData(writer, file.reader(), header.data_size);
} }
@ -1188,7 +1191,7 @@ pub const Compiler = struct {
}; };
const parsed = try literals.parseQuotedAsciiString(self.allocator, bytes, .{ const parsed = try literals.parseQuotedAsciiString(self.allocator, bytes, .{
.start_column = column, .start_column = column,
.diagnostics = .{ .diagnostics = self.diagnostics, .token = literal_node.token }, .diagnostics = self.errContext(literal_node.token),
.output_code_page = self.output_code_pages.getForToken(literal_node.token), .output_code_page = self.output_code_pages.getForToken(literal_node.token),
}); });
errdefer self.allocator.free(parsed); errdefer self.allocator.free(parsed);
@ -1202,7 +1205,8 @@ pub const Compiler = struct {
}; };
const parsed_string = try literals.parseQuotedWideString(self.allocator, bytes, .{ const parsed_string = try literals.parseQuotedWideString(self.allocator, bytes, .{
.start_column = column, .start_column = column,
.diagnostics = .{ .diagnostics = self.diagnostics, .token = literal_node.token }, .diagnostics = self.errContext(literal_node.token),
.output_code_page = self.output_code_pages.getForToken(literal_node.token),
}); });
errdefer self.allocator.free(parsed_string); errdefer self.allocator.free(parsed_string);
return .{ .wide_string = parsed_string }; return .{ .wide_string = parsed_string };
@ -1259,7 +1263,7 @@ pub const Compiler = struct {
header.applyMemoryFlags(common_resource_attributes, self.source); header.applyMemoryFlags(common_resource_attributes, self.source);
try header.write(writer, .{ .diagnostics = self.diagnostics, .token = id_token }); try header.write(writer, self.errContext(id_token));
} }
pub fn writeResourceDataNoPadding(writer: anytype, data_reader: anytype, data_size: u32) !void { pub fn writeResourceDataNoPadding(writer: anytype, data_reader: anytype, data_size: u32) !void {
@ -1297,7 +1301,8 @@ pub const Compiler = struct {
const column = literal.token.calculateColumn(self.source, 8, null); const column = literal.token.calculateColumn(self.source, 8, null);
return res.parseAcceleratorKeyString(bytes, is_virt, .{ return res.parseAcceleratorKeyString(bytes, is_virt, .{
.start_column = column, .start_column = column,
.diagnostics = .{ .diagnostics = self.diagnostics, .token = literal.token }, .diagnostics = self.errContext(literal.token),
.output_code_page = self.output_code_pages.getForToken(literal.token),
}); });
} }
} }
@ -1332,7 +1337,7 @@ pub const Compiler = struct {
header.applyMemoryFlags(node.common_resource_attributes, self.source); header.applyMemoryFlags(node.common_resource_attributes, self.source);
header.applyOptionalStatements(node.optional_statements, self.source, self.input_code_pages); header.applyOptionalStatements(node.optional_statements, self.source, self.input_code_pages);
try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id }); try header.write(writer, self.errContext(node.id));
var data_fbs = std.io.fixedBufferStream(data_buffer.items); var data_fbs = std.io.fixedBufferStream(data_buffer.items);
try writeResourceData(writer, data_fbs.reader(), data_size); try writeResourceData(writer, data_fbs.reader(), data_size);
@ -1348,6 +1353,16 @@ pub const Compiler = struct {
const modifier = rc.AcceleratorTypeAndOptions.map.get(type_or_option.slice(self.source)).?; const modifier = rc.AcceleratorTypeAndOptions.map.get(type_or_option.slice(self.source)).?;
modifiers.apply(modifier); modifiers.apply(modifier);
} }
if ((modifiers.isSet(.control) or modifiers.isSet(.shift)) and !modifiers.isSet(.virtkey)) {
try self.addErrorDetails(.{
.err = .accelerator_shift_or_control_without_virtkey,
.type = .warning,
// We know that one of SHIFT or CONTROL was specified, so there's at least one item
// in this list.
.token = accelerator.type_and_options[0],
.token_span_end = accelerator.type_and_options[accelerator.type_and_options.len - 1],
});
}
if (accelerator.event.isNumberExpression() and !modifiers.explicit_ascii_or_virtkey) { if (accelerator.event.isNumberExpression() and !modifiers.explicit_ascii_or_virtkey) {
return self.addErrorDetailsAndFail(.{ return self.addErrorDetailsAndFail(.{
.err = .accelerator_type_required, .err = .accelerator_type_required,
@ -1399,7 +1414,7 @@ pub const Compiler = struct {
var limited_writer = limitedWriter(data_buffer.writer(), std.math.maxInt(u32)); var limited_writer = limitedWriter(data_buffer.writer(), std.math.maxInt(u32));
const data_writer = limited_writer.writer(); const data_writer = limited_writer.writer();
const resource = Resource.fromString(.{ const resource = ResourceType.fromString(.{
.slice = node.type.slice(self.source), .slice = node.type.slice(self.source),
.code_page = self.input_code_pages.getForToken(node.type), .code_page = self.input_code_pages.getForToken(node.type),
}); });
@ -1414,8 +1429,6 @@ pub const Compiler = struct {
menu.deinit(self.allocator); menu.deinit(self.allocator);
} }
} }
var skipped_menu_or_classes = std.ArrayList(*Node.SimpleStatement).init(self.allocator);
defer skipped_menu_or_classes.deinit();
var last_menu: *Node.SimpleStatement = undefined; var last_menu: *Node.SimpleStatement = undefined;
var last_class: *Node.SimpleStatement = undefined; var last_class: *Node.SimpleStatement = undefined;
var last_menu_would_be_forced_ordinal = false; var last_menu_would_be_forced_ordinal = false;
@ -1445,9 +1458,6 @@ pub const Compiler = struct {
}, },
.class => { .class => {
const is_duplicate = optional_statement_values.class != null; const is_duplicate = optional_statement_values.class != null;
if (is_duplicate) {
try skipped_menu_or_classes.append(last_class);
}
const forced_ordinal = is_duplicate and optional_statement_values.class.? == .ordinal; const forced_ordinal = is_duplicate and optional_statement_values.class.? == .ordinal;
// In the Win32 RC compiler, if any CLASS values that are interpreted as // In the Win32 RC compiler, if any CLASS values that are interpreted as
// an ordinal exist, it affects all future CLASS statements and forces // an ordinal exist, it affects all future CLASS statements and forces
@ -1475,9 +1485,6 @@ pub const Compiler = struct {
}, },
.menu => { .menu => {
const is_duplicate = optional_statement_values.menu != null; const is_duplicate = optional_statement_values.menu != null;
if (is_duplicate) {
try skipped_menu_or_classes.append(last_menu);
}
const forced_ordinal = is_duplicate and optional_statement_values.menu.? == .ordinal; const forced_ordinal = is_duplicate and optional_statement_values.menu.? == .ordinal;
// In the Win32 RC compiler, if any MENU values that are interpreted as // In the Win32 RC compiler, if any MENU values that are interpreted as
// an ordinal exist, it affects all future MENU statements and forces // an ordinal exist, it affects all future MENU statements and forces
@ -1561,22 +1568,6 @@ pub const Compiler = struct {
} }
} }
for (skipped_menu_or_classes.items) |simple_statement| {
const statement_identifier = simple_statement.identifier;
const statement_type = rc.OptionalStatements.dialog_map.get(statement_identifier.slice(self.source)) orelse continue;
try self.addErrorDetails(.{
.err = .duplicate_menu_or_class_skipped,
.type = .warning,
.token = simple_statement.identifier,
.token_span_start = simple_statement.base.getFirstToken(),
.token_span_end = simple_statement.base.getLastToken(),
.extra = .{ .menu_or_class = switch (statement_type) {
.menu => .menu,
.class => .class,
else => unreachable,
} },
});
}
// The Win32 RC compiler miscompiles the value in the following scenario: // The Win32 RC compiler miscompiles the value in the following scenario:
// Multiple CLASS parameters are specified and any of them are treated as a number, then // Multiple CLASS parameters are specified and any of them are treated as a number, then
// the last CLASS is always treated as a number no matter what // the last CLASS is always treated as a number no matter what
@ -1739,7 +1730,7 @@ pub const Compiler = struct {
header.applyMemoryFlags(node.common_resource_attributes, self.source); header.applyMemoryFlags(node.common_resource_attributes, self.source);
header.applyOptionalStatements(node.optional_statements, self.source, self.input_code_pages); header.applyOptionalStatements(node.optional_statements, self.source, self.input_code_pages);
try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id }); try header.write(writer, self.errContext(node.id));
var data_fbs = std.io.fixedBufferStream(data_buffer.items); var data_fbs = std.io.fixedBufferStream(data_buffer.items);
try writeResourceData(writer, data_fbs.reader(), data_size); try writeResourceData(writer, data_fbs.reader(), data_size);
@ -1749,7 +1740,7 @@ pub const Compiler = struct {
self: *Compiler, self: *Compiler,
node: *Node.Dialog, node: *Node.Dialog,
data_writer: anytype, data_writer: anytype,
resource: Resource, resource: ResourceType,
optional_statement_values: *const DialogOptionalStatementValues, optional_statement_values: *const DialogOptionalStatementValues,
x: Number, x: Number,
y: Number, y: Number,
@ -1809,7 +1800,7 @@ pub const Compiler = struct {
self: *Compiler, self: *Compiler,
control: *Node.ControlStatement, control: *Node.ControlStatement,
data_writer: anytype, data_writer: anytype,
resource: Resource, resource: ResourceType,
bytes_written_so_far: u32, bytes_written_so_far: u32,
controls_by_id: *std.AutoHashMap(u32, *const Node.ControlStatement), controls_by_id: *std.AutoHashMap(u32, *const Node.ControlStatement),
) !void { ) !void {
@ -2053,7 +2044,7 @@ pub const Compiler = struct {
header.applyMemoryFlags(node.common_resource_attributes, self.source); header.applyMemoryFlags(node.common_resource_attributes, self.source);
try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id }); try header.write(writer, self.errContext(node.id));
var data_fbs = std.io.fixedBufferStream(data_buffer.items); var data_fbs = std.io.fixedBufferStream(data_buffer.items);
try writeResourceData(writer, data_fbs.reader(), data_size); try writeResourceData(writer, data_fbs.reader(), data_size);
@ -2067,7 +2058,7 @@ pub const Compiler = struct {
node: *Node.FontStatement, node: *Node.FontStatement,
}; };
pub fn writeDialogFont(self: *Compiler, resource: Resource, values: FontStatementValues, writer: anytype) !void { pub fn writeDialogFont(self: *Compiler, resource: ResourceType, values: FontStatementValues, writer: anytype) !void {
const node = values.node; const node = values.node;
const point_size = evaluateNumberExpression(node.point_size, self.source, self.input_code_pages); const point_size = evaluateNumberExpression(node.point_size, self.source, self.input_code_pages);
try writer.writeInt(u16, point_size.asWord(), .little); try writer.writeInt(u16, point_size.asWord(), .little);
@ -2104,7 +2095,7 @@ pub const Compiler = struct {
.slice = node.type.slice(self.source), .slice = node.type.slice(self.source),
.code_page = self.input_code_pages.getForToken(node.type), .code_page = self.input_code_pages.getForToken(node.type),
}; };
const resource = Resource.fromString(type_bytes); const resource = ResourceType.fromString(type_bytes);
std.debug.assert(resource == .menu or resource == .menuex); std.debug.assert(resource == .menu or resource == .menuex);
self.writeMenuData(node, data_writer, resource) catch |err| switch (err) { self.writeMenuData(node, data_writer, resource) catch |err| switch (err) {
@ -2128,7 +2119,7 @@ pub const Compiler = struct {
header.applyMemoryFlags(node.common_resource_attributes, self.source); header.applyMemoryFlags(node.common_resource_attributes, self.source);
header.applyOptionalStatements(node.optional_statements, self.source, self.input_code_pages); header.applyOptionalStatements(node.optional_statements, self.source, self.input_code_pages);
try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id }); try header.write(writer, self.errContext(node.id));
var data_fbs = std.io.fixedBufferStream(data_buffer.items); var data_fbs = std.io.fixedBufferStream(data_buffer.items);
try writeResourceData(writer, data_fbs.reader(), data_size); try writeResourceData(writer, data_fbs.reader(), data_size);
@ -2136,7 +2127,7 @@ pub const Compiler = struct {
/// Expects `data_writer` to be a LimitedWriter limited to u32, meaning all writes to /// Expects `data_writer` to be a LimitedWriter limited to u32, meaning all writes to
/// the writer within this function could return error.NoSpaceLeft /// the writer within this function could return error.NoSpaceLeft
pub fn writeMenuData(self: *Compiler, node: *Node.Menu, data_writer: anytype, resource: Resource) !void { pub fn writeMenuData(self: *Compiler, node: *Node.Menu, data_writer: anytype, resource: ResourceType) !void {
// menu header // menu header
const version: u16 = if (resource == .menu) 0 else 1; const version: u16 = if (resource == .menu) 0 else 1;
try data_writer.writeInt(u16, version, .little); try data_writer.writeInt(u16, version, .little);
@ -2393,7 +2384,7 @@ pub const Compiler = struct {
header.applyMemoryFlags(node.common_resource_attributes, self.source); header.applyMemoryFlags(node.common_resource_attributes, self.source);
try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id }); try header.write(writer, self.errContext(node.id));
var data_fbs = std.io.fixedBufferStream(data_buffer.items); var data_fbs = std.io.fixedBufferStream(data_buffer.items);
try writeResourceData(writer, data_fbs.reader(), data_size); try writeResourceData(writer, data_fbs.reader(), data_size);
@ -2525,14 +2516,14 @@ pub const Compiler = struct {
// It might be nice to have these errors point to the ids rather than the // It might be nice to have these errors point to the ids rather than the
// string tokens, but that would mean storing the id token of each string // string tokens, but that would mean storing the id token of each string
// which doesn't seem worth it just for slightly better error messages. // which doesn't seem worth it just for slightly better error messages.
try self.addErrorDetails(ErrorDetails{ try self.addErrorDetails(.{
.err = .string_already_defined, .err = .string_already_defined,
.token = string.string, .token = string.string,
.extra = .{ .string_and_language = .{ .id = string_id, .language = language } }, .extra = .{ .string_and_language = .{ .id = string_id, .language = language } },
}); });
const existing_def_table = self.state.string_tables.tables.getPtr(language).?; const existing_def_table = self.state.string_tables.tables.getPtr(language).?;
const existing_definition = existing_def_table.get(string_id).?; const existing_definition = existing_def_table.get(string_id).?;
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .string_already_defined, .err = .string_already_defined,
.type = .note, .type = .note,
.token = existing_definition, .token = existing_definition,
@ -2628,7 +2619,7 @@ pub const Compiler = struct {
pub fn init(allocator: Allocator, id_bytes: SourceBytes, type_bytes: SourceBytes, data_size: DWORD, language: res.Language, version: DWORD, characteristics: DWORD) InitError!ResourceHeader { pub fn init(allocator: Allocator, id_bytes: SourceBytes, type_bytes: SourceBytes, data_size: DWORD, language: res.Language, version: DWORD, characteristics: DWORD) InitError!ResourceHeader {
const type_value = type: { const type_value = type: {
const resource_type = Resource.fromString(type_bytes); const resource_type = ResourceType.fromString(type_bytes);
if (res.RT.fromResource(resource_type)) |rt_constant| { if (res.RT.fromResource(resource_type)) |rt_constant| {
break :type NameOrOrdinal{ .ordinal = @intFromEnum(rt_constant) }; break :type NameOrOrdinal{ .ordinal = @intFromEnum(rt_constant) };
} else { } else {
@ -2673,7 +2664,7 @@ pub const Compiler = struct {
padding_after_name: u2, padding_after_name: u2,
}; };
fn calcSize(self: ResourceHeader) error{Overflow}!SizeInfo { pub fn calcSize(self: ResourceHeader) error{Overflow}!SizeInfo {
var header_size: u32 = 8; var header_size: u32 = 8;
header_size = try std.math.add( header_size = try std.math.add(
u32, u32,
@ -2699,6 +2690,7 @@ pub const Compiler = struct {
const size_info = self.calcSize() catch { const size_info = self.calcSize() catch {
try err_ctx.diagnostics.append(.{ try err_ctx.diagnostics.append(.{
.err = .resource_data_size_exceeds_max, .err = .resource_data_size_exceeds_max,
.code_page = err_ctx.code_page,
.token = err_ctx.token, .token = err_ctx.token,
}); });
return error.CompileError; return error.CompileError;
@ -2706,7 +2698,7 @@ pub const Compiler = struct {
return self.writeSizeInfo(writer, size_info); return self.writeSizeInfo(writer, size_info);
} }
fn writeSizeInfo(self: ResourceHeader, writer: anytype, size_info: SizeInfo) !void { pub fn writeSizeInfo(self: ResourceHeader, writer: anytype, size_info: SizeInfo) !void {
try writer.writeInt(DWORD, self.data_size, .little); // DataSize try writer.writeInt(DWORD, self.data_size, .little); // DataSize
try writer.writeInt(DWORD, size_info.bytes, .little); // HeaderSize try writer.writeInt(DWORD, size_info.bytes, .little); // HeaderSize
try self.type_value.write(writer); // TYPE try self.type_value.write(writer); // TYPE
@ -2863,19 +2855,44 @@ pub const Compiler = struct {
self.sourceBytesForToken(token), self.sourceBytesForToken(token),
.{ .{
.start_column = token.calculateColumn(self.source, 8, null), .start_column = token.calculateColumn(self.source, 8, null),
.diagnostics = .{ .diagnostics = self.diagnostics, .token = token }, .diagnostics = self.errContext(token),
.output_code_page = self.output_code_pages.getForToken(token),
}, },
); );
} }
fn addErrorDetails(self: *Compiler, details: ErrorDetails) Allocator.Error!void { fn addErrorDetailsWithCodePage(self: *Compiler, details: ErrorDetails) Allocator.Error!void {
try self.diagnostics.append(details); try self.diagnostics.append(details);
} }
fn addErrorDetailsAndFail(self: *Compiler, details: ErrorDetails) error{ CompileError, OutOfMemory } { /// Code page is looked up in input_code_pages using the token
try self.addErrorDetails(details); fn addErrorDetails(self: *Compiler, details_without_code_page: errors.ErrorDetailsWithoutCodePage) Allocator.Error!void {
const details = ErrorDetails{
.err = details_without_code_page.err,
.code_page = self.input_code_pages.getForToken(details_without_code_page.token),
.token = details_without_code_page.token,
.token_span_start = details_without_code_page.token_span_start,
.token_span_end = details_without_code_page.token_span_end,
.type = details_without_code_page.type,
.print_source_line = details_without_code_page.print_source_line,
.extra = details_without_code_page.extra,
};
try self.addErrorDetailsWithCodePage(details);
}
/// Code page is looked up in input_code_pages using the token
fn addErrorDetailsAndFail(self: *Compiler, details_without_code_page: errors.ErrorDetailsWithoutCodePage) error{ CompileError, OutOfMemory } {
try self.addErrorDetails(details_without_code_page);
return error.CompileError; return error.CompileError;
} }
fn errContext(self: *Compiler, token: Token) errors.DiagnosticsContext {
return .{
.diagnostics = self.diagnostics,
.token = token,
.code_page = self.input_code_pages.getForToken(token),
};
}
}; };
pub const OpenSearchPathError = std.fs.Dir.OpenError; pub const OpenSearchPathError = std.fs.Dir.OpenError;
@ -3247,7 +3264,8 @@ pub const StringTable = struct {
const bytes = SourceBytes{ .slice = slice, .code_page = code_page }; const bytes = SourceBytes{ .slice = slice, .code_page = code_page };
const utf16_string = try literals.parseQuotedStringAsWideString(compiler.allocator, bytes, .{ const utf16_string = try literals.parseQuotedStringAsWideString(compiler.allocator, bytes, .{
.start_column = column, .start_column = column,
.diagnostics = .{ .diagnostics = compiler.diagnostics, .token = string_token }, .diagnostics = compiler.errContext(string_token),
.output_code_page = compiler.output_code_pages.getForToken(string_token),
}); });
defer compiler.allocator.free(utf16_string); defer compiler.allocator.free(utf16_string);

99
lib/compiler/resinator/disjoint_code_page.zig Normal file
View File

@ -0,0 +1,99 @@
const std = @import("std");
const lex = @import("lex.zig");
const SourceMappings = @import("source_mapping.zig").SourceMappings;
const SupportedCodePage = @import("code_pages.zig").SupportedCodePage;
pub fn hasDisjointCodePage(source: []const u8, source_mappings: ?*const SourceMappings, default_code_page: SupportedCodePage) bool {
var line_handler = lex.LineHandler{ .buffer = source };
var i: usize = 0;
while (i < source.len) {
const codepoint = default_code_page.codepointAt(i, source) orelse break;
const c = codepoint.value;
switch (c) {
'\r', '\n' => {
_ = line_handler.incrementLineNumber(i);
// Any lines that are not from the root file interrupt the disjoint code page
if (source_mappings != null and !source_mappings.?.isRootFile(line_handler.line_number)) return false;
},
// whitespace is ignored
' ',
'\t',
// NBSP, this should technically be in the TODO below, but it is treated as whitespace
// due to a (misguided) special casing in the lexer, see the TODO in lex.zig
'\u{A0}',
=> {},
// TODO: All of the below are treated as whitespace by the Win32 RC preprocessor, which also
// means they are trimmed from the file during preprocessing. This means that these characters
// should be treated like ' ', '\t' above, but since the resinator preprocessor does not treat
// them as whitespace *or* trim whitespace, files with these characters are likely going to
// error. So, in the future some sort of emulation of/rejection of the Win32 behavior might
// make handling these codepoints specially make sense, but for now it doesn't really matter
// so they are not handled specially for simplicity's sake.
//'\u{1680}',
//'\u{180E}',
//'\u{2001}',
//'\u{2002}',
//'\u{2003}',
//'\u{2004}',
//'\u{2005}',
//'\u{2006}',
//'\u{2007}',
//'\u{2008}',
//'\u{2009}',
//'\u{200A}',
//'\u{2028}',
//'\u{2029}',
//'\u{202F}',
//'\u{205F}',
//'\u{3000}',
'#' => {
if (source_mappings != null and !source_mappings.?.isRootFile(line_handler.line_number)) {
return false;
}
const start_i = i;
while (i < source.len and source[i] != '\r' and source[i] != '\n') : (i += 1) {}
const line = source[start_i..i];
_ = (lex.parsePragmaCodePage(line) catch |err| switch (err) {
error.NotPragma => return false,
error.NotCodePagePragma => continue,
error.CodePagePragmaUnsupportedCodePage => continue,
else => continue,
}) orelse return false; // DEFAULT interrupts disjoint code page
// If we got a code page, then it is a disjoint code page pragma
return true;
},
else => {
// Any other character interrupts the disjoint code page
return false;
},
}
i += codepoint.byte_len;
}
return false;
}
test hasDisjointCodePage {
try std.testing.expect(hasDisjointCodePage("#pragma code_page(65001)\n", null, .windows1252));
// NBSP is a special case
try std.testing.expect(hasDisjointCodePage("\xA0\n#pragma code_page(65001)\n", null, .windows1252));
try std.testing.expect(hasDisjointCodePage("\u{A0}\n#pragma code_page(1252)\n", null, .utf8));
// other preprocessor commands don't interrupt
try std.testing.expect(hasDisjointCodePage("#pragma foo\n#pragma code_page(65001)\n", null, .windows1252));
// invalid code page doesn't interrupt
try std.testing.expect(hasDisjointCodePage("#pragma code_page(1234567)\n#pragma code_page(65001)\n", null, .windows1252));
try std.testing.expect(!hasDisjointCodePage("#if 1\n#endif\n#pragma code_page(65001)", null, .windows1252));
try std.testing.expect(!hasDisjointCodePage("// comment\n#pragma code_page(65001)", null, .windows1252));
try std.testing.expect(!hasDisjointCodePage("/* comment */\n#pragma code_page(65001)", null, .windows1252));
}
test "multiline comment edge case" {
// TODO
if (true) return error.SkipZigTest;
try std.testing.expect(hasDisjointCodePage("/* comment */#pragma code_page(65001)", null, .windows1252));
}

583
lib/compiler/resinator/errors.zig
View File

@ -8,7 +8,8 @@ const ico = @import("ico.zig");
const bmp = @import("bmp.zig"); const bmp = @import("bmp.zig");
const parse = @import("parse.zig"); const parse = @import("parse.zig");
const lang = @import("lang.zig"); const lang = @import("lang.zig");
const CodePage = @import("code_pages.zig").CodePage; const code_pages = @import("code_pages.zig");
const SupportedCodePage = code_pages.SupportedCodePage;
const builtin = @import("builtin"); const builtin = @import("builtin");
const native_endian = builtin.cpu.arch.endian(); const native_endian = builtin.cpu.arch.endian();
@ -64,7 +65,7 @@ pub const Diagnostics = struct {
defer std.debug.unlockStdErr(); defer std.debug.unlockStdErr();
const stderr = std.io.getStdErr().writer(); const stderr = std.io.getStdErr().writer();
for (self.errors.items) |err_details| { for (self.errors.items) |err_details| {
renderErrorMessage(self.allocator, stderr, tty_config, cwd, err_details, source, self.strings.items, source_mappings) catch return; renderErrorMessage(stderr, tty_config, cwd, err_details, source, self.strings.items, source_mappings) catch return;
} }
} }
@ -94,32 +95,22 @@ pub const Diagnostics = struct {
pub const DiagnosticsContext = struct { pub const DiagnosticsContext = struct {
diagnostics: *Diagnostics, diagnostics: *Diagnostics,
token: Token, token: Token,
/// Code page of the source file at the token location
code_page: SupportedCodePage,
}; };
pub const ErrorDetails = struct { pub const ErrorDetails = struct {
err: Error, err: Error,
token: Token, token: Token,
/// Code page of the source file at the token location
code_page: SupportedCodePage,
/// If non-null, should be before `token`. If null, `token` is assumed to be the start. /// If non-null, should be before `token`. If null, `token` is assumed to be the start.
token_span_start: ?Token = null, token_span_start: ?Token = null,
/// If non-null, should be after `token`. If null, `token` is assumed to be the end. /// If non-null, should be after `token`. If null, `token` is assumed to be the end.
token_span_end: ?Token = null, token_span_end: ?Token = null,
type: Type = .err, type: Type = .err,
print_source_line: bool = true, print_source_line: bool = true,
extra: union { extra: Extra = .{ .none = {} },
none: void,
expected: Token.Id,
number: u32,
expected_types: ExpectedTypes,
resource: rc.Resource,
string_and_language: StringAndLanguage,
file_open_error: FileOpenError,
icon_read_error: IconReadError,
icon_dir: IconDirContext,
bmp_read_error: BitmapReadError,
accelerator_error: AcceleratorError,
statement_with_u16_param: StatementWithU16Param,
menu_or_class: enum { class, menu },
} = .{ .none = {} },
pub const Type = enum { pub const Type = enum {
/// Fatal error, stops compilation /// Fatal error, stops compilation
@ -137,9 +128,25 @@ pub const ErrorDetails = struct {
hint, hint,
}; };
pub const Extra = union {
none: void,
expected: Token.Id,
number: u32,
expected_types: ExpectedTypes,
resource: rc.ResourceType,
string_and_language: StringAndLanguage,
file_open_error: FileOpenError,
icon_read_error: IconReadError,
icon_dir: IconDirContext,
bmp_read_error: BitmapReadError,
accelerator_error: AcceleratorError,
statement_with_u16_param: StatementWithU16Param,
menu_or_class: enum { class, menu },
};
comptime { comptime {
// all fields in the extra union should be 32 bits or less // all fields in the extra union should be 32 bits or less
for (std.meta.fields(std.meta.fieldInfo(ErrorDetails, .extra).type)) |field| { for (std.meta.fields(Extra)) |field| {
std.debug.assert(@bitSizeOf(field.type) <= 32); std.debug.assert(@bitSizeOf(field.type) <= 32);
} }
} }
@ -321,6 +328,8 @@ pub const ErrorDetails = struct {
close_paren_expression, close_paren_expression,
unary_plus_expression, unary_plus_expression,
rc_could_miscompile_control_params, rc_could_miscompile_control_params,
dangling_literal_at_eof,
disjoint_code_page,
// Compiler // Compiler
/// `string_and_language` is populated /// `string_and_language` is populated
@ -331,6 +340,7 @@ pub const ErrorDetails = struct {
/// `accelerator_error` is populated /// `accelerator_error` is populated
invalid_accelerator_key, invalid_accelerator_key,
accelerator_type_required, accelerator_type_required,
accelerator_shift_or_control_without_virtkey,
rc_would_miscompile_control_padding, rc_would_miscompile_control_padding,
rc_would_miscompile_control_class_ordinal, rc_would_miscompile_control_class_ordinal,
/// `icon_dir` is populated /// `icon_dir` is populated
@ -356,11 +366,6 @@ pub const ErrorDetails = struct {
/// `number` is populated and contains a string index for which the string contains /// `number` is populated and contains a string index for which the string contains
/// the bytes of a `u64` (native endian). The `u64` contains the number of miscompiled bytes. /// the bytes of a `u64` (native endian). The `u64` contains the number of miscompiled bytes.
rc_would_miscompile_bmp_palette_padding, rc_would_miscompile_bmp_palette_padding,
/// `number` is populated and contains a string index for which the string contains
/// the bytes of two `u64`s (native endian). The first contains the number of missing
/// palette bytes and the second contains the max number of missing palette bytes.
/// If type is `.note`, then `extra` is `none`.
bmp_too_many_missing_palette_bytes,
resource_header_size_exceeds_max, resource_header_size_exceeds_max,
resource_data_size_exceeds_max, resource_data_size_exceeds_max,
control_extra_data_size_exceeds_max, control_extra_data_size_exceeds_max,
@ -383,15 +388,16 @@ pub const ErrorDetails = struct {
rc_would_miscompile_dialog_menu_or_class_id_forced_ordinal, rc_would_miscompile_dialog_menu_or_class_id_forced_ordinal,
rc_would_miscompile_dialog_menu_id_starts_with_digit, rc_would_miscompile_dialog_menu_id_starts_with_digit,
dialog_menu_id_was_uppercased, dialog_menu_id_was_uppercased,
/// `menu_or_class` is populated and contains the type of the parameter statement duplicate_optional_statement_skipped,
duplicate_menu_or_class_skipped,
invalid_digit_character_in_ordinal, invalid_digit_character_in_ordinal,
// Literals // Literals
/// `number` is populated /// `number` is populated
rc_would_miscompile_codepoint_byte_swap, rc_would_miscompile_codepoint_whitespace,
/// `number` is populated /// `number` is populated
rc_would_miscompile_codepoint_skip, rc_would_miscompile_codepoint_skip,
/// `number` is populated
rc_would_miscompile_codepoint_bom,
tab_converted_to_spaces, tab_converted_to_spaces,
// General (used in various places) // General (used in various places)
@ -403,10 +409,50 @@ pub const ErrorDetails = struct {
failed_to_open_cwd, failed_to_open_cwd,
}; };
fn formatToken(
ctx: TokenFormatContext,
comptime fmt: []const u8,
options: std.fmt.FormatOptions,
writer: anytype,
) !void {
_ = fmt;
_ = options;
switch (ctx.token.id) {
.eof => return writer.writeAll(ctx.token.id.nameForErrorDisplay()),
else => {},
}
const slice = ctx.token.slice(ctx.source);
var src_i: usize = 0;
while (src_i < slice.len) {
const codepoint = ctx.code_page.codepointAt(src_i, slice) orelse break;
defer src_i += codepoint.byte_len;
const display_codepoint = codepointForDisplay(codepoint) orelse continue;
var buf: [4]u8 = undefined;
const utf8_len = std.unicode.utf8Encode(display_codepoint, &buf) catch unreachable;
try writer.writeAll(buf[0..utf8_len]);
}
}
const TokenFormatContext = struct {
token: Token,
source: []const u8,
code_page: SupportedCodePage,
};
fn fmtToken(self: ErrorDetails, source: []const u8) std.fmt.Formatter(formatToken) {
return .{ .data = .{
.token = self.token,
.code_page = self.code_page,
.source = source,
} };
}
pub fn render(self: ErrorDetails, writer: anytype, source: []const u8, strings: []const []const u8) !void { pub fn render(self: ErrorDetails, writer: anytype, source: []const u8, strings: []const []const u8) !void {
switch (self.err) { switch (self.err) {
.unfinished_string_literal => { .unfinished_string_literal => {
return writer.print("unfinished string literal at '{s}', expected closing '\"'", .{self.token.nameForErrorDisplay(source)}); return writer.print("unfinished string literal at '{s}', expected closing '\"'", .{self.fmtToken(source)});
}, },
.string_literal_too_long => { .string_literal_too_long => {
return writer.print("string literal too long (max is currently {} characters)", .{self.extra.number}); return writer.print("string literal too long (max is currently {} characters)", .{self.extra.number});
@ -474,33 +520,33 @@ pub const ErrorDetails = struct {
number_slice.len += 1; number_slice.len += 1;
} }
const number = std.fmt.parseUnsigned(u16, number_slice, 10) catch unreachable; const number = std.fmt.parseUnsigned(u16, number_slice, 10) catch unreachable;
const code_page = CodePage.getByIdentifier(number) catch unreachable; const code_page = code_pages.getByIdentifier(number) catch unreachable;
// TODO: Improve or maybe add a note making it more clear that the code page // TODO: Improve or maybe add a note making it more clear that the code page
// is valid and that the code page is unsupported purely due to a limitation // is valid and that the code page is unsupported purely due to a limitation
// in this compiler. // in this compiler.
return writer.print("unsupported code page '{s} (id={})' in #pragma code_page", .{ @tagName(code_page), number }); return writer.print("unsupported code page '{s} (id={})' in #pragma code_page", .{ @tagName(code_page), number });
}, },
.unfinished_raw_data_block => { .unfinished_raw_data_block => {
return writer.print("unfinished raw data block at '{s}', expected closing '}}' or 'END'", .{self.token.nameForErrorDisplay(source)}); return writer.print("unfinished raw data block at '{s}', expected closing '}}' or 'END'", .{self.fmtToken(source)});
}, },
.unfinished_string_table_block => { .unfinished_string_table_block => {
return writer.print("unfinished STRINGTABLE block at '{s}', expected closing '}}' or 'END'", .{self.token.nameForErrorDisplay(source)}); return writer.print("unfinished STRINGTABLE block at '{s}', expected closing '}}' or 'END'", .{self.fmtToken(source)});
}, },
.expected_token => { .expected_token => {
return writer.print("expected '{s}', got '{s}'", .{ self.extra.expected.nameForErrorDisplay(), self.token.nameForErrorDisplay(source) }); return writer.print("expected '{s}', got '{s}'", .{ self.extra.expected.nameForErrorDisplay(), self.fmtToken(source) });
}, },
.expected_something_else => { .expected_something_else => {
try writer.writeAll("expected "); try writer.writeAll("expected ");
try self.extra.expected_types.writeCommaSeparated(writer); try self.extra.expected_types.writeCommaSeparated(writer);
return writer.print("; got '{s}'", .{self.token.nameForErrorDisplay(source)}); return writer.print("; got '{s}'", .{self.fmtToken(source)});
}, },
.resource_type_cant_use_raw_data => switch (self.type) { .resource_type_cant_use_raw_data => switch (self.type) {
.err, .warning => try writer.print("expected '<filename>', found '{s}' (resource type '{s}' can't use raw data)", .{ self.token.nameForErrorDisplay(source), self.extra.resource.nameForErrorDisplay() }), .err, .warning => try writer.print("expected '<filename>', found '{s}' (resource type '{s}' can't use raw data)", .{ self.fmtToken(source), self.extra.resource.nameForErrorDisplay() }),
.note => try writer.print("if '{s}' is intended to be a filename, it must be specified as a quoted string literal", .{self.token.nameForErrorDisplay(source)}), .note => try writer.print("if '{s}' is intended to be a filename, it must be specified as a quoted string literal", .{self.fmtToken(source)}),
.hint => return, .hint => return,
}, },
.id_must_be_ordinal => { .id_must_be_ordinal => {
try writer.print("id of resource type '{s}' must be an ordinal (u16), got '{s}'", .{ self.extra.resource.nameForErrorDisplay(), self.token.nameForErrorDisplay(source) }); try writer.print("id of resource type '{s}' must be an ordinal (u16), got '{s}'", .{ self.extra.resource.nameForErrorDisplay(), self.fmtToken(source) });
}, },
.name_or_id_not_allowed => { .name_or_id_not_allowed => {
try writer.print("name or id is not allowed for resource type '{s}'", .{self.extra.resource.nameForErrorDisplay()}); try writer.print("name or id is not allowed for resource type '{s}'", .{self.extra.resource.nameForErrorDisplay()});
@ -516,7 +562,7 @@ pub const ErrorDetails = struct {
try writer.writeAll("ASCII character not equivalent to virtual key code"); try writer.writeAll("ASCII character not equivalent to virtual key code");
}, },
.empty_menu_not_allowed => { .empty_menu_not_allowed => {
try writer.print("empty menu of type '{s}' not allowed", .{self.token.nameForErrorDisplay(source)}); try writer.print("empty menu of type '{s}' not allowed", .{self.fmtToken(source)});
}, },
.rc_would_miscompile_version_value_padding => switch (self.type) { .rc_would_miscompile_version_value_padding => switch (self.type) {
.err, .warning => return writer.print("the padding before this quoted string value would be miscompiled by the Win32 RC compiler", .{}), .err, .warning => return writer.print("the padding before this quoted string value would be miscompiled by the Win32 RC compiler", .{}),
@ -570,19 +616,18 @@ pub const ErrorDetails = struct {
.note => return writer.print("to avoid the potential miscompilation, consider adding a comma after the style parameter", .{}), .note => return writer.print("to avoid the potential miscompilation, consider adding a comma after the style parameter", .{}),
.hint => return, .hint => return,
}, },
.dangling_literal_at_eof => {
try writer.writeAll("dangling literal at end-of-file; this is not a problem, but it is likely a mistake");
},
.disjoint_code_page => switch (self.type) {
.err, .warning => return writer.print("#pragma code_page as the first thing in the .rc script can cause the input and output code pages to become out-of-sync", .{}),
.note => return writer.print("to avoid unexpected behavior, add a comment (or anything else) above the #pragma code_page line", .{}),
.hint => return,
},
.string_already_defined => switch (self.type) { .string_already_defined => switch (self.type) {
.err, .warning => { .err, .warning => {
const language_id = self.extra.string_and_language.language.asInt(); const language = self.extra.string_and_language.language;
const language_name = language_name: { return writer.print("string with id {d} (0x{X}) already defined for language {}", .{ self.extra.string_and_language.id, self.extra.string_and_language.id, language });
if (std.meta.intToEnum(lang.LanguageId, language_id)) |lang_enum_val| {
break :language_name @tagName(lang_enum_val);
} else |_| {}
if (language_id == lang.LOCALE_CUSTOM_UNSPECIFIED) {
break :language_name "LOCALE_CUSTOM_UNSPECIFIED";
}
break :language_name "<UNKNOWN>";
};
return writer.print("string with id {d} (0x{X}) already defined for language {s} (0x{X})", .{ self.extra.string_and_language.id, self.extra.string_and_language.id, language_name, language_id });
}, },
.note => return writer.print("previous definition of string with id {d} (0x{X}) here", .{ self.extra.string_and_language.id, self.extra.string_and_language.id }), .note => return writer.print("previous definition of string with id {d} (0x{X}) here", .{ self.extra.string_and_language.id, self.extra.string_and_language.id }),
.hint => return, .hint => return,
@ -597,14 +642,17 @@ pub const ErrorDetails = struct {
try writer.print("unable to open file '{s}': {s}", .{ strings[self.extra.file_open_error.filename_string_index], @tagName(self.extra.file_open_error.err) }); try writer.print("unable to open file '{s}': {s}", .{ strings[self.extra.file_open_error.filename_string_index], @tagName(self.extra.file_open_error.err) });
}, },
.invalid_accelerator_key => { .invalid_accelerator_key => {
try writer.print("invalid accelerator key '{s}': {s}", .{ self.token.nameForErrorDisplay(source), @tagName(self.extra.accelerator_error.err) }); try writer.print("invalid accelerator key '{s}': {s}", .{ self.fmtToken(source), @tagName(self.extra.accelerator_error.err) });
}, },
.accelerator_type_required => { .accelerator_type_required => {
try writer.print("accelerator type [ASCII or VIRTKEY] required when key is an integer", .{}); try writer.writeAll("accelerator type [ASCII or VIRTKEY] required when key is an integer");
},
.accelerator_shift_or_control_without_virtkey => {
try writer.writeAll("SHIFT or CONTROL used without VIRTKEY");
}, },
.rc_would_miscompile_control_padding => switch (self.type) { .rc_would_miscompile_control_padding => switch (self.type) {
.err, .warning => return writer.print("the padding before this control would be miscompiled by the Win32 RC compiler (it would insert 2 extra bytes of padding)", .{}), .err, .warning => return writer.print("the padding before this control would be miscompiled by the Win32 RC compiler (it would insert 2 extra bytes of padding)", .{}),
.note => return writer.print("to avoid the potential miscompilation, consider removing any 'control data' blocks from the controls in this dialog", .{}), .note => return writer.print("to avoid the potential miscompilation, consider adding one more byte to the control data of the control preceding this one", .{}),
.hint => return, .hint => return,
}, },
.rc_would_miscompile_control_class_ordinal => switch (self.type) { .rc_would_miscompile_control_class_ordinal => switch (self.type) {
@ -625,7 +673,7 @@ pub const ErrorDetails = struct {
try writer.print("resource with format '{s}' (at index {}) is not allowed in {s} resource groups", .{ @tagName(self.extra.icon_dir.icon_format), self.extra.icon_dir.index, @tagName(self.extra.icon_dir.icon_type) }); try writer.print("resource with format '{s}' (at index {}) is not allowed in {s} resource groups", .{ @tagName(self.extra.icon_dir.icon_format), self.extra.icon_dir.index, @tagName(self.extra.icon_dir.icon_type) });
}, },
.icon_dir_and_resource_type_mismatch => { .icon_dir_and_resource_type_mismatch => {
const unexpected_type: rc.Resource = if (self.extra.resource == .icon) .cursor else .icon; const unexpected_type: rc.ResourceType = if (self.extra.resource == .icon) .cursor else .icon;
// TODO: Better wording // TODO: Better wording
try writer.print("resource type '{s}' does not match type '{s}' specified in the file", .{ self.extra.resource.nameForErrorDisplay(), unexpected_type.nameForErrorDisplay() }); try writer.print("resource type '{s}' does not match type '{s}' specified in the file", .{ self.extra.resource.nameForErrorDisplay(), unexpected_type.nameForErrorDisplay() });
}, },
@ -663,23 +711,15 @@ pub const ErrorDetails = struct {
.bmp_missing_palette_bytes => { .bmp_missing_palette_bytes => {
const bytes = strings[self.extra.number]; const bytes = strings[self.extra.number];
const missing_bytes = std.mem.readInt(u64, bytes[0..8], native_endian); const missing_bytes = std.mem.readInt(u64, bytes[0..8], native_endian);
try writer.print("bitmap has {d} missing color palette bytes which will be padded with zeroes", .{missing_bytes}); try writer.print("bitmap has {d} missing color palette bytes", .{missing_bytes});
}, },
.rc_would_miscompile_bmp_palette_padding => { .rc_would_miscompile_bmp_palette_padding => {
try writer.writeAll("the Win32 RC compiler would erroneously pad out the missing bytes");
if (self.extra.number != 0) {
const bytes = strings[self.extra.number]; const bytes = strings[self.extra.number];
const miscompiled_bytes = std.mem.readInt(u64, bytes[0..8], native_endian); const miscompiled_bytes = std.mem.readInt(u64, bytes[0..8], native_endian);
try writer.print("the missing color palette bytes would be miscompiled by the Win32 RC compiler (the added padding bytes would include {d} bytes of the pixel data)", .{miscompiled_bytes}); try writer.print(" (and the added padding bytes would include {d} bytes of the pixel data)", .{miscompiled_bytes});
}, }
.bmp_too_many_missing_palette_bytes => switch (self.type) {
.err, .warning => {
const bytes = strings[self.extra.number];
const missing_bytes = std.mem.readInt(u64, bytes[0..8], native_endian);
const max_missing_bytes = std.mem.readInt(u64, bytes[8..16], native_endian);
try writer.print("bitmap has {} missing color palette bytes which exceeds the maximum of {}", .{ missing_bytes, max_missing_bytes });
},
// TODO: command line option
.note => try writer.writeAll("the maximum number of missing color palette bytes is configurable via <<TODO command line option>>"),
.hint => return,
}, },
.resource_header_size_exceeds_max => { .resource_header_size_exceeds_max => {
try writer.print("resource's header length exceeds maximum of {} bytes", .{std.math.maxInt(u32)}); try writer.print("resource's header length exceeds maximum of {} bytes", .{std.math.maxInt(u32)});
@ -749,23 +789,22 @@ pub const ErrorDetails = struct {
.hint => return, .hint => return,
}, },
.dialog_menu_id_was_uppercased => return, .dialog_menu_id_was_uppercased => return,
.duplicate_menu_or_class_skipped => { .duplicate_optional_statement_skipped => {
return writer.print("this {s} was ignored; when multiple {s} statements are specified, only the last takes precedence", .{ return writer.writeAll("this statement was ignored; when multiple statements of the same type are specified, only the last takes precedence");
@tagName(self.extra.menu_or_class),
@tagName(self.extra.menu_or_class),
});
}, },
.invalid_digit_character_in_ordinal => { .invalid_digit_character_in_ordinal => {
return writer.writeAll("non-ASCII digit characters are not allowed in ordinal (number) values"); return writer.writeAll("non-ASCII digit characters are not allowed in ordinal (number) values");
}, },
.rc_would_miscompile_codepoint_byte_swap => switch (self.type) { .rc_would_miscompile_codepoint_whitespace => {
.err, .warning => return writer.print("codepoint U+{X} within a string literal would be miscompiled by the Win32 RC compiler (the bytes of the UTF-16 code unit would be swapped)", .{self.extra.number}), const treated_as = self.extra.number >> 8;
.note => return writer.print("to avoid the potential miscompilation, an integer escape sequence in a wide string literal could be used instead: L\"\\x{X}\"", .{self.extra.number}), return writer.print("codepoint U+{X:0>4} within a string literal would be miscompiled by the Win32 RC compiler (it would get treated as U+{X:0>4})", .{ self.extra.number, treated_as });
.hint => return,
}, },
.rc_would_miscompile_codepoint_skip => switch (self.type) { .rc_would_miscompile_codepoint_skip => {
.err, .warning => return writer.print("codepoint U+{X} within a string literal would be miscompiled by the Win32 RC compiler (the codepoint would be missing from the compiled resource)", .{self.extra.number}), return writer.print("codepoint U+{X:0>4} within a string literal would be miscompiled by the Win32 RC compiler (the codepoint would be missing from the compiled resource)", .{self.extra.number});
.note => return writer.print("to avoid the potential miscompilation, an integer escape sequence in a wide string literal could be used instead: L\"\\x{X}\"", .{self.extra.number}), },
.rc_would_miscompile_codepoint_bom => switch (self.type) {
.err, .warning => return writer.print("codepoint U+{X:0>4} within a string literal would cause the entire file to be miscompiled by the Win32 RC compiler", .{self.extra.number}),
.note => return writer.writeAll("the presence of this codepoint causes all non-ASCII codepoints to be byteswapped by the Win32 RC preprocessor"),
.hint => return, .hint => return,
}, },
.tab_converted_to_spaces => switch (self.type) { .tab_converted_to_spaces => switch (self.type) {
@ -790,14 +829,7 @@ pub const ErrorDetails = struct {
after_len: usize, after_len: usize,
}; };
pub fn visualTokenInfo(self: ErrorDetails, source_line_start: usize, source_line_end: usize) VisualTokenInfo { pub fn visualTokenInfo(self: ErrorDetails, source_line_start: usize, source_line_end: usize, source: []const u8) VisualTokenInfo {
// Note: A perfect solution here would involve full grapheme cluster
// awareness, but oh well. This will give incorrect offsets
// if there are any multibyte codepoints within the relevant span,
// and even more inflated for grapheme clusters.
//
// We mitigate this slightly when we know we'll be pointing at
// something that displays as 1 character.
return switch (self.err) { return switch (self.err) {
// These can technically be more than 1 byte depending on encoding, // These can technically be more than 1 byte depending on encoding,
// but they always refer to one visual character/grapheme. // but they always refer to one visual character/grapheme.
@ -808,27 +840,65 @@ pub const ErrorDetails = struct {
.illegal_private_use_character, .illegal_private_use_character,
=> .{ => .{
.before_len = 0, .before_len = 0,
.point_offset = self.token.start - source_line_start, .point_offset = cellCount(self.code_page, source, source_line_start, self.token.start),
.after_len = 0, .after_len = 0,
}, },
else => .{ else => .{
.before_len = before: { .before_len = before: {
const start = @max(source_line_start, if (self.token_span_start) |span_start| span_start.start else self.token.start); const start = @max(source_line_start, if (self.token_span_start) |span_start| span_start.start else self.token.start);
break :before self.token.start - start; break :before cellCount(self.code_page, source, start, self.token.start);
}, },
.point_offset = self.token.start - source_line_start, .point_offset = cellCount(self.code_page, source, source_line_start, self.token.start),
.after_len = after: { .after_len = after: {
const end = @min(source_line_end, if (self.token_span_end) |span_end| span_end.end else self.token.end); const end = @min(source_line_end, if (self.token_span_end) |span_end| span_end.end else self.token.end);
// end may be less than start when pointing to EOF // end may be less than start when pointing to EOF
if (end <= self.token.start) break :after 0; if (end <= self.token.start) break :after 0;
break :after end - self.token.start - 1; break :after cellCount(self.code_page, source, self.token.start, end) - 1;
}, },
}, },
}; };
} }
}; };
pub fn renderErrorMessage(allocator: std.mem.Allocator, writer: anytype, tty_config: std.io.tty.Config, cwd: std.fs.Dir, err_details: ErrorDetails, source: []const u8, strings: []const []const u8, source_mappings: ?SourceMappings) !void { /// Convenience struct only useful when the code page can be inferred from the token
pub const ErrorDetailsWithoutCodePage = blk: {
const details_info = @typeInfo(ErrorDetails);
const fields = details_info.@"struct".fields;
var fields_without_codepage: [fields.len - 1]std.builtin.Type.StructField = undefined;
var i: usize = 0;
for (fields) |field| {
if (std.mem.eql(u8, field.name, "code_page")) continue;
fields_without_codepage[i] = field;
i += 1;
}
std.debug.assert(i == fields_without_codepage.len);
break :blk @Type(.{ .@"struct" = .{
.layout = .auto,
.fields = &fields_without_codepage,
.decls = &.{},
.is_tuple = false,
} });
};
fn cellCount(code_page: SupportedCodePage, source: []const u8, start_index: usize, end_index: usize) usize {
// Note: This is an imperfect solution. A proper implementation here would
// involve full grapheme cluster awareness + grapheme width data, but oh well.
var codepoint_count: usize = 0;
var index: usize = start_index;
while (index < end_index) {
const codepoint = code_page.codepointAt(index, source) orelse break;
defer index += codepoint.byte_len;
_ = codepointForDisplay(codepoint) orelse continue;
codepoint_count += 1;
// no need to count more than we will display
if (codepoint_count >= max_source_line_codepoints + truncated_str.len) break;
}
return codepoint_count;
}
const truncated_str = "<...truncated...>";
pub fn renderErrorMessage(writer: anytype, tty_config: std.io.tty.Config, cwd: std.fs.Dir, err_details: ErrorDetails, source: []const u8, strings: []const []const u8, source_mappings: ?SourceMappings) !void {
if (err_details.type == .hint) return; if (err_details.type == .hint) return;
const source_line_start = err_details.token.getLineStartForErrorDisplay(source); const source_line_start = err_details.token.getLineStartForErrorDisplay(source);
@ -884,45 +954,61 @@ pub fn renderErrorMessage(allocator: std.mem.Allocator, writer: anytype, tty_con
} }
const source_line = err_details.token.getLineForErrorDisplay(source, source_line_start); const source_line = err_details.token.getLineForErrorDisplay(source, source_line_start);
const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len); const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len, source);
const truncated_visual_info = ErrorDetails.VisualTokenInfo{
.before_len = if (visual_info.point_offset > max_source_line_codepoints and visual_info.before_len > 0)
(visual_info.before_len + 1) -| (visual_info.point_offset - max_source_line_codepoints)
else
visual_info.before_len,
.point_offset = @min(max_source_line_codepoints + 1, visual_info.point_offset),
.after_len = if (visual_info.point_offset > max_source_line_codepoints)
@min(truncated_str.len - 3, visual_info.after_len)
else
@min(max_source_line_codepoints - visual_info.point_offset + (truncated_str.len - 2), visual_info.after_len),
};
// Need this to determine if the 'line originated from' note is worth printing // Need this to determine if the 'line originated from' note is worth printing
var source_line_for_display_buf = try std.ArrayList(u8).initCapacity(allocator, source_line.len); var source_line_for_display_buf: [max_source_line_bytes]u8 = undefined;
defer source_line_for_display_buf.deinit(); const source_line_for_display = writeSourceSlice(&source_line_for_display_buf, source_line, err_details.code_page);
try writeSourceSlice(source_line_for_display_buf.writer(), source_line);
// TODO: General handling of long lines, not tied to this specific error try writer.writeAll(source_line_for_display.line);
if (err_details.err == .string_literal_too_long) { if (source_line_for_display.truncated) {
const before_slice = source_line[0..@min(source_line.len, visual_info.point_offset + 16)];
try writeSourceSlice(writer, before_slice);
try tty_config.setColor(writer, .dim); try tty_config.setColor(writer, .dim);
try writer.writeAll("<...truncated...>"); try writer.writeAll(truncated_str);
try tty_config.setColor(writer, .reset); try tty_config.setColor(writer, .reset);
} else {
try writer.writeAll(source_line_for_display_buf.items);
} }
try writer.writeByte('\n'); try writer.writeByte('\n');
try tty_config.setColor(writer, .green); try tty_config.setColor(writer, .green);
const num_spaces = visual_info.point_offset - visual_info.before_len; const num_spaces = truncated_visual_info.point_offset - truncated_visual_info.before_len;
try writer.writeByteNTimes(' ', num_spaces); try writer.writeByteNTimes(' ', num_spaces);
try writer.writeByteNTimes('~', visual_info.before_len); try writer.writeByteNTimes('~', truncated_visual_info.before_len);
try writer.writeByte('^'); try writer.writeByte('^');
if (visual_info.after_len > 0) { try writer.writeByteNTimes('~', truncated_visual_info.after_len);
var num_squiggles = visual_info.after_len;
if (err_details.err == .string_literal_too_long) {
num_squiggles = @min(num_squiggles, 15);
}
try writer.writeByteNTimes('~', num_squiggles);
}
try writer.writeByte('\n'); try writer.writeByte('\n');
try tty_config.setColor(writer, .reset); try tty_config.setColor(writer, .reset);
if (corresponding_span != null and corresponding_file != null) { if (corresponding_span != null and corresponding_file != null) {
var corresponding_lines = try CorrespondingLines.init(allocator, cwd, err_details, source_line_for_display_buf.items, corresponding_span.?, corresponding_file.?); var worth_printing_lines: bool = true;
defer corresponding_lines.deinit(allocator); var initial_lines_err: ?anyerror = null;
var corresponding_lines: ?CorrespondingLines = CorrespondingLines.init(
if (!corresponding_lines.worth_printing_note) return; cwd,
err_details,
source_line_for_display.line,
corresponding_span.?,
corresponding_file.?,
) catch |err| switch (err) {
error.NotWorthPrintingLines => blk: {
worth_printing_lines = false;
break :blk null;
},
error.NotWorthPrintingNote => return,
else => |e| blk: {
initial_lines_err = e;
break :blk null;
},
};
defer if (corresponding_lines) |*cl| cl.deinit();
try tty_config.setColor(writer, .bold); try tty_config.setColor(writer, .bold);
if (corresponding_file) |file| { if (corresponding_file) |file| {
@ -947,85 +1033,222 @@ pub fn renderErrorMessage(allocator: std.mem.Allocator, writer: anytype, tty_con
try writer.print(" of file '{s}'\n", .{corresponding_file.?}); try writer.print(" of file '{s}'\n", .{corresponding_file.?});
try tty_config.setColor(writer, .reset); try tty_config.setColor(writer, .reset);
if (!corresponding_lines.worth_printing_lines) return; if (!worth_printing_lines) return;
if (corresponding_lines.lines_is_error_message) { const write_lines_err: ?anyerror = write_lines: {
if (initial_lines_err) |err| break :write_lines err;
while (corresponding_lines.?.next() catch |err| {
break :write_lines err;
}) |display_line| {
try writer.writeAll(display_line.line);
if (display_line.truncated) {
try tty_config.setColor(writer, .dim);
try writer.writeAll(truncated_str);
try tty_config.setColor(writer, .reset);
}
try writer.writeByte('\n');
}
break :write_lines null;
};
if (write_lines_err) |err| {
try tty_config.setColor(writer, .red); try tty_config.setColor(writer, .red);
try writer.writeAll(" | "); try writer.writeAll(" | ");
try tty_config.setColor(writer, .reset); try tty_config.setColor(writer, .reset);
try tty_config.setColor(writer, .dim); try tty_config.setColor(writer, .dim);
try writer.writeAll(corresponding_lines.lines.items); try writer.print("unable to print line(s) from file: {s}\n", .{@errorName(err)});
try tty_config.setColor(writer, .reset); try tty_config.setColor(writer, .reset);
try writer.writeAll("\n\n");
return;
} }
try writer.writeByte('\n');
try writer.writeAll(corresponding_lines.lines.items);
try writer.writeAll("\n\n");
} }
} }
const VisualLine = struct {
line: []u8,
truncated: bool,
};
const CorrespondingLines = struct { const CorrespondingLines = struct {
worth_printing_note: bool = true, // enough room for one more codepoint, just so that we don't have to keep
worth_printing_lines: bool = true, // track of this being truncated, since the extra codepoint will ensure
lines: std.ArrayListUnmanaged(u8) = .empty, // the visual line will need to truncate in that case.
lines_is_error_message: bool = false, line_buf: [max_source_line_bytes + 4]u8 = undefined,
line_len: usize = 0,
visual_line_buf: [max_source_line_bytes]u8 = undefined,
visual_line_len: usize = 0,
truncated: bool = false,
line_num: usize = 1,
initial_line: bool = true,
last_byte: u8 = 0,
at_eof: bool = false,
span: SourceMappings.CorrespondingSpan,
file: std.fs.File,
buffered_reader: BufferedReaderType,
code_page: SupportedCodePage,
pub fn init(allocator: std.mem.Allocator, cwd: std.fs.Dir, err_details: ErrorDetails, lines_for_comparison: []const u8, corresponding_span: SourceMappings.CorrespondingSpan, corresponding_file: []const u8) !CorrespondingLines { const BufferedReaderType = std.io.BufferedReader(512, std.fs.File.Reader);
var corresponding_lines = CorrespondingLines{};
pub fn init(cwd: std.fs.Dir, err_details: ErrorDetails, line_for_comparison: []const u8, corresponding_span: SourceMappings.CorrespondingSpan, corresponding_file: []const u8) !CorrespondingLines {
// We don't do line comparison for this error, so don't print the note if the line // We don't do line comparison for this error, so don't print the note if the line
// number is different // number is different
if (err_details.err == .string_literal_too_long and err_details.token.line_number == corresponding_span.start_line) { if (err_details.err == .string_literal_too_long and err_details.token.line_number != corresponding_span.start_line) {
corresponding_lines.worth_printing_note = false; return error.NotWorthPrintingNote;
return corresponding_lines;
} }
// Don't print the originating line for this error, we know it's really long // Don't print the originating line for this error, we know it's really long
if (err_details.err == .string_literal_too_long) { if (err_details.err == .string_literal_too_long) {
corresponding_lines.worth_printing_lines = false; return error.NotWorthPrintingLines;
return corresponding_lines;
} }
var writer = corresponding_lines.lines.writer(allocator); var corresponding_lines = CorrespondingLines{
if (utils.openFileNotDir(cwd, corresponding_file, .{})) |file| { .span = corresponding_span,
defer file.close(); .file = try utils.openFileNotDir(cwd, corresponding_file, .{}),
var buffered_reader = std.io.bufferedReader(file.reader()); .buffered_reader = undefined,
writeLinesFromStream(writer, buffered_reader.reader(), corresponding_span.start_line, corresponding_span.end_line) catch |err| switch (err) { .code_page = err_details.code_page,
error.LinesNotFound => {
corresponding_lines.lines.clearRetainingCapacity();
try writer.print("unable to print line(s) from file: {s}", .{@errorName(err)});
corresponding_lines.lines_is_error_message = true;
return corresponding_lines;
},
else => |e| return e,
}; };
} else |err| { corresponding_lines.buffered_reader = BufferedReaderType{
corresponding_lines.lines.clearRetainingCapacity(); .unbuffered_reader = corresponding_lines.file.reader(),
try writer.print("unable to print line(s) from file: {s}", .{@errorName(err)}); };
corresponding_lines.lines_is_error_message = true; errdefer corresponding_lines.deinit();
return corresponding_lines;
} var fbs = std.io.fixedBufferStream(&corresponding_lines.line_buf);
const writer = fbs.writer();
try corresponding_lines.writeLineFromStreamVerbatim(
writer,
corresponding_lines.buffered_reader.reader(),
corresponding_span.start_line,
);
const visual_line = writeSourceSlice(
&corresponding_lines.visual_line_buf,
corresponding_lines.line_buf[0..corresponding_lines.line_len],
err_details.code_page,
);
corresponding_lines.visual_line_len = visual_line.line.len;
corresponding_lines.truncated = visual_line.truncated;
// If the lines are the same as they were before preprocessing, skip printing the note entirely // If the lines are the same as they were before preprocessing, skip printing the note entirely
if (std.mem.eql(u8, lines_for_comparison, corresponding_lines.lines.items)) { if (corresponding_span.start_line == corresponding_span.end_line and std.mem.eql(
corresponding_lines.worth_printing_note = false; u8,
line_for_comparison,
corresponding_lines.visual_line_buf[0..corresponding_lines.visual_line_len],
)) {
return error.NotWorthPrintingNote;
} }
return corresponding_lines; return corresponding_lines;
} }
pub fn deinit(self: *CorrespondingLines, allocator: std.mem.Allocator) void { pub fn next(self: *CorrespondingLines) !?VisualLine {
self.lines.deinit(allocator); if (self.initial_line) {
self.initial_line = false;
return .{
.line = self.visual_line_buf[0..self.visual_line_len],
.truncated = self.truncated,
};
}
if (self.line_num > self.span.end_line) return null;
if (self.at_eof) return error.LinesNotFound;
self.line_len = 0;
self.visual_line_len = 0;
var fbs = std.io.fixedBufferStream(&self.line_buf);
const writer = fbs.writer();
try self.writeLineFromStreamVerbatim(
writer,
self.buffered_reader.reader(),
self.line_num,
);
const visual_line = writeSourceSlice(
&self.visual_line_buf,
self.line_buf[0..self.line_len],
self.code_page,
);
self.visual_line_len = visual_line.line.len;
return visual_line;
}
fn writeLineFromStreamVerbatim(self: *CorrespondingLines, writer: anytype, input: anytype, line_num: usize) !void {
while (try readByteOrEof(input)) |byte| {
switch (byte) {
'\n', '\r' => {
if (!utils.isLineEndingPair(self.last_byte, byte)) {
const line_complete = self.line_num == line_num;
self.line_num += 1;
if (line_complete) {
self.last_byte = byte;
return;
}
} else {
// reset last_byte to a non-line ending so that
// consecutive CRLF pairs don't get treated as one
// long line ending 'pair'
self.last_byte = 0;
continue;
}
},
else => {
if (self.line_num == line_num) {
if (writer.writeByte(byte)) {
self.line_len += 1;
} else |err| switch (err) {
error.NoSpaceLeft => {},
else => |e| return e,
}
}
},
}
self.last_byte = byte;
}
self.at_eof = true;
// hacky way to get next to return null
self.line_num += 1;
}
fn readByteOrEof(reader: anytype) !?u8 {
return reader.readByte() catch |err| switch (err) {
error.EndOfStream => return null,
else => |e| return e,
};
}
pub fn deinit(self: *CorrespondingLines) void {
self.file.close();
} }
}; };
fn writeSourceSlice(writer: anytype, slice: []const u8) !void { const max_source_line_codepoints = 120;
for (slice) |c| try writeSourceByte(writer, c); const max_source_line_bytes = max_source_line_codepoints * 4;
fn writeSourceSlice(buf: []u8, slice: []const u8, code_page: SupportedCodePage) VisualLine {
var src_i: usize = 0;
var dest_i: usize = 0;
var codepoint_count: usize = 0;
while (src_i < slice.len) {
const codepoint = code_page.codepointAt(src_i, slice) orelse break;
defer src_i += codepoint.byte_len;
const display_codepoint = codepointForDisplay(codepoint) orelse continue;
codepoint_count += 1;
if (codepoint_count > max_source_line_codepoints) {
return .{ .line = buf[0..dest_i], .truncated = true };
}
const utf8_len = std.unicode.utf8Encode(display_codepoint, buf[dest_i..]) catch unreachable;
dest_i += utf8_len;
}
return .{ .line = buf[0..dest_i], .truncated = false };
} }
inline fn writeSourceByte(writer: anytype, byte: u8) !void { fn codepointForDisplay(codepoint: code_pages.Codepoint) ?u21 {
switch (byte) { return switch (codepoint.value) {
'\x00'...'\x08', '\x0E'...'\x1F', '\x7F' => try writer.writeAll("<EFBFBD>"), '\x00'...'\x08',
'\x0E'...'\x1F',
'\x7F',
code_pages.Codepoint.invalid,
=> '<27>',
// \r is seemingly ignored by the RC compiler so skipping it when printing source lines // \r is seemingly ignored by the RC compiler so skipping it when printing source lines
// could help avoid confusing output (e.g. RC\rDATA if printed verbatim would show up // could help avoid confusing output (e.g. RC\rDATA if printed verbatim would show up
// in the console as DATA but the compiler reads it as RCDATA) // in the console as DATA but the compiler reads it as RCDATA)
@ -1033,44 +1256,8 @@ inline fn writeSourceByte(writer: anytype, byte: u8) !void {
// NOTE: This is irrelevant when using the clang preprocessor, because unpaired \r // NOTE: This is irrelevant when using the clang preprocessor, because unpaired \r
// characters get converted to \n, but may become relevant if another // characters get converted to \n, but may become relevant if another
// preprocessor is used instead. // preprocessor is used instead.
'\r' => {}, '\r' => null,
'\t', '\x0B', '\x0C' => try writer.writeByte(' '), '\t', '\x0B', '\x0C' => ' ',
else => try writer.writeByte(byte), else => |v| v,
}
}
pub fn writeLinesFromStream(writer: anytype, input: anytype, start_line: usize, end_line: usize) !void {
var line_num: usize = 1;
var last_byte: u8 = 0;
while (try readByteOrEof(input)) |byte| {
switch (byte) {
'\n', '\r' => {
if (!utils.isLineEndingPair(last_byte, byte)) {
if (line_num == end_line) return;
if (line_num >= start_line) try writeSourceByte(writer, byte);
line_num += 1;
} else {
// reset last_byte to a non-line ending so that
// consecutive CRLF pairs don't get treated as one
// long line ending 'pair'
last_byte = 0;
continue;
}
},
else => {
if (line_num >= start_line) try writeSourceByte(writer, byte);
},
}
last_byte = byte;
}
if (line_num != end_line) {
return error.LinesNotFound;
}
}
pub fn readByteOrEof(reader: anytype) !?u8 {
return reader.readByte() catch |err| switch (err) {
error.EndOfStream => return null,
else => |e| return e,
}; };
} }

2
lib/compiler/resinator/lang.zig
View File

@ -119,7 +119,7 @@ test tagToId {
} }
test "exhaustive tagToId" { test "exhaustive tagToId" {
inline for (@typeInfo(LanguageId).Enum.fields) |field| { inline for (@typeInfo(LanguageId).@"enum".fields) |field| {
const id = tagToId(field.name) catch |err| { const id = tagToId(field.name) catch |err| {
std.debug.print("tag: {s}\n", .{field.name}); std.debug.print("tag: {s}\n", .{field.name});
return err; return err;

185
lib/compiler/resinator/lex.zig
View File

@ -8,7 +8,7 @@ const std = @import("std");
const ErrorDetails = @import("errors.zig").ErrorDetails; const ErrorDetails = @import("errors.zig").ErrorDetails;
const columnWidth = @import("literals.zig").columnWidth; const columnWidth = @import("literals.zig").columnWidth;
const code_pages = @import("code_pages.zig"); const code_pages = @import("code_pages.zig");
const CodePage = code_pages.CodePage; const SupportedCodePage = code_pages.SupportedCodePage;
const SourceMappings = @import("source_mapping.zig").SourceMappings; const SourceMappings = @import("source_mapping.zig").SourceMappings;
const isNonAsciiDigit = @import("utils.zig").isNonAsciiDigit; const isNonAsciiDigit = @import("utils.zig").isNonAsciiDigit;
@ -62,13 +62,6 @@ pub const Token = struct {
return buffer[self.start..self.end]; return buffer[self.start..self.end];
} }
pub fn nameForErrorDisplay(self: Token, buffer: []const u8) []const u8 {
return switch (self.id) {
.eof => self.id.nameForErrorDisplay(),
else => self.slice(buffer),
};
}
/// Returns 0-based column /// Returns 0-based column
pub fn calculateColumn(token: Token, source: []const u8, tab_columns: usize, maybe_line_start: ?usize) usize { pub fn calculateColumn(token: Token, source: []const u8, tab_columns: usize, maybe_line_start: ?usize) usize {
const line_start = maybe_line_start orelse token.getLineStartForColumnCalc(source); const line_start = maybe_line_start orelse token.getLineStartForColumnCalc(source);
@ -214,18 +207,19 @@ pub const Lexer = struct {
line_handler: LineHandler, line_handler: LineHandler,
at_start_of_line: bool = true, at_start_of_line: bool = true,
error_context_token: ?Token = null, error_context_token: ?Token = null,
current_code_page: CodePage, current_code_page: SupportedCodePage,
default_code_page: CodePage, default_code_page: SupportedCodePage,
source_mappings: ?*SourceMappings, source_mappings: ?*SourceMappings,
max_string_literal_codepoints: u15, max_string_literal_codepoints: u15,
/// Needed to determine whether or not the output code page should /// Needed to determine whether or not the output code page should
/// be set in the parser. /// be set in the parser.
seen_pragma_code_pages: u2 = 0, seen_pragma_code_pages: u2 = 0,
last_pragma_code_page_token: ?Token = null,
pub const Error = LexError; pub const Error = LexError;
pub const LexerOptions = struct { pub const LexerOptions = struct {
default_code_page: CodePage = .windows1252, default_code_page: SupportedCodePage = .windows1252,
source_mappings: ?*SourceMappings = null, source_mappings: ?*SourceMappings = null,
max_string_literal_codepoints: u15 = default_max_string_literal_codepoints, max_string_literal_codepoints: u15 = default_max_string_literal_codepoints,
}; };
@ -291,6 +285,8 @@ pub const Lexer = struct {
}, },
// NBSP only counts as whitespace at the start of a line (but // NBSP only counts as whitespace at the start of a line (but
// can be intermixed with other whitespace). Who knows why. // can be intermixed with other whitespace). Who knows why.
// TODO: This should either be removed, or it should also include
// the codepoints listed in disjoint_code_page.zig
'\xA0' => if (self.at_start_of_line) { '\xA0' => if (self.at_start_of_line) {
result.start = self.index + codepoint.byte_len; result.start = self.index + codepoint.byte_len;
} else { } else {
@ -305,12 +301,8 @@ pub const Lexer = struct {
} }
self.at_start_of_line = false; self.at_start_of_line = false;
}, },
// Semi-colon acts as a line-terminator, but in this lexing mode
// that's only true if it's at the start of a line.
';' => { ';' => {
if (self.at_start_of_line) {
state = .semicolon; state = .semicolon;
}
self.at_start_of_line = false; self.at_start_of_line = false;
}, },
else => { else => {
@ -345,7 +337,11 @@ pub const Lexer = struct {
} }
} else { // got EOF } else { // got EOF
switch (state) { switch (state) {
.start, .semicolon => {}, .start => {},
.semicolon => {
// Skip past everything up to the EOF
result.start = self.index;
},
.literal => { .literal => {
result.id = .literal; result.id = .literal;
}, },
@ -357,6 +353,10 @@ pub const Lexer = struct {
} }
result.end = self.index; result.end = self.index;
// EOF tokens must have their start index match the end index
std.debug.assert(result.id != .eof or result.start == result.end);
return result; return result;
} }
@ -796,7 +796,11 @@ pub const Lexer = struct {
} }
} else { // got EOF } else { // got EOF
switch (state) { switch (state) {
.start, .semicolon => {}, .start => {},
.semicolon => {
// Skip past everything up to the EOF
result.start = self.index;
},
.literal_or_quoted_wide_string, .literal, .e, .en, .b, .be, .beg, .begi => { .literal_or_quoted_wide_string, .literal, .e, .en, .b, .be, .beg, .begi => {
result.id = .literal; result.id = .literal;
}, },
@ -835,6 +839,9 @@ pub const Lexer = struct {
} }
} }
// EOF tokens must have their start index match the end index
std.debug.assert(result.id != .eof or result.start == result.end);
return result; return result;
} }
@ -878,7 +885,7 @@ pub const Lexer = struct {
// and miscompilations when used within string literals. We avoid the miscompilation // and miscompilations when used within string literals. We avoid the miscompilation
// within string literals and emit a warning, but outside of string literals it makes // within string literals and emit a warning, but outside of string literals it makes
// more sense to just disallow these codepoints. // more sense to just disallow these codepoints.
0x900, 0xA00, 0xA0D, 0x2000, 0xFFFE, 0xD00 => if (!in_string_literal) error.IllegalCodepointOutsideStringLiterals else return, 0x900, 0xA00, 0xA0D, 0x2000, 0xD00, 0xFFFE, 0xFFFF => if (!in_string_literal) error.IllegalCodepointOutsideStringLiterals else return,
else => return, else => return,
}; };
self.error_context_token = .{ self.error_context_token = .{
@ -899,14 +906,83 @@ pub const Lexer = struct {
}; };
errdefer self.error_context_token = token; errdefer self.error_context_token = token;
const full_command = self.buffer[start..end]; const full_command = self.buffer[start..end];
const code_page = (parsePragmaCodePage(full_command) catch |err| switch (err) {
error.NotPragma, error.NotCodePagePragma => return,
else => |e| return e,
}) orelse self.default_code_page;
// https://learn.microsoft.com/en-us/windows/win32/menurc/pragma-directives
// > This pragma is not supported in an included resource file (.rc)
//
// Even though the Win32 behavior is to just ignore such directives silently,
// this is an error in the lexer to allow for emitting warnings/errors when
// such directives are found if that's wanted. The intention is for the lexer
// to still be able to work correctly after this error is returned.
if (self.source_mappings) |source_mappings| {
if (!source_mappings.isRootFile(token.line_number)) {
return error.CodePagePragmaInIncludedFile;
}
}
self.seen_pragma_code_pages +|= 1;
self.last_pragma_code_page_token = token;
self.current_code_page = code_page;
}
pub fn getErrorDetails(self: Self, lex_err: LexError) ErrorDetails {
const err = switch (lex_err) {
error.UnfinishedStringLiteral => ErrorDetails.Error.unfinished_string_literal,
error.StringLiteralTooLong => return .{
.err = .string_literal_too_long,
.code_page = self.current_code_page,
.token = self.error_context_token.?,
.extra = .{ .number = self.max_string_literal_codepoints },
},
error.InvalidNumberWithExponent => ErrorDetails.Error.invalid_number_with_exponent,
error.InvalidDigitCharacterInNumberLiteral => ErrorDetails.Error.invalid_digit_character_in_number_literal,
error.IllegalByte => ErrorDetails.Error.illegal_byte,
error.IllegalByteOutsideStringLiterals => ErrorDetails.Error.illegal_byte_outside_string_literals,
error.IllegalCodepointOutsideStringLiterals => ErrorDetails.Error.illegal_codepoint_outside_string_literals,
error.IllegalByteOrderMark => ErrorDetails.Error.illegal_byte_order_mark,
error.IllegalPrivateUseCharacter => ErrorDetails.Error.illegal_private_use_character,
error.FoundCStyleEscapedQuote => ErrorDetails.Error.found_c_style_escaped_quote,
error.CodePagePragmaMissingLeftParen => ErrorDetails.Error.code_page_pragma_missing_left_paren,
error.CodePagePragmaMissingRightParen => ErrorDetails.Error.code_page_pragma_missing_right_paren,
error.CodePagePragmaInvalidCodePage => ErrorDetails.Error.code_page_pragma_invalid_code_page,
error.CodePagePragmaNotInteger => ErrorDetails.Error.code_page_pragma_not_integer,
error.CodePagePragmaOverflow => ErrorDetails.Error.code_page_pragma_overflow,
error.CodePagePragmaUnsupportedCodePage => ErrorDetails.Error.code_page_pragma_unsupported_code_page,
error.CodePagePragmaInIncludedFile => ErrorDetails.Error.code_page_pragma_in_included_file,
};
return .{
.err = err,
.code_page = self.current_code_page,
.token = self.error_context_token.?,
};
}
};
fn parseCodePageNum(str: []const u8) !u32 {
var x: u32 = 0;
for (str) |c| {
const digit = try std.fmt.charToDigit(c, 10);
if (x != 0) x = try std.math.mul(u32, x, 10);
x = try std.math.add(u32, x, digit);
}
return x;
}
/// Returns `null` when the code_page is set to DEFAULT
pub fn parsePragmaCodePage(full_command: []const u8) !?SupportedCodePage {
var command = full_command; var command = full_command;
// Anything besides exactly this is ignored by the Windows RC implementation // Anything besides exactly this is ignored by the Windows RC implementation
const expected_directive = "#pragma"; const expected_directive = "#pragma";
if (!std.mem.startsWith(u8, command, expected_directive)) return; if (!std.mem.startsWith(u8, command, expected_directive)) return error.NotPragma;
command = command[expected_directive.len..]; command = command[expected_directive.len..];
if (command.len == 0 or !std.ascii.isWhitespace(command[0])) return; if (command.len == 0 or !std.ascii.isWhitespace(command[0])) return error.NotCodePagePragma;
while (command.len > 0 and std.ascii.isWhitespace(command[0])) { while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
command = command[1..]; command = command[1..];
} }
@ -914,7 +990,7 @@ pub const Lexer = struct {
// Note: CoDe_PaGeZ is also treated as "code_page" by the Windows RC implementation, // Note: CoDe_PaGeZ is also treated as "code_page" by the Windows RC implementation,
// and it will error with 'Missing left parenthesis in code_page #pragma' // and it will error with 'Missing left parenthesis in code_page #pragma'
const expected_extension = "code_page"; const expected_extension = "code_page";
if (!std.ascii.startsWithIgnoreCase(command, expected_extension)) return; if (!std.ascii.startsWithIgnoreCase(command, expected_extension)) return error.NotCodePagePragma;
command = command[expected_extension.len..]; command = command[expected_extension.len..];
while (command.len > 0 and std.ascii.isWhitespace(command[0])) { while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
@ -948,9 +1024,9 @@ pub const Lexer = struct {
return error.CodePagePragmaMissingRightParen; return error.CodePagePragmaMissingRightParen;
} }
const code_page = code_page: { const code_page: ?SupportedCodePage = code_page: {
if (std.ascii.eqlIgnoreCase("DEFAULT", num_str)) { if (std.ascii.eqlIgnoreCase("DEFAULT", num_str)) {
break :code_page self.default_code_page; break :code_page null;
} }
// The Win32 compiler behaves fairly strangely around maxInt(u32): // The Win32 compiler behaves fairly strangely around maxInt(u32):
@ -978,69 +1054,14 @@ pub const Lexer = struct {
return error.CodePagePragmaInvalidCodePage; return error.CodePagePragmaInvalidCodePage;
} }
break :code_page code_pages.CodePage.getByIdentifierEnsureSupported(@intCast(num)) catch |err| switch (err) { break :code_page code_pages.getByIdentifierEnsureSupported(@intCast(num)) catch |err| switch (err) {
error.InvalidCodePage => return error.CodePagePragmaInvalidCodePage, error.InvalidCodePage => return error.CodePagePragmaInvalidCodePage,
error.UnsupportedCodePage => return error.CodePagePragmaUnsupportedCodePage, error.UnsupportedCodePage => return error.CodePagePragmaUnsupportedCodePage,
}; };
}; };
// https://learn.microsoft.com/en-us/windows/win32/menurc/pragma-directives return code_page;
// > This pragma is not supported in an included resource file (.rc) }
//
// Even though the Win32 behavior is to just ignore such directives silently,
// this is an error in the lexer to allow for emitting warnings/errors when
// such directives are found if that's wanted. The intention is for the lexer
// to still be able to work correctly after this error is returned.
if (self.source_mappings) |source_mappings| {
if (!source_mappings.isRootFile(token.line_number)) {
return error.CodePagePragmaInIncludedFile;
}
}
self.seen_pragma_code_pages +|= 1;
self.current_code_page = code_page;
}
fn parseCodePageNum(str: []const u8) !u32 {
var x: u32 = 0;
for (str) |c| {
const digit = try std.fmt.charToDigit(c, 10);
if (x != 0) x = try std.math.mul(u32, x, 10);
x = try std.math.add(u32, x, digit);
}
return x;
}
pub fn getErrorDetails(self: Self, lex_err: LexError) ErrorDetails {
const err = switch (lex_err) {
error.UnfinishedStringLiteral => ErrorDetails.Error.unfinished_string_literal,
error.StringLiteralTooLong => return .{
.err = .string_literal_too_long,
.token = self.error_context_token.?,
.extra = .{ .number = self.max_string_literal_codepoints },
},
error.InvalidNumberWithExponent => ErrorDetails.Error.invalid_number_with_exponent,
error.InvalidDigitCharacterInNumberLiteral => ErrorDetails.Error.invalid_digit_character_in_number_literal,
error.IllegalByte => ErrorDetails.Error.illegal_byte,
error.IllegalByteOutsideStringLiterals => ErrorDetails.Error.illegal_byte_outside_string_literals,
error.IllegalCodepointOutsideStringLiterals => ErrorDetails.Error.illegal_codepoint_outside_string_literals,
error.IllegalByteOrderMark => ErrorDetails.Error.illegal_byte_order_mark,
error.IllegalPrivateUseCharacter => ErrorDetails.Error.illegal_private_use_character,
error.FoundCStyleEscapedQuote => ErrorDetails.Error.found_c_style_escaped_quote,
error.CodePagePragmaMissingLeftParen => ErrorDetails.Error.code_page_pragma_missing_left_paren,
error.CodePagePragmaMissingRightParen => ErrorDetails.Error.code_page_pragma_missing_right_paren,
error.CodePagePragmaInvalidCodePage => ErrorDetails.Error.code_page_pragma_invalid_code_page,
error.CodePagePragmaNotInteger => ErrorDetails.Error.code_page_pragma_not_integer,
error.CodePagePragmaOverflow => ErrorDetails.Error.code_page_pragma_overflow,
error.CodePagePragmaUnsupportedCodePage => ErrorDetails.Error.code_page_pragma_unsupported_code_page,
error.CodePagePragmaInIncludedFile => ErrorDetails.Error.code_page_pragma_in_included_file,
};
return .{
.err = err,
.token = self.error_context_token.?,
};
}
};
fn testLexNormal(source: []const u8, expected_tokens: []const Token.Id) !void { fn testLexNormal(source: []const u8, expected_tokens: []const Token.Id) !void {
var lexer = Lexer.init(source, .{}); var lexer = Lexer.init(source, .{});
@ -1074,7 +1095,7 @@ test "normal: string literals" {
test "superscript chars and code pages" { test "superscript chars and code pages" {
const firstToken = struct { const firstToken = struct {
pub fn firstToken(source: []const u8, default_code_page: CodePage, comptime lex_method: Lexer.LexMethod) LexError!Token { pub fn firstToken(source: []const u8, default_code_page: SupportedCodePage, comptime lex_method: Lexer.LexMethod) LexError!Token {
var lexer = Lexer.init(source, .{ .default_code_page = default_code_page }); var lexer = Lexer.init(source, .{ .default_code_page = default_code_page });
return lexer.next(lex_method); return lexer.next(lex_method);
} }

319
lib/compiler/resinator/literals.zig
View File

@ -1,6 +1,6 @@
const std = @import("std"); const std = @import("std");
const code_pages = @import("code_pages.zig"); const code_pages = @import("code_pages.zig");
const CodePage = code_pages.CodePage; const SupportedCodePage = code_pages.SupportedCodePage;
const windows1252 = @import("windows1252.zig"); const windows1252 = @import("windows1252.zig");
const ErrorDetails = @import("errors.zig").ErrorDetails; const ErrorDetails = @import("errors.zig").ErrorDetails;
const DiagnosticsContext = @import("errors.zig").DiagnosticsContext; const DiagnosticsContext = @import("errors.zig").DiagnosticsContext;
@ -18,7 +18,7 @@ pub fn isValidNumberDataLiteral(str: []const u8) bool {
pub const SourceBytes = struct { pub const SourceBytes = struct {
slice: []const u8, slice: []const u8,
code_page: CodePage, code_page: SupportedCodePage,
}; };
pub const StringType = enum { ascii, wide }; pub const StringType = enum { ascii, wide };
@ -53,7 +53,7 @@ pub const StringType = enum { ascii, wide };
/// branches should never actually be hit during this function. /// branches should never actually be hit during this function.
pub const IterativeStringParser = struct { pub const IterativeStringParser = struct {
source: []const u8, source: []const u8,
code_page: CodePage, code_page: SupportedCodePage,
/// The type of the string inferred by the prefix (L"" or "") /// The type of the string inferred by the prefix (L"" or "")
/// This is what matters for things like the maximum digits in an /// This is what matters for things like the maximum digits in an
/// escape sequence, whether or not invalid escape sequences are skipped, etc. /// escape sequence, whether or not invalid escape sequences are skipped, etc.
@ -98,32 +98,55 @@ pub const IterativeStringParser = struct {
pub const ParsedCodepoint = struct { pub const ParsedCodepoint = struct {
codepoint: u21, codepoint: u21,
/// Note: If this is true, `codepoint` will be a value with a max of maxInt(u16). /// Note: If this is true, `codepoint` will have an effective maximum value
/// This is enforced by using saturating arithmetic, so in e.g. a wide string literal the /// of 0xFFFF, as `codepoint` is calculated using wrapping arithmetic on a u16.
/// octal escape sequence \7777777 (2,097,151) will be parsed into the value 0xFFFF (65,535). /// If the value needs to be truncated to a smaller integer (e.g. for ASCII string
/// If the value needs to be truncated to a smaller integer (for ASCII string literals), then that /// literals), then that must be done by the caller.
/// must be done by the caller.
from_escaped_integer: bool = false, from_escaped_integer: bool = false,
/// Denotes that the codepoint is:
/// - Escaped (has a \ in front of it), and
/// - Has a value >= U+10000, meaning it would be encoded as a surrogate
/// pair in UTF-16, and
/// - Is part of a wide string literal
///
/// Normally in wide string literals, invalid escapes are omitted
/// during parsing (the codepoints are not returned at all during
/// the `next` call), but this is a special case in which the
/// escape only applies to the high surrogate pair of the codepoint.
///
/// TODO: Maybe just return the low surrogate codepoint by itself in this case.
escaped_surrogate_pair: bool = false,
}; };
pub fn next(self: *IterativeStringParser) std.mem.Allocator.Error!?ParsedCodepoint { pub fn next(self: *IterativeStringParser) std.mem.Allocator.Error!?ParsedCodepoint {
const result = try self.nextUnchecked(); const result = try self.nextUnchecked();
if (self.diagnostics != null and result != null and !result.?.from_escaped_integer) { if (self.diagnostics != null and result != null and !result.?.from_escaped_integer) {
switch (result.?.codepoint) { switch (result.?.codepoint) {
0x900, 0xA00, 0xA0D, 0x2000, 0xFFFE, 0xD00 => { 0x0900, 0x0A00, 0x0A0D, 0x2000, 0x0D00 => {
const err: ErrorDetails.Error = if (result.?.codepoint == 0xD00) const err: ErrorDetails.Error = if (result.?.codepoint == 0xD00)
.rc_would_miscompile_codepoint_skip .rc_would_miscompile_codepoint_skip
else else
.rc_would_miscompile_codepoint_byte_swap; .rc_would_miscompile_codepoint_whitespace;
try self.diagnostics.?.diagnostics.append(ErrorDetails{ try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = err, .err = err,
.type = .warning, .type = .warning,
.code_page = self.code_page,
.token = self.diagnostics.?.token,
.extra = .{ .number = result.?.codepoint },
});
},
0xFFFE, 0xFFFF => {
try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = .rc_would_miscompile_codepoint_bom,
.type = .warning,
.code_page = self.code_page,
.token = self.diagnostics.?.token, .token = self.diagnostics.?.token,
.extra = .{ .number = result.?.codepoint }, .extra = .{ .number = result.?.codepoint },
}); });
try self.diagnostics.?.diagnostics.append(ErrorDetails{ try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = err, .err = .rc_would_miscompile_codepoint_bom,
.type = .note, .type = .note,
.code_page = self.code_page,
.token = self.diagnostics.?.token, .token = self.diagnostics.?.token,
.print_source_line = false, .print_source_line = false,
.extra = .{ .number = result.?.codepoint }, .extra = .{ .number = result.?.codepoint },
@ -188,11 +211,13 @@ pub const IterativeStringParser = struct {
try self.diagnostics.?.diagnostics.append(ErrorDetails{ try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = .tab_converted_to_spaces, .err = .tab_converted_to_spaces,
.type = .warning, .type = .warning,
.code_page = self.code_page,
.token = self.diagnostics.?.token, .token = self.diagnostics.?.token,
}); });
try self.diagnostics.?.diagnostics.append(ErrorDetails{ try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = .tab_converted_to_spaces, .err = .tab_converted_to_spaces,
.type = .note, .type = .note,
.code_page = self.code_page,
.token = self.diagnostics.?.token, .token = self.diagnostics.?.token,
.print_source_line = false, .print_source_line = false,
}); });
@ -246,8 +271,9 @@ pub const IterativeStringParser = struct {
switch (c) { switch (c) {
'a', 'A' => { 'a', 'A' => {
self.index += codepoint.byte_len; self.index += codepoint.byte_len;
// might be a bug in RC, but matches its behavior
return .{ .codepoint = '\x08' }; return .{ .codepoint = '\x08' };
}, // might be a bug in RC, but matches its behavior },
'n' => { 'n' => {
self.index += codepoint.byte_len; self.index += codepoint.byte_len;
return .{ .codepoint = '\n' }; return .{ .codepoint = '\n' };
@ -269,7 +295,65 @@ pub const IterativeStringParser = struct {
backtrack = true; backtrack = true;
}, },
else => switch (self.declared_string_type) { else => switch (self.declared_string_type) {
.wide => {}, // invalid escape sequences are skipped in wide strings .wide => {
// All invalid escape sequences are skipped in wide strings,
// but there is a special case around \<tab> where the \
// is skipped but the tab character is processed.
// It's actually a bit weirder than that, though, since
// the preprocessor is the one that does the <tab> -> spaces
// conversion, so it goes something like this:
//
// Before preprocessing: L"\<tab>"
// After preprocessing: L"\ "
//
// So the parser only sees an escaped space character followed
// by some other number of spaces >= 0.
//
// However, our preprocessor keeps tab characters intact, so we emulate
// the above behavior by skipping the \ and then outputting one less
// space than normal for the <tab> character.
if (c == '\t') {
// Only warn about a tab getting converted to spaces once per string
if (self.diagnostics != null and !self.seen_tab) {
try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = .tab_converted_to_spaces,
.type = .warning,
.code_page = self.code_page,
.token = self.diagnostics.?.token,
});
try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = .tab_converted_to_spaces,
.type = .note,
.code_page = self.code_page,
.token = self.diagnostics.?.token,
.print_source_line = false,
});
self.seen_tab = true;
}
const cols = columnsUntilTabStop(self.column, 8);
// If the tab character would only be converted to a single space,
// then we can just skip both the \ and the <tab> and move on.
if (cols > 1) {
self.num_pending_spaces = @intCast(cols - 2);
self.index += codepoint.byte_len;
return .{ .codepoint = ' ' };
}
}
// There's a second special case when the codepoint would be encoded
// as a surrogate pair in UTF-16, as the escape 'applies' to the
// high surrogate pair only in this instance. This is a side-effect
// of the Win32 RC compiler preprocessor outputting UTF-16 and the
// compiler itself seemingly working on code units instead of code points
// in this particular instance.
//
// We emulate this behavior by emitting the codepoint, but with a marker
// that indicates that it needs to be handled specially.
if (c >= 0x10000 and c != code_pages.Codepoint.invalid) {
self.index += codepoint.byte_len;
return .{ .codepoint = c, .escaped_surrogate_pair = true };
}
},
.ascii => { .ascii => {
// we intentionally avoid incrementing self.index // we intentionally avoid incrementing self.index
// to handle the current char in the next call, // to handle the current char in the next call,
@ -303,6 +387,9 @@ pub const IterativeStringParser = struct {
}, },
.escaped_octal => switch (c) { .escaped_octal => switch (c) {
'0'...'7' => { '0'...'7' => {
// Note: We use wrapping arithmetic on a u16 here since there's been no observed
// string parsing scenario where an escaped integer with a value >= the u16
// max is interpreted as anything but the truncated u16 value.
string_escape_n *%= 8; string_escape_n *%= 8;
string_escape_n +%= std.fmt.charToDigit(@intCast(c), 8) catch unreachable; string_escape_n +%= std.fmt.charToDigit(@intCast(c), 8) catch unreachable;
string_escape_i += 1; string_escape_i += 1;
@ -367,7 +454,7 @@ pub const IterativeStringParser = struct {
pub const StringParseOptions = struct { pub const StringParseOptions = struct {
start_column: usize = 0, start_column: usize = 0,
diagnostics: ?DiagnosticsContext = null, diagnostics: ?DiagnosticsContext = null,
output_code_page: CodePage = .windows1252, output_code_page: SupportedCodePage,
}; };
pub fn parseQuotedString( pub fn parseQuotedString(
@ -389,14 +476,12 @@ pub fn parseQuotedString(
while (try iterative_parser.next()) |parsed| { while (try iterative_parser.next()) |parsed| {
const c = parsed.codepoint; const c = parsed.codepoint;
if (parsed.from_escaped_integer) {
// We truncate here to get the correct behavior for ascii strings
try buf.append(std.mem.nativeToLittle(T, @truncate(c)));
} else {
switch (literal_type) { switch (literal_type) {
.ascii => switch (options.output_code_page) { .ascii => switch (options.output_code_page) {
.windows1252 => { .windows1252 => {
if (windows1252.bestFitFromCodepoint(c)) |best_fit| { if (parsed.from_escaped_integer) {
try buf.append(@truncate(c));
} else if (windows1252.bestFitFromCodepoint(c)) |best_fit| {
try buf.append(best_fit); try buf.append(best_fit);
} else if (c < 0x10000 or c == code_pages.Codepoint.invalid) { } else if (c < 0x10000 or c == code_pages.Codepoint.invalid) {
try buf.append('?'); try buf.append('?');
@ -406,31 +491,39 @@ pub fn parseQuotedString(
}, },
.utf8 => { .utf8 => {
var codepoint_to_encode = c; var codepoint_to_encode = c;
if (c == code_pages.Codepoint.invalid) { if (parsed.from_escaped_integer) {
codepoint_to_encode = @as(T, @truncate(c));
}
const escaped_integer_outside_ascii_range = parsed.from_escaped_integer and codepoint_to_encode > 0x7F;
if (escaped_integer_outside_ascii_range or c == code_pages.Codepoint.invalid) {
codepoint_to_encode = '<27>'; codepoint_to_encode = '<27>';
} }
var utf8_buf: [4]u8 = undefined; var utf8_buf: [4]u8 = undefined;
const utf8_len = std.unicode.utf8Encode(codepoint_to_encode, &utf8_buf) catch unreachable; const utf8_len = std.unicode.utf8Encode(codepoint_to_encode, &utf8_buf) catch unreachable;
try buf.appendSlice(utf8_buf[0..utf8_len]); try buf.appendSlice(utf8_buf[0..utf8_len]);
}, },
else => unreachable, // Unsupported code page
}, },
.wide => { .wide => {
if (c == code_pages.Codepoint.invalid) { // Parsing any string type as a wide string is handled separately, see parseQuotedStringAsWideString
std.debug.assert(iterative_parser.declared_string_type == .wide);
if (parsed.from_escaped_integer) {
try buf.append(std.mem.nativeToLittle(u16, @truncate(c)));
} else if (c == code_pages.Codepoint.invalid) {
try buf.append(std.mem.nativeToLittle(u16, '<27>')); try buf.append(std.mem.nativeToLittle(u16, '<27>'));
} else if (c < 0x10000) { } else if (c < 0x10000) {
const short: u16 = @intCast(c); const short: u16 = @intCast(c);
try buf.append(std.mem.nativeToLittle(u16, short)); try buf.append(std.mem.nativeToLittle(u16, short));
} else { } else {
if (!parsed.escaped_surrogate_pair) {
const high = @as(u16, @intCast((c - 0x10000) >> 10)) + 0xD800; const high = @as(u16, @intCast((c - 0x10000) >> 10)) + 0xD800;
try buf.append(std.mem.nativeToLittle(u16, high)); try buf.append(std.mem.nativeToLittle(u16, high));
}
const low = @as(u16, @intCast(c & 0x3FF)) + 0xDC00; const low = @as(u16, @intCast(c & 0x3FF)) + 0xDC00;
try buf.append(std.mem.nativeToLittle(u16, low)); try buf.append(std.mem.nativeToLittle(u16, low));
} }
}, },
} }
} }
}
if (literal_type == .wide) { if (literal_type == .wide) {
return buf.toOwnedSliceSentinel(0); return buf.toOwnedSliceSentinel(0);
@ -449,9 +542,59 @@ pub fn parseQuotedWideString(allocator: std.mem.Allocator, bytes: SourceBytes, o
return parseQuotedString(.wide, allocator, bytes, options); return parseQuotedString(.wide, allocator, bytes, options);
} }
/// Parses any string type into a wide string.
/// If the string is declared as a wide string (L""), then it is handled normally.
/// Otherwise, things are fairly normal with the exception of escaped integers.
/// Escaped integers are handled by:
/// - Truncating the escape to a u8
/// - Reinterpeting the u8 as a byte from the *output* code page
/// - Outputting the codepoint that corresponds to the interpreted byte, or <EFBFBD> if no such
/// interpretation is possible
/// For example, if the code page is UTF-8, then while \x80 is a valid start byte, it's
/// interpreted as a single byte, so it ends up being seen as invalid and <EFBFBD> is outputted.
/// If the code page is Windows-1252, then \x80 is interpreted to be € which has the
/// codepoint U+20AC, so the UTF-16 encoding of U+20AC is outputted.
pub fn parseQuotedStringAsWideString(allocator: std.mem.Allocator, bytes: SourceBytes, options: StringParseOptions) ![:0]u16 { pub fn parseQuotedStringAsWideString(allocator: std.mem.Allocator, bytes: SourceBytes, options: StringParseOptions) ![:0]u16 {
std.debug.assert(bytes.slice.len >= 2); // "" std.debug.assert(bytes.slice.len >= 2); // ""
return parseQuotedString(.wide, allocator, bytes, options);
if (bytes.slice[0] == 'l' or bytes.slice[0] == 'L') {
return parseQuotedWideString(allocator, bytes, options);
}
// Note: We're only handling the case of parsing an ASCII string into a wide string from here on out.
// TODO: The logic below is similar to that in AcceleratorKeyCodepointTranslator, might be worth merging the two
var buf = try std.ArrayList(u16).initCapacity(allocator, bytes.slice.len);
errdefer buf.deinit();
var iterative_parser = IterativeStringParser.init(bytes, options);
while (try iterative_parser.next()) |parsed| {
const c = parsed.codepoint;
if (parsed.from_escaped_integer) {
std.debug.assert(c != code_pages.Codepoint.invalid);
const byte_to_interpret: u8 = @truncate(c);
const code_unit_to_encode: u16 = switch (options.output_code_page) {
.windows1252 => windows1252.toCodepoint(byte_to_interpret),
.utf8 => if (byte_to_interpret > 0x7F) '<27>' else byte_to_interpret,
};
try buf.append(std.mem.nativeToLittle(u16, code_unit_to_encode));
} else if (c == code_pages.Codepoint.invalid) {
try buf.append(std.mem.nativeToLittle(u16, '<27>'));
} else if (c < 0x10000) {
const short: u16 = @intCast(c);
try buf.append(std.mem.nativeToLittle(u16, short));
} else {
if (!parsed.escaped_surrogate_pair) {
const high = @as(u16, @intCast((c - 0x10000) >> 10)) + 0xD800;
try buf.append(std.mem.nativeToLittle(u16, high));
}
const low = @as(u16, @intCast(c & 0x3FF)) + 0xDC00;
try buf.append(std.mem.nativeToLittle(u16, low));
}
}
return buf.toOwnedSliceSentinel(0);
} }
test "parse quoted ascii string" { test "parse quoted ascii string" {
@ -464,133 +607,155 @@ test "parse quoted ascii string" {
\\"hello" \\"hello"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// hex with 0 digits // hex with 0 digits
try std.testing.expectEqualSlices(u8, "\x00", try parseQuotedAsciiString(arena, .{ try std.testing.expectEqualSlices(u8, "\x00", try parseQuotedAsciiString(arena, .{
.slice = .slice =
\\"\x" \\"\x"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// hex max of 2 digits // hex max of 2 digits
try std.testing.expectEqualSlices(u8, "\xFFf", try parseQuotedAsciiString(arena, .{ try std.testing.expectEqualSlices(u8, "\xFFf", try parseQuotedAsciiString(arena, .{
.slice = .slice =
\\"\XfFf" \\"\XfFf"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// octal with invalid octal digit // octal with invalid octal digit
try std.testing.expectEqualSlices(u8, "\x019", try parseQuotedAsciiString(arena, .{ try std.testing.expectEqualSlices(u8, "\x019", try parseQuotedAsciiString(arena, .{
.slice = .slice =
\\"\19" \\"\19"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// escaped quotes // escaped quotes
try std.testing.expectEqualSlices(u8, " \" ", try parseQuotedAsciiString(arena, .{ try std.testing.expectEqualSlices(u8, " \" ", try parseQuotedAsciiString(arena, .{
.slice = .slice =
\\" "" " \\" "" "
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// backslash right before escaped quotes // backslash right before escaped quotes
try std.testing.expectEqualSlices(u8, "\"", try parseQuotedAsciiString(arena, .{ try std.testing.expectEqualSlices(u8, "\"", try parseQuotedAsciiString(arena, .{
.slice = .slice =
\\"\""" \\"\"""
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// octal overflow // octal overflow
try std.testing.expectEqualSlices(u8, "\x01", try parseQuotedAsciiString(arena, .{ try std.testing.expectEqualSlices(u8, "\x01", try parseQuotedAsciiString(arena, .{
.slice = .slice =
\\"\401" \\"\401"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// escapes // escapes
try std.testing.expectEqualSlices(u8, "\x08\n\r\t\\", try parseQuotedAsciiString(arena, .{ try std.testing.expectEqualSlices(u8, "\x08\n\r\t\\", try parseQuotedAsciiString(arena, .{
.slice = .slice =
\\"\a\n\r\t\\" \\"\a\n\r\t\\"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// uppercase escapes // uppercase escapes
try std.testing.expectEqualSlices(u8, "\x08\\N\\R\t\\", try parseQuotedAsciiString(arena, .{ try std.testing.expectEqualSlices(u8, "\x08\\N\\R\t\\", try parseQuotedAsciiString(arena, .{
.slice = .slice =
\\"\A\N\R\T\\" \\"\A\N\R\T\\"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// backslash on its own // backslash on its own
try std.testing.expectEqualSlices(u8, "\\", try parseQuotedAsciiString(arena, .{ try std.testing.expectEqualSlices(u8, "\\", try parseQuotedAsciiString(arena, .{
.slice = .slice =
\\"\" \\"\"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// unrecognized escapes // unrecognized escapes
try std.testing.expectEqualSlices(u8, "\\b", try parseQuotedAsciiString(arena, .{ try std.testing.expectEqualSlices(u8, "\\b", try parseQuotedAsciiString(arena, .{
.slice = .slice =
\\"\b" \\"\b"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// escaped carriage returns // escaped carriage returns
try std.testing.expectEqualSlices(u8, "\\", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "\\", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"\\\r\r\r\r\r\"", .code_page = .windows1252 }, .{ .slice = "\"\\\r\r\r\r\r\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// escaped newlines // escaped newlines
try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"\\\n\n\n\n\n\"", .code_page = .windows1252 }, .{ .slice = "\"\\\n\n\n\n\n\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// escaped CRLF pairs // escaped CRLF pairs
try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"\\\r\n\r\n\r\n\r\n\r\n\"", .code_page = .windows1252 }, .{ .slice = "\"\\\r\n\r\n\r\n\r\n\r\n\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// escaped newlines with other whitespace // escaped newlines with other whitespace
try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"\\\n \t\r\n \r\t\n \t\"", .code_page = .windows1252 }, .{ .slice = "\"\\\n \t\r\n \r\t\n \t\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// literal tab characters get converted to spaces (dependent on source file columns) // literal tab characters get converted to spaces (dependent on source file columns)
try std.testing.expectEqualSlices(u8, " ", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, " ", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"\t\"", .code_page = .windows1252 }, .{ .slice = "\"\t\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqualSlices(u8, "abc ", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "abc ", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"abc\t\"", .code_page = .windows1252 }, .{ .slice = "\"abc\t\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqualSlices(u8, "abcdefg ", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "abcdefg ", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"abcdefg\t\"", .code_page = .windows1252 }, .{ .slice = "\"abcdefg\t\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqualSlices(u8, "\\ ", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "\\ ", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"\\\t\"", .code_page = .windows1252 }, .{ .slice = "\"\\\t\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// literal CR's get dropped // literal CR's get dropped
try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"\r\r\r\r\r\"", .code_page = .windows1252 }, .{ .slice = "\"\r\r\r\r\r\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// contiguous newlines and whitespace get collapsed to <space><newline> // contiguous newlines and whitespace get collapsed to <space><newline>
try std.testing.expectEqualSlices(u8, " \n", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, " \n", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"\n\r\r \r\n \t \"", .code_page = .windows1252 }, .{ .slice = "\"\n\r\r \r\n \t \"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
} }
@ -602,32 +767,32 @@ test "parse quoted ascii string with utf8 code page" {
try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"\"", .code_page = .utf8 }, .{ .slice = "\"\"", .code_page = .utf8 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Codepoints that don't have a Windows-1252 representation get converted to ? // Codepoints that don't have a Windows-1252 representation get converted to ?
try std.testing.expectEqualSlices(u8, "?????????", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "?????????", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"кириллица\"", .code_page = .utf8 }, .{ .slice = "\"кириллица\"", .code_page = .utf8 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Codepoints that have a best fit mapping get converted accordingly, // Codepoints that have a best fit mapping get converted accordingly,
// these are box drawing codepoints // these are box drawing codepoints
try std.testing.expectEqualSlices(u8, "\x2b\x2d\x2b", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "\x2b\x2d\x2b", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"┌─┐\"", .code_page = .utf8 }, .{ .slice = "\"┌─┐\"", .code_page = .utf8 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Invalid UTF-8 gets converted to ? depending on well-formedness // Invalid UTF-8 gets converted to ? depending on well-formedness
try std.testing.expectEqualSlices(u8, "????", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "????", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"\xf0\xf0\x80\x80\x80\"", .code_page = .utf8 }, .{ .slice = "\"\xf0\xf0\x80\x80\x80\"", .code_page = .utf8 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Codepoints that would require a UTF-16 surrogate pair get converted to ?? // Codepoints that would require a UTF-16 surrogate pair get converted to ??
try std.testing.expectEqualSlices(u8, "??", try parseQuotedAsciiString( try std.testing.expectEqualSlices(u8, "??", try parseQuotedAsciiString(
arena, arena,
.{ .slice = "\"\xF2\xAF\xBA\xB4\"", .code_page = .utf8 }, .{ .slice = "\"\xF2\xAF\xBA\xB4\"", .code_page = .utf8 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Output code page changes how invalid UTF-8 gets converted, since it // Output code page changes how invalid UTF-8 gets converted, since it
@ -652,6 +817,18 @@ test "parse quoted ascii string with utf8 code page" {
)); ));
} }
test "parse quoted string with different input/output code pages" {
var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_allocator.deinit();
const arena = arena_allocator.allocator();
try std.testing.expectEqualSlices(u8, "€<EFBFBD><EFBFBD><EFBFBD>\x60\x7F", try parseQuotedAsciiString(
arena,
.{ .slice = "\"\x80\\x8a\\600\\612\\540\\577\"", .code_page = .windows1252 },
.{ .output_code_page = .utf8 },
));
}
test "parse quoted wide string" { test "parse quoted wide string" {
var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator); var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_allocator.deinit(); defer arena_allocator.deinit();
@ -662,52 +839,62 @@ test "parse quoted wide string" {
\\L"hello" \\L"hello"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// hex with 0 digits // hex with 0 digits
try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{0x0}, try parseQuotedWideString(arena, .{ try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{0x0}, try parseQuotedWideString(arena, .{
.slice = .slice =
\\L"\x" \\L"\x"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// hex max of 4 digits // hex max of 4 digits
try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0xFFFF), std.mem.nativeToLittle(u16, 'f') }, try parseQuotedWideString(arena, .{ try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0xFFFF), std.mem.nativeToLittle(u16, 'f') }, try parseQuotedWideString(arena, .{
.slice = .slice =
\\L"\XfFfFf" \\L"\XfFfFf"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// octal max of 7 digits // octal max of 7 digits
try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0x9493), std.mem.nativeToLittle(u16, '3'), std.mem.nativeToLittle(u16, '3') }, try parseQuotedWideString(arena, .{ try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0x9493), std.mem.nativeToLittle(u16, '3'), std.mem.nativeToLittle(u16, '3') }, try parseQuotedWideString(arena, .{
.slice = .slice =
\\L"\111222333" \\L"\111222333"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// octal overflow // octal overflow
try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{std.mem.nativeToLittle(u16, 0xFF01)}, try parseQuotedWideString(arena, .{ try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{std.mem.nativeToLittle(u16, 0xFF01)}, try parseQuotedWideString(arena, .{
.slice = .slice =
\\L"\777401" \\L"\777401"
, ,
.code_page = .windows1252, .code_page = .windows1252,
}, .{})); }, .{
.output_code_page = .windows1252,
}));
// literal tab characters get converted to spaces (dependent on source file columns) // literal tab characters get converted to spaces (dependent on source file columns)
try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("abcdefg "), try parseQuotedWideString( try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("abcdefg "), try parseQuotedWideString(
arena, arena,
.{ .slice = "L\"abcdefg\t\"", .code_page = .windows1252 }, .{ .slice = "L\"abcdefg\t\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Windows-1252 conversion // Windows-1252 conversion
try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("ðð€€€"), try parseQuotedWideString( try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("ðð€€€"), try parseQuotedWideString(
arena, arena,
.{ .slice = "L\"\xf0\xf0\x80\x80\x80\"", .code_page = .windows1252 }, .{ .slice = "L\"\xf0\xf0\x80\x80\x80\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Invalid escape sequences are skipped // Invalid escape sequences are skipped
try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral(""), try parseQuotedWideString( try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral(""), try parseQuotedWideString(
arena, arena,
.{ .slice = "L\"\\H\"", .code_page = .windows1252 }, .{ .slice = "L\"\\H\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
} }
@ -719,18 +906,18 @@ test "parse quoted wide string with utf8 code page" {
try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{}, try parseQuotedWideString( try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{}, try parseQuotedWideString(
arena, arena,
.{ .slice = "L\"\"", .code_page = .utf8 }, .{ .slice = "L\"\"", .code_page = .utf8 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("кириллица"), try parseQuotedWideString( try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("кириллица"), try parseQuotedWideString(
arena, arena,
.{ .slice = "L\"кириллица\"", .code_page = .utf8 }, .{ .slice = "L\"кириллица\"", .code_page = .utf8 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Invalid UTF-8 gets converted to <EFBFBD> depending on well-formedness // Invalid UTF-8 gets converted to <EFBFBD> depending on well-formedness
try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("<EFBFBD><EFBFBD><EFBFBD><EFBFBD>"), try parseQuotedWideString( try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("<EFBFBD><EFBFBD><EFBFBD><EFBFBD>"), try parseQuotedWideString(
arena, arena,
.{ .slice = "L\"\xf0\xf0\x80\x80\x80\"", .code_page = .utf8 }, .{ .slice = "L\"\xf0\xf0\x80\x80\x80\"", .code_page = .utf8 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
} }
@ -742,29 +929,29 @@ test "parse quoted ascii string as wide string" {
try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("кириллица"), try parseQuotedStringAsWideString( try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("кириллица"), try parseQuotedStringAsWideString(
arena, arena,
.{ .slice = "\"кириллица\"", .code_page = .utf8 }, .{ .slice = "\"кириллица\"", .code_page = .utf8 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Whether or not invalid escapes are skipped is still determined by the L prefix // Whether or not invalid escapes are skipped is still determined by the L prefix
try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("\\H"), try parseQuotedStringAsWideString( try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("\\H"), try parseQuotedStringAsWideString(
arena, arena,
.{ .slice = "\"\\H\"", .code_page = .windows1252 }, .{ .slice = "\"\\H\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral(""), try parseQuotedStringAsWideString( try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral(""), try parseQuotedStringAsWideString(
arena, arena,
.{ .slice = "L\"\\H\"", .code_page = .windows1252 }, .{ .slice = "L\"\\H\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Maximum escape sequence value is also determined by the L prefix // Maximum escape sequence value is also determined by the L prefix
try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0x12), std.mem.nativeToLittle(u16, '3'), std.mem.nativeToLittle(u16, '4') }, try parseQuotedStringAsWideString( try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0x12), std.mem.nativeToLittle(u16, '3'), std.mem.nativeToLittle(u16, '4') }, try parseQuotedStringAsWideString(
arena, arena,
.{ .slice = "\"\\x1234\"", .code_page = .windows1252 }, .{ .slice = "\"\\x1234\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{std.mem.nativeToLittle(u16, 0x1234)}, try parseQuotedStringAsWideString( try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{std.mem.nativeToLittle(u16, 0x1234)}, try parseQuotedStringAsWideString(
arena, arena,
.{ .slice = "L\"\\x1234\"", .code_page = .windows1252 }, .{ .slice = "L\"\\x1234\"", .code_page = .windows1252 },
.{}, .{ .output_code_page = .windows1252 },
)); ));
} }

36
lib/compiler/resinator/main.zig
View File

@ -7,6 +7,7 @@ const Diagnostics = @import("errors.zig").Diagnostics;
const cli = @import("cli.zig"); const cli = @import("cli.zig");
const preprocess = @import("preprocess.zig"); const preprocess = @import("preprocess.zig");
const renderErrorMessage = @import("utils.zig").renderErrorMessage; const renderErrorMessage = @import("utils.zig").renderErrorMessage;
const hasDisjointCodePage = @import("disjoint_code_page.zig").hasDisjointCodePage;
const aro = @import("aro"); const aro = @import("aro");
pub fn main() !void { pub fn main() !void {
@ -64,7 +65,7 @@ pub fn main() !void {
if (!zig_integration) { if (!zig_integration) {
// print any warnings/notes // print any warnings/notes
cli_diagnostics.renderToStdErr(args, stderr_config); cli_diagnostics.renderToStdErr(cli_args, stderr_config);
// If there was something printed, then add an extra newline separator // If there was something printed, then add an extra newline separator
// so that there is a clear separation between the cli diagnostics and whatever // so that there is a clear separation between the cli diagnostics and whatever
// gets printed after // gets printed after
@ -179,16 +180,30 @@ pub fn main() !void {
// Note: We still want to run this when no-preprocess is set because: // Note: We still want to run this when no-preprocess is set because:
// 1. We want to print accurate line numbers after removing multiline comments // 1. We want to print accurate line numbers after removing multiline comments
// 2. We want to be able to handle an already-preprocessed input with #line commands in it // 2. We want to be able to handle an already-preprocessed input with #line commands in it
var mapping_results = try parseAndRemoveLineCommands(allocator, full_input, full_input, .{ .initial_filename = options.input_filename }); var mapping_results = parseAndRemoveLineCommands(allocator, full_input, full_input, .{ .initial_filename = options.input_filename }) catch |err| switch (err) {
defer mapping_results.mappings.deinit(allocator); error.InvalidLineCommand => {
// TODO: Maybe output the invalid line command
const final_input = removeComments(mapping_results.result, mapping_results.result, &mapping_results.mappings) catch |err| switch (err) { try renderErrorMessage(stderr.writer(), stderr_config, .err, "invalid line command in the preprocessed source", .{});
error.InvalidSourceMappingCollapse => { if (options.preprocess == .no) {
try error_handler.emitMessage(allocator, .err, "failed during comment removal; this is a known bug", .{}); try renderErrorMessage(stderr.writer(), stderr_config, .note, "line commands must be of the format: #line <num> \"<path>\"", .{});
} else {
try renderErrorMessage(stderr.writer(), stderr_config, .note, "this is likely to be a bug, please report it", .{});
}
std.process.exit(1); std.process.exit(1);
}, },
else => |e| return e, error.LineNumberOverflow => {
// TODO: Better error message
try renderErrorMessage(stderr.writer(), stderr_config, .err, "line number count exceeded maximum of {}", .{std.math.maxInt(usize)});
std.process.exit(1);
},
error.OutOfMemory => |e| return e,
}; };
defer mapping_results.mappings.deinit(allocator);
const default_code_page = options.default_code_page orelse .windows1252;
const has_disjoint_code_page = hasDisjointCodePage(mapping_results.result, &mapping_results.mappings, default_code_page);
const final_input = try removeComments(mapping_results.result, mapping_results.result, &mapping_results.mappings);
var output_file = std.fs.cwd().createFile(options.output_filename, .{}) catch |err| { var output_file = std.fs.cwd().createFile(options.output_filename, .{}) catch |err| {
try error_handler.emitMessage(allocator, .err, "unable to create output file '{s}': {s}", .{ options.output_filename, @errorName(err) }); try error_handler.emitMessage(allocator, .err, "unable to create output file '{s}': {s}", .{ options.output_filename, @errorName(err) });
@ -211,7 +226,8 @@ pub fn main() !void {
.extra_include_paths = options.extra_include_paths.items, .extra_include_paths = options.extra_include_paths.items,
.system_include_paths = include_paths, .system_include_paths = include_paths,
.default_language_id = options.default_language_id, .default_language_id = options.default_language_id,
.default_code_page = options.default_code_page orelse .windows1252, .default_code_page = default_code_page,
.disjoint_code_page = has_disjoint_code_page,
.verbose = options.verbose, .verbose = options.verbose,
.null_terminate_string_table_strings = options.null_terminate_string_table_strings, .null_terminate_string_table_strings = options.null_terminate_string_table_strings,
.max_string_literal_codepoints = options.max_string_literal_codepoints, .max_string_literal_codepoints = options.max_string_literal_codepoints,
@ -513,7 +529,7 @@ fn diagnosticsToErrorBundle(
}; };
if (err_details.print_source_line) { if (err_details.print_source_line) {
const source_line = err_details.token.getLineForErrorDisplay(source, source_line_start); const source_line = err_details.token.getLineForErrorDisplay(source, source_line_start);
const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len); const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len, source);
src_loc.span_start = @intCast(visual_info.point_offset - visual_info.before_len); src_loc.span_start = @intCast(visual_info.point_offset - visual_info.before_len);
src_loc.span_main = @intCast(visual_info.point_offset); src_loc.span_main = @intCast(visual_info.point_offset);
src_loc.span_end = @intCast(visual_info.point_offset + 1 + visual_info.after_len); src_loc.span_end = @intCast(visual_info.point_offset + 1 + visual_info.after_len);

243
lib/compiler/resinator/parse.zig
View File

@ -4,9 +4,10 @@ const Token = @import("lex.zig").Token;
const Node = @import("ast.zig").Node; const Node = @import("ast.zig").Node;
const Tree = @import("ast.zig").Tree; const Tree = @import("ast.zig").Tree;
const CodePageLookup = @import("ast.zig").CodePageLookup; const CodePageLookup = @import("ast.zig").CodePageLookup;
const Resource = @import("rc.zig").Resource; const ResourceType = @import("rc.zig").ResourceType;
const Allocator = std.mem.Allocator; const Allocator = std.mem.Allocator;
const ErrorDetails = @import("errors.zig").ErrorDetails; const ErrorDetails = @import("errors.zig").ErrorDetails;
const ErrorDetailsWithoutCodePage = @import("errors.zig").ErrorDetailsWithoutCodePage;
const Diagnostics = @import("errors.zig").Diagnostics; const Diagnostics = @import("errors.zig").Diagnostics;
const SourceBytes = @import("literals.zig").SourceBytes; const SourceBytes = @import("literals.zig").SourceBytes;
const Compiler = @import("compile.zig").Compiler; const Compiler = @import("compile.zig").Compiler;
@ -30,6 +31,7 @@ pub const Parser = struct {
pub const Options = struct { pub const Options = struct {
warn_instead_of_error_on_invalid_code_page: bool = false, warn_instead_of_error_on_invalid_code_page: bool = false,
disjoint_code_page: bool = false,
}; };
pub fn init(lexer: *Lexer, options: Options) Parser { pub fn init(lexer: *Lexer, options: Options) Parser {
@ -47,6 +49,7 @@ pub const Parser = struct {
diagnostics: *Diagnostics, diagnostics: *Diagnostics,
input_code_page_lookup: CodePageLookup, input_code_page_lookup: CodePageLookup,
output_code_page_lookup: CodePageLookup, output_code_page_lookup: CodePageLookup,
warned_about_disjoint_code_page: bool,
}; };
pub fn parse(self: *Self, allocator: Allocator, diagnostics: *Diagnostics) Error!*Tree { pub fn parse(self: *Self, allocator: Allocator, diagnostics: *Diagnostics) Error!*Tree {
@ -61,6 +64,7 @@ pub const Parser = struct {
.diagnostics = diagnostics, .diagnostics = diagnostics,
.input_code_page_lookup = CodePageLookup.init(arena.allocator(), self.lexer.default_code_page), .input_code_page_lookup = CodePageLookup.init(arena.allocator(), self.lexer.default_code_page),
.output_code_page_lookup = CodePageLookup.init(arena.allocator(), self.lexer.default_code_page), .output_code_page_lookup = CodePageLookup.init(arena.allocator(), self.lexer.default_code_page),
.warned_about_disjoint_code_page = false,
}; };
const parsed_root = try self.parseRoot(); const parsed_root = try self.parseRoot();
@ -116,7 +120,7 @@ pub const Parser = struct {
const maybe_common_resource_attribute = try self.lookaheadToken(.normal); const maybe_common_resource_attribute = try self.lookaheadToken(.normal);
if (maybe_common_resource_attribute.id == .literal and rc.CommonResourceAttributes.map.has(maybe_common_resource_attribute.slice(self.lexer.buffer))) { if (maybe_common_resource_attribute.id == .literal and rc.CommonResourceAttributes.map.has(maybe_common_resource_attribute.slice(self.lexer.buffer))) {
try common_resource_attributes.append(self.state.arena, maybe_common_resource_attribute); try common_resource_attributes.append(self.state.arena, maybe_common_resource_attribute);
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
} else { } else {
break; break;
} }
@ -130,8 +134,13 @@ pub const Parser = struct {
/// optional statements (if any). If there are no optional statements, the /// optional statements (if any). If there are no optional statements, the
/// current token is unchanged. /// current token is unchanged.
/// The returned slice is allocated by the parser's arena /// The returned slice is allocated by the parser's arena
fn parseOptionalStatements(self: *Self, resource: Resource) ![]*Node { fn parseOptionalStatements(self: *Self, resource: ResourceType) ![]*Node {
var optional_statements: std.ArrayListUnmanaged(*Node) = .empty; var optional_statements: std.ArrayListUnmanaged(*Node) = .empty;
const num_statement_types = @typeInfo(rc.OptionalStatements).@"enum".fields.len;
var statement_type_has_duplicates = [_]bool{false} ** num_statement_types;
var last_statement_per_type = [_]?*Node{null} ** num_statement_types;
while (true) { while (true) {
const lookahead_token = try self.lookaheadToken(.normal); const lookahead_token = try self.lookaheadToken(.normal);
if (lookahead_token.id != .literal) break; if (lookahead_token.id != .literal) break;
@ -140,7 +149,13 @@ pub const Parser = struct {
.dialog, .dialogex => rc.OptionalStatements.dialog_map.get(slice) orelse break, .dialog, .dialogex => rc.OptionalStatements.dialog_map.get(slice) orelse break,
else => break, else => break,
}; };
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
const type_i = @intFromEnum(optional_statement_type);
if (last_statement_per_type[type_i] != null) {
statement_type_has_duplicates[type_i] = true;
}
switch (optional_statement_type) { switch (optional_statement_type) {
.language => { .language => {
const language = try self.parseLanguageStatement(); const language = try self.parseLanguageStatement();
@ -166,7 +181,7 @@ pub const Parser = struct {
try self.nextToken(.normal); try self.nextToken(.normal);
const value = self.state.token; const value = self.state.token;
if (!value.isStringLiteral()) { if (!value.isStringLiteral()) {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_something_else, .err = .expected_something_else,
.token = value, .token = value,
.extra = .{ .expected_types = .{ .extra = .{ .expected_types = .{
@ -223,7 +238,7 @@ pub const Parser = struct {
try self.nextToken(.normal); try self.nextToken(.normal);
const typeface = self.state.token; const typeface = self.state.token;
if (!typeface.isStringLiteral()) { if (!typeface.isStringLiteral()) {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_something_else, .err = .expected_something_else,
.token = typeface, .token = typeface,
.extra = .{ .expected_types = .{ .extra = .{ .expected_types = .{
@ -272,7 +287,42 @@ pub const Parser = struct {
try optional_statements.append(self.state.arena, &node.base); try optional_statements.append(self.state.arena, &node.base);
}, },
} }
last_statement_per_type[type_i] = optional_statements.items[optional_statements.items.len - 1];
} }
for (optional_statements.items) |optional_statement| {
const type_i = type_i: {
switch (optional_statement.id) {
.simple_statement => {
const simple_statement: *Node.SimpleStatement = @alignCast(@fieldParentPtr("base", optional_statement));
const statement_identifier = simple_statement.identifier;
const slice = statement_identifier.slice(self.lexer.buffer);
const optional_statement_type = rc.OptionalStatements.map.get(slice) orelse
rc.OptionalStatements.dialog_map.get(slice).?;
break :type_i @intFromEnum(optional_statement_type);
},
.font_statement => {
break :type_i @intFromEnum(rc.OptionalStatements.font);
},
.language_statement => {
break :type_i @intFromEnum(rc.OptionalStatements.language);
},
else => unreachable,
}
};
if (!statement_type_has_duplicates[type_i]) continue;
if (optional_statement == last_statement_per_type[type_i].?) continue;
try self.addErrorDetails(.{
.err = .duplicate_optional_statement_skipped,
.type = .warning,
.token = optional_statement.getFirstToken(),
.token_span_start = optional_statement.getFirstToken(),
.token_span_end = optional_statement.getLastToken(),
});
}
return optional_statements.toOwnedSlice(self.state.arena); return optional_statements.toOwnedSlice(self.state.arena);
} }
@ -311,12 +361,13 @@ pub const Parser = struct {
const maybe_end_token = try self.lookaheadToken(.normal); const maybe_end_token = try self.lookaheadToken(.normal);
switch (maybe_end_token.id) { switch (maybe_end_token.id) {
.end => { .end => {
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
break; break;
}, },
.eof => { .eof => {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsWithCodePageAndFail(.{
.err = .unfinished_string_table_block, .err = .unfinished_string_table_block,
.code_page = self.lexer.current_code_page,
.token = maybe_end_token, .token = maybe_end_token,
}); });
}, },
@ -328,7 +379,7 @@ pub const Parser = struct {
try self.nextToken(.normal); try self.nextToken(.normal);
if (self.state.token.id != .quoted_ascii_string and self.state.token.id != .quoted_wide_string) { if (self.state.token.id != .quoted_ascii_string and self.state.token.id != .quoted_wide_string) {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_something_else, .err = .expected_something_else,
.token = self.state.token, .token = self.state.token,
.extra = .{ .expected_types = .{ .string_literal = true } }, .extra = .{ .expected_types = .{ .string_literal = true } },
@ -345,7 +396,7 @@ pub const Parser = struct {
} }
if (strings.items.len == 0) { if (strings.items.len == 0) {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_token, // TODO: probably a more specific error message .err = .expected_token, // TODO: probably a more specific error message
.token = self.state.token, .token = self.state.token,
.extra = .{ .expected = .number }, .extra = .{ .expected = .number },
@ -374,7 +425,12 @@ pub const Parser = struct {
// of projects. So, we have special compatibility for this particular case. // of projects. So, we have special compatibility for this particular case.
const maybe_eof = try self.lookaheadToken(.whitespace_delimiter_only); const maybe_eof = try self.lookaheadToken(.whitespace_delimiter_only);
if (maybe_eof.id == .eof) { if (maybe_eof.id == .eof) {
// TODO: emit warning try self.addErrorDetails(.{
.err = .dangling_literal_at_eof,
.type = .warning,
.token = first_token,
});
var context = try self.state.arena.alloc(Token, 2); var context = try self.state.arena.alloc(Token, 2);
context[0] = first_token; context[0] = first_token;
context[1] = maybe_eof; context[1] = maybe_eof;
@ -413,12 +469,12 @@ pub const Parser = struct {
if (maybe_ordinal == null) { if (maybe_ordinal == null) {
const would_be_win32_rc_ordinal = res.NameOrOrdinal.maybeNonAsciiOrdinalFromString(id_bytes); const would_be_win32_rc_ordinal = res.NameOrOrdinal.maybeNonAsciiOrdinalFromString(id_bytes);
if (would_be_win32_rc_ordinal) |win32_rc_ordinal| { if (would_be_win32_rc_ordinal) |win32_rc_ordinal| {
try self.addErrorDetails(ErrorDetails{ try self.addErrorDetails(.{
.err = .id_must_be_ordinal, .err = .id_must_be_ordinal,
.token = id_token, .token = id_token,
.extra = .{ .resource = resource }, .extra = .{ .resource = resource },
}); });
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .win32_non_ascii_ordinal, .err = .win32_non_ascii_ordinal,
.token = id_token, .token = id_token,
.type = .note, .type = .note,
@ -426,7 +482,7 @@ pub const Parser = struct {
.extra = .{ .number = win32_rc_ordinal.ordinal }, .extra = .{ .number = win32_rc_ordinal.ordinal },
}); });
} else { } else {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .id_must_be_ordinal, .err = .id_must_be_ordinal,
.token = id_token, .token = id_token,
.extra = .{ .resource = resource }, .extra = .{ .resource = resource },
@ -451,7 +507,7 @@ pub const Parser = struct {
const lookahead = try self.lookaheadToken(.normal); const lookahead = try self.lookaheadToken(.normal);
switch (lookahead.id) { switch (lookahead.id) {
.end, .eof => { .end, .eof => {
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
break; break;
}, },
else => {}, else => {},
@ -739,19 +795,19 @@ pub const Parser = struct {
const maybe_begin = try self.lookaheadToken(.normal); const maybe_begin = try self.lookaheadToken(.normal);
if (maybe_begin.id == .begin) { if (maybe_begin.id == .begin) {
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
if (!resource.canUseRawData()) { if (!resource.canUseRawData()) {
try self.addErrorDetails(ErrorDetails{ try self.addErrorDetails(.{
.err = .resource_type_cant_use_raw_data, .err = .resource_type_cant_use_raw_data,
.token = maybe_begin, .token = self.state.token,
.extra = .{ .resource = resource }, .extra = .{ .resource = resource },
}); });
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .resource_type_cant_use_raw_data, .err = .resource_type_cant_use_raw_data,
.type = .note, .type = .note,
.print_source_line = false, .print_source_line = false,
.token = maybe_begin, .token = self.state.token,
}); });
} }
@ -802,11 +858,12 @@ pub const Parser = struct {
const maybe_end_token = try self.lookaheadToken(.normal); const maybe_end_token = try self.lookaheadToken(.normal);
switch (maybe_end_token.id) { switch (maybe_end_token.id) {
.comma => { .comma => {
try self.nextToken(.normal);
// comma as the first token in a raw data block is an error // comma as the first token in a raw data block is an error
if (raw_data.items.len == 0) { if (raw_data.items.len == 0) {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_something_else, .err = .expected_something_else,
.token = maybe_end_token, .token = self.state.token,
.extra = .{ .expected_types = .{ .extra = .{ .expected_types = .{
.number = true, .number = true,
.number_expression = true, .number_expression = true,
@ -815,16 +872,16 @@ pub const Parser = struct {
}); });
} }
// otherwise just skip over commas // otherwise just skip over commas
self.nextToken(.normal) catch unreachable;
continue; continue;
}, },
.end => { .end => {
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
break; break;
}, },
.eof => { .eof => {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsWithCodePageAndFail(.{
.err = .unfinished_raw_data_block, .err = .unfinished_raw_data_block,
.code_page = self.lexer.current_code_page,
.token = maybe_end_token, .token = maybe_end_token,
}); });
}, },
@ -836,10 +893,12 @@ pub const Parser = struct {
if (expression.isNumberExpression()) { if (expression.isNumberExpression()) {
const maybe_close_paren = try self.lookaheadToken(.normal); const maybe_close_paren = try self.lookaheadToken(.normal);
if (maybe_close_paren.id == .close_paren) { if (maybe_close_paren.id == .close_paren) {
// advance to ensure that the code page lookup is populated for this token
try self.nextToken(.normal);
// <number expression>) is an error // <number expression>) is an error
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_token, .err = .expected_token,
.token = maybe_close_paren, .token = self.state.token,
.extra = .{ .expected = .operator }, .extra = .{ .expected = .operator },
}); });
} }
@ -852,10 +911,10 @@ pub const Parser = struct {
/// begin on the next token. /// begin on the next token.
/// After return, the current token will be the token immediately before the end of the /// After return, the current token will be the token immediately before the end of the
/// control statement (or unchanged if the function returns null). /// control statement (or unchanged if the function returns null).
fn parseControlStatement(self: *Self, resource: Resource) Error!?*Node { fn parseControlStatement(self: *Self, resource: ResourceType) Error!?*Node {
const control_token = try self.lookaheadToken(.normal); const control_token = try self.lookaheadToken(.normal);
const control = rc.Control.map.get(control_token.slice(self.lexer.buffer)) orelse return null; const control = rc.Control.map.get(control_token.slice(self.lexer.buffer)) orelse return null;
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
try self.skipAnyCommas(); try self.skipAnyCommas();
@ -867,7 +926,7 @@ pub const Parser = struct {
text = self.state.token; text = self.state.token;
}, },
else => { else => {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_something_else, .err = .expected_something_else,
.token = self.state.token, .token = self.state.token,
.extra = .{ .expected_types = .{ .extra = .{ .expected_types = .{
@ -920,14 +979,16 @@ pub const Parser = struct {
// the style parameter. // the style parameter.
const lookahead_token = try self.lookaheadToken(.normal); const lookahead_token = try self.lookaheadToken(.normal);
if (lookahead_token.id != .comma and lookahead_token.id != .eof) { if (lookahead_token.id != .comma and lookahead_token.id != .eof) {
try self.addErrorDetails(.{ try self.addErrorDetailsWithCodePage(.{
.err = .rc_could_miscompile_control_params, .err = .rc_could_miscompile_control_params,
.type = .warning, .type = .warning,
.code_page = self.lexer.current_code_page,
.token = lookahead_token, .token = lookahead_token,
}); });
try self.addErrorDetails(.{ try self.addErrorDetailsWithCodePage(.{
.err = .rc_could_miscompile_control_params, .err = .rc_could_miscompile_control_params,
.type = .note, .type = .note,
.code_page = self.lexer.current_code_page,
.token = style.?.getFirstToken(), .token = style.?.getFirstToken(),
.token_span_end = style.?.getLastToken(), .token_span_end = style.?.getLastToken(),
}); });
@ -987,7 +1048,7 @@ pub const Parser = struct {
fn parseToolbarButtonStatement(self: *Self) Error!?*Node { fn parseToolbarButtonStatement(self: *Self) Error!?*Node {
const keyword_token = try self.lookaheadToken(.normal); const keyword_token = try self.lookaheadToken(.normal);
const button_type = rc.ToolbarButton.map.get(keyword_token.slice(self.lexer.buffer)) orelse return null; const button_type = rc.ToolbarButton.map.get(keyword_token.slice(self.lexer.buffer)) orelse return null;
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
switch (button_type) { switch (button_type) {
.separator => { .separator => {
@ -1014,10 +1075,10 @@ pub const Parser = struct {
/// begin on the next token. /// begin on the next token.
/// After return, the current token will be the token immediately before the end of the /// After return, the current token will be the token immediately before the end of the
/// menuitem statement (or unchanged if the function returns null). /// menuitem statement (or unchanged if the function returns null).
fn parseMenuItemStatement(self: *Self, resource: Resource, top_level_menu_id_token: Token, nesting_level: u32) Error!?*Node { fn parseMenuItemStatement(self: *Self, resource: ResourceType, top_level_menu_id_token: Token, nesting_level: u32) Error!?*Node {
const menuitem_token = try self.lookaheadToken(.normal); const menuitem_token = try self.lookaheadToken(.normal);
const menuitem = rc.MenuItem.map.get(menuitem_token.slice(self.lexer.buffer)) orelse return null; const menuitem = rc.MenuItem.map.get(menuitem_token.slice(self.lexer.buffer)) orelse return null;
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
if (nesting_level > max_nested_menu_level) { if (nesting_level > max_nested_menu_level) {
try self.addErrorDetails(.{ try self.addErrorDetails(.{
@ -1050,7 +1111,7 @@ pub const Parser = struct {
} else { } else {
const text = self.state.token; const text = self.state.token;
if (!text.isStringLiteral()) { if (!text.isStringLiteral()) {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_something_else, .err = .expected_something_else,
.token = text, .token = text,
.extra = .{ .expected_types = .{ .extra = .{ .expected_types = .{
@ -1070,7 +1131,7 @@ pub const Parser = struct {
if (!rc.MenuItem.Option.map.has(option_token.slice(self.lexer.buffer))) { if (!rc.MenuItem.Option.map.has(option_token.slice(self.lexer.buffer))) {
break; break;
} }
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
try options.append(self.state.arena, option_token); try options.append(self.state.arena, option_token);
try self.skipAnyCommas(); try self.skipAnyCommas();
} }
@ -1089,7 +1150,7 @@ pub const Parser = struct {
try self.nextToken(.normal); try self.nextToken(.normal);
const text = self.state.token; const text = self.state.token;
if (!text.isStringLiteral()) { if (!text.isStringLiteral()) {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_something_else, .err = .expected_something_else,
.token = text, .token = text,
.extra = .{ .expected_types = .{ .extra = .{ .expected_types = .{
@ -1105,7 +1166,7 @@ pub const Parser = struct {
if (!rc.MenuItem.Option.map.has(option_token.slice(self.lexer.buffer))) { if (!rc.MenuItem.Option.map.has(option_token.slice(self.lexer.buffer))) {
break; break;
} }
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
try options.append(self.state.arena, option_token); try options.append(self.state.arena, option_token);
try self.skipAnyCommas(); try self.skipAnyCommas();
} }
@ -1146,7 +1207,7 @@ pub const Parser = struct {
try self.nextToken(.normal); try self.nextToken(.normal);
const text = self.state.token; const text = self.state.token;
if (!text.isStringLiteral()) { if (!text.isStringLiteral()) {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_something_else, .err = .expected_something_else,
.token = text, .token = text,
.extra = .{ .expected_types = .{ .extra = .{ .expected_types = .{
@ -1257,7 +1318,7 @@ pub const Parser = struct {
fn parseVersionStatement(self: *Self) Error!?*Node { fn parseVersionStatement(self: *Self) Error!?*Node {
const type_token = try self.lookaheadToken(.normal); const type_token = try self.lookaheadToken(.normal);
const statement_type = rc.VersionInfo.map.get(type_token.slice(self.lexer.buffer)) orelse return null; const statement_type = rc.VersionInfo.map.get(type_token.slice(self.lexer.buffer)) orelse return null;
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
switch (statement_type) { switch (statement_type) {
.file_version, .product_version => { .file_version, .product_version => {
var parts_buffer: [4]*Node = undefined; var parts_buffer: [4]*Node = undefined;
@ -1301,7 +1362,7 @@ pub const Parser = struct {
fn parseVersionBlockOrValue(self: *Self, top_level_version_id_token: Token, nesting_level: u32) Error!?*Node { fn parseVersionBlockOrValue(self: *Self, top_level_version_id_token: Token, nesting_level: u32) Error!?*Node {
const keyword_token = try self.lookaheadToken(.normal); const keyword_token = try self.lookaheadToken(.normal);
const keyword = rc.VersionBlock.map.get(keyword_token.slice(self.lexer.buffer)) orelse return null; const keyword = rc.VersionBlock.map.get(keyword_token.slice(self.lexer.buffer)) orelse return null;
self.nextToken(.normal) catch unreachable; try self.nextToken(.normal);
if (nesting_level > max_nested_version_level) { if (nesting_level > max_nested_version_level) {
try self.addErrorDetails(.{ try self.addErrorDetails(.{
@ -1541,7 +1602,7 @@ pub const Parser = struct {
} }
}; };
pub fn toErrorDetails(options: ParseExpressionOptions, token: Token) ErrorDetails { pub fn toErrorDetails(options: ParseExpressionOptions, token: Token) ErrorDetailsWithoutCodePage {
// TODO: expected_types_override interaction with is_known_to_be_number_expression? // TODO: expected_types_override interaction with is_known_to_be_number_expression?
const expected_types = options.expected_types_override orelse ErrorDetails.ExpectedTypes{ const expected_types = options.expected_types_override orelse ErrorDetails.ExpectedTypes{
.number = options.allowed_types.number, .number = options.allowed_types.number,
@ -1549,7 +1610,7 @@ pub const Parser = struct {
.string_literal = options.allowed_types.string and !options.is_known_to_be_number_expression, .string_literal = options.allowed_types.string and !options.is_known_to_be_number_expression,
.literal = options.allowed_types.literal and !options.is_known_to_be_number_expression, .literal = options.allowed_types.literal and !options.is_known_to_be_number_expression,
}; };
return ErrorDetails{ return .{
.err = .expected_something_else, .err = .expected_something_else,
.token = token, .token = token,
.extra = .{ .expected_types = expected_types }, .extra = .{ .expected_types = expected_types },
@ -1690,7 +1751,7 @@ pub const Parser = struct {
try self.addErrorDetails(options.toErrorDetails(self.state.token)); try self.addErrorDetails(options.toErrorDetails(self.state.token));
if (is_close_paren_expression) { if (is_close_paren_expression) {
try self.addErrorDetails(ErrorDetails{ try self.addErrorDetails(.{
.err = .close_paren_expression, .err = .close_paren_expression,
.type = .note, .type = .note,
.token = self.state.token, .token = self.state.token,
@ -1698,7 +1759,7 @@ pub const Parser = struct {
}); });
} }
if (is_unary_plus_expression) { if (is_unary_plus_expression) {
try self.addErrorDetails(ErrorDetails{ try self.addErrorDetails(.{
.err = .unary_plus_expression, .err = .unary_plus_expression,
.type = .note, .type = .note,
.token = self.state.token, .token = self.state.token,
@ -1739,7 +1800,7 @@ pub const Parser = struct {
}); });
if (!rhs_node.isNumberExpression()) { if (!rhs_node.isNumberExpression()) {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_something_else, .err = .expected_something_else,
.token = rhs_node.getFirstToken(), .token = rhs_node.getFirstToken(),
.token_span_end = rhs_node.getLastToken(), .token_span_end = rhs_node.getLastToken(),
@ -1781,16 +1842,39 @@ pub const Parser = struct {
fn parseOptionalTokenAdvanced(self: *Self, id: Token.Id, comptime method: Lexer.LexMethod) Error!bool { fn parseOptionalTokenAdvanced(self: *Self, id: Token.Id, comptime method: Lexer.LexMethod) Error!bool {
const maybe_token = try self.lookaheadToken(method); const maybe_token = try self.lookaheadToken(method);
if (maybe_token.id != id) return false; if (maybe_token.id != id) return false;
self.nextToken(method) catch unreachable; try self.nextToken(method);
return true; return true;
} }
fn addErrorDetails(self: *Self, details: ErrorDetails) Allocator.Error!void { fn addErrorDetailsWithCodePage(self: *Self, details: ErrorDetails) Allocator.Error!void {
try self.state.diagnostics.append(details); try self.state.diagnostics.append(details);
} }
fn addErrorDetailsAndFail(self: *Self, details: ErrorDetails) Error { fn addErrorDetailsWithCodePageAndFail(self: *Self, details: ErrorDetails) Error {
try self.addErrorDetails(details); try self.addErrorDetailsWithCodePage(details);
return error.ParseError;
}
/// Code page is looked up in input_code_page_lookup using the token, meaning the token
/// must come from nextToken (i.e. it can't be a lookahead token).
fn addErrorDetails(self: *Self, details_without_code_page: ErrorDetailsWithoutCodePage) Allocator.Error!void {
const details = ErrorDetails{
.err = details_without_code_page.err,
.code_page = self.state.input_code_page_lookup.getForToken(details_without_code_page.token),
.token = details_without_code_page.token,
.token_span_start = details_without_code_page.token_span_start,
.token_span_end = details_without_code_page.token_span_end,
.type = details_without_code_page.type,
.print_source_line = details_without_code_page.print_source_line,
.extra = details_without_code_page.extra,
};
try self.addErrorDetailsWithCodePage(details);
}
/// Code page is looked up in input_code_page_lookup using the token, meaning the token
/// must come from nextToken (i.e. it can't be a lookahead token).
fn addErrorDetailsAndFail(self: *Self, details_without_code_page: ErrorDetailsWithoutCodePage) Error {
try self.addErrorDetails(details_without_code_page);
return error.ParseError; return error.ParseError;
} }
@ -1798,35 +1882,34 @@ pub const Parser = struct {
self.state.token = token: while (true) { self.state.token = token: while (true) {
const token = self.lexer.next(method) catch |err| switch (err) { const token = self.lexer.next(method) catch |err| switch (err) {
error.CodePagePragmaInIncludedFile => { error.CodePagePragmaInIncludedFile => {
// The Win32 RC compiler silently ignores such `#pragma code_point` directives, // The Win32 RC compiler silently ignores such `#pragma code_page` directives,
// but we want to both ignore them *and* emit a warning // but we want to both ignore them *and* emit a warning
try self.addErrorDetails(.{ var details = self.lexer.getErrorDetails(err);
.err = .code_page_pragma_in_included_file, details.type = .warning;
.type = .warning, try self.addErrorDetailsWithCodePage(details);
.token = self.lexer.error_context_token.?,
});
continue; continue;
}, },
error.CodePagePragmaInvalidCodePage => { error.CodePagePragmaInvalidCodePage => {
var details = self.lexer.getErrorDetails(err); var details = self.lexer.getErrorDetails(err);
if (!self.options.warn_instead_of_error_on_invalid_code_page) { if (!self.options.warn_instead_of_error_on_invalid_code_page) {
return self.addErrorDetailsAndFail(details); return self.addErrorDetailsWithCodePageAndFail(details);
} }
details.type = .warning; details.type = .warning;
try self.addErrorDetails(details); try self.addErrorDetailsWithCodePage(details);
continue; continue;
}, },
error.InvalidDigitCharacterInNumberLiteral => { error.InvalidDigitCharacterInNumberLiteral => {
const details = self.lexer.getErrorDetails(err); const details = self.lexer.getErrorDetails(err);
try self.addErrorDetails(details); try self.addErrorDetailsWithCodePage(details);
return self.addErrorDetailsAndFail(.{ return self.addErrorDetailsWithCodePageAndFail(.{
.err = details.err, .err = details.err,
.type = .note, .type = .note,
.code_page = self.lexer.current_code_page,
.token = details.token, .token = details.token,
.print_source_line = false, .print_source_line = false,
}); });
}, },
else => return self.addErrorDetailsAndFail(self.lexer.getErrorDetails(err)), else => return self.addErrorDetailsWithCodePageAndFail(self.lexer.getErrorDetails(err)),
}; };
break :token token; break :token token;
}; };
@ -1835,7 +1918,29 @@ pub const Parser = struct {
// But only set the output code page to the current code page if we are past the first code_page pragma in the file. // But only set the output code page to the current code page if we are past the first code_page pragma in the file.
// Otherwise, we want to fill the lookup using the default code page so that lookups still work for lines that // Otherwise, we want to fill the lookup using the default code page so that lookups still work for lines that
// don't have an explicit output code page set. // don't have an explicit output code page set.
const output_code_page = if (self.lexer.seen_pragma_code_pages > 1) self.lexer.current_code_page else self.state.output_code_page_lookup.default_code_page; const is_disjoint_code_page = self.options.disjoint_code_page and self.lexer.seen_pragma_code_pages == 1;
const output_code_page = if (is_disjoint_code_page)
self.state.output_code_page_lookup.default_code_page
else
self.lexer.current_code_page;
if (is_disjoint_code_page and !self.state.warned_about_disjoint_code_page) {
try self.addErrorDetailsWithCodePage(.{
.err = .disjoint_code_page,
.type = .warning,
.code_page = self.state.input_code_page_lookup.getForLineNum(self.lexer.last_pragma_code_page_token.?.line_number),
.token = self.lexer.last_pragma_code_page_token.?,
});
try self.addErrorDetailsWithCodePage(.{
.err = .disjoint_code_page,
.type = .note,
.code_page = self.state.input_code_page_lookup.getForLineNum(self.lexer.last_pragma_code_page_token.?.line_number),
.token = self.lexer.last_pragma_code_page_token.?,
.print_source_line = false,
});
self.state.warned_about_disjoint_code_page = true;
}
try self.state.output_code_page_lookup.setForToken(self.state.token, output_code_page); try self.state.output_code_page_lookup.setForToken(self.state.token, output_code_page);
} }
@ -1846,7 +1951,7 @@ pub const Parser = struct {
// Ignore this error and get the next valid token, we'll deal with this // Ignore this error and get the next valid token, we'll deal with this
// properly when getting the token for real // properly when getting the token for real
error.CodePagePragmaInIncludedFile => continue, error.CodePagePragmaInIncludedFile => continue,
else => return self.addErrorDetailsAndFail(self.state.lookahead_lexer.getErrorDetails(err)), else => return self.addErrorDetailsWithCodePageAndFail(self.state.lookahead_lexer.getErrorDetails(err)),
}; };
}; };
} }
@ -1860,7 +1965,7 @@ pub const Parser = struct {
switch (self.state.token.id) { switch (self.state.token.id) {
.literal => {}, .literal => {},
else => { else => {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_token, .err = .expected_token,
.token = self.state.token, .token = self.state.token,
.extra = .{ .expected = .literal }, .extra = .{ .expected = .literal },
@ -1871,7 +1976,7 @@ pub const Parser = struct {
fn check(self: *Self, expected_token_id: Token.Id) !void { fn check(self: *Self, expected_token_id: Token.Id) !void {
if (self.state.token.id != expected_token_id) { if (self.state.token.id != expected_token_id) {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_token, .err = .expected_token,
.token = self.state.token, .token = self.state.token,
.extra = .{ .expected = expected_token_id }, .extra = .{ .expected = expected_token_id },
@ -1879,14 +1984,14 @@ pub const Parser = struct {
} }
} }
fn checkResource(self: *Self) !Resource { fn checkResource(self: *Self) !ResourceType {
switch (self.state.token.id) { switch (self.state.token.id) {
.literal => return Resource.fromString(.{ .literal => return ResourceType.fromString(.{
.slice = self.state.token.slice(self.lexer.buffer), .slice = self.state.token.slice(self.lexer.buffer),
.code_page = self.lexer.current_code_page, .code_page = self.lexer.current_code_page,
}), }),
else => { else => {
return self.addErrorDetailsAndFail(ErrorDetails{ return self.addErrorDetailsAndFail(.{
.err = .expected_token, .err = .expected_token,
.token = self.state.token, .token = self.state.token,
.extra = .{ .expected = .literal }, .extra = .{ .expected = .literal },

1
lib/compiler/resinator/preprocess.zig
View File

@ -96,6 +96,7 @@ pub fn appendAroArgs(arena: Allocator, argv: *std.ArrayList([]const u8), options
"--emulate=msvc", "--emulate=msvc",
"-nostdinc", "-nostdinc",
"-DRC_INVOKED", "-DRC_INVOKED",
"-D_WIN32", // undocumented, but defined by default
}); });
for (options.extra_include_paths.items) |extra_include_path| { for (options.extra_include_paths.items) |extra_include_path| {
try argv.append("-I"); try argv.append("-I");

14
lib/compiler/resinator/rc.zig
View File

@ -5,7 +5,7 @@ const SourceBytes = @import("literals.zig").SourceBytes;
// https://learn.microsoft.com/en-us/windows/win32/menurc/about-resource-files // https://learn.microsoft.com/en-us/windows/win32/menurc/about-resource-files
pub const Resource = enum { pub const ResourceType = enum {
accelerators, accelerators,
bitmap, bitmap,
cursor, cursor,
@ -48,7 +48,7 @@ pub const Resource = enum {
manifest_num, manifest_num,
const map = std.StaticStringMapWithEql( const map = std.StaticStringMapWithEql(
Resource, ResourceType,
std.static_string_map.eqlAsciiIgnoreCase, std.static_string_map.eqlAsciiIgnoreCase,
).initComptime(.{ ).initComptime(.{
.{ "ACCELERATORS", .accelerators }, .{ "ACCELERATORS", .accelerators },
@ -72,7 +72,7 @@ pub const Resource = enum {
.{ "VXD", .vxd }, .{ "VXD", .vxd },
}); });
pub fn fromString(bytes: SourceBytes) Resource { pub fn fromString(bytes: SourceBytes) ResourceType {
const maybe_ordinal = res.NameOrOrdinal.maybeOrdinalFromString(bytes); const maybe_ordinal = res.NameOrOrdinal.maybeOrdinalFromString(bytes);
if (maybe_ordinal) |ordinal| { if (maybe_ordinal) |ordinal| {
if (ordinal.ordinal >= 256) return .user_defined; if (ordinal.ordinal >= 256) return .user_defined;
@ -81,8 +81,8 @@ pub const Resource = enum {
return map.get(bytes.slice) orelse .user_defined; return map.get(bytes.slice) orelse .user_defined;
} }
// TODO: Some comptime validation that RT <-> Resource conversion is synced? // TODO: Some comptime validation that RT <-> ResourceType conversion is synced?
pub fn fromRT(rt: res.RT) Resource { pub fn fromRT(rt: res.RT) ResourceType {
return switch (rt) { return switch (rt) {
.ACCELERATOR => .accelerators, .ACCELERATOR => .accelerators,
.ANICURSOR => .anicursor_num, .ANICURSOR => .anicursor_num,
@ -111,7 +111,7 @@ pub const Resource = enum {
}; };
} }
pub fn canUseRawData(resource: Resource) bool { pub fn canUseRawData(resource: ResourceType) bool {
return switch (resource) { return switch (resource) {
.user_defined, .user_defined,
.html, .html,
@ -125,7 +125,7 @@ pub const Resource = enum {
}; };
} }
pub fn nameForErrorDisplay(resource: Resource) []const u8 { pub fn nameForErrorDisplay(resource: ResourceType) []const u8 {
return switch (resource) { return switch (resource) {
// zig fmt: off // zig fmt: off
.accelerators, .bitmap, .cursor, .dialog, .dialogex, .dlginclude, .dlginit, .font, .accelerators, .bitmap, .cursor, .dialog, .dialogex, .dlginclude, .dlginit, .font,

213
lib/compiler/resinator/res.zig
View File

@ -1,10 +1,10 @@
const std = @import("std"); const std = @import("std");
const rc = @import("rc.zig"); const rc = @import("rc.zig");
const Resource = rc.Resource; const ResourceType = rc.ResourceType;
const CommonResourceAttributes = rc.CommonResourceAttributes; const CommonResourceAttributes = rc.CommonResourceAttributes;
const Allocator = std.mem.Allocator; const Allocator = std.mem.Allocator;
const windows1252 = @import("windows1252.zig"); const windows1252 = @import("windows1252.zig");
const CodePage = @import("code_pages.zig").CodePage; const SupportedCodePage = @import("code_pages.zig").SupportedCodePage;
const literals = @import("literals.zig"); const literals = @import("literals.zig");
const SourceBytes = literals.SourceBytes; const SourceBytes = literals.SourceBytes;
const Codepoint = @import("code_pages.zig").Codepoint; const Codepoint = @import("code_pages.zig").Codepoint;
@ -40,7 +40,7 @@ pub const RT = enum(u8) {
/// Returns null if the resource type is user-defined /// Returns null if the resource type is user-defined
/// Asserts that the resource is not `stringtable` /// Asserts that the resource is not `stringtable`
pub fn fromResource(resource: Resource) ?RT { pub fn fromResource(resource: ResourceType) ?RT {
return switch (resource) { return switch (resource) {
.accelerators => .ACCELERATOR, .accelerators => .ACCELERATOR,
.bitmap => .BITMAP, .bitmap => .BITMAP,
@ -162,6 +162,27 @@ pub const Language = packed struct(u16) {
pub fn asInt(self: Language) u16 { pub fn asInt(self: Language) u16 {
return @bitCast(self); return @bitCast(self);
} }
pub fn format(
language: Language,
comptime fmt: []const u8,
options: std.fmt.FormatOptions,
out_stream: anytype,
) !void {
_ = fmt;
_ = options;
const language_id = language.asInt();
const language_name = language_name: {
if (std.meta.intToEnum(lang.LanguageId, language_id)) |lang_enum_val| {
break :language_name @tagName(lang_enum_val);
} else |_| {}
if (language_id == lang.LOCALE_CUSTOM_UNSPECIFIED) {
break :language_name "LOCALE_CUSTOM_UNSPECIFIED";
}
break :language_name "<UNKNOWN>";
};
try out_stream.print("{s} (0x{X})", .{ language_name, language_id });
}
}; };
/// https://learn.microsoft.com/en-us/windows/win32/api/winuser/ns-winuser-dlgitemtemplate#remarks /// https://learn.microsoft.com/en-us/windows/win32/api/winuser/ns-winuser-dlgitemtemplate#remarks
@ -423,6 +444,50 @@ pub const NameOrOrdinal = union(enum) {
.name => return null, .name => return null,
} }
} }
pub fn format(
self: NameOrOrdinal,
comptime fmt: []const u8,
options: std.fmt.FormatOptions,
out_stream: anytype,
) !void {
_ = fmt;
_ = options;
switch (self) {
.name => |name| {
try out_stream.print("{s}", .{std.unicode.fmtUtf16Le(name)});
},
.ordinal => |ordinal| {
try out_stream.print("{d}", .{ordinal});
},
}
}
fn formatResourceType(
self: NameOrOrdinal,
comptime fmt: []const u8,
options: std.fmt.FormatOptions,
out_stream: anytype,
) !void {
_ = fmt;
_ = options;
switch (self) {
.name => |name| {
try out_stream.print("{s}", .{std.unicode.fmtUtf16Le(name)});
},
.ordinal => |ordinal| {
if (std.enums.tagName(RT, @enumFromInt(ordinal))) |predefined_type_name| {
try out_stream.print("{s}", .{predefined_type_name});
} else {
try out_stream.print("{d}", .{ordinal});
}
},
}
}
pub fn fmtResourceType(type_value: NameOrOrdinal) std.fmt.Formatter(formatResourceType) {
return .{ .data = type_value };
}
}; };
fn expectNameOrOrdinal(expected: NameOrOrdinal, actual: NameOrOrdinal) !void { fn expectNameOrOrdinal(expected: NameOrOrdinal, actual: NameOrOrdinal) !void {
@ -603,12 +668,33 @@ pub const AcceleratorModifiers = struct {
const AcceleratorKeyCodepointTranslator = struct { const AcceleratorKeyCodepointTranslator = struct {
string_type: literals.StringType, string_type: literals.StringType,
output_code_page: SupportedCodePage,
pub fn translate(self: @This(), maybe_parsed: ?literals.IterativeStringParser.ParsedCodepoint) ?u21 { pub fn translate(self: @This(), maybe_parsed: ?literals.IterativeStringParser.ParsedCodepoint) ?u21 {
const parsed = maybe_parsed orelse return null; const parsed = maybe_parsed orelse return null;
if (parsed.codepoint == Codepoint.invalid) return 0xFFFD; if (parsed.codepoint == Codepoint.invalid) return 0xFFFD;
if (parsed.from_escaped_integer and self.string_type == .ascii) { if (parsed.from_escaped_integer) {
return windows1252.toCodepoint(@truncate(parsed.codepoint)); switch (self.string_type) {
.ascii => {
const truncated: u8 = @truncate(parsed.codepoint);
switch (self.output_code_page) {
.utf8 => switch (truncated) {
0...0x7F => return truncated,
else => return 0xFFFD,
},
.windows1252 => return windows1252.toCodepoint(truncated),
}
},
.wide => {
const truncated: u16 = @truncate(parsed.codepoint);
return truncated;
},
}
}
if (parsed.escaped_surrogate_pair) {
// The codepoint of only the low surrogate
const low = @as(u16, @intCast(parsed.codepoint & 0x3FF)) + 0xDC00;
return low;
} }
return parsed.codepoint; return parsed.codepoint;
} }
@ -623,14 +709,17 @@ pub fn parseAcceleratorKeyString(bytes: SourceBytes, is_virt: bool, options: lit
} }
var parser = literals.IterativeStringParser.init(bytes, options); var parser = literals.IterativeStringParser.init(bytes, options);
var translator = AcceleratorKeyCodepointTranslator{ .string_type = parser.declared_string_type }; var translator = AcceleratorKeyCodepointTranslator{
.string_type = parser.declared_string_type,
.output_code_page = options.output_code_page,
};
const first_codepoint = translator.translate(try parser.next()) orelse return error.EmptyAccelerator; const first_codepoint = translator.translate(try parser.next()) orelse return error.EmptyAccelerator;
// 0 is treated as a terminator, so this is equivalent to an empty string // 0 is treated as a terminator, so this is equivalent to an empty string
if (first_codepoint == 0) return error.EmptyAccelerator; if (first_codepoint == 0) return error.EmptyAccelerator;
if (first_codepoint == '^') { if (first_codepoint == '^') {
// Note: Emitting this warning unconditonally whenever ^ is the first character // Note: Emitting this warning unconditionally whenever ^ is the first character
// matches the Win32 RC behavior, but it's questionable whether or not // matches the Win32 RC behavior, but it's questionable whether or not
// the warning should be emitted for ^^ since that results in the ASCII // the warning should be emitted for ^^ since that results in the ASCII
// character ^ being written to the .res. // character ^ being written to the .res.
@ -638,11 +727,18 @@ pub fn parseAcceleratorKeyString(bytes: SourceBytes, is_virt: bool, options: lit
try options.diagnostics.?.diagnostics.append(.{ try options.diagnostics.?.diagnostics.append(.{
.err = .ascii_character_not_equivalent_to_virtual_key_code, .err = .ascii_character_not_equivalent_to_virtual_key_code,
.type = .warning, .type = .warning,
.code_page = bytes.code_page,
.token = options.diagnostics.?.token, .token = options.diagnostics.?.token,
}); });
} }
const c = translator.translate(try parser.next()) orelse return error.InvalidControlCharacter; const c = translator.translate(try parser.next()) orelse return error.InvalidControlCharacter;
const third_codepoint = translator.translate(try parser.next());
// 0 is treated as a terminator, so a 0 in the third position is fine but
// anything else is too many codepoints for an accelerator
if (third_codepoint != null and third_codepoint.? != 0) return error.InvalidControlCharacter;
switch (c) { switch (c) {
'^' => return '^', // special case '^' => return '^', // special case
'a'...'z', 'A'...'Z' => return std.ascii.toUpper(@intCast(c)) - 0x40, 'a'...'z', 'A'...'Z' => return std.ascii.toUpper(@intCast(c)) - 0x40,
@ -699,44 +795,44 @@ test "accelerator keys" {
try std.testing.expectEqual(@as(u16, 1), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 1), try parseAcceleratorKeyString(
.{ .slice = "\"^a\"", .code_page = .windows1252 }, .{ .slice = "\"^a\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqual(@as(u16, 1), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 1), try parseAcceleratorKeyString(
.{ .slice = "\"^A\"", .code_page = .windows1252 }, .{ .slice = "\"^A\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqual(@as(u16, 26), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 26), try parseAcceleratorKeyString(
.{ .slice = "\"^Z\"", .code_page = .windows1252 }, .{ .slice = "\"^Z\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqual(@as(u16, '^'), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, '^'), try parseAcceleratorKeyString(
.{ .slice = "\"^^\"", .code_page = .windows1252 }, .{ .slice = "\"^^\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqual(@as(u16, 'a'), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 'a'), try parseAcceleratorKeyString(
.{ .slice = "\"a\"", .code_page = .windows1252 }, .{ .slice = "\"a\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqual(@as(u16, 0x6162), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0x6162), try parseAcceleratorKeyString(
.{ .slice = "\"ab\"", .code_page = .windows1252 }, .{ .slice = "\"ab\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqual(@as(u16, 'C'), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 'C'), try parseAcceleratorKeyString(
.{ .slice = "\"c\"", .code_page = .windows1252 }, .{ .slice = "\"c\"", .code_page = .windows1252 },
true, true,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqual(@as(u16, 0x6363), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0x6363), try parseAcceleratorKeyString(
.{ .slice = "\"cc\"", .code_page = .windows1252 }, .{ .slice = "\"cc\"", .code_page = .windows1252 },
true, true,
.{}, .{ .output_code_page = .windows1252 },
)); ));
// \x00 or any escape that evaluates to zero acts as a terminator, everything past it // \x00 or any escape that evaluates to zero acts as a terminator, everything past it
@ -744,93 +840,93 @@ test "accelerator keys" {
try std.testing.expectEqual(@as(u16, 'a'), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 'a'), try parseAcceleratorKeyString(
.{ .slice = "\"a\\0bcdef\"", .code_page = .windows1252 }, .{ .slice = "\"a\\0bcdef\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
// \x80 is € in Windows-1252, which is Unicode codepoint 20AC // \x80 is € in Windows-1252, which is Unicode codepoint 20AC
try std.testing.expectEqual(@as(u16, 0x20AC), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0x20AC), try parseAcceleratorKeyString(
.{ .slice = "\"\x80\"", .code_page = .windows1252 }, .{ .slice = "\"\x80\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
// This depends on the code page, though, with codepage 65001, \x80 // This depends on the code page, though, with codepage 65001, \x80
// on its own is invalid UTF-8 so it gets converted to the replacement character // on its own is invalid UTF-8 so it gets converted to the replacement character
try std.testing.expectEqual(@as(u16, 0xFFFD), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0xFFFD), try parseAcceleratorKeyString(
.{ .slice = "\"\x80\"", .code_page = .utf8 }, .{ .slice = "\"\x80\"", .code_page = .utf8 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqual(@as(u16, 0xCCAC), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0xCCAC), try parseAcceleratorKeyString(
.{ .slice = "\"\x80\x80\"", .code_page = .windows1252 }, .{ .slice = "\"\x80\x80\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
// This also behaves the same with escaped characters // This also behaves the same with escaped characters
try std.testing.expectEqual(@as(u16, 0x20AC), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0x20AC), try parseAcceleratorKeyString(
.{ .slice = "\"\\x80\"", .code_page = .windows1252 }, .{ .slice = "\"\\x80\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Even with utf8 code page // Even with utf8 code page
try std.testing.expectEqual(@as(u16, 0x20AC), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0x20AC), try parseAcceleratorKeyString(
.{ .slice = "\"\\x80\"", .code_page = .utf8 }, .{ .slice = "\"\\x80\"", .code_page = .utf8 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqual(@as(u16, 0xCCAC), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0xCCAC), try parseAcceleratorKeyString(
.{ .slice = "\"\\x80\\x80\"", .code_page = .windows1252 }, .{ .slice = "\"\\x80\\x80\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Wide string with the actual characters behaves like the ASCII string version // Wide string with the actual characters behaves like the ASCII string version
try std.testing.expectEqual(@as(u16, 0xCCAC), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0xCCAC), try parseAcceleratorKeyString(
.{ .slice = "L\"\x80\x80\"", .code_page = .windows1252 }, .{ .slice = "L\"\x80\x80\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
// But wide string with escapes behaves differently // But wide string with escapes behaves differently
try std.testing.expectEqual(@as(u16, 0x8080), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0x8080), try parseAcceleratorKeyString(
.{ .slice = "L\"\\x80\\x80\"", .code_page = .windows1252 }, .{ .slice = "L\"\\x80\\x80\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
// and invalid escapes within wide strings get skipped // and invalid escapes within wide strings get skipped
try std.testing.expectEqual(@as(u16, 'z'), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 'z'), try parseAcceleratorKeyString(
.{ .slice = "L\"\\Hz\"", .code_page = .windows1252 }, .{ .slice = "L\"\\Hz\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
// any non-A-Z codepoints are illegal // any non-A-Z codepoints are illegal
try std.testing.expectError(error.ControlCharacterOutOfRange, parseAcceleratorKeyString( try std.testing.expectError(error.ControlCharacterOutOfRange, parseAcceleratorKeyString(
.{ .slice = "\"^\x83\"", .code_page = .windows1252 }, .{ .slice = "\"^\x83\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectError(error.ControlCharacterOutOfRange, parseAcceleratorKeyString( try std.testing.expectError(error.ControlCharacterOutOfRange, parseAcceleratorKeyString(
.{ .slice = "\"^1\"", .code_page = .windows1252 }, .{ .slice = "\"^1\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectError(error.InvalidControlCharacter, parseAcceleratorKeyString( try std.testing.expectError(error.InvalidControlCharacter, parseAcceleratorKeyString(
.{ .slice = "\"^\"", .code_page = .windows1252 }, .{ .slice = "\"^\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectError(error.EmptyAccelerator, parseAcceleratorKeyString( try std.testing.expectError(error.EmptyAccelerator, parseAcceleratorKeyString(
.{ .slice = "\"\"", .code_page = .windows1252 }, .{ .slice = "\"\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectError(error.AcceleratorTooLong, parseAcceleratorKeyString( try std.testing.expectError(error.AcceleratorTooLong, parseAcceleratorKeyString(
.{ .slice = "\"hello\"", .code_page = .windows1252 }, .{ .slice = "\"hello\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectError(error.ControlCharacterOutOfRange, parseAcceleratorKeyString( try std.testing.expectError(error.ControlCharacterOutOfRange, parseAcceleratorKeyString(
.{ .slice = "\"^\x80\"", .code_page = .windows1252 }, .{ .slice = "\"^\x80\"", .code_page = .windows1252 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Invalid UTF-8 gets converted to 0xFFFD, multiple invalids get shifted and added together // Invalid UTF-8 gets converted to 0xFFFD, multiple invalids get shifted and added together
@ -838,40 +934,81 @@ test "accelerator keys" {
try std.testing.expectEqual(@as(u16, 0xFCFD), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0xFCFD), try parseAcceleratorKeyString(
.{ .slice = "\"\x80\x80\"", .code_page = .utf8 }, .{ .slice = "\"\x80\x80\"", .code_page = .utf8 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqual(@as(u16, 0xFCFD), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0xFCFD), try parseAcceleratorKeyString(
.{ .slice = "L\"\x80\x80\"", .code_page = .utf8 }, .{ .slice = "L\"\x80\x80\"", .code_page = .utf8 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
// Codepoints >= 0x10000 // Codepoints >= 0x10000
try std.testing.expectEqual(@as(u16, 0xDD00), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0xDD00), try parseAcceleratorKeyString(
.{ .slice = "\"\xF0\x90\x84\x80\"", .code_page = .utf8 }, .{ .slice = "\"\xF0\x90\x84\x80\"", .code_page = .utf8 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqual(@as(u16, 0xDD00), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0xDD00), try parseAcceleratorKeyString(
.{ .slice = "L\"\xF0\x90\x84\x80\"", .code_page = .utf8 }, .{ .slice = "L\"\xF0\x90\x84\x80\"", .code_page = .utf8 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectEqual(@as(u16, 0x9C01), try parseAcceleratorKeyString( try std.testing.expectEqual(@as(u16, 0x9C01), try parseAcceleratorKeyString(
.{ .slice = "\"\xF4\x80\x80\x81\"", .code_page = .utf8 }, .{ .slice = "\"\xF4\x80\x80\x81\"", .code_page = .utf8 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
// anything before or after a codepoint >= 0x10000 causes an error // anything before or after a codepoint >= 0x10000 causes an error
try std.testing.expectError(error.AcceleratorTooLong, parseAcceleratorKeyString( try std.testing.expectError(error.AcceleratorTooLong, parseAcceleratorKeyString(
.{ .slice = "\"a\xF0\x90\x80\x80\"", .code_page = .utf8 }, .{ .slice = "\"a\xF0\x90\x80\x80\"", .code_page = .utf8 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
)); ));
try std.testing.expectError(error.AcceleratorTooLong, parseAcceleratorKeyString( try std.testing.expectError(error.AcceleratorTooLong, parseAcceleratorKeyString(
.{ .slice = "\"\xF0\x90\x80\x80a\"", .code_page = .utf8 }, .{ .slice = "\"\xF0\x90\x80\x80a\"", .code_page = .utf8 },
false, false,
.{}, .{ .output_code_page = .windows1252 },
));
// Misc special cases
try std.testing.expectEqual(@as(u16, 0xFFFD), try parseAcceleratorKeyString(
.{ .slice = "\"\\777\"", .code_page = .utf8 },
false,
.{ .output_code_page = .utf8 },
));
try std.testing.expectEqual(@as(u16, 0xFFFF), try parseAcceleratorKeyString(
.{ .slice = "L\"\\7777777\"", .code_page = .utf8 },
false,
.{ .output_code_page = .utf8 },
));
try std.testing.expectEqual(@as(u16, 0x01), try parseAcceleratorKeyString(
.{ .slice = "L\"\\200001\"", .code_page = .utf8 },
false,
.{ .output_code_page = .utf8 },
));
// Escape of a codepoint >= 0x10000 omits the high surrogate pair
try std.testing.expectEqual(@as(u16, 0xDF48), try parseAcceleratorKeyString(
.{ .slice = "L\"\\𐍈\"", .code_page = .utf8 },
false,
.{ .output_code_page = .utf8 },
));
// Invalid escape code is skipped, allows for 2 codepoints afterwards
try std.testing.expectEqual(@as(u16, 0x7878), try parseAcceleratorKeyString(
.{ .slice = "L\"\\kxx\"", .code_page = .utf8 },
false,
.{ .output_code_page = .utf8 },
));
// Escape of a codepoint >= 0x10000 allows for a codepoint afterwards
try std.testing.expectEqual(@as(u16, 0x4878), try parseAcceleratorKeyString(
.{ .slice = "L\"\\𐍈x\"", .code_page = .utf8 },
false,
.{ .output_code_page = .utf8 },
));
// Input code page of 1252, output code page of utf-8
try std.testing.expectEqual(@as(u16, 0xFFFD), try parseAcceleratorKeyString(
.{ .slice = "\"\\270\"", .code_page = .windows1252 },
false,
.{ .output_code_page = .utf8 },
)); ));
} }

479
lib/compiler/resinator/source_mapping.zig
View File

@ -38,7 +38,7 @@ pub const ParseAndRemoveLineCommandsOptions = struct {
/// ///
/// If `options.initial_filename` is provided, that filename is guaranteed to be /// If `options.initial_filename` is provided, that filename is guaranteed to be
/// within the `mappings.files` table and `root_filename_offset` will be set appropriately. /// within the `mappings.files` table and `root_filename_offset` will be set appropriately.
pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf: []u8, options: ParseAndRemoveLineCommandsOptions) !ParseLineCommandsResult { pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf: []u8, options: ParseAndRemoveLineCommandsOptions) error{ OutOfMemory, InvalidLineCommand, LineNumberOverflow }!ParseLineCommandsResult {
var parse_result = ParseLineCommandsResult{ var parse_result = ParseLineCommandsResult{
.result = undefined, .result = undefined,
.mappings = .{}, .mappings = .{},
@ -53,12 +53,41 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
parse_result.mappings.root_filename_offset = try parse_result.mappings.files.put(allocator, initial_filename); parse_result.mappings.root_filename_offset = try parse_result.mappings.files.put(allocator, initial_filename);
} }
// This implementation attempts to be comment and string aware in order
// to avoid errant #line <num> "<filename>" within multiline comments
// leading to problems in the source mapping after comments are removed,
// but it is not a perfect implementation (intentionally).
//
// The current implementation does not handle cases like
// /* foo */ #line ...
// #line ... // foo
// #line ... /* foo ...
// etc
//
// (the first example will not be recognized as a #line command, the second
// and third will error with InvalidLineCommand)
//
// This is fine, though, since #line commands are generated by the
// preprocessor so in normal circumstances they will be well-formed and
// consistent. The only realistic way the imperfect implementation could
// affect a 'real' use-case would be someone taking the output of a
// preprocessor, editing it manually to add comments before/after #line
// commands, and then running it through resinator with /:no-preprocess.
std.debug.assert(buf.len >= source.len); std.debug.assert(buf.len >= source.len);
var result = UncheckedSliceWriter{ .slice = buf }; var result = UncheckedSliceWriter{ .slice = buf };
const State = enum { const State = enum {
line_start, line_start,
preprocessor, preprocessor,
non_preprocessor, non_preprocessor,
forward_slash,
line_comment,
multiline_comment,
multiline_comment_end,
single_quoted,
single_quoted_escape,
double_quoted,
double_quoted_escape,
}; };
var state: State = .line_start; var state: State = .line_start;
var index: usize = 0; var index: usize = 0;
@ -66,8 +95,8 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
var preprocessor_start: usize = 0; var preprocessor_start: usize = 0;
var line_number: usize = 1; var line_number: usize = 1;
while (index < source.len) : (index += 1) { while (index < source.len) : (index += 1) {
const c = source[index]; var c = source[index];
switch (state) { state: switch (state) {
.line_start => switch (c) { .line_start => switch (c) {
'#' => { '#' => {
preprocessor_start = index; preprocessor_start = index;
@ -93,6 +122,27 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
pending_start = index; pending_start = index;
} }
}, },
'/' => {
if (!current_mapping.ignore_contents) {
result.writeSlice(source[pending_start orelse index .. index + 1]);
pending_start = null;
}
state = .forward_slash;
},
'\'' => {
if (!current_mapping.ignore_contents) {
result.writeSlice(source[pending_start orelse index .. index + 1]);
pending_start = null;
}
state = .single_quoted;
},
'"' => {
if (!current_mapping.ignore_contents) {
result.writeSlice(source[pending_start orelse index .. index + 1]);
pending_start = null;
}
state = .double_quoted;
},
else => { else => {
state = .non_preprocessor; state = .non_preprocessor;
if (pending_start != null) { if (pending_start != null) {
@ -107,29 +157,40 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
} }
}, },
}, },
.preprocessor => switch (c) { .forward_slash => switch (c) {
'\r', '\n' => { '\r', '\n' => {
// Now that we have the full line we can decide what to do with it
const preprocessor_str = source[preprocessor_start..index];
const is_crlf = formsLineEndingPair(source, c, index + 1); const is_crlf = formsLineEndingPair(source, c, index + 1);
if (std.mem.startsWith(u8, preprocessor_str, "#line")) {
try handleLineCommand(allocator, preprocessor_str, &current_mapping);
} else {
if (!current_mapping.ignore_contents) { if (!current_mapping.ignore_contents) {
try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping); try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
const line_ending_len: usize = if (is_crlf) 2 else 1; result.write(c);
result.writeSlice(source[pending_start.? .. index + line_ending_len]); if (is_crlf) result.write(source[index + 1]);
line_number += 1; line_number += 1;
} }
}
if (is_crlf) index += 1; if (is_crlf) index += 1;
state = .line_start; state = .line_start;
pending_start = null; pending_start = null;
}, },
else => {}, '/' => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .line_comment;
}, },
.non_preprocessor => switch (c) { '*' => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .multiline_comment;
},
else => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .non_preprocessor;
},
},
.line_comment => switch (c) {
'\r', '\n' => { '\r', '\n' => {
const is_crlf = formsLineEndingPair(source, c, index + 1); const is_crlf = formsLineEndingPair(source, c, index + 1);
if (!current_mapping.ignore_contents) { if (!current_mapping.ignore_contents) {
@ -149,11 +210,248 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
} }
}, },
}, },
.multiline_comment => switch (c) {
'\r', '\n' => {
const is_crlf = formsLineEndingPair(source, c, index + 1);
if (!current_mapping.ignore_contents) {
try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
result.write(c);
if (is_crlf) result.write(source[index + 1]);
line_number += 1;
}
if (is_crlf) index += 1;
pending_start = null;
},
'*' => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .multiline_comment_end;
},
else => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
},
},
.multiline_comment_end => switch (c) {
'\r', '\n' => {
const is_crlf = formsLineEndingPair(source, c, index + 1);
if (!current_mapping.ignore_contents) {
try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
result.write(c);
if (is_crlf) result.write(source[index + 1]);
line_number += 1;
}
if (is_crlf) index += 1;
state = .multiline_comment;
pending_start = null;
},
'/' => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .non_preprocessor;
},
'*' => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
// stay in multiline_comment_end state
},
else => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .multiline_comment;
},
},
.single_quoted => switch (c) {
'\r', '\n' => {
const is_crlf = formsLineEndingPair(source, c, index + 1);
if (!current_mapping.ignore_contents) {
try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
result.write(c);
if (is_crlf) result.write(source[index + 1]);
line_number += 1;
}
if (is_crlf) index += 1;
state = .line_start;
pending_start = null;
},
'\\' => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .single_quoted_escape;
},
'\'' => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .non_preprocessor;
},
else => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
},
},
.single_quoted_escape => switch (c) {
'\r', '\n' => {
const is_crlf = formsLineEndingPair(source, c, index + 1);
if (!current_mapping.ignore_contents) {
try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
result.write(c);
if (is_crlf) result.write(source[index + 1]);
line_number += 1;
}
if (is_crlf) index += 1;
state = .line_start;
pending_start = null;
},
else => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .single_quoted;
},
},
.double_quoted => switch (c) {
'\r', '\n' => {
const is_crlf = formsLineEndingPair(source, c, index + 1);
if (!current_mapping.ignore_contents) {
try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
result.write(c);
if (is_crlf) result.write(source[index + 1]);
line_number += 1;
}
if (is_crlf) index += 1;
state = .line_start;
pending_start = null;
},
'\\' => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .double_quoted_escape;
},
'"' => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .non_preprocessor;
},
else => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
},
},
.double_quoted_escape => switch (c) {
'\r', '\n' => {
const is_crlf = formsLineEndingPair(source, c, index + 1);
if (!current_mapping.ignore_contents) {
try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
result.write(c);
if (is_crlf) result.write(source[index + 1]);
line_number += 1;
}
if (is_crlf) index += 1;
state = .line_start;
pending_start = null;
},
else => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .double_quoted;
},
},
.preprocessor => switch (c) {
'\r', '\n' => {
// Now that we have the full line we can decide what to do with it
const preprocessor_str = source[preprocessor_start..index];
if (std.mem.startsWith(u8, preprocessor_str, "#line")) {
try handleLineCommand(allocator, preprocessor_str, &current_mapping);
const is_crlf = formsLineEndingPair(source, c, index + 1);
if (is_crlf) index += 1;
state = .line_start;
pending_start = null;
} else {
// Backtrack and reparse the line in the non_preprocessor state,
// since it's possible that this line contains a multiline comment
// start, etc.
state = .non_preprocessor;
index = pending_start.?;
pending_start = null;
// TODO: This is a hacky way to implement this, c needs to be
// updated since we're using continue :state here
c = source[index];
// continue to avoid the index += 1 of the while loop
continue :state .non_preprocessor;
}
},
else => {},
},
.non_preprocessor => switch (c) {
'\r', '\n' => {
const is_crlf = formsLineEndingPair(source, c, index + 1);
if (!current_mapping.ignore_contents) {
try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
result.write(c);
if (is_crlf) result.write(source[index + 1]);
line_number += 1;
}
if (is_crlf) index += 1;
state = .line_start;
pending_start = null;
},
'/' => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .forward_slash;
},
'\'' => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .single_quoted;
},
'"' => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
state = .double_quoted;
},
else => {
if (!current_mapping.ignore_contents) {
result.write(c);
}
},
},
} }
} else { } else {
switch (state) { switch (state) {
.line_start => {}, .line_start => {},
.non_preprocessor => { .forward_slash,
.line_comment,
.multiline_comment,
.multiline_comment_end,
.single_quoted,
.single_quoted_escape,
.double_quoted,
.double_quoted_escape,
.non_preprocessor,
=> {
try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping); try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
}, },
.preprocessor => { .preprocessor => {
@ -207,34 +505,40 @@ pub fn handleLineEnd(allocator: Allocator, post_processed_line_number: usize, ma
try mapping.set(post_processed_line_number, current_mapping.line_num, filename_offset); try mapping.set(post_processed_line_number, current_mapping.line_num, filename_offset);
current_mapping.line_num += 1; current_mapping.line_num = std.math.add(usize, current_mapping.line_num, 1) catch return error.LineNumberOverflow;
current_mapping.pending = false; current_mapping.pending = false;
} }
// TODO: Might want to provide diagnostics on invalid line commands instead of just returning // TODO: Might want to provide diagnostics on invalid line commands instead of just returning
pub fn handleLineCommand(allocator: Allocator, line_command: []const u8, current_mapping: *CurrentMapping) error{OutOfMemory}!void { pub fn handleLineCommand(allocator: Allocator, line_command: []const u8, current_mapping: *CurrentMapping) error{ OutOfMemory, InvalidLineCommand }!void {
// TODO: Are there other whitespace characters that should be included? // TODO: Are there other whitespace characters that should be included?
var tokenizer = std.mem.tokenizeAny(u8, line_command, " \t"); var tokenizer = std.mem.tokenizeAny(u8, line_command, " \t");
const line_directive = tokenizer.next() orelse return; // #line const line_directive = tokenizer.next() orelse return error.InvalidLineCommand; // #line
if (!std.mem.eql(u8, line_directive, "#line")) return; if (!std.mem.eql(u8, line_directive, "#line")) return error.InvalidLineCommand;
const linenum_str = tokenizer.next() orelse return; const linenum_str = tokenizer.next() orelse return error.InvalidLineCommand;
const linenum = std.fmt.parseUnsigned(usize, linenum_str, 10) catch return; const linenum = std.fmt.parseUnsigned(usize, linenum_str, 10) catch return error.InvalidLineCommand;
if (linenum == 0) return error.InvalidLineCommand;
var filename_literal = tokenizer.rest(); var filename_literal = tokenizer.rest();
while (filename_literal.len > 0 and std.ascii.isWhitespace(filename_literal[filename_literal.len - 1])) { while (filename_literal.len > 0 and std.ascii.isWhitespace(filename_literal[filename_literal.len - 1])) {
filename_literal.len -= 1; filename_literal.len -= 1;
} }
if (filename_literal.len < 2) return; if (filename_literal.len < 2) return error.InvalidLineCommand;
const is_quoted = filename_literal[0] == '"' and filename_literal[filename_literal.len - 1] == '"'; const is_quoted = filename_literal[0] == '"' and filename_literal[filename_literal.len - 1] == '"';
if (!is_quoted) return; if (!is_quoted) return error.InvalidLineCommand;
const filename = parseFilename(allocator, filename_literal[1 .. filename_literal.len - 1]) catch |err| switch (err) { const unquoted_filename = filename_literal[1 .. filename_literal.len - 1];
// Ignore <builtin> and <command line>
if (std.mem.eql(u8, unquoted_filename, "<builtin>") or std.mem.eql(u8, unquoted_filename, "<command line>")) return;
const filename = parseFilename(allocator, unquoted_filename) catch |err| switch (err) {
error.OutOfMemory => |e| return e, error.OutOfMemory => |e| return e,
else => return, else => return error.InvalidLineCommand,
}; };
defer allocator.free(filename); defer allocator.free(filename);
// \x00 bytes in the filename is incompatible with how StringTable works // \x00 bytes in the filename is incompatible with how StringTable works
if (std.mem.indexOfScalar(u8, filename, '\x00') != null) return; if (std.mem.indexOfScalar(u8, filename, '\x00') != null) return error.InvalidLineCommand;
current_mapping.line_num = linenum; current_mapping.line_num = linenum;
current_mapping.filename.clearRetainingCapacity(); current_mapping.filename.clearRetainingCapacity();
@ -494,8 +798,12 @@ pub const SourceMappings = struct {
if (node.key.filename_offset != filename_offset) { if (node.key.filename_offset != filename_offset) {
break :need_new_node true; break :need_new_node true;
} }
const exist_delta = @as(i64, @intCast(node.key.corresponding_start_line)) - @as(i64, @intCast(node.key.start_line)); // TODO: These use i65 to avoid truncation when any of the line number values
const cur_delta = @as(i64, @intCast(corresponding_line_num)) - @as(i64, @intCast(line_num)); // use all 64 bits of the usize. In reality, line numbers can't really
// get that large so limiting the line number and using a smaller iX
// type here might be a better solution.
const exist_delta = @as(i65, @intCast(node.key.corresponding_start_line)) - @as(i65, @intCast(node.key.start_line));
const cur_delta = @as(i65, @intCast(corresponding_line_num)) - @as(i65, @intCast(line_num));
if (exist_delta != cur_delta) { if (exist_delta != cur_delta) {
break :need_new_node true; break :need_new_node true;
} }
@ -578,15 +886,8 @@ pub const SourceMappings = struct {
inorder_node.key.start_line -= span_diff; inorder_node.key.start_line -= span_diff;
// This can only really happen if there are #line commands within // This can only really happen if there are #line commands within
// a multiline comment, which in theory should be skipped over. // a multiline comment, which should be skipped over.
// However, currently, parseAndRemoveLineCommands is not aware of std.debug.assert(prev.key.start_line <= inorder_node.key.start_line);
// comments at all.
//
// TODO: Make parseAndRemoveLineCommands aware of comments/strings
// and turn this into an assertion
if (prev.key.start_line > inorder_node.key.start_line) {
return error.InvalidSourceMappingCollapse;
}
prev = inorder_node; prev = inorder_node;
} }
self.end_line -= span_diff; self.end_line -= span_diff;
@ -594,7 +895,7 @@ pub const SourceMappings = struct {
/// Returns true if the line is from the main/root file (i.e. not a file that has been /// Returns true if the line is from the main/root file (i.e. not a file that has been
/// `#include`d). /// `#include`d).
pub fn isRootFile(self: *SourceMappings, line_num: usize) bool { pub fn isRootFile(self: *const SourceMappings, line_num: usize) bool {
const source = self.get(line_num) orelse return false; const source = self.get(line_num) orelse return false;
return source.filename_offset == self.root_filename_offset; return source.filename_offset == self.root_filename_offset;
} }
@ -803,9 +1104,6 @@ test "in place" {
} }
test "line command within a multiline comment" { test "line command within a multiline comment" {
// TODO: Enable once parseAndRemoveLineCommands is comment-aware
if (true) return error.SkipZigTest;
try testParseAndRemoveLineCommands( try testParseAndRemoveLineCommands(
\\/* \\/*
\\#line 1 "irrelevant.rc" \\#line 1 "irrelevant.rc"
@ -825,4 +1123,103 @@ test "line command within a multiline comment" {
\\ \\
\\*/ \\*/
, .{ .initial_filename = "blah.rc" }); , .{ .initial_filename = "blah.rc" });
// * but without / directly after
try testParseAndRemoveLineCommands(
\\/** /
\\#line 1 "irrelevant.rc"
\\*/
, &[_]ExpectedSourceSpan{
.{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
.{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
.{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
},
\\/** /
\\#line 1 "irrelevant.rc"
\\*/
, .{ .initial_filename = "blah.rc" });
// /** and **/
try testParseAndRemoveLineCommands(
\\/**
\\#line 1 "irrelevant.rc"
\\**/
\\foo
, &[_]ExpectedSourceSpan{
.{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
.{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
.{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
.{ .start_line = 20, .end_line = 20, .filename = "blah.rc" },
},
\\/**
\\#line 1 "irrelevant.rc"
\\**/
\\#line 20 "blah.rc"
\\foo
, .{ .initial_filename = "blah.rc" });
}
test "whitespace preservation" {
try testParseAndRemoveLineCommands(
\\ /
\\/
, &[_]ExpectedSourceSpan{
.{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
.{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
},
\\ /
\\/
, .{ .initial_filename = "blah.rc" });
}
test "preprocessor line with a multiline comment after" {
try testParseAndRemoveLineCommands(
\\#pragma test /*
\\
\\*/
, &[_]ExpectedSourceSpan{
.{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
.{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
.{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
},
\\#pragma test /*
\\
\\*/
, .{ .initial_filename = "blah.rc" });
}
test "comment after line command" {
var mut_source = "#line 1 \"blah.rc\" /*".*;
try std.testing.expectError(error.InvalidLineCommand, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
}
test "line command with 0 as line number" {
var mut_source = "#line 0 \"blah.rc\"".*;
try std.testing.expectError(error.InvalidLineCommand, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
}
test "line number limits" {
// TODO: Avoid usize for line numbers
if (@sizeOf(usize) != 8) return error.SkipZigTest;
// greater than i64 max
try testParseAndRemoveLineCommands(
\\
, &[_]ExpectedSourceSpan{
.{ .start_line = 11111111111111111111, .end_line = 11111111111111111111, .filename = "blah.rc" },
},
\\#line 11111111111111111111 "blah.rc"
, .{ .initial_filename = "blah.rc" });
// equal to u64 max, overflows on line number increment
{
var mut_source = "#line 18446744073709551615 \"blah.rc\"".*;
try std.testing.expectError(error.LineNumberOverflow, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
}
// greater than u64 max
{
var mut_source = "#line 18446744073709551616 \"blah.rc\"".*;
try std.testing.expectError(error.InvalidLineCommand, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
}
} }