Merge pull request #5422 from pixelherodev/error_tests

[Stage2/Testing] ZIR tests for expected errors
This commit is contained in:
Andrew Kelley 2020-06-16 03:50:56 -04:00 committed by GitHub
commit f595545c10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 306 additions and 176 deletions

View File

@ -6,8 +6,7 @@ const zir = @import("zir.zig");
const Package = @import("Package.zig");
test "self-hosted" {
var ctx: TestContext = undefined;
try ctx.init();
var ctx = TestContext.init();
defer ctx.deinit();
try @import("stage2_tests").addCases(&ctx);
@ -15,46 +14,152 @@ test "self-hosted" {
try ctx.run();
}
pub const TestContext = struct {
zir_cmp_output_cases: std.ArrayList(ZIRCompareOutputCase),
zir_transform_cases: std.ArrayList(ZIRTransformCase),
const ErrorMsg = struct {
msg: []const u8,
line: u32,
column: u32,
};
pub const TestContext = struct {
// TODO: remove these. They are deprecated.
zir_cmp_output_cases: std.ArrayList(ZIRCompareOutputCase),
/// TODO: find a way to treat cases as individual tests (shouldn't show "1 test passed" if there are 200 cases)
zir_cases: std.ArrayList(ZIRCase),
// TODO: remove
pub const ZIRCompareOutputCase = struct {
name: []const u8,
src_list: []const []const u8,
expected_stdout_list: []const []const u8,
};
pub const ZIRTransformCase = struct {
pub const ZIRUpdateType = enum {
/// A transformation update transforms the input ZIR and tests against
/// the expected output
Transformation,
/// An error update attempts to compile bad code, and ensures that it
/// fails to compile, and for the expected reasons
Error,
/// An execution update compiles and runs the input ZIR, feeding in
/// provided input and ensuring that the outputs match what is expected
Execution,
/// A compilation update checks that the ZIR compiles without any issues
Compiles,
};
pub const ZIRUpdate = struct {
/// The input to the current update. We simulate an incremental update
/// with the file's contents changed to this value each update.
///
/// This value can change entirely between updates, which would be akin
/// to deleting the source file and creating a new one from scratch; or
/// you can keep it mostly consistent, with small changes, testing the
/// effects of the incremental compilation.
src: [:0]const u8,
case: union(ZIRUpdateType) {
/// The expected output ZIR
Transformation: [:0]const u8,
/// A slice containing the expected errors *in sequential order*.
Error: []const ErrorMsg,
/// Input to feed to the program, and expected outputs.
///
/// If stdout, stderr, and exit_code are all null, addZIRCase will
/// discard the test. To test for successful compilation, use a
/// dedicated Compile update instead.
Execution: struct {
stdin: ?[]const u8,
stdout: ?[]const u8,
stderr: ?[]const u8,
exit_code: ?u8,
},
/// A Compiles test checks only that compilation of the given ZIR
/// succeeds. To test outputs, use an Execution test. It is good to
/// use a Compiles test before an Execution, as the overhead should
/// be low (due to incremental compilation) and TODO: provide a way
/// to check changed / new / etc decls in testing mode
/// (usingnamespace a debug info struct with a comptime flag?)
Compiles: void,
},
};
/// A ZIRCase consists of a set of *updates*. A update can transform ZIR,
/// compile it, ensure that compilation fails, and more. The same Module is
/// used for each update, so each update's source is treated as a single file
/// being updated by the test harness and incrementally compiled.
pub const ZIRCase = struct {
name: []const u8,
cross_target: std.zig.CrossTarget,
updates: std.ArrayList(Update),
/// The platform the ZIR targets. For non-native platforms, an emulator
/// such as QEMU is required for tests to complete.
target: std.zig.CrossTarget,
updates: std.ArrayList(ZIRUpdate),
pub const Update = struct {
expected: Expected,
src: [:0]const u8,
};
pub const Expected = union(enum) {
zir: []const u8,
errors: []const []const u8,
};
pub fn addZIR(case: *ZIRTransformCase, src: [:0]const u8, zir_text: []const u8) void {
case.updates.append(.{
/// Adds a subcase in which the module is updated with new ZIR, and the
/// resulting ZIR is validated.
pub fn addTransform(self: *ZIRCase, src: [:0]const u8, result: [:0]const u8) void {
self.updates.append(.{
.src = src,
.expected = .{ .zir = zir_text },
.case = .{ .Transformation = result },
}) catch unreachable;
}
pub fn addError(case: *ZIRTransformCase, src: [:0]const u8, errors: []const []const u8) void {
case.updates.append(.{
.src = src,
.expected = .{ .errors = errors },
}) catch unreachable;
/// Adds a subcase in which the module is updated with invalid ZIR, and
/// ensures that compilation fails for the expected reasons.
///
/// Errors must be specified in sequential order.
pub fn addError(self: *ZIRCase, src: [:0]const u8, errors: []const []const u8) void {
var array = self.updates.allocator.alloc(ErrorMsg, errors.len) catch unreachable;
for (errors) |e, i| {
if (e[0] != ':') {
std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{});
}
var cur = e[1..];
var line_index = std.mem.indexOf(u8, cur, ":");
if (line_index == null) {
std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{});
}
const line = std.fmt.parseInt(u32, cur[0..line_index.?], 10) catch @panic("Unable to parse line number");
cur = cur[line_index.? + 1 ..];
const column_index = std.mem.indexOf(u8, cur, ":");
if (column_index == null) {
std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{});
}
const column = std.fmt.parseInt(u32, cur[0..column_index.?], 10) catch @panic("Unable to parse column number");
cur = cur[column_index.? + 2 ..];
if (!std.mem.eql(u8, cur[0..7], "error: ")) {
std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{});
}
const msg = cur[7..];
if (line == 0 or column == 0) {
@panic("Invalid test: error line and column must be specified starting at one!");
}
array[i] = .{
.msg = msg,
.line = line - 1,
.column = column - 1,
};
}
self.updates.append(.{ .src = src, .case = .{ .Error = array } }) catch unreachable;
}
};
pub fn addZIRMulti(
ctx: *TestContext,
name: []const u8,
target: std.zig.CrossTarget,
) *ZIRCase {
const case = ZIRCase{
.name = name,
.target = target,
.updates = std.ArrayList(ZIRUpdate).init(ctx.zir_cases.allocator),
};
ctx.zir_cases.append(case) catch unreachable;
return &ctx.zir_cases.items[ctx.zir_cases.items.len - 1];
}
pub fn addZIRCompareOutput(
ctx: *TestContext,
name: []const u8,
@ -71,67 +176,161 @@ pub const TestContext = struct {
pub fn addZIRTransform(
ctx: *TestContext,
name: []const u8,
cross_target: std.zig.CrossTarget,
target: std.zig.CrossTarget,
src: [:0]const u8,
expected_zir: []const u8,
result: [:0]const u8,
) void {
const case = ctx.zir_transform_cases.addOne() catch unreachable;
case.* = .{
.name = name,
.cross_target = cross_target,
.updates = std.ArrayList(ZIRTransformCase.Update).init(std.heap.page_allocator),
};
case.updates.append(.{
.src = src,
.expected = .{ .zir = expected_zir },
}) catch unreachable;
var c = ctx.addZIRMulti(name, target);
c.addTransform(src, result);
}
pub fn addZIRMulti(
pub fn addZIRError(
ctx: *TestContext,
name: []const u8,
cross_target: std.zig.CrossTarget,
) *ZIRTransformCase {
const case = ctx.zir_transform_cases.addOne() catch unreachable;
case.* = .{
.name = name,
.cross_target = cross_target,
.updates = std.ArrayList(ZIRTransformCase.Update).init(std.heap.page_allocator),
};
return case;
target: std.zig.CrossTarget,
src: [:0]const u8,
expected_errors: []const []const u8,
) void {
var c = ctx.addZIRMulti(name, target);
c.addError(src, expected_errors);
}
fn init(self: *TestContext) !void {
self.* = .{
.zir_cmp_output_cases = std.ArrayList(ZIRCompareOutputCase).init(std.heap.page_allocator),
.zir_transform_cases = std.ArrayList(ZIRTransformCase).init(std.heap.page_allocator),
fn init() TestContext {
const allocator = std.heap.page_allocator;
return .{
.zir_cmp_output_cases = std.ArrayList(ZIRCompareOutputCase).init(allocator),
.zir_cases = std.ArrayList(ZIRCase).init(allocator),
};
}
fn deinit(self: *TestContext) void {
self.zir_cmp_output_cases.deinit();
self.zir_transform_cases.deinit();
for (self.zir_cases.items) |c| {
for (c.updates.items) |u| {
if (u.case == .Error) {
c.updates.allocator.free(u.case.Error);
}
}
c.updates.deinit();
}
self.zir_cases.deinit();
self.* = undefined;
}
fn run(self: *TestContext) !void {
var progress = std.Progress{};
const root_node = try progress.start("zir", self.zir_cmp_output_cases.items.len +
self.zir_transform_cases.items.len);
const root_node = try progress.start("zir", self.zir_cases.items.len);
defer root_node.end();
const native_info = try std.zig.system.NativeTargetInfo.detect(std.heap.page_allocator, .{});
for (self.zir_cases.items) |case| {
std.testing.base_allocator_instance.reset();
const info = try std.zig.system.NativeTargetInfo.detect(std.testing.allocator, case.target);
try self.runOneZIRCase(std.testing.allocator, root_node, case, info.target);
try std.testing.allocator_instance.validate();
}
// TODO: wipe the rest of this function
for (self.zir_cmp_output_cases.items) |case| {
std.testing.base_allocator_instance.reset();
try self.runOneZIRCmpOutputCase(std.testing.allocator, root_node, case, native_info.target);
try std.testing.allocator_instance.validate();
}
for (self.zir_transform_cases.items) |case| {
std.testing.base_allocator_instance.reset();
const info = try std.zig.system.NativeTargetInfo.detect(std.testing.allocator, case.cross_target);
try self.runOneZIRTransformCase(std.testing.allocator, root_node, case, info.target);
try std.testing.allocator_instance.validate();
}
fn runOneZIRCase(self: *TestContext, allocator: *Allocator, root_node: *std.Progress.Node, case: ZIRCase, target: std.Target) !void {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const tmp_src_path = "test_case.zir";
const root_pkg = try Package.create(allocator, tmp.dir, ".", tmp_src_path);
defer root_pkg.destroy();
var prg_node = root_node.start(case.name, case.updates.items.len);
prg_node.activate();
defer prg_node.end();
var module = try Module.init(allocator, .{
.target = target,
// This is an Executable, as opposed to e.g. a *library*. This does
// not mean no ZIR is generated.
//
// TODO: support tests for object file building, and library builds
// and linking. This will require a rework to support multi-file
// tests.
.output_mode = .Obj,
// TODO: support testing optimizations
.optimize_mode = .Debug,
.bin_file_dir = tmp.dir,
.bin_file_path = "test_case.o",
.root_pkg = root_pkg,
});
defer module.deinit();
for (case.updates.items) |update| {
var update_node = prg_node.start("update", 4);
update_node.activate();
defer update_node.end();
var sync_node = update_node.start("write", null);
sync_node.activate();
try tmp.dir.writeFile(tmp_src_path, update.src);
sync_node.end();
var module_node = update_node.start("parse/analysis/codegen", null);
module_node.activate();
try module.update();
module_node.end();
switch (update.case) {
.Transformation => |expected_output| {
var emit_node = update_node.start("emit", null);
emit_node.activate();
var new_zir_module = try zir.emit(allocator, module);
defer new_zir_module.deinit(allocator);
emit_node.end();
var write_node = update_node.start("write", null);
write_node.activate();
var out_zir = std.ArrayList(u8).init(allocator);
defer out_zir.deinit();
try new_zir_module.writeToStream(allocator, out_zir.outStream());
write_node.end();
std.testing.expectEqualSlices(u8, expected_output, out_zir.items);
},
.Error => |e| {
var handled_errors = try allocator.alloc(bool, e.len);
defer allocator.free(handled_errors);
for (handled_errors) |*h| {
h.* = false;
}
var all_errors = try module.getAllErrorsAlloc();
defer all_errors.deinit(allocator);
for (all_errors.list) |a| {
for (e) |ex, i| {
if (a.line == ex.line and a.column == ex.column and std.mem.eql(u8, ex.msg, a.msg)) {
handled_errors[i] = true;
break;
}
} else {
std.debug.warn("{}\nUnexpected error:\n================\n:{}:{}: error: {}\n================\nTest failed.\n", .{ case.name, a.line + 1, a.column + 1, a.msg });
std.process.exit(1);
}
}
for (handled_errors) |h, i| {
if (!h) {
const er = e[i];
std.debug.warn("{}\nDid not receive error:\n================\n{}:{}: {}\n================\nTest failed.\n", .{ case.name, er.line, er.column, er.msg });
std.process.exit(1);
}
}
},
else => return error.unimplemented,
}
}
}
@ -208,118 +407,4 @@ pub const TestContext = struct {
}
}
}
fn runOneZIRTransformCase(
self: *TestContext,
allocator: *Allocator,
root_node: *std.Progress.Node,
case: ZIRTransformCase,
target: std.Target,
) !void {
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var update_node = root_node.start(case.name, case.updates.items.len);
update_node.activate();
defer update_node.end();
const tmp_src_path = "test-case.zir";
const root_pkg = try Package.create(allocator, tmp.dir, ".", tmp_src_path);
defer root_pkg.destroy();
var module = try Module.init(allocator, .{
.target = target,
.output_mode = .Obj,
.optimize_mode = .Debug,
.bin_file_dir = tmp.dir,
.bin_file_path = "test-case.o",
.root_pkg = root_pkg,
});
defer module.deinit();
for (case.updates.items) |update| {
var prg_node = update_node.start("", 3);
prg_node.activate();
defer prg_node.end();
try tmp.dir.writeFile(tmp_src_path, update.src);
var module_node = prg_node.start("parse/analysis/codegen", null);
module_node.activate();
try module.update();
module_node.end();
switch (update.expected) {
.zir => |expected_zir| {
var emit_node = prg_node.start("emit", null);
emit_node.activate();
var new_zir_module = try zir.emit(allocator, module);
defer new_zir_module.deinit(allocator);
emit_node.end();
var write_node = prg_node.start("write", null);
write_node.activate();
var out_zir = std.ArrayList(u8).init(allocator);
defer out_zir.deinit();
try new_zir_module.writeToStream(allocator, out_zir.outStream());
write_node.end();
std.testing.expectEqualSlices(u8, expected_zir, out_zir.items);
},
.errors => |expected_errors| {
var all_errors = try module.getAllErrorsAlloc();
defer all_errors.deinit(module.allocator);
for (expected_errors) |expected_error| {
for (all_errors.list) |full_err_msg| {
const text = try std.fmt.allocPrint(allocator, ":{}:{}: error: {}", .{
full_err_msg.line + 1,
full_err_msg.column + 1,
full_err_msg.msg,
});
defer allocator.free(text);
if (std.mem.eql(u8, text, expected_error)) {
break;
}
} else {
std.debug.warn(
"{}\nExpected this error:\n================\n{}\n================\nBut found these errors:\n================\n",
.{ case.name, expected_error },
);
for (all_errors.list) |full_err_msg| {
std.debug.warn(":{}:{}: error: {}\n", .{
full_err_msg.line + 1,
full_err_msg.column + 1,
full_err_msg.msg,
});
}
std.debug.warn("================\nTest failed\n", .{});
std.process.exit(1);
}
}
},
}
}
}
};
fn debugPrintErrors(src: []const u8, errors: var) void {
std.debug.warn("\n", .{});
var nl = true;
var line: usize = 1;
for (src) |byte| {
if (nl) {
std.debug.warn("{: >3}| ", .{line});
nl = false;
}
if (byte == '\n') {
nl = true;
line += 1;
}
std.debug.warn("{c}", .{byte});
}
std.debug.warn("\n", .{});
for (errors) |err_msg| {
const loc = std.zig.findLineColumn(src, err_msg.byte_offset);
std.debug.warn("{}:{}: error: {}\n", .{ loc.line + 1, loc.column + 1, err_msg.msg });
}
}

View File

@ -1,8 +1,53 @@
const TestContext = @import("../../src-self-hosted/test.zig").TestContext;
const std = @import("std");
const ErrorMsg = @import("../../src-self-hosted/Module.zig").ErrorMsg;
const linux_x64 = std.zig.CrossTarget{
.cpu_arch = .x86_64,
.os_tag = .linux,
};
pub fn addCases(ctx: *TestContext) !void {
ctx.addZIRError("call undefined local", linux_x64,
\\@noreturn = primitive(noreturn)
\\
\\@start_fnty = fntype([], @noreturn, cc=Naked)
\\@start = fn(@start_fnty, {
\\ %0 = call(%test, [])
\\})
// TODO: address inconsistency in this message and the one in the next test
, &[_][]const u8{":5:13: error: unrecognized identifier: %test"});
ctx.addZIRError("call with non-existent target", linux_x64,
\\@noreturn = primitive(noreturn)
\\
\\@start_fnty = fntype([], @noreturn, cc=Naked)
\\@start = fn(@start_fnty, {
\\ %0 = call(@notafunc, [])
\\})
\\@0 = str("_start")
\\@1 = ref(@0)
\\@2 = export(@1, @start)
, &[_][]const u8{":5:13: error: use of undeclared identifier 'notafunc'"});
// TODO: this error should occur at the call site, not the fntype decl
ctx.addZIRError("call naked function", linux_x64,
\\@noreturn = primitive(noreturn)
\\
\\@start_fnty = fntype([], @noreturn, cc=Naked)
\\@s = fn(@start_fnty, {})
\\@start = fn(@start_fnty, {
\\ %0 = call(@s, [])
\\})
\\@0 = str("_start")
\\@1 = ref(@0)
\\@2 = export(@1, @start)
, &[_][]const u8{":4:9: error: unable to call function with naked calling convention"});
// TODO: re-enable these tests.
// https://github.com/ziglang/zig/issues/1364
// TODO: add Zig AST -> ZIR testing pipeline
//try ctx.testCompileError(
// \\export fn entry() void {}

View File

@ -92,7 +92,7 @@ pub fn addCases(ctx: *TestContext) void {
{
var case = ctx.addZIRMulti("reference cycle with compile error in the cycle", linux_x64);
case.addZIR(
case.addTransform(
\\@void = primitive(void)
\\@fnty = fntype([], @void, cc=C)
\\
@ -171,7 +171,7 @@ pub fn addCases(ctx: *TestContext) void {
// Now we remove the call to `a`. `a` and `b` form a cycle, but no entry points are
// referencing either of them. This tests that the cycle is detected, and the error
// goes away.
case.addZIR(
case.addTransform(
\\@void = primitive(void)
\\@fnty = fntype([], @void, cc=C)
\\