mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2025-12-06 06:13:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: improve test batch/sequence iterator

Browse Source 
With this improved iterator, type of test is now inferred from
the filename, enabling us to put all cases in one common parent
directory, and iterate over that, thus automating a lot of tasks.
This commit is contained in:
Jakub Konka 2022-05-03 10:29:12 +02:00
parent a839ccc153
commit 9d79b740bc
1 changed files with 205 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

242
src/test.zig
View File

@ -44,7 +44,7 @@ test {
// TODO make this incremental once the bug is solved that it triggers
// See: https://github.com/ziglang/zig/issues/11344
ctx.addTestCasesFromDir(dir, .independent);
ctx.addTestCasesFromDir(dir);
}
{
@ -55,7 +55,7 @@ test {
var dir = try std.fs.cwd().openDir(dir_path, .{ .iterate = true });
defer dir.close();
ctx.addTestCasesFromDir(dir, .incremental);
ctx.addTestCasesFromDir(dir);
}
try @import("test_cases").addCases(&ctx);
@ -395,6 +395,134 @@ const TestManifest = struct {
}
};
const TestStrategy = enum {
/// Execute tests as independent compilations, unless they are explicitly
/// incremental ("foo.0.zig", "foo.1.zig", etc.)
independent,
/// Execute all tests as incremental updates to a single compilation. Explicitly
/// incremental tests ("foo.0.zig", "foo.1.zig", etc.) still execute in order
incremental,
};
/// Iterates a set of filenames extracting batches that are either incremental
/// ("foo.0.zig", "foo.1.zig", etc.) or independent ("foo.zig", "bar.zig", etc.).
/// Assumes filenames are sorted.
const TestIterator = struct {
start: usize = 0,
end: usize = 0,
filenames: []const []const u8,
const Error = error{InvalidIncrementalTestIndex};
fn next(it: *TestIterator) Error!?[]const []const u8 {
try it.nextInner();
if (it.start == it.end) return null;
return it.filenames[it.start..it.end];
}
fn nextInner(it: *TestIterator) Error!void {
it.start = it.end;
if (it.end == it.filenames.len) return;
if (it.end + 1 == it.filenames.len) {
it.end += 1;
return;
}
const remaining = it.filenames[it.end..];
var i: usize = 0;
while (i < remaining.len - 1) : (i += 1) {
// First, check if this file is part of an incremental update sequence
// Split filename into "<base_name>.<index>.<file_ext>"
const prev_parts = getTestFileNameParts(remaining[i]);
const new_parts = getTestFileNameParts(remaining[i + 1]);
// If base_name and file_ext match, these files are in the same test sequence
// and the new one should be the incremented version of the previous test
if (std.mem.eql(u8, prev_parts.base_name, new_parts.base_name) and
std.mem.eql(u8, prev_parts.file_ext, new_parts.file_ext))
{
// This is "foo.X.zig" followed by "foo.Y.zig". Make sure that X = Y + 1
if (prev_parts.test_index == null)
return error.InvalidIncrementalTestIndex;
if (new_parts.test_index == null)
return error.InvalidIncrementalTestIndex;
if (new_parts.test_index.? != prev_parts.test_index.? + 1)
return error.InvalidIncrementalTestIndex;
} else {
// This is not the same test sequence, so the new file must be the first file
// in a new sequence ("*.0.zig") or an independent test file ("*.zig")
if (new_parts.test_index != null and new_parts.test_index.? != 0)
return error.InvalidIncrementalTestIndex;
it.end += i + 1;
break;
}
} else {
it.end += remaining.len;
}
}
};
/// For a filename in the format "<filename>.X.<ext>" or "<filename>.<ext>", returns
/// "<filename>", "<ext>" and X parsed as a decimal number. If X is not present, or
/// cannot be parsed as a decimal number, it is treated as part of <filename>
fn getTestFileNameParts(name: []const u8) struct {
base_name: []const u8,
file_ext: []const u8,
test_index: ?usize,
} {
const file_ext = std.fs.path.extension(name);
const trimmed = name[0 .. name.len - file_ext.len]; // Trim off ".<ext>"
const maybe_index = std.fs.path.extension(trimmed); // Extract ".X"
// Attempt to parse index
const index: ?usize = if (maybe_index.len > 0)
std.fmt.parseInt(usize, maybe_index[1..], 10) catch null
else
null;
// Adjust "<filename>" extent based on parsing success
const base_name_end = trimmed.len - if (index != null) maybe_index.len else 0;
return .{
.base_name = name[0..base_name_end],
.file_ext = if (file_ext.len > 0) file_ext[1..] else file_ext,
.test_index = index,
};
}
/// Sort test filenames in-place, so that incremental test cases ("foo.0.zig",
/// "foo.1.zig", etc.) are contiguous and appear in numerical order.
fn sortTestFilenames(
filenames: [][]const u8,
) void {
const Context = struct {
pub fn lessThan(_: @This(), a: []const u8, b: []const u8) bool {
const a_parts = getTestFileNameParts(a);
const b_parts = getTestFileNameParts(b);
// Sort "<base_name>.X.<file_ext>" based on "<base_name>" and "<file_ext>" first
return switch (std.mem.order(u8, a_parts.base_name, b_parts.base_name)) {
.lt => true,
.gt => false,
.eq => switch (std.mem.order(u8, a_parts.file_ext, b_parts.file_ext)) {
.lt => true,
.gt => false,
.eq => b: { // a and b differ only in their ".X" part
// Sort "<base_name>.<file_ext>" before any "<base_name>.X.<file_ext>"
if (a_parts.test_index == null) break :b true;
if (b_parts.test_index == null) break :b false;
// Make sure that incremental tests appear in linear order
return a_parts.test_index.? < b_parts.test_index.?;
},
},
};
}
};
std.sort.sort([]const u8, filenames, Context{}, Context.lessThan);
}
pub const TestContext = struct {
arena: Allocator,
cases: std.ArrayList(Case),
@ -895,100 +1023,29 @@ pub const TestContext = struct {
case.compiles(fixed_src);
}
const Strategy = enum {
/// Execute tests as independent compilations, unless they are explicitly
/// incremental ("foo.0.zig", "foo.1.zig", etc.)
independent,
/// Execute all tests as incremental updates to a single compilation. Explicitly
/// incremental tests ("foo.0.zig", "foo.1.zig", etc.) still execute in order
incremental,
};
/// Adds a test for each file in the provided directory, using the selected strategy.
/// Adds a test for each file in the provided directory.
/// Testing strategy (TestStrategy) is inferred automatically from filenames.
/// Recurses nested directories.
///
/// Each file should include a test manifest as a contiguous block of comments at
/// the end of the file. The first line should be the test type, followed by a set of
/// key-value config values, followed by a blank line, then the expected output.
pub fn addTestCasesFromDir(ctx: *TestContext, dir: std.fs.Dir, strategy: Strategy) void {
pub fn addTestCasesFromDir(ctx: *TestContext, dir: std.fs.Dir) void {
var current_file: []const u8 = "none";
ctx.addTestCasesFromDirInner(dir, strategy, &current_file) catch |err| {
ctx.addTestCasesFromDirInner(dir, &current_file) catch |err| {
std.debug.panic("test harness failed to process file '{s}': {s}\n", .{
current_file, @errorName(err),
});
};
}
/// For a filename in the format "<filename>.X.<ext>" or "<filename>.<ext>", returns
/// "<filename>", "<ext>" and X parsed as a decimal number. If X is not present, or
/// cannot be parsed as a decimal number, it is treated as part of <filename>
fn getTestFileNameParts(name: []const u8) struct {
base_name: []const u8,
file_ext: []const u8,
test_index: ?usize,
} {
const file_ext = std.fs.path.extension(name);
const trimmed = name[0 .. name.len - file_ext.len]; // Trim off ".<ext>"
const maybe_index = std.fs.path.extension(trimmed); // Extract ".X"
// Attempt to parse index
const index: ?usize = if (maybe_index.len > 0)
std.fmt.parseInt(usize, maybe_index[1..], 10) catch null
else
null;
// Adjust "<filename>" extent based on parsing success
const base_name_end = trimmed.len - if (index != null) maybe_index.len else 0;
return .{
.base_name = name[0..base_name_end],
.file_ext = if (file_ext.len > 0) file_ext[1..] else file_ext,
.test_index = index,
};
}
/// Sort test filenames in-place, so that incremental test cases ("foo.0.zig",
/// "foo.1.zig", etc.) are contiguous and appear in numerical order.
fn sortTestFilenames(
filenames: [][]const u8,
) void {
const Context = struct {
pub fn lessThan(_: @This(), a: []const u8, b: []const u8) bool {
const a_parts = getTestFileNameParts(a);
const b_parts = getTestFileNameParts(b);
// Sort "<base_name>.X.<file_ext>" based on "<base_name>" and "<file_ext>" first
return switch (std.mem.order(u8, a_parts.base_name, b_parts.base_name)) {
.lt => true,
.gt => false,
.eq => switch (std.mem.order(u8, a_parts.file_ext, b_parts.file_ext)) {
.lt => true,
.gt => false,
.eq => b: { // a and b differ only in their ".X" part
// Sort "<base_name>.<file_ext>" before any "<base_name>.X.<file_ext>"
if (a_parts.test_index == null) break :b true;
if (b_parts.test_index == null) break :b false;
// Make sure that incremental tests appear in linear order
return a_parts.test_index.? < b_parts.test_index.?;
},
},
};
}
};
std.sort.sort([]const u8, filenames, Context{}, Context.lessThan);
}
fn addTestCasesFromDirInner(
ctx: *TestContext,
dir: std.fs.Dir,
strategy: Strategy,
/// This is kept up to date with the currently being processed file so
/// that if any errors occur the caller knows it happened during this file.
current_file: *[]const u8,
) !void {
var cases = std.ArrayList(usize).init(ctx.arena);
var it = try dir.walk(ctx.arena);
var filenames = std.ArrayList([]const u8).init(ctx.arena);
@ -1006,32 +1063,14 @@ pub const TestContext = struct {
// Sort filenames, so that incremental tests are contiguous and in-order
sortTestFilenames(filenames.items);
var prev_filename: []const u8 = "";
for (filenames.items) |filename| {
var test_it = TestIterator{ .filenames = filenames.items };
while (try test_it.next()) |batch| {
const strategy: TestStrategy = if (batch.len > 1) .incremental else .independent;
var cases = std.ArrayList(usize).init(ctx.arena);
for (batch) |filename| {
current_file.* = filename;
// First, check if this file is part of an incremental update sequence
// Split filename into "<base_name>.<index>.<file_ext>"
const prev_parts = getTestFileNameParts(prev_filename);
const new_parts = getTestFileNameParts(filename);
// If base_name and file_ext match, these files are in the same test sequence
// and the new one should be the incremented version of the previous test
if (std.mem.eql(u8, prev_parts.base_name, new_parts.base_name) and
std.mem.eql(u8, prev_parts.file_ext, new_parts.file_ext))
{
// This is "foo.X.zig" followed by "foo.Y.zig". Make sure that X = Y + 1
if (prev_parts.test_index == null) return error.InvalidIncrementalTestIndex;
if (new_parts.test_index == null) return error.InvalidIncrementalTestIndex;
if (new_parts.test_index.? != prev_parts.test_index.? + 1) return error.InvalidIncrementalTestIndex;
} else {
// This is not the same test sequence, so the new file must be the first file
// in a new sequence ("*.0.zig") or an independent test file ("*.zig")
if (new_parts.test_index != null and new_parts.test_index.? != 0) return error.InvalidIncrementalTestIndex;
cases.clearRetainingCapacity();
}
prev_filename = filename;
const max_file_size = 10 * 1024 * 1024;
const src = try dir.readFileAllocOptions(ctx.arena, filename, max_file_size, null, 1, 0);
@ -1108,6 +1147,7 @@ pub const TestContext = struct {
}
}
}
}
fn init(gpa: Allocator, arena: Allocator) TestContext {
return .{