diff --git a/src/test.zig b/src/test.zig index 8c2a15e844..c0a7fe6dc0 100644 --- a/src/test.zig +++ b/src/test.zig @@ -72,6 +72,17 @@ test { } } + { + const dir_path = try std.fs.path.join(arena, &.{ + std.fs.path.dirname(@src().file).?, "..", "test", "incremental", + }); + + var dir = try std.fs.cwd().openDir(dir_path, .{ .iterate = true }); + defer dir.close(); + + ctx.addTestCasesFromDir(dir); + } + try @import("test_cases").addCases(&ctx); try ctx.run(); @@ -154,6 +165,125 @@ const ErrorMsg = union(enum) { } }; +/// Manifest syntax example: +/// (see https://github.com/ziglang/zig/issues/11288) +/// +/// error +/// backend=stage1,stage2 +/// output_mode=exe +/// +/// :3:19: error: foo +/// +/// run +/// target=x86_64-linux,aarch64-macos +/// +/// I am expected stdout! Hello! +/// +/// cli +/// +/// build test +const TestManifest = struct { + @"type": Type, + config_map: std.StringHashMap([]const u8), + trailing_bytes: []const u8 = "", + + const Type = enum { + @"error", + run, + cli, + }; + + const TrailingIterator = struct { + inner: std.mem.TokenIterator(u8), + + fn next(self: *TrailingIterator) ?[]const u8 { + const next_inner = self.inner.next() orelse return null; + return std.mem.trim(u8, next_inner, " \t"); + } + }; + + fn ConfigValueIterator(comptime T: type, comptime ParseFn: type) type { + return struct { + inner: std.mem.SplitIterator(u8), + parse_fn: ParseFn, + + fn next(self: *@This()) ?T { + const next_raw = self.inner.next() orelse return null; + return self.parse_fn(next_raw); + } + }; + } + + fn parse(arena: Allocator, bytes: []const u8) !TestManifest { + var it = std.mem.tokenize(u8, bytes, "\r\n"); + + // First line is the test type + const tt: Type = blk: { + const line = it.next() orelse return error.MissingTestCaseType; + const raw = std.mem.trim(u8, line[2..], " \t"); + if (std.mem.eql(u8, raw, "error")) { + break :blk .@"error"; + } else if (std.mem.eql(u8, raw, "run")) { + break :blk .run; + } else if (std.mem.eql(u8, raw, "cli")) { + break :blk .cli; + } else { + std.log.warn("unknown test case type requested: {s}", .{raw}); + return error.UnknownTestCaseType; + } + }; + + var manifest: TestManifest = .{ + .@"type" = tt, + .config_map = std.StringHashMap([]const u8).init(arena), + }; + + // Any subsequent line until a blank comment line is key=value(s) pair + while (it.next()) |line| { + const trimmed = std.mem.trim(u8, line[2..], " \t"); + if (trimmed.len == 0) break; + + // Parse key=value(s) + var kv_it = std.mem.split(u8, trimmed, "="); + const key = kv_it.next() orelse return error.MissingKeyForConfig; + try manifest.config_map.putNoClobber(key, kv_it.next() orelse return error.MissingValuesForConfig); + } + + // Finally, trailing is expected output + manifest.trailing_bytes = bytes[it.index..]; + + return manifest; + } + + fn getConfigValues( + self: TestManifest, + key: []const u8, + comptime T: type, + parse_fn: anytype, + ) ?ConfigValueIterator(T, @TypeOf(parse_fn)) { + const bytes = self.config_map.get(key) orelse return null; + return ConfigValueIterator(T, @TypeOf(parse_fn)){ + .inner = std.mem.split(u8, bytes, ","), + .parse_fn = parse_fn, + }; + } + + fn trailing(self: TestManifest) TrailingIterator { + return .{ + .inner = std.mem.tokenize(u8, self.trailing_bytes, "\r\n"), + }; + } + + fn trailingAlloc(self: TestManifest, arena: Allocator) ![]const []const u8 { + var out = std.ArrayList([]const u8).init(arena); + var it = self.trailing(); + while (it.next()) |line| { + try out.append(line); + } + return out.toOwnedSlice(); + } +}; + pub const TestContext = struct { arena: Allocator, cases: std.ArrayList(Case), @@ -197,6 +327,10 @@ pub const TestContext = struct { stage1, stage2, llvm, + + fn parse(str: []const u8) ?Backend { + return std.meta.stringToEnum(Backend, str); + } }; /// A `Case` consists of a list of `Update`. The same `Compilation` is used for each @@ -661,6 +795,10 @@ pub const TestContext = struct { /// Execute all tests as incremental updates to a single compilation. Explicitly /// incremental tests ("foo.0.zig", "foo.1.zig", etc.) still execute in order incremental, + + fn parse(str: []const u8) ?Strategy { + return std.meta.stringToEnum(Strategy, str); + } }; /// Adds a compile-error test for each file in the provided directory, using the @@ -689,6 +827,15 @@ pub const TestContext = struct { }; } + pub fn addTestCasesFromDir(ctx: *TestContext, dir: std.fs.Dir) void { + var current_file: []const u8 = "none"; + addTestCasesFromDirInner(ctx, dir, ¤t_file) catch |err| { + std.debug.panic("test harness failed to process file '{s}': {s}\n", .{ + current_file, @errorName(err), + }); + }; + } + /// For a filename in the format ".X." or ".", returns /// "", "" and X parsed as a decimal number. If X is not present, or /// cannot be parsed as a decimal number, it is treated as part of @@ -749,6 +896,159 @@ pub const TestContext = struct { std.sort.sort([]const u8, filenames, Context{}, Context.lessThan); } + fn addTestCasesFromDirInner( + ctx: *TestContext, + dir: std.fs.Dir, + /// This is kept up to date with the currently being processed file so + /// that if any errors occur the caller knows it happened during this file. + current_file: *[]const u8, + ) !void { + var opt_case: ?*Case = null; + + var it = dir.iterate(); + var filenames = std.ArrayList([]const u8).init(ctx.arena); + defer filenames.deinit(); + + while (try it.next()) |entry| { + if (entry.kind != .File) continue; + + // Ignore stuff such as .swp files + switch (Compilation.classifyFileExt(entry.name)) { + .unknown => continue, + else => {}, + } + try filenames.append(try ctx.arena.dupe(u8, entry.name)); + } + + // Sort filenames, so that incremental tests are contiguous and in-order + sortTestFilenames(filenames.items); + + var prev_filename: []const u8 = ""; + for (filenames.items) |filename| { + current_file.* = filename; + + { // First, check if this file is part of an incremental update sequence + + // Split filename into ".." + const prev_parts = getTestFileNameParts(prev_filename); + const new_parts = getTestFileNameParts(filename); + + // If base_name and file_ext match, these files are in the same test sequence + // and the new one should be the incremented version of the previous test + if (std.mem.eql(u8, prev_parts.base_name, new_parts.base_name) and + std.mem.eql(u8, prev_parts.file_ext, new_parts.file_ext)) + { + + // This is "foo.X.zig" followed by "foo.Y.zig". Make sure that X = Y + 1 + if (prev_parts.test_index == null) return error.InvalidIncrementalTestIndex; + if (new_parts.test_index == null) return error.InvalidIncrementalTestIndex; + if (new_parts.test_index.? != prev_parts.test_index.? + 1) return error.InvalidIncrementalTestIndex; + } else { + + // This is not the same test sequence, so the new file must be the first file + // in a new sequence ("*.0.zig") or an independent test file ("*.zig") + if (new_parts.test_index != null and new_parts.test_index.? != 0) return error.InvalidIncrementalTestIndex; + + // if (strategy == .independent) + // opt_case = null; // Generate a new independent test case for this update + } + } + prev_filename = filename; + + const max_file_size = 10 * 1024 * 1024; + const src = try dir.readFileAllocOptions(ctx.arena, filename, max_file_size, null, 1, 0); + + // The manifest is the last contiguous block of comments in the file + // We scan for the beginning by searching backward for the first non-empty line that does not start with "//" + var manifest_start: ?usize = null; + var manifest_end: usize = src.len; + if (src.len > 0) { + var cursor: usize = src.len - 1; + while (true) { + // Move to beginning of line + while (cursor > 0 and src[cursor - 1] != '\n') cursor -= 1; + + if (std.mem.startsWith(u8, src[cursor..], "//")) { + manifest_start = cursor; // Contiguous comment line, include in manifest + } else { + if (manifest_start != null) break; // Encountered non-comment line, end of manifest + + // We ignore all-whitespace lines following the comment block, but anything else + // means that there is no manifest present. + if (std.mem.trim(u8, src[cursor..manifest_end], " \r\n\t").len == 0) { + manifest_end = cursor; + } else break; // If it's not whitespace, there is no manifest + } + + // Move to previous line + if (cursor != 0) cursor -= 1 else break; + } + } + + if (manifest_start) |start| { + // Parse the manifest + var mani = try TestManifest.parse(ctx.arena, src[start..manifest_end]); + const strategy = mani.getConfigValues("strategy", Strategy, Strategy.parse).?.next().?; + const backend = mani.getConfigValues("backend", Backend, Backend.parse).?.next().?; + + switch (mani.@"type") { + .@"error" => { + const case = opt_case orelse case: { + const case = try ctx.cases.addOne(); + case.* = .{ + .name = "none", + .target = .{}, + .backend = backend, + .updates = std.ArrayList(TestContext.Update).init(ctx.cases.allocator), + .is_test = false, + .output_mode = .Obj, + .files = std.ArrayList(TestContext.File).init(ctx.cases.allocator), + }; + opt_case = case; + break :case case; + }; + const errors = try mani.trailingAlloc(ctx.arena); + + switch (strategy) { + .independent => { + case.addError(src, errors); + }, + .incremental => { + case.addErrorNamed("update", src, errors); + }, + } + }, + .run => { + const case = opt_case orelse case: { + const case = try ctx.cases.addOne(); + case.* = .{ + .name = "none", + .target = .{}, + .backend = backend, + .updates = std.ArrayList(TestContext.Update).init(ctx.cases.allocator), + .is_test = false, + .output_mode = .Exe, + .files = std.ArrayList(TestContext.File).init(ctx.cases.allocator), + }; + opt_case = case; + break :case case; + }; + + var output = std.ArrayList(u8).init(ctx.arena); + var trailing_it = mani.trailing(); + while (trailing_it.next()) |line| { + try output.appendSlice(line); + } + case.addCompareOutput(src, output.toOwnedSlice()); + }, + .cli => @panic("TODO cli tests"), + } + } else { + return error.MissingManifest; + } + } + } + fn addErrorCasesFromDirInner( ctx: *TestContext, name: []const u8, diff --git a/test/incremental/add.0.zig b/test/incremental/add.0.zig new file mode 100644 index 0000000000..f271bfed84 --- /dev/null +++ b/test/incremental/add.0.zig @@ -0,0 +1,12 @@ +pub fn main() void { + add(3, 4); +} + +fn add(a: u32, b: u32) void { + if (a + b != 7) unreachable; +} + +// run +// backend=stage2 +// strategy=incremental +// diff --git a/test/incremental/add.1.zig b/test/incremental/add.1.zig new file mode 100644 index 0000000000..a8cc3fa836 --- /dev/null +++ b/test/incremental/add.1.zig @@ -0,0 +1,14 @@ +pub fn main() void { + if (x - 7 != 0) unreachable; +} + +fn add(a: u32, b: u32) u32 { + return a + b; +} + +const x = add(3, 4); + +// run +// backend=stage2 +// strategy=incremental +// diff --git a/test/incremental/add.2.zig b/test/incremental/add.2.zig new file mode 100644 index 0000000000..0f4c18e28f --- /dev/null +++ b/test/incremental/add.2.zig @@ -0,0 +1,14 @@ +pub fn main() void { + var x: usize = 3; + const y = add(1, 2, x); + if (y - 6 != 0) unreachable; +} + +inline fn add(a: usize, b: usize, c: usize) usize { + return a + b + c; +} + +// run +// backend=stage2 +// strategy=incremental +//