diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 8bb03939dc..76be8610ef 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -17,6 +17,12 @@ const runner = @This(); pub const root = @import("@build"); pub const dependencies = @import("@dependencies"); +pub const std_options: std.Options = .{ + .side_channels_mitigations = .none, + .http_disable_tls = true, + .crypto_fork_safety = false, +}; + pub fn main() !void { // Here we use an ArenaAllocator backed by a page allocator because a build is a short-lived, // one shot program. We don't need to waste time freeing memory and finding places to squish @@ -106,6 +112,7 @@ pub fn main() !void { var watch = false; var fuzz = false; var debounce_interval_ms: u16 = 50; + var listen_port: u16 = 0; while (nextArg(args, &arg_idx)) |arg| { if (mem.startsWith(u8, arg, "-Z")) { @@ -203,6 +210,14 @@ pub fn main() !void { next_arg, @errorName(err), }); }; + } else if (mem.eql(u8, arg, "--port")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected u16 after '{s}'", .{arg}); + listen_port = std.fmt.parseUnsigned(u16, next_arg, 10) catch |err| { + fatal("unable to parse port '{s}' as unsigned 16-bit integer: {s}\n", .{ + next_arg, @errorName(err), + }); + }; } else if (mem.eql(u8, arg, "--debug-log")) { const next_arg = nextArgOrFatal(args, &arg_idx); try debug_log_scopes.append(next_arg); @@ -403,7 +418,19 @@ pub fn main() !void { else => return err, }; if (fuzz) { - Fuzz.start(&run.thread_pool, run.step_stack.keys(), run.ttyconf, main_progress_node); + const listen_address = std.net.Address.parseIp("127.0.0.1", listen_port) catch unreachable; + try Fuzz.start( + gpa, + arena, + global_cache_directory, + zig_lib_directory, + zig_exe, + &run.thread_pool, + run.step_stack.keys(), + run.ttyconf, + listen_address, + main_progress_node, + ); } if (!watch) return cleanExit(); diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 64f6f230fe..65580fcd3f 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -28,6 +28,7 @@ pub fn main() void { @panic("unable to parse command line args"); var listen = false; + var opt_cache_dir: ?[]const u8 = null; for (args[1..]) |arg| { if (std.mem.eql(u8, arg, "--listen=-")) { @@ -35,13 +36,18 @@ pub fn main() void { } else if (std.mem.startsWith(u8, arg, "--seed=")) { testing.random_seed = std.fmt.parseUnsigned(u32, arg["--seed=".len..], 0) catch @panic("unable to parse --seed command line argument"); + } else if (std.mem.startsWith(u8, arg, "--cache-dir")) { + opt_cache_dir = arg["--cache-dir=".len..]; } else { @panic("unrecognized command line argument"); } } fba.reset(); - if (builtin.fuzz) fuzzer_init(); + if (builtin.fuzz) { + const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument"); + fuzzer_init(FuzzerSlice.fromSlice(cache_dir)); + } if (listen) { return mainServer() catch @panic("internal test runner failure"); @@ -60,6 +66,11 @@ fn mainServer() !void { }); defer server.deinit(); + if (builtin.fuzz) { + const coverage_id = fuzzer_coverage_id(); + try server.serveU64Message(.coverage_id, coverage_id); + } + while (true) { const hdr = try server.receiveMessage(); switch (hdr.tag) { @@ -316,15 +327,22 @@ const FuzzerSlice = extern struct { ptr: [*]const u8, len: usize, + /// Inline to avoid fuzzer instrumentation. inline fn toSlice(s: FuzzerSlice) []const u8 { return s.ptr[0..s.len]; } + + /// Inline to avoid fuzzer instrumentation. + inline fn fromSlice(s: []const u8) FuzzerSlice { + return .{ .ptr = s.ptr, .len = s.len }; + } }; var is_fuzz_test: bool = undefined; extern fn fuzzer_next() FuzzerSlice; -extern fn fuzzer_init() void; +extern fn fuzzer_init(cache_dir: FuzzerSlice) void; +extern fn fuzzer_coverage_id() u64; pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 { @disableInstrumentation(); diff --git a/lib/docs/wasm/main.zig b/lib/docs/wasm/main.zig index f5ce02d7d6..214f28c24b 100644 --- a/lib/docs/wasm/main.zig +++ b/lib/docs/wasm/main.zig @@ -53,7 +53,7 @@ export fn unpack(tar_ptr: [*]u8, tar_len: usize) void { const tar_bytes = tar_ptr[0..tar_len]; //log.debug("received {d} bytes of tar file", .{tar_bytes.len}); - unpack_inner(tar_bytes) catch |err| { + unpackInner(tar_bytes) catch |err| { fatal("unable to unpack tar: {s}", .{@errorName(err)}); }; } @@ -750,7 +750,7 @@ export fn decl_type_html(decl_index: Decl.Index) String { const Oom = error{OutOfMemory}; -fn unpack_inner(tar_bytes: []u8) !void { +fn unpackInner(tar_bytes: []u8) !void { var fbs = std.io.fixedBufferStream(tar_bytes); var file_name_buffer: [1024]u8 = undefined; var link_name_buffer: [1024]u8 = undefined; diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 7edab785a9..ede3663cdc 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -17,7 +17,8 @@ fn logOverride( args: anytype, ) void { const f = if (log_file) |f| f else f: { - const f = fuzzer.dir.createFile("libfuzzer.log", .{}) catch @panic("failed to open fuzzer log file"); + const f = fuzzer.cache_dir.createFile("tmp/libfuzzer.log", .{}) catch + @panic("failed to open fuzzer log file"); log_file = f; break :f f; }; @@ -114,7 +115,10 @@ const Fuzzer = struct { /// Stored in a memory-mapped file so that it can be shared with other /// processes and viewed while the fuzzer is running. seen_pcs: MemoryMappedList, - dir: std.fs.Dir, + cache_dir: std.fs.Dir, + /// Identifies the file name that will be used to store coverage + /// information, available to other processes. + coverage_id: u64, const SeenPcsHeader = extern struct { n_runs: usize, @@ -189,18 +193,31 @@ const Fuzzer = struct { id: Run.Id, }; - fn init(f: *Fuzzer, dir: std.fs.Dir) !void { - f.dir = dir; + fn init(f: *Fuzzer, cache_dir: std.fs.Dir) !void { + const flagged_pcs = f.flagged_pcs; + + f.cache_dir = cache_dir; + + // Choose a file name for the coverage based on a hash of the PCs that will be stored within. + const pc_digest = d: { + var hasher = std.hash.Wyhash.init(0); + for (flagged_pcs) |flagged_pc| { + hasher.update(std.mem.asBytes(&flagged_pc.addr)); + } + break :d f.coverage.run_id_hasher.final(); + }; + f.coverage_id = pc_digest; + const hex_digest = std.fmt.hex(pc_digest); + const coverage_file_path = "v/" ++ hex_digest; // Layout of this file: // - Header // - list of PC addresses (usize elements) // - list of hit flag, 1 bit per address (stored in u8 elements) - const coverage_file = dir.createFile("coverage", .{ + const coverage_file = createFileBail(cache_dir, coverage_file_path, .{ .read = true, .truncate = false, - }) catch |err| fatal("unable to create coverage file: {s}", .{@errorName(err)}); - const flagged_pcs = f.flagged_pcs; + }); const n_bitset_elems = (flagged_pcs.len + 7) / 8; const bytes_len = @sizeOf(SeenPcsHeader) + flagged_pcs.len * @sizeOf(usize) + n_bitset_elems; const existing_len = coverage_file.getEndPos() catch |err| { @@ -217,7 +234,8 @@ const Fuzzer = struct { fatal("unable to init coverage memory map: {s}", .{@errorName(err)}); }; if (existing_len != 0) { - const existing_pcs = std.mem.bytesAsSlice(usize, f.seen_pcs.items[@sizeOf(SeenPcsHeader)..][0 .. flagged_pcs.len * @sizeOf(usize)]); + const existing_pcs_bytes = f.seen_pcs.items[@sizeOf(SeenPcsHeader)..][0 .. flagged_pcs.len * @sizeOf(usize)]; + const existing_pcs = std.mem.bytesAsSlice(usize, existing_pcs_bytes); for (existing_pcs, flagged_pcs, 0..) |old, new, i| { if (old != new.addr) { fatal("incompatible existing coverage file (differing PC at index {d}: {x} != {x})", .{ @@ -380,6 +398,21 @@ const Fuzzer = struct { } }; +fn createFileBail(dir: std.fs.Dir, sub_path: []const u8, flags: std.fs.File.CreateFlags) std.fs.File { + return dir.createFile(sub_path, flags) catch |err| switch (err) { + error.FileNotFound => { + const dir_name = std.fs.path.dirname(sub_path).?; + dir.makePath(dir_name) catch |e| { + fatal("unable to make path '{s}': {s}", .{ dir_name, @errorName(e) }); + }; + return dir.createFile(sub_path, flags) catch |e| { + fatal("unable to create file '{s}': {s}", .{ sub_path, @errorName(e) }); + }; + }, + else => fatal("unable to create file '{s}': {s}", .{ sub_path, @errorName(err) }), + }; +} + fn oom(err: anytype) noreturn { switch (err) { error.OutOfMemory => @panic("out of memory"), @@ -397,25 +430,35 @@ var fuzzer: Fuzzer = .{ .n_runs = 0, .recent_cases = .{}, .coverage = undefined, - .dir = undefined, + .cache_dir = undefined, .seen_pcs = undefined, + .coverage_id = undefined, }; +/// Invalid until `fuzzer_init` is called. +export fn fuzzer_coverage_id() u64 { + return fuzzer.coverage_id; +} + export fn fuzzer_next() Fuzzer.Slice { return Fuzzer.Slice.fromZig(fuzzer.next() catch |err| switch (err) { error.OutOfMemory => @panic("out of memory"), }); } -export fn fuzzer_init() void { +export fn fuzzer_init(cache_dir_struct: Fuzzer.Slice) void { if (module_count_8bc == 0) fatal("__sanitizer_cov_8bit_counters_init was never called", .{}); if (module_count_pcs == 0) fatal("__sanitizer_cov_pcs_init was never called", .{}); - // TODO: move this to .zig-cache/f - const fuzz_dir = std.fs.cwd().makeOpenPath("f", .{ .iterate = true }) catch |err| { - fatal("unable to open fuzz directory 'f': {s}", .{@errorName(err)}); - }; - fuzzer.init(fuzz_dir) catch |err| fatal("unable to init fuzzer: {s}", .{@errorName(err)}); + const cache_dir_path = cache_dir_struct.toZig(); + const cache_dir = if (cache_dir_path.len == 0) + std.fs.cwd() + else + std.fs.cwd().makeOpenPath(cache_dir_path, .{ .iterate = true }) catch |err| { + fatal("unable to open fuzz directory '{s}': {s}", .{ cache_dir_path, @errorName(err) }); + }; + + fuzzer.init(cache_dir) catch |err| fatal("unable to init fuzzer: {s}", .{@errorName(err)}); } /// Like `std.ArrayListUnmanaged(u8)` but backed by memory mapping. diff --git a/lib/fuzzer/index.html b/lib/fuzzer/index.html new file mode 100644 index 0000000000..c1ef059ad6 --- /dev/null +++ b/lib/fuzzer/index.html @@ -0,0 +1,76 @@ + + + + + Zig Documentation + + + + + + + diff --git a/lib/fuzzer/main.js b/lib/fuzzer/main.js new file mode 100644 index 0000000000..9b0d4cd8c3 --- /dev/null +++ b/lib/fuzzer/main.js @@ -0,0 +1,40 @@ +(function() { + let wasm_promise = fetch("main.wasm"); + let sources_promise = fetch("sources.tar").then(function(response) { + if (!response.ok) throw new Error("unable to download sources"); + return response.arrayBuffer(); + }); + var wasm_exports = null; + + const text_decoder = new TextDecoder(); + const text_encoder = new TextEncoder(); + + WebAssembly.instantiateStreaming(wasm_promise, { + js: { + log: function(ptr, len) { + const msg = decodeString(ptr, len); + console.log(msg); + }, + panic: function (ptr, len) { + const msg = decodeString(ptr, len); + throw new Error("panic: " + msg); + }, + }, + }).then(function(obj) { + wasm_exports = obj.instance.exports; + window.wasm = obj; // for debugging + + sources_promise.then(function(buffer) { + const js_array = new Uint8Array(buffer); + const ptr = wasm_exports.alloc(js_array.length); + const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length); + wasm_array.set(js_array); + wasm_exports.unpack(ptr, js_array.length); + }); + }); + + function decodeString(ptr, len) { + if (len === 0) return ""; + return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len)); + } +})(); diff --git a/lib/fuzzer/wasm/main.zig b/lib/fuzzer/wasm/main.zig new file mode 100644 index 0000000000..09b9d81068 --- /dev/null +++ b/lib/fuzzer/wasm/main.zig @@ -0,0 +1,99 @@ +const std = @import("std"); +const assert = std.debug.assert; + +const Walk = @import("Walk"); + +const gpa = std.heap.wasm_allocator; +const log = std.log; + +const js = struct { + extern "js" fn log(ptr: [*]const u8, len: usize) void; + extern "js" fn panic(ptr: [*]const u8, len: usize) noreturn; +}; + +pub const std_options: std.Options = .{ + .logFn = logFn, +}; + +pub fn panic(msg: []const u8, st: ?*std.builtin.StackTrace, addr: ?usize) noreturn { + _ = st; + _ = addr; + log.err("panic: {s}", .{msg}); + @trap(); +} + +fn logFn( + comptime message_level: log.Level, + comptime scope: @TypeOf(.enum_literal), + comptime format: []const u8, + args: anytype, +) void { + const level_txt = comptime message_level.asText(); + const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; + var buf: [500]u8 = undefined; + const line = std.fmt.bufPrint(&buf, level_txt ++ prefix2 ++ format, args) catch l: { + buf[buf.len - 3 ..][0..3].* = "...".*; + break :l &buf; + }; + js.log(line.ptr, line.len); +} + +export fn alloc(n: usize) [*]u8 { + const slice = gpa.alloc(u8, n) catch @panic("OOM"); + return slice.ptr; +} + +export fn unpack(tar_ptr: [*]u8, tar_len: usize) void { + const tar_bytes = tar_ptr[0..tar_len]; + log.debug("received {d} bytes of tar file", .{tar_bytes.len}); + + unpackInner(tar_bytes) catch |err| { + fatal("unable to unpack tar: {s}", .{@errorName(err)}); + }; +} + +fn unpackInner(tar_bytes: []u8) !void { + var fbs = std.io.fixedBufferStream(tar_bytes); + var file_name_buffer: [1024]u8 = undefined; + var link_name_buffer: [1024]u8 = undefined; + var it = std.tar.iterator(fbs.reader(), .{ + .file_name_buffer = &file_name_buffer, + .link_name_buffer = &link_name_buffer, + }); + while (try it.next()) |tar_file| { + switch (tar_file.kind) { + .file => { + if (tar_file.size == 0 and tar_file.name.len == 0) break; + if (std.mem.endsWith(u8, tar_file.name, ".zig")) { + log.debug("found file: '{s}'", .{tar_file.name}); + const file_name = try gpa.dupe(u8, tar_file.name); + if (std.mem.indexOfScalar(u8, file_name, '/')) |pkg_name_end| { + const pkg_name = file_name[0..pkg_name_end]; + const gop = try Walk.modules.getOrPut(gpa, pkg_name); + const file: Walk.File.Index = @enumFromInt(Walk.files.entries.len); + if (!gop.found_existing or + std.mem.eql(u8, file_name[pkg_name_end..], "/root.zig") or + std.mem.eql(u8, file_name[pkg_name_end + 1 .. file_name.len - ".zig".len], pkg_name)) + { + gop.value_ptr.* = file; + } + const file_bytes = tar_bytes[fbs.pos..][0..@intCast(tar_file.size)]; + assert(file == try Walk.add_file(file_name, file_bytes)); + } + } else { + log.warn("skipping: '{s}' - the tar creation should have done that", .{tar_file.name}); + } + }, + else => continue, + } + } +} + +fn fatal(comptime format: []const u8, args: anytype) noreturn { + var buf: [500]u8 = undefined; + const line = std.fmt.bufPrint(&buf, format, args) catch l: { + buf[buf.len - 3 ..][0..3].* = "...".*; + break :l &buf; + }; + js.panic(line.ptr, line.len); +} diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 7612ad0d6d..03743cf52e 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -2300,22 +2300,26 @@ pub const LazyPath = union(enum) { } pub fn path(lazy_path: LazyPath, b: *Build, sub_path: []const u8) LazyPath { + return lazy_path.join(b.allocator, sub_path) catch @panic("OOM"); + } + + pub fn join(lazy_path: LazyPath, arena: Allocator, sub_path: []const u8) Allocator.Error!LazyPath { return switch (lazy_path) { .src_path => |src| .{ .src_path = .{ .owner = src.owner, - .sub_path = b.pathResolve(&.{ src.sub_path, sub_path }), + .sub_path = try fs.path.resolve(arena, &.{ src.sub_path, sub_path }), } }, .generated => |gen| .{ .generated = .{ .file = gen.file, .up = gen.up, - .sub_path = b.pathResolve(&.{ gen.sub_path, sub_path }), + .sub_path = try fs.path.resolve(arena, &.{ gen.sub_path, sub_path }), } }, .cwd_relative => |cwd_relative| .{ - .cwd_relative = b.pathResolve(&.{ cwd_relative, sub_path }), + .cwd_relative = try fs.path.resolve(arena, &.{ cwd_relative, sub_path }), }, .dependency => |dep| .{ .dependency = .{ .dependency = dep.dependency, - .sub_path = b.pathResolve(&.{ dep.sub_path, sub_path }), + .sub_path = try fs.path.resolve(arena, &.{ dep.sub_path, sub_path }), } }, }; } diff --git a/lib/std/Build/Fuzz.zig b/lib/std/Build/Fuzz.zig index 2628b92516..e26f587eac 100644 --- a/lib/std/Build/Fuzz.zig +++ b/lib/std/Build/Fuzz.zig @@ -1,59 +1,479 @@ +const builtin = @import("builtin"); const std = @import("../std.zig"); -const Fuzz = @This(); +const Build = std.Build; const Step = std.Build.Step; const assert = std.debug.assert; const fatal = std.process.fatal; +const Allocator = std.mem.Allocator; +const log = std.log; + +const Fuzz = @This(); const build_runner = @import("root"); pub fn start( + gpa: Allocator, + arena: Allocator, + global_cache_directory: Build.Cache.Directory, + zig_lib_directory: Build.Cache.Directory, + zig_exe_path: []const u8, thread_pool: *std.Thread.Pool, all_steps: []const *Step, ttyconf: std.io.tty.Config, + listen_address: std.net.Address, prog_node: std.Progress.Node, -) void { - const count = block: { +) Allocator.Error!void { + const fuzz_run_steps = block: { const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0); defer rebuild_node.end(); - var count: usize = 0; var wait_group: std.Thread.WaitGroup = .{}; defer wait_group.wait(); + var fuzz_run_steps: std.ArrayListUnmanaged(*Step.Run) = .{}; + defer fuzz_run_steps.deinit(gpa); for (all_steps) |step| { const run = step.cast(Step.Run) orelse continue; if (run.fuzz_tests.items.len > 0 and run.producer != null) { thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, ttyconf, rebuild_node }); - count += 1; + try fuzz_run_steps.append(gpa, run); } } - if (count == 0) fatal("no fuzz tests found", .{}); - rebuild_node.setEstimatedTotalItems(count); - break :block count; + if (fuzz_run_steps.items.len == 0) fatal("no fuzz tests found", .{}); + rebuild_node.setEstimatedTotalItems(fuzz_run_steps.items.len); + break :block try arena.dupe(*Step.Run, fuzz_run_steps.items); }; // Detect failure. - for (all_steps) |step| { - const run = step.cast(Step.Run) orelse continue; - if (run.fuzz_tests.items.len > 0 and run.rebuilt_executable == null) + for (fuzz_run_steps) |run| { + assert(run.fuzz_tests.items.len > 0); + if (run.rebuilt_executable == null) fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); } + var web_server: WebServer = .{ + .gpa = gpa, + .global_cache_directory = global_cache_directory, + .zig_lib_directory = zig_lib_directory, + .zig_exe_path = zig_exe_path, + .msg_queue = .{}, + .mutex = .{}, + .listen_address = listen_address, + .fuzz_run_steps = fuzz_run_steps, + }; + + const web_server_thread = std.Thread.spawn(.{}, WebServer.run, .{&web_server}) catch |err| { + fatal("unable to spawn web server thread: {s}", .{@errorName(err)}); + }; + defer web_server_thread.join(); + { - const fuzz_node = prog_node.start("Fuzzing", count); + const fuzz_node = prog_node.start("Fuzzing", fuzz_run_steps.len); defer fuzz_node.end(); var wait_group: std.Thread.WaitGroup = .{}; defer wait_group.wait(); - for (all_steps) |step| { - const run = step.cast(Step.Run) orelse continue; + for (fuzz_run_steps) |run| { for (run.fuzz_tests.items) |unit_test_index| { assert(run.rebuilt_executable != null); - thread_pool.spawnWg(&wait_group, fuzzWorkerRun, .{ run, unit_test_index, ttyconf, fuzz_node }); + thread_pool.spawnWg(&wait_group, fuzzWorkerRun, .{ + run, &web_server, unit_test_index, ttyconf, fuzz_node, + }); } } } - fatal("all fuzz workers crashed", .{}); + log.err("all fuzz workers crashed", .{}); } +pub const WebServer = struct { + gpa: Allocator, + global_cache_directory: Build.Cache.Directory, + zig_lib_directory: Build.Cache.Directory, + zig_exe_path: []const u8, + /// Messages from fuzz workers. Protected by mutex. + msg_queue: std.ArrayListUnmanaged(Msg), + mutex: std.Thread.Mutex, + listen_address: std.net.Address, + fuzz_run_steps: []const *Step.Run, + + const Msg = union(enum) { + coverage_id: u64, + }; + + fn run(ws: *WebServer) void { + var http_server = ws.listen_address.listen(.{ + .reuse_address = true, + }) catch |err| { + log.err("failed to listen to port {d}: {s}", .{ ws.listen_address.in.getPort(), @errorName(err) }); + return; + }; + const port = http_server.listen_address.in.getPort(); + log.info("web interface listening at http://127.0.0.1:{d}/", .{port}); + + while (true) { + const connection = http_server.accept() catch |err| { + log.err("failed to accept connection: {s}", .{@errorName(err)}); + return; + }; + _ = std.Thread.spawn(.{}, accept, .{ ws, connection }) catch |err| { + log.err("unable to spawn connection thread: {s}", .{@errorName(err)}); + connection.stream.close(); + continue; + }; + } + } + + fn accept(ws: *WebServer, connection: std.net.Server.Connection) void { + defer connection.stream.close(); + + var read_buffer: [8000]u8 = undefined; + var server = std.http.Server.init(connection, &read_buffer); + while (server.state == .ready) { + var request = server.receiveHead() catch |err| switch (err) { + error.HttpConnectionClosing => return, + else => { + log.err("closing http connection: {s}", .{@errorName(err)}); + return; + }, + }; + serveRequest(ws, &request) catch |err| switch (err) { + error.AlreadyReported => return, + else => |e| { + log.err("unable to serve {s}: {s}", .{ request.head.target, @errorName(e) }); + return; + }, + }; + } + } + + fn serveRequest(ws: *WebServer, request: *std.http.Server.Request) !void { + if (std.mem.eql(u8, request.head.target, "/") or + std.mem.eql(u8, request.head.target, "/debug") or + std.mem.eql(u8, request.head.target, "/debug/")) + { + try serveFile(ws, request, "fuzzer/index.html", "text/html"); + } else if (std.mem.eql(u8, request.head.target, "/main.js") or + std.mem.eql(u8, request.head.target, "/debug/main.js")) + { + try serveFile(ws, request, "fuzzer/main.js", "application/javascript"); + } else if (std.mem.eql(u8, request.head.target, "/main.wasm")) { + try serveWasm(ws, request, .ReleaseFast); + } else if (std.mem.eql(u8, request.head.target, "/debug/main.wasm")) { + try serveWasm(ws, request, .Debug); + } else if (std.mem.eql(u8, request.head.target, "/sources.tar") or + std.mem.eql(u8, request.head.target, "/debug/sources.tar")) + { + try serveSourcesTar(ws, request); + } else { + try request.respond("not found", .{ + .status = .not_found, + .extra_headers = &.{ + .{ .name = "content-type", .value = "text/plain" }, + }, + }); + } + } + + fn serveFile( + ws: *WebServer, + request: *std.http.Server.Request, + name: []const u8, + content_type: []const u8, + ) !void { + const gpa = ws.gpa; + // The desired API is actually sendfile, which will require enhancing std.http.Server. + // We load the file with every request so that the user can make changes to the file + // and refresh the HTML page without restarting this server. + const file_contents = ws.zig_lib_directory.handle.readFileAlloc(gpa, name, 10 * 1024 * 1024) catch |err| { + log.err("failed to read '{}{s}': {s}", .{ ws.zig_lib_directory, name, @errorName(err) }); + return error.AlreadyReported; + }; + defer gpa.free(file_contents); + try request.respond(file_contents, .{ + .extra_headers = &.{ + .{ .name = "content-type", .value = content_type }, + cache_control_header, + }, + }); + } + + fn serveWasm( + ws: *WebServer, + request: *std.http.Server.Request, + optimize_mode: std.builtin.OptimizeMode, + ) !void { + const gpa = ws.gpa; + + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + // Do the compilation every request, so that the user can edit the files + // and see the changes without restarting the server. + const wasm_binary_path = try buildWasmBinary(ws, arena, optimize_mode); + // std.http.Server does not have a sendfile API yet. + const file_contents = try std.fs.cwd().readFileAlloc(gpa, wasm_binary_path, 10 * 1024 * 1024); + defer gpa.free(file_contents); + try request.respond(file_contents, .{ + .extra_headers = &.{ + .{ .name = "content-type", .value = "application/wasm" }, + cache_control_header, + }, + }); + } + + fn buildWasmBinary( + ws: *WebServer, + arena: Allocator, + optimize_mode: std.builtin.OptimizeMode, + ) ![]const u8 { + const gpa = ws.gpa; + + const main_src_path: Build.Cache.Path = .{ + .root_dir = ws.zig_lib_directory, + .sub_path = "fuzzer/wasm/main.zig", + }; + const walk_src_path: Build.Cache.Path = .{ + .root_dir = ws.zig_lib_directory, + .sub_path = "docs/wasm/Walk.zig", + }; + + var argv: std.ArrayListUnmanaged([]const u8) = .{}; + + try argv.appendSlice(arena, &.{ + ws.zig_exe_path, + "build-exe", + "-fno-entry", + "-O", + @tagName(optimize_mode), + "-target", + "wasm32-freestanding", + "-mcpu", + "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext", + "--cache-dir", + ws.global_cache_directory.path orelse ".", + "--global-cache-dir", + ws.global_cache_directory.path orelse ".", + "--name", + "fuzzer", + "-rdynamic", + "--dep", + "Walk", + try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}), + try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}), + "--listen=-", + }); + + var child = std.process.Child.init(argv.items, gpa); + child.stdin_behavior = .Pipe; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; + try child.spawn(); + + var poller = std.io.poll(gpa, enum { stdout, stderr }, .{ + .stdout = child.stdout.?, + .stderr = child.stderr.?, + }); + defer poller.deinit(); + + try sendMessage(child.stdin.?, .update); + try sendMessage(child.stdin.?, .exit); + + const Header = std.zig.Server.Message.Header; + var result: ?[]const u8 = null; + var result_error_bundle = std.zig.ErrorBundle.empty; + + const stdout = poller.fifo(.stdout); + + poll: while (true) { + while (stdout.readableLength() < @sizeOf(Header)) { + if (!(try poller.poll())) break :poll; + } + const header = stdout.reader().readStruct(Header) catch unreachable; + while (stdout.readableLength() < header.bytes_len) { + if (!(try poller.poll())) break :poll; + } + const body = stdout.readableSliceOfLen(header.bytes_len); + + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) { + return error.ZigProtocolVersionMismatch; + } + }, + .error_bundle => { + const EbHdr = std.zig.Server.Message.ErrorBundle; + const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body)); + const extra_bytes = + body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len]; + const string_bytes = + body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len]; + // TODO: use @ptrCast when the compiler supports it + const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes); + const extra_array = try arena.alloc(u32, unaligned_extra.len); + @memcpy(extra_array, unaligned_extra); + result_error_bundle = .{ + .string_bytes = try arena.dupe(u8, string_bytes), + .extra = extra_array, + }; + }, + .emit_bin_path => { + const EbpHdr = std.zig.Server.Message.EmitBinPath; + const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body)); + if (!ebp_hdr.flags.cache_hit) { + log.info("source changes detected; rebuilt wasm component", .{}); + } + result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]); + }, + else => {}, // ignore other messages + } + + stdout.discard(body.len); + } + + const stderr = poller.fifo(.stderr); + if (stderr.readableLength() > 0) { + const owned_stderr = try stderr.toOwnedSlice(); + defer gpa.free(owned_stderr); + std.debug.print("{s}", .{owned_stderr}); + } + + // Send EOF to stdin. + child.stdin.?.close(); + child.stdin = null; + + switch (try child.wait()) { + .Exited => |code| { + if (code != 0) { + log.err( + "the following command exited with error code {d}:\n{s}", + .{ code, try Build.Step.allocPrintCmd(arena, null, argv.items) }, + ); + return error.WasmCompilationFailed; + } + }, + .Signal, .Stopped, .Unknown => { + log.err( + "the following command terminated unexpectedly:\n{s}", + .{try Build.Step.allocPrintCmd(arena, null, argv.items)}, + ); + return error.WasmCompilationFailed; + }, + } + + if (result_error_bundle.errorMessageCount() > 0) { + const color = std.zig.Color.auto; + result_error_bundle.renderToStdErr(color.renderOptions()); + log.err("the following command failed with {d} compilation errors:\n{s}", .{ + result_error_bundle.errorMessageCount(), + try Build.Step.allocPrintCmd(arena, null, argv.items), + }); + return error.WasmCompilationFailed; + } + + return result orelse { + log.err("child process failed to report result\n{s}", .{ + try Build.Step.allocPrintCmd(arena, null, argv.items), + }); + return error.WasmCompilationFailed; + }; + } + + fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 0, + }; + try file.writeAll(std.mem.asBytes(&header)); + } + + fn serveSourcesTar(ws: *WebServer, request: *std.http.Server.Request) !void { + const gpa = ws.gpa; + + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + var send_buffer: [0x4000]u8 = undefined; + var response = request.respondStreaming(.{ + .send_buffer = &send_buffer, + .respond_options = .{ + .extra_headers = &.{ + .{ .name = "content-type", .value = "application/x-tar" }, + cache_control_header, + }, + }, + }); + const w = response.writer(); + + const DedupeTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false); + var dedupe_table: DedupeTable = .{}; + defer dedupe_table.deinit(gpa); + + for (ws.fuzz_run_steps) |run_step| { + const compile_step_inputs = run_step.producer.?.step.inputs.table; + for (compile_step_inputs.keys(), compile_step_inputs.values()) |dir_path, *file_list| { + try dedupe_table.ensureUnusedCapacity(gpa, file_list.items.len); + for (file_list.items) |sub_path| { + // Special file "." means the entire directory. + if (std.mem.eql(u8, sub_path, ".")) continue; + const joined_path = try dir_path.join(arena, sub_path); + _ = dedupe_table.getOrPutAssumeCapacity(joined_path); + } + } + } + + const deduped_paths = dedupe_table.keys(); + const SortContext = struct { + pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool { + _ = this; + return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) { + .lt => true, + .gt => false, + .eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path), + }; + } + }; + std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan); + + for (deduped_paths) |joined_path| { + var file = joined_path.root_dir.handle.openFile(joined_path.sub_path, .{}) catch |err| { + log.err("failed to open {}: {s}", .{ joined_path, @errorName(err) }); + continue; + }; + defer file.close(); + + const stat = file.stat() catch |err| { + log.err("failed to stat {}: {s}", .{ joined_path, @errorName(err) }); + continue; + }; + if (stat.kind != .file) + continue; + + const padding = p: { + const remainder = stat.size % 512; + break :p if (remainder > 0) 512 - remainder else 0; + }; + + var file_header = std.tar.output.Header.init(); + file_header.typeflag = .regular; + try file_header.setPath(joined_path.root_dir.path orelse ".", joined_path.sub_path); + try file_header.setSize(stat.size); + try file_header.updateChecksum(); + try w.writeAll(std.mem.asBytes(&file_header)); + try w.writeFile(file); + try w.writeByteNTimes(0, padding); + } + + // intentionally omitting the pointless trailer + //try w.writeByteNTimes(0, 512 * 2); + try response.end(); + } + + const cache_control_header: std.http.Header = .{ + .name = "cache-control", + .value = "max-age=0, must-revalidate", + }; +}; + fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void { const gpa = run.step.owner.allocator; const stderr = std.io.getStdErr(); @@ -88,6 +508,7 @@ fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog fn fuzzWorkerRun( run: *Step.Run, + web_server: *WebServer, unit_test_index: u32, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node, @@ -98,7 +519,7 @@ fn fuzzWorkerRun( const prog_node = parent_prog_node.start(test_name, 0); defer prog_node.end(); - run.rerunInFuzzMode(unit_test_index, prog_node) catch |err| switch (err) { + run.rerunInFuzzMode(web_server, unit_test_index, prog_node) catch |err| switch (err) { error.MakeFailed => { const stderr = std.io.getStdErr(); std.debug.lockStdErr(); diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 8f3236d867..47a6e49a82 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -559,7 +559,8 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 { }, .zig_lib => zl: { if (s.cast(Step.Compile)) |compile| { - if (compile.zig_lib_dir) |lp| { + if (compile.zig_lib_dir) |zig_lib_dir| { + const lp = try zig_lib_dir.join(arena, sub_path); try addWatchInput(s, lp); break :zl; } diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index c2d25cd82c..e494e969f0 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -205,6 +205,7 @@ pub fn enableTestRunnerMode(run: *Run) void { run.stdio = .zig_test; run.addArgs(&.{ std.fmt.allocPrint(arena, "--seed=0x{x}", .{b.graph.random_seed}) catch @panic("OOM"), + std.fmt.allocPrint(arena, "--cache-dir={s}", .{b.cache_root.path orelse ""}) catch @panic("OOM"), "--listen=-", }); } @@ -845,7 +846,12 @@ fn make(step: *Step, options: Step.MakeOptions) !void { ); } -pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress.Node) !void { +pub fn rerunInFuzzMode( + run: *Run, + web_server: *std.Build.Fuzz.WebServer, + unit_test_index: u32, + prog_node: std.Progress.Node, +) !void { const step = &run.step; const b = step.owner; const arena = b.allocator; @@ -877,7 +883,10 @@ pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress. const has_side_effects = false; const rand_int = std.crypto.random.int(u64); const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); - try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, unit_test_index); + try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, .{ + .unit_test_index = unit_test_index, + .web_server = web_server, + }); } fn populateGeneratedPaths( @@ -952,13 +961,18 @@ fn termMatches(expected: ?std.process.Child.Term, actual: std.process.Child.Term }; } +const FuzzContext = struct { + web_server: *std.Build.Fuzz.WebServer, + unit_test_index: u32, +}; + fn runCommand( run: *Run, argv: []const []const u8, has_side_effects: bool, output_dir_path: []const u8, prog_node: std.Progress.Node, - fuzz_unit_test_index: ?u32, + fuzz_context: ?FuzzContext, ) !void { const step = &run.step; const b = step.owner; @@ -977,7 +991,7 @@ fn runCommand( var interp_argv = std.ArrayList([]const u8).init(b.allocator); defer interp_argv.deinit(); - const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node, fuzz_unit_test_index) catch |err| term: { + const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node, fuzz_context) catch |err| term: { // InvalidExe: cpu arch mismatch // FileNotFound: can happen with a wrong dynamic linker path if (err == error.InvalidExe or err == error.FileNotFound) interpret: { @@ -1113,7 +1127,7 @@ fn runCommand( try Step.handleVerbose2(step.owner, cwd, run.env_map, interp_argv.items); - break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node, fuzz_unit_test_index) catch |e| { + break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node, fuzz_context) catch |e| { if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; return step.fail("unable to spawn interpreter {s}: {s}", .{ @@ -1133,7 +1147,7 @@ fn runCommand( const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items; - if (fuzz_unit_test_index != null) { + if (fuzz_context != null) { try step.handleChildProcessTerm(result.term, cwd, final_argv); return; } @@ -1298,12 +1312,12 @@ fn spawnChildAndCollect( argv: []const []const u8, has_side_effects: bool, prog_node: std.Progress.Node, - fuzz_unit_test_index: ?u32, + fuzz_context: ?FuzzContext, ) !ChildProcResult { const b = run.step.owner; const arena = b.allocator; - if (fuzz_unit_test_index != null) { + if (fuzz_context != null) { assert(!has_side_effects); assert(run.stdio == .zig_test); } @@ -1357,7 +1371,7 @@ fn spawnChildAndCollect( var timer = try std.time.Timer.start(); const result = if (run.stdio == .zig_test) - evalZigTest(run, &child, prog_node, fuzz_unit_test_index) + evalZigTest(run, &child, prog_node, fuzz_context) else evalGeneric(run, &child); @@ -1383,7 +1397,7 @@ fn evalZigTest( run: *Run, child: *std.process.Child, prog_node: std.Progress.Node, - fuzz_unit_test_index: ?u32, + fuzz_context: ?FuzzContext, ) !StdIoResult { const gpa = run.step.owner.allocator; const arena = run.step.owner.allocator; @@ -1394,8 +1408,8 @@ fn evalZigTest( }); defer poller.deinit(); - if (fuzz_unit_test_index) |index| { - try sendRunTestMessage(child.stdin.?, .start_fuzzing, index); + if (fuzz_context) |fuzz| { + try sendRunTestMessage(child.stdin.?, .start_fuzzing, fuzz.unit_test_index); } else { run.fuzz_tests.clearRetainingCapacity(); try sendMessage(child.stdin.?, .query_test_metadata); @@ -1437,7 +1451,7 @@ fn evalZigTest( } }, .test_metadata => { - assert(fuzz_unit_test_index == null); + assert(fuzz_context == null); const TmHdr = std.zig.Server.Message.TestMetadata; const tm_hdr = @as(*align(1) const TmHdr, @ptrCast(body)); test_count = tm_hdr.tests_len; @@ -1466,7 +1480,7 @@ fn evalZigTest( try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node); }, .test_results => { - assert(fuzz_unit_test_index == null); + assert(fuzz_context == null); const md = metadata.?; const TrHdr = std.zig.Server.Message.TestResults; @@ -1500,6 +1514,16 @@ fn evalZigTest( try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node); }, + .coverage_id => { + const web_server = fuzz_context.?.web_server; + const msg_ptr: *align(1) const u64 = @ptrCast(body); + const coverage_id = msg_ptr.*; + { + web_server.mutex.lock(); + defer web_server.mutex.unlock(); + try web_server.msg_queue.append(web_server.gpa, .{ .coverage_id = coverage_id }); + } + }, else => {}, // ignore other messages } diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index f1e564d43e..93ce6cc01f 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -28,6 +28,10 @@ pub const Message = struct { /// The remaining bytes is the file path relative to that prefix. /// The prefixes are hard-coded in Compilation.create (cwd, zig lib dir, local cache dir) file_system_inputs, + /// Body is a u64le that indicates the file path within the cache used + /// to store coverage information. The integer is a hash of the PCs + /// stored within that file. + coverage_id, _, }; @@ -180,6 +184,14 @@ pub fn serveMessage( try s.out.writevAll(iovecs[0 .. bufs.len + 1]); } +pub fn serveU64Message(s: *Server, tag: OutMessage.Tag, int: u64) !void { + const msg_le = bswap(int); + return s.serveMessage(.{ + .tag = tag, + .bytes_len = @sizeOf(u64), + }, &.{std.mem.asBytes(&msg_le)}); +} + pub fn serveEmitBinPath( s: *Server, fs_path: []const u8, @@ -187,7 +199,7 @@ pub fn serveEmitBinPath( ) !void { try s.serveMessage(.{ .tag = .emit_bin_path, - .bytes_len = @as(u32, @intCast(fs_path.len + @sizeOf(OutMessage.EmitBinPath))), + .bytes_len = @intCast(fs_path.len + @sizeOf(OutMessage.EmitBinPath)), }, &.{ std.mem.asBytes(&header), fs_path, @@ -201,7 +213,7 @@ pub fn serveTestResults( const msg_le = bswap(msg); try s.serveMessage(.{ .tag = .test_results, - .bytes_len = @as(u32, @intCast(@sizeOf(OutMessage.TestResults))), + .bytes_len = @intCast(@sizeOf(OutMessage.TestResults)), }, &.{ std.mem.asBytes(&msg_le), }); @@ -209,14 +221,14 @@ pub fn serveTestResults( pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void { const eb_hdr: OutMessage.ErrorBundle = .{ - .extra_len = @as(u32, @intCast(error_bundle.extra.len)), - .string_bytes_len = @as(u32, @intCast(error_bundle.string_bytes.len)), + .extra_len = @intCast(error_bundle.extra.len), + .string_bytes_len = @intCast(error_bundle.string_bytes.len), }; const bytes_len = @sizeOf(OutMessage.ErrorBundle) + 4 * error_bundle.extra.len + error_bundle.string_bytes.len; try s.serveMessage(.{ .tag = .error_bundle, - .bytes_len = @as(u32, @intCast(bytes_len)), + .bytes_len = @intCast(bytes_len), }, &.{ std.mem.asBytes(&eb_hdr), // TODO: implement @ptrCast between slices changing the length @@ -251,7 +263,7 @@ pub fn serveTestMetadata(s: *Server, test_metadata: TestMetadata) !void { return s.serveMessage(.{ .tag = .test_metadata, - .bytes_len = @as(u32, @intCast(bytes_len)), + .bytes_len = @intCast(bytes_len), }, &.{ std.mem.asBytes(&header), // TODO: implement @ptrCast between slices changing the length diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index c375818770..b63bde5633 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -1840,3 +1840,48 @@ fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !v try std.testing.expectEqual(source.len, last_token.loc.start); try std.testing.expectEqual(source.len, last_token.loc.end); } + +test "fuzzable properties upheld" { + const source = std.testing.fuzzInput(.{}); + const source0 = try std.testing.allocator.dupeZ(u8, source); + defer std.testing.allocator.free(source0); + var tokenizer = Tokenizer.init(source0); + var tokenization_failed = false; + while (true) { + const token = tokenizer.next(); + + // Property: token end location after start location (or equal) + try std.testing.expect(token.loc.end >= token.loc.start); + + switch (token.tag) { + .invalid => { + tokenization_failed = true; + + // Property: invalid token always ends at newline or eof + try std.testing.expect(source0[token.loc.end] == '\n' or source0[token.loc.end] == 0); + }, + .eof => { + // Property: EOF token is always 0-length at end of source. + try std.testing.expectEqual(source0.len, token.loc.start); + try std.testing.expectEqual(source0.len, token.loc.end); + break; + }, + else => continue, + } + } + + if (source0.len > 0) for (source0, source0[1..][0..source0.len]) |cur, next| { + // Property: No null byte allowed except at end. + if (cur == 0) { + try std.testing.expect(tokenization_failed); + } + // Property: No ASCII control characters other than \n and \t are allowed. + if (std.ascii.isControl(cur) and cur != '\n' and cur != '\t') { + try std.testing.expect(tokenization_failed); + } + // Property: All '\r' must be followed by '\n'. + if (cur == '\r' and next != '\n') { + try std.testing.expect(tokenization_failed); + } + }; +}