From 46aa416c48c283849059292267ac25a6d0db76d6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 10 Feb 2018 20:55:13 -0500 Subject: [PATCH] std.os and std.io API update * move std.io.File to std.os.File * add `zig fmt` to self hosted compiler * introduce std.io.BufferedAtomicFile API * introduce std.os.AtomicFile API * add `std.os.default_file_mode` * change FileMode on posix from being a usize to a u32 * add std.os.File.mode to return mode of an open file * std.os.copyFile copies the mode from the source file instead of using the default file mode for the dest file * move `std.os.line_sep` to `std.cstr.line_sep` --- CMakeLists.txt | 3 +- doc/docgen.zig | 4 +- example/cat/main.zig | 4 +- src-self-hosted/main.zig | 13 +- std/build.zig | 15 +- std/cstr.zig | 7 + std/debug/index.zig | 6 +- std/elf.zig | 7 +- std/io.zig | 338 +++++++-------------------------------- std/io_test.zig | 4 +- std/os/child_process.zig | 18 +-- std/os/file.zig | 311 +++++++++++++++++++++++++++++++++++ std/os/index.zig | 159 +++++++++++++----- std/zig/parser.zig | 5 +- test/compare_output.zig | 6 +- 15 files changed, 546 insertions(+), 354 deletions(-) create mode 100644 std/os/file.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 46a1ecbb0d..bdc3465830 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -437,11 +437,12 @@ set(ZIG_STD_FILES "os/child_process.zig" "os/darwin.zig" "os/darwin_errno.zig" + "os/file.zig" "os/get_user_id.zig" "os/index.zig" - "os/linux/index.zig" "os/linux/errno.zig" "os/linux/i386.zig" + "os/linux/index.zig" "os/linux/x86_64.zig" "os/path.zig" "os/windows/error.zig" diff --git a/doc/docgen.zig b/doc/docgen.zig index 9272258cf1..f9aac826ad 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -31,10 +31,10 @@ pub fn main() !void { const out_file_name = try (args_it.next(allocator) ?? @panic("expected output arg")); defer allocator.free(out_file_name); - var in_file = try io.File.openRead(allocator, in_file_name); + var in_file = try os.File.openRead(allocator, in_file_name); defer in_file.close(); - var out_file = try io.File.openWrite(allocator, out_file_name); + var out_file = try os.File.openWrite(allocator, out_file_name); defer out_file.close(); var file_in_stream = io.FileInStream.init(&in_file); diff --git a/example/cat/main.zig b/example/cat/main.zig index 97fbada7a9..de0d323bed 100644 --- a/example/cat/main.zig +++ b/example/cat/main.zig @@ -20,7 +20,7 @@ pub fn main() !void { } else if (arg[0] == '-') { return usage(exe); } else { - var file = io.File.openRead(allocator, arg) catch |err| { + var file = os.File.openRead(allocator, arg) catch |err| { warn("Unable to open file: {}\n", @errorName(err)); return err; }; @@ -41,7 +41,7 @@ fn usage(exe: []const u8) !void { return error.Invalid; } -fn cat_file(stdout: &io.File, file: &io.File) !void { +fn cat_file(stdout: &os.File, file: &os.File) !void { var buf: [1024 * 4]u8 = undefined; while (true) { diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 2b6e20253d..4d59783098 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -562,7 +562,7 @@ fn printZen() !void { fn fmtMain(allocator: &mem.Allocator, file_paths: []const []const u8) !void { for (file_paths) |file_path| { - var file = try io.File.openRead(allocator, file_path); + var file = try os.File.openRead(allocator, file_path); defer file.close(); const source_code = io.readFileAlloc(allocator, file_path) catch |err| { @@ -574,7 +574,14 @@ fn fmtMain(allocator: &mem.Allocator, file_paths: []const []const u8) !void { var tokenizer = std.zig.Tokenizer.init(source_code); var parser = std.zig.Parser.init(&tokenizer, allocator, file_path); defer parser.deinit(); - warn("opened {} (todo tokenize and parse and render)\n", file_path); + + const tree = try parser.parse(); + defer tree.deinit(); + + const baf = try io.BufferedAtomicFile.create(allocator, file_path); + defer baf.destroy(); + + try parser.renderSource(baf.stream(), tree.root_node); } } @@ -602,7 +609,7 @@ fn testZigInstallPrefix(allocator: &mem.Allocator, test_path: []const u8) ![]u8 const test_index_file = try os.path.join(allocator, test_zig_dir, "std", "index.zig"); defer allocator.free(test_index_file); - var file = try io.File.openRead(allocator, test_index_file); + var file = try os.File.openRead(allocator, test_index_file); file.close(); return test_zig_dir; diff --git a/std/build.zig b/std/build.zig index edd91697aa..e6b6676261 100644 --- a/std/build.zig +++ b/std/build.zig @@ -624,10 +624,10 @@ pub const Builder = struct { } fn copyFile(self: &Builder, source_path: []const u8, dest_path: []const u8) !void { - return self.copyFileMode(source_path, dest_path, 0o666); + return self.copyFileMode(source_path, dest_path, os.default_file_mode); } - fn copyFileMode(self: &Builder, source_path: []const u8, dest_path: []const u8, mode: usize) !void { + fn copyFileMode(self: &Builder, source_path: []const u8, dest_path: []const u8, mode: os.FileMode) !void { if (self.verbose) { warn("cp {} {}\n", source_path, dest_path); } @@ -1833,10 +1833,13 @@ const InstallArtifactStep = struct { const self = @fieldParentPtr(Self, "step", step); const builder = self.builder; - const mode = switch (self.artifact.kind) { - LibExeObjStep.Kind.Obj => unreachable, - LibExeObjStep.Kind.Exe => usize(0o755), - LibExeObjStep.Kind.Lib => if (self.artifact.static) usize(0o666) else usize(0o755), + const mode = switch (builtin.os) { + builtin.Os.windows => {}, + else => switch (self.artifact.kind) { + LibExeObjStep.Kind.Obj => unreachable, + LibExeObjStep.Kind.Exe => u32(0o755), + LibExeObjStep.Kind.Lib => if (self.artifact.static) u32(0o666) else u32(0o755), + }, }; try builder.copyFileMode(self.artifact.getOutputPath(), self.dest_file, mode); if (self.artifact.kind == LibExeObjStep.Kind.Lib and !self.artifact.static) { diff --git a/std/cstr.zig b/std/cstr.zig index f905b57c91..d396dcbce3 100644 --- a/std/cstr.zig +++ b/std/cstr.zig @@ -1,8 +1,15 @@ const std = @import("index.zig"); +const builtin = @import("builtin"); const debug = std.debug; const mem = std.mem; const assert = debug.assert; +pub const line_sep = switch (builtin.os) { + builtin.Os.windows => "\r\n", + else => "\n", +}; + + pub fn len(ptr: &const u8) usize { var count: usize = 0; while (ptr[count] != 0) : (count += 1) {} diff --git a/std/debug/index.zig b/std/debug/index.zig index 9c27d84375..5426a197f2 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -13,7 +13,7 @@ pub const FailingAllocator = @import("failing_allocator.zig").FailingAllocator; /// Tries to write to stderr, unbuffered, and ignores any error returned. /// Does not append a newline. /// TODO atomic/multithread support -var stderr_file: io.File = undefined; +var stderr_file: os.File = undefined; var stderr_file_out_stream: io.FileOutStream = undefined; var stderr_stream: ?&io.OutStream(io.FileOutStream.Error) = null; pub fn warn(comptime fmt: []const u8, args: ...) void { @@ -265,7 +265,7 @@ pub fn openSelfDebugInfo(allocator: &mem.Allocator) !&ElfStackTrace { } fn printLineFromFile(allocator: &mem.Allocator, out_stream: var, line_info: &const LineInfo) !void { - var f = try io.File.openRead(allocator, line_info.file_name); + var f = try os.File.openRead(allocator, line_info.file_name); defer f.close(); // TODO fstat and make sure that the file has the correct size @@ -298,7 +298,7 @@ fn printLineFromFile(allocator: &mem.Allocator, out_stream: var, line_info: &con } pub const ElfStackTrace = struct { - self_exe_file: io.File, + self_exe_file: os.File, elf: elf.Elf, debug_info: &elf.SectionHeader, debug_abbrev: &elf.SectionHeader, diff --git a/std/elf.zig b/std/elf.zig index 2e139870b1..7e20fa000f 100644 --- a/std/elf.zig +++ b/std/elf.zig @@ -1,6 +1,7 @@ const builtin = @import("builtin"); const std = @import("index.zig"); const io = std.io; +const os = std.os; const math = std.math; const mem = std.mem; const debug = std.debug; @@ -63,7 +64,7 @@ pub const SectionHeader = struct { }; pub const Elf = struct { - in_file: &io.File, + in_file: &os.File, auto_close_stream: bool, is_64: bool, endian: builtin.Endian, @@ -76,7 +77,7 @@ pub const Elf = struct { string_section: &SectionHeader, section_headers: []SectionHeader, allocator: &mem.Allocator, - prealloc_file: io.File, + prealloc_file: os.File, /// Call close when done. pub fn openPath(elf: &Elf, allocator: &mem.Allocator, path: []const u8) !void { @@ -86,7 +87,7 @@ pub const Elf = struct { } /// Call close when done. - pub fn openFile(elf: &Elf, allocator: &mem.Allocator, file: &io.File) !void { + pub fn openFile(elf: &Elf, allocator: &mem.Allocator, file: &os.File) !void { elf.allocator = allocator; elf.in_file = file; elf.auto_close_stream = false; diff --git a/std/io.zig b/std/io.zig index 7457416b29..94685c4d03 100644 --- a/std/io.zig +++ b/std/io.zig @@ -1,12 +1,6 @@ const std = @import("index.zig"); const builtin = @import("builtin"); const Os = builtin.Os; -const system = switch(builtin.os) { - Os.linux => @import("os/linux/index.zig"), - Os.macosx, Os.ios => @import("os/darwin.zig"), - Os.windows => @import("os/windows/index.zig"), - else => @compileError("Unsupported OS"), -}; const c = std.c; const math = std.math; @@ -16,23 +10,18 @@ const os = std.os; const mem = std.mem; const Buffer = std.Buffer; const fmt = std.fmt; +const File = std.os.File; const is_posix = builtin.os != builtin.Os.windows; const is_windows = builtin.os == builtin.Os.windows; -test "import io tests" { - comptime { - _ = @import("io_test.zig"); - } -} - const GetStdIoErrs = os.WindowsGetStdHandleErrs; pub fn getStdErr() GetStdIoErrs!File { const handle = if (is_windows) - try os.windowsGetStdHandle(system.STD_ERROR_HANDLE) + try os.windowsGetStdHandle(os.windows.STD_ERROR_HANDLE) else if (is_posix) - system.STDERR_FILENO + os.posix.STDERR_FILENO else unreachable; return File.openHandle(handle); @@ -40,9 +29,9 @@ pub fn getStdErr() GetStdIoErrs!File { pub fn getStdOut() GetStdIoErrs!File { const handle = if (is_windows) - try os.windowsGetStdHandle(system.STD_OUTPUT_HANDLE) + try os.windowsGetStdHandle(os.windows.STD_OUTPUT_HANDLE) else if (is_posix) - system.STDOUT_FILENO + os.posix.STDOUT_FILENO else unreachable; return File.openHandle(handle); @@ -50,9 +39,9 @@ pub fn getStdOut() GetStdIoErrs!File { pub fn getStdIn() GetStdIoErrs!File { const handle = if (is_windows) - try os.windowsGetStdHandle(system.STD_INPUT_HANDLE) + try os.windowsGetStdHandle(os.windows.STD_INPUT_HANDLE) else if (is_posix) - system.STDIN_FILENO + os.posix.STDIN_FILENO else unreachable; return File.openHandle(handle); @@ -104,260 +93,10 @@ pub const FileOutStream = struct { } }; -pub const File = struct { - /// The OS-specific file descriptor or file handle. - handle: os.FileHandle, - - const OpenError = os.WindowsOpenError || os.PosixOpenError; - - /// `path` needs to be copied in memory to add a null terminating byte, hence the allocator. - /// Call close to clean up. - pub fn openRead(allocator: &mem.Allocator, path: []const u8) OpenError!File { - if (is_posix) { - const flags = system.O_LARGEFILE|system.O_RDONLY; - const fd = try os.posixOpen(allocator, path, flags, 0); - return openHandle(fd); - } else if (is_windows) { - const handle = try os.windowsOpen(allocator, path, system.GENERIC_READ, system.FILE_SHARE_READ, - system.OPEN_EXISTING, system.FILE_ATTRIBUTE_NORMAL); - return openHandle(handle); - } else { - unreachable; - } - } - - /// Calls `openWriteMode` with 0o666 for the mode. - pub fn openWrite(allocator: &mem.Allocator, path: []const u8) !File { - return openWriteMode(allocator, path, 0o666); - - } - - /// `path` needs to be copied in memory to add a null terminating byte, hence the allocator. - /// Call close to clean up. - pub fn openWriteMode(allocator: &mem.Allocator, path: []const u8, mode: usize) !File { - if (is_posix) { - const flags = system.O_LARGEFILE|system.O_WRONLY|system.O_CREAT|system.O_CLOEXEC|system.O_TRUNC; - const fd = try os.posixOpen(allocator, path, flags, mode); - return openHandle(fd); - } else if (is_windows) { - const handle = try os.windowsOpen(allocator, path, system.GENERIC_WRITE, - system.FILE_SHARE_WRITE|system.FILE_SHARE_READ|system.FILE_SHARE_DELETE, - system.CREATE_ALWAYS, system.FILE_ATTRIBUTE_NORMAL); - return openHandle(handle); - } else { - unreachable; - } - - } - - pub fn openHandle(handle: os.FileHandle) File { - return File { - .handle = handle, - }; - } - - - /// Upon success, the stream is in an uninitialized state. To continue using it, - /// you must use the open() function. - pub fn close(self: &File) void { - os.close(self.handle); - self.handle = undefined; - } - - /// Calls `os.isTty` on `self.handle`. - pub fn isTty(self: &File) bool { - return os.isTty(self.handle); - } - - pub fn seekForward(self: &File, amount: isize) !void { - switch (builtin.os) { - Os.linux, Os.macosx, Os.ios => { - const result = system.lseek(self.handle, amount, system.SEEK_CUR); - const err = system.getErrno(result); - if (err > 0) { - return switch (err) { - system.EBADF => error.BadFd, - system.EINVAL => error.Unseekable, - system.EOVERFLOW => error.Unseekable, - system.ESPIPE => error.Unseekable, - system.ENXIO => error.Unseekable, - else => os.unexpectedErrorPosix(err), - }; - } - }, - Os.windows => { - if (system.SetFilePointerEx(self.handle, amount, null, system.FILE_CURRENT) == 0) { - const err = system.GetLastError(); - return switch (err) { - system.ERROR.INVALID_PARAMETER => error.BadFd, - else => os.unexpectedErrorWindows(err), - }; - } - }, - else => @compileError("unsupported OS"), - } - } - - pub fn seekTo(self: &File, pos: usize) !void { - switch (builtin.os) { - Os.linux, Os.macosx, Os.ios => { - const ipos = try math.cast(isize, pos); - const result = system.lseek(self.handle, ipos, system.SEEK_SET); - const err = system.getErrno(result); - if (err > 0) { - return switch (err) { - system.EBADF => error.BadFd, - system.EINVAL => error.Unseekable, - system.EOVERFLOW => error.Unseekable, - system.ESPIPE => error.Unseekable, - system.ENXIO => error.Unseekable, - else => os.unexpectedErrorPosix(err), - }; - } - }, - Os.windows => { - const ipos = try math.cast(isize, pos); - if (system.SetFilePointerEx(self.handle, ipos, null, system.FILE_BEGIN) == 0) { - const err = system.GetLastError(); - return switch (err) { - system.ERROR.INVALID_PARAMETER => error.BadFd, - else => os.unexpectedErrorWindows(err), - }; - } - }, - else => @compileError("unsupported OS: " ++ @tagName(builtin.os)), - } - } - - pub fn getPos(self: &File) !usize { - switch (builtin.os) { - Os.linux, Os.macosx, Os.ios => { - const result = system.lseek(self.handle, 0, system.SEEK_CUR); - const err = system.getErrno(result); - if (err > 0) { - return switch (err) { - system.EBADF => error.BadFd, - system.EINVAL => error.Unseekable, - system.EOVERFLOW => error.Unseekable, - system.ESPIPE => error.Unseekable, - system.ENXIO => error.Unseekable, - else => os.unexpectedErrorPosix(err), - }; - } - return result; - }, - Os.windows => { - var pos : system.LARGE_INTEGER = undefined; - if (system.SetFilePointerEx(self.handle, 0, &pos, system.FILE_CURRENT) == 0) { - const err = system.GetLastError(); - return switch (err) { - system.ERROR.INVALID_PARAMETER => error.BadFd, - else => os.unexpectedErrorWindows(err), - }; - } - - assert(pos >= 0); - if (@sizeOf(@typeOf(pos)) > @sizeOf(usize)) { - if (pos > @maxValue(usize)) { - return error.FilePosLargerThanPointerRange; - } - } - - return usize(pos); - }, - else => @compileError("unsupported OS"), - } - } - - pub fn getEndPos(self: &File) !usize { - if (is_posix) { - var stat: system.Stat = undefined; - const err = system.getErrno(system.fstat(self.handle, &stat)); - if (err > 0) { - return switch (err) { - system.EBADF => error.BadFd, - system.ENOMEM => error.SystemResources, - else => os.unexpectedErrorPosix(err), - }; - } - - return usize(stat.size); - } else if (is_windows) { - var file_size: system.LARGE_INTEGER = undefined; - if (system.GetFileSizeEx(self.handle, &file_size) == 0) { - const err = system.GetLastError(); - return switch (err) { - else => os.unexpectedErrorWindows(err), - }; - } - if (file_size < 0) - return error.Overflow; - return math.cast(usize, u64(file_size)); - } else { - unreachable; - } - } - - pub const ReadError = error {}; - - pub fn read(self: &File, buffer: []u8) !usize { - if (is_posix) { - var index: usize = 0; - while (index < buffer.len) { - const amt_read = system.read(self.handle, &buffer[index], buffer.len - index); - const read_err = system.getErrno(amt_read); - if (read_err > 0) { - switch (read_err) { - system.EINTR => continue, - system.EINVAL => unreachable, - system.EFAULT => unreachable, - system.EBADF => return error.BadFd, - system.EIO => return error.Io, - else => return os.unexpectedErrorPosix(read_err), - } - } - if (amt_read == 0) return index; - index += amt_read; - } - return index; - } else if (is_windows) { - var index: usize = 0; - while (index < buffer.len) { - const want_read_count = system.DWORD(math.min(system.DWORD(@maxValue(system.DWORD)), buffer.len - index)); - var amt_read: system.DWORD = undefined; - if (system.ReadFile(self.handle, @ptrCast(&c_void, &buffer[index]), want_read_count, &amt_read, null) == 0) { - const err = system.GetLastError(); - return switch (err) { - system.ERROR.OPERATION_ABORTED => continue, - system.ERROR.BROKEN_PIPE => return index, - else => os.unexpectedErrorWindows(err), - }; - } - if (amt_read == 0) return index; - index += amt_read; - } - return index; - } else { - unreachable; - } - } - - pub const WriteError = os.WindowsWriteError || os.PosixWriteError; - - fn write(self: &File, bytes: []const u8) WriteError!void { - if (is_posix) { - try os.posixWrite(self.handle, bytes); - } else if (is_windows) { - try os.windowsWrite(self.handle, bytes); - } else { - @compileError("Unsupported OS"); - } - } -}; - -pub fn InStream(comptime Error: type) type { +pub fn InStream(comptime ReadError: type) type { return struct { const Self = this; + pub const Error = ReadError; /// Return the number of bytes read. If the number read is smaller than buf.len, it /// means the stream reached the end. Reaching the end of a stream is not an error @@ -486,9 +225,10 @@ pub fn InStream(comptime Error: type) type { }; } -pub fn OutStream(comptime Error: type) type { +pub fn OutStream(comptime WriteError: type) type { return struct { const Self = this; + pub const Error = WriteError; writeFn: fn(self: &Self, bytes: []const u8) Error!void, @@ -614,10 +354,11 @@ pub fn BufferedOutStream(comptime Error: type) type { return BufferedOutStreamCustom(os.page_size, Error); } -pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime Error: type) type { +pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamError: type) type { return struct { const Self = this; - const Stream = OutStream(Error); + pub const Stream = OutStream(Error); + pub const Error = OutStreamError; pub stream: Stream, @@ -638,9 +379,6 @@ pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime Error: type } pub fn flush(self: &Self) !void { - if (self.index == 0) - return; - try self.unbuffered_out_stream.write(self.buffer[0..self.index]); self.index = 0; } @@ -692,3 +430,51 @@ pub const BufferOutStream = struct { } }; + +pub const BufferedAtomicFile = struct { + atomic_file: os.AtomicFile, + file_stream: FileOutStream, + buffered_stream: BufferedOutStream(FileOutStream.Error), + + pub fn create(allocator: &mem.Allocator, dest_path: []const u8) !&BufferedAtomicFile { + // TODO with well defined copy elision we don't need this allocation + var self = try allocator.create(BufferedAtomicFile); + errdefer allocator.destroy(self); + + *self = BufferedAtomicFile { + .atomic_file = undefined, + .file_stream = undefined, + .buffered_stream = undefined, + }; + + self.atomic_file = try os.AtomicFile.init(allocator, dest_path, os.default_file_mode); + errdefer self.atomic_file.deinit(); + + self.file_stream = FileOutStream.init(&self.atomic_file.file); + self.buffered_stream = BufferedOutStream(FileOutStream.Error).init(&self.file_stream.stream); + return self; + } + + /// always call destroy, even after successful finish() + pub fn destroy(self: &BufferedAtomicFile) void { + const allocator = self.atomic_file.allocator; + self.atomic_file.deinit(); + allocator.destroy(self); + } + + pub fn finish(self: &BufferedAtomicFile) !void { + try self.buffered_stream.flush(); + try self.atomic_file.finish(); + } + + pub fn stream(self: &BufferedAtomicFile) &OutStream(FileOutStream.Error) { + return &self.buffered_stream.stream; + } +}; + +test "import io tests" { + comptime { + _ = @import("io_test.zig"); + } +} + diff --git a/std/io_test.zig b/std/io_test.zig index 7cf67a51f2..993ec84d20 100644 --- a/std/io_test.zig +++ b/std/io_test.zig @@ -13,7 +13,7 @@ test "write a file, read it, then delete it" { rng.fillBytes(data[0..]); const tmp_file_name = "temp_test_file.txt"; { - var file = try io.File.openWrite(allocator, tmp_file_name); + var file = try os.File.openWrite(allocator, tmp_file_name); defer file.close(); var file_out_stream = io.FileOutStream.init(&file); @@ -25,7 +25,7 @@ test "write a file, read it, then delete it" { try buf_stream.flush(); } { - var file = try io.File.openRead(allocator, tmp_file_name); + var file = try os.File.openRead(allocator, tmp_file_name); defer file.close(); const file_size = try file.getEndPos(); diff --git a/std/os/child_process.zig b/std/os/child_process.zig index 7b0be8ef82..c85202c9ed 100644 --- a/std/os/child_process.zig +++ b/std/os/child_process.zig @@ -24,9 +24,9 @@ pub const ChildProcess = struct { pub allocator: &mem.Allocator, - pub stdin: ?io.File, - pub stdout: ?io.File, - pub stderr: ?io.File, + pub stdin: ?os.File, + pub stdout: ?os.File, + pub stderr: ?os.File, pub term: ?(SpawnError!Term), @@ -428,17 +428,17 @@ pub const ChildProcess = struct { // we are the parent const pid = i32(pid_result); if (self.stdin_behavior == StdIo.Pipe) { - self.stdin = io.File.openHandle(stdin_pipe[1]); + self.stdin = os.File.openHandle(stdin_pipe[1]); } else { self.stdin = null; } if (self.stdout_behavior == StdIo.Pipe) { - self.stdout = io.File.openHandle(stdout_pipe[0]); + self.stdout = os.File.openHandle(stdout_pipe[0]); } else { self.stdout = null; } if (self.stderr_behavior == StdIo.Pipe) { - self.stderr = io.File.openHandle(stderr_pipe[0]); + self.stderr = os.File.openHandle(stderr_pipe[0]); } else { self.stderr = null; } @@ -620,17 +620,17 @@ pub const ChildProcess = struct { }; if (g_hChildStd_IN_Wr) |h| { - self.stdin = io.File.openHandle(h); + self.stdin = os.File.openHandle(h); } else { self.stdin = null; } if (g_hChildStd_OUT_Rd) |h| { - self.stdout = io.File.openHandle(h); + self.stdout = os.File.openHandle(h); } else { self.stdout = null; } if (g_hChildStd_ERR_Rd) |h| { - self.stderr = io.File.openHandle(h); + self.stderr = os.File.openHandle(h); } else { self.stderr = null; } diff --git a/std/os/file.zig b/std/os/file.zig new file mode 100644 index 0000000000..772fbf7c73 --- /dev/null +++ b/std/os/file.zig @@ -0,0 +1,311 @@ +const std = @import("../index.zig"); +const builtin = @import("builtin"); +const os = std.os; +const mem = std.mem; +const math = std.math; +const assert = std.debug.assert; +const posix = os.posix; +const windows = os.windows; +const Os = builtin.Os; + +const is_posix = builtin.os != builtin.Os.windows; +const is_windows = builtin.os == builtin.Os.windows; + +pub const File = struct { + /// The OS-specific file descriptor or file handle. + handle: os.FileHandle, + + const OpenError = os.WindowsOpenError || os.PosixOpenError; + + /// `path` needs to be copied in memory to add a null terminating byte, hence the allocator. + /// Call close to clean up. + pub fn openRead(allocator: &mem.Allocator, path: []const u8) OpenError!File { + if (is_posix) { + const flags = posix.O_LARGEFILE|posix.O_RDONLY; + const fd = try os.posixOpen(allocator, path, flags, 0); + return openHandle(fd); + } else if (is_windows) { + const handle = try os.windowsOpen(allocator, path, windows.GENERIC_READ, windows.FILE_SHARE_READ, + windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL); + return openHandle(handle); + } else { + @compileError("TODO implement openRead for this OS"); + } + } + + /// Calls `openWriteMode` with os.default_file_mode for the mode. + pub fn openWrite(allocator: &mem.Allocator, path: []const u8) OpenError!File { + return openWriteMode(allocator, path, os.default_file_mode); + + } + + /// If the path does not exist it will be created. + /// If a file already exists in the destination it will be truncated. + /// `path` needs to be copied in memory to add a null terminating byte, hence the allocator. + /// Call close to clean up. + pub fn openWriteMode(allocator: &mem.Allocator, path: []const u8, file_mode: os.FileMode) OpenError!File { + if (is_posix) { + const flags = posix.O_LARGEFILE|posix.O_WRONLY|posix.O_CREAT|posix.O_CLOEXEC|posix.O_TRUNC; + const fd = try os.posixOpen(allocator, path, flags, file_mode); + return openHandle(fd); + } else if (is_windows) { + const handle = try os.windowsOpen(allocator, path, windows.GENERIC_WRITE, + windows.FILE_SHARE_WRITE|windows.FILE_SHARE_READ|windows.FILE_SHARE_DELETE, + windows.CREATE_ALWAYS, windows.FILE_ATTRIBUTE_NORMAL); + return openHandle(handle); + } else { + @compileError("TODO implement openWriteMode for this OS"); + } + + } + + /// If the path does not exist it will be created. + /// If a file already exists in the destination this returns OpenError.PathAlreadyExists + /// `path` needs to be copied in memory to add a null terminating byte, hence the allocator. + /// Call close to clean up. + pub fn openWriteNoClobber(allocator: &mem.Allocator, path: []const u8, file_mode: os.FileMode) OpenError!File { + if (is_posix) { + const flags = posix.O_LARGEFILE|posix.O_WRONLY|posix.O_CREAT|posix.O_CLOEXEC|posix.O_EXCL; + const fd = try os.posixOpen(allocator, path, flags, file_mode); + return openHandle(fd); + } else if (is_windows) { + const handle = try os.windowsOpen(allocator, path, windows.GENERIC_WRITE, + windows.FILE_SHARE_WRITE|windows.FILE_SHARE_READ|windows.FILE_SHARE_DELETE, + windows.CREATE_NEW, windows.FILE_ATTRIBUTE_NORMAL); + return openHandle(handle); + } else { + @compileError("TODO implement openWriteMode for this OS"); + } + + } + + pub fn openHandle(handle: os.FileHandle) File { + return File { + .handle = handle, + }; + } + + + /// Upon success, the stream is in an uninitialized state. To continue using it, + /// you must use the open() function. + pub fn close(self: &File) void { + os.close(self.handle); + self.handle = undefined; + } + + /// Calls `os.isTty` on `self.handle`. + pub fn isTty(self: &File) bool { + return os.isTty(self.handle); + } + + pub fn seekForward(self: &File, amount: isize) !void { + switch (builtin.os) { + Os.linux, Os.macosx, Os.ios => { + const result = posix.lseek(self.handle, amount, posix.SEEK_CUR); + const err = posix.getErrno(result); + if (err > 0) { + return switch (err) { + posix.EBADF => error.BadFd, + posix.EINVAL => error.Unseekable, + posix.EOVERFLOW => error.Unseekable, + posix.ESPIPE => error.Unseekable, + posix.ENXIO => error.Unseekable, + else => os.unexpectedErrorPosix(err), + }; + } + }, + Os.windows => { + if (windows.SetFilePointerEx(self.handle, amount, null, windows.FILE_CURRENT) == 0) { + const err = windows.GetLastError(); + return switch (err) { + windows.ERROR.INVALID_PARAMETER => error.BadFd, + else => os.unexpectedErrorWindows(err), + }; + } + }, + else => @compileError("unsupported OS"), + } + } + + pub fn seekTo(self: &File, pos: usize) !void { + switch (builtin.os) { + Os.linux, Os.macosx, Os.ios => { + const ipos = try math.cast(isize, pos); + const result = posix.lseek(self.handle, ipos, posix.SEEK_SET); + const err = posix.getErrno(result); + if (err > 0) { + return switch (err) { + posix.EBADF => error.BadFd, + posix.EINVAL => error.Unseekable, + posix.EOVERFLOW => error.Unseekable, + posix.ESPIPE => error.Unseekable, + posix.ENXIO => error.Unseekable, + else => os.unexpectedErrorPosix(err), + }; + } + }, + Os.windows => { + const ipos = try math.cast(isize, pos); + if (windows.SetFilePointerEx(self.handle, ipos, null, windows.FILE_BEGIN) == 0) { + const err = windows.GetLastError(); + return switch (err) { + windows.ERROR.INVALID_PARAMETER => error.BadFd, + else => os.unexpectedErrorWindows(err), + }; + } + }, + else => @compileError("unsupported OS: " ++ @tagName(builtin.os)), + } + } + + pub fn getPos(self: &File) !usize { + switch (builtin.os) { + Os.linux, Os.macosx, Os.ios => { + const result = posix.lseek(self.handle, 0, posix.SEEK_CUR); + const err = posix.getErrno(result); + if (err > 0) { + return switch (err) { + posix.EBADF => error.BadFd, + posix.EINVAL => error.Unseekable, + posix.EOVERFLOW => error.Unseekable, + posix.ESPIPE => error.Unseekable, + posix.ENXIO => error.Unseekable, + else => os.unexpectedErrorPosix(err), + }; + } + return result; + }, + Os.windows => { + var pos : windows.LARGE_INTEGER = undefined; + if (windows.SetFilePointerEx(self.handle, 0, &pos, windows.FILE_CURRENT) == 0) { + const err = windows.GetLastError(); + return switch (err) { + windows.ERROR.INVALID_PARAMETER => error.BadFd, + else => os.unexpectedErrorWindows(err), + }; + } + + assert(pos >= 0); + if (@sizeOf(@typeOf(pos)) > @sizeOf(usize)) { + if (pos > @maxValue(usize)) { + return error.FilePosLargerThanPointerRange; + } + } + + return usize(pos); + }, + else => @compileError("unsupported OS"), + } + } + + pub fn getEndPos(self: &File) !usize { + if (is_posix) { + var stat: posix.Stat = undefined; + const err = posix.getErrno(posix.fstat(self.handle, &stat)); + if (err > 0) { + return switch (err) { + posix.EBADF => error.BadFd, + posix.ENOMEM => error.SystemResources, + else => os.unexpectedErrorPosix(err), + }; + } + + return usize(stat.size); + } else if (is_windows) { + var file_size: windows.LARGE_INTEGER = undefined; + if (windows.GetFileSizeEx(self.handle, &file_size) == 0) { + const err = windows.GetLastError(); + return switch (err) { + else => os.unexpectedErrorWindows(err), + }; + } + if (file_size < 0) + return error.Overflow; + return math.cast(usize, u64(file_size)); + } else { + @compileError("TODO support getEndPos on this OS"); + } + } + + pub const ModeError = error { + BadFd, + SystemResources, + Unexpected, + }; + + fn mode(self: &File) ModeError!FileMode { + if (is_posix) { + var stat: posix.Stat = undefined; + const err = posix.getErrno(posix.fstat(self.handle, &stat)); + if (err > 0) { + return switch (err) { + posix.EBADF => error.BadFd, + posix.ENOMEM => error.SystemResources, + else => os.unexpectedErrorPosix(err), + }; + } + + return stat.mode; + } else if (is_windows) { + return {}; + } else { + @compileError("TODO support file mode on this OS"); + } + } + + pub const ReadError = error {}; + + pub fn read(self: &File, buffer: []u8) !usize { + if (is_posix) { + var index: usize = 0; + while (index < buffer.len) { + const amt_read = posix.read(self.handle, &buffer[index], buffer.len - index); + const read_err = posix.getErrno(amt_read); + if (read_err > 0) { + switch (read_err) { + posix.EINTR => continue, + posix.EINVAL => unreachable, + posix.EFAULT => unreachable, + posix.EBADF => return error.BadFd, + posix.EIO => return error.Io, + else => return os.unexpectedErrorPosix(read_err), + } + } + if (amt_read == 0) return index; + index += amt_read; + } + return index; + } else if (is_windows) { + var index: usize = 0; + while (index < buffer.len) { + const want_read_count = windows.DWORD(math.min(windows.DWORD(@maxValue(windows.DWORD)), buffer.len - index)); + var amt_read: windows.DWORD = undefined; + if (windows.ReadFile(self.handle, @ptrCast(&c_void, &buffer[index]), want_read_count, &amt_read, null) == 0) { + const err = windows.GetLastError(); + return switch (err) { + windows.ERROR.OPERATION_ABORTED => continue, + windows.ERROR.BROKEN_PIPE => return index, + else => os.unexpectedErrorWindows(err), + }; + } + if (amt_read == 0) return index; + index += amt_read; + } + return index; + } else { + unreachable; + } + } + + pub const WriteError = os.WindowsWriteError || os.PosixWriteError; + + fn write(self: &File, bytes: []const u8) WriteError!void { + if (is_posix) { + try os.posixWrite(self.handle, bytes); + } else if (is_windows) { + try os.windowsWrite(self.handle, bytes); + } else { + @compileError("Unsupported OS"); + } + } +}; diff --git a/std/os/index.zig b/std/os/index.zig index b8d5f073e0..1d31d6893e 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -17,10 +17,16 @@ pub const posix = switch(builtin.os) { pub const ChildProcess = @import("child_process.zig").ChildProcess; pub const path = @import("path.zig"); +pub const File = @import("file.zig").File; -pub const line_sep = switch (builtin.os) { - Os.windows => "\r\n", - else => "\n", +pub const FileMode = switch (builtin.os) { + Os.windows => void, + else => u32, +}; + +pub const default_file_mode = switch (builtin.os) { + Os.windows => {}, + else => 0o666, }; pub const page_size = 4 * 1024; @@ -672,27 +678,27 @@ const b64_fs_encoder = base64.Base64Encoder.init( pub fn atomicSymLink(allocator: &Allocator, existing_path: []const u8, new_path: []const u8) !void { if (symLink(allocator, existing_path, new_path)) { return; - } else |err| { - if (err != error.PathAlreadyExists) { - return err; - } + } else |err| switch (err) { + error.PathAlreadyExists => {}, + else => return err, // TODO zig should know this set does not include PathAlreadyExists } + const dirname = os.path.dirname(new_path); + var rand_buf: [12]u8 = undefined; - const tmp_path = try allocator.alloc(u8, new_path.len + base64.Base64Encoder.calcSize(rand_buf.len)); + const tmp_path = try allocator.alloc(u8, dirname.len + 1 + base64.Base64Encoder.calcSize(rand_buf.len)); defer allocator.free(tmp_path); - mem.copy(u8, tmp_path[0..], new_path); + mem.copy(u8, tmp_path[0..], dirname); + tmp_path[dirname.len] = os.path.sep; while (true) { try getRandomBytes(rand_buf[0..]); - b64_fs_encoder.encode(tmp_path[new_path.len..], rand_buf); + b64_fs_encoder.encode(tmp_path[dirname.len + 1 ..], rand_buf); + if (symLink(allocator, existing_path, tmp_path)) { return rename(allocator, tmp_path, new_path); - } else |err| { - if (err == error.PathAlreadyExists) { - continue; - } else { - return err; - } + } else |err| switch (err) { + error.PathAlreadyExists => continue, + else => return err, // TODO zig should know this set does not include PathAlreadyExists } } @@ -750,37 +756,108 @@ pub fn deleteFilePosix(allocator: &Allocator, file_path: []const u8) !void { } } -/// Calls ::copyFileMode with 0o666 for the mode. +/// Guaranteed to be atomic. However until https://patchwork.kernel.org/patch/9636735/ is +/// merged and readily available, +/// there is a possibility of power loss or application termination leaving temporary files present +/// in the same directory as dest_path. +/// Destination file will have the same mode as the source file. pub fn copyFile(allocator: &Allocator, source_path: []const u8, dest_path: []const u8) !void { - return copyFileMode(allocator, source_path, dest_path, 0o666); -} - -// TODO instead of accepting a mode argument, use the mode from fstat'ing the source path once open -/// Guaranteed to be atomic. -pub fn copyFileMode(allocator: &Allocator, source_path: []const u8, dest_path: []const u8, mode: usize) !void { - var rand_buf: [12]u8 = undefined; - const tmp_path = try allocator.alloc(u8, dest_path.len + base64.Base64Encoder.calcSize(rand_buf.len)); - defer allocator.free(tmp_path); - mem.copy(u8, tmp_path[0..], dest_path); - try getRandomBytes(rand_buf[0..]); - b64_fs_encoder.encode(tmp_path[dest_path.len..], rand_buf); - - var out_file = try io.File.openWriteMode(allocator, tmp_path, mode); - defer out_file.close(); - errdefer _ = deleteFile(allocator, tmp_path); - - var in_file = try io.File.openRead(allocator, source_path); + var in_file = try os.File.openRead(allocator, source_path); defer in_file.close(); + const mode = try in_file.mode(); + + var atomic_file = try AtomicFile.init(allocator, dest_path, mode); + defer atomic_file.deinit(); + var buf: [page_size]u8 = undefined; while (true) { const amt = try in_file.read(buf[0..]); - try out_file.write(buf[0..amt]); - if (amt != buf.len) - return rename(allocator, tmp_path, dest_path); + try atomic_file.file.write(buf[0..amt]); + if (amt != buf.len) { + return atomic_file.finish(); + } } } +/// Guaranteed to be atomic. However until https://patchwork.kernel.org/patch/9636735/ is +/// merged and readily available, +/// there is a possibility of power loss or application termination leaving temporary files present +pub fn copyFileMode(allocator: &Allocator, source_path: []const u8, dest_path: []const u8, mode: FileMode) !void { + var in_file = try os.File.openRead(allocator, source_path); + defer in_file.close(); + + var atomic_file = try AtomicFile.init(allocator, dest_path, mode); + defer atomic_file.deinit(); + + var buf: [page_size]u8 = undefined; + while (true) { + const amt = try in_file.read(buf[0..]); + try atomic_file.file.write(buf[0..amt]); + if (amt != buf.len) { + return atomic_file.finish(); + } + } +} + +pub const AtomicFile = struct { + allocator: &Allocator, + file: os.File, + tmp_path: []u8, + dest_path: []const u8, + finished: bool, + + /// dest_path must remain valid for the lifetime of AtomicFile + /// call finish to atomically replace dest_path with contents + pub fn init(allocator: &Allocator, dest_path: []const u8, mode: FileMode) !AtomicFile { + const dirname = os.path.dirname(dest_path); + + var rand_buf: [12]u8 = undefined; + const tmp_path = try allocator.alloc(u8, dirname.len + 1 + base64.Base64Encoder.calcSize(rand_buf.len)); + errdefer allocator.free(tmp_path); + mem.copy(u8, tmp_path[0..], dirname); + tmp_path[dirname.len] = os.path.sep; + + while (true) { + try getRandomBytes(rand_buf[0..]); + b64_fs_encoder.encode(tmp_path[dirname.len + 1 ..], rand_buf); + + const file = os.File.openWriteNoClobber(allocator, tmp_path, mode) catch |err| switch (err) { + error.PathAlreadyExists => continue, + // TODO zig should figure out that this error set does not include PathAlreadyExists since + // it is handled in the above switch + else => return err, + }; + + return AtomicFile { + .allocator = allocator, + .file = file, + .tmp_path = tmp_path, + .dest_path = dest_path, + .finished = false, + }; + } + } + + /// always call deinit, even after successful finish() + pub fn deinit(self: &AtomicFile) void { + if (!self.finished) { + self.file.close(); + deleteFile(self.allocator, self.tmp_path) catch {}; + self.allocator.free(self.tmp_path); + self.finished = true; + } + } + + pub fn finish(self: &AtomicFile) !void { + assert(!self.finished); + self.file.close(); + try rename(self.allocator, self.tmp_path, self.dest_path); + self.allocator.free(self.tmp_path); + self.finished = true; + } +}; + pub fn rename(allocator: &Allocator, old_path: []const u8, new_path: []const u8) !void { const full_buf = try allocator.alloc(u8, old_path.len + new_path.len + 2); defer allocator.free(full_buf); @@ -1620,19 +1697,19 @@ pub fn unexpectedErrorWindows(err: windows.DWORD) (error{Unexpected}) { return error.Unexpected; } -pub fn openSelfExe() !io.File { +pub fn openSelfExe() !os.File { switch (builtin.os) { Os.linux => { const proc_file_path = "/proc/self/exe"; var fixed_buffer_mem: [proc_file_path.len + 1]u8 = undefined; var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]); - return io.File.openRead(&fixed_allocator.allocator, proc_file_path); + return os.File.openRead(&fixed_allocator.allocator, proc_file_path); }, Os.macosx, Os.ios => { var fixed_buffer_mem: [darwin.PATH_MAX * 2]u8 = undefined; var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]); const self_exe_path = try selfExePath(&fixed_allocator.allocator); - return io.File.openRead(&fixed_allocator.allocator, self_exe_path); + return os.File.openRead(&fixed_allocator.allocator, self_exe_path); }, else => @compileError("Unsupported OS"), } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 9a2044c232..601e91fe7f 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1038,11 +1038,8 @@ var fixed_buffer_mem: [100 * 1024]u8 = undefined; fn testParse(source: []const u8, allocator: &mem.Allocator) ![]u8 { var padded_source: [0x100]u8 = undefined; std.mem.copy(u8, padded_source[0..source.len], source); - padded_source[source.len + 0] = '\n'; - padded_source[source.len + 1] = '\n'; - padded_source[source.len + 2] = '\n'; - var tokenizer = Tokenizer.init(padded_source[0..source.len + 3]); + var tokenizer = Tokenizer.init(padded_source[0..source.len]); var parser = Parser.init(&tokenizer, allocator, "(memory buffer)"); defer parser.deinit(); diff --git a/test/compare_output.zig b/test/compare_output.zig index dfdeeba9b3..2ca2ba262e 100644 --- a/test/compare_output.zig +++ b/test/compare_output.zig @@ -1,4 +1,6 @@ -const os = @import("std").os; +const builtin = @import("builtin"); +const std = @import("std"); +const os = std.os; const tests = @import("tests.zig"); pub fn addCases(cases: &tests.CompareOutputContext) void { @@ -8,7 +10,7 @@ pub fn addCases(cases: &tests.CompareOutputContext) void { \\ _ = c.puts(c"Hello, world!"); \\ return 0; \\} - , "Hello, world!" ++ os.line_sep); + , "Hello, world!" ++ std.cstr.line_sep); cases.addCase(x: { var tc = cases.create("multiple files with private function",