From f47655eb6d42590a89f177e7354c998a8d88582d Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Sat, 16 Jun 2018 23:47:51 +0200 Subject: [PATCH 01/56] pointer reform: missed change in windows specific code. --- std/os/file.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/os/file.zig b/std/os/file.zig index 56da4f73a6..757c49ff74 100644 --- a/std/os/file.zig +++ b/std/os/file.zig @@ -242,7 +242,7 @@ pub const File = struct { }, Os.windows => { var pos: windows.LARGE_INTEGER = undefined; - if (windows.SetFilePointerEx(self.handle, 0, *pos, windows.FILE_CURRENT) == 0) { + if (windows.SetFilePointerEx(self.handle, 0, &pos, windows.FILE_CURRENT) == 0) { const err = windows.GetLastError(); return switch (err) { windows.ERROR.INVALID_PARAMETER => error.BadFd, From 2ec9a11646c792a046b4601e0b99f8e182416a6c Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Sat, 21 Jul 2018 20:30:11 +0200 Subject: [PATCH 02/56] Very much WIP base implementation for #721. Currently does: - read COFF executable file - locate and load corresponding .pdb file - expose .pdb content as streams (PDB format) --- CMakeLists.txt | 2 + std/coff.zig | 238 +++++++++++++++++++++++++++++++++++ std/debug/index.zig | 47 ++++++- std/index.zig | 4 + std/os/index.zig | 8 +- std/os/windows/index.zig | 2 + std/pdb.zig | 265 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 563 insertions(+), 3 deletions(-) create mode 100644 std/coff.zig create mode 100644 std/pdb.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index dd4770ad72..4ddf0bd66e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -427,6 +427,7 @@ set(ZIG_STD_FILES "c/index.zig" "c/linux.zig" "c/windows.zig" + "coff.zig" "crypto/blake2.zig" "crypto/hmac.zig" "crypto/index.zig" @@ -544,6 +545,7 @@ set(ZIG_STD_FILES "os/windows/index.zig" "os/windows/util.zig" "os/zen.zig" + "pdb.zig" "rand/index.zig" "rand/ziggurat.zig" "segmented_list.zig" diff --git a/std/coff.zig b/std/coff.zig new file mode 100644 index 0000000000..475b4fcbc1 --- /dev/null +++ b/std/coff.zig @@ -0,0 +1,238 @@ +const builtin = @import("builtin"); +const std = @import("index.zig"); +const io = std.io; +const mem = std.mem; +const os = std.os; + +const ArrayList = std.ArrayList; + +// CoffHeader.machine values +// see https://msdn.microsoft.com/en-us/library/windows/desktop/ms680313(v=vs.85).aspx +const IMAGE_FILE_MACHINE_I386 = 0x014c; +const IMAGE_FILE_MACHINE_IA64 = 0x0200; +const IMAGE_FILE_MACHINE_AMD64 = 0x8664; + +// OptionalHeader.magic values +// see https://msdn.microsoft.com/en-us/library/windows/desktop/ms680339(v=vs.85).aspx +const IMAGE_NT_OPTIONAL_HDR32_MAGIC = 0x10b; +const IMAGE_NT_OPTIONAL_HDR64_MAGIC = 0x20b; + +const IMAGE_NUMBEROF_DIRECTORY_ENTRIES = 16; +const DEBUG_DIRECTORY = 6; + +pub const CoffError = error { + InvalidPEMagic, + InvalidPEHeader, + InvalidMachine, + MissingCoffSection, +}; + +pub const Coff = struct { + in_file: os.File, + allocator: *mem.Allocator, + + coff_header: CoffHeader, + pe_header: OptionalHeader, + sections: ArrayList(Section), + + guid: [16]u8, + age: u32, + + pub fn loadHeader(self: *Coff) !void { + const pe_pointer_offset = 0x3C; + + var file_stream = io.FileInStream.init(&self.in_file); + const in = &file_stream.stream; + + var magic: [2]u8 = undefined; + try in.readNoEof(magic[0..]); + if (!mem.eql(u8, magic, "MZ")) + return error.InvalidPEMagic; + + // Seek to PE File Header (coff header) + try self.in_file.seekTo(pe_pointer_offset); + const pe_magic_offset = try in.readIntLe(u32); + try self.in_file.seekTo(pe_magic_offset); + + var pe_header_magic: [4]u8 = undefined; + try in.readNoEof(pe_header_magic[0..]); + if (!mem.eql(u8, pe_header_magic, []u8{'P', 'E', 0, 0})) + return error.InvalidPEHeader; + + self.coff_header = CoffHeader { + .machine = try in.readIntLe(u16), + .number_of_sections = try in.readIntLe(u16), + .timedate_stamp = try in.readIntLe(u32), + .pointer_to_symbol_table = try in.readIntLe(u32), + .number_of_symbols = try in.readIntLe(u32), + .size_of_optional_header = try in.readIntLe(u16), + .characteristics = try in.readIntLe(u16), + }; + + switch (self.coff_header.machine) { + IMAGE_FILE_MACHINE_I386, + IMAGE_FILE_MACHINE_AMD64, + IMAGE_FILE_MACHINE_IA64 + => {}, + else => return error.InvalidMachine, + } + + try self.loadOptionalHeader(&file_stream); + } + + fn loadOptionalHeader(self: *Coff, file_stream: *io.FileInStream) !void { + const in = &file_stream.stream; + self.pe_header.magic = try in.readIntLe(u16); + std.debug.warn("reading pe optional\n"); + // For now we're only interested in finding the reference to the .pdb, + // so we'll skip most of this header, which size is different in 32 + // 64 bits by the way. + var skip_size: u16 = undefined; + if (self.pe_header.magic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) { + skip_size = 2 * @sizeOf(u8) + 8 * @sizeOf(u16) + 18 * @sizeOf(u32); + } + else if (self.pe_header.magic == IMAGE_NT_OPTIONAL_HDR64_MAGIC) { + skip_size = 2 * @sizeOf(u8) + 8 * @sizeOf(u16) + 12 * @sizeOf(u32) + 5 * @sizeOf(u64); + } + else + return error.InvalidPEMagic; + + std.debug.warn("skipping {}\n", skip_size); + try self.in_file.seekForward(skip_size); + + const number_of_rva_and_sizes = try in.readIntLe(u32); + //std.debug.warn("indicating {} data dirs\n", number_of_rva_and_sizes); + if (number_of_rva_and_sizes != IMAGE_NUMBEROF_DIRECTORY_ENTRIES) + return error.InvalidPEHeader; + + for (self.pe_header.data_directory) |*data_dir| { + data_dir.* = OptionalHeader.DataDirectory { + .virtual_address = try in.readIntLe(u32), + .size = try in.readIntLe(u32), + }; + //std.debug.warn("data_dir @ {x}, size {}\n", data_dir.virtual_address, data_dir.size); + } + std.debug.warn("loaded data directories\n"); + } + + pub fn getPdbPath(self: *Coff, buffer: []u8) !usize { + try self.loadSections(); + const header = (self.getSection(".rdata") orelse return error.MissingCoffSection).header; + + // The linker puts a chunk that contains the .pdb path right after the + // debug_directory. + const debug_dir = &self.pe_header.data_directory[DEBUG_DIRECTORY]; + const file_offset = debug_dir.virtual_address - header.virtual_address + header.pointer_to_raw_data; + std.debug.warn("file offset {x}\n", file_offset); + try self.in_file.seekTo(file_offset + debug_dir.size); + + var file_stream = io.FileInStream.init(&self.in_file); + const in = &file_stream.stream; + + var cv_signature: [4]u8 = undefined; // CodeView signature + try in.readNoEof(cv_signature[0..]); + // 'RSDS' indicates PDB70 format, used by lld. + if (!mem.eql(u8, cv_signature, "RSDS")) + return error.InvalidPEMagic; + std.debug.warn("cv_signature {}\n", cv_signature); + try in.readNoEof(self.guid[0..]); + self.age = try in.readIntLe(u32); + + // Finally read the null-terminated string. + var byte = try in.readByte(); + var i: usize = 0; + while (byte != 0 and i < buffer.len) : (i += 1) { + buffer[i] = byte; + byte = try in.readByte(); + } + + if (byte != 0 and i == buffer.len) + return error.NameTooLong; + + return i; + } + + pub fn loadSections(self: *Coff) !void { + if (self.sections.len != 0) + return; + + self.sections = ArrayList(Section).init(self.allocator); + + var file_stream = io.FileInStream.init(&self.in_file); + const in = &file_stream.stream; + + var name: [8]u8 = undefined; + + var i: u16 = 0; + while (i < self.coff_header.number_of_sections) : (i += 1) { + try in.readNoEof(name[0..]); + try self.sections.append(Section { + .header = SectionHeader { + .name = name, + .misc = SectionHeader.Misc { .physical_address = try in.readIntLe(u32) }, + .virtual_address = try in.readIntLe(u32), + .size_of_raw_data = try in.readIntLe(u32), + .pointer_to_raw_data = try in.readIntLe(u32), + .pointer_to_relocations = try in.readIntLe(u32), + .pointer_to_line_numbers = try in.readIntLe(u32), + .number_of_relocations = try in.readIntLe(u16), + .number_of_line_numbers = try in.readIntLe(u16), + .characteristics = try in.readIntLe(u32), + }, + }); + } + std.debug.warn("loaded {} sections\n", self.coff_header.number_of_sections); + } + + pub fn getSection(self: *Coff, comptime name: []const u8) ?*Section { + for (self.sections.toSlice()) |*sec| { + if (mem.eql(u8, sec.header.name[0..name.len], name)) { + return sec; + } + } + return null; + } + +}; + +const CoffHeader = struct { + machine: u16, + number_of_sections: u16, + timedate_stamp: u32, + pointer_to_symbol_table: u32, + number_of_symbols: u32, + size_of_optional_header: u16, + characteristics: u16 +}; + +const OptionalHeader = struct { + const DataDirectory = struct { + virtual_address: u32, + size: u32 + }; + + magic: u16, + data_directory: [IMAGE_NUMBEROF_DIRECTORY_ENTRIES]DataDirectory, +}; + +const Section = struct { + header: SectionHeader, +}; + +const SectionHeader = struct { + const Misc = union { + physical_address: u32, + virtual_size: u32 + }; + + name: [8]u8, + misc: Misc, + virtual_address: u32, + size_of_raw_data: u32, + pointer_to_raw_data: u32, + pointer_to_relocations: u32, + pointer_to_line_numbers: u32, + number_of_relocations: u16, + number_of_line_numbers: u16, + characteristics: u32, +}; \ No newline at end of file diff --git a/std/debug/index.zig b/std/debug/index.zig index 25f7a58b25..5d00b5a873 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -6,6 +6,9 @@ const os = std.os; const elf = std.elf; const DW = std.dwarf; const macho = std.macho; +const coff = std.coff; +const pdb = std.pdb; +const windows = os.windows; const ArrayList = std.ArrayList; const builtin = @import("builtin"); @@ -197,7 +200,13 @@ fn printSourceAtAddress(debug_info: *ElfStackTrace, out_stream: var, address: us const ptr_hex = "0x{x}"; switch (builtin.os) { - builtin.Os.windows => return error.UnsupportedDebugInfo, + builtin.Os.windows => { + const base_address = @ptrToInt(windows.GetModuleHandleA(null)); // returned HMODULE points to our executable file in memory + const relative_address = address - base_address; + std.debug.warn("{x} - {x} => {x}\n", address, base_address, relative_address); + try debug_info.pdb.getSourceLine(relative_address); + return error.UnsupportedDebugInfo; + }, builtin.Os.macosx => { // TODO(bnoordhuis) It's theoretically possible to obtain the // compilation unit from the symbtab but it's not that useful @@ -288,7 +297,38 @@ pub fn openSelfDebugInfo(allocator: *mem.Allocator) !*ElfStackTrace { return st; }, builtin.ObjectFormat.coff => { - return error.TodoSupportCoffDebugInfo; + var coff_file: coff.Coff = undefined; + coff_file.in_file = try os.openSelfExe(); + coff_file.allocator = allocator; + defer coff_file.in_file.close(); + + try coff_file.loadHeader(); + + var path: [windows.MAX_PATH]u8 = undefined; + const len = try coff_file.getPdbPath(path[0..]); + std.debug.warn("pdb path {}\n", path[0..len]); + + const st = try allocator.create(ElfStackTrace); + errdefer allocator.destroy(st); + st.* = ElfStackTrace { + .pdb = undefined, + }; + + try st.pdb.openFile(allocator, path[0..len]); + + var pdb_stream = st.pdb.getStream(pdb.StreamType.Pdb) orelse return error.CorruptedFile; + std.debug.warn("pdb real filepos {}\n", pdb_stream.getFilePos()); + const version = try pdb_stream.stream.readIntLe(u32); + const signature = try pdb_stream.stream.readIntLe(u32); + const age = try pdb_stream.stream.readIntLe(u32); + var guid: [16]u8 = undefined; + try pdb_stream.stream.readNoEof(guid[0..]); + if (!mem.eql(u8, coff_file.guid, guid) or coff_file.age != age) + return error.CorruptedFile; + std.debug.warn("v {} s {} a {}\n", version, signature, age); + // We validated the executable and pdb match. + + return st; }, builtin.ObjectFormat.wasm => { return error.TodoSupportCOFFDebugInfo; @@ -339,6 +379,9 @@ pub const ElfStackTrace = switch (builtin.os) { self.symbol_table.deinit(); } }, + builtin.Os.windows => struct { + pdb: pdb.Pdb, + }, else => struct { self_exe_file: os.File, elf: elf.Elf, diff --git a/std/index.zig b/std/index.zig index 8abfa3db88..a54c5ac465 100644 --- a/std/index.zig +++ b/std/index.zig @@ -13,6 +13,7 @@ pub const atomic = @import("atomic/index.zig"); pub const base64 = @import("base64.zig"); pub const build = @import("build.zig"); pub const c = @import("c/index.zig"); +pub const coff = @import("coff.zig"); pub const crypto = @import("crypto/index.zig"); pub const cstr = @import("cstr.zig"); pub const debug = @import("debug/index.zig"); @@ -30,6 +31,7 @@ pub const math = @import("math/index.zig"); pub const mem = @import("mem.zig"); pub const net = @import("net.zig"); pub const os = @import("os/index.zig"); +pub const pdb = @import("pdb.zig"); pub const rand = @import("rand/index.zig"); pub const sort = @import("sort.zig"); pub const unicode = @import("unicode.zig"); @@ -49,6 +51,7 @@ test "std" { _ = @import("base64.zig"); _ = @import("build.zig"); _ = @import("c/index.zig"); + _ = @import("coff.zig"); _ = @import("crypto/index.zig"); _ = @import("cstr.zig"); _ = @import("debug/index.zig"); @@ -67,6 +70,7 @@ test "std" { _ = @import("heap.zig"); _ = @import("os/index.zig"); _ = @import("rand/index.zig"); + _ = @import("pdb.zig"); _ = @import("sort.zig"); _ = @import("unicode.zig"); _ = @import("zig/index.zig"); diff --git a/std/os/index.zig b/std/os/index.zig index 62eeb7e43e..45bad41a02 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -1896,13 +1896,19 @@ pub fn openSelfExe() !os.File { const self_exe_path = try selfExePath(&fixed_allocator.allocator); return os.File.openRead(&fixed_allocator.allocator, self_exe_path); }, + Os.windows => { + var fixed_buffer_mem: [windows.MAX_PATH * 2]u8 = undefined; + var fixed_allocator = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]); + const self_exe_path = try selfExePath(&fixed_allocator.allocator); + return os.File.openRead(&fixed_allocator.allocator, self_exe_path); + }, else => @compileError("Unsupported OS"), } } test "openSelfExe" { switch (builtin.os) { - Os.linux, Os.macosx, Os.ios => (try openSelfExe()).close(), + Os.linux, Os.macosx, Os.ios, Os.windows => (try openSelfExe()).close(), else => return, // Unsupported OS. } } diff --git a/std/os/windows/index.zig b/std/os/windows/index.zig index d631c6adbf..6eb9fc38f3 100644 --- a/std/os/windows/index.zig +++ b/std/os/windows/index.zig @@ -105,6 +105,8 @@ pub extern "kernel32" stdcallcc fn GetFinalPathNameByHandleA( dwFlags: DWORD, ) DWORD; +pub extern "kernel32" stdcallcc fn GetModuleHandleA(lpModuleName: ?LPCSTR) HMODULE; + pub extern "kernel32" stdcallcc fn GetProcessHeap() ?HANDLE; pub extern "kernel32" stdcallcc fn GetSystemTimeAsFileTime(*FILETIME) void; diff --git a/std/pdb.zig b/std/pdb.zig new file mode 100644 index 0000000000..8c5a82880e --- /dev/null +++ b/std/pdb.zig @@ -0,0 +1,265 @@ +const builtin = @import("builtin"); +const std = @import("index.zig"); +const io = std.io; +const math = std.math; +const mem = std.mem; +const os = std.os; +const warn = std.debug.warn; + +const ArrayList = std.ArrayList; + +pub const PdbError = error { + InvalidPdbMagic, + CorruptedFile, +}; + +pub const StreamType = enum(u16) { + Pdb = 1, + Tpi = 2, + Dbi = 3, + Ipi = 4, +}; + +pub const Pdb = struct { + in_file: os.File, + allocator: *mem.Allocator, + + msf: Msf, + + pub fn openFile(self: *Pdb, allocator: *mem.Allocator, file_name: []u8) !void { + self.in_file = try os.File.openRead(allocator, file_name[0..]); + self.allocator = allocator; + + try self.msf.openFile(allocator, &self.in_file); + } + + pub fn getStream(self: *Pdb, stream: StreamType) ?*MsfStream { + const id = u16(stream); + if (id < self.msf.streams.len) + return &self.msf.streams.items[id]; + return null; + } + + pub fn getSourceLine(self: *Pdb, address: usize) !void { + const dbi = self.getStream(StreamType.Dbi) orelse return error.CorruptedFile; + + // Dbi Header + try dbi.seekForward(@sizeOf(u32) * 3 + @sizeOf(u16) * 6); + warn("dbi stream at {} (file offset)\n", dbi.getFilePos()); + const module_info_size = try dbi.stream.readIntLe(u32); + const section_contribution_size = try dbi.stream.readIntLe(u32); + const section_map_size = try dbi.stream.readIntLe(u32); + const source_info_size = try dbi.stream.readIntLe(u32); + warn("module_info_size: {}\n", module_info_size); + warn("section_contribution_size: {}\n", section_contribution_size); + warn("section_map_size: {}\n", section_map_size); + warn("source_info_size: {}\n", source_info_size); + try dbi.seekForward(@sizeOf(u32) * 5 + @sizeOf(u16) * 2); + warn("after header dbi stream at {} (file offset)\n", dbi.getFilePos()); + + // Module Info Substream + try dbi.seekForward(@sizeOf(u32) + @sizeOf(u16) + @sizeOf(u8) * 2); + const offset = try dbi.stream.readIntLe(u32); + const size = try dbi.stream.readIntLe(u32); + try dbi.seekForward(@sizeOf(u32)); + const module_index = try dbi.stream.readIntLe(u16); + warn("module {} of size {} at {}\n", module_index, size, offset); + + // TODO: locate corresponding source line information + } +}; + +// see https://llvm.org/docs/PDB/MsfFile.html +const Msf = struct { + superblock: SuperBlock, + directory: MsfStream, + streams: ArrayList(MsfStream), + + fn openFile(self: *Msf, allocator: *mem.Allocator, file: *os.File) !void { + var file_stream = io.FileInStream.init(file); + const in = &file_stream.stream; + + var magic: SuperBlock.FileMagicBuffer = undefined; + try in.readNoEof(magic[0..]); + warn("magic: '{}'\n", magic); + + if (!mem.eql(u8, magic, SuperBlock.FileMagic)) + return error.InvalidPdbMagic; + + self.superblock = SuperBlock { + .block_size = try in.readIntLe(u32), + .free_block_map_block = try in.readIntLe(u32), + .num_blocks = try in.readIntLe(u32), + .num_directory_bytes = try in.readIntLe(u32), + .unknown = try in.readIntLe(u32), + .block_map_addr = try in.readIntLe(u32), + }; + + switch (self.superblock.block_size) { + 512, 1024, 2048, 4096 => {}, // llvm only uses 4096 + else => return error.InvalidPdbMagic + } + + if (self.superblock.fileSize() != try file.getEndPos()) + return error.CorruptedFile; // Should always stand. + + self.directory = try MsfStream.init( + self.superblock.block_size, + self.superblock.blocksOccupiedByDirectoryStream(), + self.superblock.blockMapAddr(), + file, + allocator + ); + + const stream_count = try self.directory.stream.readIntLe(u32); + warn("stream count {}\n", stream_count); + + var stream_sizes = ArrayList(u32).init(allocator); + try stream_sizes.resize(stream_count); + for (stream_sizes.toSlice()) |*s| { + const size = try self.directory.stream.readIntLe(u32); + s.* = blockCountFromSize(size, self.superblock.block_size); + warn("stream {}B {} blocks\n", size, s.*); + } + + self.streams = ArrayList(MsfStream).init(allocator); + try self.streams.resize(stream_count); + for (self.streams.toSlice()) |*ss, i| { + ss.* = try MsfStream.init( + self.superblock.block_size, + stream_sizes.items[i], + try file.getPos(), // We're reading the jagged array of block indices when creating streams so the file is always at the right position. + file, + allocator + ); + } + } +}; + +fn blockCountFromSize(size: u32, block_size: u32) u32 { + return (size + block_size - 1) / block_size; +} + +const SuperBlock = struct { + const FileMagic = "Microsoft C/C++ MSF 7.00\r\n" ++ []u8 { 0x1A, 'D', 'S', 0, 0, 0}; + const FileMagicBuffer = @typeOf(FileMagic); + + block_size: u32, + free_block_map_block: u32, + num_blocks: u32, + num_directory_bytes: u32, + unknown: u32, + block_map_addr: u32, + + fn fileSize(self: *const SuperBlock) usize { + return self.num_blocks * self.block_size; + } + + fn blockMapAddr(self: *const SuperBlock) usize { + return self.block_size * self.block_map_addr; + } + + fn blocksOccupiedByDirectoryStream(self: *const SuperBlock) u32 { + return blockCountFromSize(self.num_directory_bytes, self.block_size); + } +}; + +const MsfStream = struct { + in_file: *os.File, + pos: usize, + blocks: ArrayList(u32), + block_size: u32, + + fn init(block_size: u32, block_count: u32, pos: usize, file: *os.File, allocator: *mem.Allocator) !MsfStream { + var stream = MsfStream { + .in_file = file, + .pos = 0, + .blocks = ArrayList(u32).init(allocator), + .block_size = block_size, + .stream = Stream { + .readFn = readFn, + }, + }; + + try stream.blocks.resize(block_count); + + var file_stream = io.FileInStream.init(file); + const in = &file_stream.stream; + try file.seekTo(pos); + + warn("stream with blocks"); + var i: u32 = 0; + while (i < block_count) : (i += 1) { + stream.blocks.items[i] = try in.readIntLe(u32); + warn(" {}", stream.blocks.items[i]); + } + warn("\n"); + + return stream; + } + + fn read(self: *MsfStream, buffer: []u8) !usize { + var block_id = self.pos / self.block_size; + var block = self.blocks.items[block_id]; + var offset = self.pos % self.block_size; + + try self.in_file.seekTo(block * self.block_size + offset); + var file_stream = io.FileInStream.init(self.in_file); + const in = &file_stream.stream; + + var size: usize = 0; + for (buffer) |*byte| { + byte.* = try in.readByte(); + + offset += 1; + size += 1; + + // If we're at the end of a block, go to the next one. + if (offset == self.block_size) + { + offset = 0; + block_id += 1; + block = self.blocks.items[block_id]; + try self.in_file.seekTo(block * self.block_size); + } + } + + self.pos += size; + return size; + } + + fn seekForward(self: *MsfStream, len: usize) !void { + self.pos += len; + if (self.pos >= self.blocks.len * self.block_size) + return error.EOF; + } + + fn seekTo(self: *MsfStream, len: usize) !void { + self.pos = len; + if (self.pos >= self.blocks.len * self.block_size) + return error.EOF; + } + + fn getSize(self: *const MsfStream) usize { + return self.blocks.len * self.block_size; + } + + fn getFilePos(self: *const MsfStream) usize { + const block_id = self.pos / self.block_size; + const block = self.blocks.items[block_id]; + const offset = self.pos % self.block_size; + + return block * self.block_size + offset; + } + + /// Implementation of InStream trait for Pdb.MsfStream + pub const Error = @typeOf(read).ReturnType.ErrorSet; + pub const Stream = io.InStream(Error); + + stream: Stream, + + fn readFn(in_stream: *Stream, buffer: []u8) Error!usize { + const self = @fieldParentPtr(MsfStream, "stream", in_stream); + return self.read(buffer); + } +}; \ No newline at end of file From 0839ed1f945dbbe3ad1f780a5334ad16ad12fd5e Mon Sep 17 00:00:00 2001 From: kristopher tate Date: Mon, 27 Aug 2018 01:22:50 +0900 Subject: [PATCH 03/56] src/ir.cpp: check return value of `const_ptr_pointee` to protect against dereferencing null pointers; --- src/ir.cpp | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/ir.cpp b/src/ir.cpp index 5bf39ee691..48547298ae 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -10982,6 +10982,7 @@ static IrInstruction *ir_get_deref(IrAnalyze *ira, IrInstruction *source_instruc ptr->value.data.x_ptr.mut == ConstPtrMutComptimeVar) { ConstExprValue *pointee = const_ptr_pointee(ira->codegen, &ptr->value); + if (!pointee) return ira->codegen->invalid_instruction; if (pointee->special != ConstValSpecialRuntime) { IrInstruction *result = ir_create_const(&ira->new_irb, source_instruction->scope, source_instruction->source_node, child_type); @@ -13655,6 +13656,7 @@ static TypeTableEntry *ir_analyze_dereference(IrAnalyze *ira, IrInstructionUnOp return ira->codegen->builtin_types.entry_invalid; ConstExprValue *pointee = const_ptr_pointee(ira->codegen, comptime_value); + if (!pointee) return ira->codegen->builtin_types.entry_invalid; if (pointee->type == child_type) { ConstExprValue *out_val = ir_build_const_from(ira, &un_op_instruction->base); copy_const_val(out_val, pointee, value->value.data.x_ptr.mut == ConstPtrMutComptimeConst); @@ -14058,6 +14060,7 @@ static TypeTableEntry *ir_analyze_instruction_elem_ptr(IrAnalyze *ira, IrInstruc ptr_type = ptr_type->data.pointer.child_type; if (orig_array_ptr_val->special != ConstValSpecialRuntime) { orig_array_ptr_val = const_ptr_pointee(ira->codegen, orig_array_ptr_val); + if (!orig_array_ptr_val) return ira->codegen->builtin_types.entry_invalid; } } if (array_type->data.array.len == 0) { @@ -14099,6 +14102,7 @@ static TypeTableEntry *ir_analyze_instruction_elem_ptr(IrAnalyze *ira, IrInstruc if (!ptr_val) return ira->codegen->builtin_types.entry_invalid; ConstExprValue *args_val = const_ptr_pointee(ira->codegen, ptr_val); + if (!args_val) return ira->codegen->builtin_types.entry_invalid; size_t start = args_val->data.x_arg_tuple.start_index; size_t end = args_val->data.x_arg_tuple.end_index; uint64_t elem_index_val; @@ -14380,6 +14384,7 @@ static IrInstruction *ir_analyze_container_field_ptr(IrAnalyze *ira, Buf *field_ if (ptr_val->data.x_ptr.special != ConstPtrSpecialHardCodedAddr) { ConstExprValue *struct_val = const_ptr_pointee(ira->codegen, ptr_val); + if (!struct_val) return ira->codegen->invalid_instruction; if (type_is_invalid(struct_val->type)) return ira->codegen->invalid_instruction; ConstExprValue *field_val = &struct_val->data.x_struct.fields[field->src_index]; @@ -14422,6 +14427,7 @@ static IrInstruction *ir_analyze_container_field_ptr(IrAnalyze *ira, Buf *field_ if (ptr_val->data.x_ptr.special != ConstPtrSpecialHardCodedAddr) { ConstExprValue *union_val = const_ptr_pointee(ira->codegen, ptr_val); + if (!union_val) return ira->codegen->invalid_instruction; if (type_is_invalid(union_val->type)) return ira->codegen->invalid_instruction; @@ -14618,6 +14624,7 @@ static TypeTableEntry *ir_analyze_instruction_field_ptr(IrAnalyze *ira, IrInstru assert(container_ptr->value.type->id == TypeTableEntryIdPointer); ConstExprValue *child_val = const_ptr_pointee(ira->codegen, container_ptr_val); + if (!child_val) return ira->codegen->builtin_types.entry_invalid; if (buf_eql_str(field_name, "len")) { ConstExprValue *len_val = create_const_vals(1); @@ -14642,6 +14649,7 @@ static TypeTableEntry *ir_analyze_instruction_field_ptr(IrAnalyze *ira, IrInstru assert(container_ptr->value.type->id == TypeTableEntryIdPointer); ConstExprValue *child_val = const_ptr_pointee(ira->codegen, container_ptr_val); + if (!child_val) return ira->codegen->builtin_types.entry_invalid; TypeTableEntry *child_type = child_val->data.x_type; if (type_is_invalid(child_type)) { @@ -14910,6 +14918,7 @@ static TypeTableEntry *ir_analyze_instruction_field_ptr(IrAnalyze *ira, IrInstru return ira->codegen->builtin_types.entry_invalid; ConstExprValue *namespace_val = const_ptr_pointee(ira->codegen, container_ptr_val); + if (!namespace_val) return ira->codegen->builtin_types.entry_invalid; assert(namespace_val->special == ConstValSpecialStatic); ImportTableEntry *namespace_import = namespace_val->data.x_import; @@ -14986,6 +14995,7 @@ static TypeTableEntry *ir_analyze_instruction_store_ptr(IrAnalyze *ira, IrInstru if (ptr->value.data.x_ptr.mut == ConstPtrMutComptimeVar) { if (instr_is_comptime(casted_value)) { ConstExprValue *dest_val = const_ptr_pointee(ira->codegen, &ptr->value); + if (!dest_val) return ira->codegen->builtin_types.entry_invalid; if (dest_val->special != ConstValSpecialRuntime) { *dest_val = casted_value->value; if (!ira->new_irb.current_basic_block->must_be_comptime_source_instr) { @@ -14997,6 +15007,7 @@ static TypeTableEntry *ir_analyze_instruction_store_ptr(IrAnalyze *ira, IrInstru ir_add_error(ira, &store_ptr_instruction->base, buf_sprintf("cannot store runtime value in compile time variable")); ConstExprValue *dest_val = const_ptr_pointee(ira->codegen, &ptr->value); + if (!dest_val) return ira->codegen->builtin_types.entry_invalid; dest_val->type = ira->codegen->builtin_types.entry_invalid; return ira->codegen->builtin_types.entry_invalid; @@ -15850,7 +15861,7 @@ static TypeTableEntry *ir_analyze_instruction_switch_target(IrAnalyze *ira, ConstExprValue *pointee_val = nullptr; if (instr_is_comptime(target_value_ptr)) { pointee_val = const_ptr_pointee(ira->codegen, &target_value_ptr->value); - if (pointee_val->special == ConstValSpecialRuntime) + if (pointee_val && pointee_val->special == ConstValSpecialRuntime) pointee_val = nullptr; } if ((err = ensure_complete_type(ira->codegen, target_type))) @@ -15981,6 +15992,7 @@ static TypeTableEntry *ir_analyze_instruction_switch_var(IrAnalyze *ira, IrInstr return ira->codegen->builtin_types.entry_invalid; ConstExprValue *pointee_val = const_ptr_pointee(ira->codegen, target_val_ptr); + if (!pointee_val) return ira->codegen->builtin_types.entry_invalid; ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); out_val->data.x_ptr.special = ConstPtrSpecialRef; out_val->data.x_ptr.mut = target_val_ptr->data.x_ptr.mut; @@ -18732,11 +18744,14 @@ static TypeTableEntry *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstructio TypeTableEntry *child_array_type = array_type->data.pointer.child_type; assert(child_array_type->id == TypeTableEntryIdArray); parent_ptr = const_ptr_pointee(ira->codegen, &ptr_ptr->value); + if (!parent_ptr) return ira->codegen->builtin_types.entry_invalid; array_val = const_ptr_pointee(ira->codegen, parent_ptr); + if (!array_val) return ira->codegen->builtin_types.entry_invalid; rel_end = child_array_type->data.array.len; abs_offset = 0; } else { array_val = const_ptr_pointee(ira->codegen, &ptr_ptr->value); + if (!array_val) return ira->codegen->builtin_types.entry_invalid; rel_end = array_type->data.array.len; parent_ptr = nullptr; abs_offset = 0; @@ -18744,6 +18759,7 @@ static TypeTableEntry *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstructio } else if (array_type->id == TypeTableEntryIdPointer) { assert(array_type->data.pointer.ptr_len == PtrLenUnknown); parent_ptr = const_ptr_pointee(ira->codegen, &ptr_ptr->value); + if (!parent_ptr) return ira->codegen->builtin_types.entry_invalid; if (parent_ptr->special == ConstValSpecialUndef) { array_val = nullptr; abs_offset = 0; @@ -18775,6 +18791,7 @@ static TypeTableEntry *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstructio } } else if (is_slice(array_type)) { ConstExprValue *slice_ptr = const_ptr_pointee(ira->codegen, &ptr_ptr->value); + if (!slice_ptr) return ira->codegen->builtin_types.entry_invalid; parent_ptr = &slice_ptr->data.x_struct.fields[slice_ptr_index]; ConstExprValue *len_val = &slice_ptr->data.x_struct.fields[slice_len_index]; @@ -19175,6 +19192,7 @@ static TypeTableEntry *ir_analyze_instruction_overflow_op(IrAnalyze *ira, IrInst BigInt *op1_bigint = &casted_op1->value.data.x_bigint; BigInt *op2_bigint = &casted_op2->value.data.x_bigint; ConstExprValue *pointee_val = const_ptr_pointee(ira->codegen, &casted_result_ptr->value); + if (!pointee_val) return ira->codegen->builtin_types.entry_invalid; BigInt *dest_bigint = &pointee_val->data.x_bigint; switch (instruction->op) { case IrOverflowOpAdd: @@ -19275,6 +19293,7 @@ static TypeTableEntry *ir_analyze_instruction_unwrap_err_code(IrAnalyze *ira, if (!ptr_val) return ira->codegen->builtin_types.entry_invalid; ConstExprValue *err_union_val = const_ptr_pointee(ira->codegen, ptr_val); + if (!err_union_val) return ira->codegen->builtin_types.entry_invalid; if (err_union_val->special != ConstValSpecialRuntime) { ErrorTableEntry *err = err_union_val->data.x_err_union.err; assert(err); @@ -19323,6 +19342,7 @@ static TypeTableEntry *ir_analyze_instruction_unwrap_err_payload(IrAnalyze *ira, if (!ptr_val) return ira->codegen->builtin_types.entry_invalid; ConstExprValue *err_union_val = const_ptr_pointee(ira->codegen, ptr_val); + if (!err_union_val) return ira->codegen->builtin_types.entry_invalid; if (err_union_val->special != ConstValSpecialRuntime) { ErrorTableEntry *err = err_union_val->data.x_err_union.err; if (err != nullptr) { From 6a3fad1d596a59846d072a005f77f6c59da09824 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 26 Aug 2018 13:04:58 -0400 Subject: [PATCH 04/56] Revert "src/ir.cpp: check return value of `const_ptr_pointee` to protect against dereferencing null pointers;" This reverts commit 0839ed1f945dbbe3ad1f780a5334ad16ad12fd5e. I realized too late there is a better fix. See PR #1419 --- src/ir.cpp | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index 48547298ae..5bf39ee691 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -10982,7 +10982,6 @@ static IrInstruction *ir_get_deref(IrAnalyze *ira, IrInstruction *source_instruc ptr->value.data.x_ptr.mut == ConstPtrMutComptimeVar) { ConstExprValue *pointee = const_ptr_pointee(ira->codegen, &ptr->value); - if (!pointee) return ira->codegen->invalid_instruction; if (pointee->special != ConstValSpecialRuntime) { IrInstruction *result = ir_create_const(&ira->new_irb, source_instruction->scope, source_instruction->source_node, child_type); @@ -13656,7 +13655,6 @@ static TypeTableEntry *ir_analyze_dereference(IrAnalyze *ira, IrInstructionUnOp return ira->codegen->builtin_types.entry_invalid; ConstExprValue *pointee = const_ptr_pointee(ira->codegen, comptime_value); - if (!pointee) return ira->codegen->builtin_types.entry_invalid; if (pointee->type == child_type) { ConstExprValue *out_val = ir_build_const_from(ira, &un_op_instruction->base); copy_const_val(out_val, pointee, value->value.data.x_ptr.mut == ConstPtrMutComptimeConst); @@ -14060,7 +14058,6 @@ static TypeTableEntry *ir_analyze_instruction_elem_ptr(IrAnalyze *ira, IrInstruc ptr_type = ptr_type->data.pointer.child_type; if (orig_array_ptr_val->special != ConstValSpecialRuntime) { orig_array_ptr_val = const_ptr_pointee(ira->codegen, orig_array_ptr_val); - if (!orig_array_ptr_val) return ira->codegen->builtin_types.entry_invalid; } } if (array_type->data.array.len == 0) { @@ -14102,7 +14099,6 @@ static TypeTableEntry *ir_analyze_instruction_elem_ptr(IrAnalyze *ira, IrInstruc if (!ptr_val) return ira->codegen->builtin_types.entry_invalid; ConstExprValue *args_val = const_ptr_pointee(ira->codegen, ptr_val); - if (!args_val) return ira->codegen->builtin_types.entry_invalid; size_t start = args_val->data.x_arg_tuple.start_index; size_t end = args_val->data.x_arg_tuple.end_index; uint64_t elem_index_val; @@ -14384,7 +14380,6 @@ static IrInstruction *ir_analyze_container_field_ptr(IrAnalyze *ira, Buf *field_ if (ptr_val->data.x_ptr.special != ConstPtrSpecialHardCodedAddr) { ConstExprValue *struct_val = const_ptr_pointee(ira->codegen, ptr_val); - if (!struct_val) return ira->codegen->invalid_instruction; if (type_is_invalid(struct_val->type)) return ira->codegen->invalid_instruction; ConstExprValue *field_val = &struct_val->data.x_struct.fields[field->src_index]; @@ -14427,7 +14422,6 @@ static IrInstruction *ir_analyze_container_field_ptr(IrAnalyze *ira, Buf *field_ if (ptr_val->data.x_ptr.special != ConstPtrSpecialHardCodedAddr) { ConstExprValue *union_val = const_ptr_pointee(ira->codegen, ptr_val); - if (!union_val) return ira->codegen->invalid_instruction; if (type_is_invalid(union_val->type)) return ira->codegen->invalid_instruction; @@ -14624,7 +14618,6 @@ static TypeTableEntry *ir_analyze_instruction_field_ptr(IrAnalyze *ira, IrInstru assert(container_ptr->value.type->id == TypeTableEntryIdPointer); ConstExprValue *child_val = const_ptr_pointee(ira->codegen, container_ptr_val); - if (!child_val) return ira->codegen->builtin_types.entry_invalid; if (buf_eql_str(field_name, "len")) { ConstExprValue *len_val = create_const_vals(1); @@ -14649,7 +14642,6 @@ static TypeTableEntry *ir_analyze_instruction_field_ptr(IrAnalyze *ira, IrInstru assert(container_ptr->value.type->id == TypeTableEntryIdPointer); ConstExprValue *child_val = const_ptr_pointee(ira->codegen, container_ptr_val); - if (!child_val) return ira->codegen->builtin_types.entry_invalid; TypeTableEntry *child_type = child_val->data.x_type; if (type_is_invalid(child_type)) { @@ -14918,7 +14910,6 @@ static TypeTableEntry *ir_analyze_instruction_field_ptr(IrAnalyze *ira, IrInstru return ira->codegen->builtin_types.entry_invalid; ConstExprValue *namespace_val = const_ptr_pointee(ira->codegen, container_ptr_val); - if (!namespace_val) return ira->codegen->builtin_types.entry_invalid; assert(namespace_val->special == ConstValSpecialStatic); ImportTableEntry *namespace_import = namespace_val->data.x_import; @@ -14995,7 +14986,6 @@ static TypeTableEntry *ir_analyze_instruction_store_ptr(IrAnalyze *ira, IrInstru if (ptr->value.data.x_ptr.mut == ConstPtrMutComptimeVar) { if (instr_is_comptime(casted_value)) { ConstExprValue *dest_val = const_ptr_pointee(ira->codegen, &ptr->value); - if (!dest_val) return ira->codegen->builtin_types.entry_invalid; if (dest_val->special != ConstValSpecialRuntime) { *dest_val = casted_value->value; if (!ira->new_irb.current_basic_block->must_be_comptime_source_instr) { @@ -15007,7 +14997,6 @@ static TypeTableEntry *ir_analyze_instruction_store_ptr(IrAnalyze *ira, IrInstru ir_add_error(ira, &store_ptr_instruction->base, buf_sprintf("cannot store runtime value in compile time variable")); ConstExprValue *dest_val = const_ptr_pointee(ira->codegen, &ptr->value); - if (!dest_val) return ira->codegen->builtin_types.entry_invalid; dest_val->type = ira->codegen->builtin_types.entry_invalid; return ira->codegen->builtin_types.entry_invalid; @@ -15861,7 +15850,7 @@ static TypeTableEntry *ir_analyze_instruction_switch_target(IrAnalyze *ira, ConstExprValue *pointee_val = nullptr; if (instr_is_comptime(target_value_ptr)) { pointee_val = const_ptr_pointee(ira->codegen, &target_value_ptr->value); - if (pointee_val && pointee_val->special == ConstValSpecialRuntime) + if (pointee_val->special == ConstValSpecialRuntime) pointee_val = nullptr; } if ((err = ensure_complete_type(ira->codegen, target_type))) @@ -15992,7 +15981,6 @@ static TypeTableEntry *ir_analyze_instruction_switch_var(IrAnalyze *ira, IrInstr return ira->codegen->builtin_types.entry_invalid; ConstExprValue *pointee_val = const_ptr_pointee(ira->codegen, target_val_ptr); - if (!pointee_val) return ira->codegen->builtin_types.entry_invalid; ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); out_val->data.x_ptr.special = ConstPtrSpecialRef; out_val->data.x_ptr.mut = target_val_ptr->data.x_ptr.mut; @@ -18744,14 +18732,11 @@ static TypeTableEntry *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstructio TypeTableEntry *child_array_type = array_type->data.pointer.child_type; assert(child_array_type->id == TypeTableEntryIdArray); parent_ptr = const_ptr_pointee(ira->codegen, &ptr_ptr->value); - if (!parent_ptr) return ira->codegen->builtin_types.entry_invalid; array_val = const_ptr_pointee(ira->codegen, parent_ptr); - if (!array_val) return ira->codegen->builtin_types.entry_invalid; rel_end = child_array_type->data.array.len; abs_offset = 0; } else { array_val = const_ptr_pointee(ira->codegen, &ptr_ptr->value); - if (!array_val) return ira->codegen->builtin_types.entry_invalid; rel_end = array_type->data.array.len; parent_ptr = nullptr; abs_offset = 0; @@ -18759,7 +18744,6 @@ static TypeTableEntry *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstructio } else if (array_type->id == TypeTableEntryIdPointer) { assert(array_type->data.pointer.ptr_len == PtrLenUnknown); parent_ptr = const_ptr_pointee(ira->codegen, &ptr_ptr->value); - if (!parent_ptr) return ira->codegen->builtin_types.entry_invalid; if (parent_ptr->special == ConstValSpecialUndef) { array_val = nullptr; abs_offset = 0; @@ -18791,7 +18775,6 @@ static TypeTableEntry *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstructio } } else if (is_slice(array_type)) { ConstExprValue *slice_ptr = const_ptr_pointee(ira->codegen, &ptr_ptr->value); - if (!slice_ptr) return ira->codegen->builtin_types.entry_invalid; parent_ptr = &slice_ptr->data.x_struct.fields[slice_ptr_index]; ConstExprValue *len_val = &slice_ptr->data.x_struct.fields[slice_len_index]; @@ -19192,7 +19175,6 @@ static TypeTableEntry *ir_analyze_instruction_overflow_op(IrAnalyze *ira, IrInst BigInt *op1_bigint = &casted_op1->value.data.x_bigint; BigInt *op2_bigint = &casted_op2->value.data.x_bigint; ConstExprValue *pointee_val = const_ptr_pointee(ira->codegen, &casted_result_ptr->value); - if (!pointee_val) return ira->codegen->builtin_types.entry_invalid; BigInt *dest_bigint = &pointee_val->data.x_bigint; switch (instruction->op) { case IrOverflowOpAdd: @@ -19293,7 +19275,6 @@ static TypeTableEntry *ir_analyze_instruction_unwrap_err_code(IrAnalyze *ira, if (!ptr_val) return ira->codegen->builtin_types.entry_invalid; ConstExprValue *err_union_val = const_ptr_pointee(ira->codegen, ptr_val); - if (!err_union_val) return ira->codegen->builtin_types.entry_invalid; if (err_union_val->special != ConstValSpecialRuntime) { ErrorTableEntry *err = err_union_val->data.x_err_union.err; assert(err); @@ -19342,7 +19323,6 @@ static TypeTableEntry *ir_analyze_instruction_unwrap_err_payload(IrAnalyze *ira, if (!ptr_val) return ira->codegen->builtin_types.entry_invalid; ConstExprValue *err_union_val = const_ptr_pointee(ira->codegen, ptr_val); - if (!err_union_val) return ira->codegen->builtin_types.entry_invalid; if (err_union_val->special != ConstValSpecialRuntime) { ErrorTableEntry *err = err_union_val->data.x_err_union.err; if (err != nullptr) { From 68e2794e1543f91ab8f984127018820897cc0521 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 26 Aug 2018 13:13:26 -0400 Subject: [PATCH 05/56] ir: const_ptr_pointee asserts that its return value is non-null --- src/ir.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index 5bf39ee691..ae69f9c817 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -155,18 +155,22 @@ static TypeTableEntry *adjust_slice_align(CodeGen *g, TypeTableEntry *slice_type ConstExprValue *const_ptr_pointee(CodeGen *g, ConstExprValue *const_val) { assert(get_codegen_ptr_type(const_val->type) != nullptr); assert(const_val->special == ConstValSpecialStatic); + ConstExprValue *result; switch (const_val->data.x_ptr.special) { case ConstPtrSpecialInvalid: zig_unreachable(); case ConstPtrSpecialRef: - return const_val->data.x_ptr.data.ref.pointee; + result = const_val->data.x_ptr.data.ref.pointee; + break; case ConstPtrSpecialBaseArray: expand_undef_array(g, const_val->data.x_ptr.data.base_array.array_val); - return &const_val->data.x_ptr.data.base_array.array_val->data.x_array.s_none.elements[ + result = &const_val->data.x_ptr.data.base_array.array_val->data.x_array.s_none.elements[ const_val->data.x_ptr.data.base_array.elem_index]; + break; case ConstPtrSpecialBaseStruct: - return &const_val->data.x_ptr.data.base_struct.struct_val->data.x_struct.fields[ + result = &const_val->data.x_ptr.data.base_struct.struct_val->data.x_struct.fields[ const_val->data.x_ptr.data.base_struct.field_index]; + break; case ConstPtrSpecialHardCodedAddr: zig_unreachable(); case ConstPtrSpecialDiscard: @@ -174,7 +178,8 @@ ConstExprValue *const_ptr_pointee(CodeGen *g, ConstExprValue *const_val) { case ConstPtrSpecialFunction: zig_unreachable(); } - zig_unreachable(); + assert(result != nullptr); + return result; } static bool ir_should_inline(IrExecutable *exec, Scope *scope) { From 526d8425abf91b33e828b964f7d55d70e17780bf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Aug 2018 16:14:48 -0400 Subject: [PATCH 06/56] fix false negative determining if function is generic This solves the smaller test case of #1421 but the other test case is still an assertion failure. --- src/analyze.cpp | 7 ++++++- test/behavior.zig | 1 + test/cases/bugs/1421.zig | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 test/cases/bugs/1421.zig diff --git a/src/analyze.cpp b/src/analyze.cpp index a8b3ea7132..54dd84f166 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -1575,7 +1575,7 @@ static TypeTableEntry *analyze_fn_type(CodeGen *g, AstNode *proto_node, Scope *c switch (type_entry->id) { case TypeTableEntryIdInvalid: - return g->builtin_types.entry_invalid; + zig_unreachable(); case TypeTableEntryIdUnreachable: case TypeTableEntryIdUndefined: case TypeTableEntryIdNull: @@ -1703,6 +1703,11 @@ static TypeTableEntry *analyze_fn_type(CodeGen *g, AstNode *proto_node, Scope *c case TypeTableEntryIdUnion: case TypeTableEntryIdFn: case TypeTableEntryIdPromise: + if ((err = type_ensure_zero_bits_known(g, fn_type_id.return_type))) + return g->builtin_types.entry_invalid; + if (type_requires_comptime(fn_type_id.return_type)) { + return get_generic_fn_type(g, &fn_type_id); + } break; } diff --git a/test/behavior.zig b/test/behavior.zig index 5a26e206bf..a092889099 100644 --- a/test/behavior.zig +++ b/test/behavior.zig @@ -11,6 +11,7 @@ comptime { _ = @import("cases/bugs/1111.zig"); _ = @import("cases/bugs/1230.zig"); _ = @import("cases/bugs/1277.zig"); + _ = @import("cases/bugs/1421.zig"); _ = @import("cases/bugs/394.zig"); _ = @import("cases/bugs/655.zig"); _ = @import("cases/bugs/656.zig"); diff --git a/test/cases/bugs/1421.zig b/test/cases/bugs/1421.zig new file mode 100644 index 0000000000..fcbb8b70e4 --- /dev/null +++ b/test/cases/bugs/1421.zig @@ -0,0 +1,14 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const assert = std.debug.assert; + +const S = struct { + fn method() builtin.TypeInfo { + return @typeInfo(S); + } +}; + +test "functions with return type required to be comptime are generic" { + const ti = S.method(); + assert(builtin.TypeId(ti) == builtin.TypeId.Struct); +} From 2f2215c9f46f0a75ce65803af84984b4a5f3741c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Aug 2018 16:26:36 -0400 Subject: [PATCH 07/56] this was intended to be included in the previous commit --- src/analyze.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/analyze.cpp b/src/analyze.cpp index 54dd84f166..ccab9170e9 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -1680,14 +1680,6 @@ static TypeTableEntry *analyze_fn_type(CodeGen *g, AstNode *proto_node, Scope *c case TypeTableEntryIdBlock: case TypeTableEntryIdBoundFn: case TypeTableEntryIdMetaType: - if (!calling_convention_allows_zig_types(fn_type_id.cc)) { - add_node_error(g, fn_proto->return_type, - buf_sprintf("return type '%s' not allowed in function with calling convention '%s'", - buf_ptr(&fn_type_id.return_type->name), - calling_convention_name(fn_type_id.cc))); - return g->builtin_types.entry_invalid; - } - return get_generic_fn_type(g, &fn_type_id); case TypeTableEntryIdUnreachable: case TypeTableEntryIdVoid: case TypeTableEntryIdBool: From 009e90f446ec7a81d602906fda9b17a6af1859f5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Aug 2018 17:11:43 -0400 Subject: [PATCH 08/56] fix @typeInfo unable to distinguish compile error vs no-payload closes #1421 closes #1426 --- src/ir.cpp | 135 ++++++++++++++++++++-------------------- test/compile_errors.zig | 17 +++++ 2 files changed, 83 insertions(+), 69 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index ae69f9c817..710456179c 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -14444,8 +14444,7 @@ static IrInstruction *ir_analyze_container_field_ptr(IrAnalyze *ira, Buf *field_ ConstExprValue *payload_val = union_val->data.x_union.payload; TypeTableEntry *field_type = field->type_entry; - if (field_type->id == TypeTableEntryIdVoid) - { + if (field_type->id == TypeTableEntryIdVoid) { assert(payload_val == nullptr); payload_val = create_const_vals(1); payload_val->special = ConstValSpecialStatic; @@ -16797,12 +16796,11 @@ static TypeTableEntry *ir_type_info_get_type(IrAnalyze *ira, const char *type_na return var->value->data.x_type; } -static bool ir_make_type_info_defs(IrAnalyze *ira, ConstExprValue *out_val, ScopeDecls *decls_scope) -{ +static Error ir_make_type_info_defs(IrAnalyze *ira, ConstExprValue *out_val, ScopeDecls *decls_scope) { Error err; TypeTableEntry *type_info_definition_type = ir_type_info_get_type(ira, "Definition", nullptr); if ((err = ensure_complete_type(ira->codegen, type_info_definition_type))) - return false; + return err; ensure_field_index(type_info_definition_type, "name", 0); ensure_field_index(type_info_definition_type, "is_pub", 1); @@ -16810,38 +16808,33 @@ static bool ir_make_type_info_defs(IrAnalyze *ira, ConstExprValue *out_val, Scop TypeTableEntry *type_info_definition_data_type = ir_type_info_get_type(ira, "Data", type_info_definition_type); if ((err = ensure_complete_type(ira->codegen, type_info_definition_data_type))) - return false; + return err; TypeTableEntry *type_info_fn_def_type = ir_type_info_get_type(ira, "FnDef", type_info_definition_data_type); if ((err = ensure_complete_type(ira->codegen, type_info_fn_def_type))) - return false; + return err; TypeTableEntry *type_info_fn_def_inline_type = ir_type_info_get_type(ira, "Inline", type_info_fn_def_type); if ((err = ensure_complete_type(ira->codegen, type_info_fn_def_inline_type))) - return false; + return err; // Loop through our definitions once to figure out how many definitions we will generate info for. auto decl_it = decls_scope->decl_table.entry_iterator(); decltype(decls_scope->decl_table)::Entry *curr_entry = nullptr; int definition_count = 0; - while ((curr_entry = decl_it.next()) != nullptr) - { + while ((curr_entry = decl_it.next()) != nullptr) { // If the definition is unresolved, force it to be resolved again. - if (curr_entry->value->resolution == TldResolutionUnresolved) - { + if (curr_entry->value->resolution == TldResolutionUnresolved) { resolve_top_level_decl(ira->codegen, curr_entry->value, false, curr_entry->value->source_node); - if (curr_entry->value->resolution != TldResolutionOk) - { - return false; + if (curr_entry->value->resolution != TldResolutionOk) { + return ErrorSemanticAnalyzeFail; } } // Skip comptime blocks and test functions. - if (curr_entry->value->id != TldIdCompTime) - { - if (curr_entry->value->id == TldIdFn) - { + if (curr_entry->value->id != TldIdCompTime) { + if (curr_entry->value->id == TldIdFn) { FnTableEntry *fn_entry = ((TldFn *)curr_entry->value)->fn_entry; if (fn_entry->is_test) continue; @@ -16863,13 +16856,11 @@ static bool ir_make_type_info_defs(IrAnalyze *ira, ConstExprValue *out_val, Scop decl_it = decls_scope->decl_table.entry_iterator(); curr_entry = nullptr; int definition_index = 0; - while ((curr_entry = decl_it.next()) != nullptr) - { + while ((curr_entry = decl_it.next()) != nullptr) { // Skip comptime blocks and test functions. - if (curr_entry->value->id == TldIdCompTime) + if (curr_entry->value->id == TldIdCompTime) { continue; - else if (curr_entry->value->id == TldIdFn) - { + } else if (curr_entry->value->id == TldIdFn) { FnTableEntry *fn_entry = ((TldFn *)curr_entry->value)->fn_entry; if (fn_entry->is_test) continue; @@ -16892,13 +16883,12 @@ static bool ir_make_type_info_defs(IrAnalyze *ira, ConstExprValue *out_val, Scop inner_fields[2].data.x_union.parent.data.p_struct.struct_val = definition_val; inner_fields[2].data.x_union.parent.data.p_struct.field_index = 1; - switch (curr_entry->value->id) - { + switch (curr_entry->value->id) { case TldIdVar: { VariableTableEntry *var = ((TldVar *)curr_entry->value)->var; if ((err = ensure_complete_type(ira->codegen, var->value->type))) - return false; + return ErrorSemanticAnalyzeFail; if (var->value->type->id == TypeTableEntryIdMetaType) { @@ -17029,7 +17019,7 @@ static bool ir_make_type_info_defs(IrAnalyze *ira, ConstExprValue *out_val, Scop { TypeTableEntry *type_entry = ((TldContainer *)curr_entry->value)->type_entry; if ((err = ensure_complete_type(ira->codegen, type_entry))) - return false; + return ErrorSemanticAnalyzeFail; // This is a type. bigint_init_unsigned(&inner_fields[2].data.x_union.tag, 0); @@ -17051,7 +17041,7 @@ static bool ir_make_type_info_defs(IrAnalyze *ira, ConstExprValue *out_val, Scop } assert(definition_index == definition_count); - return true; + return ErrorNone; } static ConstExprValue *create_ptr_like_type_info(IrAnalyze *ira, TypeTableEntry *ptr_type_entry) { @@ -17109,30 +17099,31 @@ static ConstExprValue *create_ptr_like_type_info(IrAnalyze *ira, TypeTableEntry return result; }; -static ConstExprValue *ir_make_type_info_value(IrAnalyze *ira, TypeTableEntry *type_entry) { +static void make_enum_field_val(IrAnalyze *ira, ConstExprValue *enum_field_val, TypeEnumField *enum_field, + TypeTableEntry *type_info_enum_field_type) +{ + enum_field_val->special = ConstValSpecialStatic; + enum_field_val->type = type_info_enum_field_type; + + ConstExprValue *inner_fields = create_const_vals(2); + inner_fields[1].special = ConstValSpecialStatic; + inner_fields[1].type = ira->codegen->builtin_types.entry_usize; + + ConstExprValue *name = create_const_str_lit(ira->codegen, enum_field->name); + init_const_slice(ira->codegen, &inner_fields[0], name, 0, buf_len(enum_field->name), true); + + bigint_init_bigint(&inner_fields[1].data.x_bigint, &enum_field->value); + + enum_field_val->data.x_struct.fields = inner_fields; +} + +static Error ir_make_type_info_value(IrAnalyze *ira, TypeTableEntry *type_entry, ConstExprValue **out) { Error err; assert(type_entry != nullptr); assert(!type_is_invalid(type_entry)); if ((err = ensure_complete_type(ira->codegen, type_entry))) - return nullptr; - - const auto make_enum_field_val = [ira](ConstExprValue *enum_field_val, TypeEnumField *enum_field, - TypeTableEntry *type_info_enum_field_type) { - enum_field_val->special = ConstValSpecialStatic; - enum_field_val->type = type_info_enum_field_type; - - ConstExprValue *inner_fields = create_const_vals(2); - inner_fields[1].special = ConstValSpecialStatic; - inner_fields[1].type = ira->codegen->builtin_types.entry_usize; - - ConstExprValue *name = create_const_str_lit(ira->codegen, enum_field->name); - init_const_slice(ira->codegen, &inner_fields[0], name, 0, buf_len(enum_field->name), true); - - bigint_init_bigint(&inner_fields[1].data.x_bigint, &enum_field->value); - - enum_field_val->data.x_struct.fields = inner_fields; - }; + return err; if (type_entry == ira->codegen->builtin_types.entry_global_error_set) { zig_panic("TODO implement @typeInfo for global error set"); @@ -17155,13 +17146,16 @@ static ConstExprValue *ir_make_type_info_value(IrAnalyze *ira, TypeTableEntry *t case TypeTableEntryIdBlock: case TypeTableEntryIdArgTuple: case TypeTableEntryIdOpaque: - return nullptr; + *out = nullptr; + return ErrorNone; default: { // Lookup an available value in our cache. auto entry = ira->codegen->type_info_cache.maybe_get(type_entry); - if (entry != nullptr) - return entry->value; + if (entry != nullptr) { + *out = entry->value; + return ErrorNone; + } // Fallthrough if we don't find one. } @@ -17312,15 +17306,15 @@ static ConstExprValue *ir_make_type_info_value(IrAnalyze *ira, TypeTableEntry *t { TypeEnumField *enum_field = &type_entry->data.enumeration.fields[enum_field_index]; ConstExprValue *enum_field_val = &enum_field_array->data.x_array.s_none.elements[enum_field_index]; - make_enum_field_val(enum_field_val, enum_field, type_info_enum_field_type); + make_enum_field_val(ira, enum_field_val, enum_field, type_info_enum_field_type); enum_field_val->data.x_struct.parent.id = ConstParentIdArray; enum_field_val->data.x_struct.parent.data.p_array.array_val = enum_field_array; enum_field_val->data.x_struct.parent.data.p_array.elem_index = enum_field_index; } // defs: []TypeInfo.Definition ensure_field_index(result->type, "defs", 3); - if (!ir_make_type_info_defs(ira, &fields[3], type_entry->data.enumeration.decls_scope)) - return nullptr; + if ((err = ir_make_type_info_defs(ira, &fields[3], type_entry->data.enumeration.decls_scope))) + return err; break; } @@ -17346,8 +17340,7 @@ static ConstExprValue *ir_make_type_info_value(IrAnalyze *ira, TypeTableEntry *t error_array->data.x_array.s_none.elements = create_const_vals(error_count); init_const_slice(ira->codegen, &fields[0], error_array, 0, error_count, false); - for (uint32_t error_index = 0; error_index < error_count; error_index++) - { + for (uint32_t error_index = 0; error_index < error_count; error_index++) { ErrorTableEntry *error = type_entry->data.error_set.errors[error_index]; ConstExprValue *error_val = &error_array->data.x_array.s_none.elements[error_index]; @@ -17425,9 +17418,9 @@ static ConstExprValue *ir_make_type_info_value(IrAnalyze *ira, TypeTableEntry *t tag_type->type = ira->codegen->builtin_types.entry_type; tag_type->data.x_type = type_entry->data.unionation.tag_type; fields[1].data.x_optional = tag_type; - } - else + } else { fields[1].data.x_optional = nullptr; + } // fields: []TypeInfo.UnionField ensure_field_index(result->type, "fields", 2); @@ -17460,7 +17453,7 @@ static ConstExprValue *ir_make_type_info_value(IrAnalyze *ira, TypeTableEntry *t inner_fields[1].data.x_optional = nullptr; } else { inner_fields[1].data.x_optional = create_const_vals(1); - make_enum_field_val(inner_fields[1].data.x_optional, union_field->enum_field, type_info_enum_field_type); + make_enum_field_val(ira, inner_fields[1].data.x_optional, union_field->enum_field, type_info_enum_field_type); } inner_fields[2].special = ConstValSpecialStatic; @@ -17477,8 +17470,8 @@ static ConstExprValue *ir_make_type_info_value(IrAnalyze *ira, TypeTableEntry *t } // defs: []TypeInfo.Definition ensure_field_index(result->type, "defs", 3); - if (!ir_make_type_info_defs(ira, &fields[3], type_entry->data.unionation.decls_scope)) - return nullptr; + if ((err = ir_make_type_info_defs(ira, &fields[3], type_entry->data.unionation.decls_scope))) + return err; break; } @@ -17551,8 +17544,8 @@ static ConstExprValue *ir_make_type_info_value(IrAnalyze *ira, TypeTableEntry *t } // defs: []TypeInfo.Definition ensure_field_index(result->type, "defs", 2); - if (!ir_make_type_info_defs(ira, &fields[2], type_entry->data.structure.decls_scope)) - return nullptr; + if ((err = ir_make_type_info_defs(ira, &fields[2], type_entry->data.structure.decls_scope))) + return err; break; } @@ -17665,7 +17658,8 @@ static ConstExprValue *ir_make_type_info_value(IrAnalyze *ira, TypeTableEntry *t { TypeTableEntry *fn_type = type_entry->data.bound_fn.fn_type; assert(fn_type->id == TypeTableEntryIdFn); - result = ir_make_type_info_value(ira, fn_type); + if ((err = ir_make_type_info_value(ira, fn_type, &result))) + return err; break; } @@ -17673,12 +17667,14 @@ static ConstExprValue *ir_make_type_info_value(IrAnalyze *ira, TypeTableEntry *t assert(result != nullptr); ira->codegen->type_info_cache.put(type_entry, result); - return result; + *out = result; + return ErrorNone; } static TypeTableEntry *ir_analyze_instruction_type_info(IrAnalyze *ira, IrInstructionTypeInfo *instruction) { + Error err; IrInstruction *type_value = instruction->type_value->other; TypeTableEntry *type_entry = ir_resolve_type(ira, type_value); if (type_is_invalid(type_entry)) @@ -17686,15 +17682,16 @@ static TypeTableEntry *ir_analyze_instruction_type_info(IrAnalyze *ira, TypeTableEntry *result_type = ir_type_info_get_type(ira, nullptr, nullptr); + ConstExprValue *payload; + if ((err = ir_make_type_info_value(ira, type_entry, &payload))) + return ira->codegen->builtin_types.entry_invalid; + ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); out_val->type = result_type; bigint_init_unsigned(&out_val->data.x_union.tag, type_id_index(type_entry)); - - ConstExprValue *payload = ir_make_type_info_value(ira, type_entry); out_val->data.x_union.payload = payload; - if (payload != nullptr) - { + if (payload != nullptr) { assert(payload->type->id == TypeTableEntryIdStruct); payload->data.x_struct.parent.id = ConstParentIdUnion; payload->data.x_struct.parent.data.p_union.union_val = out_val; diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 56b2c51d74..d0795c4550 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -1,6 +1,23 @@ const tests = @import("tests.zig"); pub fn addCases(cases: *tests.CompileErrorContext) void { + cases.add( + "@typeInfo causing depend on itself compile error", + \\const start = struct { + \\ fn crash() bug() { + \\ return bug; + \\ } + \\}; + \\fn bug() void { + \\ _ = @typeInfo(start).Struct; + \\} + \\export fn entry() void { + \\ var boom = start.crash(); + \\} + , + ".tmp_source.zig:2:5: error: 'crash' depends on itself", + ); + cases.add( "@handle() called outside of function definition", \\var handle_undef: promise = undefined; From ecc54640243ba84ffa3656b73aa7dd6b53474462 Mon Sep 17 00:00:00 2001 From: tgschultz Date: Mon, 27 Aug 2018 16:25:33 -0500 Subject: [PATCH 09/56] Handle unions differently in std.fmt (#1432) * Handle unions differently in std.fmt Print the active tag's value in tagged unions. Untagged unions considered unsafe to print and treated like a pointer or an array. --- std/fmt/index.zig | 115 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 98 insertions(+), 17 deletions(-) diff --git a/std/fmt/index.zig b/std/fmt/index.zig index 82e9a5ba39..17039f3953 100644 --- a/std/fmt/index.zig +++ b/std/fmt/index.zig @@ -163,26 +163,47 @@ pub fn formatType( } break :cf false; }; - if (has_cust_fmt) return value.format(fmt, context, Errors, output); + try output(context, @typeName(T)); - if (comptime @typeId(T) == builtin.TypeId.Enum) { - try output(context, "."); - try formatType(@tagName(value), "", context, Errors, output); - return; + switch (comptime @typeId(T)) { + builtin.TypeId.Enum => { + try output(context, "."); + try formatType(@tagName(value), "", context, Errors, output); + return; + }, + builtin.TypeId.Struct => { + comptime var field_i = 0; + inline while (field_i < @memberCount(T)) : (field_i += 1) { + if (field_i == 0) { + try output(context, "{ ."); + } else { + try output(context, ", ."); + } + try output(context, @memberName(T, field_i)); + try output(context, " = "); + try formatType(@field(value, @memberName(T, field_i)), "", context, Errors, output); + } + try output(context, " }"); + }, + builtin.TypeId.Union => { + const info = @typeInfo(T).Union; + if(info.tag_type) |UnionTagType| { + try output(context, "{ ."); + try output(context, @tagName(UnionTagType(value))); + try output(context, " = "); + inline for(info.fields) |u_field| { + if(@enumToInt(UnionTagType(value)) == u_field.enum_field.?.value) { + try formatType(@field(value, u_field.name), "", context, Errors, output); + } + } + try output(context, " }"); + } else { + try format(context, Errors, output, "@{x}", @ptrToInt(&value)); + } + }, + else => unreachable, } - comptime var field_i = 0; - inline while (field_i < @memberCount(T)) : (field_i += 1) { - if (field_i == 0) { - try output(context, "{ ."); - } else { - try output(context, ", ."); - } - try output(context, @memberName(T, field_i)); - try output(context, " = "); - try formatType(@field(value, @memberName(T, field_i)), "", context, Errors, output); - } - try output(context, " }"); return; }, builtin.TypeId.Pointer => |ptr_info| switch (ptr_info.size) { @@ -1194,6 +1215,66 @@ test "fmt.format" { try testFmt("point: (10.200,2.220)\n", "point: {}\n", value); try testFmt("dim: 10.200x2.220\n", "dim: {d}\n", value); } + //struct format + { + const S = struct { + a: u32, + b: error, + }; + + const inst = S { + .a = 456, + .b = error.Unused, + }; + + try testFmt("S{ .a = 456, .b = error.Unused }", "{}", inst); + } + //union format + { + const TU = union(enum) + { + float: f32, + int: u32, + }; + + const UU = union + { + float: f32, + int: u32, + }; + + const EU = extern union + { + float: f32, + int: u32, + }; + + const tu_inst = TU{ .int = 123, }; + const uu_inst = UU{ .int = 456, }; + const eu_inst = EU{ .float = 321.123, }; + + try testFmt("TU{ .int = 123 }", "{}", tu_inst); + + var buf: [100]u8 = undefined; + const uu_result = try bufPrint(buf[0..], "{}", uu_inst); + debug.assert(mem.eql(u8, uu_result[0..3], "UU@")); + + const eu_result = try bufPrint(buf[0..], "{}", eu_inst); + debug.assert(mem.eql(u8, uu_result[0..3], "EU@")); + } + //enum format + { + const E = enum + { + One, + Two, + Three, + }; + + const inst = E.Two; + + try testFmt("E.Two", "{}", inst); + } } fn testFmt(expected: []const u8, comptime template: []const u8, args: ...) !void { From c48be3a742cd82a8007512bfe145374e4b3750be Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Aug 2018 17:44:58 -0400 Subject: [PATCH 10/56] langref: document exporting a library closes #1431 --- doc/langref.html.in | 55 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/doc/langref.html.in b/doc/langref.html.in index 465f3c56a7..495b19580c 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -7057,6 +7057,61 @@ const c = @cImport({ }); {#code_end#} {#see_also|@cImport|@cInclude|@cDefine|@cUndef|@import#} + {#header_close#} + {#header_open|Exporting a C Library#} +

+ One of the primary use cases for Zig is exporting a library with the C ABI for other programming languages + to call into. The export keyword in front of functions, variables, and types causes them to + be part of the library API: +

+

mathtest.zig

+ {#code_begin|syntax#} +export fn add(a: i32, b: i32) i32 { + return a + b; +} + {#code_end#} +

To make a shared library:

+ 
$ zig build-lib mathtest.zig
+
+

To make a static library:

+ 
$ zig build-lib mathtest.zig --static
+
+

Here is an example with the {#link|Zig Build System#}:

+

test.c

+ 
// This header is generated by zig from mathtest.zig
+#include "mathtest.h"
+#include <assert.h>
+
+int main(int argc, char **argv) {
+    assert(add(42, 1337) == 1379);
+    return 0;
+}
+

build.zig

+ {#code_begin|syntax#} +const Builder = @import("std").build.Builder; + +pub fn build(b: *Builder) void { + const lib = b.addSharedLibrary("mathtest", "mathtest.zig", b.version(1, 0, 0)); + + const exe = b.addCExecutable("test"); + exe.addCompileFlags([][]const u8{"-std=c99"}); + exe.addSourceFile("test.c"); + exe.linkLibrary(lib); + + b.default_step.dependOn(&exe.step); + + const run_cmd = b.addCommand(".", b.env_map, [][]const u8{exe.getOutputPath()}); + run_cmd.step.dependOn(&exe.step); + + const test_step = b.step("test", "Test the program"); + test_step.dependOn(&run_cmd.step); +} + {#code_end#} +

terminal

+ 
$ zig build
+$ ./test
+$ echo $?
+0
{#header_close#} {#header_open|Mixing Object Files#}

From e2a9f2ef988e6ff4ec9f943ead9285c67c56ffd7 Mon Sep 17 00:00:00 2001 From: raulgrell Date: Mon, 27 Aug 2018 16:37:37 +0100 Subject: [PATCH 11/56] Allow implicit cast from *T and [*]T to ?*c_void --- src/ir.cpp | 20 ++++++++++++++++---- test/cases/cast.zig | 24 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index 710456179c..b4af7f42e2 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -60,7 +60,7 @@ enum ConstCastResultId { ConstCastResultIdType, ConstCastResultIdUnresolvedInferredErrSet, ConstCastResultIdAsyncAllocatorType, - ConstCastResultIdNullWrapPtr, + ConstCastResultIdNullWrapPtr }; struct ConstCastOnly; @@ -8471,9 +8471,9 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, TypeTableEntry if (wanted_type == actual_type) return result; - // * and [*] can do a const-cast-only to ?* and ?[*], respectively - // but not if there is a mutable parent pointer - // and not if the pointer is zero bits + // *T and [*]T may const-cast-only to ?*U and ?[*]U, respectively + // but not if we want a mutable pointer + // and not if the actual pointer has zero bits if (!wanted_is_mutable && wanted_type->id == TypeTableEntryIdOptional && wanted_type->data.maybe.child_type->id == TypeTableEntryIdPointer && actual_type->id == TypeTableEntryIdPointer && type_has_bits(actual_type)) @@ -8488,6 +8488,18 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, TypeTableEntry return result; } + // *T and [*]T can always cast to ?*c_void + if (wanted_type->id == TypeTableEntryIdPointer && + wanted_type->data.pointer.ptr_len == PtrLenSingle && + wanted_type->data.pointer.child_type == g->builtin_types.entry_c_void && + actual_type->id == TypeTableEntryIdPointer && + (!actual_type->data.pointer.is_const || wanted_type->data.pointer.is_const) && + (!actual_type->data.pointer.is_volatile || wanted_type->data.pointer.is_volatile)) + { + assert(actual_type->data.pointer.alignment >= wanted_type->data.pointer.alignment); + return result; + } + // pointer const if (wanted_type->id == TypeTableEntryIdPointer && actual_type->id == TypeTableEntryIdPointer) { ConstCastOnly child = types_match_const_cast_only(ira, wanted_type->data.pointer.child_type, diff --git a/test/cases/cast.zig b/test/cases/cast.zig index df37bd1dd9..dde714b2be 100644 --- a/test/cases/cast.zig +++ b/test/cases/cast.zig @@ -495,4 +495,28 @@ test "implicit cast from *[N]T to ?[*]T" { x.?[0] = 8; y[3] = 6; assert(std.mem.eql(u16, x.?[0..4], y[0..4])); +} + +test "implicit cast from *T to ?*c_void" { + var a: u8 = 1; + incrementVoidPtrValue(&a); + std.debug.assert(a == 2); +} + +fn incrementVoidPtrValue(value: ?*c_void) void { + @ptrCast(*u8, value.?).* += 1; +} + +test "implicit cast from [*]T to ?*c_void" { + var a = []u8{3, 2, 1}; + incrementVoidPtrArray(a[0..].ptr, 3); + std.debug.assert(std.mem.eql(u8, a, []u8{4, 3, 2})); +} + +fn incrementVoidPtrArray(array: ?*c_void, len: usize) void { + var n: usize = 0; + while(n < len) : (n += 1) { + std.debug.warn("{}", n); + @ptrCast([*]u8, array.?)[n] += 1; + } } \ No newline at end of file From 45d9d9f953df2167b35878ea02551d9c86ad8a0f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Aug 2018 18:31:28 -0400 Subject: [PATCH 12/56] minor fixups --- src/ir.cpp | 2 +- test/cases/cast.zig | 31 +++++++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index b4af7f42e2..3117410cd2 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -8488,7 +8488,7 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, TypeTableEntry return result; } - // *T and [*]T can always cast to ?*c_void + // *T and [*]T can always cast to *c_void if (wanted_type->id == TypeTableEntryIdPointer && wanted_type->data.pointer.ptr_len == PtrLenSingle && wanted_type->data.pointer.child_type == g->builtin_types.entry_c_void && diff --git a/test/cases/cast.zig b/test/cases/cast.zig index dde714b2be..cc5e4b4394 100644 --- a/test/cases/cast.zig +++ b/test/cases/cast.zig @@ -487,20 +487,20 @@ fn MakeType(comptime T: type) type { } test "implicit cast from *[N]T to ?[*]T" { - var x: ?[*]u16 = null; - var y: [4]u16 = [4]u16 {0, 1, 2, 3}; + var x: ?[*]u16 = null; + var y: [4]u16 = [4]u16{ 0, 1, 2, 3 }; - x = &y; - assert(std.mem.eql(u16, x.?[0..4], y[0..4])); - x.?[0] = 8; - y[3] = 6; - assert(std.mem.eql(u16, x.?[0..4], y[0..4])); + x = &y; + assert(std.mem.eql(u16, x.?[0..4], y[0..4])); + x.?[0] = 8; + y[3] = 6; + assert(std.mem.eql(u16, x.?[0..4], y[0..4])); } test "implicit cast from *T to ?*c_void" { - var a: u8 = 1; - incrementVoidPtrValue(&a); - std.debug.assert(a == 2); + var a: u8 = 1; + incrementVoidPtrValue(&a); + std.debug.assert(a == 2); } fn incrementVoidPtrValue(value: ?*c_void) void { @@ -508,15 +508,14 @@ fn incrementVoidPtrValue(value: ?*c_void) void { } test "implicit cast from [*]T to ?*c_void" { - var a = []u8{3, 2, 1}; - incrementVoidPtrArray(a[0..].ptr, 3); - std.debug.assert(std.mem.eql(u8, a, []u8{4, 3, 2})); + var a = []u8{ 3, 2, 1 }; + incrementVoidPtrArray(a[0..].ptr, 3); + assert(std.mem.eql(u8, a, []u8{ 4, 3, 2 })); } fn incrementVoidPtrArray(array: ?*c_void, len: usize) void { var n: usize = 0; - while(n < len) : (n += 1) { - std.debug.warn("{}", n); + while (n < len) : (n += 1) { @ptrCast([*]u8, array.?)[n] += 1; } -} \ No newline at end of file +} From 4f2d49fd138be67ad01d9c1b9086cb6a41530944 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Aug 2018 19:21:38 -0400 Subject: [PATCH 13/56] std.zig.parse: fix parsing of doc comments after fields closes #1404 --- std/zig/parse.zig | 71 +++++++++++++++++++++++++++++++++-------- std/zig/parser_test.zig | 17 ++++++++++ 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/std/zig/parse.zig b/std/zig/parse.zig index fb49d2a2ba..b9c05f8da0 100644 --- a/std/zig/parse.zig +++ b/std/zig/parse.zig @@ -340,7 +340,12 @@ pub fn parse(allocator: *mem.Allocator, source: []const u8) !ast.Tree { const node_ptr = try ctx.container_decl.fields_and_decls.addOne(); node_ptr.* = &node.base; - stack.append(State{ .FieldListCommaOrEnd = ctx.container_decl }) catch unreachable; + try stack.append(State{ + .FieldListCommaOrEnd = FieldCtx{ + .doc_comments = &node.doc_comments, + .container_decl = ctx.container_decl, + }, + }); try stack.append(State{ .Expression = OptionalCtx{ .Required = &node.type_expr } }); try stack.append(State{ .ExpectToken = Token.Id.Colon }); continue; @@ -458,7 +463,12 @@ pub fn parse(allocator: *mem.Allocator, source: []const u8) !ast.Tree { const node_ptr = try container_decl.fields_and_decls.addOne(); node_ptr.* = &node.base; - try stack.append(State{ .FieldListCommaOrEnd = container_decl }); + try stack.append(State{ + .FieldListCommaOrEnd = FieldCtx{ + .doc_comments = &node.doc_comments, + .container_decl = container_decl, + }, + }); try stack.append(State{ .TypeExprBegin = OptionalCtx{ .Required = &node.type_expr } }); try stack.append(State{ .ExpectToken = Token.Id.Colon }); continue; @@ -473,7 +483,12 @@ pub fn parse(allocator: *mem.Allocator, source: []const u8) !ast.Tree { }); try container_decl.fields_and_decls.push(&node.base); - stack.append(State{ .FieldListCommaOrEnd = container_decl }) catch unreachable; + try stack.append(State{ + .FieldListCommaOrEnd = FieldCtx{ + .doc_comments = &node.doc_comments, + .container_decl = container_decl, + }, + }); try stack.append(State{ .FieldInitValue = OptionalCtx{ .RequiredNull = &node.value_expr } }); try stack.append(State{ .TypeExprBegin = OptionalCtx{ .RequiredNull = &node.type_expr } }); try stack.append(State{ .IfToken = Token.Id.Colon }); @@ -488,7 +503,12 @@ pub fn parse(allocator: *mem.Allocator, source: []const u8) !ast.Tree { }); try container_decl.fields_and_decls.push(&node.base); - stack.append(State{ .FieldListCommaOrEnd = container_decl }) catch unreachable; + try stack.append(State{ + .FieldListCommaOrEnd = FieldCtx{ + .doc_comments = &node.doc_comments, + .container_decl = container_decl, + }, + }); try stack.append(State{ .Expression = OptionalCtx{ .RequiredNull = &node.value } }); try stack.append(State{ .IfToken = Token.Id.Equal }); continue; @@ -1265,17 +1285,35 @@ pub fn parse(allocator: *mem.Allocator, source: []const u8) !ast.Tree { }, } }, - State.FieldListCommaOrEnd => |container_decl| { - switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RBrace)) { - ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| { - container_decl.rbrace_token = end; - continue; - } else { - try stack.append(State{ .ContainerDecl = container_decl }); + State.FieldListCommaOrEnd => |field_ctx| { + const end_token = nextToken(&tok_it, &tree); + const end_token_index = end_token.index; + const end_token_ptr = end_token.ptr; + switch (end_token_ptr.id) { + Token.Id.Comma => { + if (eatToken(&tok_it, &tree, Token.Id.DocComment)) |doc_comment_token| { + const loc = tree.tokenLocation(end_token_ptr.end, doc_comment_token); + if (loc.line == 0) { + try pushDocComment(arena, doc_comment_token, field_ctx.doc_comments); + } else { + prevToken(&tok_it, &tree); + } + } + + try stack.append(State{ .ContainerDecl = field_ctx.container_decl }); continue; }, - ExpectCommaOrEndResult.parse_error => |e| { - try tree.errors.push(e); + Token.Id.RBrace => { + field_ctx.container_decl.rbrace_token = end_token_index; + continue; + }, + else => { + try tree.errors.push(Error{ + .ExpectedCommaOrEnd = Error.ExpectedCommaOrEnd{ + .token = end_token_index, + .end_id = end_token_ptr.id, + }, + }); return tree; }, } @@ -2813,6 +2851,11 @@ const ExprListCtx = struct { ptr: *TokenIndex, }; +const FieldCtx = struct { + container_decl: *ast.Node.ContainerDecl, + doc_comments: *?*ast.Node.DocComment, +}; + fn ListSave(comptime List: type) type { return struct { list: *List, @@ -2950,7 +2993,7 @@ const State = union(enum) { ExprListCommaOrEnd: ExprListCtx, FieldInitListItemOrEnd: ListSave(ast.Node.SuffixOp.Op.InitList), FieldInitListCommaOrEnd: ListSave(ast.Node.SuffixOp.Op.InitList), - FieldListCommaOrEnd: *ast.Node.ContainerDecl, + FieldListCommaOrEnd: FieldCtx, FieldInitValue: OptionalCtx, ErrorTagListItemOrEnd: ListSave(ast.Node.ErrorSetDecl.DeclList), ErrorTagListCommaOrEnd: ListSave(ast.Node.ErrorSetDecl.DeclList), diff --git a/std/zig/parser_test.zig b/std/zig/parser_test.zig index 582bffdf3d..6ea25e54f1 100644 --- a/std/zig/parser_test.zig +++ b/std/zig/parser_test.zig @@ -1,3 +1,20 @@ +test "zig fmt: correctly move doc comments on struct fields" { + try testTransform( + \\pub const section_64 = extern struct { + \\ sectname: [16]u8, /// name of this section + \\ segname: [16]u8, /// segment this section goes in + \\}; + , + \\pub const section_64 = extern struct { + \\ /// name of this section + \\ sectname: [16]u8, + \\ /// segment this section goes in + \\ segname: [16]u8, + \\}; + \\ + ); +} + test "zig fmt: preserve space between async fn definitions" { try testCanonical( \\async fn a() void {} From fb6d3859e80faf98335c55414bb73490977d2089 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Aug 2018 19:25:18 -0400 Subject: [PATCH 14/56] zig fmt --- std/event/tcp.zig | 2 +- std/fmt/index.zig | 46 ++- std/macho.zig | 732 ++++++++++++++++++++++++++---------- std/os/windows/advapi32.zig | 19 +- std/os/windows/index.zig | 12 +- std/os/windows/ole32.zig | 1 - std/os/zen.zig | 49 ++- std/rb.zig | 28 +- 8 files changed, 621 insertions(+), 268 deletions(-) diff --git a/std/event/tcp.zig b/std/event/tcp.zig index 19cce4a5e5..d8b97659a9 100644 --- a/std/event/tcp.zig +++ b/std/event/tcp.zig @@ -89,7 +89,7 @@ pub const Server = struct { error.ProcessFdQuotaExceeded => { errdefer std.os.emfile_promise_queue.remove(&self.waiting_for_emfile_node); suspend { - self.waiting_for_emfile_node = PromiseNode.init( @handle() ); + self.waiting_for_emfile_node = PromiseNode.init(@handle()); std.os.emfile_promise_queue.append(&self.waiting_for_emfile_node); } continue; diff --git a/std/fmt/index.zig b/std/fmt/index.zig index 17039f3953..6d23eebd0b 100644 --- a/std/fmt/index.zig +++ b/std/fmt/index.zig @@ -188,12 +188,12 @@ pub fn formatType( }, builtin.TypeId.Union => { const info = @typeInfo(T).Union; - if(info.tag_type) |UnionTagType| { + if (info.tag_type) |UnionTagType| { try output(context, "{ ."); try output(context, @tagName(UnionTagType(value))); try output(context, " = "); - inline for(info.fields) |u_field| { - if(@enumToInt(UnionTagType(value)) == u_field.enum_field.?.value) { + inline for (info.fields) |u_field| { + if (@enumToInt(UnionTagType(value)) == u_field.enum_field.?.value) { try formatType(@field(value, u_field.name), "", context, Errors, output); } } @@ -1221,58 +1221,54 @@ test "fmt.format" { a: u32, b: error, }; - - const inst = S { + + const inst = S{ .a = 456, .b = error.Unused, }; - + try testFmt("S{ .a = 456, .b = error.Unused }", "{}", inst); } //union format { - const TU = union(enum) - { + const TU = union(enum) { float: f32, int: u32, }; - - const UU = union - { + + const UU = union { float: f32, int: u32, }; - - const EU = extern union - { + + const EU = extern union { float: f32, int: u32, }; - - const tu_inst = TU{ .int = 123, }; - const uu_inst = UU{ .int = 456, }; - const eu_inst = EU{ .float = 321.123, }; - + + const tu_inst = TU{ .int = 123 }; + const uu_inst = UU{ .int = 456 }; + const eu_inst = EU{ .float = 321.123 }; + try testFmt("TU{ .int = 123 }", "{}", tu_inst); - + var buf: [100]u8 = undefined; const uu_result = try bufPrint(buf[0..], "{}", uu_inst); debug.assert(mem.eql(u8, uu_result[0..3], "UU@")); - + const eu_result = try bufPrint(buf[0..], "{}", eu_inst); debug.assert(mem.eql(u8, uu_result[0..3], "EU@")); } //enum format { - const E = enum - { + const E = enum { One, Two, Three, }; - + const inst = E.Two; - + try testFmt("E.Two", "{}", inst); } } diff --git a/std/macho.zig b/std/macho.zig index 4325810b03..e1bbd755c6 100644 --- a/std/macho.zig +++ b/std/macho.zig @@ -1,4 +1,3 @@ - pub const mach_header = extern struct { magic: u32, cputype: cpu_type_t, @@ -25,26 +24,43 @@ pub const load_command = extern struct { cmdsize: u32, }; - /// The symtab_command contains the offsets and sizes of the link-edit 4.3BSD /// "stab" style symbol table information as described in the header files /// and . pub const symtab_command = extern struct { - cmd: u32, /// LC_SYMTAB - cmdsize: u32, /// sizeof(struct symtab_command) - symoff: u32, /// symbol table offset - nsyms: u32, /// number of symbol table entries - stroff: u32, /// string table offset - strsize: u32, /// string table size in bytes + /// LC_SYMTAB + cmd: u32, + + /// sizeof(struct symtab_command) + cmdsize: u32, + + /// symbol table offset + symoff: u32, + + /// number of symbol table entries + nsyms: u32, + + /// string table offset + stroff: u32, + + /// string table size in bytes + strsize: u32, }; /// The linkedit_data_command contains the offsets and sizes of a blob -/// of data in the __LINKEDIT segment. +/// of data in the __LINKEDIT segment. const linkedit_data_command = extern struct { - cmd: u32,/// LC_CODE_SIGNATURE, LC_SEGMENT_SPLIT_INFO, LC_FUNCTION_STARTS, LC_DATA_IN_CODE, LC_DYLIB_CODE_SIGN_DRS or LC_LINKER_OPTIMIZATION_HINT. - cmdsize: u32, /// sizeof(struct linkedit_data_command) - dataoff: u32 , /// file offset of data in __LINKEDIT segment - datasize: u32 , /// file size of data in __LINKEDIT segment + /// LC_CODE_SIGNATURE, LC_SEGMENT_SPLIT_INFO, LC_FUNCTION_STARTS, LC_DATA_IN_CODE, LC_DYLIB_CODE_SIGN_DRS or LC_LINKER_OPTIMIZATION_HINT. + cmd: u32, + + /// sizeof(struct linkedit_data_command) + cmdsize: u32, + + /// file offset of data in __LINKEDIT segment + dataoff: u32, + + /// file size of data in __LINKEDIT segment + datasize: u32, }; /// The segment load command indicates that a part of this file is to be @@ -58,16 +74,35 @@ const linkedit_data_command = extern struct { /// section structures directly follow the segment command and their size is /// reflected in cmdsize. pub const segment_command = extern struct { - cmd: u32,/// LC_SEGMENT - cmdsize: u32,/// includes sizeof section structs - segname: [16]u8,/// segment name - vmaddr: u32,/// memory address of this segment - vmsize: u32,/// memory size of this segment - fileoff: u32,/// file offset of this segment - filesize: u32,/// amount to map from the file - maxprot: vm_prot_t,/// maximum VM protection - initprot: vm_prot_t,/// initial VM protection - nsects: u32,/// number of sections in segment + /// LC_SEGMENT + cmd: u32, + + /// includes sizeof section structs + cmdsize: u32, + + /// segment name + segname: [16]u8, + + /// memory address of this segment + vmaddr: u32, + + /// memory size of this segment + vmsize: u32, + + /// file offset of this segment + fileoff: u32, + + /// amount to map from the file + filesize: u32, + + /// maximum VM protection + maxprot: vm_prot_t, + + /// initial VM protection + initprot: vm_prot_t, + + /// number of sections in segment + nsects: u32, flags: u32, }; @@ -76,17 +111,36 @@ pub const segment_command = extern struct { /// sections then section_64 structures directly follow the 64-bit segment /// command and their size is reflected in cmdsize. pub const segment_command_64 = extern struct { - cmd: u32, /// LC_SEGMENT_64 - cmdsize: u32, /// includes sizeof section_64 structs - segname: [16]u8, /// segment name - vmaddr: u64, /// memory address of this segment - vmsize: u64, /// memory size of this segment - fileoff: u64, /// file offset of this segment - filesize: u64, /// amount to map from the file - maxprot: vm_prot_t, /// maximum VM protection - initprot: vm_prot_t, /// initial VM protection - nsects: u32, /// number of sections in segment - flags: u32, + /// LC_SEGMENT_64 + cmd: u32, + + /// includes sizeof section_64 structs + cmdsize: u32, + + /// segment name + segname: [16]u8, + + /// memory address of this segment + vmaddr: u64, + + /// memory size of this segment + vmsize: u64, + + /// file offset of this segment + fileoff: u64, + + /// amount to map from the file + filesize: u64, + + /// maximum VM protection + maxprot: vm_prot_t, + + /// initial VM protection + initprot: vm_prot_t, + + /// number of sections in segment + nsects: u32, + flags: u32, }; /// A segment is made up of zero or more sections. Non-MH_OBJECT files have @@ -115,32 +169,76 @@ pub const segment_command_64 = extern struct { /// fields of the section structure for mach object files is described in the /// header file . pub const @"section" = extern struct { - sectname: [16]u8, /// name of this section - segname: [16]u8, /// segment this section goes in - addr: u32, /// memory address of this section - size: u32, /// size in bytes of this section - offset: u32, /// file offset of this section - @"align": u32, /// section alignment (power of 2) - reloff: u32, /// file offset of relocation entries - nreloc: u32, /// number of relocation entries - flags: u32, /// flags (section type and attributes - reserved1: u32, /// reserved (for offset or index) - reserved2: u32, /// reserved (for count or sizeof) + /// name of this section + sectname: [16]u8, + + /// segment this section goes in + segname: [16]u8, + + /// memory address of this section + addr: u32, + + /// size in bytes of this section + size: u32, + + /// file offset of this section + offset: u32, + + /// section alignment (power of 2) + @"align": u32, + + /// file offset of relocation entries + reloff: u32, + + /// number of relocation entries + nreloc: u32, + + /// flags (section type and attributes + flags: u32, + + /// reserved (for offset or index) + reserved1: u32, + + /// reserved (for count or sizeof) + reserved2: u32, }; pub const section_64 = extern struct { - sectname: [16]u8, /// name of this section - segname: [16]u8, /// segment this section goes in - addr: u64, /// memory address of this section - size: u64, /// size in bytes of this section - offset: u32, /// file offset of this section - @"align": u32, /// section alignment (power of 2) - reloff: u32, /// file offset of relocation entries - nreloc: u32, /// number of relocation entries - flags: u32, /// flags (section type and attributes - reserved1: u32, /// reserved (for offset or index) - reserved2: u32, /// reserved (for count or sizeof) - reserved3: u32, /// reserved + /// name of this section + sectname: [16]u8, + + /// segment this section goes in + segname: [16]u8, + + /// memory address of this section + addr: u64, + + /// size in bytes of this section + size: u64, + + /// file offset of this section + offset: u32, + + /// section alignment (power of 2) + @"align": u32, + + /// file offset of relocation entries + reloff: u32, + + /// number of relocation entries + nreloc: u32, + + /// flags (section type and attributes + flags: u32, + + /// reserved (for offset or index) + reserved1: u32, + + /// reserved (for count or sizeof) + reserved2: u32, + + /// reserved + reserved3: u32, }; pub const nlist = extern struct { @@ -168,116 +266,287 @@ pub const nlist_64 = extern struct { /// simply be ignored. pub const LC_REQ_DYLD = 0x80000000; -pub const LC_SEGMENT = 0x1; /// segment of this file to be mapped -pub const LC_SYMTAB = 0x2; /// link-edit stab symbol table info -pub const LC_SYMSEG = 0x3; /// link-edit gdb symbol table info (obsolete) -pub const LC_THREAD = 0x4; /// thread -pub const LC_UNIXTHREAD = 0x5; /// unix thread (includes a stack) -pub const LC_LOADFVMLIB = 0x6; /// load a specified fixed VM shared library -pub const LC_IDFVMLIB = 0x7; /// fixed VM shared library identification -pub const LC_IDENT = 0x8; /// object identification info (obsolete) -pub const LC_FVMFILE = 0x9; /// fixed VM file inclusion (internal use) -pub const LC_PREPAGE = 0xa; /// prepage command (internal use) -pub const LC_DYSYMTAB = 0xb; /// dynamic link-edit symbol table info -pub const LC_LOAD_DYLIB = 0xc; /// load a dynamically linked shared library -pub const LC_ID_DYLIB = 0xd; /// dynamically linked shared lib ident -pub const LC_LOAD_DYLINKER = 0xe; /// load a dynamic linker -pub const LC_ID_DYLINKER = 0xf; /// dynamic linker identification -pub const LC_PREBOUND_DYLIB = 0x10; /// modules prebound for a dynamically -pub const LC_ROUTINES = 0x11; /// image routines -pub const LC_SUB_FRAMEWORK = 0x12; /// sub framework -pub const LC_SUB_UMBRELLA = 0x13; /// sub umbrella -pub const LC_SUB_CLIENT = 0x14; /// sub client -pub const LC_SUB_LIBRARY = 0x15; /// sub library -pub const LC_TWOLEVEL_HINTS = 0x16; /// two-level namespace lookup hints -pub const LC_PREBIND_CKSUM = 0x17; /// prebind checksum +/// segment of this file to be mapped +pub const LC_SEGMENT = 0x1; + +/// link-edit stab symbol table info +pub const LC_SYMTAB = 0x2; + +/// link-edit gdb symbol table info (obsolete) +pub const LC_SYMSEG = 0x3; + +/// thread +pub const LC_THREAD = 0x4; + +/// unix thread (includes a stack) +pub const LC_UNIXTHREAD = 0x5; + +/// load a specified fixed VM shared library +pub const LC_LOADFVMLIB = 0x6; + +/// fixed VM shared library identification +pub const LC_IDFVMLIB = 0x7; + +/// object identification info (obsolete) +pub const LC_IDENT = 0x8; + +/// fixed VM file inclusion (internal use) +pub const LC_FVMFILE = 0x9; + +/// prepage command (internal use) +pub const LC_PREPAGE = 0xa; + +/// dynamic link-edit symbol table info +pub const LC_DYSYMTAB = 0xb; + +/// load a dynamically linked shared library +pub const LC_LOAD_DYLIB = 0xc; + +/// dynamically linked shared lib ident +pub const LC_ID_DYLIB = 0xd; + +/// load a dynamic linker +pub const LC_LOAD_DYLINKER = 0xe; + +/// dynamic linker identification +pub const LC_ID_DYLINKER = 0xf; + +/// modules prebound for a dynamically +pub const LC_PREBOUND_DYLIB = 0x10; + +/// image routines +pub const LC_ROUTINES = 0x11; + +/// sub framework +pub const LC_SUB_FRAMEWORK = 0x12; + +/// sub umbrella +pub const LC_SUB_UMBRELLA = 0x13; + +/// sub client +pub const LC_SUB_CLIENT = 0x14; + +/// sub library +pub const LC_SUB_LIBRARY = 0x15; + +/// two-level namespace lookup hints +pub const LC_TWOLEVEL_HINTS = 0x16; + +/// prebind checksum +pub const LC_PREBIND_CKSUM = 0x17; /// load a dynamically linked shared library that is allowed to be missing /// (all symbols are weak imported). pub const LC_LOAD_WEAK_DYLIB = (0x18 | LC_REQ_DYLD); -pub const LC_SEGMENT_64 = 0x19; /// 64-bit segment of this file to be mapped -pub const LC_ROUTINES_64 = 0x1a; /// 64-bit image routines -pub const LC_UUID = 0x1b; /// the uuid -pub const LC_RPATH = (0x1c | LC_REQ_DYLD); /// runpath additions -pub const LC_CODE_SIGNATURE = 0x1d; /// local of code signature -pub const LC_SEGMENT_SPLIT_INFO = 0x1e; /// local of info to split segments -pub const LC_REEXPORT_DYLIB = (0x1f | LC_REQ_DYLD); /// load and re-export dylib -pub const LC_LAZY_LOAD_DYLIB = 0x20; /// delay load of dylib until first use -pub const LC_ENCRYPTION_INFO = 0x21; /// encrypted segment information -pub const LC_DYLD_INFO = 0x22; /// compressed dyld information -pub const LC_DYLD_INFO_ONLY = (0x22|LC_REQ_DYLD); /// compressed dyld information only -pub const LC_LOAD_UPWARD_DYLIB = (0x23 | LC_REQ_DYLD); /// load upward dylib -pub const LC_VERSION_MIN_MACOSX = 0x24; /// build for MacOSX min OS version -pub const LC_VERSION_MIN_IPHONEOS = 0x25; /// build for iPhoneOS min OS version -pub const LC_FUNCTION_STARTS = 0x26; /// compressed table of function start addresses -pub const LC_DYLD_ENVIRONMENT = 0x27; /// string for dyld to treat like environment variable -pub const LC_MAIN = (0x28|LC_REQ_DYLD); /// replacement for LC_UNIXTHREAD -pub const LC_DATA_IN_CODE = 0x29; /// table of non-instructions in __text -pub const LC_SOURCE_VERSION = 0x2A; /// source version used to build binary -pub const LC_DYLIB_CODE_SIGN_DRS = 0x2B; /// Code signing DRs copied from linked dylibs -pub const LC_ENCRYPTION_INFO_64 = 0x2C; /// 64-bit encrypted segment information -pub const LC_LINKER_OPTION = 0x2D; /// linker options in MH_OBJECT files -pub const LC_LINKER_OPTIMIZATION_HINT = 0x2E; /// optimization hints in MH_OBJECT files -pub const LC_VERSION_MIN_TVOS = 0x2F; /// build for AppleTV min OS version -pub const LC_VERSION_MIN_WATCHOS = 0x30; /// build for Watch min OS version -pub const LC_NOTE = 0x31; /// arbitrary data included within a Mach-O file -pub const LC_BUILD_VERSION = 0x32; /// build for platform min OS version +/// 64-bit segment of this file to be mapped +pub const LC_SEGMENT_64 = 0x19; -pub const MH_MAGIC = 0xfeedface; /// the mach magic number -pub const MH_CIGAM = 0xcefaedfe; /// NXSwapInt(MH_MAGIC) +/// 64-bit image routines +pub const LC_ROUTINES_64 = 0x1a; -pub const MH_MAGIC_64 = 0xfeedfacf; /// the 64-bit mach magic number -pub const MH_CIGAM_64 = 0xcffaedfe; /// NXSwapInt(MH_MAGIC_64) +/// the uuid +pub const LC_UUID = 0x1b; -pub const MH_OBJECT = 0x1; /// relocatable object file -pub const MH_EXECUTE = 0x2; /// demand paged executable file -pub const MH_FVMLIB = 0x3; /// fixed VM shared library file -pub const MH_CORE = 0x4; /// core file -pub const MH_PRELOAD = 0x5; /// preloaded executable file -pub const MH_DYLIB = 0x6; /// dynamically bound shared library -pub const MH_DYLINKER = 0x7; /// dynamic link editor -pub const MH_BUNDLE = 0x8; /// dynamically bound bundle file -pub const MH_DYLIB_STUB = 0x9; /// shared library stub for static linking only, no section contents -pub const MH_DSYM = 0xa; /// companion file with only debug sections -pub const MH_KEXT_BUNDLE = 0xb; /// x86_64 kexts +/// runpath additions +pub const LC_RPATH = (0x1c | LC_REQ_DYLD); + +/// local of code signature +pub const LC_CODE_SIGNATURE = 0x1d; + +/// local of info to split segments +pub const LC_SEGMENT_SPLIT_INFO = 0x1e; + +/// load and re-export dylib +pub const LC_REEXPORT_DYLIB = (0x1f | LC_REQ_DYLD); + +/// delay load of dylib until first use +pub const LC_LAZY_LOAD_DYLIB = 0x20; + +/// encrypted segment information +pub const LC_ENCRYPTION_INFO = 0x21; + +/// compressed dyld information +pub const LC_DYLD_INFO = 0x22; + +/// compressed dyld information only +pub const LC_DYLD_INFO_ONLY = (0x22 | LC_REQ_DYLD); + +/// load upward dylib +pub const LC_LOAD_UPWARD_DYLIB = (0x23 | LC_REQ_DYLD); + +/// build for MacOSX min OS version +pub const LC_VERSION_MIN_MACOSX = 0x24; + +/// build for iPhoneOS min OS version +pub const LC_VERSION_MIN_IPHONEOS = 0x25; + +/// compressed table of function start addresses +pub const LC_FUNCTION_STARTS = 0x26; + +/// string for dyld to treat like environment variable +pub const LC_DYLD_ENVIRONMENT = 0x27; + +/// replacement for LC_UNIXTHREAD +pub const LC_MAIN = (0x28 | LC_REQ_DYLD); + +/// table of non-instructions in __text +pub const LC_DATA_IN_CODE = 0x29; + +/// source version used to build binary +pub const LC_SOURCE_VERSION = 0x2A; + +/// Code signing DRs copied from linked dylibs +pub const LC_DYLIB_CODE_SIGN_DRS = 0x2B; + +/// 64-bit encrypted segment information +pub const LC_ENCRYPTION_INFO_64 = 0x2C; + +/// linker options in MH_OBJECT files +pub const LC_LINKER_OPTION = 0x2D; + +/// optimization hints in MH_OBJECT files +pub const LC_LINKER_OPTIMIZATION_HINT = 0x2E; + +/// build for AppleTV min OS version +pub const LC_VERSION_MIN_TVOS = 0x2F; + +/// build for Watch min OS version +pub const LC_VERSION_MIN_WATCHOS = 0x30; + +/// arbitrary data included within a Mach-O file +pub const LC_NOTE = 0x31; + +/// build for platform min OS version +pub const LC_BUILD_VERSION = 0x32; + +/// the mach magic number +pub const MH_MAGIC = 0xfeedface; + +/// NXSwapInt(MH_MAGIC) +pub const MH_CIGAM = 0xcefaedfe; + +/// the 64-bit mach magic number +pub const MH_MAGIC_64 = 0xfeedfacf; + +/// NXSwapInt(MH_MAGIC_64) +pub const MH_CIGAM_64 = 0xcffaedfe; + +/// relocatable object file +pub const MH_OBJECT = 0x1; + +/// demand paged executable file +pub const MH_EXECUTE = 0x2; + +/// fixed VM shared library file +pub const MH_FVMLIB = 0x3; + +/// core file +pub const MH_CORE = 0x4; + +/// preloaded executable file +pub const MH_PRELOAD = 0x5; + +/// dynamically bound shared library +pub const MH_DYLIB = 0x6; + +/// dynamic link editor +pub const MH_DYLINKER = 0x7; + +/// dynamically bound bundle file +pub const MH_BUNDLE = 0x8; + +/// shared library stub for static linking only, no section contents +pub const MH_DYLIB_STUB = 0x9; + +/// companion file with only debug sections +pub const MH_DSYM = 0xa; + +/// x86_64 kexts +pub const MH_KEXT_BUNDLE = 0xb; // Constants for the flags field of the mach_header -pub const MH_NOUNDEFS = 0x1; /// the object file has no undefined references -pub const MH_INCRLINK = 0x2; /// the object file is the output of an incremental link against a base file and can't be link edited again -pub const MH_DYLDLINK = 0x4; /// the object file is input for the dynamic linker and can't be staticly link edited again -pub const MH_BINDATLOAD = 0x8; /// the object file's undefined references are bound by the dynamic linker when loaded. -pub const MH_PREBOUND = 0x10; /// the file has its dynamic undefined references prebound. -pub const MH_SPLIT_SEGS = 0x20; /// the file has its read-only and read-write segments split -pub const MH_LAZY_INIT = 0x40; /// the shared library init routine is to be run lazily via catching memory faults to its writeable segments (obsolete) -pub const MH_TWOLEVEL = 0x80; /// the image is using two-level name space bindings -pub const MH_FORCE_FLAT = 0x100; /// the executable is forcing all images to use flat name space bindings -pub const MH_NOMULTIDEFS = 0x200; /// this umbrella guarantees no multiple defintions of symbols in its sub-images so the two-level namespace hints can always be used. -pub const MH_NOFIXPREBINDING = 0x400; /// do not have dyld notify the prebinding agent about this executable -pub const MH_PREBINDABLE = 0x800; /// the binary is not prebound but can have its prebinding redone. only used when MH_PREBOUND is not set. -pub const MH_ALLMODSBOUND = 0x1000; /// indicates that this binary binds to all two-level namespace modules of its dependent libraries. only used when MH_PREBINDABLE and MH_TWOLEVEL are both set. -pub const MH_SUBSECTIONS_VIA_SYMBOLS = 0x2000;/// safe to divide up the sections into sub-sections via symbols for dead code stripping -pub const MH_CANONICAL = 0x4000; /// the binary has been canonicalized via the unprebind operation -pub const MH_WEAK_DEFINES = 0x8000; /// the final linked image contains external weak symbols -pub const MH_BINDS_TO_WEAK = 0x10000; /// the final linked image uses weak symbols +/// the object file has no undefined references +pub const MH_NOUNDEFS = 0x1; -pub const MH_ALLOW_STACK_EXECUTION = 0x20000;/// When this bit is set, all stacks in the task will be given stack execution privilege. Only used in MH_EXECUTE filetypes. -pub const MH_ROOT_SAFE = 0x40000; /// When this bit is set, the binary declares it is safe for use in processes with uid zero - -pub const MH_SETUID_SAFE = 0x80000; /// When this bit is set, the binary declares it is safe for use in processes when issetugid() is true +/// the object file is the output of an incremental link against a base file and can't be link edited again +pub const MH_INCRLINK = 0x2; -pub const MH_NO_REEXPORTED_DYLIBS = 0x100000; /// When this bit is set on a dylib, the static linker does not need to examine dependent dylibs to see if any are re-exported -pub const MH_PIE = 0x200000; /// When this bit is set, the OS will load the main executable at a random address. Only used in MH_EXECUTE filetypes. -pub const MH_DEAD_STRIPPABLE_DYLIB = 0x400000; /// Only for use on dylibs. When linking against a dylib that has this bit set, the static linker will automatically not create a LC_LOAD_DYLIB load command to the dylib if no symbols are being referenced from the dylib. -pub const MH_HAS_TLV_DESCRIPTORS = 0x800000; /// Contains a section of type S_THREAD_LOCAL_VARIABLES +/// the object file is input for the dynamic linker and can't be staticly link edited again +pub const MH_DYLDLINK = 0x4; -pub const MH_NO_HEAP_EXECUTION = 0x1000000; /// When this bit is set, the OS will run the main executable with a non-executable heap even on platforms (e.g. i386) that don't require it. Only used in MH_EXECUTE filetypes. +/// the object file's undefined references are bound by the dynamic linker when loaded. +pub const MH_BINDATLOAD = 0x8; -pub const MH_APP_EXTENSION_SAFE = 0x02000000; /// The code was linked for use in an application extension. +/// the file has its dynamic undefined references prebound. +pub const MH_PREBOUND = 0x10; -pub const MH_NLIST_OUTOFSYNC_WITH_DYLDINFO = 0x04000000; /// The external symbols listed in the nlist symbol table do not include all the symbols listed in the dyld info. +/// the file has its read-only and read-write segments split +pub const MH_SPLIT_SEGS = 0x20; +/// the shared library init routine is to be run lazily via catching memory faults to its writeable segments (obsolete) +pub const MH_LAZY_INIT = 0x40; + +/// the image is using two-level name space bindings +pub const MH_TWOLEVEL = 0x80; + +/// the executable is forcing all images to use flat name space bindings +pub const MH_FORCE_FLAT = 0x100; + +/// this umbrella guarantees no multiple defintions of symbols in its sub-images so the two-level namespace hints can always be used. +pub const MH_NOMULTIDEFS = 0x200; + +/// do not have dyld notify the prebinding agent about this executable +pub const MH_NOFIXPREBINDING = 0x400; + +/// the binary is not prebound but can have its prebinding redone. only used when MH_PREBOUND is not set. +pub const MH_PREBINDABLE = 0x800; + +/// indicates that this binary binds to all two-level namespace modules of its dependent libraries. only used when MH_PREBINDABLE and MH_TWOLEVEL are both set. +pub const MH_ALLMODSBOUND = 0x1000; + +/// safe to divide up the sections into sub-sections via symbols for dead code stripping +pub const MH_SUBSECTIONS_VIA_SYMBOLS = 0x2000; + +/// the binary has been canonicalized via the unprebind operation +pub const MH_CANONICAL = 0x4000; + +/// the final linked image contains external weak symbols +pub const MH_WEAK_DEFINES = 0x8000; + +/// the final linked image uses weak symbols +pub const MH_BINDS_TO_WEAK = 0x10000; + +/// When this bit is set, all stacks in the task will be given stack execution privilege. Only used in MH_EXECUTE filetypes. +pub const MH_ALLOW_STACK_EXECUTION = 0x20000; + +/// When this bit is set, the binary declares it is safe for use in processes with uid zero +pub const MH_ROOT_SAFE = 0x40000; + +/// When this bit is set, the binary declares it is safe for use in processes when issetugid() is true +pub const MH_SETUID_SAFE = 0x80000; + +/// When this bit is set on a dylib, the static linker does not need to examine dependent dylibs to see if any are re-exported +pub const MH_NO_REEXPORTED_DYLIBS = 0x100000; + +/// When this bit is set, the OS will load the main executable at a random address. Only used in MH_EXECUTE filetypes. +pub const MH_PIE = 0x200000; + +/// Only for use on dylibs. When linking against a dylib that has this bit set, the static linker will automatically not create a LC_LOAD_DYLIB load command to the dylib if no symbols are being referenced from the dylib. +pub const MH_DEAD_STRIPPABLE_DYLIB = 0x400000; + +/// Contains a section of type S_THREAD_LOCAL_VARIABLES +pub const MH_HAS_TLV_DESCRIPTORS = 0x800000; + +/// When this bit is set, the OS will run the main executable with a non-executable heap even on platforms (e.g. i386) that don't require it. Only used in MH_EXECUTE filetypes. +pub const MH_NO_HEAP_EXECUTION = 0x1000000; + +/// The code was linked for use in an application extension. +pub const MH_APP_EXTENSION_SAFE = 0x02000000; + +/// The external symbols listed in the nlist symbol table do not include all the symbols listed in the dyld info. +pub const MH_NLIST_OUTOFSYNC_WITH_DYLDINFO = 0x04000000; /// The flags field of a section structure is separated into two parts a section /// type and section attributes. The section types are mutually exclusive (it @@ -285,52 +554,129 @@ pub const MH_NLIST_OUTOFSYNC_WITH_DYLDINFO = 0x04000000; /// The external symbol /// than one attribute). /// 256 section types pub const SECTION_TYPE = 0x000000ff; -pub const SECTION_ATTRIBUTES = 0xffffff00; /// 24 section attributes -pub const S_REGULAR = 0x0; /// regular section -pub const S_ZEROFILL = 0x1; /// zero fill on demand section -pub const S_CSTRING_LITERALS = 0x2; /// section with only literal C string -pub const S_4BYTE_LITERALS = 0x3; /// section with only 4 byte literals -pub const S_8BYTE_LITERALS = 0x4; /// section with only 8 byte literals -pub const S_LITERAL_POINTERS = 0x5; /// section with only pointers to +/// 24 section attributes +pub const SECTION_ATTRIBUTES = 0xffffff00; +/// regular section +pub const S_REGULAR = 0x0; -pub const N_STAB = 0xe0; /// if any of these bits set, a symbolic debugging entry -pub const N_PEXT = 0x10; /// private external symbol bit -pub const N_TYPE = 0x0e; /// mask for the type bits -pub const N_EXT = 0x01; /// external symbol bit, set for external symbols +/// zero fill on demand section +pub const S_ZEROFILL = 0x1; +/// section with only literal C string +pub const S_CSTRING_LITERALS = 0x2; -pub const N_GSYM = 0x20; /// global symbol: name,,NO_SECT,type,0 -pub const N_FNAME = 0x22; /// procedure name (f77 kludge): name,,NO_SECT,0,0 -pub const N_FUN = 0x24; /// procedure: name,,n_sect,linenumber,address -pub const N_STSYM = 0x26; /// static symbol: name,,n_sect,type,address -pub const N_LCSYM = 0x28; /// .lcomm symbol: name,,n_sect,type,address -pub const N_BNSYM = 0x2e; /// begin nsect sym: 0,,n_sect,0,address -pub const N_AST = 0x32; /// AST file path: name,,NO_SECT,0,0 -pub const N_OPT = 0x3c; /// emitted with gcc2_compiled and in gcc source -pub const N_RSYM = 0x40; /// register sym: name,,NO_SECT,type,register -pub const N_SLINE = 0x44; /// src line: 0,,n_sect,linenumber,address -pub const N_ENSYM = 0x4e; /// end nsect sym: 0,,n_sect,0,address -pub const N_SSYM = 0x60; /// structure elt: name,,NO_SECT,type,struct_offset -pub const N_SO = 0x64; /// source file name: name,,n_sect,0,address -pub const N_OSO = 0x66; /// object file name: name,,0,0,st_mtime -pub const N_LSYM = 0x80; /// local sym: name,,NO_SECT,type,offset -pub const N_BINCL = 0x82; /// include file beginning: name,,NO_SECT,0,sum -pub const N_SOL = 0x84; /// #included file name: name,,n_sect,0,address -pub const N_PARAMS = 0x86; /// compiler parameters: name,,NO_SECT,0,0 -pub const N_VERSION = 0x88; /// compiler version: name,,NO_SECT,0,0 -pub const N_OLEVEL = 0x8A; /// compiler -O level: name,,NO_SECT,0,0 -pub const N_PSYM = 0xa0; /// parameter: name,,NO_SECT,type,offset -pub const N_EINCL = 0xa2; /// include file end: name,,NO_SECT,0,0 -pub const N_ENTRY = 0xa4; /// alternate entry: name,,n_sect,linenumber,address -pub const N_LBRAC = 0xc0; /// left bracket: 0,,NO_SECT,nesting level,address -pub const N_EXCL = 0xc2; /// deleted include file: name,,NO_SECT,0,sum -pub const N_RBRAC = 0xe0; /// right bracket: 0,,NO_SECT,nesting level,address -pub const N_BCOMM = 0xe2; /// begin common: name,,NO_SECT,0,0 -pub const N_ECOMM = 0xe4; /// end common: name,,n_sect,0,0 -pub const N_ECOML = 0xe8; /// end common (local name): 0,,n_sect,0,address -pub const N_LENG = 0xfe; /// second stab entry with length information +/// section with only 4 byte literals +pub const S_4BYTE_LITERALS = 0x3; + +/// section with only 8 byte literals +pub const S_8BYTE_LITERALS = 0x4; + +/// section with only pointers to +pub const S_LITERAL_POINTERS = 0x5; + +/// if any of these bits set, a symbolic debugging entry +pub const N_STAB = 0xe0; + +/// private external symbol bit +pub const N_PEXT = 0x10; + +/// mask for the type bits +pub const N_TYPE = 0x0e; + +/// external symbol bit, set for external symbols +pub const N_EXT = 0x01; + +/// global symbol: name,,NO_SECT,type,0 +pub const N_GSYM = 0x20; + +/// procedure name (f77 kludge): name,,NO_SECT,0,0 +pub const N_FNAME = 0x22; + +/// procedure: name,,n_sect,linenumber,address +pub const N_FUN = 0x24; + +/// static symbol: name,,n_sect,type,address +pub const N_STSYM = 0x26; + +/// .lcomm symbol: name,,n_sect,type,address +pub const N_LCSYM = 0x28; + +/// begin nsect sym: 0,,n_sect,0,address +pub const N_BNSYM = 0x2e; + +/// AST file path: name,,NO_SECT,0,0 +pub const N_AST = 0x32; + +/// emitted with gcc2_compiled and in gcc source +pub const N_OPT = 0x3c; + +/// register sym: name,,NO_SECT,type,register +pub const N_RSYM = 0x40; + +/// src line: 0,,n_sect,linenumber,address +pub const N_SLINE = 0x44; + +/// end nsect sym: 0,,n_sect,0,address +pub const N_ENSYM = 0x4e; + +/// structure elt: name,,NO_SECT,type,struct_offset +pub const N_SSYM = 0x60; + +/// source file name: name,,n_sect,0,address +pub const N_SO = 0x64; + +/// object file name: name,,0,0,st_mtime +pub const N_OSO = 0x66; + +/// local sym: name,,NO_SECT,type,offset +pub const N_LSYM = 0x80; + +/// include file beginning: name,,NO_SECT,0,sum +pub const N_BINCL = 0x82; + +/// #included file name: name,,n_sect,0,address +pub const N_SOL = 0x84; + +/// compiler parameters: name,,NO_SECT,0,0 +pub const N_PARAMS = 0x86; + +/// compiler version: name,,NO_SECT,0,0 +pub const N_VERSION = 0x88; + +/// compiler -O level: name,,NO_SECT,0,0 +pub const N_OLEVEL = 0x8A; + +/// parameter: name,,NO_SECT,type,offset +pub const N_PSYM = 0xa0; + +/// include file end: name,,NO_SECT,0,0 +pub const N_EINCL = 0xa2; + +/// alternate entry: name,,n_sect,linenumber,address +pub const N_ENTRY = 0xa4; + +/// left bracket: 0,,NO_SECT,nesting level,address +pub const N_LBRAC = 0xc0; + +/// deleted include file: name,,NO_SECT,0,sum +pub const N_EXCL = 0xc2; + +/// right bracket: 0,,NO_SECT,nesting level,address +pub const N_RBRAC = 0xe0; + +/// begin common: name,,NO_SECT,0,0 +pub const N_BCOMM = 0xe2; + +/// end common: name,,n_sect,0,0 +pub const N_ECOMM = 0xe4; + +/// end common (local name): 0,,n_sect,0,address +pub const N_ECOML = 0xe8; + +/// second stab entry with length information +pub const N_LENG = 0xfe; /// If a segment contains any sections marked with S_ATTR_DEBUG then all /// sections in that segment must have this attribute. No section other than @@ -339,10 +685,10 @@ pub const N_LENG = 0xfe; /// second stab entry with length information /// a section type S_REGULAR. The static linker will not copy section contents /// from sections with this attribute into its output file. These sections /// generally contain DWARF debugging info. -pub const S_ATTR_DEBUG = 0x02000000; /// a debug section +/// a debug section +pub const S_ATTR_DEBUG = 0x02000000; pub const cpu_type_t = integer_t; pub const cpu_subtype_t = integer_t; pub const integer_t = c_int; pub const vm_prot_t = c_int; - diff --git a/std/os/windows/advapi32.zig b/std/os/windows/advapi32.zig index 2f3195475c..d0a3ec41e9 100644 --- a/std/os/windows/advapi32.zig +++ b/std/os/windows/advapi32.zig @@ -23,11 +23,22 @@ pub extern "advapi32" stdcallcc fn CryptReleaseContext(hProv: HCRYPTPROV, dwFlag pub extern "advapi32" stdcallcc fn CryptGenRandom(hProv: HCRYPTPROV, dwLen: DWORD, pbBuffer: [*]BYTE) BOOL; -pub extern "advapi32" stdcallcc fn RegOpenKeyExW(hKey: HKEY, lpSubKey: LPCWSTR, ulOptions: DWORD, samDesired: REGSAM, - phkResult: &HKEY,) LSTATUS; +pub extern "advapi32" stdcallcc fn RegOpenKeyExW( + hKey: HKEY, + lpSubKey: LPCWSTR, + ulOptions: DWORD, + samDesired: REGSAM, + phkResult: &HKEY, +) LSTATUS; -pub extern "advapi32" stdcallcc fn RegQueryValueExW(hKey: HKEY, lpValueName: LPCWSTR, lpReserved: LPDWORD, - lpType: LPDWORD, lpData: LPBYTE, lpcbData: LPDWORD,) LSTATUS; +pub extern "advapi32" stdcallcc fn RegQueryValueExW( + hKey: HKEY, + lpValueName: LPCWSTR, + lpReserved: LPDWORD, + lpType: LPDWORD, + lpData: LPBYTE, + lpcbData: LPDWORD, +) LSTATUS; // RtlGenRandom is known as SystemFunction036 under advapi32 // http://msdn.microsoft.com/en-us/library/windows/desktop/aa387694.aspx */ diff --git a/std/os/windows/index.zig b/std/os/windows/index.zig index bb055468a5..5c68176c5a 100644 --- a/std/os/windows/index.zig +++ b/std/os/windows/index.zig @@ -288,27 +288,27 @@ pub const GUID = extern struct { assert(str[index] == '{'); index += 1; - guid.Data1 = std.fmt.parseUnsigned(c_ulong, str[index..index + 8], 16) catch unreachable; + guid.Data1 = std.fmt.parseUnsigned(c_ulong, str[index .. index + 8], 16) catch unreachable; index += 8; assert(str[index] == '-'); index += 1; - guid.Data2 = std.fmt.parseUnsigned(c_ushort, str[index..index + 4], 16) catch unreachable; + guid.Data2 = std.fmt.parseUnsigned(c_ushort, str[index .. index + 4], 16) catch unreachable; index += 4; assert(str[index] == '-'); index += 1; - guid.Data3 = std.fmt.parseUnsigned(c_ushort, str[index..index + 4], 16) catch unreachable; + guid.Data3 = std.fmt.parseUnsigned(c_ushort, str[index .. index + 4], 16) catch unreachable; index += 4; assert(str[index] == '-'); index += 1; - guid.Data4[0] = std.fmt.parseUnsigned(u8, str[index..index + 2], 16) catch unreachable; + guid.Data4[0] = std.fmt.parseUnsigned(u8, str[index .. index + 2], 16) catch unreachable; index += 2; - guid.Data4[1] = std.fmt.parseUnsigned(u8, str[index..index + 2], 16) catch unreachable; + guid.Data4[1] = std.fmt.parseUnsigned(u8, str[index .. index + 2], 16) catch unreachable; index += 2; assert(str[index] == '-'); @@ -316,7 +316,7 @@ pub const GUID = extern struct { var i: usize = 2; while (i < guid.Data4.len) : (i += 1) { - guid.Data4[i] = std.fmt.parseUnsigned(u8, str[index..index + 2], 16) catch unreachable; + guid.Data4[i] = std.fmt.parseUnsigned(u8, str[index .. index + 2], 16) catch unreachable; index += 2; } diff --git a/std/os/windows/ole32.zig b/std/os/windows/ole32.zig index 84d8089d07..cc737e344c 100644 --- a/std/os/windows/ole32.zig +++ b/std/os/windows/ole32.zig @@ -5,7 +5,6 @@ pub extern "ole32.dll" stdcallcc fn CoUninitialize() void; pub extern "ole32.dll" stdcallcc fn CoGetCurrentProcess() DWORD; pub extern "ole32.dll" stdcallcc fn CoInitializeEx(pvReserved: LPVOID, dwCoInit: DWORD) HRESULT; - pub const COINIT_APARTMENTTHREADED = COINIT.COINIT_APARTMENTTHREADED; pub const COINIT_MULTITHREADED = COINIT.COINIT_MULTITHREADED; pub const COINIT_DISABLE_OLE1DDE = COINIT.COINIT_DISABLE_OLE1DDE; diff --git a/std/os/zen.zig b/std/os/zen.zig index 55b6d91128..6ac480b890 100644 --- a/std/os/zen.zig +++ b/std/os/zen.zig @@ -6,32 +6,32 @@ const assert = std.debug.assert; ////////////////////////// pub const Message = struct { -sender: MailboxId, + sender: MailboxId, receiver: MailboxId, - code: usize, - args: [5]usize, - payload: ?[]const u8, + code: usize, + args: [5]usize, + payload: ?[]const u8, pub fn from(mailbox_id: *const MailboxId) Message { - return Message { - .sender = MailboxId.Undefined, + return Message{ + .sender = MailboxId.Undefined, .receiver = mailbox_id.*, - .code = undefined, - .args = undefined, - .payload = null, + .code = undefined, + .args = undefined, + .payload = null, }; } pub fn to(mailbox_id: *const MailboxId, msg_code: usize, args: ...) Message { - var message = Message { - .sender = MailboxId.This, + var message = Message{ + .sender = MailboxId.This, .receiver = mailbox_id.*, - .code = msg_code, - .args = undefined, - .payload = null, + .code = msg_code, + .args = undefined, + .payload = null, }; - assert (args.len <= message.args.len); + assert(args.len <= message.args.len); comptime var i = 0; inline while (i < args.len) : (i += 1) { message.args[i] = args[i]; @@ -111,8 +111,7 @@ pub fn read(fd: i32, buf: [*]u8, count: usize) usize { pub fn write(fd: i32, buf: [*]const u8, count: usize) usize { switch (fd) { STDOUT_FILENO, STDERR_FILENO => { - send(Message.to(Server.Terminal, 1) - .withPayload(buf[0..count])); + send(Message.to(Server.Terminal, 1).withPayload(buf[0..count])); }, else => unreachable, } @@ -124,14 +123,14 @@ pub fn write(fd: i32, buf: [*]const u8, count: usize) usize { /////////////////////////// pub const Syscall = enum(usize) { - exit = 0, - send = 1, - receive = 2, - subscribeIRQ = 3, - inb = 4, - outb = 5, - map = 6, - createThread = 7, + exit = 0, + send = 1, + receive = 2, + subscribeIRQ = 3, + inb = 4, + outb = 5, + map = 6, + createThread = 7, }; //////////////////// diff --git a/std/rb.zig b/std/rb.zig index d523069846..e8c4738b52 100644 --- a/std/rb.zig +++ b/std/rb.zig @@ -9,9 +9,7 @@ const Color = enum(u1) { const Red = Color.Red; const Black = Color.Black; -const ReplaceError = error { - NotEqual, -}; +const ReplaceError = error{NotEqual}; /// Insert this into your struct that you want to add to a red-black tree. /// Do not use a pointer. Turn the *rb.Node results of the functions in rb @@ -27,7 +25,9 @@ const ReplaceError = error { pub const Node = struct { left: ?*Node, right: ?*Node, - parent_and_color: usize, /// parent | color + + /// parent | color + parent_and_color: usize, pub fn next(constnode: *Node) ?*Node { var node = constnode; @@ -130,7 +130,7 @@ pub const Node = struct { pub const Tree = struct { root: ?*Node, - compareFn: fn(*Node, *Node) mem.Compare, + compareFn: fn (*Node, *Node) mem.Compare, /// If you have a need for a version that caches this, please file a bug. pub fn first(tree: *Tree) ?*Node { @@ -180,7 +180,7 @@ pub const Tree = struct { while (node.get_parent()) |*parent| { if (parent.*.is_black()) break; - // the root is always black + // the root is always black var grandpa = parent.*.get_parent() orelse unreachable; if (parent.* == grandpa.left) { @@ -206,7 +206,7 @@ pub const Tree = struct { } } else { var maybe_uncle = grandpa.left; - + if (maybe_uncle) |uncle| { if (uncle.is_black()) break; @@ -259,7 +259,7 @@ pub const Tree = struct { if (node.left == null) { next = node.right.?; // Not both null as per above } else if (node.right == null) { - next = node.left.?; // Not both null as per above + next = node.left.?; // Not both null as per above } else next = node.right.?.get_first(); // Just checked for null above @@ -313,7 +313,7 @@ pub const Tree = struct { var parent = maybe_parent.?; if (node == parent.left) { var sibling = parent.right.?; // Same number of black nodes. - + if (sibling.is_red()) { sibling.set_color(Black); parent.set_color(Red); @@ -321,7 +321,8 @@ pub const Tree = struct { sibling = parent.right.?; // Just rotated } if ((if (sibling.left) |n| n.is_black() else true) and - (if (sibling.right) |n| n.is_black() else true)) { + (if (sibling.right) |n| n.is_black() else true)) + { sibling.set_color(Red); node = parent; maybe_parent = parent.get_parent(); @@ -341,7 +342,7 @@ pub const Tree = struct { break; } else { var sibling = parent.left.?; // Same number of black nodes. - + if (sibling.is_red()) { sibling.set_color(Black); parent.set_color(Red); @@ -349,7 +350,8 @@ pub const Tree = struct { sibling = parent.left.?; // Just rotated } if ((if (sibling.left) |n| n.is_black() else true) and - (if (sibling.right) |n| n.is_black() else true)) { + (if (sibling.right) |n| n.is_black() else true)) + { sibling.set_color(Red); node = parent; maybe_parent = parent.get_parent(); @@ -397,7 +399,7 @@ pub const Tree = struct { new.* = old.*; } - pub fn init(tree: *Tree, f: fn(*Node, *Node) mem.Compare) void { + pub fn init(tree: *Tree, f: fn (*Node, *Node) mem.Compare) void { tree.root = null; tree.compareFn = f; } From 048f506aa6b78dd13a593b6bb12a317c5a047bc7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Aug 2018 20:59:28 -0400 Subject: [PATCH 15/56] langref: document labeled blocks, labeled for, labeled while closes #1327 --- doc/langref.html.in | 86 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/doc/langref.html.in b/doc/langref.html.in index 495b19580c..31d923e84b 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -2179,6 +2179,39 @@ test "@tagName" { sorts the order of the tag and union field by the largest alignment.

{#header_close#} + {#header_open|blocks#} +

+ Blocks are used to limit the scope of variable declarations: +

+ {#code_begin|test_err|undeclared identifier#} +test "access variable after block scope" { + { + var x: i32 = 1; + } + x += 1; +} + {#code_end#} +

Blocks are expressions. When labeled, break can be used + to return a value from the block: +

+ {#code_begin|test#} +const std = @import("std"); +const assert = std.debug.assert; + +test "labeled break from labeled block expression" { + var y: i32 = 123; + + const x = blk: { + y += 1; + break :blk y; + }; + assert(x == 124); + assert(y == 124); +} + {#code_end#} +

Here, blk can be any name.

+ {#see_also|Labeled while|Labeled for#} + {#header_close#} {#header_open|switch#} {#code_begin|test|switch#} const assert = @import("std").debug.assert; @@ -2374,6 +2407,28 @@ fn rangeHasNumber(begin: usize, end: usize, number: usize) bool { } else false; } {#code_end#} + {#header_open|Labeled while#} +

When a while loop is labeled, it can be referenced from a break + or continue from within a nested loop:

+ {#code_begin|test#} +test "nested break" { + outer: while (true) { + while (true) { + break :outer; + } + } +} + +test "nested continue" { + var i: usize = 0; + outer: while (i < 10) : (i += 1) { + while (true) { + continue :outer; + } + } +} + {#code_end#} + {#header_close#} {#header_open|while with Optionals#}

Just like {#link|if#} expressions, while loops can take an optional as the @@ -2560,6 +2615,37 @@ test "for else" { }; } {#code_end#} + {#header_open|Labeled for#} +

When a for loop is labeled, it can be referenced from a break + or continue from within a nested loop:

+ {#code_begin|test#} +const std = @import("std"); +const assert = std.debug.assert; + +test "nested break" { + var count: usize = 0; + outer: for ([]i32{ 1, 2, 3, 4, 5 }) |_| { + for ([]i32{ 1, 2, 3, 4, 5 }) |_| { + count += 1; + break :outer; + } + } + assert(count == 1); +} + +test "nested continue" { + var count: usize = 0; + outer: for ([]i32{ 1, 2, 3, 4, 5, 6, 7, 8 }) |_| { + for ([]i32{ 1, 2, 3, 4, 5 }) |_| { + count += 1; + continue :outer; + } + } + + assert(count == 8); +} + {#code_end#} + {#header_close#} {#header_open|inline for#}

For loops can be inlined. This causes the loop to be unrolled, which From 444edd9aed84ebfd9153817259e2a4e1228b7120 Mon Sep 17 00:00:00 2001 From: Shawn Landden Date: Fri, 10 Aug 2018 19:01:37 -0700 Subject: [PATCH 16/56] std.crypto: add chaCha20 v3 --- CMakeLists.txt | 1 + std/crypto/chacha20.zig | 431 ++++++++++++++++++++++++++++++++++++++++ std/crypto/index.zig | 5 + 3 files changed, 437 insertions(+) create mode 100644 std/crypto/chacha20.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index d7487ce905..8e753b5137 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -451,6 +451,7 @@ set(ZIG_STD_FILES "crypto/sha1.zig" "crypto/sha2.zig" "crypto/sha3.zig" + "crypto/chacha20.zig" "cstr.zig" "debug/failing_allocator.zig" "debug/index.zig" diff --git a/std/crypto/chacha20.zig b/std/crypto/chacha20.zig new file mode 100644 index 0000000000..3030569ae6 --- /dev/null +++ b/std/crypto/chacha20.zig @@ -0,0 +1,431 @@ +// Based on public domain Supercop by Daniel J. Bernstein + +const std = @import("../index.zig"); +const mem = std.mem; +const endian = std.endian; +const assert = std.debug.assert; +const builtin = @import("builtin"); + +const QuarterRound = struct { + a: usize, + b: usize, + c: usize, + d: usize, +}; + +fn Rp(a: usize, b: usize, c: usize, d: usize) QuarterRound { + return QuarterRound{ + .a = a, + .b = b, + .c = c, + .d = d, + }; +} + +fn rotate(a: u32, b: u5) u32 { + return ((a << b) | + (a >> @intCast(u5, (32 - @intCast(u6, b)))) + ); +} + +// The chacha family of ciphers are based on the salsa family. +fn salsa20_wordtobyte(input: [16]u32) [64]u8 { + var x: [16]u32 = undefined; + var out: [64]u8 = undefined; + + for (x) |_, i| + x[i] = input[i]; + const rounds = comptime []QuarterRound{ + Rp( 0, 4, 8,12), + Rp( 1, 5, 9,13), + Rp( 2, 6,10,14), + Rp( 3, 7,11,15), + Rp( 0, 5,10,15), + Rp( 1, 6,11,12), + Rp( 2, 7, 8,13), + Rp( 3, 4, 9,14), + }; + comptime var j: usize = 20; + inline while (j > 0) : (j -=2) { + for (rounds) |r| { + x[r.a] +%= x[r.b]; x[r.d] = rotate(x[r.d] ^ x[r.a], 16); + x[r.c] +%= x[r.d]; x[r.b] = rotate(x[r.b] ^ x[r.c], 12); + x[r.a] +%= x[r.b]; x[r.d] = rotate(x[r.d] ^ x[r.a], 8); + x[r.c] +%= x[r.d]; x[r.b] = rotate(x[r.b] ^ x[r.c], 7); + } + } + for (x) |_, i| + x[i] +%= input[i]; + for (x) |_, i| + mem.writeInt(out[4 * i .. 4 * i + 4], x[i], builtin.Endian.Little); + return out; +} + +fn chaCha20_internal(out: []u8, in: []const u8, key: [8]u32, counter: [4]u32) void { + var ctx: [16]u32 = undefined; + var remaining: usize = if (in.len > out.len) in.len else out.len; + var cursor: usize = 0; + + const c = "expand 32-byte k"; + const constant_le = []u32{ + mem.readIntLE(u32, c[0..4]), + mem.readIntLE(u32, c[4..8]), + mem.readIntLE(u32, c[8..12]), + mem.readIntLE(u32, c[12..16]), + }; + + mem.copy(u32, ctx[0..], constant_le[0..4]); + mem.copy(u32, ctx[4..12], key[0..8]); + mem.copy(u32, ctx[12..16], counter[0..4]); + + while (true) { + var buf = salsa20_wordtobyte(ctx); + + if (remaining < 64) { + var i: usize = 0; + while (i < remaining) : (i += 1) + out[cursor + i] = in[cursor + i] ^ buf[i]; + return; + } + + comptime var i: usize = 0; + inline while (i < 64) : (i += 1) + out[cursor + i] = in[cursor + i] ^ buf[i]; + + cursor += 64; + remaining -= 64; + + ctx[12] += 1; + } +} + +/// ChaCha20 avoids the possibility of timing attacks, as there are no branches +/// on secret key data. +/// +/// in and out should be the same length. +/// counter should generally be 0 or 1 +/// +/// ChaCha20 is self-reversing. To decrypt just run the cipher with the same +/// counter, nonce, and key. +pub fn chaCha20IETF(out: []u8, in: []const u8, counter: u32, key: [32]u8, nonce: [12]u8) void { + assert(in.len >= out.len); + assert((in.len >> 6) + counter <= @maxValue(u32)); + + var k: [8]u32 = undefined; + var c: [4]u32 = undefined; + + k[0] = mem.readIntLE(u32, key[0..4]); + k[1] = mem.readIntLE(u32, key[4..8]); + k[2] = mem.readIntLE(u32, key[8..12]); + k[3] = mem.readIntLE(u32, key[12..16]); + k[4] = mem.readIntLE(u32, key[16..20]); + k[5] = mem.readIntLE(u32, key[20..24]); + k[6] = mem.readIntLE(u32, key[24..28]); + k[7] = mem.readIntLE(u32, key[28..32]); + + c[0] = counter; + c[1] = mem.readIntLE(u32, nonce[0..4]); + c[2] = mem.readIntLE(u32, nonce[4..8]); + c[3] = mem.readIntLE(u32, nonce[8..12]); + chaCha20_internal(out, in, k, c); +} + +/// This is the original ChaCha20 before RFC 7539, which recommends using the +/// orgininal version on applications such as disk or file encryption that might +/// exceed the 256 GiB limit of the 96-bit nonce version. +pub fn chaCha20With64BitNonce(out: []u8, in: []const u8, counter: u64, key: [32]u8, nonce: [8]u8) void { + assert(in.len >= out.len); + assert(counter +% (in.len >> 6) >= counter); + + var cursor: u64 = 0; + var k: [8]u32 = undefined; + var c: [4]u32 = undefined; + + k[0] = mem.readIntLE(u32, key[0..4]); + k[1] = mem.readIntLE(u32, key[4..8]); + k[2] = mem.readIntLE(u32, key[8..12]); + k[3] = mem.readIntLE(u32, key[12..16]); + k[4] = mem.readIntLE(u32, key[16..20]); + k[5] = mem.readIntLE(u32, key[20..24]); + k[6] = mem.readIntLE(u32, key[24..28]); + k[7] = mem.readIntLE(u32, key[28..32]); + + c[0] = @truncate(u32, counter); + c[1] = @truncate(u32, counter >> 32); + c[2] = mem.readIntLE(u32, nonce[0..4]); + c[3] = mem.readIntLE(u32, nonce[4..8]); + + const block_size = (1 << 6); + const big_block = (block_size << 32); + + // first partial big block + if (((@intCast(u64, @maxValue(u32) - @truncate(u32, counter)) + 1) << 6) < in.len) { + chaCha20_internal(out[cursor..big_block], in[cursor..big_block], k, c); + cursor = big_block - cursor; + c[1] += 1; + if (comptime @sizeOf(usize) > 4) { + // A big block is giant: 256 GiB, but we can avoid this limitation + var remaining_blocks: u32 = @intCast(u32, (in.len / big_block)); + var i: u32 = 0; + while (remaining_blocks > 0) : (remaining_blocks -= 1) { + chaCha20_internal(out[cursor..cursor + big_block], in[cursor..cursor + big_block], k, c); + c[1] += 1; // upper 32-bit of counter, generic chaCha20_internal() doesn't + // know about this. + cursor += big_block; + } + } + } + + chaCha20_internal(out[cursor..], in[cursor..], k, c); +} + +// https://tools.ietf.org/html/rfc7539#section-2.4.2 +test "crypto.chacha20 test vector sunscreen" { + const expected_result = []u8{ + 0x6e, 0x2e, 0x35, 0x9a, 0x25, 0x68, 0xf9, 0x80, + 0x41, 0xba, 0x07, 0x28, 0xdd, 0x0d, 0x69, 0x81, + 0xe9, 0x7e, 0x7a, 0xec, 0x1d, 0x43, 0x60, 0xc2, + 0x0a, 0x27, 0xaf, 0xcc, 0xfd, 0x9f, 0xae, 0x0b, + 0xf9, 0x1b, 0x65, 0xc5, 0x52, 0x47, 0x33, 0xab, + 0x8f, 0x59, 0x3d, 0xab, 0xcd, 0x62, 0xb3, 0x57, + 0x16, 0x39, 0xd6, 0x24, 0xe6, 0x51, 0x52, 0xab, + 0x8f, 0x53, 0x0c, 0x35, 0x9f, 0x08, 0x61, 0xd8, + 0x07, 0xca, 0x0d, 0xbf, 0x50, 0x0d, 0x6a, 0x61, + 0x56, 0xa3, 0x8e, 0x08, 0x8a, 0x22, 0xb6, 0x5e, + 0x52, 0xbc, 0x51, 0x4d, 0x16, 0xcc, 0xf8, 0x06, + 0x81, 0x8c, 0xe9, 0x1a, 0xb7, 0x79, 0x37, 0x36, + 0x5a, 0xf9, 0x0b, 0xbf, 0x74, 0xa3, 0x5b, 0xe6, + 0xb4, 0x0b, 0x8e, 0xed, 0xf2, 0x78, 0x5e, 0x42, + 0x87, 0x4d, + }; + const input = "Ladies and Gentlemen of the class of '99: If I could offer you only one tip for the future, sunscreen would be it."; + var result: [114]u8 = undefined; + const key = []u8{ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9,10,11,12,13,14,15, + 16,17,18,19,20,21,22,23, + 24,25,26,27,28,29,30,31, + }; + const nonce = []u8{ + 0, 0, 0, 0, + 0, 0, 0, 0x4a, + 0, 0, 0, 0, + }; + + chaCha20IETF(result[0..], input[0..], 1, key, nonce); + assert(mem.eql(u8, expected_result, result)); + + // Chacha20 is self-reversing. + var plaintext: [114]u8 = undefined; + chaCha20IETF(plaintext[0..], result[0..], 1, key, nonce); + assert(mem.compare(u8, input, plaintext) == mem.Compare.Equal); +} + +// https://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-04#section-7 +test "crypto.chacha20 test vector 1" { + const expected_result = []u8{ + 0x76, 0xb8, 0xe0, 0xad, 0xa0, 0xf1, 0x3d, 0x90, + 0x40, 0x5d, 0x6a, 0xe5, 0x53, 0x86, 0xbd, 0x28, + 0xbd, 0xd2, 0x19, 0xb8, 0xa0, 0x8d, 0xed, 0x1a, + 0xa8, 0x36, 0xef, 0xcc, 0x8b, 0x77, 0x0d, 0xc7, + 0xda, 0x41, 0x59, 0x7c, 0x51, 0x57, 0x48, 0x8d, + 0x77, 0x24, 0xe0, 0x3f, 0xb8, 0xd8, 0x4a, 0x37, + 0x6a, 0x43, 0xb8, 0xf4, 0x15, 0x18, 0xa1, 0x1c, + 0xc3, 0x87, 0xb6, 0x69, 0xb2, 0xee, 0x65, 0x86, + }; + const input = []u8{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + var result: [64]u8 = undefined; + const key = []u8{ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + }; + const nonce = []u8{0, 0, 0, 0, 0, 0, 0, 0}; + + chaCha20With64BitNonce(result[0..], input[0..], 0, key, nonce); + assert(mem.eql(u8, expected_result, result)); +} + +test "crypto.chacha20 test vector 2" { + const expected_result = []u8{ + 0x45, 0x40, 0xf0, 0x5a, 0x9f, 0x1f, 0xb2, 0x96, + 0xd7, 0x73, 0x6e, 0x7b, 0x20, 0x8e, 0x3c, 0x96, + 0xeb, 0x4f, 0xe1, 0x83, 0x46, 0x88, 0xd2, 0x60, + 0x4f, 0x45, 0x09, 0x52, 0xed, 0x43, 0x2d, 0x41, + 0xbb, 0xe2, 0xa0, 0xb6, 0xea, 0x75, 0x66, 0xd2, + 0xa5, 0xd1, 0xe7, 0xe2, 0x0d, 0x42, 0xaf, 0x2c, + 0x53, 0xd7, 0x92, 0xb1, 0xc4, 0x3f, 0xea, 0x81, + 0x7e, 0x9a, 0xd2, 0x75, 0xae, 0x54, 0x69, 0x63, + }; + const input = []u8{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + var result: [64]u8 = undefined; + const key = []u8{ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, + }; + const nonce = []u8{0, 0, 0, 0, 0, 0, 0, 0}; + + chaCha20With64BitNonce(result[0..], input[0..], 0, key, nonce); + assert(mem.eql(u8, expected_result, result)); +} + +test "crypto.chacha20 test vector 3" { + const expected_result = []u8{ + 0xde, 0x9c, 0xba, 0x7b, 0xf3, 0xd6, 0x9e, 0xf5, + 0xe7, 0x86, 0xdc, 0x63, 0x97, 0x3f, 0x65, 0x3a, + 0x0b, 0x49, 0xe0, 0x15, 0xad, 0xbf, 0xf7, 0x13, + 0x4f, 0xcb, 0x7d, 0xf1, 0x37, 0x82, 0x10, 0x31, + 0xe8, 0x5a, 0x05, 0x02, 0x78, 0xa7, 0x08, 0x45, + 0x27, 0x21, 0x4f, 0x73, 0xef, 0xc7, 0xfa, 0x5b, + 0x52, 0x77, 0x06, 0x2e, 0xb7, 0xa0, 0x43, 0x3e, + 0x44, 0x5f, 0x41, 0xe3, + }; + const input = []u8{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }; + var result: [60]u8 = undefined; + const key = []u8{ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + }; + const nonce = []u8{0, 0, 0, 0, 0, 0, 0, 1}; + + chaCha20With64BitNonce(result[0..], input[0..], 0, key, nonce); + assert(mem.eql(u8, expected_result, result)); +} + +test "crypto.chacha20 test vector 4" { + const expected_result = []u8{ + 0xef, 0x3f, 0xdf, 0xd6, 0xc6, 0x15, 0x78, 0xfb, + 0xf5, 0xcf, 0x35, 0xbd, 0x3d, 0xd3, 0x3b, 0x80, + 0x09, 0x63, 0x16, 0x34, 0xd2, 0x1e, 0x42, 0xac, + 0x33, 0x96, 0x0b, 0xd1, 0x38, 0xe5, 0x0d, 0x32, + 0x11, 0x1e, 0x4c, 0xaf, 0x23, 0x7e, 0xe5, 0x3c, + 0xa8, 0xad, 0x64, 0x26, 0x19, 0x4a, 0x88, 0x54, + 0x5d, 0xdc, 0x49, 0x7a, 0x0b, 0x46, 0x6e, 0x7d, + 0x6b, 0xbd, 0xb0, 0x04, 0x1b, 0x2f, 0x58, 0x6b, + }; + const input = []u8{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + var result: [64]u8 = undefined; + const key = []u8{ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + }; + const nonce = []u8{1, 0, 0, 0, 0, 0, 0, 0}; + + chaCha20With64BitNonce(result[0..], input[0..], 0, key, nonce); + assert(mem.eql(u8, expected_result, result)); +} + +test "crypto.chacha20 test vector 5" { + const expected_result = []u8{ + 0xf7, 0x98, 0xa1, 0x89, 0xf1, 0x95, 0xe6, 0x69, + 0x82, 0x10, 0x5f, 0xfb, 0x64, 0x0b, 0xb7, 0x75, + 0x7f, 0x57, 0x9d, 0xa3, 0x16, 0x02, 0xfc, 0x93, + 0xec, 0x01, 0xac, 0x56, 0xf8, 0x5a, 0xc3, 0xc1, + 0x34, 0xa4, 0x54, 0x7b, 0x73, 0x3b, 0x46, 0x41, + 0x30, 0x42, 0xc9, 0x44, 0x00, 0x49, 0x17, 0x69, + 0x05, 0xd3, 0xbe, 0x59, 0xea, 0x1c, 0x53, 0xf1, + 0x59, 0x16, 0x15, 0x5c, 0x2b, 0xe8, 0x24, 0x1a, + + 0x38, 0x00, 0x8b, 0x9a, 0x26, 0xbc, 0x35, 0x94, + 0x1e, 0x24, 0x44, 0x17, 0x7c, 0x8a, 0xde, 0x66, + 0x89, 0xde, 0x95, 0x26, 0x49, 0x86, 0xd9, 0x58, + 0x89, 0xfb, 0x60, 0xe8, 0x46, 0x29, 0xc9, 0xbd, + 0x9a, 0x5a, 0xcb, 0x1c, 0xc1, 0x18, 0xbe, 0x56, + 0x3e, 0xb9, 0xb3, 0xa4, 0xa4, 0x72, 0xf8, 0x2e, + 0x09, 0xa7, 0xe7, 0x78, 0x49, 0x2b, 0x56, 0x2e, + 0xf7, 0x13, 0x0e, 0x88, 0xdf, 0xe0, 0x31, 0xc7, + + 0x9d, 0xb9, 0xd4, 0xf7, 0xc7, 0xa8, 0x99, 0x15, + 0x1b, 0x9a, 0x47, 0x50, 0x32, 0xb6, 0x3f, 0xc3, + 0x85, 0x24, 0x5f, 0xe0, 0x54, 0xe3, 0xdd, 0x5a, + 0x97, 0xa5, 0xf5, 0x76, 0xfe, 0x06, 0x40, 0x25, + 0xd3, 0xce, 0x04, 0x2c, 0x56, 0x6a, 0xb2, 0xc5, + 0x07, 0xb1, 0x38, 0xdb, 0x85, 0x3e, 0x3d, 0x69, + 0x59, 0x66, 0x09, 0x96, 0x54, 0x6c, 0xc9, 0xc4, + 0xa6, 0xea, 0xfd, 0xc7, 0x77, 0xc0, 0x40, 0xd7, + + 0x0e, 0xaf, 0x46, 0xf7, 0x6d, 0xad, 0x39, 0x79, + 0xe5, 0xc5, 0x36, 0x0c, 0x33, 0x17, 0x16, 0x6a, + 0x1c, 0x89, 0x4c, 0x94, 0xa3, 0x71, 0x87, 0x6a, + 0x94, 0xdf, 0x76, 0x28, 0xfe, 0x4e, 0xaa, 0xf2, + 0xcc, 0xb2, 0x7d, 0x5a, 0xaa, 0xe0, 0xad, 0x7a, + 0xd0, 0xf9, 0xd4, 0xb6, 0xad, 0x3b, 0x54, 0x09, + 0x87, 0x46, 0xd4, 0x52, 0x4d, 0x38, 0x40, 0x7a, + 0x6d, 0xeb, 0x3a, 0xb7, 0x8f, 0xab, 0x78, 0xc9, + }; + const input = []u8{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + var result: [256]u8 = undefined; + const key = []u8{ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + }; + const nonce = []u8{ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + }; + + chaCha20With64BitNonce(result[0..], input[0..], 0, key, nonce); + assert(mem.eql(u8, expected_result, result)); +} diff --git a/std/crypto/index.zig b/std/crypto/index.zig index 2f39020228..1b18a5dc6f 100644 --- a/std/crypto/index.zig +++ b/std/crypto/index.zig @@ -24,6 +24,10 @@ pub const HmacMd5 = hmac.HmacMd5; pub const HmacSha1 = hmac.Sha1; pub const HmacSha256 = hmac.Sha256; +const import_chaCha20 = @import("chacha20.zig"); +pub const chaCha20IETF = import_chaCha20.chaCha20IETF; +pub const chaCha20With64BitNonce = import_chaCha20.chaCha20With64BitNonce; + test "crypto" { _ = @import("md5.zig"); _ = @import("sha1.zig"); @@ -31,4 +35,5 @@ test "crypto" { _ = @import("sha3.zig"); _ = @import("blake2.zig"); _ = @import("hmac.zig"); + _ = @import("chacha20.zig"); } From 87eb95f816b01c0133de47eb3c94ac470f9d8bf2 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Mon, 27 Aug 2018 22:55:53 -0700 Subject: [PATCH 17/56] speed up chacha20 The main changes are: Unrolling the inner rounds of salsa20_wordtobyte which doubles the speed. Passing the slice explicitly instead of returning the array saves a copy (can optimize out in future with copy elision) and gives ~10% improvement. Inlining the outer loop gives ~15-20% improvement but it costs an extra 4Kb of code space. I think the tradeoff is worthwhile here. The other inline loops are small and can be done by the compiler if it is worthwhile. The rotate function replacement doesn't alter the performance from the former. The modified throughput test I've used to benchmark is as follows. Interestingly we need to allocate memory instead of using a fixed buffer else Zig optimizes the whole thing out. https://github.com/ziglang/zig/pull/1369#issuecomment-416456628 --- std/crypto/chacha20.zig | 46 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/std/crypto/chacha20.zig b/std/crypto/chacha20.zig index 3030569ae6..836c8c8878 100644 --- a/std/crypto/chacha20.zig +++ b/std/crypto/chacha20.zig @@ -22,19 +22,15 @@ fn Rp(a: usize, b: usize, c: usize, d: usize) QuarterRound { }; } -fn rotate(a: u32, b: u5) u32 { - return ((a << b) | - (a >> @intCast(u5, (32 - @intCast(u6, b)))) - ); -} - // The chacha family of ciphers are based on the salsa family. -fn salsa20_wordtobyte(input: [16]u32) [64]u8 { +fn salsa20_wordtobyte(out: []u8, input: [16]u32) void { + assert(out.len >= 64); + var x: [16]u32 = undefined; - var out: [64]u8 = undefined; for (x) |_, i| x[i] = input[i]; + const rounds = comptime []QuarterRound{ Rp( 0, 4, 8,12), Rp( 1, 5, 9,13), @@ -45,20 +41,21 @@ fn salsa20_wordtobyte(input: [16]u32) [64]u8 { Rp( 2, 7, 8,13), Rp( 3, 4, 9,14), }; - comptime var j: usize = 20; - inline while (j > 0) : (j -=2) { - for (rounds) |r| { - x[r.a] +%= x[r.b]; x[r.d] = rotate(x[r.d] ^ x[r.a], 16); - x[r.c] +%= x[r.d]; x[r.b] = rotate(x[r.b] ^ x[r.c], 12); - x[r.a] +%= x[r.b]; x[r.d] = rotate(x[r.d] ^ x[r.a], 8); - x[r.c] +%= x[r.d]; x[r.b] = rotate(x[r.b] ^ x[r.c], 7); + + comptime var j: usize = 0; + inline while (j < 20) : (j += 2) { + // two-round cycles + inline for (rounds) |r| { + x[r.a] +%= x[r.b]; x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], u32(16)); + x[r.c] +%= x[r.d]; x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], u32(12)); + x[r.a] +%= x[r.b]; x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], u32(8)); + x[r.c] +%= x[r.d]; x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], u32(7)); } } - for (x) |_, i| - x[i] +%= input[i]; - for (x) |_, i| - mem.writeInt(out[4 * i .. 4 * i + 4], x[i], builtin.Endian.Little); - return out; + + for (x) |_, i| { + mem.writeInt(out[4 * i .. 4 * i + 4], x[i] +% input[i], builtin.Endian.Little); + } } fn chaCha20_internal(out: []u8, in: []const u8, key: [8]u32, counter: [4]u32) void { @@ -73,13 +70,14 @@ fn chaCha20_internal(out: []u8, in: []const u8, key: [8]u32, counter: [4]u32) vo mem.readIntLE(u32, c[8..12]), mem.readIntLE(u32, c[12..16]), }; - + mem.copy(u32, ctx[0..], constant_le[0..4]); mem.copy(u32, ctx[4..12], key[0..8]); mem.copy(u32, ctx[12..16], counter[0..4]); while (true) { - var buf = salsa20_wordtobyte(ctx); + var buf: [64]u8 = undefined; + salsa20_wordtobyte(buf[0..], ctx); if (remaining < 64) { var i: usize = 0; @@ -88,8 +86,8 @@ fn chaCha20_internal(out: []u8, in: []const u8, key: [8]u32, counter: [4]u32) vo return; } - comptime var i: usize = 0; - inline while (i < 64) : (i += 1) + var i: usize = 0; + while (i < 64) : (i += 1) out[cursor + i] = in[cursor + i] ^ buf[i]; cursor += 64; From 09cc1dc66067f378a1508e34c0714b659b445724 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 28 Aug 2018 15:24:28 -0400 Subject: [PATCH 18/56] fix crash when var in inline loop has different types closes #917 closes #845 closes #741 closes #740 --- src/all_types.hpp | 5 ++++ src/ir.cpp | 64 +++++++++++++++++++++++++++------------------ test/cases/eval.zig | 13 +++++++++ test/cases/for.zig | 6 ++--- 4 files changed, 58 insertions(+), 30 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index a8dd9dde83..6f0dd9baac 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1807,6 +1807,11 @@ struct VariableTableEntry { VarLinkage linkage; IrInstruction *decl_instruction; uint32_t align_bytes; + + // In an inline loop, multiple variables may be created, + // In this case, a reference to a variable should follow + // this pointer to the redefined variable. + VariableTableEntry *next_var; }; struct ErrorTableEntry { diff --git a/src/ir.cpp b/src/ir.cpp index 3117410cd2..4e64b3e002 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -3280,7 +3280,8 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, } static VariableTableEntry *create_local_var(CodeGen *codegen, AstNode *node, Scope *parent_scope, - Buf *name, bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstruction *is_comptime) + Buf *name, bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstruction *is_comptime, + bool skip_name_check) { VariableTableEntry *variable_entry = allocate(1); variable_entry->parent_scope = parent_scope; @@ -3293,29 +3294,30 @@ static VariableTableEntry *create_local_var(CodeGen *codegen, AstNode *node, Sco if (name) { buf_init_from_buf(&variable_entry->name, name); - VariableTableEntry *existing_var = find_variable(codegen, parent_scope, name); - if (existing_var && !existing_var->shadowable) { - ErrorMsg *msg = add_node_error(codegen, node, - buf_sprintf("redeclaration of variable '%s'", buf_ptr(name))); - add_error_note(codegen, msg, existing_var->decl_node, buf_sprintf("previous declaration is here")); - variable_entry->value->type = codegen->builtin_types.entry_invalid; - } else { - TypeTableEntry *type = get_primitive_type(codegen, name); - if (type != nullptr) { - add_node_error(codegen, node, - buf_sprintf("variable shadows type '%s'", buf_ptr(&type->name))); + if (!skip_name_check) { + VariableTableEntry *existing_var = find_variable(codegen, parent_scope, name); + if (existing_var && !existing_var->shadowable) { + ErrorMsg *msg = add_node_error(codegen, node, + buf_sprintf("redeclaration of variable '%s'", buf_ptr(name))); + add_error_note(codegen, msg, existing_var->decl_node, buf_sprintf("previous declaration is here")); variable_entry->value->type = codegen->builtin_types.entry_invalid; } else { - Tld *tld = find_decl(codegen, parent_scope, name); - if (tld != nullptr) { - ErrorMsg *msg = add_node_error(codegen, node, - buf_sprintf("redefinition of '%s'", buf_ptr(name))); - add_error_note(codegen, msg, tld->source_node, buf_sprintf("previous definition is here")); + TypeTableEntry *type = get_primitive_type(codegen, name); + if (type != nullptr) { + add_node_error(codegen, node, + buf_sprintf("variable shadows type '%s'", buf_ptr(&type->name))); variable_entry->value->type = codegen->builtin_types.entry_invalid; + } else { + Tld *tld = find_decl(codegen, parent_scope, name); + if (tld != nullptr) { + ErrorMsg *msg = add_node_error(codegen, node, + buf_sprintf("redefinition of '%s'", buf_ptr(name))); + add_error_note(codegen, msg, tld->source_node, buf_sprintf("previous definition is here")); + variable_entry->value->type = codegen->builtin_types.entry_invalid; + } } } } - } else { assert(is_shadowable); // TODO make this name not actually be in scope. user should be able to make a variable called "_anon" @@ -3338,14 +3340,9 @@ static VariableTableEntry *ir_create_var(IrBuilder *irb, AstNode *node, Scope *s bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstruction *is_comptime) { bool is_underscored = name ? buf_eql_str(name, "_") : false; - VariableTableEntry *var = create_local_var( irb->codegen - , node - , scope - , (is_underscored ? nullptr : name) - , src_is_const - , gen_is_const - , (is_underscored ? true : is_shadowable) - , is_comptime ); + VariableTableEntry *var = create_local_var(irb->codegen, node, scope, + (is_underscored ? nullptr : name), src_is_const, gen_is_const, + (is_underscored ? true : is_shadowable), is_comptime, false); if (is_comptime != nullptr || gen_is_const) { var->mem_slot_index = exec_next_mem_slot(irb->exec); var->owner_exec = irb->exec; @@ -12495,6 +12492,17 @@ static TypeTableEntry *ir_analyze_instruction_decl_var(IrAnalyze *ira, IrInstruc } } + if (var->value->type != nullptr && var->value->type != result_type && !is_comptime_var) { + // This is at least the second time we've seen this variable declaration during analysis. + // This means that this is actually a different variable due to, e.g. an inline while loop. + // We make a new variable so that it can hold a different type, and so the debug info can + // be distinct. + VariableTableEntry *new_var = create_local_var(ira->codegen, var->decl_node, var->child_scope, + &var->name, var->src_is_const, var->gen_is_const, var->shadowable, var->is_comptime, true); + var->next_var = new_var; + var = new_var; + } + var->value->type = result_type; assert(var->value->type); @@ -12977,6 +12985,10 @@ static IrInstruction *ir_get_var_ptr(IrAnalyze *ira, IrInstruction *instruction, VariableTableEntry *var) { Error err; + while (var->next_var != nullptr) { + var = var->next_var; + } + if (var->mem_slot_index != SIZE_MAX && var->owner_exec->analysis == nullptr) { assert(ira->codegen->errors.length != 0); return ira->codegen->invalid_instruction; diff --git a/test/cases/eval.zig b/test/cases/eval.zig index 9da475994d..96b9e14cf5 100644 --- a/test/cases/eval.zig +++ b/test/cases/eval.zig @@ -652,3 +652,16 @@ fn loopNTimes(comptime n: usize) void { comptime var i = 0; inline while (i < n) : (i += 1) {} } + +test "variable inside inline loop that has different types on different iterations" { + testVarInsideInlineLoop(true, u32(42)); +} + +fn testVarInsideInlineLoop(args: ...) void { + comptime var i = 0; + inline while (i < args.len) : (i += 1) { + const x = args[i]; + if (i == 0) assert(x); + if (i == 1) assert(x == 42); + } +} diff --git a/test/cases/for.zig b/test/cases/for.zig index 59d90c1b85..f943515cc1 100644 --- a/test/cases/for.zig +++ b/test/cases/for.zig @@ -71,8 +71,7 @@ fn testBreakOuter() void { var array = "aoeu"; var count: usize = 0; outer: for (array) |_| { - // TODO shouldn't get error for redeclaring "_" - for (array) |_2| { + for (array) |_| { count += 1; break :outer; } @@ -89,8 +88,7 @@ fn testContinueOuter() void { var array = "aoeu"; var counter: usize = 0; outer: for (array) |_| { - // TODO shouldn't get error for redeclaring "_" - for (array) |_2| { + for (array) |_| { counter += 1; continue :outer; } From 901b5c1566a497822b98ba9327578839bac3df50 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 28 Aug 2018 15:39:32 -0400 Subject: [PATCH 19/56] add compile error for function prototype with no body closes #1231 --- src/analyze.cpp | 7 +++++++ test/compile_errors.zig | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/analyze.cpp b/src/analyze.cpp index ccab9170e9..405e010ba4 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -3242,6 +3242,13 @@ static void add_top_level_decl(CodeGen *g, ScopeDecls *decls_scope, Tld *tld) { } else if (tld->id == TldIdFn) { assert(tld->source_node->type == NodeTypeFnProto); is_export = tld->source_node->data.fn_proto.is_export; + + if (!is_export && !tld->source_node->data.fn_proto.is_extern && + tld->source_node->data.fn_proto.fn_def_node == nullptr) + { + add_node_error(g, tld->source_node, buf_sprintf("non-extern function has no body")); + return; + } } if (is_export) { g->resolve_queue.append(tld); diff --git a/test/compile_errors.zig b/test/compile_errors.zig index d0795c4550..5405b7b8a8 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -1,6 +1,16 @@ const tests = @import("tests.zig"); pub fn addCases(cases: *tests.CompileErrorContext) void { + cases.add( + "function protoype with no body", + \\fn foo() void; + \\export fn entry() void { + \\ foo(); + \\} + , + ".tmp_source.zig:1:1: error: non-extern function has no body", + ); + cases.add( "@typeInfo causing depend on itself compile error", \\const start = struct { From b65cca37ceb2b4181a4e713c270c546682acea23 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 28 Aug 2018 15:48:39 -0400 Subject: [PATCH 20/56] add test coverage for invalid switch expression parameter closes #604 --- test/compile_errors.zig | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 5405b7b8a8..9979ed666c 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -1,6 +1,22 @@ const tests = @import("tests.zig"); pub fn addCases(cases: *tests.CompileErrorContext) void { + cases.add( + "switch with invalid expression parameter", + \\export fn entry() void { + \\ Test(i32); + \\} + \\fn Test(comptime T: type) void { + \\ const x = switch (T) { + \\ []u8 => |x| 123, + \\ i32 => |x| 456, + \\ else => unreachable, + \\ }; + \\} + , + ".tmp_source.zig:7:17: error: switch on type 'type' provides no expression parameter", + ); + cases.add( "function protoype with no body", \\fn foo() void; From 833477abf5a7df2c8861a2df11fae2105016e087 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 28 Aug 2018 18:55:51 -0400 Subject: [PATCH 21/56] fix unresolved path preventing PDB loading --- std/debug/index.zig | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/std/debug/index.zig b/std/debug/index.zig index 2828f42a8b..38d60a6818 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -393,15 +393,19 @@ fn openSelfDebugInfoWindows(allocator: *mem.Allocator) !DebugInfo { try coff_file.loadHeader(); - var path: [windows.MAX_PATH]u8 = undefined; - const len = try coff_file.getPdbPath(path[0..]); - std.debug.warn("pdb path {}\n", path[0..len]); + var path_buf: [windows.MAX_PATH]u8 = undefined; + const len = try coff_file.getPdbPath(path_buf[0..]); + const raw_path = path_buf[0..len]; + std.debug.warn("pdb raw path {}\n", raw_path); + + const path = try os.path.resolve(allocator, raw_path); + std.debug.warn("pdb resolved path {}\n", path); var di = DebugInfo{ .pdb = undefined, }; - try di.pdb.openFile(allocator, path[0..len]); + try di.pdb.openFile(allocator, path); var pdb_stream = di.pdb.getStream(pdb.StreamType.Pdb) orelse return error.InvalidDebugInfo; std.debug.warn("pdb real filepos {}\n", pdb_stream.getFilePos()); From f1b71053de6c5e71e2e1f73daeaff29280b69350 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 29 Aug 2018 16:35:51 -0400 Subject: [PATCH 22/56] use RtlCaptureStackBackTrace on windows --- CMakeLists.txt | 1 + std/debug/index.zig | 24 +++++++++++++++++++++++- std/os/file.zig | 1 + std/os/index.zig | 6 ++++-- std/os/windows/index.zig | 1 + std/os/windows/ntdll.zig | 3 +++ 6 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 std/os/windows/ntdll.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d8ace6a61..5664f1db19 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -578,6 +578,7 @@ set(ZIG_STD_FILES "os/windows/error.zig" "os/windows/index.zig" "os/windows/kernel32.zig" + "os/windows/ntdll.zig" "os/windows/ole32.zig" "os/windows/shell32.zig" "os/windows/shlwapi.zig" diff --git a/std/debug/index.zig b/std/debug/index.zig index 38d60a6818..1bf38a9fbe 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -195,6 +195,10 @@ pub inline fn getReturnAddress(frame_count: usize) usize { } pub fn writeCurrentStackTrace(out_stream: var, allocator: *mem.Allocator, debug_info: *DebugInfo, tty_color: bool, start_addr: ?usize) !void { + switch (builtin.os) { + builtin.Os.windows => return writeCurrentStackTraceWindows(out_stream, allocator, debug_info, tty_color, start_addr), + else => {}, + } const AddressState = union(enum) { NotLookingForStartAddress, LookingForStartAddress: usize, @@ -227,6 +231,24 @@ pub fn writeCurrentStackTrace(out_stream: var, allocator: *mem.Allocator, debug_ } } +pub fn writeCurrentStackTraceWindows(out_stream: var, allocator: *mem.Allocator, debug_info: *DebugInfo, + tty_color: bool, start_addr: ?usize) !void +{ + var addr_buf: [1024]usize = undefined; + const casted_len = @intCast(u32, addr_buf.len); // TODO shouldn't need this cast + const n = windows.RtlCaptureStackBackTrace(0, casted_len, @ptrCast(**c_void, &addr_buf), null); + const addrs = addr_buf[0..n]; + var start_i: usize = if (start_addr) |saddr| blk: { + for (addrs) |addr, i| { + if (addr == saddr) break :blk i; + } + return; + } else 0; + for (addrs[start_i..]) |addr| { + try printSourceAtAddress(debug_info, out_stream, addr, tty_color); + } +} + pub fn printSourceAtAddress(debug_info: *DebugInfo, out_stream: var, address: usize, tty_color: bool) !void { switch (builtin.os) { builtin.Os.macosx => return printSourceAtAddressMacOs(debug_info, out_stream, address, tty_color), @@ -237,7 +259,7 @@ pub fn printSourceAtAddress(debug_info: *DebugInfo, out_stream: var, address: us } fn printSourceAtAddressWindows(di: *DebugInfo, out_stream: var, address: usize, tty_color: bool) !void { - const base_address = @ptrToInt(windows.GetModuleHandleW(null)); // returned HMODULE points to our executable file in memory + const base_address = os.getBaseAddress(); const relative_address = address - base_address; std.debug.warn("{x} - {x} => {x}\n", address, base_address, relative_address); try di.pdb.getSourceLine(relative_address); diff --git a/std/os/file.zig b/std/os/file.zig index d63cb1deaa..e90275ec7f 100644 --- a/std/os/file.zig +++ b/std/os/file.zig @@ -271,6 +271,7 @@ pub const File = struct { const err = windows.GetLastError(); return switch (err) { windows.ERROR.INVALID_PARAMETER => unreachable, + windows.ERROR.INVALID_HANDLE => unreachable, else => os.unexpectedErrorWindows(err), }; } diff --git a/std/os/index.zig b/std/os/index.zig index 74f94dce5a..03337b63bc 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -661,6 +661,7 @@ pub fn getBaseAddress() usize { return phdr - @sizeOf(ElfHeader); }, builtin.Os.macosx => return @ptrToInt(&std.c._mh_execute_header), + builtin.Os.windows => return @ptrToInt(windows.GetModuleHandleW(null)), else => @compileError("Unsupported OS"), } } @@ -2069,7 +2070,7 @@ fn testWindowsCmdLine(input_cmd_line: [*]const u8, expected_args: []const []cons } // TODO make this a build variable that you can set -const unexpected_error_tracing = false; +const unexpected_error_tracing = true; const UnexpectedError = error{ /// The Operating System returned an undocumented error code. Unexpected, @@ -2088,8 +2089,9 @@ pub fn unexpectedErrorPosix(errno: usize) UnexpectedError { /// Call this when you made a windows DLL call or something that does SetLastError /// and you get an unexpected error. pub fn unexpectedErrorWindows(err: windows.DWORD) UnexpectedError { - if (true) { + if (unexpected_error_tracing) { debug.warn("unexpected GetLastError(): {}\n", err); + @breakpoint(); debug.dumpCurrentStackTrace(null); } return error.Unexpected; diff --git a/std/os/windows/index.zig b/std/os/windows/index.zig index 5c68176c5a..ca6299dc5e 100644 --- a/std/os/windows/index.zig +++ b/std/os/windows/index.zig @@ -3,6 +3,7 @@ const assert = std.debug.assert; pub use @import("advapi32.zig"); pub use @import("kernel32.zig"); +pub use @import("ntdll.zig"); pub use @import("ole32.zig"); pub use @import("shell32.zig"); pub use @import("shlwapi.zig"); diff --git a/std/os/windows/ntdll.zig b/std/os/windows/ntdll.zig new file mode 100644 index 0000000000..acb78a59f4 --- /dev/null +++ b/std/os/windows/ntdll.zig @@ -0,0 +1,3 @@ +use @import("index.zig"); + +pub extern "NtDll" stdcallcc fn RtlCaptureStackBackTrace(FramesToSkip: DWORD, FramesToCapture: DWORD, BackTrace: **c_void, BackTraceHash: ?*DWORD) WORD; From 686663239af6afd8dea814a9fe6a8885f06d6cb3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 29 Aug 2018 19:00:24 -0400 Subject: [PATCH 23/56] printing info from the ModuleInfo substream of DebugInfo --- std/coff.zig | 8 +-- std/debug/index.zig | 10 ++-- std/io.zig | 8 +-- std/os/file.zig | 19 +++---- std/pdb.zig | 132 ++++++++++++++++++++++++++++++++------------ 5 files changed, 120 insertions(+), 57 deletions(-) diff --git a/std/coff.zig b/std/coff.zig index 475b4fcbc1..cce001d618 100644 --- a/std/coff.zig +++ b/std/coff.zig @@ -41,7 +41,7 @@ pub const Coff = struct { pub fn loadHeader(self: *Coff) !void { const pe_pointer_offset = 0x3C; - var file_stream = io.FileInStream.init(&self.in_file); + var file_stream = io.FileInStream.init(self.in_file); const in = &file_stream.stream; var magic: [2]u8 = undefined; @@ -126,7 +126,7 @@ pub const Coff = struct { std.debug.warn("file offset {x}\n", file_offset); try self.in_file.seekTo(file_offset + debug_dir.size); - var file_stream = io.FileInStream.init(&self.in_file); + var file_stream = io.FileInStream.init(self.in_file); const in = &file_stream.stream; var cv_signature: [4]u8 = undefined; // CodeView signature @@ -158,7 +158,7 @@ pub const Coff = struct { self.sections = ArrayList(Section).init(self.allocator); - var file_stream = io.FileInStream.init(&self.in_file); + var file_stream = io.FileInStream.init(self.in_file); const in = &file_stream.stream; var name: [8]u8 = undefined; @@ -235,4 +235,4 @@ const SectionHeader = struct { number_of_relocations: u16, number_of_line_numbers: u16, characteristics: u32, -}; \ No newline at end of file +}; diff --git a/std/debug/index.zig b/std/debug/index.zig index 1bf38a9fbe..36c5a6bdc9 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -40,7 +40,7 @@ pub fn getStderrStream() !*io.OutStream(io.FileOutStream.Error) { return st; } else { stderr_file = try io.getStdErr(); - stderr_file_out_stream = io.FileOutStream.init(&stderr_file); + stderr_file_out_stream = io.FileOutStream.init(stderr_file); const st = &stderr_file_out_stream.stream; stderr_stream = st; return st; @@ -73,7 +73,7 @@ pub fn dumpCurrentStackTrace(start_addr: ?usize) void { stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", @errorName(err)) catch return; return; }; - writeCurrentStackTrace(stderr, getDebugInfoAllocator(), debug_info, wantTtyColor(), start_addr) catch |err| { + writeCurrentStackTrace(stderr, debug_info, wantTtyColor(), start_addr) catch |err| { stderr.print("Unable to dump stack trace: {}\n", @errorName(err)) catch return; return; }; @@ -194,9 +194,9 @@ pub inline fn getReturnAddress(frame_count: usize) usize { return @intToPtr(*const usize, fp + @sizeOf(usize)).*; } -pub fn writeCurrentStackTrace(out_stream: var, allocator: *mem.Allocator, debug_info: *DebugInfo, tty_color: bool, start_addr: ?usize) !void { +pub fn writeCurrentStackTrace(out_stream: var, debug_info: *DebugInfo, tty_color: bool, start_addr: ?usize) !void { switch (builtin.os) { - builtin.Os.windows => return writeCurrentStackTraceWindows(out_stream, allocator, debug_info, tty_color, start_addr), + builtin.Os.windows => return writeCurrentStackTraceWindows(out_stream, debug_info, tty_color, start_addr), else => {}, } const AddressState = union(enum) { @@ -231,7 +231,7 @@ pub fn writeCurrentStackTrace(out_stream: var, allocator: *mem.Allocator, debug_ } } -pub fn writeCurrentStackTraceWindows(out_stream: var, allocator: *mem.Allocator, debug_info: *DebugInfo, +pub fn writeCurrentStackTraceWindows(out_stream: var, debug_info: *DebugInfo, tty_color: bool, start_addr: ?usize) !void { var addr_buf: [1024]usize = undefined; diff --git a/std/io.zig b/std/io.zig index 369f6eede3..2de52493b1 100644 --- a/std/io.zig +++ b/std/io.zig @@ -34,13 +34,13 @@ pub fn getStdIn() GetStdIoErrs!File { /// Implementation of InStream trait for File pub const FileInStream = struct { - file: *File, + file: File, stream: Stream, pub const Error = @typeOf(File.read).ReturnType.ErrorSet; pub const Stream = InStream(Error); - pub fn init(file: *File) FileInStream { + pub fn init(file: File) FileInStream { return FileInStream{ .file = file, .stream = Stream{ .readFn = readFn }, @@ -55,13 +55,13 @@ pub const FileInStream = struct { /// Implementation of OutStream trait for File pub const FileOutStream = struct { - file: *File, + file: File, stream: Stream, pub const Error = File.WriteError; pub const Stream = OutStream(Error); - pub fn init(file: *File) FileOutStream { + pub fn init(file: File) FileOutStream { return FileOutStream{ .file = file, .stream = Stream{ .writeFn = writeFn }, diff --git a/std/os/file.zig b/std/os/file.zig index e90275ec7f..020a5dca54 100644 --- a/std/os/file.zig +++ b/std/os/file.zig @@ -205,17 +205,16 @@ pub const File = struct { /// Upon success, the stream is in an uninitialized state. To continue using it, /// you must use the open() function. - pub fn close(self: *File) void { + pub fn close(self: File) void { os.close(self.handle); - self.handle = undefined; } /// Calls `os.isTty` on `self.handle`. - pub fn isTty(self: *File) bool { + pub fn isTty(self: File) bool { return os.isTty(self.handle); } - pub fn seekForward(self: *File, amount: isize) !void { + pub fn seekForward(self: File, amount: isize) !void { switch (builtin.os) { Os.linux, Os.macosx, Os.ios => { const result = posix.lseek(self.handle, amount, posix.SEEK_CUR); @@ -246,7 +245,7 @@ pub const File = struct { } } - pub fn seekTo(self: *File, pos: usize) !void { + pub fn seekTo(self: File, pos: usize) !void { switch (builtin.os) { Os.linux, Os.macosx, Os.ios => { const ipos = try math.cast(isize, pos); @@ -280,7 +279,7 @@ pub const File = struct { } } - pub fn getPos(self: *File) !usize { + pub fn getPos(self: File) !usize { switch (builtin.os) { Os.linux, Os.macosx, Os.ios => { const result = posix.lseek(self.handle, 0, posix.SEEK_CUR); @@ -316,7 +315,7 @@ pub const File = struct { } } - pub fn getEndPos(self: *File) !usize { + pub fn getEndPos(self: File) !usize { if (is_posix) { const stat = try os.posixFStat(self.handle); return @intCast(usize, stat.size); @@ -341,7 +340,7 @@ pub const File = struct { Unexpected, }; - pub fn mode(self: *File) ModeError!Mode { + pub fn mode(self: File) ModeError!Mode { if (is_posix) { var stat: posix.Stat = undefined; const err = posix.getErrno(posix.fstat(self.handle, &stat)); @@ -375,7 +374,7 @@ pub const File = struct { Unexpected, }; - pub fn read(self: *File, buffer: []u8) ReadError!usize { + pub fn read(self: File, buffer: []u8) ReadError!usize { if (is_posix) { var index: usize = 0; while (index < buffer.len) { @@ -423,7 +422,7 @@ pub const File = struct { pub const WriteError = os.WindowsWriteError || os.PosixWriteError; - pub fn write(self: *File, bytes: []const u8) WriteError!void { + pub fn write(self: File, bytes: []const u8) WriteError!void { if (is_posix) { try os.posixWrite(self.handle, bytes); } else if (is_windows) { diff --git a/std/pdb.zig b/std/pdb.zig index 08c1d25f65..a83011df6a 100644 --- a/std/pdb.zig +++ b/std/pdb.zig @@ -8,9 +8,58 @@ const warn = std.debug.warn; const ArrayList = std.ArrayList; -pub const PdbError = error { - InvalidPdbMagic, - CorruptedFile, +// https://llvm.org/docs/PDB/DbiStream.html#stream-header +const DbiStreamHeader = packed struct { + VersionSignature: i32, + VersionHeader: u32, + Age: u32, + GlobalStreamIndex: u16, + BuildNumber: u16, + PublicStreamIndex: u16, + PdbDllVersion: u16, + SymRecordStream: u16, + PdbDllRbld: u16, + ModInfoSize: u32, + SectionContributionSize: i32, + SectionMapSize: i32, + SourceInfoSize: i32, + TypeServerSize: i32, + MFCTypeServerIndex: u32, + OptionalDbgHeaderSize: i32, + ECSubstreamSize: i32, + Flags: u16, + Machine: u16, + Padding: u32, +}; + +const SectionContribEntry = packed struct { + Section: u16, + Padding1: [2]u8, + Offset: i32, + Size: i32, + Characteristics: u32, + ModuleIndex: u16, + Padding2: [2]u8, + DataCrc: u32, + RelocCrc: u32, +}; + +const ModInfo = packed struct { + Unused1: u32, + SectionContr: SectionContribEntry, + Flags: u16, + ModuleSymStream: u16, + SymByteSize: u32, + C11ByteSize: u32, + C13ByteSize: u32, + SourceFileCount: u16, + Padding: [2]u8, + Unused2: u32, + SourceFileNameIndex: u32, + PdbFilePathNameIndex: u32, + // These fields are variable length + //ModuleName: char[], + //ObjFileName: char[], }; pub const StreamType = enum(u16) { @@ -30,7 +79,7 @@ pub const Pdb = struct { self.in_file = try os.File.openRead(file_name[0..]); self.allocator = allocator; - try self.msf.openFile(allocator, &self.in_file); + try self.msf.openFile(allocator, self.in_file); } pub fn getStream(self: *Pdb, stream: StreamType) ?*MsfStream { @@ -41,29 +90,32 @@ pub const Pdb = struct { } pub fn getSourceLine(self: *Pdb, address: usize) !void { - const dbi = self.getStream(StreamType.Dbi) orelse return error.CorruptedFile; + const dbi = self.getStream(StreamType.Dbi) orelse return error.InvalidDebugInfo; // Dbi Header - try dbi.seekForward(@sizeOf(u32) * 3 + @sizeOf(u16) * 6); - warn("dbi stream at {} (file offset)\n", dbi.getFilePos()); - const module_info_size = try dbi.stream.readIntLe(u32); - const section_contribution_size = try dbi.stream.readIntLe(u32); - const section_map_size = try dbi.stream.readIntLe(u32); - const source_info_size = try dbi.stream.readIntLe(u32); - warn("module_info_size: {}\n", module_info_size); - warn("section_contribution_size: {}\n", section_contribution_size); - warn("section_map_size: {}\n", section_map_size); - warn("source_info_size: {}\n", source_info_size); - try dbi.seekForward(@sizeOf(u32) * 5 + @sizeOf(u16) * 2); + var header: DbiStreamHeader = undefined; + try dbi.stream.readStruct(DbiStreamHeader, &header); + std.debug.warn("{}\n", header); warn("after header dbi stream at {} (file offset)\n", dbi.getFilePos()); // Module Info Substream - try dbi.seekForward(@sizeOf(u32) + @sizeOf(u16) + @sizeOf(u8) * 2); - const offset = try dbi.stream.readIntLe(u32); - const size = try dbi.stream.readIntLe(u32); - try dbi.seekForward(@sizeOf(u32)); - const module_index = try dbi.stream.readIntLe(u16); - warn("module {} of size {} at {}\n", module_index, size, offset); + var mod_info_offset: usize = 0; + while (mod_info_offset < header.ModInfoSize) { + var mod_info: ModInfo = undefined; + try dbi.stream.readStruct(ModInfo, &mod_info); + std.debug.warn("{}\n", mod_info); + mod_info_offset += @sizeOf(ModInfo); + + const module_name = try dbi.readNullTermString(self.allocator); + std.debug.warn("module_name {}\n", module_name); + mod_info_offset += module_name.len + 1; + + const obj_file_name = try dbi.readNullTermString(self.allocator); + std.debug.warn("obj_file_name {}\n", obj_file_name); + mod_info_offset += obj_file_name.len + 1; + } + std.debug.warn("end modules\n"); + // TODO: locate corresponding source line information } @@ -75,7 +127,7 @@ const Msf = struct { directory: MsfStream, streams: ArrayList(MsfStream), - fn openFile(self: *Msf, allocator: *mem.Allocator, file: *os.File) !void { + fn openFile(self: *Msf, allocator: *mem.Allocator, file: os.File) !void { var file_stream = io.FileInStream.init(file); const in = &file_stream.stream; @@ -84,7 +136,7 @@ const Msf = struct { warn("magic: '{}'\n", magic); if (!mem.eql(u8, magic, SuperBlock.FileMagic)) - return error.InvalidPdbMagic; + return error.InvalidDebugInfo; self.superblock = SuperBlock { .block_size = try in.readIntLe(u32), @@ -97,11 +149,11 @@ const Msf = struct { switch (self.superblock.block_size) { 512, 1024, 2048, 4096 => {}, // llvm only uses 4096 - else => return error.InvalidPdbMagic + else => return error.InvalidDebugInfo } if (self.superblock.fileSize() != try file.getEndPos()) - return error.CorruptedFile; // Should always stand. + return error.InvalidDebugInfo; // Should always stand. self.directory = try MsfStream.init( self.superblock.block_size, @@ -165,12 +217,18 @@ const SuperBlock = struct { }; const MsfStream = struct { - in_file: *os.File, + in_file: os.File, pos: usize, blocks: ArrayList(u32), block_size: u32, - fn init(block_size: u32, block_count: u32, pos: usize, file: *os.File, allocator: *mem.Allocator) !MsfStream { + /// Implementation of InStream trait for Pdb.MsfStream + stream: Stream, + + pub const Error = @typeOf(read).ReturnType.ErrorSet; + pub const Stream = io.InStream(Error); + + fn init(block_size: u32, block_count: u32, pos: usize, file: os.File, allocator: *mem.Allocator) !MsfStream { var stream = MsfStream { .in_file = file, .pos = 0, @@ -198,6 +256,18 @@ const MsfStream = struct { return stream; } + fn readNullTermString(self: *MsfStream, allocator: *mem.Allocator) ![]u8 { + var list = ArrayList(u8).init(allocator); + defer list.deinit(); + while (true) { + const byte = try self.stream.readByte(); + if (byte == 0) { + return list.toSlice(); + } + try list.append(byte); + } + } + fn read(self: *MsfStream, buffer: []u8) !usize { var block_id = self.pos / self.block_size; var block = self.blocks.items[block_id]; @@ -252,12 +322,6 @@ const MsfStream = struct { return block * self.block_size + offset; } - /// Implementation of InStream trait for Pdb.MsfStream - pub const Error = @typeOf(read).ReturnType.ErrorSet; - pub const Stream = io.InStream(Error); - - stream: Stream, - fn readFn(in_stream: *Stream, buffer: []u8) Error!usize { const self = @fieldParentPtr(MsfStream, "stream", in_stream); return self.read(buffer); From 65b89f598c63719ab80553163424feb2d8e6f9e4 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Thu, 30 Aug 2018 18:02:19 +1200 Subject: [PATCH 24/56] Add poly1305 and x25519 crypto primitives These are translated from [monocypher](https://monocypher.org/) which has fairly competitive performance while remaining quite simple. Initial performance comparision: Zig: Poly1305: 1423 MiB/s X25519: 8671 exchanges per second Monocypher: Poly1305: 1567 MiB/s X25519: 10539 exchanges per second There is room for improvement and no real effort has been made at all in optimization beyond a direct translation. --- CMakeLists.txt | 4 +- std/crypto/index.zig | 5 + std/crypto/poly1305.zig | 220 +++++++++++++ std/crypto/x25519.zig | 667 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 895 insertions(+), 1 deletion(-) create mode 100644 std/crypto/poly1305.zig create mode 100644 std/crypto/x25519.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e753b5137..26833b5b01 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -445,13 +445,15 @@ set(ZIG_STD_FILES "c/linux.zig" "c/windows.zig" "crypto/blake2.zig" + "crypto/chacha20.zig" "crypto/hmac.zig" "crypto/index.zig" "crypto/md5.zig" "crypto/sha1.zig" "crypto/sha2.zig" "crypto/sha3.zig" - "crypto/chacha20.zig" + "crypto/poly1305.zig" + "crypto/x25519.zig" "cstr.zig" "debug/failing_allocator.zig" "debug/index.zig" diff --git a/std/crypto/index.zig b/std/crypto/index.zig index 1b18a5dc6f..f861b61a20 100644 --- a/std/crypto/index.zig +++ b/std/crypto/index.zig @@ -28,6 +28,9 @@ const import_chaCha20 = @import("chacha20.zig"); pub const chaCha20IETF = import_chaCha20.chaCha20IETF; pub const chaCha20With64BitNonce = import_chaCha20.chaCha20With64BitNonce; +const poly1305 = @import("poly1305.zig"); +const x25519 = @import("x25519.zig"); + test "crypto" { _ = @import("md5.zig"); _ = @import("sha1.zig"); @@ -36,4 +39,6 @@ test "crypto" { _ = @import("blake2.zig"); _ = @import("hmac.zig"); _ = @import("chacha20.zig"); + _ = @import("poly1305.zig"); + _ = @import("x25519.zig"); } diff --git a/std/crypto/poly1305.zig b/std/crypto/poly1305.zig new file mode 100644 index 0000000000..bce90db3d5 --- /dev/null +++ b/std/crypto/poly1305.zig @@ -0,0 +1,220 @@ +// Translated from monocypher which is licensed under CC-0/BSD-3. +// +// https://monocypher.org/ + +const std = @import("../index.zig"); +const builtin = @import("builtin"); + +const Endian = builtin.Endian; +const readInt = std.mem.readInt; +const writeInt = std.mem.writeInt; + +const crypto_poly1305_ctx = struct { + // constant multiplier (from the secret key) + r: [4]u32, + // accumulated hash + h: [5]u32, + // chunk of the message + c: [5]u32, + // random number added at the end (from the secret key) + pad: [4]u32, + // How many bytes are there in the chunk. + c_idx: usize, + + fn secure_zero(self: *crypto_poly1305_ctx) void { + std.mem.secureZero(u8, @ptrCast([*]u8, self)[0..@sizeOf(crypto_poly1305_ctx)]); + } +}; + +// h = (h + c) * r +// preconditions: +// ctx->h <= 4_ffffffff_ffffffff_ffffffff_ffffffff +// ctx->c <= 1_ffffffff_ffffffff_ffffffff_ffffffff +// ctx->r <= 0ffffffc_0ffffffc_0ffffffc_0fffffff +// Postcondition: +// ctx->h <= 4_ffffffff_ffffffff_ffffffff_ffffffff +fn poly_block(ctx: *crypto_poly1305_ctx) void { + // s = h + c, without carry propagation + const s0 = u64(ctx.h[0]) + ctx.c[0]; // s0 <= 1_fffffffe + const s1 = u64(ctx.h[1]) + ctx.c[1]; // s1 <= 1_fffffffe + const s2 = u64(ctx.h[2]) + ctx.c[2]; // s2 <= 1_fffffffe + const s3 = u64(ctx.h[3]) + ctx.c[3]; // s3 <= 1_fffffffe + const s4 = u64(ctx.h[4]) + ctx.c[4]; // s4 <= 5 + + // Local all the things! + const r0 = ctx.r[0]; // r0 <= 0fffffff + const r1 = ctx.r[1]; // r1 <= 0ffffffc + const r2 = ctx.r[2]; // r2 <= 0ffffffc + const r3 = ctx.r[3]; // r3 <= 0ffffffc + const rr0 = (r0 >> 2) * 5; // rr0 <= 13fffffb // lose 2 bits... + const rr1 = (r1 >> 2) + r1; // rr1 <= 13fffffb // rr1 == (r1 >> 2) * 5 + const rr2 = (r2 >> 2) + r2; // rr2 <= 13fffffb // rr1 == (r2 >> 2) * 5 + const rr3 = (r3 >> 2) + r3; // rr3 <= 13fffffb // rr1 == (r3 >> 2) * 5 + + // (h + c) * r, without carry propagation + const x0 = s0 * r0 + s1 * rr3 + s2 * rr2 + s3 * rr1 + s4 * rr0; //<=97ffffe007fffff8 + const x1 = s0 * r1 + s1 * r0 + s2 * rr3 + s3 * rr2 + s4 * rr1; //<=8fffffe20ffffff6 + const x2 = s0 * r2 + s1 * r1 + s2 * r0 + s3 * rr3 + s4 * rr2; //<=87ffffe417fffff4 + const x3 = s0 * r3 + s1 * r2 + s2 * r1 + s3 * r0 + s4 * rr3; //<=7fffffe61ffffff2 + const x4 = s4 * (r0 & 3); // ...recover 2 bits //<= f + + // partial reduction modulo 2^130 - 5 + const _u5 = @truncate(u32, x4 + (x3 >> 32)); // u5 <= 7ffffff5 + const _u0 = (_u5 >> 2) * 5 + (x0 & 0xffffffff); + const _u1 = (_u0 >> 32) + (x1 & 0xffffffff) + (x0 >> 32); + const _u2 = (_u1 >> 32) + (x2 & 0xffffffff) + (x1 >> 32); + const _u3 = (_u2 >> 32) + (x3 & 0xffffffff) + (x2 >> 32); + const _u4 = (_u3 >> 32) + (_u5 & 3); + + // Update the hash + ctx.h[0] = @truncate(u32, _u0); // u0 <= 1_9ffffff0 + ctx.h[1] = @truncate(u32, _u1); // u1 <= 1_97ffffe0 + ctx.h[2] = @truncate(u32, _u2); // u2 <= 1_8fffffe2 + ctx.h[3] = @truncate(u32, _u3); // u3 <= 1_87ffffe4 + ctx.h[4] = @truncate(u32, _u4); // u4 <= 4 +} + +// (re-)initializes the input counter and input buffer +fn poly_clear_c(ctx: *crypto_poly1305_ctx) void { + ctx.c[0] = 0; + ctx.c[1] = 0; + ctx.c[2] = 0; + ctx.c[3] = 0; + ctx.c_idx = 0; +} + +fn poly_take_input(ctx: *crypto_poly1305_ctx, input: u8) void { + const word = ctx.c_idx >> 2; + const byte = ctx.c_idx & 3; + ctx.c[word] |= std.math.shl(u32, input, byte * 8); + ctx.c_idx += 1; +} + +fn poly_update(ctx: *crypto_poly1305_ctx, message: []const u8) void { + for (message) |b| { + poly_take_input(ctx, b); + if (ctx.c_idx == 16) { + poly_block(ctx); + poly_clear_c(ctx); + } + } +} + +pub fn crypto_poly1305_init(ctx: *crypto_poly1305_ctx, key: [32]u8) void { + // Initial hash is zero + { + var i: usize = 0; + while (i < 5) : (i += 1) { + ctx.h[i] = 0; + } + } + // add 2^130 to every input block + ctx.c[4] = 1; + poly_clear_c(ctx); + + // load r and pad (r has some of its bits cleared) + { + var i: usize = 0; + while (i < 1) : (i += 1) { + ctx.r[0] = readInt(key[0..4], u32, Endian.Little) & 0x0fffffff; + } + } + { + var i: usize = 1; + while (i < 4) : (i += 1) { + ctx.r[i] = readInt(key[i * 4 .. i * 4 + 4], u32, Endian.Little) & 0x0ffffffc; + } + } + { + var i: usize = 0; + while (i < 4) : (i += 1) { + ctx.pad[i] = readInt(key[i * 4 + 16 .. i * 4 + 16 + 4], u32, Endian.Little); + } + } +} + +inline fn alignto(x: usize, block_size: usize) usize { + return ((~x) +% 1) & (block_size - 1); +} + +pub fn crypto_poly1305_update(ctx: *crypto_poly1305_ctx, message: []const u8) void { + // Align ourselves with block boundaries + const alignm = std.math.min(alignto(ctx.c_idx, 16), message.len); + poly_update(ctx, message[0..alignm]); + + var nmessage = message[alignm..]; + + // Process the message block by block + const nb_blocks = nmessage.len >> 4; + var i: usize = 0; + while (i < nb_blocks) : (i += 1) { + ctx.c[0] = readInt(nmessage[0..4], u32, Endian.Little); + ctx.c[1] = readInt(nmessage[4..8], u32, Endian.Little); + ctx.c[2] = readInt(nmessage[8..12], u32, Endian.Little); + ctx.c[3] = readInt(nmessage[12..16], u32, Endian.Little); + poly_block(ctx); + nmessage = nmessage[16..]; + } + if (nb_blocks > 0) { + poly_clear_c(ctx); + } + + // remaining bytes + poly_update(ctx, nmessage[0..]); +} + +pub fn crypto_poly1305_final(ctx: *crypto_poly1305_ctx, mac: []u8) void { + // Process the last block (if any) + if (ctx.c_idx != 0) { + // move the final 1 according to remaining input length + // (We may add less than 2^130 to the last input block) + ctx.c[4] = 0; + poly_take_input(ctx, 1); + // one last hash update + poly_block(ctx); + } + + // check if we should subtract 2^130-5 by performing the + // corresponding carry propagation. + const _u0 = u64(5) + ctx.h[0]; // <= 1_00000004 + const _u1 = (_u0 >> 32) + ctx.h[1]; // <= 1_00000000 + const _u2 = (_u1 >> 32) + ctx.h[2]; // <= 1_00000000 + const _u3 = (_u2 >> 32) + ctx.h[3]; // <= 1_00000000 + const _u4 = (_u3 >> 32) + ctx.h[4]; // <= 5 + // u4 indicates how many times we should subtract 2^130-5 (0 or 1) + + // h + pad, minus 2^130-5 if u4 exceeds 3 + const uu0 = (_u4 >> 2) * 5 + ctx.h[0] + ctx.pad[0]; // <= 2_00000003 + const uu1 = (uu0 >> 32) + ctx.h[1] + ctx.pad[1]; // <= 2_00000000 + const uu2 = (uu1 >> 32) + ctx.h[2] + ctx.pad[2]; // <= 2_00000000 + const uu3 = (uu2 >> 32) + ctx.h[3] + ctx.pad[3]; // <= 2_00000000 + + writeInt(mac[0..], uu0, Endian.Little); + writeInt(mac[4..], uu1, Endian.Little); + writeInt(mac[8..], uu2, Endian.Little); + writeInt(mac[12..], uu3, Endian.Little); + + ctx.secure_zero(); +} + +pub fn crypto_poly1305(mac: []u8, message: []const u8, key: [32]u8) void { + std.debug.assert(mac.len >= 16); + + var ctx: crypto_poly1305_ctx = undefined; + crypto_poly1305_init(&ctx, key); + crypto_poly1305_update(&ctx, message); + crypto_poly1305_final(&ctx, mac); +} + +test "poly1305 rfc7439 vector1" { + const expected_mac = "\xa8\x06\x1d\xc1\x30\x51\x36\xc6\xc2\x2b\x8b\xaf\x0c\x01\x27\xa9"; + + const msg = "Cryptographic Forum Research Group"; + const key = "\x85\xd6\xbe\x78\x57\x55\x6d\x33\x7f\x44\x52\xfe\x42\xd5\x06\xa8" ++ + "\x01\x03\x80\x8a\xfb\x0d\xb2\xfd\x4a\xbf\xf6\xaf\x41\x49\xf5\x1b"; + + var mac: [16]u8 = undefined; + crypto_poly1305(mac[0..], msg, key); + + std.debug.assert(std.mem.eql(u8, mac, expected_mac)); +} diff --git a/std/crypto/x25519.zig b/std/crypto/x25519.zig new file mode 100644 index 0000000000..b5b85e7104 --- /dev/null +++ b/std/crypto/x25519.zig @@ -0,0 +1,667 @@ +// Translated from monocypher which is licensed under CC-0/BSD-3. +// +// https://monocypher.org/ + +const std = @import("../index.zig"); +const builtin = @import("builtin"); + +const Endian = builtin.Endian; +const readInt = std.mem.readInt; +const writeInt = std.mem.writeInt; + +// Constant time compare to zero. +fn zerocmp(comptime T: type, a: []const T) bool { + var s: T = 0; + for (a) |b| { + s |= b; + } + return s == 0; +} + +//////////////////////////////////// +/// Arithmetic modulo 2^255 - 19 /// +//////////////////////////////////// +// Taken from Supercop's ref10 implementation. +// A bit bigger than TweetNaCl, over 4 times faster. + +// field element +const Fe = struct { + b: [10]i32, + + fn secure_zero(self: *Fe) void { + std.mem.secureZero(u8, @ptrCast([*]u8, self)[0..@sizeOf(Fe)]); + } +}; + +fn fe_0(h: *Fe) void { + for (h.b) |*e| { + e.* = 0; + } +} + +fn fe_1(h: *Fe) void { + for (h.b[1..]) |*e| { + e.* = 0; + } + h.b[0] = 1; +} + +fn fe_copy(h: *Fe, f: *const Fe) void { + for (h.b) |_, i| { + h.b[i] = f.b[i]; + } +} + +fn fe_neg(h: *Fe, f: *const Fe) void { + for (h.b) |_, i| { + h.b[i] = -f.b[i]; + } +} + +fn fe_add(h: *Fe, f: *const Fe, g: *const Fe) void { + for (h.b) |_, i| { + h.b[i] = f.b[i] + g.b[i]; + } +} + +fn fe_sub(h: *Fe, f: *const Fe, g: *const Fe) void { + for (h.b) |_, i| { + h.b[i] = f.b[i] - g.b[i]; + } +} + +fn fe_cswap(f: *Fe, g: *Fe, b: i32) void { + for (f.b) |_, i| { + const x = (f.b[i] ^ g.b[i]) & -b; + f.b[i] ^= x; + g.b[i] ^= x; + } +} + +fn fe_ccopy(f: *Fe, g: *const Fe, b: i32) void { + for (f.b) |_, i| { + const x = (f.b[i] ^ g.b[i]) & -b; + f.b[i] ^= x; + } +} + +inline fn carryround(c: []i64, t: []i64, comptime i: comptime_int, comptime shift: comptime_int, comptime mult: comptime_int) void { + const j = (i + 1) % 10; + + c[i] = (t[i] + (i64(1) << shift)) >> (shift + 1); + t[j] += c[i] * mult; + t[i] -= c[i] * (i64(1) << (shift + 1)); +} + +fn feCarry1(h: *Fe, t: []i64) void { + var c: [10]i64 = undefined; + + var sc = c[0..]; + var st = t[0..]; + + carryround(sc, st, 9, 24, 19); + carryround(sc, st, 1, 24, 1); + carryround(sc, st, 3, 24, 1); + carryround(sc, st, 5, 24, 1); + carryround(sc, st, 7, 24, 1); + carryround(sc, st, 0, 25, 1); + carryround(sc, st, 2, 25, 1); + carryround(sc, st, 4, 25, 1); + carryround(sc, st, 6, 25, 1); + carryround(sc, st, 8, 25, 1); + + for (h.b) |_, i| { + h.b[i] = @intCast(i32, t[i]); + } +} + +fn feCarry2(h: *Fe, t: []i64) void { + var c: [10]i64 = undefined; + + var sc = c[0..]; + var st = t[0..]; + + carryround(sc, st, 0, 25, 1); + carryround(sc, st, 4, 25, 1); + carryround(sc, st, 1, 24, 1); + carryround(sc, st, 5, 24, 1); + carryround(sc, st, 2, 25, 1); + carryround(sc, st, 6, 25, 1); + carryround(sc, st, 3, 24, 1); + carryround(sc, st, 7, 24, 1); + carryround(sc, st, 4, 25, 1); + carryround(sc, st, 8, 25, 1); + carryround(sc, st, 9, 24, 19); + carryround(sc, st, 0, 25, 1); + + for (h.b) |_, i| { + h.b[i] = @intCast(i32, t[i]); + } +} + +// TODO: Use readInt(u24) but double check alignment since currently it produces different values. +fn load24_le(s: []const u8) u32 { + return s[0] | (u32(s[1]) << 8) | (u32(s[2]) << 16); +} + +fn fe_frombytes(h: *Fe, s: [32]u8) void { + var t: [10]i64 = undefined; + + t[0] = readInt(s[0..4], u32, Endian.Little); + t[1] = load24_le(s[4..7]) << 6; + t[2] = load24_le(s[7..10]) << 5; + t[3] = load24_le(s[10..13]) << 3; + t[4] = load24_le(s[13..16]) << 2; + t[5] = readInt(s[16..20], u32, Endian.Little); + t[6] = load24_le(s[20..23]) << 7; + t[7] = load24_le(s[23..26]) << 5; + t[8] = load24_le(s[26..29]) << 4; + t[9] = (load24_le(s[29..32]) & 0x7fffff) << 2; + + feCarry1(h, t[0..]); +} + +fn fe_mul_small(h: *Fe, f: *const Fe, comptime g: comptime_int) void { + var t: [10]i64 = undefined; + + for (t[0..]) |_, i| { + t[i] = i64(f.b[i]) * g; + } + + feCarry1(h, t[0..]); +} + +fn fe_mul121666(h: *Fe, f: *const Fe) void { + fe_mul_small(h, f, 121666); +} + +fn fe_mul(h: *Fe, f1: *const Fe, g1: *const Fe) void { + const f = f1.b; + const g = g1.b; + + var F: [10]i32 = undefined; + var G: [10]i32 = undefined; + + F[1] = f[1] * 2; + F[3] = f[3] * 2; + F[5] = f[5] * 2; + F[7] = f[7] * 2; + F[9] = f[9] * 2; + + G[1] = g[1] * 19; + G[2] = g[2] * 19; + G[3] = g[3] * 19; + G[4] = g[4] * 19; + G[5] = g[5] * 19; + G[6] = g[6] * 19; + G[7] = g[7] * 19; + G[8] = g[8] * 19; + G[9] = g[9] * 19; + + // t's become h + var t: [10]i64 = undefined; + + t[0] = f[0] * i64(g[0]) + F[1] * i64(G[9]) + f[2] * i64(G[8]) + F[3] * i64(G[7]) + f[4] * i64(G[6]) + F[5] * i64(G[5]) + f[6] * i64(G[4]) + F[7] * i64(G[3]) + f[8] * i64(G[2]) + F[9] * i64(G[1]); + t[1] = f[0] * i64(g[1]) + f[1] * i64(g[0]) + f[2] * i64(G[9]) + f[3] * i64(G[8]) + f[4] * i64(G[7]) + f[5] * i64(G[6]) + f[6] * i64(G[5]) + f[7] * i64(G[4]) + f[8] * i64(G[3]) + f[9] * i64(G[2]); + t[2] = f[0] * i64(g[2]) + F[1] * i64(g[1]) + f[2] * i64(g[0]) + F[3] * i64(G[9]) + f[4] * i64(G[8]) + F[5] * i64(G[7]) + f[6] * i64(G[6]) + F[7] * i64(G[5]) + f[8] * i64(G[4]) + F[9] * i64(G[3]); + t[3] = f[0] * i64(g[3]) + f[1] * i64(g[2]) + f[2] * i64(g[1]) + f[3] * i64(g[0]) + f[4] * i64(G[9]) + f[5] * i64(G[8]) + f[6] * i64(G[7]) + f[7] * i64(G[6]) + f[8] * i64(G[5]) + f[9] * i64(G[4]); + t[4] = f[0] * i64(g[4]) + F[1] * i64(g[3]) + f[2] * i64(g[2]) + F[3] * i64(g[1]) + f[4] * i64(g[0]) + F[5] * i64(G[9]) + f[6] * i64(G[8]) + F[7] * i64(G[7]) + f[8] * i64(G[6]) + F[9] * i64(G[5]); + t[5] = f[0] * i64(g[5]) + f[1] * i64(g[4]) + f[2] * i64(g[3]) + f[3] * i64(g[2]) + f[4] * i64(g[1]) + f[5] * i64(g[0]) + f[6] * i64(G[9]) + f[7] * i64(G[8]) + f[8] * i64(G[7]) + f[9] * i64(G[6]); + t[6] = f[0] * i64(g[6]) + F[1] * i64(g[5]) + f[2] * i64(g[4]) + F[3] * i64(g[3]) + f[4] * i64(g[2]) + F[5] * i64(g[1]) + f[6] * i64(g[0]) + F[7] * i64(G[9]) + f[8] * i64(G[8]) + F[9] * i64(G[7]); + t[7] = f[0] * i64(g[7]) + f[1] * i64(g[6]) + f[2] * i64(g[5]) + f[3] * i64(g[4]) + f[4] * i64(g[3]) + f[5] * i64(g[2]) + f[6] * i64(g[1]) + f[7] * i64(g[0]) + f[8] * i64(G[9]) + f[9] * i64(G[8]); + t[8] = f[0] * i64(g[8]) + F[1] * i64(g[7]) + f[2] * i64(g[6]) + F[3] * i64(g[5]) + f[4] * i64(g[4]) + F[5] * i64(g[3]) + f[6] * i64(g[2]) + F[7] * i64(g[1]) + f[8] * i64(g[0]) + F[9] * i64(G[9]); + t[9] = f[0] * i64(g[9]) + f[1] * i64(g[8]) + f[2] * i64(g[7]) + f[3] * i64(g[6]) + f[4] * i64(g[5]) + f[5] * i64(g[4]) + f[6] * i64(g[3]) + f[7] * i64(g[2]) + f[8] * i64(g[1]) + f[9] * i64(g[0]); + + feCarry2(h, t[0..]); +} + +// we could use fe_mul() for this, but this is significantly faster +fn fe_sq(h: *Fe, fz: *const Fe) void { + const f0 = fz.b[0]; + const f1 = fz.b[1]; + const f2 = fz.b[2]; + const f3 = fz.b[3]; + const f4 = fz.b[4]; + const f5 = fz.b[5]; + const f6 = fz.b[6]; + const f7 = fz.b[7]; + const f8 = fz.b[8]; + const f9 = fz.b[9]; + + const f0_2 = f0 * 2; + const f1_2 = f1 * 2; + const f2_2 = f2 * 2; + const f3_2 = f3 * 2; + const f4_2 = f4 * 2; + const f5_2 = f5 * 2; + const f6_2 = f6 * 2; + const f7_2 = f7 * 2; + const f5_38 = f5 * 38; + const f6_19 = f6 * 19; + const f7_38 = f7 * 38; + const f8_19 = f8 * 19; + const f9_38 = f9 * 38; + + var t: [10]i64 = undefined; + + t[0] = f0 * i64(f0) + f1_2 * i64(f9_38) + f2_2 * i64(f8_19) + f3_2 * i64(f7_38) + f4_2 * i64(f6_19) + f5 * i64(f5_38); + t[1] = f0_2 * i64(f1) + f2 * i64(f9_38) + f3_2 * i64(f8_19) + f4 * i64(f7_38) + f5_2 * i64(f6_19); + t[2] = f0_2 * i64(f2) + f1_2 * i64(f1) + f3_2 * i64(f9_38) + f4_2 * i64(f8_19) + f5_2 * i64(f7_38) + f6 * i64(f6_19); + t[3] = f0_2 * i64(f3) + f1_2 * i64(f2) + f4 * i64(f9_38) + f5_2 * i64(f8_19) + f6 * i64(f7_38); + t[4] = f0_2 * i64(f4) + f1_2 * i64(f3_2) + f2 * i64(f2) + f5_2 * i64(f9_38) + f6_2 * i64(f8_19) + f7 * i64(f7_38); + t[5] = f0_2 * i64(f5) + f1_2 * i64(f4) + f2_2 * i64(f3) + f6 * i64(f9_38) + f7_2 * i64(f8_19); + t[6] = f0_2 * i64(f6) + f1_2 * i64(f5_2) + f2_2 * i64(f4) + f3_2 * i64(f3) + f7_2 * i64(f9_38) + f8 * i64(f8_19); + t[7] = f0_2 * i64(f7) + f1_2 * i64(f6) + f2_2 * i64(f5) + f3_2 * i64(f4) + f8 * i64(f9_38); + t[8] = f0_2 * i64(f8) + f1_2 * i64(f7_2) + f2_2 * i64(f6) + f3_2 * i64(f5_2) + f4 * i64(f4) + f9 * i64(f9_38); + t[9] = f0_2 * i64(f9) + f1_2 * i64(f8) + f2_2 * i64(f7) + f3_2 * i64(f6) + f4 * i64(f5_2); + + feCarry2(h, t[0..]); +} + +fn fe_sq2(h: *Fe, f: *const Fe) void { + fe_sq(h, f); + fe_mul_small(h, h, 2); +} + +// This could be simplified, but it would be slower +fn fe_invert(out: *Fe, z: *const Fe) void { + var i: usize = undefined; + + var t: [4]Fe = undefined; + var t0 = &t[0]; + var t1 = &t[1]; + var t2 = &t[2]; + var t3 = &t[3]; + + fe_sq(t0, z); + fe_sq(t1, t0); + fe_sq(t1, t1); + fe_mul(t1, z, t1); + fe_mul(t0, t0, t1); + + fe_sq(t2, t0); + fe_mul(t1, t1, t2); + + fe_sq(t2, t1); + i = 1; + while (i < 5) : (i += 1) fe_sq(t2, t2); + fe_mul(t1, t2, t1); + + fe_sq(t2, t1); + i = 1; + while (i < 10) : (i += 1) fe_sq(t2, t2); + fe_mul(t2, t2, t1); + + fe_sq(t3, t2); + i = 1; + while (i < 20) : (i += 1) fe_sq(t3, t3); + fe_mul(t2, t3, t2); + + fe_sq(t2, t2); + i = 1; + while (i < 10) : (i += 1) fe_sq(t2, t2); + fe_mul(t1, t2, t1); + + fe_sq(t2, t1); + i = 1; + while (i < 50) : (i += 1) fe_sq(t2, t2); + fe_mul(t2, t2, t1); + + fe_sq(t3, t2); + i = 1; + while (i < 100) : (i += 1) fe_sq(t3, t3); + fe_mul(t2, t3, t2); + + fe_sq(t2, t2); + i = 1; + while (i < 50) : (i += 1) fe_sq(t2, t2); + fe_mul(t1, t2, t1); + + fe_sq(t1, t1); + i = 1; + while (i < 5) : (i += 1) fe_sq(t1, t1); + fe_mul(out, t1, t0); + + t0.secure_zero(); + t1.secure_zero(); + t2.secure_zero(); + t3.secure_zero(); +} + +// This could be simplified, but it would be slower +fn fe_pow22523(out: *Fe, z: *const Fe) void { + var i: usize = undefined; + + var t: [3]Fe = undefined; + var t0 = &t[0]; + var t1 = &t[1]; + var t2 = &t[2]; + + fe_sq(t0, z); + fe_sq(t1, t0); + fe_sq(t1, t1); + fe_mul(t1, z, t1); + fe_mul(t0, t0, t1); + + fe_sq(t0, t0); + fe_mul(t0, t1, t0); + + fe_sq(t1, t0); + i = 1; + while (i < 5) : (i += 1) fe_sq(t1, t1); + fe_mul(t0, t1, t0); + + fe_sq(t1, t0); + i = 1; + while (i < 10) : (i += 1) fe_sq(t1, t1); + fe_mul(t1, t1, t0); + + fe_sq(t2, t1); + i = 1; + while (i < 20) : (i += 1) fe_sq(t2, t2); + fe_mul(t1, t2, t1); + + fe_sq(t1, t1); + i = 1; + while (i < 10) : (i += 1) fe_sq(t1, t1); + fe_mul(t0, t1, t0); + + fe_sq(t1, t0); + i = 1; + while (i < 50) : (i += 1) fe_sq(t1, t1); + fe_mul(t1, t1, t0); + + fe_sq(t2, t1); + i = 1; + while (i < 100) : (i += 1) fe_sq(t2, t2); + fe_mul(t1, t2, t1); + + fe_sq(t1, t1); + i = 1; + while (i < 50) : (i += 1) fe_sq(t1, t1); + fe_mul(t0, t1, t0); + + fe_sq(t0, t0); + i = 1; + while (i < 2) : (i += 1) fe_sq(t0, t0); + fe_mul(out, t0, z); + + t0.secure_zero(); + t1.secure_zero(); + t2.secure_zero(); +} + +inline fn tobytesround(c: []i64, t: []i64, comptime i: comptime_int, comptime shift: comptime_int) void { + c[i] = t[i] >> shift; + if (i + 1 < 10) { + t[i + 1] += c[i]; + } + t[i] -= c[i] * (i32(1) << shift); +} + +fn fe_tobytes(s: []u8, h: *const Fe) void { + std.debug.assert(s.len >= 32); + + var t: [10]i64 = undefined; + for (h.b[0..]) |_, i| { + t[i] = h.b[i]; + } + + var q = (19 * t[9] + ((i32(1) << 24))) >> 25; + { + var i: usize = 0; + while (i < 5) : (i += 1) { + q += t[2 * i]; + q >>= 26; + q += t[2 * i + 1]; + q >>= 25; + } + } + t[0] += 19 * q; + + var c: [10]i64 = undefined; + + var st = t[0..]; + var sc = c[0..]; + + tobytesround(sc, st, 0, 26); + tobytesround(sc, st, 1, 25); + tobytesround(sc, st, 2, 26); + tobytesround(sc, st, 3, 25); + tobytesround(sc, st, 4, 26); + tobytesround(sc, st, 5, 25); + tobytesround(sc, st, 6, 26); + tobytesround(sc, st, 7, 25); + tobytesround(sc, st, 8, 26); + tobytesround(sc, st, 9, 25); + + var ut: [10]u32 = undefined; + for (ut[0..]) |_, i| { + ut[i] = @bitCast(u32, @intCast(i32, t[i])); + } + + writeInt(s[0..], (ut[0] >> 0) | (ut[1] << 26), Endian.Little); + writeInt(s[4..], (ut[1] >> 6) | (ut[2] << 19), Endian.Little); + writeInt(s[8..], (ut[2] >> 13) | (ut[3] << 13), Endian.Little); + writeInt(s[12..], (ut[3] >> 19) | (ut[4] << 6), Endian.Little); + writeInt(s[16..], (ut[5] >> 0) | (ut[6] << 25), Endian.Little); + writeInt(s[20..], (ut[6] >> 7) | (ut[7] << 19), Endian.Little); + writeInt(s[24..], (ut[7] >> 13) | (ut[8] << 12), Endian.Little); + writeInt(s[28..], (ut[8] >> 20) | (ut[9] << 6), Endian.Little); + + std.mem.secureZero(i64, t[0..]); +} + +// Parity check. Returns 0 if even, 1 if odd +fn fe_isnegative(f: *const Fe) bool { + var s: [32]u8 = undefined; + fe_tobytes(s[0..], f); + const isneg = s[0] & 1; + s.secure_zero(); + return isneg; +} + +fn fe_isnonzero(f: *const Fe) bool { + var s: [32]u8 = undefined; + fe_tobytes(s[0..], f); + const isnonzero = zerocmp(u8, s[0..]); + s.secure_zero(); + return isneg; +} + +/////////////// +/// X-25519 /// Taken from Supercop's ref10 implementation. +/////////////// +fn trim_scalar(s: []u8) void { + s[0] &= 248; + s[31] &= 127; + s[31] |= 64; +} + +fn scalar_bit(s: []const u8, i: usize) i32 { + return (s[i >> 3] >> @intCast(u3, i & 7)) & 1; +} + +pub fn crypto_x25519(raw_shared_secret: []u8, your_secret_key: [32]u8, their_public_key: [32]u8) bool { + std.debug.assert(raw_shared_secret.len >= 32); + + var storage: [7]Fe = undefined; + + var x1 = &storage[0]; + var x2 = &storage[1]; + var z2 = &storage[2]; + var x3 = &storage[3]; + var z3 = &storage[4]; + var t0 = &storage[5]; + var t1 = &storage[6]; + + // computes the scalar product + fe_frombytes(x1, their_public_key); + + // restrict the possible scalar values + var e: [32]u8 = undefined; + for (e[0..]) |_, i| { + e[i] = your_secret_key[i]; + } + trim_scalar(e[0..]); + + // computes the actual scalar product (the result is in x2 and z2) + + // Montgomery ladder + // In projective coordinates, to avoid divisons: x = X / Z + // We don't care about the y coordinate, it's only 1 bit of information + fe_1(x2); + fe_0(z2); // "zero" point + fe_copy(x3, x1); + fe_1(z3); + + var swap: i32 = 0; + var pos: isize = 254; + while (pos >= 0) : (pos -= 1) { + // constant time conditional swap before ladder step + const b = scalar_bit(e, @intCast(usize, pos)); + swap ^= b; // xor trick avoids swapping at the end of the loop + fe_cswap(x2, x3, swap); + fe_cswap(z2, z3, swap); + swap = b; // anticipates one last swap after the loop + + // Montgomery ladder step: replaces (P2, P3) by (P2*2, P2+P3) + // with differential addition + fe_sub(t0, x3, z3); + fe_sub(t1, x2, z2); + fe_add(x2, x2, z2); + fe_add(z2, x3, z3); + fe_mul(z3, t0, x2); + fe_mul(z2, z2, t1); + fe_sq(t0, t1); + fe_sq(t1, x2); + fe_add(x3, z3, z2); + fe_sub(z2, z3, z2); + fe_mul(x2, t1, t0); + fe_sub(t1, t1, t0); + fe_sq(z2, z2); + fe_mul121666(z3, t1); + fe_sq(x3, x3); + fe_add(t0, t0, z3); + fe_mul(z3, x1, z2); + fe_mul(z2, t1, t0); + } + + // last swap is necessary to compensate for the xor trick + // Note: after this swap, P3 == P2 + P1. + fe_cswap(x2, x3, swap); + fe_cswap(z2, z3, swap); + + // normalises the coordinates: x == X / Z + fe_invert(z2, z2); + fe_mul(x2, x2, z2); + fe_tobytes(raw_shared_secret, x2); + + x1.secure_zero(); + x2.secure_zero(); + x3.secure_zero(); + t0.secure_zero(); + t1.secure_zero(); + z2.secure_zero(); + z3.secure_zero(); + std.mem.secureZero(u8, e[0..]); + + // Returns false if the output is all zero + // (happens with some malicious public keys) + return !zerocmp(u8, raw_shared_secret); +} + +pub fn crypto_x25519_public_key(public_key: []u8, secret_key: [32]u8) void { + var base_point = []u8{9} ++ []u8{0} ** 31; + crypto_x25519(public_key, secret_key, base_point); +} + +test "x25519 rfc7748 vector1" { + const secret_key = "\xa5\x46\xe3\x6b\xf0\x52\x7c\x9d\x3b\x16\x15\x4b\x82\x46\x5e\xdd\x62\x14\x4c\x0a\xc1\xfc\x5a\x18\x50\x6a\x22\x44\xba\x44\x9a\xc4"; + const public_key = "\xe6\xdb\x68\x67\x58\x30\x30\xdb\x35\x94\xc1\xa4\x24\xb1\x5f\x7c\x72\x66\x24\xec\x26\xb3\x35\x3b\x10\xa9\x03\xa6\xd0\xab\x1c\x4c"; + + const expected_output = "\xc3\xda\x55\x37\x9d\xe9\xc6\x90\x8e\x94\xea\x4d\xf2\x8d\x08\x4f\x32\xec\xcf\x03\x49\x1c\x71\xf7\x54\xb4\x07\x55\x77\xa2\x85\x52"; + + var output: [32]u8 = undefined; + + std.debug.assert(crypto_x25519(output[0..], secret_key, public_key)); + std.debug.assert(std.mem.eql(u8, output, expected_output)); +} + +test "x25519 rfc7748 vector2" { + const secret_key = "\x4b\x66\xe9\xd4\xd1\xb4\x67\x3c\x5a\xd2\x26\x91\x95\x7d\x6a\xf5\xc1\x1b\x64\x21\xe0\xea\x01\xd4\x2c\xa4\x16\x9e\x79\x18\xba\x0d"; + const public_key = "\xe5\x21\x0f\x12\x78\x68\x11\xd3\xf4\xb7\x95\x9d\x05\x38\xae\x2c\x31\xdb\xe7\x10\x6f\xc0\x3c\x3e\xfc\x4c\xd5\x49\xc7\x15\xa4\x93"; + + const expected_output = "\x95\xcb\xde\x94\x76\xe8\x90\x7d\x7a\xad\xe4\x5c\xb4\xb8\x73\xf8\x8b\x59\x5a\x68\x79\x9f\xa1\x52\xe6\xf8\xf7\x64\x7a\xac\x79\x57"; + + var output: [32]u8 = undefined; + + std.debug.assert(crypto_x25519(output[0..], secret_key, public_key)); + std.debug.assert(std.mem.eql(u8, output, expected_output)); +} + +test "x25519 rfc7748 one iteration" { + const initial_value = "\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"; + const expected_output = "\x42\x2c\x8e\x7a\x62\x27\xd7\xbc\xa1\x35\x0b\x3e\x2b\xb7\x27\x9f\x78\x97\xb8\x7b\xb6\x85\x4b\x78\x3c\x60\xe8\x03\x11\xae\x30\x79"; + + var k: [32]u8 = initial_value; + var u: [32]u8 = initial_value; + + var i: usize = 0; + while (i < 1) : (i += 1) { + var output: [32]u8 = undefined; + std.debug.assert(crypto_x25519(output[0..], k, u)); + + std.mem.copy(u8, u[0..], k[0..]); + std.mem.copy(u8, k[0..], output[0..]); + } + + std.debug.assert(std.mem.eql(u8, k[0..], expected_output)); +} + +test "x25519 rfc7748 1,000 iterations" { + // These iteration tests are slow so we always skip them. Results have been verified. + if (true) { + return error.SkipZigTest; + } + + const initial_value = "\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"; + const expected_output = "\x68\x4c\xf5\x9b\xa8\x33\x09\x55\x28\x00\xef\x56\x6f\x2f\x4d\x3c\x1c\x38\x87\xc4\x93\x60\xe3\x87\x5f\x2e\xb9\x4d\x99\x53\x2c\x51"; + + var k: [32]u8 = initial_value; + var u: [32]u8 = initial_value; + + var i: usize = 0; + while (i < 1000) : (i += 1) { + var output: [32]u8 = undefined; + std.debug.assert(crypto_x25519(output[0..], k, u)); + + std.mem.copy(u8, u[0..], k[0..]); + std.mem.copy(u8, k[0..], output[0..]); + } + + std.debug.assert(std.mem.eql(u8, k[0..], expected_output)); +} + +test "x25519 rfc7748 1,000,000 iterations" { + if (true) { + return error.SkipZigTest; + } + + const initial_value = "\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"; + const expected_output = "\x7c\x39\x11\xe0\xab\x25\x86\xfd\x86\x44\x97\x29\x7e\x57\x5e\x6f\x3b\xc6\x01\xc0\x88\x3c\x30\xdf\x5f\x4d\xd2\xd2\x4f\x66\x54\x24"; + + var k: [32]u8 = initial_value; + var u: [32]u8 = initial_value; + + var i: usize = 0; + while (i < 1000000) : (i += 1) { + var output: [32]u8 = undefined; + std.debug.assert(crypto_x25519(output[0..], k, u)); + + std.mem.copy(u8, u[0..], k[0..]); + std.mem.copy(u8, k[0..], output[0..]); + } + + std.debug.assert(std.mem.eql(u8, k[0..], expected_output)); +} From 96117e20cc7c076cbf9d8243602c6e4205fdd0d4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Aug 2018 03:44:34 -0400 Subject: [PATCH 25/56] reading the module information substream --- std/pdb.zig | 163 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 93 insertions(+), 70 deletions(-) diff --git a/std/pdb.zig b/std/pdb.zig index a83011df6a..6f086b59e0 100644 --- a/std/pdb.zig +++ b/std/pdb.zig @@ -76,7 +76,7 @@ pub const Pdb = struct { msf: Msf, pub fn openFile(self: *Pdb, allocator: *mem.Allocator, file_name: []u8) !void { - self.in_file = try os.File.openRead(file_name[0..]); + self.in_file = try os.File.openRead(file_name); self.allocator = allocator; try self.msf.openFile(allocator, self.in_file); @@ -85,7 +85,7 @@ pub const Pdb = struct { pub fn getStream(self: *Pdb, stream: StreamType) ?*MsfStream { const id = @enumToInt(stream); if (id < self.msf.streams.len) - return &self.msf.streams.items[id]; + return &self.msf.streams[id]; return null; } @@ -101,17 +101,31 @@ pub const Pdb = struct { // Module Info Substream var mod_info_offset: usize = 0; while (mod_info_offset < header.ModInfoSize) { + const march_forward_bytes = dbi.getFilePos() % 4; + if (march_forward_bytes != 0) { + try dbi.seekForward(march_forward_bytes); + mod_info_offset += march_forward_bytes; + } var mod_info: ModInfo = undefined; try dbi.stream.readStruct(ModInfo, &mod_info); std.debug.warn("{}\n", mod_info); mod_info_offset += @sizeOf(ModInfo); const module_name = try dbi.readNullTermString(self.allocator); - std.debug.warn("module_name {}\n", module_name); + std.debug.warn("module_name '{}'\n", module_name); mod_info_offset += module_name.len + 1; + //if (mem.eql(u8, module_name, "piler_rt.obj")) { + // std.debug.warn("detected bad thing\n"); + // try dbi.seekTo(dbi.pos - + // "c:\\msys64\\home\\andy\\zig\\build-llvm6-msvc-release\\.\\zig-cache\\compiler_rt.obj\x00".len - + // @sizeOf(ModInfo)); + // mod_info_offset -= module_name.len + 1; + // continue; + //} + const obj_file_name = try dbi.readNullTermString(self.allocator); - std.debug.warn("obj_file_name {}\n", obj_file_name); + std.debug.warn("obj_file_name '{}'\n", obj_file_name); mod_info_offset += obj_file_name.len + 1; } std.debug.warn("end modules\n"); @@ -123,66 +137,55 @@ pub const Pdb = struct { // see https://llvm.org/docs/PDB/MsfFile.html const Msf = struct { - superblock: SuperBlock, directory: MsfStream, - streams: ArrayList(MsfStream), + streams: []MsfStream, fn openFile(self: *Msf, allocator: *mem.Allocator, file: os.File) !void { var file_stream = io.FileInStream.init(file); const in = &file_stream.stream; - var magic: SuperBlock.FileMagicBuffer = undefined; - try in.readNoEof(magic[0..]); - warn("magic: '{}'\n", magic); - - if (!mem.eql(u8, magic, SuperBlock.FileMagic)) + var superblock: SuperBlock = undefined; + try in.readStruct(SuperBlock, &superblock); + + if (!mem.eql(u8, superblock.FileMagic, SuperBlock.file_magic)) return error.InvalidDebugInfo; - self.superblock = SuperBlock { - .block_size = try in.readIntLe(u32), - .free_block_map_block = try in.readIntLe(u32), - .num_blocks = try in.readIntLe(u32), - .num_directory_bytes = try in.readIntLe(u32), - .unknown = try in.readIntLe(u32), - .block_map_addr = try in.readIntLe(u32), - }; - - switch (self.superblock.block_size) { - 512, 1024, 2048, 4096 => {}, // llvm only uses 4096 + switch (superblock.BlockSize) { + // llvm only supports 4096 but we can handle any of these values + 512, 1024, 2048, 4096 => {}, else => return error.InvalidDebugInfo } - if (self.superblock.fileSize() != try file.getEndPos()) - return error.InvalidDebugInfo; // Should always stand. + if (superblock.NumBlocks * superblock.BlockSize != try file.getEndPos()) + return error.InvalidDebugInfo; self.directory = try MsfStream.init( - self.superblock.block_size, - self.superblock.blocksOccupiedByDirectoryStream(), - self.superblock.blockMapAddr(), + superblock.BlockSize, + blockCountFromSize(superblock.NumDirectoryBytes, superblock.BlockSize), + superblock.BlockSize * superblock.BlockMapAddr, file, - allocator + allocator, ); const stream_count = try self.directory.stream.readIntLe(u32); warn("stream count {}\n", stream_count); - var stream_sizes = ArrayList(u32).init(allocator); - try stream_sizes.resize(stream_count); - for (stream_sizes.toSlice()) |*s| { + const stream_sizes = try allocator.alloc(u32, stream_count); + for (stream_sizes) |*s| { const size = try self.directory.stream.readIntLe(u32); - s.* = blockCountFromSize(size, self.superblock.block_size); + s.* = blockCountFromSize(size, superblock.BlockSize); warn("stream {}B {} blocks\n", size, s.*); } - self.streams = ArrayList(MsfStream).init(allocator); - try self.streams.resize(stream_count); - for (self.streams.toSlice()) |*ss, i| { - ss.* = try MsfStream.init( - self.superblock.block_size, - stream_sizes.items[i], - try file.getPos(), // We're reading the jagged array of block indices when creating streams so the file is always at the right position. + self.streams = try allocator.alloc(MsfStream, stream_count); + for (self.streams) |*stream, i| { + stream.* = try MsfStream.init( + superblock.BlockSize, + stream_sizes[i], + // MsfStream.init expects the file to be at the part where it reads [N]u32 + try file.getPos(), file, - allocator + allocator, ); } } @@ -192,34 +195,53 @@ fn blockCountFromSize(size: u32, block_size: u32) u32 { return (size + block_size - 1) / block_size; } -const SuperBlock = struct { - const FileMagic = "Microsoft C/C++ MSF 7.00\r\n" ++ []u8 { 0x1A, 'D', 'S', 0, 0, 0}; - const FileMagicBuffer = @typeOf(FileMagic); +// https://llvm.org/docs/PDB/MsfFile.html#the-superblock +const SuperBlock = packed struct { + /// The LLVM docs list a space between C / C++ but empirically this is not the case. + const file_magic = "Microsoft C/C++ MSF 7.00\r\n\x1a\x44\x53\x00\x00\x00"; - block_size: u32, - free_block_map_block: u32, - num_blocks: u32, - num_directory_bytes: u32, - unknown: u32, - block_map_addr: u32, + FileMagic: [file_magic.len]u8, - fn fileSize(self: *const SuperBlock) usize { - return self.num_blocks * self.block_size; - } + /// The block size of the internal file system. Valid values are 512, 1024, + /// 2048, and 4096 bytes. Certain aspects of the MSF file layout vary depending + /// on the block sizes. For the purposes of LLVM, we handle only block sizes of + /// 4KiB, and all further discussion assumes a block size of 4KiB. + BlockSize: u32, - fn blockMapAddr(self: *const SuperBlock) usize { - return self.block_size * self.block_map_addr; - } + /// The index of a block within the file, at which begins a bitfield representing + /// the set of all blocks within the file which are “free” (i.e. the data within + /// that block is not used). See The Free Block Map for more information. Important: + /// FreeBlockMapBlock can only be 1 or 2! + FreeBlockMapBlock: u32, + + /// The total number of blocks in the file. NumBlocks * BlockSize should equal the + /// size of the file on disk. + NumBlocks: u32, + + /// The size of the stream directory, in bytes. The stream directory contains + /// information about each stream’s size and the set of blocks that it occupies. + /// It will be described in more detail later. + NumDirectoryBytes: u32, + + Unknown: u32, + + /// The index of a block within the MSF file. At this block is an array of + /// ulittle32_t’s listing the blocks that the stream directory resides on. + /// For large MSF files, the stream directory (which describes the block + /// layout of each stream) may not fit entirely on a single block. As a + /// result, this extra layer of indirection is introduced, whereby this + /// block contains the list of blocks that the stream directory occupies, + /// and the stream directory itself can be stitched together accordingly. + /// The number of ulittle32_t’s in this array is given by + /// ceil(NumDirectoryBytes / BlockSize). + BlockMapAddr: u32, - fn blocksOccupiedByDirectoryStream(self: *const SuperBlock) u32 { - return blockCountFromSize(self.num_directory_bytes, self.block_size); - } }; const MsfStream = struct { in_file: os.File, pos: usize, - blocks: ArrayList(u32), + blocks: []u32, block_size: u32, /// Implementation of InStream trait for Pdb.MsfStream @@ -232,15 +254,13 @@ const MsfStream = struct { var stream = MsfStream { .in_file = file, .pos = 0, - .blocks = ArrayList(u32).init(allocator), + .blocks = try allocator.alloc(u32, block_count), .block_size = block_size, .stream = Stream { .readFn = readFn, }, }; - try stream.blocks.resize(block_count); - var file_stream = io.FileInStream.init(file); const in = &file_stream.stream; try file.seekTo(pos); @@ -248,8 +268,8 @@ const MsfStream = struct { warn("stream with blocks"); var i: u32 = 0; while (i < block_count) : (i += 1) { - stream.blocks.items[i] = try in.readIntLe(u32); - warn(" {}", stream.blocks.items[i]); + stream.blocks[i] = try in.readIntLe(u32); + warn(" {}", stream.blocks[i]); } warn("\n"); @@ -270,9 +290,13 @@ const MsfStream = struct { fn read(self: *MsfStream, buffer: []u8) !usize { var block_id = self.pos / self.block_size; - var block = self.blocks.items[block_id]; + var block = self.blocks[block_id]; var offset = self.pos % self.block_size; + //std.debug.warn("seek {} read {}B: block_id={} block={} offset={}\n", + // block * self.block_size + offset, + // buffer.len, block_id, block, offset); + try self.in_file.seekTo(block * self.block_size + offset); var file_stream = io.FileInStream.init(self.in_file); const in = &file_stream.stream; @@ -285,11 +309,10 @@ const MsfStream = struct { size += 1; // If we're at the end of a block, go to the next one. - if (offset == self.block_size) - { + if (offset == self.block_size) { offset = 0; block_id += 1; - block = self.blocks.items[block_id]; + block = self.blocks[block_id]; try self.in_file.seekTo(block * self.block_size); } } @@ -314,9 +337,9 @@ const MsfStream = struct { return self.blocks.len * self.block_size; } - fn getFilePos(self: *const MsfStream) usize { + fn getFilePos(self: MsfStream) usize { const block_id = self.pos / self.block_size; - const block = self.blocks.items[block_id]; + const block = self.blocks[block_id]; const offset = self.pos % self.block_size; return block * self.block_size + offset; From 44f908d2e63d761e183e266248e777fe74184f44 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Aug 2018 15:33:50 -0400 Subject: [PATCH 26/56] figuring out which module an address belongs in --- std/debug/index.zig | 35 ++-- std/pdb.zig | 427 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 420 insertions(+), 42 deletions(-) diff --git a/std/debug/index.zig b/std/debug/index.zig index 36c5a6bdc9..5bdfb52186 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -408,26 +408,36 @@ pub fn openSelfDebugInfo(allocator: *mem.Allocator) !DebugInfo { } fn openSelfDebugInfoWindows(allocator: *mem.Allocator) !DebugInfo { - var coff_file: coff.Coff = undefined; - coff_file.in_file = try os.openSelfExe(); - coff_file.allocator = allocator; - defer coff_file.in_file.close(); + const self_file = try os.openSelfExe(); + defer self_file.close(); - try coff_file.loadHeader(); + const coff_obj = try allocator.createOne(coff.Coff); + coff_obj.* = coff.Coff{ + .in_file = self_file, + .allocator = allocator, + .coff_header = undefined, + .pe_header = undefined, + .sections = undefined, + .guid = undefined, + .age = undefined, + }; + + var di = DebugInfo{ + .coff = coff_obj, + .pdb = undefined, + }; + + try di.coff.loadHeader(); var path_buf: [windows.MAX_PATH]u8 = undefined; - const len = try coff_file.getPdbPath(path_buf[0..]); + const len = try di.coff.getPdbPath(path_buf[0..]); const raw_path = path_buf[0..len]; std.debug.warn("pdb raw path {}\n", raw_path); const path = try os.path.resolve(allocator, raw_path); std.debug.warn("pdb resolved path {}\n", path); - var di = DebugInfo{ - .pdb = undefined, - }; - - try di.pdb.openFile(allocator, path); + try di.pdb.openFile(di.coff, path); var pdb_stream = di.pdb.getStream(pdb.StreamType.Pdb) orelse return error.InvalidDebugInfo; std.debug.warn("pdb real filepos {}\n", pdb_stream.getFilePos()); @@ -436,7 +446,7 @@ fn openSelfDebugInfoWindows(allocator: *mem.Allocator) !DebugInfo { const age = try pdb_stream.stream.readIntLe(u32); var guid: [16]u8 = undefined; try pdb_stream.stream.readNoEof(guid[0..]); - if (!mem.eql(u8, coff_file.guid, guid) or coff_file.age != age) + if (!mem.eql(u8, di.coff.guid, guid) or di.coff.age != age) return error.InvalidDebugInfo; std.debug.warn("v {} s {} a {}\n", version, signature, age); // We validated the executable and pdb match. @@ -644,6 +654,7 @@ pub const DebugInfo = switch (builtin.os) { }, builtin.Os.windows => struct { pdb: pdb.Pdb, + coff: *coff.Coff, }, builtin.Os.linux => struct { self_exe_file: os.File, diff --git a/std/pdb.zig b/std/pdb.zig index 6f086b59e0..14c584a9f7 100644 --- a/std/pdb.zig +++ b/std/pdb.zig @@ -5,6 +5,7 @@ const math = std.math; const mem = std.mem; const os = std.os; const warn = std.debug.warn; +const Coff = std.coff.Coff; const ArrayList = std.ArrayList; @@ -20,8 +21,8 @@ const DbiStreamHeader = packed struct { SymRecordStream: u16, PdbDllRbld: u16, ModInfoSize: u32, - SectionContributionSize: i32, - SectionMapSize: i32, + SectionContributionSize: u32, + SectionMapSize: u32, SourceInfoSize: i32, TypeServerSize: i32, MFCTypeServerIndex: u32, @@ -35,8 +36,8 @@ const DbiStreamHeader = packed struct { const SectionContribEntry = packed struct { Section: u16, Padding1: [2]u8, - Offset: i32, - Size: i32, + Offset: u32, + Size: u32, Characteristics: u32, ModuleIndex: u16, Padding2: [2]u8, @@ -62,6 +63,22 @@ const ModInfo = packed struct { //ObjFileName: char[], }; +const SectionMapHeader = packed struct { + Count: u16, /// Number of segment descriptors + LogCount: u16, /// Number of logical segment descriptors +}; + +const SectionMapEntry = packed struct { + Flags: u16 , /// See the SectionMapEntryFlags enum below. + Ovl: u16 , /// Logical overlay number + Group: u16 , /// Group index into descriptor array. + Frame: u16 , + SectionName: u16 , /// Byte index of segment / group name in string table, or 0xFFFF. + ClassName: u16 , /// Byte index of class in string table, or 0xFFFF. + Offset: u32 , /// Byte offset of the logical segment within physical segment. If group is set in flags, this is the offset of the group. + SectionLength: u32 , /// Byte count of the segment or group. +}; + pub const StreamType = enum(u16) { Pdb = 1, Tpi = 2, @@ -69,24 +86,306 @@ pub const StreamType = enum(u16) { Ipi = 4, }; +const Module = struct { + mod_info: ModInfo, + module_name: []u8, + obj_file_name: []u8, +}; + +/// Distinguishes individual records in the Symbols subsection of a .debug$S +/// section. Equivalent to SYM_ENUM_e in cvinfo.h. +pub const SymbolRecordKind = enum(u16) { + InlineesSym = 4456, + ScopeEndSym = 6, + InlineSiteEnd = 4430, + ProcEnd = 4431, + Thunk32Sym = 4354, + TrampolineSym = 4396, + SectionSym = 4406, + CoffGroupSym = 4407, + ExportSym = 4408, + ProcSym = 4367, + GlobalProcSym = 4368, + ProcIdSym = 4422, + GlobalProcIdSym = 4423, + DPCProcSym = 4437, + DPCProcIdSym = 4438, + RegisterSym = 4358, + PublicSym32 = 4366, + ProcRefSym = 4389, + LocalProcRef = 4391, + EnvBlockSym = 4413, + InlineSiteSym = 4429, + LocalSym = 4414, + DefRangeSym = 4415, + DefRangeSubfieldSym = 4416, + DefRangeRegisterSym = 4417, + DefRangeFramePointerRelSym = 4418, + DefRangeSubfieldRegisterSym = 4419, + DefRangeFramePointerRelFullScopeSym = 4420, + DefRangeRegisterRelSym = 4421, + BlockSym = 4355, + LabelSym = 4357, + ObjNameSym = 4353, + Compile2Sym = 4374, + Compile3Sym = 4412, + FrameProcSym = 4114, + CallSiteInfoSym = 4409, + FileStaticSym = 4435, + HeapAllocationSiteSym = 4446, + FrameCookieSym = 4410, + CallerSym = 4442, + CalleeSym = 4443, + UDTSym = 4360, + CobolUDT = 4361, + BuildInfoSym = 4428, + BPRelativeSym = 4363, + RegRelativeSym = 4369, + ConstantSym = 4359, + ManagedConstant = 4397, + DataSym = 4364, + GlobalData = 4365, + ManagedLocalData = 4380, + ManagedGlobalData = 4381, + ThreadLocalDataSym = 4370, + GlobalTLS = 4371, +}; + +/// Duplicate copy of SymbolRecordKind, but using the official CV names. Useful +/// for reference purposes and when dealing with unknown record types. +pub const SymbolKind = enum(u16) { + S_COMPILE = 1, + S_REGISTER_16t = 2, + S_CONSTANT_16t = 3, + S_UDT_16t = 4, + S_SSEARCH = 5, + S_SKIP = 7, + S_CVRESERVE = 8, + S_OBJNAME_ST = 9, + S_ENDARG = 10, + S_COBOLUDT_16t = 11, + S_MANYREG_16t = 12, + S_RETURN = 13, + S_ENTRYTHIS = 14, + S_BPREL16 = 256, + S_LDATA16 = 257, + S_GDATA16 = 258, + S_PUB16 = 259, + S_LPROC16 = 260, + S_GPROC16 = 261, + S_THUNK16 = 262, + S_BLOCK16 = 263, + S_WITH16 = 264, + S_LABEL16 = 265, + S_CEXMODEL16 = 266, + S_VFTABLE16 = 267, + S_REGREL16 = 268, + S_BPREL32_16t = 512, + S_LDATA32_16t = 513, + S_GDATA32_16t = 514, + S_PUB32_16t = 515, + S_LPROC32_16t = 516, + S_GPROC32_16t = 517, + S_THUNK32_ST = 518, + S_BLOCK32_ST = 519, + S_WITH32_ST = 520, + S_LABEL32_ST = 521, + S_CEXMODEL32 = 522, + S_VFTABLE32_16t = 523, + S_REGREL32_16t = 524, + S_LTHREAD32_16t = 525, + S_GTHREAD32_16t = 526, + S_SLINK32 = 527, + S_LPROCMIPS_16t = 768, + S_GPROCMIPS_16t = 769, + S_PROCREF_ST = 1024, + S_DATAREF_ST = 1025, + S_ALIGN = 1026, + S_LPROCREF_ST = 1027, + S_OEM = 1028, + S_TI16_MAX = 4096, + S_REGISTER_ST = 4097, + S_CONSTANT_ST = 4098, + S_UDT_ST = 4099, + S_COBOLUDT_ST = 4100, + S_MANYREG_ST = 4101, + S_BPREL32_ST = 4102, + S_LDATA32_ST = 4103, + S_GDATA32_ST = 4104, + S_PUB32_ST = 4105, + S_LPROC32_ST = 4106, + S_GPROC32_ST = 4107, + S_VFTABLE32 = 4108, + S_REGREL32_ST = 4109, + S_LTHREAD32_ST = 4110, + S_GTHREAD32_ST = 4111, + S_LPROCMIPS_ST = 4112, + S_GPROCMIPS_ST = 4113, + S_COMPILE2_ST = 4115, + S_MANYREG2_ST = 4116, + S_LPROCIA64_ST = 4117, + S_GPROCIA64_ST = 4118, + S_LOCALSLOT_ST = 4119, + S_PARAMSLOT_ST = 4120, + S_ANNOTATION = 4121, + S_GMANPROC_ST = 4122, + S_LMANPROC_ST = 4123, + S_RESERVED1 = 4124, + S_RESERVED2 = 4125, + S_RESERVED3 = 4126, + S_RESERVED4 = 4127, + S_LMANDATA_ST = 4128, + S_GMANDATA_ST = 4129, + S_MANFRAMEREL_ST = 4130, + S_MANREGISTER_ST = 4131, + S_MANSLOT_ST = 4132, + S_MANMANYREG_ST = 4133, + S_MANREGREL_ST = 4134, + S_MANMANYREG2_ST = 4135, + S_MANTYPREF = 4136, + S_UNAMESPACE_ST = 4137, + S_ST_MAX = 4352, + S_WITH32 = 4356, + S_MANYREG = 4362, + S_LPROCMIPS = 4372, + S_GPROCMIPS = 4373, + S_MANYREG2 = 4375, + S_LPROCIA64 = 4376, + S_GPROCIA64 = 4377, + S_LOCALSLOT = 4378, + S_PARAMSLOT = 4379, + S_MANFRAMEREL = 4382, + S_MANREGISTER = 4383, + S_MANSLOT = 4384, + S_MANMANYREG = 4385, + S_MANREGREL = 4386, + S_MANMANYREG2 = 4387, + S_UNAMESPACE = 4388, + S_DATAREF = 4390, + S_ANNOTATIONREF = 4392, + S_TOKENREF = 4393, + S_GMANPROC = 4394, + S_LMANPROC = 4395, + S_ATTR_FRAMEREL = 4398, + S_ATTR_REGISTER = 4399, + S_ATTR_REGREL = 4400, + S_ATTR_MANYREG = 4401, + S_SEPCODE = 4402, + S_LOCAL_2005 = 4403, + S_DEFRANGE_2005 = 4404, + S_DEFRANGE2_2005 = 4405, + S_DISCARDED = 4411, + S_LPROCMIPS_ID = 4424, + S_GPROCMIPS_ID = 4425, + S_LPROCIA64_ID = 4426, + S_GPROCIA64_ID = 4427, + S_DEFRANGE_HLSL = 4432, + S_GDATA_HLSL = 4433, + S_LDATA_HLSL = 4434, + S_LOCAL_DPC_GROUPSHARED = 4436, + S_DEFRANGE_DPC_PTR_TAG = 4439, + S_DPC_SYM_TAG_MAP = 4440, + S_ARMSWITCHTABLE = 4441, + S_POGODATA = 4444, + S_INLINESITE2 = 4445, + S_MOD_TYPEREF = 4447, + S_REF_MINIPDB = 4448, + S_PDBMAP = 4449, + S_GDATA_HLSL32 = 4450, + S_LDATA_HLSL32 = 4451, + S_GDATA_HLSL32_EX = 4452, + S_LDATA_HLSL32_EX = 4453, + S_FASTLINK = 4455, + S_INLINEES = 4456, + S_END = 6, + S_INLINESITE_END = 4430, + S_PROC_ID_END = 4431, + S_THUNK32 = 4354, + S_TRAMPOLINE = 4396, + S_SECTION = 4406, + S_COFFGROUP = 4407, + S_EXPORT = 4408, + S_LPROC32 = 4367, + S_GPROC32 = 4368, + S_LPROC32_ID = 4422, + S_GPROC32_ID = 4423, + S_LPROC32_DPC = 4437, + S_LPROC32_DPC_ID = 4438, + S_REGISTER = 4358, + S_PUB32 = 4366, + S_PROCREF = 4389, + S_LPROCREF = 4391, + S_ENVBLOCK = 4413, + S_INLINESITE = 4429, + S_LOCAL = 4414, + S_DEFRANGE = 4415, + S_DEFRANGE_SUBFIELD = 4416, + S_DEFRANGE_REGISTER = 4417, + S_DEFRANGE_FRAMEPOINTER_REL = 4418, + S_DEFRANGE_SUBFIELD_REGISTER = 4419, + S_DEFRANGE_FRAMEPOINTER_REL_FULL_SCOPE = 4420, + S_DEFRANGE_REGISTER_REL = 4421, + S_BLOCK32 = 4355, + S_LABEL32 = 4357, + S_OBJNAME = 4353, + S_COMPILE2 = 4374, + S_COMPILE3 = 4412, + S_FRAMEPROC = 4114, + S_CALLSITEINFO = 4409, + S_FILESTATIC = 4435, + S_HEAPALLOCSITE = 4446, + S_FRAMECOOKIE = 4410, + S_CALLEES = 4442, + S_CALLERS = 4443, + S_UDT = 4360, + S_COBOLUDT = 4361, + S_BUILDINFO = 4428, + S_BPREL32 = 4363, + S_REGREL32 = 4369, + S_CONSTANT = 4359, + S_MANCONSTANT = 4397, + S_LDATA32 = 4364, + S_GDATA32 = 4365, + S_LMANDATA = 4380, + S_GMANDATA = 4381, + S_LTHREAD32 = 4370, + S_GTHREAD32 = 4371, +}; + +const SectionContrSubstreamVersion = enum(u32) { + Ver60 = 0xeffe0000 + 19970605, + V2 = 0xeffe0000 + 20140516 +}; + +const RecordPrefix = packed struct { + RecordLen: u16, /// Record length, starting from &RecordKind. + RecordKind: u16, /// Record kind enum (SymRecordKind or TypeRecordKind) +}; + pub const Pdb = struct { in_file: os.File, allocator: *mem.Allocator, + coff: *Coff, msf: Msf, - pub fn openFile(self: *Pdb, allocator: *mem.Allocator, file_name: []u8) !void { + pub fn openFile(self: *Pdb, coff: *Coff, file_name: []u8) !void { self.in_file = try os.File.openRead(file_name); - self.allocator = allocator; + self.allocator = coff.allocator; + self.coff = coff; - try self.msf.openFile(allocator, self.in_file); + try self.msf.openFile(self.allocator, self.in_file); + } + + pub fn getStreamById(self: *Pdb, id: u32) ?*MsfStream { + if (id >= self.msf.streams.len) + return null; + return &self.msf.streams[id]; } pub fn getStream(self: *Pdb, stream: StreamType) ?*MsfStream { const id = @enumToInt(stream); - if (id < self.msf.streams.len) - return &self.msf.streams[id]; - return null; + return self.getStreamById(id); } pub fn getSourceLine(self: *Pdb, address: usize) !void { @@ -98,38 +397,106 @@ pub const Pdb = struct { std.debug.warn("{}\n", header); warn("after header dbi stream at {} (file offset)\n", dbi.getFilePos()); + var modules = ArrayList(Module).init(self.allocator); + // Module Info Substream var mod_info_offset: usize = 0; - while (mod_info_offset < header.ModInfoSize) { - const march_forward_bytes = dbi.getFilePos() % 4; - if (march_forward_bytes != 0) { - try dbi.seekForward(march_forward_bytes); - mod_info_offset += march_forward_bytes; - } + while (mod_info_offset != header.ModInfoSize) { var mod_info: ModInfo = undefined; try dbi.stream.readStruct(ModInfo, &mod_info); std.debug.warn("{}\n", mod_info); - mod_info_offset += @sizeOf(ModInfo); + var this_record_len: usize = @sizeOf(ModInfo); const module_name = try dbi.readNullTermString(self.allocator); std.debug.warn("module_name '{}'\n", module_name); - mod_info_offset += module_name.len + 1; - - //if (mem.eql(u8, module_name, "piler_rt.obj")) { - // std.debug.warn("detected bad thing\n"); - // try dbi.seekTo(dbi.pos - - // "c:\\msys64\\home\\andy\\zig\\build-llvm6-msvc-release\\.\\zig-cache\\compiler_rt.obj\x00".len - - // @sizeOf(ModInfo)); - // mod_info_offset -= module_name.len + 1; - // continue; - //} + this_record_len += module_name.len + 1; const obj_file_name = try dbi.readNullTermString(self.allocator); std.debug.warn("obj_file_name '{}'\n", obj_file_name); - mod_info_offset += obj_file_name.len + 1; - } - std.debug.warn("end modules\n"); + this_record_len += obj_file_name.len + 1; + const march_forward_bytes = this_record_len % 4; + if (march_forward_bytes != 0) { + try dbi.seekForward(march_forward_bytes); + this_record_len += march_forward_bytes; + } + + try modules.append(Module{ + .mod_info = mod_info, + .module_name = module_name, + .obj_file_name = obj_file_name, + }); + + mod_info_offset += this_record_len; + if (mod_info_offset > header.ModInfoSize) + return error.InvalidDebugInfo; + } + + // Section Contribution Substream + var sect_contribs = ArrayList(SectionContribEntry).init(self.allocator); + std.debug.warn("looking at Section Contributinos now\n"); + var sect_cont_offset: usize = 0; + if (header.SectionContributionSize != 0) { + const ver = @intToEnum(SectionContrSubstreamVersion, try dbi.stream.readIntLe(u32)); + if (ver != SectionContrSubstreamVersion.Ver60) + return error.InvalidDebugInfo; + sect_cont_offset += @sizeOf(u32); + } + while (sect_cont_offset != header.SectionContributionSize) { + const entry = try sect_contribs.addOne(); + try dbi.stream.readStruct(SectionContribEntry, entry); + std.debug.warn("{}\n", entry); + sect_cont_offset += @sizeOf(SectionContribEntry); + + if (sect_cont_offset > header.SectionContributionSize) + return error.InvalidDebugInfo; + } + //std.debug.warn("looking at section map now\n"); + //if (header.SectionMapSize == 0) + // return error.MissingDebugInfo; + + //var sect_map_hdr: SectionMapHeader = undefined; + //try dbi.stream.readStruct(SectionMapHeader, §_map_hdr); + + //const sect_entries = try self.allocator.alloc(SectionMapEntry, sect_map_hdr.Count); + //const as_bytes = @sliceToBytes(sect_entries); + //if (as_bytes.len + @sizeOf(SectionMapHeader) != header.SectionMapSize) + // return error.InvalidDebugInfo; + //try dbi.stream.readNoEof(as_bytes); + + //for (sect_entries) |sect_entry| { + // std.debug.warn("{}\n", sect_entry); + //} + + const mod_index = for (sect_contribs.toSlice()) |sect_contrib| { + const coff_section = self.coff.sections.toSlice()[sect_contrib.Section]; + std.debug.warn("looking in coff name: {}\n", mem.toSliceConst(u8, &coff_section.header.name)); + + const vaddr_start = coff_section.header.virtual_address + sect_contrib.Offset; + const vaddr_end = vaddr_start + sect_contrib.Size; + if (address >= vaddr_start and address < vaddr_end) { + std.debug.warn("found sect contrib: {}\n", sect_contrib); + break sect_contrib.ModuleIndex; + } + } else return error.MissingDebugInfo; + + const mod = &modules.toSlice()[mod_index]; + const modi = self.getStreamById(mod.mod_info.ModuleSymStream) orelse return error.InvalidDebugInfo; + + const signature = try modi.stream.readIntLe(u32); + if (signature != 4) + return error.InvalidDebugInfo; + + const symbols = try self.allocator.alloc(u8, mod.mod_info.SymByteSize - 4); + std.debug.warn("read {} bytes of symbol info\n", symbols.len); + try modi.stream.readNoEof(symbols); + + if (mod.mod_info.C11ByteSize != 0) + return error.InvalidDebugInfo; + + if (mod.mod_info.C13ByteSize != 0) { + std.debug.warn("read C13 line info\n"); + } // TODO: locate corresponding source line information } From 72185e7dd36d9b87c57a3ed3719b9824ca11edf9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Aug 2018 16:57:55 -0400 Subject: [PATCH 27/56] finding the function that an address is in --- std/coff.zig | 2 +- std/pdb.zig | 64 +++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/std/coff.zig b/std/coff.zig index cce001d618..2921109bd6 100644 --- a/std/coff.zig +++ b/std/coff.zig @@ -215,7 +215,7 @@ const OptionalHeader = struct { data_directory: [IMAGE_NUMBEROF_DIRECTORY_ENTRIES]DataDirectory, }; -const Section = struct { +pub const Section = struct { header: SectionHeader, }; diff --git a/std/pdb.zig b/std/pdb.zig index 14c584a9f7..e67ca7cf0c 100644 --- a/std/pdb.zig +++ b/std/pdb.zig @@ -5,7 +5,7 @@ const math = std.math; const mem = std.mem; const os = std.os; const warn = std.debug.warn; -const Coff = std.coff.Coff; +const coff = std.coff; const ArrayList = std.ArrayList; @@ -153,7 +153,7 @@ pub const SymbolRecordKind = enum(u16) { /// Duplicate copy of SymbolRecordKind, but using the official CV names. Useful /// for reference purposes and when dealing with unknown record types. -pub const SymbolKind = enum(u16) { +pub const SymbolKind = packed enum(u16) { S_COMPILE = 1, S_REGISTER_16t = 2, S_CONSTANT_16t = 3, @@ -352,6 +352,33 @@ pub const SymbolKind = enum(u16) { S_GTHREAD32 = 4371, }; +const TypeIndex = u32; + +const ProcSym = packed struct { + Parent: u32 , + End: u32 , + Next: u32 , + CodeSize: u32 , + DbgStart: u32 , + DbgEnd: u32 , + FunctionType: TypeIndex , + CodeOffset: u32, + Segment: u16, + Flags: ProcSymFlags, + Name: u8, +}; + +const ProcSymFlags = packed struct { + HasFP: bool, + HasIRET: bool, + HasFRET: bool, + IsNoReturn: bool, + IsUnreachable: bool, + HasCustomCallingConv: bool, + IsNoInline: bool, + HasOptimizedDebugInfo: bool, +}; + const SectionContrSubstreamVersion = enum(u32) { Ver60 = 0xeffe0000 + 19970605, V2 = 0xeffe0000 + 20140516 @@ -359,20 +386,20 @@ const SectionContrSubstreamVersion = enum(u32) { const RecordPrefix = packed struct { RecordLen: u16, /// Record length, starting from &RecordKind. - RecordKind: u16, /// Record kind enum (SymRecordKind or TypeRecordKind) + RecordKind: SymbolKind, /// Record kind enum (SymRecordKind or TypeRecordKind) }; pub const Pdb = struct { in_file: os.File, allocator: *mem.Allocator, - coff: *Coff, + coff: *coff.Coff, msf: Msf, - pub fn openFile(self: *Pdb, coff: *Coff, file_name: []u8) !void { + pub fn openFile(self: *Pdb, coff_ptr: *coff.Coff, file_name: []u8) !void { self.in_file = try os.File.openRead(file_name); - self.allocator = coff.allocator; - self.coff = coff; + self.allocator = coff_ptr.allocator; + self.coff = coff_ptr; try self.msf.openFile(self.allocator, self.in_file); } @@ -468,8 +495,9 @@ pub const Pdb = struct { // std.debug.warn("{}\n", sect_entry); //} + var coff_section: *coff.Section = undefined; const mod_index = for (sect_contribs.toSlice()) |sect_contrib| { - const coff_section = self.coff.sections.toSlice()[sect_contrib.Section]; + coff_section = &self.coff.sections.toSlice()[sect_contrib.Section]; std.debug.warn("looking in coff name: {}\n", mem.toSliceConst(u8, &coff_section.header.name)); const vaddr_start = coff_section.header.virtual_address + sect_contrib.Offset; @@ -490,6 +518,26 @@ pub const Pdb = struct { const symbols = try self.allocator.alloc(u8, mod.mod_info.SymByteSize - 4); std.debug.warn("read {} bytes of symbol info\n", symbols.len); try modi.stream.readNoEof(symbols); + var symbol_i: usize = 0; + const proc_sym = while (symbol_i != symbols.len) { + const prefix = @ptrCast(*RecordPrefix, &symbols[symbol_i]); + if (prefix.RecordLen < 2) + return error.InvalidDebugInfo; + if (prefix.RecordKind == SymbolKind.S_LPROC32) { + const proc_sym = @ptrCast(*ProcSym, &symbols[symbol_i + @sizeOf(RecordPrefix)]); + const vaddr_start = coff_section.header.virtual_address + proc_sym.CodeOffset; + const vaddr_end = vaddr_start + proc_sym.CodeSize; + std.debug.warn(" {}\n", proc_sym); + if (address >= vaddr_start and address < vaddr_end) { + break proc_sym; + } + } + symbol_i += prefix.RecordLen + @sizeOf(u16); + if (symbol_i > symbols.len) + return error.InvalidDebugInfo; + } else return error.MissingDebugInfo; + + std.debug.warn("found in {s}: {}\n", ([*]u8)((*[1]u8)(&proc_sym.Name)), proc_sym); if (mod.mod_info.C11ByteSize != 0) return error.InvalidDebugInfo; From 99170aa13db236b47aa8fda4a3d94b49e6b7f93c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 31 Aug 2018 01:01:37 -0400 Subject: [PATCH 28/56] finding source file, line, and column info --- std/io.zig | 2 +- std/pdb.zig | 122 +++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 118 insertions(+), 6 deletions(-) diff --git a/std/io.zig b/std/io.zig index 2de52493b1..a1a77271e1 100644 --- a/std/io.zig +++ b/std/io.zig @@ -210,7 +210,7 @@ pub fn InStream(comptime ReadError: type) type { pub fn readStruct(self: *Self, comptime T: type, ptr: *T) !void { // Only extern and packed structs have defined in-memory layout. - assert(@typeInfo(T).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto); + comptime assert(@typeInfo(T).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto); return self.readNoEof(@sliceToBytes((*[1]T)(ptr)[0..])); } }; diff --git a/std/pdb.zig b/std/pdb.zig index e67ca7cf0c..c1fb863b67 100644 --- a/std/pdb.zig +++ b/std/pdb.zig @@ -365,7 +365,8 @@ const ProcSym = packed struct { CodeOffset: u32, Segment: u16, Flags: ProcSymFlags, - Name: u8, + // following is a null terminated string + // Name: [*]u8, }; const ProcSymFlags = packed struct { @@ -389,6 +390,51 @@ const RecordPrefix = packed struct { RecordKind: SymbolKind, /// Record kind enum (SymRecordKind or TypeRecordKind) }; +const LineFragmentHeader = packed struct { + RelocOffset: u32, /// Code offset of line contribution. + RelocSegment: u16, /// Code segment of line contribution. + Flags: LineFlags, + CodeSize: u32, /// Code size of this line contribution. +}; + +const LineFlags = packed struct { + LF_HaveColumns: bool, /// CV_LINES_HAVE_COLUMNS + unused: u15, +}; + +/// The following two variable length arrays appear immediately after the +/// header. The structure definitions follow. +/// LineNumberEntry Lines[NumLines]; +/// ColumnNumberEntry Columns[NumLines]; +const LineBlockFragmentHeader = packed struct { + /// Offset of FileChecksum entry in File + /// checksums buffer. The checksum entry then + /// contains another offset into the string + /// table of the actual name. + NameIndex: u32, + NumLines: u32, + BlockSize: u32, /// code size of block, in bytes +}; + + +const LineNumberEntry = packed struct { + Offset: u32, /// Offset to start of code bytes for line number + Flags: u32, + + /// TODO runtime crash when I make the actual type of Flags this + const Flags = packed struct { + Start: u24, + End: u7, + IsStatement: bool, + }; +}; + +const ColumnNumberEntry = packed struct { + StartColumn: u16, + EndColumn: u16, +}; + + pub const Pdb = struct { in_file: os.File, allocator: *mem.Allocator, @@ -537,16 +583,82 @@ pub const Pdb = struct { return error.InvalidDebugInfo; } else return error.MissingDebugInfo; - std.debug.warn("found in {s}: {}\n", ([*]u8)((*[1]u8)(&proc_sym.Name)), proc_sym); + std.debug.warn("found in {s}: {}\n", @ptrCast([*]u8, proc_sym) + @sizeOf(ProcSym), proc_sym); if (mod.mod_info.C11ByteSize != 0) return error.InvalidDebugInfo; - if (mod.mod_info.C13ByteSize != 0) { - std.debug.warn("read C13 line info\n"); + if (mod.mod_info.C13ByteSize == 0) { + return error.MissingDebugInfo; } - // TODO: locate corresponding source line information + const line_info = try self.allocator.alloc(u8, mod.mod_info.C13ByteSize); + std.debug.warn("read C13 line info {} bytes\n", line_info.len); + try modi.stream.readNoEof(line_info); + + var line_index: usize = 0; + while (line_index != line_info.len) { + std.debug.warn("unknown bytes: {x2} {x2} {x2} {x2} {x2} {x2} {x2} {x2}\n", + line_info[line_index+0], + line_info[line_index+1], + line_info[line_index+2], + line_info[line_index+3], + line_info[line_index+4], + line_info[line_index+5], + line_info[line_index+6], + line_info[line_index+7], + ); + line_index += 8; + + const line_hdr = @ptrCast(*LineFragmentHeader, &line_info[line_index]); + if (line_hdr.RelocSegment == 0) return error.MissingDebugInfo; + std.debug.warn("{}\n", line_hdr); + line_index += @sizeOf(LineFragmentHeader); + + const block_hdr = @ptrCast(*LineBlockFragmentHeader, &line_info[line_index]); + std.debug.warn("{}\n", block_hdr); + line_index += @sizeOf(LineBlockFragmentHeader); + + const has_column = line_hdr.Flags.LF_HaveColumns; + std.debug.warn("has column: {}\n", has_column); + + const frag_vaddr_start = coff_section.header.virtual_address + line_hdr.RelocOffset; + const frag_vaddr_end = frag_vaddr_start + line_hdr.CodeSize; + if (address >= frag_vaddr_start and address < frag_vaddr_end) { + std.debug.warn("found line listing\n"); + var line_i: usize = 0; + const start_line_index = line_index; + while (line_i < block_hdr.NumLines) : (line_i += 1) { + const line_num_entry = @ptrCast(*LineNumberEntry, &line_info[line_index]); + line_index += @sizeOf(LineNumberEntry); + const flags = @ptrCast(*LineNumberEntry.Flags, &line_num_entry.Flags); + std.debug.warn("{} {}\n", line_num_entry, flags); + const vaddr_start = frag_vaddr_start + line_num_entry.Offset; + const vaddr_end = if (flags.End == 0) frag_vaddr_end else vaddr_start + flags.End; + std.debug.warn("test {x} <= {x} < {x}\n", vaddr_start, address, vaddr_end); + if (address >= vaddr_start and address < vaddr_end) { + std.debug.warn("{} line {}\n", block_hdr.NameIndex, flags.Start); + if (has_column) { + line_index = start_line_index + @sizeOf(LineNumberEntry) * block_hdr.NumLines; + line_index += @sizeOf(ColumnNumberEntry) * line_i; + const col_num_entry = @ptrCast(*ColumnNumberEntry, &line_info[line_index]); + std.debug.warn("col {}\n", col_num_entry.StartColumn); + } + return; + } + } + return error.MissingDebugInfo; + } else { + line_index += @sizeOf(LineNumberEntry) * block_hdr.NumLines; + if (has_column) + line_index += @sizeOf(ColumnNumberEntry) * block_hdr.NumLines; + } + + if (line_index > mod.mod_info.C13ByteSize) + return error.InvalidDebugInfo; + } + + std.debug.warn("end line info\n"); } }; From a7527389ccdf4e3ab6ce3f83ff9a55003b2c5692 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Fri, 31 Aug 2018 18:40:09 +1200 Subject: [PATCH 29/56] Make poly1305 and x25519 more idiomatic zig This also adjusts the current hash/hmac functions to have a consistent interface allowing easier switching/testing. --- std/crypto/blake2.zig | 16 +- std/crypto/hmac.zig | 91 +++++----- std/crypto/index.zig | 8 +- std/crypto/md5.zig | 8 +- std/crypto/poly1305.zig | 355 +++++++++++++++++++++------------------- std/crypto/sha1.zig | 8 +- std/crypto/sha2.zig | 16 +- std/crypto/sha3.zig | 12 +- std/crypto/x25519.zig | 233 +++++++++++++------------- 9 files changed, 392 insertions(+), 355 deletions(-) diff --git a/std/crypto/blake2.zig b/std/crypto/blake2.zig index 947133e4cf..467ddde5db 100644 --- a/std/crypto/blake2.zig +++ b/std/crypto/blake2.zig @@ -34,8 +34,8 @@ pub const Blake2s256 = Blake2s(256); fn Blake2s(comptime out_len: usize) type { return struct { const Self = this; - const block_size = 64; - const digest_size = out_len / 8; + const block_length = 64; + const digest_length = out_len / 8; const iv = [8]u32{ 0x6A09E667, @@ -250,8 +250,8 @@ test "blake2s256 streaming" { } test "blake2s256 aligned final" { - var block = []u8{0} ** Blake2s256.block_size; - var out: [Blake2s256.digest_size]u8 = undefined; + var block = []u8{0} ** Blake2s256.block_length; + var out: [Blake2s256.digest_length]u8 = undefined; var h = Blake2s256.init(); h.update(block); @@ -267,8 +267,8 @@ pub const Blake2b512 = Blake2b(512); fn Blake2b(comptime out_len: usize) type { return struct { const Self = this; - const block_size = 128; - const digest_size = out_len / 8; + const block_length = 128; + const digest_length = out_len / 8; const iv = [8]u64{ 0x6a09e667f3bcc908, @@ -483,8 +483,8 @@ test "blake2b512 streaming" { } test "blake2b512 aligned final" { - var block = []u8{0} ** Blake2b512.block_size; - var out: [Blake2b512.digest_size]u8 = undefined; + var block = []u8{0} ** Blake2b512.block_length; + var out: [Blake2b512.digest_length]u8 = undefined; var h = Blake2b512.init(); h.update(block); diff --git a/std/crypto/hmac.zig b/std/crypto/hmac.zig index 1415e88cf4..23eeff2a00 100644 --- a/std/crypto/hmac.zig +++ b/std/crypto/hmac.zig @@ -7,46 +7,63 @@ pub const HmacMd5 = Hmac(crypto.Md5); pub const HmacSha1 = Hmac(crypto.Sha1); pub const HmacSha256 = Hmac(crypto.Sha256); -pub fn Hmac(comptime H: type) type { +pub fn Hmac(comptime Hash: type) type { return struct { - const digest_size = H.digest_size; + const Self = this; + pub const mac_length = Hash.digest_length; + pub const minimum_key_length = 0; - pub fn hash(output: []u8, key: []const u8, message: []const u8) void { - debug.assert(output.len >= H.digest_size); - debug.assert(H.digest_size <= H.block_size); // HMAC makes this assumption - var scratch: [H.block_size]u8 = undefined; + o_key_pad: [Hash.block_length]u8, + i_key_pad: [Hash.block_length]u8, + scratch: [Hash.block_length]u8, + hash: Hash, + + // HMAC(k, m) = H(o_key_pad | H(i_key_pad | msg)) where | is concatenation + pub fn create(out: []u8, msg: []const u8, key: []const u8) void { + var ctx = Self.init(key); + ctx.update(msg); + ctx.final(out[0..]); + } + + pub fn init(key: []const u8) Self { + var ctx: Self = undefined; // Normalize key length to block size of hash - if (key.len > H.block_size) { - H.hash(key, scratch[0..H.digest_size]); - mem.set(u8, scratch[H.digest_size..H.block_size], 0); - } else if (key.len < H.block_size) { - mem.copy(u8, scratch[0..key.len], key); - mem.set(u8, scratch[key.len..H.block_size], 0); + if (key.len > Hash.block_length) { + Hash.hash(key, ctx.scratch[0..mac_length]); + mem.set(u8, ctx.scratch[mac_length..Hash.block_length], 0); + } else if (key.len < Hash.block_length) { + mem.copy(u8, ctx.scratch[0..key.len], key); + mem.set(u8, ctx.scratch[key.len..Hash.block_length], 0); } else { - mem.copy(u8, scratch[0..], key); + mem.copy(u8, ctx.scratch[0..], key); } - var o_key_pad: [H.block_size]u8 = undefined; - for (o_key_pad) |*b, i| { - b.* = scratch[i] ^ 0x5c; + for (ctx.o_key_pad) |*b, i| { + b.* = ctx.scratch[i] ^ 0x5c; } - var i_key_pad: [H.block_size]u8 = undefined; - for (i_key_pad) |*b, i| { - b.* = scratch[i] ^ 0x36; + for (ctx.i_key_pad) |*b, i| { + b.* = ctx.scratch[i] ^ 0x36; } - // HMAC(k, m) = H(o_key_pad | H(i_key_pad | message)) where | is concatenation - var hmac = H.init(); - hmac.update(i_key_pad[0..]); - hmac.update(message); - hmac.final(scratch[0..H.digest_size]); + ctx.hash = Hash.init(); + ctx.hash.update(ctx.i_key_pad[0..]); + return ctx; + } - hmac.reset(); - hmac.update(o_key_pad[0..]); - hmac.update(scratch[0..H.digest_size]); - hmac.final(output[0..H.digest_size]); + pub fn update(ctx: *Self, msg: []const u8) void { + ctx.hash.update(msg); + } + + pub fn final(ctx: *Self, out: []u8) void { + debug.assert(Hash.block_length >= out.len and out.len >= mac_length); + + ctx.hash.final(ctx.scratch[0..mac_length]); + ctx.hash.reset(); + ctx.hash.update(ctx.o_key_pad[0..]); + ctx.hash.update(ctx.scratch[0..mac_length]); + ctx.hash.final(out[0..mac_length]); } }; } @@ -54,28 +71,28 @@ pub fn Hmac(comptime H: type) type { const htest = @import("test.zig"); test "hmac md5" { - var out: [crypto.Md5.digest_size]u8 = undefined; - HmacMd5.hash(out[0..], "", ""); + var out: [HmacMd5.mac_length]u8 = undefined; + HmacMd5.create(out[0..], "", ""); htest.assertEqual("74e6f7298a9c2d168935f58c001bad88", out[0..]); - HmacMd5.hash(out[0..], "key", "The quick brown fox jumps over the lazy dog"); + HmacMd5.create(out[0..], "The quick brown fox jumps over the lazy dog", "key"); htest.assertEqual("80070713463e7749b90c2dc24911e275", out[0..]); } test "hmac sha1" { - var out: [crypto.Sha1.digest_size]u8 = undefined; - HmacSha1.hash(out[0..], "", ""); + var out: [HmacSha1.mac_length]u8 = undefined; + HmacSha1.create(out[0..], "", ""); htest.assertEqual("fbdb1d1b18aa6c08324b7d64b71fb76370690e1d", out[0..]); - HmacSha1.hash(out[0..], "key", "The quick brown fox jumps over the lazy dog"); + HmacSha1.create(out[0..], "The quick brown fox jumps over the lazy dog", "key"); htest.assertEqual("de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9", out[0..]); } test "hmac sha256" { - var out: [crypto.Sha256.digest_size]u8 = undefined; - HmacSha256.hash(out[0..], "", ""); + var out: [HmacSha256.mac_length]u8 = undefined; + HmacSha256.create(out[0..], "", ""); htest.assertEqual("b613679a0814d9ec772f95d778c35fc5ff1697c493715653c6c712144292c5ad", out[0..]); - HmacSha256.hash(out[0..], "key", "The quick brown fox jumps over the lazy dog"); + HmacSha256.create(out[0..], "The quick brown fox jumps over the lazy dog", "key"); htest.assertEqual("f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8", out[0..]); } diff --git a/std/crypto/index.zig b/std/crypto/index.zig index f861b61a20..9f20ce514b 100644 --- a/std/crypto/index.zig +++ b/std/crypto/index.zig @@ -21,15 +21,15 @@ pub const Blake2b512 = blake2.Blake2b512; const hmac = @import("hmac.zig"); pub const HmacMd5 = hmac.HmacMd5; -pub const HmacSha1 = hmac.Sha1; -pub const HmacSha256 = hmac.Sha256; +pub const HmacSha1 = hmac.HmacSha1; +pub const HmacSha256 = hmac.HmacSha256; const import_chaCha20 = @import("chacha20.zig"); pub const chaCha20IETF = import_chaCha20.chaCha20IETF; pub const chaCha20With64BitNonce = import_chaCha20.chaCha20With64BitNonce; -const poly1305 = @import("poly1305.zig"); -const x25519 = @import("x25519.zig"); +pub const Poly1305 = @import("poly1305.zig").Poly1305; +pub const X25519 = @import("x25519.zig").X25519; test "crypto" { _ = @import("md5.zig"); diff --git a/std/crypto/md5.zig b/std/crypto/md5.zig index 23fe2313a0..20334ec7d8 100644 --- a/std/crypto/md5.zig +++ b/std/crypto/md5.zig @@ -29,8 +29,8 @@ fn Rp(a: usize, b: usize, c: usize, d: usize, k: usize, s: u32, t: u32) RoundPar pub const Md5 = struct { const Self = this; - const block_size = 64; - const digest_size = 16; + const block_length = 64; + const digest_length = 16; s: [4]u32, // Streaming Cache @@ -271,8 +271,8 @@ test "md5 streaming" { } test "md5 aligned final" { - var block = []u8{0} ** Md5.block_size; - var out: [Md5.digest_size]u8 = undefined; + var block = []u8{0} ** Md5.block_length; + var out: [Md5.digest_length]u8 = undefined; var h = Md5.init(); h.update(block); diff --git a/std/crypto/poly1305.zig b/std/crypto/poly1305.zig index bce90db3d5..1cec72890f 100644 --- a/std/crypto/poly1305.zig +++ b/std/crypto/poly1305.zig @@ -9,7 +9,12 @@ const Endian = builtin.Endian; const readInt = std.mem.readInt; const writeInt = std.mem.writeInt; -const crypto_poly1305_ctx = struct { +pub const Poly1305 = struct { + const Self = this; + + pub const mac_length = 16; + pub const minimum_key_length = 32; + // constant multiplier (from the secret key) r: [4]u32, // accumulated hash @@ -21,190 +26,198 @@ const crypto_poly1305_ctx = struct { // How many bytes are there in the chunk. c_idx: usize, - fn secure_zero(self: *crypto_poly1305_ctx) void { - std.mem.secureZero(u8, @ptrCast([*]u8, self)[0..@sizeOf(crypto_poly1305_ctx)]); + fn secure_zero(self: *Poly1305) void { + std.mem.secureZero(u8, @ptrCast([*]u8, self)[0..@sizeOf(Poly1305)]); } -}; -// h = (h + c) * r -// preconditions: -// ctx->h <= 4_ffffffff_ffffffff_ffffffff_ffffffff -// ctx->c <= 1_ffffffff_ffffffff_ffffffff_ffffffff -// ctx->r <= 0ffffffc_0ffffffc_0ffffffc_0fffffff -// Postcondition: -// ctx->h <= 4_ffffffff_ffffffff_ffffffff_ffffffff -fn poly_block(ctx: *crypto_poly1305_ctx) void { - // s = h + c, without carry propagation - const s0 = u64(ctx.h[0]) + ctx.c[0]; // s0 <= 1_fffffffe - const s1 = u64(ctx.h[1]) + ctx.c[1]; // s1 <= 1_fffffffe - const s2 = u64(ctx.h[2]) + ctx.c[2]; // s2 <= 1_fffffffe - const s3 = u64(ctx.h[3]) + ctx.c[3]; // s3 <= 1_fffffffe - const s4 = u64(ctx.h[4]) + ctx.c[4]; // s4 <= 5 + pub fn create(out: []u8, msg: []const u8, key: []const u8) void { + std.debug.assert(out.len >= mac_length); + std.debug.assert(key.len >= minimum_key_length); - // Local all the things! - const r0 = ctx.r[0]; // r0 <= 0fffffff - const r1 = ctx.r[1]; // r1 <= 0ffffffc - const r2 = ctx.r[2]; // r2 <= 0ffffffc - const r3 = ctx.r[3]; // r3 <= 0ffffffc - const rr0 = (r0 >> 2) * 5; // rr0 <= 13fffffb // lose 2 bits... - const rr1 = (r1 >> 2) + r1; // rr1 <= 13fffffb // rr1 == (r1 >> 2) * 5 - const rr2 = (r2 >> 2) + r2; // rr2 <= 13fffffb // rr1 == (r2 >> 2) * 5 - const rr3 = (r3 >> 2) + r3; // rr3 <= 13fffffb // rr1 == (r3 >> 2) * 5 + var ctx = Poly1305.init(key); + ctx.update(msg); + ctx.final(out); + } - // (h + c) * r, without carry propagation - const x0 = s0 * r0 + s1 * rr3 + s2 * rr2 + s3 * rr1 + s4 * rr0; //<=97ffffe007fffff8 - const x1 = s0 * r1 + s1 * r0 + s2 * rr3 + s3 * rr2 + s4 * rr1; //<=8fffffe20ffffff6 - const x2 = s0 * r2 + s1 * r1 + s2 * r0 + s3 * rr3 + s4 * rr2; //<=87ffffe417fffff4 - const x3 = s0 * r3 + s1 * r2 + s2 * r1 + s3 * r0 + s4 * rr3; //<=7fffffe61ffffff2 - const x4 = s4 * (r0 & 3); // ...recover 2 bits //<= f + // Initialize the MAC context. + // - key.len is sufficient size. + pub fn init(key: []const u8) Self { + var ctx: Poly1305 = undefined; - // partial reduction modulo 2^130 - 5 - const _u5 = @truncate(u32, x4 + (x3 >> 32)); // u5 <= 7ffffff5 - const _u0 = (_u5 >> 2) * 5 + (x0 & 0xffffffff); - const _u1 = (_u0 >> 32) + (x1 & 0xffffffff) + (x0 >> 32); - const _u2 = (_u1 >> 32) + (x2 & 0xffffffff) + (x1 >> 32); - const _u3 = (_u2 >> 32) + (x3 & 0xffffffff) + (x2 >> 32); - const _u4 = (_u3 >> 32) + (_u5 & 3); + // Initial hash is zero + { + var i: usize = 0; + while (i < 5) : (i += 1) { + ctx.h[i] = 0; + } + } + // add 2^130 to every input block + ctx.c[4] = 1; + poly_clear_c(&ctx); - // Update the hash - ctx.h[0] = @truncate(u32, _u0); // u0 <= 1_9ffffff0 - ctx.h[1] = @truncate(u32, _u1); // u1 <= 1_97ffffe0 - ctx.h[2] = @truncate(u32, _u2); // u2 <= 1_8fffffe2 - ctx.h[3] = @truncate(u32, _u3); // u3 <= 1_87ffffe4 - ctx.h[4] = @truncate(u32, _u4); // u4 <= 4 -} + // load r and pad (r has some of its bits cleared) + { + var i: usize = 0; + while (i < 1) : (i += 1) { + ctx.r[0] = readInt(key[0..4], u32, Endian.Little) & 0x0fffffff; + } + } + { + var i: usize = 1; + while (i < 4) : (i += 1) { + ctx.r[i] = readInt(key[i * 4 .. i * 4 + 4], u32, Endian.Little) & 0x0ffffffc; + } + } + { + var i: usize = 0; + while (i < 4) : (i += 1) { + ctx.pad[i] = readInt(key[i * 4 + 16 .. i * 4 + 16 + 4], u32, Endian.Little); + } + } -// (re-)initializes the input counter and input buffer -fn poly_clear_c(ctx: *crypto_poly1305_ctx) void { - ctx.c[0] = 0; - ctx.c[1] = 0; - ctx.c[2] = 0; - ctx.c[3] = 0; - ctx.c_idx = 0; -} + return ctx; + } -fn poly_take_input(ctx: *crypto_poly1305_ctx, input: u8) void { - const word = ctx.c_idx >> 2; - const byte = ctx.c_idx & 3; - ctx.c[word] |= std.math.shl(u32, input, byte * 8); - ctx.c_idx += 1; -} + // h = (h + c) * r + // preconditions: + // ctx->h <= 4_ffffffff_ffffffff_ffffffff_ffffffff + // ctx->c <= 1_ffffffff_ffffffff_ffffffff_ffffffff + // ctx->r <= 0ffffffc_0ffffffc_0ffffffc_0fffffff + // Postcondition: + // ctx->h <= 4_ffffffff_ffffffff_ffffffff_ffffffff + fn poly_block(ctx: *Poly1305) void { + // s = h + c, without carry propagation + const s0 = u64(ctx.h[0]) + ctx.c[0]; // s0 <= 1_fffffffe + const s1 = u64(ctx.h[1]) + ctx.c[1]; // s1 <= 1_fffffffe + const s2 = u64(ctx.h[2]) + ctx.c[2]; // s2 <= 1_fffffffe + const s3 = u64(ctx.h[3]) + ctx.c[3]; // s3 <= 1_fffffffe + const s4 = u64(ctx.h[4]) + ctx.c[4]; // s4 <= 5 -fn poly_update(ctx: *crypto_poly1305_ctx, message: []const u8) void { - for (message) |b| { - poly_take_input(ctx, b); - if (ctx.c_idx == 16) { + // Local all the things! + const r0 = ctx.r[0]; // r0 <= 0fffffff + const r1 = ctx.r[1]; // r1 <= 0ffffffc + const r2 = ctx.r[2]; // r2 <= 0ffffffc + const r3 = ctx.r[3]; // r3 <= 0ffffffc + const rr0 = (r0 >> 2) * 5; // rr0 <= 13fffffb // lose 2 bits... + const rr1 = (r1 >> 2) + r1; // rr1 <= 13fffffb // rr1 == (r1 >> 2) * 5 + const rr2 = (r2 >> 2) + r2; // rr2 <= 13fffffb // rr1 == (r2 >> 2) * 5 + const rr3 = (r3 >> 2) + r3; // rr3 <= 13fffffb // rr1 == (r3 >> 2) * 5 + + // (h + c) * r, without carry propagation + const x0 = s0 * r0 + s1 * rr3 + s2 * rr2 + s3 * rr1 + s4 * rr0; //<=97ffffe007fffff8 + const x1 = s0 * r1 + s1 * r0 + s2 * rr3 + s3 * rr2 + s4 * rr1; //<=8fffffe20ffffff6 + const x2 = s0 * r2 + s1 * r1 + s2 * r0 + s3 * rr3 + s4 * rr2; //<=87ffffe417fffff4 + const x3 = s0 * r3 + s1 * r2 + s2 * r1 + s3 * r0 + s4 * rr3; //<=7fffffe61ffffff2 + const x4 = s4 * (r0 & 3); // ...recover 2 bits //<= f + + // partial reduction modulo 2^130 - 5 + const _u5 = @truncate(u32, x4 + (x3 >> 32)); // u5 <= 7ffffff5 + const _u0 = (_u5 >> 2) * 5 + (x0 & 0xffffffff); + const _u1 = (_u0 >> 32) + (x1 & 0xffffffff) + (x0 >> 32); + const _u2 = (_u1 >> 32) + (x2 & 0xffffffff) + (x1 >> 32); + const _u3 = (_u2 >> 32) + (x3 & 0xffffffff) + (x2 >> 32); + const _u4 = (_u3 >> 32) + (_u5 & 3); + + // Update the hash + ctx.h[0] = @truncate(u32, _u0); // u0 <= 1_9ffffff0 + ctx.h[1] = @truncate(u32, _u1); // u1 <= 1_97ffffe0 + ctx.h[2] = @truncate(u32, _u2); // u2 <= 1_8fffffe2 + ctx.h[3] = @truncate(u32, _u3); // u3 <= 1_87ffffe4 + ctx.h[4] = @truncate(u32, _u4); // u4 <= 4 + } + + // (re-)initializes the input counter and input buffer + fn poly_clear_c(ctx: *Poly1305) void { + ctx.c[0] = 0; + ctx.c[1] = 0; + ctx.c[2] = 0; + ctx.c[3] = 0; + ctx.c_idx = 0; + } + + fn poly_take_input(ctx: *Poly1305, input: u8) void { + const word = ctx.c_idx >> 2; + const byte = ctx.c_idx & 3; + ctx.c[word] |= std.math.shl(u32, input, byte * 8); + ctx.c_idx += 1; + } + + fn poly_update(ctx: *Poly1305, msg: []const u8) void { + for (msg) |b| { + poly_take_input(ctx, b); + if (ctx.c_idx == 16) { + poly_block(ctx); + poly_clear_c(ctx); + } + } + } + + inline fn alignto(x: usize, block_size: usize) usize { + return ((~x) +% 1) & (block_size - 1); + } + + // Feed data into the MAC context. + pub fn update(ctx: *Self, msg: []const u8) void { + // Align ourselves with block boundaries + const alignm = std.math.min(alignto(ctx.c_idx, 16), msg.len); + poly_update(ctx, msg[0..alignm]); + + var nmsg = msg[alignm..]; + + // Process the msg block by block + const nb_blocks = nmsg.len >> 4; + var i: usize = 0; + while (i < nb_blocks) : (i += 1) { + ctx.c[0] = readInt(nmsg[0..4], u32, Endian.Little); + ctx.c[1] = readInt(nmsg[4..8], u32, Endian.Little); + ctx.c[2] = readInt(nmsg[8..12], u32, Endian.Little); + ctx.c[3] = readInt(nmsg[12..16], u32, Endian.Little); poly_block(ctx); + nmsg = nmsg[16..]; + } + if (nb_blocks > 0) { poly_clear_c(ctx); } - } -} -pub fn crypto_poly1305_init(ctx: *crypto_poly1305_ctx, key: [32]u8) void { - // Initial hash is zero - { - var i: usize = 0; - while (i < 5) : (i += 1) { - ctx.h[i] = 0; + // remaining bytes + poly_update(ctx, nmsg[0..]); + } + + // Finalize the MAC and output into buffer provided by caller. + pub fn final(ctx: *Self, out: []u8) void { + // Process the last block (if any) + if (ctx.c_idx != 0) { + // move the final 1 according to remaining input length + // (We may add less than 2^130 to the last input block) + ctx.c[4] = 0; + poly_take_input(ctx, 1); + // one last hash update + poly_block(ctx); } + + // check if we should subtract 2^130-5 by performing the + // corresponding carry propagation. + const _u0 = u64(5) + ctx.h[0]; // <= 1_00000004 + const _u1 = (_u0 >> 32) + ctx.h[1]; // <= 1_00000000 + const _u2 = (_u1 >> 32) + ctx.h[2]; // <= 1_00000000 + const _u3 = (_u2 >> 32) + ctx.h[3]; // <= 1_00000000 + const _u4 = (_u3 >> 32) + ctx.h[4]; // <= 5 + // u4 indicates how many times we should subtract 2^130-5 (0 or 1) + + // h + pad, minus 2^130-5 if u4 exceeds 3 + const uu0 = (_u4 >> 2) * 5 + ctx.h[0] + ctx.pad[0]; // <= 2_00000003 + const uu1 = (uu0 >> 32) + ctx.h[1] + ctx.pad[1]; // <= 2_00000000 + const uu2 = (uu1 >> 32) + ctx.h[2] + ctx.pad[2]; // <= 2_00000000 + const uu3 = (uu2 >> 32) + ctx.h[3] + ctx.pad[3]; // <= 2_00000000 + + writeInt(out[0..], @truncate(u32, uu0), Endian.Little); + writeInt(out[4..], @truncate(u32, uu1), Endian.Little); + writeInt(out[8..], @truncate(u32, uu2), Endian.Little); + writeInt(out[12..], @truncate(u32, uu3), Endian.Little); + + ctx.secure_zero(); } - // add 2^130 to every input block - ctx.c[4] = 1; - poly_clear_c(ctx); - - // load r and pad (r has some of its bits cleared) - { - var i: usize = 0; - while (i < 1) : (i += 1) { - ctx.r[0] = readInt(key[0..4], u32, Endian.Little) & 0x0fffffff; - } - } - { - var i: usize = 1; - while (i < 4) : (i += 1) { - ctx.r[i] = readInt(key[i * 4 .. i * 4 + 4], u32, Endian.Little) & 0x0ffffffc; - } - } - { - var i: usize = 0; - while (i < 4) : (i += 1) { - ctx.pad[i] = readInt(key[i * 4 + 16 .. i * 4 + 16 + 4], u32, Endian.Little); - } - } -} - -inline fn alignto(x: usize, block_size: usize) usize { - return ((~x) +% 1) & (block_size - 1); -} - -pub fn crypto_poly1305_update(ctx: *crypto_poly1305_ctx, message: []const u8) void { - // Align ourselves with block boundaries - const alignm = std.math.min(alignto(ctx.c_idx, 16), message.len); - poly_update(ctx, message[0..alignm]); - - var nmessage = message[alignm..]; - - // Process the message block by block - const nb_blocks = nmessage.len >> 4; - var i: usize = 0; - while (i < nb_blocks) : (i += 1) { - ctx.c[0] = readInt(nmessage[0..4], u32, Endian.Little); - ctx.c[1] = readInt(nmessage[4..8], u32, Endian.Little); - ctx.c[2] = readInt(nmessage[8..12], u32, Endian.Little); - ctx.c[3] = readInt(nmessage[12..16], u32, Endian.Little); - poly_block(ctx); - nmessage = nmessage[16..]; - } - if (nb_blocks > 0) { - poly_clear_c(ctx); - } - - // remaining bytes - poly_update(ctx, nmessage[0..]); -} - -pub fn crypto_poly1305_final(ctx: *crypto_poly1305_ctx, mac: []u8) void { - // Process the last block (if any) - if (ctx.c_idx != 0) { - // move the final 1 according to remaining input length - // (We may add less than 2^130 to the last input block) - ctx.c[4] = 0; - poly_take_input(ctx, 1); - // one last hash update - poly_block(ctx); - } - - // check if we should subtract 2^130-5 by performing the - // corresponding carry propagation. - const _u0 = u64(5) + ctx.h[0]; // <= 1_00000004 - const _u1 = (_u0 >> 32) + ctx.h[1]; // <= 1_00000000 - const _u2 = (_u1 >> 32) + ctx.h[2]; // <= 1_00000000 - const _u3 = (_u2 >> 32) + ctx.h[3]; // <= 1_00000000 - const _u4 = (_u3 >> 32) + ctx.h[4]; // <= 5 - // u4 indicates how many times we should subtract 2^130-5 (0 or 1) - - // h + pad, minus 2^130-5 if u4 exceeds 3 - const uu0 = (_u4 >> 2) * 5 + ctx.h[0] + ctx.pad[0]; // <= 2_00000003 - const uu1 = (uu0 >> 32) + ctx.h[1] + ctx.pad[1]; // <= 2_00000000 - const uu2 = (uu1 >> 32) + ctx.h[2] + ctx.pad[2]; // <= 2_00000000 - const uu3 = (uu2 >> 32) + ctx.h[3] + ctx.pad[3]; // <= 2_00000000 - - writeInt(mac[0..], uu0, Endian.Little); - writeInt(mac[4..], uu1, Endian.Little); - writeInt(mac[8..], uu2, Endian.Little); - writeInt(mac[12..], uu3, Endian.Little); - - ctx.secure_zero(); -} - -pub fn crypto_poly1305(mac: []u8, message: []const u8, key: [32]u8) void { - std.debug.assert(mac.len >= 16); - - var ctx: crypto_poly1305_ctx = undefined; - crypto_poly1305_init(&ctx, key); - crypto_poly1305_update(&ctx, message); - crypto_poly1305_final(&ctx, mac); -} +}; test "poly1305 rfc7439 vector1" { const expected_mac = "\xa8\x06\x1d\xc1\x30\x51\x36\xc6\xc2\x2b\x8b\xaf\x0c\x01\x27\xa9"; @@ -214,7 +227,7 @@ test "poly1305 rfc7439 vector1" { "\x01\x03\x80\x8a\xfb\x0d\xb2\xfd\x4a\xbf\xf6\xaf\x41\x49\xf5\x1b"; var mac: [16]u8 = undefined; - crypto_poly1305(mac[0..], msg, key); + Poly1305.create(mac[0..], msg, key); std.debug.assert(std.mem.eql(u8, mac, expected_mac)); } diff --git a/std/crypto/sha1.zig b/std/crypto/sha1.zig index 451cfb3122..6d6b4dbd3f 100644 --- a/std/crypto/sha1.zig +++ b/std/crypto/sha1.zig @@ -26,8 +26,8 @@ fn Rp(a: usize, b: usize, c: usize, d: usize, e: usize, i: u32) RoundParam { pub const Sha1 = struct { const Self = this; - const block_size = 64; - const digest_size = 20; + const block_length = 64; + const digest_length = 20; s: [5]u32, // Streaming Cache @@ -292,8 +292,8 @@ test "sha1 streaming" { } test "sha1 aligned final" { - var block = []u8{0} ** Sha1.block_size; - var out: [Sha1.digest_size]u8 = undefined; + var block = []u8{0} ** Sha1.block_length; + var out: [Sha1.digest_length]u8 = undefined; var h = Sha1.init(); h.update(block); diff --git a/std/crypto/sha2.zig b/std/crypto/sha2.zig index d1b915835c..8a25fecc43 100644 --- a/std/crypto/sha2.zig +++ b/std/crypto/sha2.zig @@ -78,8 +78,8 @@ pub const Sha256 = Sha2_32(Sha256Params); fn Sha2_32(comptime params: Sha2Params32) type { return struct { const Self = this; - const block_size = 64; - const digest_size = params.out_len / 8; + const block_length = 64; + const digest_length = params.out_len / 8; s: [8]u32, // Streaming Cache @@ -338,8 +338,8 @@ test "sha256 streaming" { } test "sha256 aligned final" { - var block = []u8{0} ** Sha256.block_size; - var out: [Sha256.digest_size]u8 = undefined; + var block = []u8{0} ** Sha256.block_length; + var out: [Sha256.digest_length]u8 = undefined; var h = Sha256.init(); h.update(block); @@ -419,8 +419,8 @@ pub const Sha512 = Sha2_64(Sha512Params); fn Sha2_64(comptime params: Sha2Params64) type { return struct { const Self = this; - const block_size = 128; - const digest_size = params.out_len / 8; + const block_length = 128; + const digest_length = params.out_len / 8; s: [8]u64, // Streaming Cache @@ -715,8 +715,8 @@ test "sha512 streaming" { } test "sha512 aligned final" { - var block = []u8{0} ** Sha512.block_size; - var out: [Sha512.digest_size]u8 = undefined; + var block = []u8{0} ** Sha512.block_length; + var out: [Sha512.digest_length]u8 = undefined; var h = Sha512.init(); h.update(block); diff --git a/std/crypto/sha3.zig b/std/crypto/sha3.zig index ae02d7a482..ade615f492 100644 --- a/std/crypto/sha3.zig +++ b/std/crypto/sha3.zig @@ -13,8 +13,8 @@ pub const Sha3_512 = Keccak(512, 0x06); fn Keccak(comptime bits: usize, comptime delim: u8) type { return struct { const Self = this; - const block_size = 200; - const digest_size = bits / 8; + const block_length = 200; + const digest_length = bits / 8; s: [200]u8, offset: usize, @@ -297,8 +297,8 @@ test "sha3-256 streaming" { } test "sha3-256 aligned final" { - var block = []u8{0} ** Sha3_256.block_size; - var out: [Sha3_256.digest_size]u8 = undefined; + var block = []u8{0} ** Sha3_256.block_length; + var out: [Sha3_256.digest_length]u8 = undefined; var h = Sha3_256.init(); h.update(block); @@ -368,8 +368,8 @@ test "sha3-512 streaming" { } test "sha3-512 aligned final" { - var block = []u8{0} ** Sha3_512.block_size; - var out: [Sha3_512.digest_size]u8 = undefined; + var block = []u8{0} ** Sha3_512.block_length; + var out: [Sha3_512.digest_length]u8 = undefined; var h = Sha3_512.init(); h.update(block); diff --git a/std/crypto/x25519.zig b/std/crypto/x25519.zig index b5b85e7104..cf88d6e1c0 100644 --- a/std/crypto/x25519.zig +++ b/std/crypto/x25519.zig @@ -9,6 +9,118 @@ const Endian = builtin.Endian; const readInt = std.mem.readInt; const writeInt = std.mem.writeInt; +// Based on Supercop's ref10 implementation. +pub const X25519 = struct { + pub const secret_length = 32; + pub const minimum_key_length = 32; + + fn trim_scalar(s: []u8) void { + s[0] &= 248; + s[31] &= 127; + s[31] |= 64; + } + + fn scalar_bit(s: []const u8, i: usize) i32 { + return (s[i >> 3] >> @intCast(u3, i & 7)) & 1; + } + + pub fn create(out: []u8, private_key: []const u8, public_key: []const u8) bool { + std.debug.assert(out.len >= secret_length); + std.debug.assert(private_key.len >= minimum_key_length); + std.debug.assert(public_key.len >= minimum_key_length); + + var storage: [7]Fe = undefined; + + var x1 = &storage[0]; + var x2 = &storage[1]; + var z2 = &storage[2]; + var x3 = &storage[3]; + var z3 = &storage[4]; + var t0 = &storage[5]; + var t1 = &storage[6]; + + // computes the scalar product + fe_frombytes(x1, public_key); + + // restrict the possible scalar values + var e: [32]u8 = undefined; + for (e[0..]) |_, i| { + e[i] = private_key[i]; + } + trim_scalar(e[0..]); + + // computes the actual scalar product (the result is in x2 and z2) + + // Montgomery ladder + // In projective coordinates, to avoid divisons: x = X / Z + // We don't care about the y coordinate, it's only 1 bit of information + fe_1(x2); + fe_0(z2); // "zero" point + fe_copy(x3, x1); + fe_1(z3); + + var swap: i32 = 0; + var pos: isize = 254; + while (pos >= 0) : (pos -= 1) { + // constant time conditional swap before ladder step + const b = scalar_bit(e, @intCast(usize, pos)); + swap ^= b; // xor trick avoids swapping at the end of the loop + fe_cswap(x2, x3, swap); + fe_cswap(z2, z3, swap); + swap = b; // anticipates one last swap after the loop + + // Montgomery ladder step: replaces (P2, P3) by (P2*2, P2+P3) + // with differential addition + fe_sub(t0, x3, z3); + fe_sub(t1, x2, z2); + fe_add(x2, x2, z2); + fe_add(z2, x3, z3); + fe_mul(z3, t0, x2); + fe_mul(z2, z2, t1); + fe_sq(t0, t1); + fe_sq(t1, x2); + fe_add(x3, z3, z2); + fe_sub(z2, z3, z2); + fe_mul(x2, t1, t0); + fe_sub(t1, t1, t0); + fe_sq(z2, z2); + fe_mul121666(z3, t1); + fe_sq(x3, x3); + fe_add(t0, t0, z3); + fe_mul(z3, x1, z2); + fe_mul(z2, t1, t0); + } + + // last swap is necessary to compensate for the xor trick + // Note: after this swap, P3 == P2 + P1. + fe_cswap(x2, x3, swap); + fe_cswap(z2, z3, swap); + + // normalises the coordinates: x == X / Z + fe_invert(z2, z2); + fe_mul(x2, x2, z2); + fe_tobytes(out, x2); + + x1.secure_zero(); + x2.secure_zero(); + x3.secure_zero(); + t0.secure_zero(); + t1.secure_zero(); + z2.secure_zero(); + z3.secure_zero(); + std.mem.secureZero(u8, e[0..]); + + // Returns false if the output is all zero + // (happens with some malicious public keys) + return !zerocmp(u8, out); + } + + pub fn createPublicKey(public_key: []const u8, private_key: []const u8) bool { + var base_point = []u8{9} ++ []u8{0} ** 31; + return create(public_key, private_key, base_point); + } +}; + // Constant time compare to zero. fn zerocmp(comptime T: type, a: []const T) bool { var s: T = 0; @@ -144,7 +256,9 @@ fn load24_le(s: []const u8) u32 { return s[0] | (u32(s[1]) << 8) | (u32(s[2]) << 16); } -fn fe_frombytes(h: *Fe, s: [32]u8) void { +fn fe_frombytes(h: *Fe, s: []const u8) void { + std.debug.assert(s.len >= 32); + var t: [10]i64 = undefined; t[0] = readInt(s[0..4], u32, Endian.Little); @@ -469,113 +583,6 @@ fn fe_isnonzero(f: *const Fe) bool { return isneg; } -/////////////// -/// X-25519 /// Taken from Supercop's ref10 implementation. -/////////////// -fn trim_scalar(s: []u8) void { - s[0] &= 248; - s[31] &= 127; - s[31] |= 64; -} - -fn scalar_bit(s: []const u8, i: usize) i32 { - return (s[i >> 3] >> @intCast(u3, i & 7)) & 1; -} - -pub fn crypto_x25519(raw_shared_secret: []u8, your_secret_key: [32]u8, their_public_key: [32]u8) bool { - std.debug.assert(raw_shared_secret.len >= 32); - - var storage: [7]Fe = undefined; - - var x1 = &storage[0]; - var x2 = &storage[1]; - var z2 = &storage[2]; - var x3 = &storage[3]; - var z3 = &storage[4]; - var t0 = &storage[5]; - var t1 = &storage[6]; - - // computes the scalar product - fe_frombytes(x1, their_public_key); - - // restrict the possible scalar values - var e: [32]u8 = undefined; - for (e[0..]) |_, i| { - e[i] = your_secret_key[i]; - } - trim_scalar(e[0..]); - - // computes the actual scalar product (the result is in x2 and z2) - - // Montgomery ladder - // In projective coordinates, to avoid divisons: x = X / Z - // We don't care about the y coordinate, it's only 1 bit of information - fe_1(x2); - fe_0(z2); // "zero" point - fe_copy(x3, x1); - fe_1(z3); - - var swap: i32 = 0; - var pos: isize = 254; - while (pos >= 0) : (pos -= 1) { - // constant time conditional swap before ladder step - const b = scalar_bit(e, @intCast(usize, pos)); - swap ^= b; // xor trick avoids swapping at the end of the loop - fe_cswap(x2, x3, swap); - fe_cswap(z2, z3, swap); - swap = b; // anticipates one last swap after the loop - - // Montgomery ladder step: replaces (P2, P3) by (P2*2, P2+P3) - // with differential addition - fe_sub(t0, x3, z3); - fe_sub(t1, x2, z2); - fe_add(x2, x2, z2); - fe_add(z2, x3, z3); - fe_mul(z3, t0, x2); - fe_mul(z2, z2, t1); - fe_sq(t0, t1); - fe_sq(t1, x2); - fe_add(x3, z3, z2); - fe_sub(z2, z3, z2); - fe_mul(x2, t1, t0); - fe_sub(t1, t1, t0); - fe_sq(z2, z2); - fe_mul121666(z3, t1); - fe_sq(x3, x3); - fe_add(t0, t0, z3); - fe_mul(z3, x1, z2); - fe_mul(z2, t1, t0); - } - - // last swap is necessary to compensate for the xor trick - // Note: after this swap, P3 == P2 + P1. - fe_cswap(x2, x3, swap); - fe_cswap(z2, z3, swap); - - // normalises the coordinates: x == X / Z - fe_invert(z2, z2); - fe_mul(x2, x2, z2); - fe_tobytes(raw_shared_secret, x2); - - x1.secure_zero(); - x2.secure_zero(); - x3.secure_zero(); - t0.secure_zero(); - t1.secure_zero(); - z2.secure_zero(); - z3.secure_zero(); - std.mem.secureZero(u8, e[0..]); - - // Returns false if the output is all zero - // (happens with some malicious public keys) - return !zerocmp(u8, raw_shared_secret); -} - -pub fn crypto_x25519_public_key(public_key: []u8, secret_key: [32]u8) void { - var base_point = []u8{9} ++ []u8{0} ** 31; - crypto_x25519(public_key, secret_key, base_point); -} - test "x25519 rfc7748 vector1" { const secret_key = "\xa5\x46\xe3\x6b\xf0\x52\x7c\x9d\x3b\x16\x15\x4b\x82\x46\x5e\xdd\x62\x14\x4c\x0a\xc1\xfc\x5a\x18\x50\x6a\x22\x44\xba\x44\x9a\xc4"; const public_key = "\xe6\xdb\x68\x67\x58\x30\x30\xdb\x35\x94\xc1\xa4\x24\xb1\x5f\x7c\x72\x66\x24\xec\x26\xb3\x35\x3b\x10\xa9\x03\xa6\xd0\xab\x1c\x4c"; @@ -584,7 +591,7 @@ test "x25519 rfc7748 vector1" { var output: [32]u8 = undefined; - std.debug.assert(crypto_x25519(output[0..], secret_key, public_key)); + std.debug.assert(X25519.create(output[0..], secret_key, public_key)); std.debug.assert(std.mem.eql(u8, output, expected_output)); } @@ -596,7 +603,7 @@ test "x25519 rfc7748 vector2" { var output: [32]u8 = undefined; - std.debug.assert(crypto_x25519(output[0..], secret_key, public_key)); + std.debug.assert(X25519.create(output[0..], secret_key, public_key)); std.debug.assert(std.mem.eql(u8, output, expected_output)); } @@ -610,7 +617,7 @@ test "x25519 rfc7748 one iteration" { var i: usize = 0; while (i < 1) : (i += 1) { var output: [32]u8 = undefined; - std.debug.assert(crypto_x25519(output[0..], k, u)); + std.debug.assert(X25519.create(output[0..], k, u)); std.mem.copy(u8, u[0..], k[0..]); std.mem.copy(u8, k[0..], output[0..]); @@ -634,7 +641,7 @@ test "x25519 rfc7748 1,000 iterations" { var i: usize = 0; while (i < 1000) : (i += 1) { var output: [32]u8 = undefined; - std.debug.assert(crypto_x25519(output[0..], k, u)); + std.debug.assert(X25519.create(output[0..], k, u)); std.mem.copy(u8, u[0..], k[0..]); std.mem.copy(u8, k[0..], output[0..]); @@ -657,7 +664,7 @@ test "x25519 rfc7748 1,000,000 iterations" { var i: usize = 0; while (i < 1000000) : (i += 1) { var output: [32]u8 = undefined; - std.debug.assert(crypto_x25519(output[0..], k, u)); + std.debug.assert(X25519.create(output[0..], k, u)); std.mem.copy(u8, u[0..], k[0..]); std.mem.copy(u8, k[0..], output[0..]); From 38399941d41f8edba9c297a7c0e3da6deb44766c Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Fri, 31 Aug 2018 18:43:42 +1200 Subject: [PATCH 30/56] std/crypto: Update throughput_test.zig to include all hash functions This avoids the need to recompile to test specific hash functions. This also adds mac/key exchange performance tests as well. --- std/crypto/throughput_test.zig | 204 +++++++++++++++++++++++++++++---- 1 file changed, 182 insertions(+), 22 deletions(-) diff --git a/std/crypto/throughput_test.zig b/std/crypto/throughput_test.zig index c21838e607..5f25c0b885 100644 --- a/std/crypto/throughput_test.zig +++ b/std/crypto/throughput_test.zig @@ -1,38 +1,198 @@ -// Modify the HashFunction variable to the one wanted to test. -// -// ``` -// zig build-exe --release-fast throughput_test.zig -// ./throughput_test -// ``` - +const builtin = @import("builtin"); const std = @import("std"); const time = std.os.time; const Timer = time.Timer; -const HashFunction = @import("md5.zig").Md5; +const crypto = @import("index.zig"); -const MiB = 1024 * 1024; -const BytesToHash = 1024 * MiB; +const KiB = 1024; +const MiB = 1024 * KiB; + +var prng = std.rand.DefaultPrng.init(0); + +const Crypto = struct { + ty: type, + name: []const u8, +}; + +const hashes = []Crypto{ + Crypto{ .ty = crypto.Md5, .name = "md5" }, + Crypto{ .ty = crypto.Sha1, .name = "sha1" }, + Crypto{ .ty = crypto.Sha256, .name = "sha256" }, + Crypto{ .ty = crypto.Sha512, .name = "sha512" }, + Crypto{ .ty = crypto.Sha3_256, .name = "sha3-256" }, + Crypto{ .ty = crypto.Sha3_512, .name = "sha3-512" }, + Crypto{ .ty = crypto.Blake2s256, .name = "blake2s" }, + Crypto{ .ty = crypto.Blake2b512, .name = "blake2b" }, +}; + +pub fn benchmarkHash(comptime Hash: var, comptime bytes: comptime_int) !u64 { + var h = Hash.init(); + + var block: [Hash.digest_length]u8 = undefined; + prng.random.bytes(block[0..]); + + var offset: usize = 0; + var timer = try Timer.start(); + const start = timer.lap(); + while (offset < bytes) : (offset += block.len) { + h.update(block[0..]); + } + const end = timer.read(); + + const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s; + const throughput = @floatToInt(u64, bytes / elapsed_s); + + return throughput; +} + +const macs = []Crypto{ + Crypto{ .ty = crypto.Poly1305, .name = "poly1305" }, + Crypto{ .ty = crypto.HmacMd5, .name = "hmac-md5" }, + Crypto{ .ty = crypto.HmacSha1, .name = "hmac-sha1" }, + Crypto{ .ty = crypto.HmacSha256, .name = "hmac-sha256" }, +}; + +pub fn benchmarkMac(comptime Mac: var, comptime bytes: comptime_int) !u64 { + std.debug.assert(32 >= Mac.mac_length and 32 >= Mac.minimum_key_length); + + var in: [1 * MiB]u8 = undefined; + prng.random.bytes(in[0..]); + + var key: [32]u8 = undefined; + prng.random.bytes(key[0..]); + + var offset: usize = 0; + var timer = try Timer.start(); + const start = timer.lap(); + while (offset < bytes) : (offset += in.len) { + Mac.create(key[0..], in[0..], key); + } + const end = timer.read(); + + const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s; + const throughput = @floatToInt(u64, bytes / elapsed_s); + + return throughput; +} + +const exchanges = []Crypto{Crypto{ .ty = crypto.X25519, .name = "x25519" }}; + +pub fn benchmarkKeyExchange(comptime DhKeyExchange: var, comptime exchange_count: comptime_int) !u64 { + std.debug.assert(DhKeyExchange.minimum_key_length >= DhKeyExchange.secret_length); + + var in: [DhKeyExchange.minimum_key_length]u8 = undefined; + prng.random.bytes(in[0..]); + + var out: [DhKeyExchange.minimum_key_length]u8 = undefined; + prng.random.bytes(out[0..]); + + var offset: usize = 0; + var timer = try Timer.start(); + const start = timer.lap(); + { + var i: usize = 0; + while (i < exchange_count) : (i += 1) { + _ = DhKeyExchange.create(out[0..], out, in); + } + } + const end = timer.read(); + + const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s; + const throughput = @floatToInt(u64, exchange_count / elapsed_s); + + return throughput; +} + +fn usage() void { + std.debug.warn( + \\throughput_test [options] + \\ + \\Options: + \\ --filter [test-name] + \\ --seed [int] + \\ --help + \\ + ); +} + +fn mode(comptime x: comptime_int) comptime_int { + return if (builtin.mode == builtin.Mode.Debug) x / 64 else x; +} + +// TODO(#1358): Replace with builtin formatted padding when available. +fn printPad(stdout: var, s: []const u8) !void { + var i: usize = 0; + while (i < 12 - s.len) : (i += 1) { + try stdout.print(" "); + } + try stdout.print("{}", s); +} pub fn main() !void { var stdout_file = try std.io.getStdOut(); var stdout_out_stream = std.io.FileOutStream.init(&stdout_file); const stdout = &stdout_out_stream.stream; - var block: [HashFunction.block_size]u8 = undefined; - std.mem.set(u8, block[0..], 0); + var buffer: [1024]u8 = undefined; + var fixed = std.heap.FixedBufferAllocator.init(buffer[0..]); + const args = try std.os.argsAlloc(&fixed.allocator); - var h = HashFunction.init(); - var offset: usize = 0; + var filter: ?[]u8 = ""; - var timer = try Timer.start(); - const start = timer.lap(); - while (offset < BytesToHash) : (offset += block.len) { - h.update(block[0..]); + var i: usize = 1; + while (i < args.len) : (i += 1) { + if (std.mem.eql(u8, args[i], "--seed")) { + i += 1; + if (i == args.len) { + usage(); + std.os.exit(1); + } + + const seed = try std.fmt.parseUnsigned(u32, args[i], 10); + prng.seed(seed); + } else if (std.mem.eql(u8, args[i], "--filter")) { + i += 1; + if (i == args.len) { + usage(); + std.os.exit(1); + } + + filter = args[i]; + } else if (std.mem.eql(u8, args[i], "--help")) { + usage(); + return; + } else { + usage(); + std.os.exit(1); + } } - const end = timer.read(); - const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s; - const throughput = @floatToInt(u64, BytesToHash / elapsed_s); + inline for (hashes) |H| { + // TODO: Inverted early continue case here segfaults compiler. Create reduced test case. + // + // if (filter != null and std.mem.indexOf(u8, H.name, filter.?) == null) { + // continue; + // } + if (filter == null or std.mem.indexOf(u8, H.name, filter.?) != null) { + const throughput = try benchmarkHash(H.ty, mode(32 * MiB)); + try printPad(stdout, H.name); + try stdout.print(": {} MiB/s\n", throughput / (1 * MiB)); + } + } - try stdout.print("{}: {} MiB/s\n", @typeName(HashFunction), throughput / (1 * MiB)); + inline for (macs) |M| { + if (filter == null or std.mem.indexOf(u8, M.name, filter.?) != null) { + const throughput = try benchmarkMac(M.ty, mode(128 * MiB)); + try printPad(stdout, M.name); + try stdout.print(": {} MiB/s\n", throughput / (1 * MiB)); + } + } + + inline for (exchanges) |E| { + if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { + const throughput = try benchmarkKeyExchange(E.ty, mode(1000)); + try printPad(stdout, E.name); + try stdout.print(": {} exchanges/s\n", throughput); + } + } } From 763845f95c33140859aa0297c9495d5f08f9afea Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Fri, 31 Aug 2018 18:45:45 +1200 Subject: [PATCH 31/56] std/crypto: zig fmt --- std/crypto/chacha20.zig | 57 ++++++++++++++------------ std/crypto/sha3.zig | 91 ++++------------------------------------- 2 files changed, 39 insertions(+), 109 deletions(-) diff --git a/std/crypto/chacha20.zig b/std/crypto/chacha20.zig index 836c8c8878..743d1033e3 100644 --- a/std/crypto/chacha20.zig +++ b/std/crypto/chacha20.zig @@ -32,24 +32,28 @@ fn salsa20_wordtobyte(out: []u8, input: [16]u32) void { x[i] = input[i]; const rounds = comptime []QuarterRound{ - Rp( 0, 4, 8,12), - Rp( 1, 5, 9,13), - Rp( 2, 6,10,14), - Rp( 3, 7,11,15), - Rp( 0, 5,10,15), - Rp( 1, 6,11,12), - Rp( 2, 7, 8,13), - Rp( 3, 4, 9,14), + Rp(0, 4, 8, 12), + Rp(1, 5, 9, 13), + Rp(2, 6, 10, 14), + Rp(3, 7, 11, 15), + Rp(0, 5, 10, 15), + Rp(1, 6, 11, 12), + Rp(2, 7, 8, 13), + Rp(3, 4, 9, 14), }; comptime var j: usize = 0; inline while (j < 20) : (j += 2) { // two-round cycles inline for (rounds) |r| { - x[r.a] +%= x[r.b]; x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], u32(16)); - x[r.c] +%= x[r.d]; x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], u32(12)); - x[r.a] +%= x[r.b]; x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], u32(8)); - x[r.c] +%= x[r.d]; x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], u32(7)); + x[r.a] +%= x[r.b]; + x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], u32(16)); + x[r.c] +%= x[r.d]; + x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], u32(12)); + x[r.a] +%= x[r.b]; + x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], u32(8)); + x[r.c] +%= x[r.d]; + x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], u32(7)); } } @@ -166,9 +170,8 @@ pub fn chaCha20With64BitNonce(out: []u8, in: []const u8, counter: u64, key: [32] var remaining_blocks: u32 = @intCast(u32, (in.len / big_block)); var i: u32 = 0; while (remaining_blocks > 0) : (remaining_blocks -= 1) { - chaCha20_internal(out[cursor..cursor + big_block], in[cursor..cursor + big_block], k, c); - c[1] += 1; // upper 32-bit of counter, generic chaCha20_internal() doesn't - // know about this. + chaCha20_internal(out[cursor .. cursor + big_block], in[cursor .. cursor + big_block], k, c); + c[1] += 1; // upper 32-bit of counter, generic chaCha20_internal() doesn't know about this. cursor += big_block; } } @@ -199,16 +202,16 @@ test "crypto.chacha20 test vector sunscreen" { const input = "Ladies and Gentlemen of the class of '99: If I could offer you only one tip for the future, sunscreen would be it."; var result: [114]u8 = undefined; const key = []u8{ - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9,10,11,12,13,14,15, - 16,17,18,19,20,21,22,23, - 24,25,26,27,28,29,30,31, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, }; const nonce = []u8{ - 0, 0, 0, 0, - 0, 0, 0, 0x4a, - 0, 0, 0, 0, - }; + 0, 0, 0, 0, + 0, 0, 0, 0x4a, + 0, 0, 0, 0, + }; chaCha20IETF(result[0..], input[0..], 1, key, nonce); assert(mem.eql(u8, expected_result, result)); @@ -248,7 +251,7 @@ test "crypto.chacha20 test vector 1" { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; - const nonce = []u8{0, 0, 0, 0, 0, 0, 0, 0}; + const nonce = []u8{ 0, 0, 0, 0, 0, 0, 0, 0 }; chaCha20With64BitNonce(result[0..], input[0..], 0, key, nonce); assert(mem.eql(u8, expected_result, result)); @@ -282,7 +285,7 @@ test "crypto.chacha20 test vector 2" { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, }; - const nonce = []u8{0, 0, 0, 0, 0, 0, 0, 0}; + const nonce = []u8{ 0, 0, 0, 0, 0, 0, 0, 0 }; chaCha20With64BitNonce(result[0..], input[0..], 0, key, nonce); assert(mem.eql(u8, expected_result, result)); @@ -316,7 +319,7 @@ test "crypto.chacha20 test vector 3" { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; - const nonce = []u8{0, 0, 0, 0, 0, 0, 0, 1}; + const nonce = []u8{ 0, 0, 0, 0, 0, 0, 0, 1 }; chaCha20With64BitNonce(result[0..], input[0..], 0, key, nonce); assert(mem.eql(u8, expected_result, result)); @@ -350,7 +353,7 @@ test "crypto.chacha20 test vector 4" { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; - const nonce = []u8{1, 0, 0, 0, 0, 0, 0, 0}; + const nonce = []u8{ 1, 0, 0, 0, 0, 0, 0, 0 }; chaCha20With64BitNonce(result[0..], input[0..], 0, key, nonce); assert(mem.eql(u8, expected_result, result)); diff --git a/std/crypto/sha3.zig b/std/crypto/sha3.zig index ade615f492..827bbd0680 100644 --- a/std/crypto/sha3.zig +++ b/std/crypto/sha3.zig @@ -87,97 +87,24 @@ fn Keccak(comptime bits: usize, comptime delim: u8) type { } const RC = []const u64{ - 0x0000000000000001, - 0x0000000000008082, - 0x800000000000808a, - 0x8000000080008000, - 0x000000000000808b, - 0x0000000080000001, - 0x8000000080008081, - 0x8000000000008009, - 0x000000000000008a, - 0x0000000000000088, - 0x0000000080008009, - 0x000000008000000a, - 0x000000008000808b, - 0x800000000000008b, - 0x8000000000008089, - 0x8000000000008003, - 0x8000000000008002, - 0x8000000000000080, - 0x000000000000800a, - 0x800000008000000a, - 0x8000000080008081, - 0x8000000000008080, - 0x0000000080000001, - 0x8000000080008008, + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, 0x8000000080008000, + 0x000000000000808b, 0x0000000080000001, 0x8000000080008081, 0x8000000000008009, + 0x000000000000008a, 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, 0x8000000000008003, + 0x8000000000008002, 0x8000000000000080, 0x000000000000800a, 0x800000008000000a, + 0x8000000080008081, 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, }; const ROTC = []const usize{ - 1, - 3, - 6, - 10, - 15, - 21, - 28, - 36, - 45, - 55, - 2, - 14, - 27, - 41, - 56, - 8, - 25, - 43, - 62, - 18, - 39, - 61, - 20, - 44, + 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44, }; const PIL = []const usize{ - 10, - 7, - 11, - 17, - 18, - 3, - 5, - 16, - 8, - 21, - 24, - 4, - 15, - 23, - 19, - 13, - 12, - 2, - 20, - 14, - 22, - 9, - 6, - 1, + 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1, }; const M5 = []const usize{ - 0, - 1, - 2, - 3, - 4, - 0, - 1, - 2, - 3, - 4, + 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, }; fn keccak_f(comptime F: usize, d: []u8) void { From b36b93fb3ef3ce96ecc457d0e379740b39d37f80 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 31 Aug 2018 15:02:41 -0400 Subject: [PATCH 32/56] awareness of debug subsections --- std/debug/index.zig | 28 ++++++ std/pdb.zig | 237 +++++++++++++++++++++----------------------- 2 files changed, 142 insertions(+), 123 deletions(-) diff --git a/std/debug/index.zig b/std/debug/index.zig index 5bdfb52186..3b32eafb01 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -451,6 +451,34 @@ fn openSelfDebugInfoWindows(allocator: *mem.Allocator) !DebugInfo { std.debug.warn("v {} s {} a {}\n", version, signature, age); // We validated the executable and pdb match. + const name_bytes_len = try pdb_stream.stream.readIntLe(u32); + const name_bytes = try allocator.alloc(u8, name_bytes_len); + try pdb_stream.stream.readNoEof(name_bytes); + + //const HashTableHeader = packed struct { + // Size: u32, + // Capacity: u32, + + // fn maxLoad(cap: u32) u32 { + // return cap * 2 / 3 + 1; + // } + //}; + //var hash_tbl_hdr: HashTableHeader = undefined; + //try pdb_stream.stream.readStruct(Header, &hash_tbl_hdr); + //if (hash_tbl_hdr.Capacity == 0) + // return error.InvalidDebugInfo; + + //if (hash_tbl_hdr.Size > HashTableHeader.maxLoad(hash_tbl_hdr.Capacity)) + // return error.InvalidDebugInfo; + + //std.debug.warn("{}\n", hash_tbl_hdr); + + //var more_buf: [100]u8 = undefined; + //const more_len = try pdb_stream.stream.read(more_buf[0..]); + //for (more_buf[0..more_len]) |x| { + // std.debug.warn("{x2} {c}\n", x, x); + //} + return di; } diff --git a/std/pdb.zig b/std/pdb.zig index c1fb863b67..1ee9b23383 100644 --- a/std/pdb.zig +++ b/std/pdb.zig @@ -92,65 +92,6 @@ const Module = struct { obj_file_name: []u8, }; -/// Distinguishes individual records in the Symbols subsection of a .debug$S -/// section. Equivalent to SYM_ENUM_e in cvinfo.h. -pub const SymbolRecordKind = enum(u16) { - InlineesSym = 4456, - ScopeEndSym = 6, - InlineSiteEnd = 4430, - ProcEnd = 4431, - Thunk32Sym = 4354, - TrampolineSym = 4396, - SectionSym = 4406, - CoffGroupSym = 4407, - ExportSym = 4408, - ProcSym = 4367, - GlobalProcSym = 4368, - ProcIdSym = 4422, - GlobalProcIdSym = 4423, - DPCProcSym = 4437, - DPCProcIdSym = 4438, - RegisterSym = 4358, - PublicSym32 = 4366, - ProcRefSym = 4389, - LocalProcRef = 4391, - EnvBlockSym = 4413, - InlineSiteSym = 4429, - LocalSym = 4414, - DefRangeSym = 4415, - DefRangeSubfieldSym = 4416, - DefRangeRegisterSym = 4417, - DefRangeFramePointerRelSym = 4418, - DefRangeSubfieldRegisterSym = 4419, - DefRangeFramePointerRelFullScopeSym = 4420, - DefRangeRegisterRelSym = 4421, - BlockSym = 4355, - LabelSym = 4357, - ObjNameSym = 4353, - Compile2Sym = 4374, - Compile3Sym = 4412, - FrameProcSym = 4114, - CallSiteInfoSym = 4409, - FileStaticSym = 4435, - HeapAllocationSiteSym = 4446, - FrameCookieSym = 4410, - CallerSym = 4442, - CalleeSym = 4443, - UDTSym = 4360, - CobolUDT = 4361, - BuildInfoSym = 4428, - BPRelativeSym = 4363, - RegRelativeSym = 4369, - ConstantSym = 4359, - ManagedConstant = 4397, - DataSym = 4364, - GlobalData = 4365, - ManagedLocalData = 4380, - ManagedGlobalData = 4381, - ThreadLocalDataSym = 4370, - GlobalTLS = 4371, -}; - /// Duplicate copy of SymbolRecordKind, but using the official CV names. Useful /// for reference purposes and when dealing with unknown record types. pub const SymbolKind = packed enum(u16) { @@ -434,6 +375,32 @@ const ColumnNumberEntry = packed struct { EndColumn: u16, }; +/// Checksum bytes follow. +const FileChecksumEntryHeader = packed struct { + FileNameOffset: u32, /// Byte offset of filename in global string table. + ChecksumSize: u8, /// Number of bytes of checksum. + ChecksumKind: u8, /// FileChecksumKind +}; + +const DebugSubsectionKind = packed enum(u32) { + None = 0, + Symbols = 0xf1, + Lines = 0xf2, + StringTable = 0xf3, + FileChecksums = 0xf4, + FrameData = 0xf5, + InlineeLines = 0xf6, + CrossScopeImports = 0xf7, + CrossScopeExports = 0xf8, + + // These appear to relate to .Net assembly info. + ILLines = 0xf9, + FuncMDTokenMap = 0xfa, + TypeMDTokenMap = 0xfb, + MergedAssemblyInput = 0xfc, + + CoffSymbolRVA = 0xfd, +}; pub const Pdb = struct { in_file: os.File, @@ -569,14 +536,17 @@ pub const Pdb = struct { const prefix = @ptrCast(*RecordPrefix, &symbols[symbol_i]); if (prefix.RecordLen < 2) return error.InvalidDebugInfo; - if (prefix.RecordKind == SymbolKind.S_LPROC32) { - const proc_sym = @ptrCast(*ProcSym, &symbols[symbol_i + @sizeOf(RecordPrefix)]); - const vaddr_start = coff_section.header.virtual_address + proc_sym.CodeOffset; - const vaddr_end = vaddr_start + proc_sym.CodeSize; - std.debug.warn(" {}\n", proc_sym); - if (address >= vaddr_start and address < vaddr_end) { - break proc_sym; - } + switch (prefix.RecordKind) { + SymbolKind.S_LPROC32 => { + const proc_sym = @ptrCast(*ProcSym, &symbols[symbol_i + @sizeOf(RecordPrefix)]); + const vaddr_start = coff_section.header.virtual_address + proc_sym.CodeOffset; + const vaddr_end = vaddr_start + proc_sym.CodeSize; + std.debug.warn(" {}\n", proc_sym); + if (address >= vaddr_start and address < vaddr_end) { + break proc_sym; + } + }, + else => {}, } symbol_i += prefix.RecordLen + @sizeOf(u16); if (symbol_i > symbols.len) @@ -592,73 +562,94 @@ pub const Pdb = struct { return error.MissingDebugInfo; } - const line_info = try self.allocator.alloc(u8, mod.mod_info.C13ByteSize); - std.debug.warn("read C13 line info {} bytes\n", line_info.len); - try modi.stream.readNoEof(line_info); + const subsect_info = try self.allocator.alloc(u8, mod.mod_info.C13ByteSize); + std.debug.warn("read C13 line info {} bytes\n", subsect_info.len); + const line_info_file_pos = modi.getFilePos(); + try modi.stream.readNoEof(subsect_info); - var line_index: usize = 0; - while (line_index != line_info.len) { - std.debug.warn("unknown bytes: {x2} {x2} {x2} {x2} {x2} {x2} {x2} {x2}\n", - line_info[line_index+0], - line_info[line_index+1], - line_info[line_index+2], - line_info[line_index+3], - line_info[line_index+4], - line_info[line_index+5], - line_info[line_index+6], - line_info[line_index+7], - ); - line_index += 8; + const DebugSubsectionHeader = packed struct { + Kind: DebugSubsectionKind, /// codeview::DebugSubsectionKind enum + Length: u32, /// number of bytes occupied by this record. + }; + var sect_offset: usize = 0; + var skip_len: usize = undefined; + var have_line_info: bool = false; + subsections: while (sect_offset != subsect_info.len) : (sect_offset += skip_len) { + const subsect_hdr = @ptrCast(*DebugSubsectionHeader, &subsect_info[sect_offset]); + skip_len = subsect_hdr.Length; + sect_offset += @sizeOf(DebugSubsectionHeader); - const line_hdr = @ptrCast(*LineFragmentHeader, &line_info[line_index]); - if (line_hdr.RelocSegment == 0) return error.MissingDebugInfo; - std.debug.warn("{}\n", line_hdr); - line_index += @sizeOf(LineFragmentHeader); + switch (subsect_hdr.Kind) { + DebugSubsectionKind.Lines => { + if (have_line_info) + continue :subsections; - const block_hdr = @ptrCast(*LineBlockFragmentHeader, &line_info[line_index]); - std.debug.warn("{}\n", block_hdr); - line_index += @sizeOf(LineBlockFragmentHeader); + var line_index: usize = sect_offset; - const has_column = line_hdr.Flags.LF_HaveColumns; - std.debug.warn("has column: {}\n", has_column); + const line_hdr = @ptrCast(*LineFragmentHeader, &subsect_info[line_index]); + if (line_hdr.RelocSegment == 0) return error.MissingDebugInfo; + std.debug.warn("{}\n", line_hdr); + line_index += @sizeOf(LineFragmentHeader); - const frag_vaddr_start = coff_section.header.virtual_address + line_hdr.RelocOffset; - const frag_vaddr_end = frag_vaddr_start + line_hdr.CodeSize; - if (address >= frag_vaddr_start and address < frag_vaddr_end) { - std.debug.warn("found line listing\n"); - var line_i: usize = 0; - const start_line_index = line_index; - while (line_i < block_hdr.NumLines) : (line_i += 1) { - const line_num_entry = @ptrCast(*LineNumberEntry, &line_info[line_index]); - line_index += @sizeOf(LineNumberEntry); - const flags = @ptrCast(*LineNumberEntry.Flags, &line_num_entry.Flags); - std.debug.warn("{} {}\n", line_num_entry, flags); - const vaddr_start = frag_vaddr_start + line_num_entry.Offset; - const vaddr_end = if (flags.End == 0) frag_vaddr_end else vaddr_start + flags.End; - std.debug.warn("test {x} <= {x} < {x}\n", vaddr_start, address, vaddr_end); - if (address >= vaddr_start and address < vaddr_end) { - std.debug.warn("{} line {}\n", block_hdr.NameIndex, flags.Start); - if (has_column) { - line_index = start_line_index + @sizeOf(LineNumberEntry) * block_hdr.NumLines; - line_index += @sizeOf(ColumnNumberEntry) * line_i; - const col_num_entry = @ptrCast(*ColumnNumberEntry, &line_info[line_index]); - std.debug.warn("col {}\n", col_num_entry.StartColumn); + const block_hdr = @ptrCast(*LineBlockFragmentHeader, &subsect_info[line_index]); + std.debug.warn("{}\n", block_hdr); + line_index += @sizeOf(LineBlockFragmentHeader); + + const has_column = line_hdr.Flags.LF_HaveColumns; + std.debug.warn("has column: {}\n", has_column); + + const frag_vaddr_start = coff_section.header.virtual_address + line_hdr.RelocOffset; + const frag_vaddr_end = frag_vaddr_start + line_hdr.CodeSize; + if (address >= frag_vaddr_start and address < frag_vaddr_end) { + std.debug.warn("found line listing\n"); + var line_i: usize = 0; + const start_line_index = line_index; + while (line_i < block_hdr.NumLines) : (line_i += 1) { + const line_num_entry = @ptrCast(*LineNumberEntry, &subsect_info[line_index]); + line_index += @sizeOf(LineNumberEntry); + const flags = @ptrCast(*LineNumberEntry.Flags, &line_num_entry.Flags); + std.debug.warn("{} {}\n", line_num_entry, flags); + const vaddr_start = frag_vaddr_start + line_num_entry.Offset; + const vaddr_end = if (flags.End == 0) frag_vaddr_end else vaddr_start + flags.End; + std.debug.warn("test {x} <= {x} < {x}\n", vaddr_start, address, vaddr_end); + if (address >= vaddr_start and address < vaddr_end) { + std.debug.warn("{} line {}\n", block_hdr.NameIndex, flags.Start); + if (has_column) { + line_index = start_line_index + @sizeOf(LineNumberEntry) * block_hdr.NumLines; + line_index += @sizeOf(ColumnNumberEntry) * line_i; + const col_num_entry = @ptrCast(*ColumnNumberEntry, &subsect_info[line_index]); + std.debug.warn("col {}\n", col_num_entry.StartColumn); + } + have_line_info = true; + continue :subsections; + } } - return; + return error.MissingDebugInfo; } - } - return error.MissingDebugInfo; - } else { - line_index += @sizeOf(LineNumberEntry) * block_hdr.NumLines; - if (has_column) - line_index += @sizeOf(ColumnNumberEntry) * block_hdr.NumLines; + + }, + DebugSubsectionKind.FileChecksums => { + var chksum_index: usize = sect_offset; + + while (chksum_index != subsect_info.len) { + const chksum_hdr = @ptrCast(*FileChecksumEntryHeader, &subsect_info[chksum_index]); + std.debug.warn("{}\n", chksum_hdr); + const len = @sizeOf(FileChecksumEntryHeader) + chksum_hdr.ChecksumSize; + chksum_index += len + (len % 4); + if (chksum_index > subsect_info.len) + return error.InvalidDebugInfo; + } + + }, + else => { + std.debug.warn("ignore subsection {}\n", @tagName(subsect_hdr.Kind)); + }, } - if (line_index > mod.mod_info.C13ByteSize) + if (sect_offset > subsect_info.len) return error.InvalidDebugInfo; } - - std.debug.warn("end line info\n"); + std.debug.warn("end subsections\n"); } }; From e036f65ac0df91b03dc6bcda8b043484321c6857 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 31 Aug 2018 23:17:17 +0200 Subject: [PATCH 33/56] Translate-c: Check for error before working on while loop body (#1445) --- src/translate_c.cpp | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/translate_c.cpp b/src/translate_c.cpp index 735a671bcc..12c474cd53 100644 --- a/src/translate_c.cpp +++ b/src/translate_c.cpp @@ -3023,12 +3023,19 @@ static int trans_stmt_extra(Context *c, TransScope *scope, const Stmt *stmt, trans_unary_operator(c, result_used, scope, (const UnaryOperator *)stmt)); case Stmt::DeclStmtClass: return trans_local_declaration(c, scope, (const DeclStmt *)stmt, out_node, out_child_scope); + case Stmt::DoStmtClass: case Stmt::WhileStmtClass: { - AstNode *while_node = trans_while_loop(c, scope, (const WhileStmt *)stmt); + AstNode *while_node = sc == Stmt::DoStmtClass + ? trans_do_loop(c, scope, (const DoStmt *)stmt) + : trans_while_loop(c, scope, (const WhileStmt *)stmt); + + if (while_node == nullptr) + return ErrorUnexpected; + assert(while_node->type == NodeTypeWhileExpr); - if (while_node->data.while_expr.body == nullptr) { + if (while_node->data.while_expr.body == nullptr) while_node->data.while_expr.body = trans_create_node(c, NodeTypeBlock); - } + return wrap_stmt(out_node, out_child_scope, scope, while_node); } case Stmt::IfStmtClass: @@ -3053,14 +3060,6 @@ static int trans_stmt_extra(Context *c, TransScope *scope, const Stmt *stmt, case Stmt::UnaryExprOrTypeTraitExprClass: return wrap_stmt(out_node, out_child_scope, scope, trans_unary_expr_or_type_trait_expr(c, scope, (const UnaryExprOrTypeTraitExpr *)stmt)); - case Stmt::DoStmtClass: { - AstNode *while_node = trans_do_loop(c, scope, (const DoStmt *)stmt); - assert(while_node->type == NodeTypeWhileExpr); - if (while_node->data.while_expr.body == nullptr) { - while_node->data.while_expr.body = trans_create_node(c, NodeTypeBlock); - } - return wrap_stmt(out_node, out_child_scope, scope, while_node); - } case Stmt::ForStmtClass: { AstNode *node = trans_for_loop(c, scope, (const ForStmt *)stmt); return wrap_stmt(out_node, out_child_scope, scope, node); From 6ddbd345aa085291f540e6d1b436edad32fe9a69 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 31 Aug 2018 19:50:03 -0400 Subject: [PATCH 34/56] figuring out where /names stream is --- std/debug/index.zig | 70 +++++++++++++++++++++++++++++++++------------ std/pdb.zig | 1 + 2 files changed, 52 insertions(+), 19 deletions(-) diff --git a/std/debug/index.zig b/std/debug/index.zig index 3b32eafb01..2883465931 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -455,33 +455,65 @@ fn openSelfDebugInfoWindows(allocator: *mem.Allocator) !DebugInfo { const name_bytes = try allocator.alloc(u8, name_bytes_len); try pdb_stream.stream.readNoEof(name_bytes); - //const HashTableHeader = packed struct { - // Size: u32, - // Capacity: u32, + const HashTableHeader = packed struct { + Size: u32, + Capacity: u32, - // fn maxLoad(cap: u32) u32 { - // return cap * 2 / 3 + 1; - // } - //}; - //var hash_tbl_hdr: HashTableHeader = undefined; - //try pdb_stream.stream.readStruct(Header, &hash_tbl_hdr); - //if (hash_tbl_hdr.Capacity == 0) - // return error.InvalidDebugInfo; + fn maxLoad(cap: u32) u32 { + return cap * 2 / 3 + 1; + } + }; + var hash_tbl_hdr: HashTableHeader = undefined; + try pdb_stream.stream.readStruct(HashTableHeader, &hash_tbl_hdr); + if (hash_tbl_hdr.Capacity == 0) + return error.InvalidDebugInfo; - //if (hash_tbl_hdr.Size > HashTableHeader.maxLoad(hash_tbl_hdr.Capacity)) - // return error.InvalidDebugInfo; + if (hash_tbl_hdr.Size > HashTableHeader.maxLoad(hash_tbl_hdr.Capacity)) + return error.InvalidDebugInfo; - //std.debug.warn("{}\n", hash_tbl_hdr); + std.debug.warn("{}\n", hash_tbl_hdr); - //var more_buf: [100]u8 = undefined; - //const more_len = try pdb_stream.stream.read(more_buf[0..]); - //for (more_buf[0..more_len]) |x| { - // std.debug.warn("{x2} {c}\n", x, x); - //} + const present = try readSparseBitVector(&pdb_stream.stream, allocator); + if (present.len != hash_tbl_hdr.Size) + return error.InvalidDebugInfo; + const deleted = try readSparseBitVector(&pdb_stream.stream, allocator); + + const Bucket = struct { + first: u32, + second: u32, + }; + const bucket_list = try allocator.alloc(Bucket, present.len); + const string_table_index = for (present) |_| { + const name_offset = try pdb_stream.stream.readIntLe(u32); + const name_index = try pdb_stream.stream.readIntLe(u32); + const name = mem.toSlice(u8, name_bytes.ptr + name_offset); + if (mem.eql(u8, name, "/names")) { + break name_index; + } + } else return error.MissingDebugInfo; + + di.pdb.string_table = di.pdb.getStreamById(string_table_index) orelse return error.InvalidDebugInfo; return di; } +fn readSparseBitVector(stream: var, allocator: *mem.Allocator) ![]usize { + const num_words = try stream.readIntLe(u32); + var word_i: usize = 0; + var list = ArrayList(usize).init(allocator); + while (word_i != num_words) : (word_i += 1) { + const word = try stream.readIntLe(u32); + var bit_i: u5 = 0; + while (true) : (bit_i += 1) { + if (word & (u32(1) << bit_i) != 0) { + try list.append(word_i * 32 + bit_i); + } + if (bit_i == @maxValue(u5)) break; + } + } + return list.toOwnedSlice(); +} + fn openSelfDebugInfoLinux(allocator: *mem.Allocator) !DebugInfo { var di = DebugInfo{ .self_exe_file = undefined, diff --git a/std/pdb.zig b/std/pdb.zig index 1ee9b23383..3aefc4f724 100644 --- a/std/pdb.zig +++ b/std/pdb.zig @@ -406,6 +406,7 @@ pub const Pdb = struct { in_file: os.File, allocator: *mem.Allocator, coff: *coff.Coff, + string_table: *MsfStream, msf: Msf, From d4474e195e21ebc935958a32f945d7ddff087ea4 Mon Sep 17 00:00:00 2001 From: kristopher tate Date: Sat, 1 Sep 2018 12:02:04 +0900 Subject: [PATCH 35/56] i#1438: src/windows_sdk.cpp: fix version guard in `find_10_version`; --- src/windows_sdk.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/windows_sdk.cpp b/src/windows_sdk.cpp index 0f9d0fc301..a63ccceb75 100644 --- a/src/windows_sdk.cpp +++ b/src/windows_sdk.cpp @@ -204,7 +204,11 @@ static ZigFindWindowsSdkError find_10_version(ZigWindowsSDKPrivate *priv) { // https://developer.microsoft.com/en-us/windows/downloads/sdk-archive c2 = 26624; } - if ((c0 > v0) || (c1 > v1) || (c2 > v2) || (c3 > v3)) { + + if ( (c0 > v0) + || (c0 == v0 && c1 > v1) + || (c0 == v0 && c1 == v1 && c2 > v2) + || (c0 == v0 && c1 == v1 && c2 == v2 && c3 > v3) ) { v0 = c0, v1 = c1, v2 = c2, v3 = c3; free((void*)priv->base.version10_ptr); priv->base.version10_ptr = strdup(ffd.cFileName); From 9e6f53dd5857d04294f8bed3fe2b3dde75196756 Mon Sep 17 00:00:00 2001 From: kristopher tate Date: Sat, 1 Sep 2018 12:02:21 +0900 Subject: [PATCH 36/56] i#1438: src/windows_sdk.cpp: fix version guard in `find_81_version`; --- src/windows_sdk.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/windows_sdk.cpp b/src/windows_sdk.cpp index a63ccceb75..c6e79f85cd 100644 --- a/src/windows_sdk.cpp +++ b/src/windows_sdk.cpp @@ -248,7 +248,8 @@ static ZigFindWindowsSdkError find_81_version(ZigWindowsSDKPrivate *priv) { if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { int c0 = 0, c1 = 0; sscanf(ffd.cFileName, "winv%d.%d", &c0, &c1); - if ((c0 > v0) || (c1 > v1)) { + + if ( (c0 > v0) || (c0 == v0 && c1 > v1) ) { v0 = c0, v1 = c1; free((void*)priv->base.version81_ptr); priv->base.version81_ptr = strdup(ffd.cFileName); From 454b2362ee5e80dd1236535eaaec70d98e0d33ba Mon Sep 17 00:00:00 2001 From: kristopher tate Date: Sat, 1 Sep 2018 19:40:05 +0900 Subject: [PATCH 37/56] std/fmt/index.zig: #1358 allow bytes to be printed-out as hex; Supports {x} for lowercase and {X} for uppercase; --- std/fmt/index.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/std/fmt/index.zig b/std/fmt/index.zig index 6d23eebd0b..a8ea5b4c5a 100644 --- a/std/fmt/index.zig +++ b/std/fmt/index.zig @@ -350,6 +350,11 @@ pub fn formatText( comptime var width = 0; if (fmt.len > 1) width = comptime (parseUnsigned(usize, fmt[1..], 10) catch unreachable); return formatBuf(bytes, width, context, Errors, output); + } else if ((fmt[0] == 'x') or (fmt[0] == 'X') ) { + for (bytes) |c| { + try formatInt(c, 16, fmt[0] == 'X', 0, context, Errors, output); + } + return; } else @compileError("Unknown format character: " ++ []u8{fmt[0]}); } return output(context, bytes); From 7a633f472daba84b03dac18557f411c949f6b875 Mon Sep 17 00:00:00 2001 From: kristopher tate Date: Sat, 1 Sep 2018 19:53:11 +0900 Subject: [PATCH 38/56] std/fmt/index.zig: #1358: test bytes printed-out as hex; --- std/fmt/index.zig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/std/fmt/index.zig b/std/fmt/index.zig index a8ea5b4c5a..8e34bd43c4 100644 --- a/std/fmt/index.zig +++ b/std/fmt/index.zig @@ -1276,6 +1276,12 @@ test "fmt.format" { try testFmt("E.Two", "{}", inst); } + //print bytes as hex + { + const some_bytes = "\xCA\xFE\xBA\xBE"; + try testFmt("lowercase: cafebabe\n", "lowercase: {x}\n", some_bytes); + try testFmt("uppercase: CAFEBABE\n", "uppercase: {X}\n", some_bytes); + } } fn testFmt(expected: []const u8, comptime template: []const u8, args: ...) !void { From 4bf54f30103ec89173111b397b064a46268873a5 Mon Sep 17 00:00:00 2001 From: Shawn Landden Date: Sat, 1 Sep 2018 22:23:34 -0700 Subject: [PATCH 39/56] std/rb.zig: fix comment --- std/rb.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/rb.zig b/std/rb.zig index e8c4738b52..e42efc32af 100644 --- a/std/rb.zig +++ b/std/rb.zig @@ -19,7 +19,7 @@ const ReplaceError = error{NotEqual}; /// node: rb.Node, /// value: i32, /// }; -/// fn number(node: *Node) Number { +/// fn number(node: *rb.Node) Number { /// return @fieldParentPtr(Number, "node", node); /// } pub const Node = struct { From 48d3fbef5c3eb0c146979d3156c094fb23f45e8e Mon Sep 17 00:00:00 2001 From: kristopher tate Date: Sun, 2 Sep 2018 15:04:20 +0900 Subject: [PATCH 40/56] std/fmt/index.zig: set width from 0 to 2; \x00 was printed as 0 and \x0E was printed as E; \x00 now correctly prints 00 and \x0E correctly prints 0E; --- std/fmt/index.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/fmt/index.zig b/std/fmt/index.zig index 8e34bd43c4..345c46441d 100644 --- a/std/fmt/index.zig +++ b/std/fmt/index.zig @@ -352,7 +352,7 @@ pub fn formatText( return formatBuf(bytes, width, context, Errors, output); } else if ((fmt[0] == 'x') or (fmt[0] == 'X') ) { for (bytes) |c| { - try formatInt(c, 16, fmt[0] == 'X', 0, context, Errors, output); + try formatInt(c, 16, fmt[0] == 'X', 2, context, Errors, output); } return; } else @compileError("Unknown format character: " ++ []u8{fmt[0]}); From d1752fbdc0d778fa92e97510f868ebbee5137329 Mon Sep 17 00:00:00 2001 From: kristopher tate Date: Sun, 2 Sep 2018 15:04:57 +0900 Subject: [PATCH 41/56] std/fmt/index.zig: test for printing double width hex bytes with zeros; Co-Authored-By: Shawn Landden --- std/fmt/index.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/std/fmt/index.zig b/std/fmt/index.zig index 345c46441d..f9b68d7941 100644 --- a/std/fmt/index.zig +++ b/std/fmt/index.zig @@ -1281,6 +1281,8 @@ test "fmt.format" { const some_bytes = "\xCA\xFE\xBA\xBE"; try testFmt("lowercase: cafebabe\n", "lowercase: {x}\n", some_bytes); try testFmt("uppercase: CAFEBABE\n", "uppercase: {X}\n", some_bytes); + const bytes_with_zeros = "\x00\x0E\xBA\xBE"; + try testFmt("lowercase: 000ebabe\n", "lowercase: {x}\n", bytes_with_zeros); } } From 528e3b43a677c0fbd05245fec9e4a6c361b89b59 Mon Sep 17 00:00:00 2001 From: Shawn Landden Date: Sat, 1 Sep 2018 23:51:44 -0700 Subject: [PATCH 42/56] these all use futex() (inaccurate comments) --- std/event/lock.zig | 2 +- std/event/locked.zig | 2 +- std/event/rwlock.zig | 2 +- std/event/rwlocked.zig | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/std/event/lock.zig b/std/event/lock.zig index 2ee9dc981f..46e0d13468 100644 --- a/std/event/lock.zig +++ b/std/event/lock.zig @@ -7,7 +7,7 @@ const AtomicOrder = builtin.AtomicOrder; const Loop = std.event.Loop; /// Thread-safe async/await lock. -/// Does not make any syscalls - coroutines which are waiting for the lock are suspended, and +/// coroutines which are waiting for the lock are suspended, and /// are resumed when the lock is released, in order. /// Allows only one actor to hold the lock. pub const Lock = struct { diff --git a/std/event/locked.zig b/std/event/locked.zig index e7ad544d78..7df56c4571 100644 --- a/std/event/locked.zig +++ b/std/event/locked.zig @@ -3,7 +3,7 @@ const Lock = std.event.Lock; const Loop = std.event.Loop; /// Thread-safe async/await lock that protects one piece of data. -/// Does not make any syscalls - coroutines which are waiting for the lock are suspended, and +/// coroutines which are waiting for the lock are suspended, and /// are resumed when the lock is released, in order. pub fn Locked(comptime T: type) type { return struct { diff --git a/std/event/rwlock.zig b/std/event/rwlock.zig index 186c81eb76..03f2ac702b 100644 --- a/std/event/rwlock.zig +++ b/std/event/rwlock.zig @@ -7,7 +7,7 @@ const AtomicOrder = builtin.AtomicOrder; const Loop = std.event.Loop; /// Thread-safe async/await lock. -/// Does not make any syscalls - coroutines which are waiting for the lock are suspended, and +/// coroutines which are waiting for the lock are suspended, and /// are resumed when the lock is released, in order. /// Many readers can hold the lock at the same time; however locking for writing is exclusive. /// When a read lock is held, it will not be released until the reader queue is empty. diff --git a/std/event/rwlocked.zig b/std/event/rwlocked.zig index ef7e83d20c..1a6e77c27a 100644 --- a/std/event/rwlocked.zig +++ b/std/event/rwlocked.zig @@ -3,7 +3,7 @@ const RwLock = std.event.RwLock; const Loop = std.event.Loop; /// Thread-safe async/await RW lock that protects one piece of data. -/// Does not make any syscalls - coroutines which are waiting for the lock are suspended, and +/// coroutines which are waiting for the lock are suspended, and /// are resumed when the lock is released, in order. pub fn RwLocked(comptime T: type) type { return struct { From fbd9bac5e7166a9b4ceb56ecb3a659abf6076311 Mon Sep 17 00:00:00 2001 From: kristopher tate Date: Sun, 2 Sep 2018 18:02:24 +0900 Subject: [PATCH 43/56] std/fmt/index.zig: add hexToBytes function under std.fmt; Depends on #1454 being implemented; --- std/fmt/index.zig | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/std/fmt/index.zig b/std/fmt/index.zig index f9b68d7941..13a0b525d7 100644 --- a/std/fmt/index.zig +++ b/std/fmt/index.zig @@ -1331,3 +1331,23 @@ pub fn isWhiteSpace(byte: u8) bool { else => false, }; } + +// depends on https://github.com/ziglang/zig/pull/1454 +pub fn hexToBytes(input: []const u8, out: []u8) !void { + if (out.len * 2 < input.len) + return error.InvalidLength; + + var i: usize = 0; + while (i < input.len) : (i += 2) { + out[i/2] = (try charToDigit(input[i], 36)) << 4; + out[i/2] += try charToDigit(input[i+1], 36); + } +} + +test "fmt.hexToBytes" { + const test_hex_str = "909A312BB12ED1F819B3521AC4C1E896F2160507FFC1C8381E3B07BB16BD1706"; + var pb: [32]u8 = undefined; + try hexToBytes(test_hex_str, pb[0..]); + try testFmt(test_hex_str, "{X}", pb); +} + From 98dc943c0784b93ed28099bb75044c536174a144 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 2 Sep 2018 15:58:08 -0400 Subject: [PATCH 44/56] rework code to avoid duplicate operations --- std/coff.zig | 8 - std/debug/index.zig | 435 ++++++++++++++++++++++++++++++++---- std/os/windows/index.zig | 13 ++ std/os/windows/kernel32.zig | 18 ++ std/pdb.zig | 287 +++--------------------- 5 files changed, 452 insertions(+), 309 deletions(-) diff --git a/std/coff.zig b/std/coff.zig index 2921109bd6..379fd1af42 100644 --- a/std/coff.zig +++ b/std/coff.zig @@ -83,7 +83,6 @@ pub const Coff = struct { fn loadOptionalHeader(self: *Coff, file_stream: *io.FileInStream) !void { const in = &file_stream.stream; self.pe_header.magic = try in.readIntLe(u16); - std.debug.warn("reading pe optional\n"); // For now we're only interested in finding the reference to the .pdb, // so we'll skip most of this header, which size is different in 32 // 64 bits by the way. @@ -97,11 +96,9 @@ pub const Coff = struct { else return error.InvalidPEMagic; - std.debug.warn("skipping {}\n", skip_size); try self.in_file.seekForward(skip_size); const number_of_rva_and_sizes = try in.readIntLe(u32); - //std.debug.warn("indicating {} data dirs\n", number_of_rva_and_sizes); if (number_of_rva_and_sizes != IMAGE_NUMBEROF_DIRECTORY_ENTRIES) return error.InvalidPEHeader; @@ -110,9 +107,7 @@ pub const Coff = struct { .virtual_address = try in.readIntLe(u32), .size = try in.readIntLe(u32), }; - //std.debug.warn("data_dir @ {x}, size {}\n", data_dir.virtual_address, data_dir.size); } - std.debug.warn("loaded data directories\n"); } pub fn getPdbPath(self: *Coff, buffer: []u8) !usize { @@ -123,7 +118,6 @@ pub const Coff = struct { // debug_directory. const debug_dir = &self.pe_header.data_directory[DEBUG_DIRECTORY]; const file_offset = debug_dir.virtual_address - header.virtual_address + header.pointer_to_raw_data; - std.debug.warn("file offset {x}\n", file_offset); try self.in_file.seekTo(file_offset + debug_dir.size); var file_stream = io.FileInStream.init(self.in_file); @@ -134,7 +128,6 @@ pub const Coff = struct { // 'RSDS' indicates PDB70 format, used by lld. if (!mem.eql(u8, cv_signature, "RSDS")) return error.InvalidPEMagic; - std.debug.warn("cv_signature {}\n", cv_signature); try in.readNoEof(self.guid[0..]); self.age = try in.readIntLe(u32); @@ -181,7 +174,6 @@ pub const Coff = struct { }, }); } - std.debug.warn("loaded {} sections\n", self.coff_header.number_of_sections); } pub fn getSection(self: *Coff, comptime name: []const u8) ?*Section { diff --git a/std/debug/index.zig b/std/debug/index.zig index 2883465931..de4d7745c6 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -20,6 +20,17 @@ pub const runtime_safety = switch (builtin.mode) { builtin.Mode.ReleaseFast, builtin.Mode.ReleaseSmall => false, }; +const Module = struct { + mod_info: pdb.ModInfo, + module_name: []u8, + obj_file_name: []u8, + + populated: bool, + symbols: []u8, + subsect_info: []u8, + checksums: []u32, +}; + /// Tries to write to stderr, unbuffered, and ignores any error returned. /// Does not append a newline. var stderr_file: os.File = undefined; @@ -258,12 +269,277 @@ pub fn printSourceAtAddress(debug_info: *DebugInfo, out_stream: var, address: us } } -fn printSourceAtAddressWindows(di: *DebugInfo, out_stream: var, address: usize, tty_color: bool) !void { +fn printSourceAtAddressWindows(di: *DebugInfo, out_stream: var, relocated_address: usize, tty_color: bool) !void { + const allocator = getDebugInfoAllocator(); const base_address = os.getBaseAddress(); - const relative_address = address - base_address; - std.debug.warn("{x} - {x} => {x}\n", address, base_address, relative_address); - try di.pdb.getSourceLine(relative_address); - return error.UnsupportedDebugInfo; + const relative_address = relocated_address - base_address; + + var coff_section: *coff.Section = undefined; + const mod_index = for (di.sect_contribs) |sect_contrib| { + coff_section = &di.coff.sections.toSlice()[sect_contrib.Section]; + + const vaddr_start = coff_section.header.virtual_address + sect_contrib.Offset; + const vaddr_end = vaddr_start + sect_contrib.Size; + if (relative_address >= vaddr_start and relative_address < vaddr_end) { + break sect_contrib.ModuleIndex; + } + } else { + // we have no information to add to the address + if (tty_color) { + try out_stream.print("???:?:?: "); + setTtyColor(TtyColor.Dim); + try out_stream.print("0x{x} in ??? (???)", relocated_address); + setTtyColor(TtyColor.Reset); + try out_stream.print("\n\n\n"); + } else { + try out_stream.print("???:?:?: 0x{x} in ??? (???)\n\n\n", relocated_address); + } + return; + }; + + const mod = &di.modules[mod_index]; + try populateModule(di, mod); + const obj_basename = os.path.basename(mod.obj_file_name); + + var symbol_i: usize = 0; + const symbol_name = while (symbol_i != mod.symbols.len) { + const prefix = @ptrCast(*pdb.RecordPrefix, &mod.symbols[symbol_i]); + if (prefix.RecordLen < 2) + return error.InvalidDebugInfo; + switch (prefix.RecordKind) { + pdb.SymbolKind.S_LPROC32 => { + const proc_sym = @ptrCast(*pdb.ProcSym, &mod.symbols[symbol_i + @sizeOf(pdb.RecordPrefix)]); + const vaddr_start = coff_section.header.virtual_address + proc_sym.CodeOffset; + const vaddr_end = vaddr_start + proc_sym.CodeSize; + if (relative_address >= vaddr_start and relative_address < vaddr_end) { + break mem.toSliceConst(u8, @ptrCast([*]u8, proc_sym) + @sizeOf(pdb.ProcSym)); + } + }, + else => {}, + } + symbol_i += prefix.RecordLen + @sizeOf(u16); + if (symbol_i > mod.symbols.len) + return error.InvalidDebugInfo; + } else "???"; + + const subsect_info = mod.subsect_info; + + var sect_offset: usize = 0; + var skip_len: usize = undefined; + const opt_line_info = subsections: while (sect_offset != subsect_info.len) : (sect_offset += skip_len) { + const subsect_hdr = @ptrCast(*pdb.DebugSubsectionHeader, &subsect_info[sect_offset]); + skip_len = subsect_hdr.Length; + sect_offset += @sizeOf(pdb.DebugSubsectionHeader); + + switch (subsect_hdr.Kind) { + pdb.DebugSubsectionKind.Lines => { + var line_index: usize = sect_offset; + + const line_hdr = @ptrCast(*pdb.LineFragmentHeader, &subsect_info[line_index]); + if (line_hdr.RelocSegment == 0) return error.MissingDebugInfo; + line_index += @sizeOf(pdb.LineFragmentHeader); + + const block_hdr = @ptrCast(*pdb.LineBlockFragmentHeader, &subsect_info[line_index]); + line_index += @sizeOf(pdb.LineBlockFragmentHeader); + + const has_column = line_hdr.Flags.LF_HaveColumns; + + const frag_vaddr_start = coff_section.header.virtual_address + line_hdr.RelocOffset; + const frag_vaddr_end = frag_vaddr_start + line_hdr.CodeSize; + if (relative_address >= frag_vaddr_start and relative_address < frag_vaddr_end) { + var line_i: usize = 0; + const start_line_index = line_index; + while (line_i < block_hdr.NumLines) : (line_i += 1) { + const line_num_entry = @ptrCast(*pdb.LineNumberEntry, &subsect_info[line_index]); + line_index += @sizeOf(pdb.LineNumberEntry); + const flags = @ptrCast(*pdb.LineNumberEntry.Flags, &line_num_entry.Flags); + const vaddr_start = frag_vaddr_start + line_num_entry.Offset; + const vaddr_end = if (flags.End == 0) frag_vaddr_end else vaddr_start + flags.End; + if (relative_address >= vaddr_start and relative_address < vaddr_end) { + const chksum_index = block_hdr.NameIndex; + std.debug.warn("looking up checksum {}\n", chksum_index); + const strtab_offset = mod.checksums[chksum_index]; + try di.pdb.string_table.seekTo(@sizeOf(pdb.PDBStringTableHeader) + strtab_offset); + const source_file_name = try di.pdb.string_table.readNullTermString(allocator); + const line = flags.Start; + const column = if (has_column) blk: { + line_index = start_line_index + @sizeOf(pdb.LineNumberEntry) * block_hdr.NumLines; + line_index += @sizeOf(pdb.ColumnNumberEntry) * line_i; + const col_num_entry = @ptrCast(*pdb.ColumnNumberEntry, &subsect_info[line_index]); + break :blk col_num_entry.StartColumn; + } else 0; + break :subsections LineInfo{ + .allocator = allocator, + .file_name = source_file_name, + .line = line, + .column = column, + }; + } + } + break :subsections null; + } + }, + else => {}, + } + + if (sect_offset > subsect_info.len) + return error.InvalidDebugInfo; + } else null; + + if (tty_color) { + if (opt_line_info) |li| { + try out_stream.print("{}:{}:{}: ", li.file_name, li.line, li.column); + } else { + try out_stream.print("???:?:?: "); + } + setTtyColor(TtyColor.Dim); + try out_stream.print("0x{x} in {} ({})", relocated_address, symbol_name, obj_basename); + setTtyColor(TtyColor.Reset); + + if (opt_line_info) |line_info| { + try out_stream.print("\n"); + if (printLineFromFile(out_stream, line_info)) { + if (line_info.column == 0) { + try out_stream.write("\n"); + } else { + { + var col_i: usize = 1; + while (col_i < line_info.column) : (col_i += 1) { + try out_stream.writeByte(' '); + } + } + setTtyColor(TtyColor.Green); + try out_stream.write("^"); + setTtyColor(TtyColor.Reset); + try out_stream.write("\n"); + } + } else |err| switch (err) { + error.EndOfFile => {}, + else => return err, + } + } else { + try out_stream.print("\n\n\n"); + } + } else { + if (opt_line_info) |li| { + try out_stream.print("{}:{}:{}: 0x{x} in {} ({})\n\n\n", li.file_name, li.line, li.column, relocated_address, symbol_name, obj_basename); + } else { + try out_stream.print("???:?:?: 0x{x} in {} ({})\n\n\n", relocated_address, symbol_name, obj_basename); + } + } +} + +const TtyColor = enum{ + Red, + Green, + Cyan, + White, + Dim, + Bold, + Reset, +}; + +/// TODO this is a special case hack right now. clean it up and maybe make it part of std.fmt +fn setTtyColor(tty_color: TtyColor) void { + const S = struct { + var attrs: windows.WORD = undefined; + var init_attrs = false; + }; + if (!S.init_attrs) { + S.init_attrs = true; + var info: windows.CONSOLE_SCREEN_BUFFER_INFO = undefined; + // TODO handle error + _ = windows.GetConsoleScreenBufferInfo(stderr_file.handle, &info); + S.attrs = info.wAttributes; + } + + // TODO handle errors + switch (tty_color) { + TtyColor.Red => { + _ = windows.SetConsoleTextAttribute(stderr_file.handle, windows.FOREGROUND_RED|windows.FOREGROUND_INTENSITY); + }, + TtyColor.Green => { + _ = windows.SetConsoleTextAttribute(stderr_file.handle, windows.FOREGROUND_GREEN|windows.FOREGROUND_INTENSITY); + }, + TtyColor.Cyan => { + _ = windows.SetConsoleTextAttribute(stderr_file.handle, + windows.FOREGROUND_GREEN|windows.FOREGROUND_BLUE|windows.FOREGROUND_INTENSITY); + }, + TtyColor.White, TtyColor.Bold => { + _ = windows.SetConsoleTextAttribute(stderr_file.handle, + windows.FOREGROUND_RED|windows.FOREGROUND_GREEN|windows.FOREGROUND_BLUE|windows.FOREGROUND_INTENSITY); + }, + TtyColor.Dim => { + _ = windows.SetConsoleTextAttribute(stderr_file.handle, + windows.FOREGROUND_RED|windows.FOREGROUND_GREEN|windows.FOREGROUND_BLUE); + }, + TtyColor.Reset => { + _ = windows.SetConsoleTextAttribute(stderr_file.handle, S.attrs); + }, + } +} + +fn populateModule(di: *DebugInfo, mod: *Module) !void { + if (mod.populated) + return; + const allocator = getDebugInfoAllocator(); + + if (mod.mod_info.C11ByteSize != 0) + return error.InvalidDebugInfo; + + if (mod.mod_info.C13ByteSize == 0) + return error.MissingDebugInfo; + + const modi = di.pdb.getStreamById(mod.mod_info.ModuleSymStream) orelse return error.MissingDebugInfo; + + const signature = try modi.stream.readIntLe(u32); + if (signature != 4) + return error.InvalidDebugInfo; + + mod.symbols = try allocator.alloc(u8, mod.mod_info.SymByteSize - 4); + try modi.stream.readNoEof(mod.symbols); + + mod.subsect_info = try allocator.alloc(u8, mod.mod_info.C13ByteSize); + try modi.stream.readNoEof(mod.subsect_info); + + var checksum_list = ArrayList(u32).init(allocator); + var sect_offset: usize = 0; + var skip_len: usize = undefined; + while (sect_offset != mod.subsect_info.len) : (sect_offset += skip_len) { + const subsect_hdr = @ptrCast(*pdb.DebugSubsectionHeader, &mod.subsect_info[sect_offset]); + skip_len = subsect_hdr.Length; + sect_offset += @sizeOf(pdb.DebugSubsectionHeader); + + switch (subsect_hdr.Kind) { + pdb.DebugSubsectionKind.FileChecksums => { + var chksum_index: usize = sect_offset; + + while (chksum_index != mod.subsect_info.len) { + const chksum_hdr = @ptrCast(*pdb.FileChecksumEntryHeader, &mod.subsect_info[chksum_index]); + std.debug.warn("{} {}\n", checksum_list.len, chksum_hdr); + try checksum_list.append(chksum_hdr.FileNameOffset); + const len = @sizeOf(pdb.FileChecksumEntryHeader) + chksum_hdr.ChecksumSize; + chksum_index += len + (len % 4); + if (chksum_index > mod.subsect_info.len) + return error.InvalidDebugInfo; + } + + }, + else => {}, + } + + if (sect_offset > mod.subsect_info.len) + return error.InvalidDebugInfo; + } + mod.checksums = checksum_list.toOwnedSlice(); + + for (mod.checksums) |strtab_offset| { + try di.pdb.string_table.seekTo(@sizeOf(pdb.PDBStringTableHeader) + strtab_offset); + const source_file_name = try di.pdb.string_table.readNullTermString(allocator); + std.debug.warn("{}={}\n", strtab_offset, source_file_name); + } + + mod.populated = true; } fn machoSearchSymbols(symbols: []const MachoSymbol, address: usize) ?*const MachoSymbol { @@ -425,6 +701,8 @@ fn openSelfDebugInfoWindows(allocator: *mem.Allocator) !DebugInfo { var di = DebugInfo{ .coff = coff_obj, .pdb = undefined, + .sect_contribs = undefined, + .modules = undefined, }; try di.coff.loadHeader(); @@ -432,15 +710,12 @@ fn openSelfDebugInfoWindows(allocator: *mem.Allocator) !DebugInfo { var path_buf: [windows.MAX_PATH]u8 = undefined; const len = try di.coff.getPdbPath(path_buf[0..]); const raw_path = path_buf[0..len]; - std.debug.warn("pdb raw path {}\n", raw_path); const path = try os.path.resolve(allocator, raw_path); - std.debug.warn("pdb resolved path {}\n", path); try di.pdb.openFile(di.coff, path); var pdb_stream = di.pdb.getStream(pdb.StreamType.Pdb) orelse return error.InvalidDebugInfo; - std.debug.warn("pdb real filepos {}\n", pdb_stream.getFilePos()); const version = try pdb_stream.stream.readIntLe(u32); const signature = try pdb_stream.stream.readIntLe(u32); const age = try pdb_stream.stream.readIntLe(u32); @@ -448,51 +723,119 @@ fn openSelfDebugInfoWindows(allocator: *mem.Allocator) !DebugInfo { try pdb_stream.stream.readNoEof(guid[0..]); if (!mem.eql(u8, di.coff.guid, guid) or di.coff.age != age) return error.InvalidDebugInfo; - std.debug.warn("v {} s {} a {}\n", version, signature, age); // We validated the executable and pdb match. - const name_bytes_len = try pdb_stream.stream.readIntLe(u32); - const name_bytes = try allocator.alloc(u8, name_bytes_len); - try pdb_stream.stream.readNoEof(name_bytes); + const string_table_index = str_tab_index: { + const name_bytes_len = try pdb_stream.stream.readIntLe(u32); + const name_bytes = try allocator.alloc(u8, name_bytes_len); + try pdb_stream.stream.readNoEof(name_bytes); - const HashTableHeader = packed struct { - Size: u32, - Capacity: u32, + const HashTableHeader = packed struct { + Size: u32, + Capacity: u32, - fn maxLoad(cap: u32) u32 { - return cap * 2 / 3 + 1; + fn maxLoad(cap: u32) u32 { + return cap * 2 / 3 + 1; + } + }; + var hash_tbl_hdr: HashTableHeader = undefined; + try pdb_stream.stream.readStruct(HashTableHeader, &hash_tbl_hdr); + if (hash_tbl_hdr.Capacity == 0) + return error.InvalidDebugInfo; + + if (hash_tbl_hdr.Size > HashTableHeader.maxLoad(hash_tbl_hdr.Capacity)) + return error.InvalidDebugInfo; + + const present = try readSparseBitVector(&pdb_stream.stream, allocator); + if (present.len != hash_tbl_hdr.Size) + return error.InvalidDebugInfo; + const deleted = try readSparseBitVector(&pdb_stream.stream, allocator); + + const Bucket = struct { + first: u32, + second: u32, + }; + const bucket_list = try allocator.alloc(Bucket, present.len); + for (present) |_| { + const name_offset = try pdb_stream.stream.readIntLe(u32); + const name_index = try pdb_stream.stream.readIntLe(u32); + const name = mem.toSlice(u8, name_bytes.ptr + name_offset); + if (mem.eql(u8, name, "/names")) { + break :str_tab_index name_index; + } } + return error.MissingDebugInfo; }; - var hash_tbl_hdr: HashTableHeader = undefined; - try pdb_stream.stream.readStruct(HashTableHeader, &hash_tbl_hdr); - if (hash_tbl_hdr.Capacity == 0) - return error.InvalidDebugInfo; - - if (hash_tbl_hdr.Size > HashTableHeader.maxLoad(hash_tbl_hdr.Capacity)) - return error.InvalidDebugInfo; - - std.debug.warn("{}\n", hash_tbl_hdr); - - const present = try readSparseBitVector(&pdb_stream.stream, allocator); - if (present.len != hash_tbl_hdr.Size) - return error.InvalidDebugInfo; - const deleted = try readSparseBitVector(&pdb_stream.stream, allocator); - - const Bucket = struct { - first: u32, - second: u32, - }; - const bucket_list = try allocator.alloc(Bucket, present.len); - const string_table_index = for (present) |_| { - const name_offset = try pdb_stream.stream.readIntLe(u32); - const name_index = try pdb_stream.stream.readIntLe(u32); - const name = mem.toSlice(u8, name_bytes.ptr + name_offset); - if (mem.eql(u8, name, "/names")) { - break name_index; - } - } else return error.MissingDebugInfo; di.pdb.string_table = di.pdb.getStreamById(string_table_index) orelse return error.InvalidDebugInfo; + di.pdb.dbi = di.pdb.getStream(pdb.StreamType.Dbi) orelse return error.MissingDebugInfo; + + const dbi = di.pdb.dbi; + + // Dbi Header + var dbi_stream_header: pdb.DbiStreamHeader = undefined; + try dbi.stream.readStruct(pdb.DbiStreamHeader, &dbi_stream_header); + const mod_info_size = dbi_stream_header.ModInfoSize; + const section_contrib_size = dbi_stream_header.SectionContributionSize; + + var modules = ArrayList(Module).init(allocator); + + // Module Info Substream + var mod_info_offset: usize = 0; + while (mod_info_offset != mod_info_size) { + var mod_info: pdb.ModInfo = undefined; + try dbi.stream.readStruct(pdb.ModInfo, &mod_info); + var this_record_len: usize = @sizeOf(pdb.ModInfo); + + const module_name = try dbi.readNullTermString(allocator); + this_record_len += module_name.len + 1; + + const obj_file_name = try dbi.readNullTermString(allocator); + this_record_len += obj_file_name.len + 1; + + const march_forward_bytes = this_record_len % 4; + if (march_forward_bytes != 0) { + try dbi.seekForward(march_forward_bytes); + this_record_len += march_forward_bytes; + } + + try modules.append(Module{ + .mod_info = mod_info, + .module_name = module_name, + .obj_file_name = obj_file_name, + + .populated = false, + .symbols = undefined, + .subsect_info = undefined, + .checksums = undefined, + }); + + mod_info_offset += this_record_len; + if (mod_info_offset > mod_info_size) + return error.InvalidDebugInfo; + } + + di.modules = modules.toOwnedSlice(); + + // Section Contribution Substream + var sect_contribs = ArrayList(pdb.SectionContribEntry).init(allocator); + var sect_cont_offset: usize = 0; + if (section_contrib_size != 0) { + const ver = @intToEnum(pdb.SectionContrSubstreamVersion, try dbi.stream.readIntLe(u32)); + if (ver != pdb.SectionContrSubstreamVersion.Ver60) + return error.InvalidDebugInfo; + sect_cont_offset += @sizeOf(u32); + } + while (sect_cont_offset != section_contrib_size) { + const entry = try sect_contribs.addOne(); + try dbi.stream.readStruct(pdb.SectionContribEntry, entry); + sect_cont_offset += @sizeOf(pdb.SectionContribEntry); + + if (sect_cont_offset > section_contrib_size) + return error.InvalidDebugInfo; + } + + di.sect_contribs = sect_contribs.toOwnedSlice(); return di; } @@ -715,6 +1058,8 @@ pub const DebugInfo = switch (builtin.os) { builtin.Os.windows => struct { pdb: pdb.Pdb, coff: *coff.Coff, + sect_contribs: []pdb.SectionContribEntry, + modules: []Module, }, builtin.Os.linux => struct { self_exe_file: os.File, diff --git a/std/os/windows/index.zig b/std/os/windows/index.zig index ca6299dc5e..9286b7d090 100644 --- a/std/os/windows/index.zig +++ b/std/os/windows/index.zig @@ -15,6 +15,7 @@ test "import" { pub const ERROR = @import("error.zig"); +pub const SHORT = c_short; pub const BOOL = c_int; pub const BOOLEAN = BYTE; pub const BYTE = u8; @@ -364,3 +365,15 @@ pub const FILE_FLAG_RANDOM_ACCESS = 0x10000000; pub const FILE_FLAG_SESSION_AWARE = 0x00800000; pub const FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000; pub const FILE_FLAG_WRITE_THROUGH = 0x80000000; + +pub const SMALL_RECT = extern struct { + Left: SHORT, + Top: SHORT, + Right: SHORT, + Bottom: SHORT, +}; + +pub const COORD = extern struct { + X: SHORT, + Y: SHORT, +}; diff --git a/std/os/windows/kernel32.zig b/std/os/windows/kernel32.zig index 65f10a5a2a..ffa4422760 100644 --- a/std/os/windows/kernel32.zig +++ b/std/os/windows/kernel32.zig @@ -72,6 +72,8 @@ pub extern "kernel32" stdcallcc fn GetCommandLineA() LPSTR; pub extern "kernel32" stdcallcc fn GetConsoleMode(in_hConsoleHandle: HANDLE, out_lpMode: *DWORD) BOOL; +pub extern "kernel32" stdcallcc fn GetConsoleScreenBufferInfo(hConsoleOutput: HANDLE, lpConsoleScreenBufferInfo: *CONSOLE_SCREEN_BUFFER_INFO) BOOL; + pub extern "kernel32" stdcallcc fn GetCurrentDirectoryA(nBufferLength: DWORD, lpBuffer: ?[*]CHAR) DWORD; pub extern "kernel32" stdcallcc fn GetCurrentDirectoryW(nBufferLength: DWORD, lpBuffer: ?[*]WCHAR) DWORD; @@ -179,6 +181,8 @@ pub extern "kernel32" stdcallcc fn ReadFile( pub extern "kernel32" stdcallcc fn RemoveDirectoryA(lpPathName: LPCSTR) BOOL; +pub extern "kernel32" stdcallcc fn SetConsoleTextAttribute(hConsoleOutput: HANDLE, wAttributes: WORD) BOOL; + pub extern "kernel32" stdcallcc fn SetFilePointerEx( in_fFile: HANDLE, in_liDistanceToMove: LARGE_INTEGER, @@ -234,3 +238,17 @@ pub const FILE_NOTIFY_CHANGE_LAST_WRITE = 16; pub const FILE_NOTIFY_CHANGE_DIR_NAME = 2; pub const FILE_NOTIFY_CHANGE_FILE_NAME = 1; pub const FILE_NOTIFY_CHANGE_ATTRIBUTES = 4; + + +pub const CONSOLE_SCREEN_BUFFER_INFO = extern struct { + dwSize: COORD, + dwCursorPosition: COORD, + wAttributes: WORD, + srWindow: SMALL_RECT, + dwMaximumWindowSize: COORD, +}; + +pub const FOREGROUND_BLUE = 1; +pub const FOREGROUND_GREEN = 2; +pub const FOREGROUND_RED = 4; +pub const FOREGROUND_INTENSITY = 8; diff --git a/std/pdb.zig b/std/pdb.zig index 3aefc4f724..907eddff04 100644 --- a/std/pdb.zig +++ b/std/pdb.zig @@ -10,7 +10,7 @@ const coff = std.coff; const ArrayList = std.ArrayList; // https://llvm.org/docs/PDB/DbiStream.html#stream-header -const DbiStreamHeader = packed struct { +pub const DbiStreamHeader = packed struct { VersionSignature: i32, VersionHeader: u32, Age: u32, @@ -33,7 +33,7 @@ const DbiStreamHeader = packed struct { Padding: u32, }; -const SectionContribEntry = packed struct { +pub const SectionContribEntry = packed struct { Section: u16, Padding1: [2]u8, Offset: u32, @@ -45,7 +45,7 @@ const SectionContribEntry = packed struct { RelocCrc: u32, }; -const ModInfo = packed struct { +pub const ModInfo = packed struct { Unused1: u32, SectionContr: SectionContribEntry, Flags: u16, @@ -63,12 +63,12 @@ const ModInfo = packed struct { //ObjFileName: char[], }; -const SectionMapHeader = packed struct { +pub const SectionMapHeader = packed struct { Count: u16, /// Number of segment descriptors LogCount: u16, /// Number of logical segment descriptors }; -const SectionMapEntry = packed struct { +pub const SectionMapEntry = packed struct { Flags: u16 , /// See the SectionMapEntryFlags enum below. Ovl: u16 , /// Logical overlay number Group: u16 , /// Group index into descriptor array. @@ -86,12 +86,6 @@ pub const StreamType = enum(u16) { Ipi = 4, }; -const Module = struct { - mod_info: ModInfo, - module_name: []u8, - obj_file_name: []u8, -}; - /// Duplicate copy of SymbolRecordKind, but using the official CV names. Useful /// for reference purposes and when dealing with unknown record types. pub const SymbolKind = packed enum(u16) { @@ -293,9 +287,9 @@ pub const SymbolKind = packed enum(u16) { S_GTHREAD32 = 4371, }; -const TypeIndex = u32; +pub const TypeIndex = u32; -const ProcSym = packed struct { +pub const ProcSym = packed struct { Parent: u32 , End: u32 , Next: u32 , @@ -310,7 +304,7 @@ const ProcSym = packed struct { // Name: [*]u8, }; -const ProcSymFlags = packed struct { +pub const ProcSymFlags = packed struct { HasFP: bool, HasIRET: bool, HasFRET: bool, @@ -321,24 +315,24 @@ const ProcSymFlags = packed struct { HasOptimizedDebugInfo: bool, }; -const SectionContrSubstreamVersion = enum(u32) { +pub const SectionContrSubstreamVersion = enum(u32) { Ver60 = 0xeffe0000 + 19970605, V2 = 0xeffe0000 + 20140516 }; -const RecordPrefix = packed struct { +pub const RecordPrefix = packed struct { RecordLen: u16, /// Record length, starting from &RecordKind. RecordKind: SymbolKind, /// Record kind enum (SymRecordKind or TypeRecordKind) }; -const LineFragmentHeader = packed struct { +pub const LineFragmentHeader = packed struct { RelocOffset: u32, /// Code offset of line contribution. RelocSegment: u16, /// Code segment of line contribution. Flags: LineFlags, CodeSize: u32, /// Code size of this line contribution. }; -const LineFlags = packed struct { +pub const LineFlags = packed struct { LF_HaveColumns: bool, /// CV_LINES_HAVE_COLUMNS unused: u15, }; @@ -347,7 +341,7 @@ const LineFlags = packed struct { /// header. The structure definitions follow. /// LineNumberEntry Lines[NumLines]; /// ColumnNumberEntry Columns[NumLines]; -const LineBlockFragmentHeader = packed struct { +pub const LineBlockFragmentHeader = packed struct { /// Offset of FileChecksum entry in File /// checksums buffer. The checksum entry then /// contains another offset into the string @@ -358,7 +352,7 @@ const LineBlockFragmentHeader = packed struct { }; -const LineNumberEntry = packed struct { +pub const LineNumberEntry = packed struct { Offset: u32, /// Offset to start of code bytes for line number Flags: u32, @@ -370,19 +364,19 @@ const LineNumberEntry = packed struct { }; }; -const ColumnNumberEntry = packed struct { +pub const ColumnNumberEntry = packed struct { StartColumn: u16, EndColumn: u16, }; /// Checksum bytes follow. -const FileChecksumEntryHeader = packed struct { +pub const FileChecksumEntryHeader = packed struct { FileNameOffset: u32, /// Byte offset of filename in global string table. ChecksumSize: u8, /// Number of bytes of checksum. ChecksumKind: u8, /// FileChecksumKind }; -const DebugSubsectionKind = packed enum(u32) { +pub const DebugSubsectionKind = packed enum(u32) { None = 0, Symbols = 0xf1, Lines = 0xf2, @@ -402,11 +396,25 @@ const DebugSubsectionKind = packed enum(u32) { CoffSymbolRVA = 0xfd, }; + +pub const DebugSubsectionHeader = packed struct { + Kind: DebugSubsectionKind, /// codeview::DebugSubsectionKind enum + Length: u32, /// number of bytes occupied by this record. +}; + + +pub const PDBStringTableHeader = packed struct { + Signature: u32, /// PDBStringTableSignature + HashVersion: u32, /// 1 or 2 + ByteSize: u32, /// Number of bytes of names buffer. +}; + pub const Pdb = struct { in_file: os.File, allocator: *mem.Allocator, coff: *coff.Coff, string_table: *MsfStream, + dbi: *MsfStream, msf: Msf, @@ -428,230 +436,6 @@ pub const Pdb = struct { const id = @enumToInt(stream); return self.getStreamById(id); } - - pub fn getSourceLine(self: *Pdb, address: usize) !void { - const dbi = self.getStream(StreamType.Dbi) orelse return error.InvalidDebugInfo; - - // Dbi Header - var header: DbiStreamHeader = undefined; - try dbi.stream.readStruct(DbiStreamHeader, &header); - std.debug.warn("{}\n", header); - warn("after header dbi stream at {} (file offset)\n", dbi.getFilePos()); - - var modules = ArrayList(Module).init(self.allocator); - - // Module Info Substream - var mod_info_offset: usize = 0; - while (mod_info_offset != header.ModInfoSize) { - var mod_info: ModInfo = undefined; - try dbi.stream.readStruct(ModInfo, &mod_info); - std.debug.warn("{}\n", mod_info); - var this_record_len: usize = @sizeOf(ModInfo); - - const module_name = try dbi.readNullTermString(self.allocator); - std.debug.warn("module_name '{}'\n", module_name); - this_record_len += module_name.len + 1; - - const obj_file_name = try dbi.readNullTermString(self.allocator); - std.debug.warn("obj_file_name '{}'\n", obj_file_name); - this_record_len += obj_file_name.len + 1; - - const march_forward_bytes = this_record_len % 4; - if (march_forward_bytes != 0) { - try dbi.seekForward(march_forward_bytes); - this_record_len += march_forward_bytes; - } - - try modules.append(Module{ - .mod_info = mod_info, - .module_name = module_name, - .obj_file_name = obj_file_name, - }); - - mod_info_offset += this_record_len; - if (mod_info_offset > header.ModInfoSize) - return error.InvalidDebugInfo; - } - - // Section Contribution Substream - var sect_contribs = ArrayList(SectionContribEntry).init(self.allocator); - std.debug.warn("looking at Section Contributinos now\n"); - var sect_cont_offset: usize = 0; - if (header.SectionContributionSize != 0) { - const ver = @intToEnum(SectionContrSubstreamVersion, try dbi.stream.readIntLe(u32)); - if (ver != SectionContrSubstreamVersion.Ver60) - return error.InvalidDebugInfo; - sect_cont_offset += @sizeOf(u32); - } - while (sect_cont_offset != header.SectionContributionSize) { - const entry = try sect_contribs.addOne(); - try dbi.stream.readStruct(SectionContribEntry, entry); - std.debug.warn("{}\n", entry); - sect_cont_offset += @sizeOf(SectionContribEntry); - - if (sect_cont_offset > header.SectionContributionSize) - return error.InvalidDebugInfo; - } - //std.debug.warn("looking at section map now\n"); - //if (header.SectionMapSize == 0) - // return error.MissingDebugInfo; - - //var sect_map_hdr: SectionMapHeader = undefined; - //try dbi.stream.readStruct(SectionMapHeader, §_map_hdr); - - //const sect_entries = try self.allocator.alloc(SectionMapEntry, sect_map_hdr.Count); - //const as_bytes = @sliceToBytes(sect_entries); - //if (as_bytes.len + @sizeOf(SectionMapHeader) != header.SectionMapSize) - // return error.InvalidDebugInfo; - //try dbi.stream.readNoEof(as_bytes); - - //for (sect_entries) |sect_entry| { - // std.debug.warn("{}\n", sect_entry); - //} - - var coff_section: *coff.Section = undefined; - const mod_index = for (sect_contribs.toSlice()) |sect_contrib| { - coff_section = &self.coff.sections.toSlice()[sect_contrib.Section]; - std.debug.warn("looking in coff name: {}\n", mem.toSliceConst(u8, &coff_section.header.name)); - - const vaddr_start = coff_section.header.virtual_address + sect_contrib.Offset; - const vaddr_end = vaddr_start + sect_contrib.Size; - if (address >= vaddr_start and address < vaddr_end) { - std.debug.warn("found sect contrib: {}\n", sect_contrib); - break sect_contrib.ModuleIndex; - } - } else return error.MissingDebugInfo; - - const mod = &modules.toSlice()[mod_index]; - const modi = self.getStreamById(mod.mod_info.ModuleSymStream) orelse return error.InvalidDebugInfo; - - const signature = try modi.stream.readIntLe(u32); - if (signature != 4) - return error.InvalidDebugInfo; - - const symbols = try self.allocator.alloc(u8, mod.mod_info.SymByteSize - 4); - std.debug.warn("read {} bytes of symbol info\n", symbols.len); - try modi.stream.readNoEof(symbols); - var symbol_i: usize = 0; - const proc_sym = while (symbol_i != symbols.len) { - const prefix = @ptrCast(*RecordPrefix, &symbols[symbol_i]); - if (prefix.RecordLen < 2) - return error.InvalidDebugInfo; - switch (prefix.RecordKind) { - SymbolKind.S_LPROC32 => { - const proc_sym = @ptrCast(*ProcSym, &symbols[symbol_i + @sizeOf(RecordPrefix)]); - const vaddr_start = coff_section.header.virtual_address + proc_sym.CodeOffset; - const vaddr_end = vaddr_start + proc_sym.CodeSize; - std.debug.warn(" {}\n", proc_sym); - if (address >= vaddr_start and address < vaddr_end) { - break proc_sym; - } - }, - else => {}, - } - symbol_i += prefix.RecordLen + @sizeOf(u16); - if (symbol_i > symbols.len) - return error.InvalidDebugInfo; - } else return error.MissingDebugInfo; - - std.debug.warn("found in {s}: {}\n", @ptrCast([*]u8, proc_sym) + @sizeOf(ProcSym), proc_sym); - - if (mod.mod_info.C11ByteSize != 0) - return error.InvalidDebugInfo; - - if (mod.mod_info.C13ByteSize == 0) { - return error.MissingDebugInfo; - } - - const subsect_info = try self.allocator.alloc(u8, mod.mod_info.C13ByteSize); - std.debug.warn("read C13 line info {} bytes\n", subsect_info.len); - const line_info_file_pos = modi.getFilePos(); - try modi.stream.readNoEof(subsect_info); - - const DebugSubsectionHeader = packed struct { - Kind: DebugSubsectionKind, /// codeview::DebugSubsectionKind enum - Length: u32, /// number of bytes occupied by this record. - }; - var sect_offset: usize = 0; - var skip_len: usize = undefined; - var have_line_info: bool = false; - subsections: while (sect_offset != subsect_info.len) : (sect_offset += skip_len) { - const subsect_hdr = @ptrCast(*DebugSubsectionHeader, &subsect_info[sect_offset]); - skip_len = subsect_hdr.Length; - sect_offset += @sizeOf(DebugSubsectionHeader); - - switch (subsect_hdr.Kind) { - DebugSubsectionKind.Lines => { - if (have_line_info) - continue :subsections; - - var line_index: usize = sect_offset; - - const line_hdr = @ptrCast(*LineFragmentHeader, &subsect_info[line_index]); - if (line_hdr.RelocSegment == 0) return error.MissingDebugInfo; - std.debug.warn("{}\n", line_hdr); - line_index += @sizeOf(LineFragmentHeader); - - const block_hdr = @ptrCast(*LineBlockFragmentHeader, &subsect_info[line_index]); - std.debug.warn("{}\n", block_hdr); - line_index += @sizeOf(LineBlockFragmentHeader); - - const has_column = line_hdr.Flags.LF_HaveColumns; - std.debug.warn("has column: {}\n", has_column); - - const frag_vaddr_start = coff_section.header.virtual_address + line_hdr.RelocOffset; - const frag_vaddr_end = frag_vaddr_start + line_hdr.CodeSize; - if (address >= frag_vaddr_start and address < frag_vaddr_end) { - std.debug.warn("found line listing\n"); - var line_i: usize = 0; - const start_line_index = line_index; - while (line_i < block_hdr.NumLines) : (line_i += 1) { - const line_num_entry = @ptrCast(*LineNumberEntry, &subsect_info[line_index]); - line_index += @sizeOf(LineNumberEntry); - const flags = @ptrCast(*LineNumberEntry.Flags, &line_num_entry.Flags); - std.debug.warn("{} {}\n", line_num_entry, flags); - const vaddr_start = frag_vaddr_start + line_num_entry.Offset; - const vaddr_end = if (flags.End == 0) frag_vaddr_end else vaddr_start + flags.End; - std.debug.warn("test {x} <= {x} < {x}\n", vaddr_start, address, vaddr_end); - if (address >= vaddr_start and address < vaddr_end) { - std.debug.warn("{} line {}\n", block_hdr.NameIndex, flags.Start); - if (has_column) { - line_index = start_line_index + @sizeOf(LineNumberEntry) * block_hdr.NumLines; - line_index += @sizeOf(ColumnNumberEntry) * line_i; - const col_num_entry = @ptrCast(*ColumnNumberEntry, &subsect_info[line_index]); - std.debug.warn("col {}\n", col_num_entry.StartColumn); - } - have_line_info = true; - continue :subsections; - } - } - return error.MissingDebugInfo; - } - - }, - DebugSubsectionKind.FileChecksums => { - var chksum_index: usize = sect_offset; - - while (chksum_index != subsect_info.len) { - const chksum_hdr = @ptrCast(*FileChecksumEntryHeader, &subsect_info[chksum_index]); - std.debug.warn("{}\n", chksum_hdr); - const len = @sizeOf(FileChecksumEntryHeader) + chksum_hdr.ChecksumSize; - chksum_index += len + (len % 4); - if (chksum_index > subsect_info.len) - return error.InvalidDebugInfo; - } - - }, - else => { - std.debug.warn("ignore subsection {}\n", @tagName(subsect_hdr.Kind)); - }, - } - - if (sect_offset > subsect_info.len) - return error.InvalidDebugInfo; - } - std.debug.warn("end subsections\n"); - } }; // see https://llvm.org/docs/PDB/MsfFile.html @@ -687,13 +471,11 @@ const Msf = struct { ); const stream_count = try self.directory.stream.readIntLe(u32); - warn("stream count {}\n", stream_count); const stream_sizes = try allocator.alloc(u32, stream_count); for (stream_sizes) |*s| { const size = try self.directory.stream.readIntLe(u32); s.* = blockCountFromSize(size, superblock.BlockSize); - warn("stream {}B {} blocks\n", size, s.*); } self.streams = try allocator.alloc(MsfStream, stream_count); @@ -784,13 +566,10 @@ const MsfStream = struct { const in = &file_stream.stream; try file.seekTo(pos); - warn("stream with blocks"); var i: u32 = 0; while (i < block_count) : (i += 1) { stream.blocks[i] = try in.readIntLe(u32); - warn(" {}", stream.blocks[i]); } - warn("\n"); return stream; } @@ -812,10 +591,6 @@ const MsfStream = struct { var block = self.blocks[block_id]; var offset = self.pos % self.block_size; - //std.debug.warn("seek {} read {}B: block_id={} block={} offset={}\n", - // block * self.block_size + offset, - // buffer.len, block_id, block, offset); - try self.in_file.seekTo(block * self.block_size + offset); var file_stream = io.FileInStream.init(self.in_file); const in = &file_stream.stream; From 4cd50865bffab670040f20b69da13d2f3e89f86f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 2 Sep 2018 17:58:50 -0400 Subject: [PATCH 45/56] fix source file lookup --- std/debug/index.zig | 144 +++++++++++++++++++++----------------------- 1 file changed, 67 insertions(+), 77 deletions(-) diff --git a/std/debug/index.zig b/std/debug/index.zig index de4d7745c6..2930819b3a 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -28,7 +28,7 @@ const Module = struct { populated: bool, symbols: []u8, subsect_info: []u8, - checksums: []u32, + checksum_offset: ?usize, }; /// Tries to write to stderr, unbuffered, and ignores any error returned. @@ -276,6 +276,7 @@ fn printSourceAtAddressWindows(di: *DebugInfo, out_stream: var, relocated_addres var coff_section: *coff.Section = undefined; const mod_index = for (di.sect_contribs) |sect_contrib| { + if (sect_contrib.Section >= di.coff.sections.len) continue; coff_section = &di.coff.sections.toSlice()[sect_contrib.Section]; const vaddr_start = coff_section.header.virtual_address + sect_contrib.Offset; @@ -326,72 +327,80 @@ fn printSourceAtAddressWindows(di: *DebugInfo, out_stream: var, relocated_addres var sect_offset: usize = 0; var skip_len: usize = undefined; - const opt_line_info = subsections: while (sect_offset != subsect_info.len) : (sect_offset += skip_len) { - const subsect_hdr = @ptrCast(*pdb.DebugSubsectionHeader, &subsect_info[sect_offset]); - skip_len = subsect_hdr.Length; - sect_offset += @sizeOf(pdb.DebugSubsectionHeader); + const opt_line_info = subsections: { + const checksum_offset = mod.checksum_offset orelse break :subsections null; + while (sect_offset != subsect_info.len) : (sect_offset += skip_len) { + const subsect_hdr = @ptrCast(*pdb.DebugSubsectionHeader, &subsect_info[sect_offset]); + skip_len = subsect_hdr.Length; + sect_offset += @sizeOf(pdb.DebugSubsectionHeader); - switch (subsect_hdr.Kind) { - pdb.DebugSubsectionKind.Lines => { - var line_index: usize = sect_offset; + switch (subsect_hdr.Kind) { + pdb.DebugSubsectionKind.Lines => { + var line_index: usize = sect_offset; - const line_hdr = @ptrCast(*pdb.LineFragmentHeader, &subsect_info[line_index]); - if (line_hdr.RelocSegment == 0) return error.MissingDebugInfo; - line_index += @sizeOf(pdb.LineFragmentHeader); + const line_hdr = @ptrCast(*pdb.LineFragmentHeader, &subsect_info[line_index]); + if (line_hdr.RelocSegment == 0) return error.MissingDebugInfo; + line_index += @sizeOf(pdb.LineFragmentHeader); - const block_hdr = @ptrCast(*pdb.LineBlockFragmentHeader, &subsect_info[line_index]); - line_index += @sizeOf(pdb.LineBlockFragmentHeader); + const block_hdr = @ptrCast(*pdb.LineBlockFragmentHeader, &subsect_info[line_index]); + line_index += @sizeOf(pdb.LineBlockFragmentHeader); - const has_column = line_hdr.Flags.LF_HaveColumns; + const has_column = line_hdr.Flags.LF_HaveColumns; - const frag_vaddr_start = coff_section.header.virtual_address + line_hdr.RelocOffset; - const frag_vaddr_end = frag_vaddr_start + line_hdr.CodeSize; - if (relative_address >= frag_vaddr_start and relative_address < frag_vaddr_end) { - var line_i: usize = 0; - const start_line_index = line_index; - while (line_i < block_hdr.NumLines) : (line_i += 1) { - const line_num_entry = @ptrCast(*pdb.LineNumberEntry, &subsect_info[line_index]); - line_index += @sizeOf(pdb.LineNumberEntry); - const flags = @ptrCast(*pdb.LineNumberEntry.Flags, &line_num_entry.Flags); - const vaddr_start = frag_vaddr_start + line_num_entry.Offset; - const vaddr_end = if (flags.End == 0) frag_vaddr_end else vaddr_start + flags.End; - if (relative_address >= vaddr_start and relative_address < vaddr_end) { - const chksum_index = block_hdr.NameIndex; - std.debug.warn("looking up checksum {}\n", chksum_index); - const strtab_offset = mod.checksums[chksum_index]; - try di.pdb.string_table.seekTo(@sizeOf(pdb.PDBStringTableHeader) + strtab_offset); - const source_file_name = try di.pdb.string_table.readNullTermString(allocator); - const line = flags.Start; - const column = if (has_column) blk: { - line_index = start_line_index + @sizeOf(pdb.LineNumberEntry) * block_hdr.NumLines; - line_index += @sizeOf(pdb.ColumnNumberEntry) * line_i; - const col_num_entry = @ptrCast(*pdb.ColumnNumberEntry, &subsect_info[line_index]); - break :blk col_num_entry.StartColumn; - } else 0; - break :subsections LineInfo{ - .allocator = allocator, - .file_name = source_file_name, - .line = line, - .column = column, - }; + const frag_vaddr_start = coff_section.header.virtual_address + line_hdr.RelocOffset; + const frag_vaddr_end = frag_vaddr_start + line_hdr.CodeSize; + if (relative_address >= frag_vaddr_start and relative_address < frag_vaddr_end) { + var line_i: usize = 0; + const start_line_index = line_index; + while (line_i < block_hdr.NumLines) : (line_i += 1) { + const line_num_entry = @ptrCast(*pdb.LineNumberEntry, &subsect_info[line_index]); + line_index += @sizeOf(pdb.LineNumberEntry); + const flags = @ptrCast(*pdb.LineNumberEntry.Flags, &line_num_entry.Flags); + const vaddr_start = frag_vaddr_start + line_num_entry.Offset; + const vaddr_end = if (flags.End == 0) frag_vaddr_end else vaddr_start + flags.End; + if (relative_address >= vaddr_start and relative_address < vaddr_end) { + const subsect_index = checksum_offset + block_hdr.NameIndex; + const chksum_hdr = @ptrCast(*pdb.FileChecksumEntryHeader, &mod.subsect_info[subsect_index]); + const strtab_offset = @sizeOf(pdb.PDBStringTableHeader) + chksum_hdr.FileNameOffset; + try di.pdb.string_table.seekTo(strtab_offset); + const source_file_name = try di.pdb.string_table.readNullTermString(allocator); + const line = flags.Start; + const column = if (has_column) blk: { + line_index = start_line_index + @sizeOf(pdb.LineNumberEntry) * block_hdr.NumLines; + line_index += @sizeOf(pdb.ColumnNumberEntry) * line_i; + const col_num_entry = @ptrCast(*pdb.ColumnNumberEntry, &subsect_info[line_index]); + break :blk col_num_entry.StartColumn; + } else 0; + break :subsections LineInfo{ + .allocator = allocator, + .file_name = source_file_name, + .line = line, + .column = column, + }; + } } + break :subsections null; } - break :subsections null; - } - }, - else => {}, - } + }, + else => {}, + } - if (sect_offset > subsect_info.len) - return error.InvalidDebugInfo; - } else null; + if (sect_offset > subsect_info.len) + return error.InvalidDebugInfo; + } else { + break :subsections null; + } + }; if (tty_color) { + setTtyColor(TtyColor.White); if (opt_line_info) |li| { - try out_stream.print("{}:{}:{}: ", li.file_name, li.line, li.column); + try out_stream.print("{}:{}:{}", li.file_name, li.line, li.column); } else { - try out_stream.print("???:?:?: "); + try out_stream.print("???:?:?"); } + setTtyColor(TtyColor.Reset); + try out_stream.print(": "); setTtyColor(TtyColor.Dim); try out_stream.print("0x{x} in {} ({})", relocated_address, symbol_name, obj_basename); setTtyColor(TtyColor.Reset); @@ -470,8 +479,7 @@ fn setTtyColor(tty_color: TtyColor) void { windows.FOREGROUND_RED|windows.FOREGROUND_GREEN|windows.FOREGROUND_BLUE|windows.FOREGROUND_INTENSITY); }, TtyColor.Dim => { - _ = windows.SetConsoleTextAttribute(stderr_file.handle, - windows.FOREGROUND_RED|windows.FOREGROUND_GREEN|windows.FOREGROUND_BLUE); + _ = windows.SetConsoleTextAttribute(stderr_file.handle, windows.FOREGROUND_INTENSITY); }, TtyColor.Reset => { _ = windows.SetConsoleTextAttribute(stderr_file.handle, S.attrs); @@ -502,7 +510,6 @@ fn populateModule(di: *DebugInfo, mod: *Module) !void { mod.subsect_info = try allocator.alloc(u8, mod.mod_info.C13ByteSize); try modi.stream.readNoEof(mod.subsect_info); - var checksum_list = ArrayList(u32).init(allocator); var sect_offset: usize = 0; var skip_len: usize = undefined; while (sect_offset != mod.subsect_info.len) : (sect_offset += skip_len) { @@ -512,18 +519,8 @@ fn populateModule(di: *DebugInfo, mod: *Module) !void { switch (subsect_hdr.Kind) { pdb.DebugSubsectionKind.FileChecksums => { - var chksum_index: usize = sect_offset; - - while (chksum_index != mod.subsect_info.len) { - const chksum_hdr = @ptrCast(*pdb.FileChecksumEntryHeader, &mod.subsect_info[chksum_index]); - std.debug.warn("{} {}\n", checksum_list.len, chksum_hdr); - try checksum_list.append(chksum_hdr.FileNameOffset); - const len = @sizeOf(pdb.FileChecksumEntryHeader) + chksum_hdr.ChecksumSize; - chksum_index += len + (len % 4); - if (chksum_index > mod.subsect_info.len) - return error.InvalidDebugInfo; - } - + mod.checksum_offset = sect_offset; + break; }, else => {}, } @@ -531,13 +528,6 @@ fn populateModule(di: *DebugInfo, mod: *Module) !void { if (sect_offset > mod.subsect_info.len) return error.InvalidDebugInfo; } - mod.checksums = checksum_list.toOwnedSlice(); - - for (mod.checksums) |strtab_offset| { - try di.pdb.string_table.seekTo(@sizeOf(pdb.PDBStringTableHeader) + strtab_offset); - const source_file_name = try di.pdb.string_table.readNullTermString(allocator); - std.debug.warn("{}={}\n", strtab_offset, source_file_name); - } mod.populated = true; } @@ -807,7 +797,7 @@ fn openSelfDebugInfoWindows(allocator: *mem.Allocator) !DebugInfo { .populated = false, .symbols = undefined, .subsect_info = undefined, - .checksums = undefined, + .checksum_offset = null, }); mod_info_offset += this_record_len; From 832caefc2a1b20deb513d43306d6723670ba9c8f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 2 Sep 2018 18:35:32 -0400 Subject: [PATCH 46/56] fix regressions --- doc/docgen.zig | 4 ++-- example/guess_number/main.zig | 2 +- src-self-hosted/errmsg.zig | 2 +- src-self-hosted/main.zig | 12 ++++++------ src-self-hosted/test.zig | 4 ++-- std/debug/index.zig | 14 +++++++------- std/elf.zig | 4 ++-- std/event/tcp.zig | 6 +++--- std/io.zig | 6 +++--- std/io_test.zig | 4 ++-- std/os/child_process.zig | 4 ++-- std/os/index.zig | 1 + std/special/build_runner.zig | 4 ++-- std/zig/parser_test.zig | 2 +- test/compare_output.zig | 30 +++++++++++++++--------------- test/tests.zig | 12 ++++++------ 16 files changed, 56 insertions(+), 55 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index 3145c4483e..c1158dc03f 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -40,11 +40,11 @@ pub fn main() !void { var out_file = try os.File.openWrite(out_file_name); defer out_file.close(); - var file_in_stream = io.FileInStream.init(&in_file); + var file_in_stream = io.FileInStream.init(in_file); const input_file_bytes = try file_in_stream.stream.readAllAlloc(allocator, max_doc_file_size); - var file_out_stream = io.FileOutStream.init(&out_file); + var file_out_stream = io.FileOutStream.init(out_file); var buffered_out_stream = io.BufferedOutStream(io.FileOutStream.Error).init(&file_out_stream.stream); var tokenizer = Tokenizer.init(in_file_name, input_file_bytes); diff --git a/example/guess_number/main.zig b/example/guess_number/main.zig index bed132b25c..062f93e7f7 100644 --- a/example/guess_number/main.zig +++ b/example/guess_number/main.zig @@ -6,7 +6,7 @@ const os = std.os; pub fn main() !void { var stdout_file = try io.getStdOut(); - var stdout_file_stream = io.FileOutStream.init(&stdout_file); + var stdout_file_stream = io.FileOutStream.init(stdout_file); const stdout = &stdout_file_stream.stream; try stdout.print("Welcome to the Guess Number Game in Zig.\n"); diff --git a/src-self-hosted/errmsg.zig b/src-self-hosted/errmsg.zig index 028c2e2174..6cf29b9441 100644 --- a/src-self-hosted/errmsg.zig +++ b/src-self-hosted/errmsg.zig @@ -272,7 +272,7 @@ pub const Msg = struct { try stream.write("\n"); } - pub fn printToFile(msg: *const Msg, file: *os.File, color: Color) !void { + pub fn printToFile(msg: *const Msg, file: os.File, color: Color) !void { const color_on = switch (color) { Color.Auto => file.isTty(), Color.On => true, diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 64c55a24e8..6a450030ca 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -55,11 +55,11 @@ pub fn main() !void { const allocator = std.heap.c_allocator; var stdout_file = try std.io.getStdOut(); - var stdout_out_stream = std.io.FileOutStream.init(&stdout_file); + var stdout_out_stream = std.io.FileOutStream.init(stdout_file); stdout = &stdout_out_stream.stream; stderr_file = try std.io.getStdErr(); - var stderr_out_stream = std.io.FileOutStream.init(&stderr_file); + var stderr_out_stream = std.io.FileOutStream.init(stderr_file); stderr = &stderr_out_stream.stream; const args = try os.argsAlloc(allocator); @@ -491,7 +491,7 @@ async fn processBuildEvents(comp: *Compilation, color: errmsg.Color) void { stderr.print("Build {} compile errors:\n", count) catch os.exit(1); for (msgs) |msg| { defer msg.destroy(); - msg.printToFile(&stderr_file, color) catch os.exit(1); + msg.printToFile(stderr_file, color) catch os.exit(1); } }, } @@ -619,7 +619,7 @@ fn cmdFmt(allocator: *Allocator, args: []const []const u8) !void { } var stdin_file = try io.getStdIn(); - var stdin = io.FileInStream.init(&stdin_file); + var stdin = io.FileInStream.init(stdin_file); const source_code = try stdin.stream.readAllAlloc(allocator, max_src_size); defer allocator.free(source_code); @@ -635,7 +635,7 @@ fn cmdFmt(allocator: *Allocator, args: []const []const u8) !void { const msg = try errmsg.Msg.createFromParseError(allocator, parse_error, &tree, ""); defer msg.destroy(); - try msg.printToFile(&stderr_file, color); + try msg.printToFile(stderr_file, color); } if (tree.errors.len != 0) { os.exit(1); @@ -772,7 +772,7 @@ async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8) FmtError!void { const msg = try errmsg.Msg.createFromParseError(fmt.loop.allocator, parse_error, &tree, file_path); defer fmt.loop.allocator.destroy(msg); - try msg.printToFile(&stderr_file, fmt.color); + try msg.printToFile(stderr_file, fmt.color); } if (tree.errors.len != 0) { fmt.any_error = true; diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index d4a45e7a04..4f377d4247 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -185,7 +185,7 @@ pub const TestContext = struct { try stderr.write("build incorrectly failed:\n"); for (msgs) |msg| { defer msg.destroy(); - try msg.printToFile(&stderr, errmsg.Color.Auto); + try msg.printToFile(stderr, errmsg.Color.Auto); } }, } @@ -234,7 +234,7 @@ pub const TestContext = struct { var stderr = try std.io.getStdErr(); for (msgs) |msg| { defer msg.destroy(); - try msg.printToFile(&stderr, errmsg.Color.Auto); + try msg.printToFile(stderr, errmsg.Color.Auto); } std.debug.warn("============\n"); return error.TestFailed; diff --git a/std/debug/index.zig b/std/debug/index.zig index 2930819b3a..8db7c75d2c 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -862,7 +862,7 @@ fn openSelfDebugInfoLinux(allocator: *mem.Allocator) !DebugInfo { di.self_exe_file = try os.openSelfExe(); errdefer di.self_exe_file.close(); - try di.elf.openFile(allocator, &di.self_exe_file); + try di.elf.openFile(allocator, di.self_exe_file); errdefer di.elf.close(); di.debug_info = (try di.elf.findSection(".debug_info")) orelse return error.MissingDebugInfo; @@ -1067,7 +1067,7 @@ pub const DebugInfo = switch (builtin.os) { } pub fn readString(self: *DebugInfo) ![]u8 { - var in_file_stream = io.FileInStream.init(&self.self_exe_file); + var in_file_stream = io.FileInStream.init(self.self_exe_file); const in_stream = &in_file_stream.stream; return readStringRaw(self.allocator(), in_stream); } @@ -1403,7 +1403,7 @@ fn parseFormValue(allocator: *mem.Allocator, in_stream: var, form_id: u64, is_64 } fn parseAbbrevTable(st: *DebugInfo) !AbbrevTable { - const in_file = &st.self_exe_file; + const in_file = st.self_exe_file; var in_file_stream = io.FileInStream.init(in_file); const in_stream = &in_file_stream.stream; var result = AbbrevTable.init(st.allocator()); @@ -1454,7 +1454,7 @@ fn getAbbrevTableEntry(abbrev_table: *const AbbrevTable, abbrev_code: u64) ?*con } fn parseDie(st: *DebugInfo, abbrev_table: *const AbbrevTable, is_64: bool) !Die { - const in_file = &st.self_exe_file; + const in_file = st.self_exe_file; var in_file_stream = io.FileInStream.init(in_file); const in_stream = &in_file_stream.stream; const abbrev_code = try readULeb128(in_stream); @@ -1676,7 +1676,7 @@ fn getLineNumberInfoMacOs(di: *DebugInfo, symbol: MachoSymbol, target_address: u fn getLineNumberInfoLinux(di: *DebugInfo, compile_unit: *const CompileUnit, target_address: usize) !LineInfo { const compile_unit_cwd = try compile_unit.die.getAttrString(di, DW.AT_comp_dir); - const in_file = &di.self_exe_file; + const in_file = di.self_exe_file; const debug_line_end = di.debug_line.offset + di.debug_line.size; var this_offset = di.debug_line.offset; var this_index: usize = 0; @@ -1856,7 +1856,7 @@ fn scanAllCompileUnits(st: *DebugInfo) !void { var this_unit_offset = st.debug_info.offset; var cu_index: usize = 0; - var in_file_stream = io.FileInStream.init(&st.self_exe_file); + var in_file_stream = io.FileInStream.init(st.self_exe_file); const in_stream = &in_file_stream.stream; while (this_unit_offset < debug_info_end) { @@ -1922,7 +1922,7 @@ fn scanAllCompileUnits(st: *DebugInfo) !void { } fn findCompileUnit(st: *DebugInfo, target_address: u64) !*const CompileUnit { - var in_file_stream = io.FileInStream.init(&st.self_exe_file); + var in_file_stream = io.FileInStream.init(st.self_exe_file); const in_stream = &in_file_stream.stream; for (st.compile_unit_list.toSlice()) |*compile_unit| { if (compile_unit.pc_range) |range| { diff --git a/std/elf.zig b/std/elf.zig index 3d81555319..a3a72dc728 100644 --- a/std/elf.zig +++ b/std/elf.zig @@ -353,7 +353,7 @@ pub const SectionHeader = struct { }; pub const Elf = struct { - in_file: *os.File, + in_file: os.File, auto_close_stream: bool, is_64: bool, endian: builtin.Endian, @@ -376,7 +376,7 @@ pub const Elf = struct { } /// Call close when done. - pub fn openFile(elf: *Elf, allocator: *mem.Allocator, file: *os.File) !void { + pub fn openFile(elf: *Elf, allocator: *mem.Allocator, file: os.File) !void { elf.allocator = allocator; elf.in_file = file; elf.auto_close_stream = false; diff --git a/std/event/tcp.zig b/std/event/tcp.zig index d8b97659a9..491acab39d 100644 --- a/std/event/tcp.zig +++ b/std/event/tcp.zig @@ -145,11 +145,11 @@ test "listen on a port, send bytes, receive bytes" { cancel @handle(); } } - async fn errorableHandler(self: *Self, _addr: *const std.net.Address, _socket: *const std.os.File) !void { + async fn errorableHandler(self: *Self, _addr: *const std.net.Address, _socket: std.os.File) !void { const addr = _addr.*; // TODO https://github.com/ziglang/zig/issues/733 - var socket = _socket.*; // TODO https://github.com/ziglang/zig/issues/733 + var socket = _socket; // TODO https://github.com/ziglang/zig/issues/733 - var adapter = std.io.FileOutStream.init(&socket); + var adapter = std.io.FileOutStream.init(socket); var stream = &adapter.stream; try stream.print("hello from server\n"); } diff --git a/std/io.zig b/std/io.zig index a1a77271e1..2b31bc0548 100644 --- a/std/io.zig +++ b/std/io.zig @@ -280,7 +280,7 @@ pub fn readFileAllocAligned(allocator: *mem.Allocator, path: []const u8, comptim const buf = try allocator.alignedAlloc(u8, A, size); errdefer allocator.free(buf); - var adapter = FileInStream.init(&file); + var adapter = FileInStream.init(file); try adapter.stream.readNoEof(buf[0..size]); return buf; } @@ -592,7 +592,7 @@ pub const BufferedAtomicFile = struct { self.atomic_file = try os.AtomicFile.init(allocator, dest_path, os.File.default_mode); errdefer self.atomic_file.deinit(); - self.file_stream = FileOutStream.init(&self.atomic_file.file); + self.file_stream = FileOutStream.init(self.atomic_file.file); self.buffered_stream = BufferedOutStream(FileOutStream.Error).init(&self.file_stream.stream); return self; } @@ -622,7 +622,7 @@ test "import io tests" { pub fn readLine(buf: []u8) !usize { var stdin = getStdIn() catch return error.StdInUnavailable; - var adapter = FileInStream.init(&stdin); + var adapter = FileInStream.init(stdin); var stream = &adapter.stream; var index: usize = 0; while (true) { diff --git a/std/io_test.zig b/std/io_test.zig index 7a44032673..7403c96994 100644 --- a/std/io_test.zig +++ b/std/io_test.zig @@ -19,7 +19,7 @@ test "write a file, read it, then delete it" { var file = try os.File.openWrite(tmp_file_name); defer file.close(); - var file_out_stream = io.FileOutStream.init(&file); + var file_out_stream = io.FileOutStream.init(file); var buf_stream = io.BufferedOutStream(io.FileOutStream.Error).init(&file_out_stream.stream); const st = &buf_stream.stream; try st.print("begin"); @@ -35,7 +35,7 @@ test "write a file, read it, then delete it" { const expected_file_size = "begin".len + data.len + "end".len; assert(file_size == expected_file_size); - var file_in_stream = io.FileInStream.init(&file); + var file_in_stream = io.FileInStream.init(file); var buf_stream = io.BufferedInStream(io.FileInStream.Error).init(&file_in_stream.stream); const st = &buf_stream.stream; const contents = try st.readAllAlloc(allocator, 2 * 1024); diff --git a/std/os/child_process.zig b/std/os/child_process.zig index b79a8de16f..decd8d04fa 100644 --- a/std/os/child_process.zig +++ b/std/os/child_process.zig @@ -209,8 +209,8 @@ pub const ChildProcess = struct { defer Buffer.deinit(&stdout); defer Buffer.deinit(&stderr); - var stdout_file_in_stream = io.FileInStream.init(&child.stdout.?); - var stderr_file_in_stream = io.FileInStream.init(&child.stderr.?); + var stdout_file_in_stream = io.FileInStream.init(child.stdout.?); + var stderr_file_in_stream = io.FileInStream.init(child.stderr.?); try stdout_file_in_stream.stream.readAllBuffer(&stdout, max_output_size); try stderr_file_in_stream.stream.readAllBuffer(&stderr, max_output_size); diff --git a/std/os/index.zig b/std/os/index.zig index 03337b63bc..9b49a05067 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -2149,6 +2149,7 @@ pub fn selfExePath(out_buffer: *[MAX_PATH_BYTES]u8) ![]u8 { switch (builtin.os) { Os.linux => return readLink(out_buffer, "/proc/self/exe"), Os.windows => { + var utf16le_buf: [windows_util.PATH_MAX_WIDE]u16 = undefined; const utf16le_slice = try selfExePathW(&utf16le_buf); // Trust that Windows gives us valid UTF-16LE. const end_index = std.unicode.utf16leToUtf8(out_buffer, utf16le_slice) catch unreachable; diff --git a/std/special/build_runner.zig b/std/special/build_runner.zig index 982c60aed8..8cf237f634 100644 --- a/std/special/build_runner.zig +++ b/std/special/build_runner.zig @@ -49,14 +49,14 @@ pub fn main() !void { var stderr_file = io.getStdErr(); var stderr_file_stream: io.FileOutStream = undefined; - var stderr_stream = if (stderr_file) |*f| x: { + var stderr_stream = if (stderr_file) |f| x: { stderr_file_stream = io.FileOutStream.init(f); break :x &stderr_file_stream.stream; } else |err| err; var stdout_file = io.getStdOut(); var stdout_file_stream: io.FileOutStream = undefined; - var stdout_stream = if (stdout_file) |*f| x: { + var stdout_stream = if (stdout_file) |f| x: { stdout_file_stream = io.FileOutStream.init(f); break :x &stdout_file_stream.stream; } else |err| err; diff --git a/std/zig/parser_test.zig b/std/zig/parser_test.zig index 6ea25e54f1..7f3ce7bd8a 100644 --- a/std/zig/parser_test.zig +++ b/std/zig/parser_test.zig @@ -1865,7 +1865,7 @@ var fixed_buffer_mem: [100 * 1024]u8 = undefined; fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *bool) ![]u8 { var stderr_file = try io.getStdErr(); - var stderr = &io.FileOutStream.init(&stderr_file).stream; + var stderr = &io.FileOutStream.init(stderr_file).stream; var tree = try std.zig.parse(allocator, source); defer tree.deinit(); diff --git a/test/compare_output.zig b/test/compare_output.zig index a18a78b419..bcd9d15b9c 100644 --- a/test/compare_output.zig +++ b/test/compare_output.zig @@ -19,7 +19,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ \\pub fn main() void { \\ privateFunction(); - \\ const stdout = &(FileOutStream.init(&(getStdOut() catch unreachable)).stream); + \\ const stdout = &FileOutStream.init(getStdOut() catch unreachable).stream; \\ stdout.print("OK 2\n") catch unreachable; \\} \\ @@ -34,7 +34,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\// purposefully conflicting function with main.zig \\// but it's private so it should be OK \\fn privateFunction() void { - \\ const stdout = &(FileOutStream.init(&(getStdOut() catch unreachable)).stream); + \\ const stdout = &FileOutStream.init(getStdOut() catch unreachable).stream; \\ stdout.print("OK 1\n") catch unreachable; \\} \\ @@ -60,7 +60,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { tc.addSourceFile("foo.zig", \\use @import("std").io; \\pub fn foo_function() void { - \\ const stdout = &(FileOutStream.init(&(getStdOut() catch unreachable)).stream); + \\ const stdout = &FileOutStream.init(getStdOut() catch unreachable).stream; \\ stdout.print("OK\n") catch unreachable; \\} ); @@ -71,7 +71,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ \\pub fn bar_function() void { \\ if (foo_function()) { - \\ const stdout = &(FileOutStream.init(&(getStdOut() catch unreachable)).stream); + \\ const stdout = &FileOutStream.init(getStdOut() catch unreachable).stream; \\ stdout.print("OK\n") catch unreachable; \\ } \\} @@ -103,7 +103,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\pub const a_text = "OK\n"; \\ \\pub fn ok() void { - \\ const stdout = &(io.FileOutStream.init(&(io.getStdOut() catch unreachable)).stream); + \\ const stdout = &io.FileOutStream.init(io.getStdOut() catch unreachable).stream; \\ stdout.print(b_text) catch unreachable; \\} ); @@ -121,7 +121,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\const io = @import("std").io; \\ \\pub fn main() void { - \\ const stdout = &(io.FileOutStream.init(&(io.getStdOut() catch unreachable)).stream); + \\ const stdout = &io.FileOutStream.init(io.getStdOut() catch unreachable).stream; \\ stdout.print("Hello, world!\n{d4} {x3} {c}\n", u32(12), u16(0x12), u8('a')) catch unreachable; \\} , "Hello, world!\n0012 012 a\n"); @@ -274,7 +274,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ var x_local : i32 = print_ok(x); \\} \\fn print_ok(val: @typeOf(x)) @typeOf(foo) { - \\ const stdout = &(io.FileOutStream.init(&(io.getStdOut() catch unreachable)).stream); + \\ const stdout = &io.FileOutStream.init(io.getStdOut() catch unreachable).stream; \\ stdout.print("OK\n") catch unreachable; \\ return 0; \\} @@ -356,7 +356,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\pub fn main() void { \\ const bar = Bar {.field2 = 13,}; \\ const foo = Foo {.field1 = bar,}; - \\ const stdout = &(io.FileOutStream.init(&(io.getStdOut() catch unreachable)).stream); + \\ const stdout = &io.FileOutStream.init(io.getStdOut() catch unreachable).stream; \\ if (!foo.method()) { \\ stdout.print("BAD\n") catch unreachable; \\ } @@ -370,7 +370,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { cases.add("defer with only fallthrough", \\const io = @import("std").io; \\pub fn main() void { - \\ const stdout = &(io.FileOutStream.init(&(io.getStdOut() catch unreachable)).stream); + \\ const stdout = &io.FileOutStream.init(io.getStdOut() catch unreachable).stream; \\ stdout.print("before\n") catch unreachable; \\ defer stdout.print("defer1\n") catch unreachable; \\ defer stdout.print("defer2\n") catch unreachable; @@ -383,7 +383,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\const io = @import("std").io; \\const os = @import("std").os; \\pub fn main() void { - \\ const stdout = &(io.FileOutStream.init(&(io.getStdOut() catch unreachable)).stream); + \\ const stdout = &io.FileOutStream.init(io.getStdOut() catch unreachable).stream; \\ stdout.print("before\n") catch unreachable; \\ defer stdout.print("defer1\n") catch unreachable; \\ defer stdout.print("defer2\n") catch unreachable; @@ -400,7 +400,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ do_test() catch return; \\} \\fn do_test() !void { - \\ const stdout = &(io.FileOutStream.init(&(io.getStdOut() catch unreachable)).stream); + \\ const stdout = &io.FileOutStream.init(io.getStdOut() catch unreachable).stream; \\ stdout.print("before\n") catch unreachable; \\ defer stdout.print("defer1\n") catch unreachable; \\ errdefer stdout.print("deferErr\n") catch unreachable; @@ -419,7 +419,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ do_test() catch return; \\} \\fn do_test() !void { - \\ const stdout = &(io.FileOutStream.init(&(io.getStdOut() catch unreachable)).stream); + \\ const stdout = &io.FileOutStream.init(io.getStdOut() catch unreachable).stream; \\ stdout.print("before\n") catch unreachable; \\ defer stdout.print("defer1\n") catch unreachable; \\ errdefer stdout.print("deferErr\n") catch unreachable; @@ -436,7 +436,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\const io = @import("std").io; \\ \\pub fn main() void { - \\ const stdout = &(io.FileOutStream.init(&(io.getStdOut() catch unreachable)).stream); + \\ const stdout = &io.FileOutStream.init(io.getStdOut() catch unreachable).stream; \\ stdout.print(foo_txt) catch unreachable; \\} , "1234\nabcd\n"); @@ -456,7 +456,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\pub fn main() !void { \\ var args_it = os.args(); \\ var stdout_file = try io.getStdOut(); - \\ var stdout_adapter = io.FileOutStream.init(&stdout_file); + \\ var stdout_adapter = io.FileOutStream.init(stdout_file); \\ const stdout = &stdout_adapter.stream; \\ var index: usize = 0; \\ _ = args_it.skip(); @@ -497,7 +497,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\pub fn main() !void { \\ var args_it = os.args(); \\ var stdout_file = try io.getStdOut(); - \\ var stdout_adapter = io.FileOutStream.init(&stdout_file); + \\ var stdout_adapter = io.FileOutStream.init(stdout_file); \\ const stdout = &stdout_adapter.stream; \\ var index: usize = 0; \\ _ = args_it.skip(); diff --git a/test/tests.zig b/test/tests.zig index aa5eed17ee..a0e1792079 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -263,8 +263,8 @@ pub const CompareOutputContext = struct { var stdout = Buffer.initNull(b.allocator); var stderr = Buffer.initNull(b.allocator); - var stdout_file_in_stream = io.FileInStream.init(&child.stdout.?); - var stderr_file_in_stream = io.FileInStream.init(&child.stderr.?); + var stdout_file_in_stream = io.FileInStream.init(child.stdout.?); + var stderr_file_in_stream = io.FileInStream.init(child.stderr.?); stdout_file_in_stream.stream.readAllBuffer(&stdout, max_stdout_size) catch unreachable; stderr_file_in_stream.stream.readAllBuffer(&stderr, max_stdout_size) catch unreachable; @@ -578,8 +578,8 @@ pub const CompileErrorContext = struct { var stdout_buf = Buffer.initNull(b.allocator); var stderr_buf = Buffer.initNull(b.allocator); - var stdout_file_in_stream = io.FileInStream.init(&child.stdout.?); - var stderr_file_in_stream = io.FileInStream.init(&child.stderr.?); + var stdout_file_in_stream = io.FileInStream.init(child.stdout.?); + var stderr_file_in_stream = io.FileInStream.init(child.stderr.?); stdout_file_in_stream.stream.readAllBuffer(&stdout_buf, max_stdout_size) catch unreachable; stderr_file_in_stream.stream.readAllBuffer(&stderr_buf, max_stdout_size) catch unreachable; @@ -842,8 +842,8 @@ pub const TranslateCContext = struct { var stdout_buf = Buffer.initNull(b.allocator); var stderr_buf = Buffer.initNull(b.allocator); - var stdout_file_in_stream = io.FileInStream.init(&child.stdout.?); - var stderr_file_in_stream = io.FileInStream.init(&child.stderr.?); + var stdout_file_in_stream = io.FileInStream.init(child.stdout.?); + var stderr_file_in_stream = io.FileInStream.init(child.stderr.?); stdout_file_in_stream.stream.readAllBuffer(&stdout_buf, max_stdout_size) catch unreachable; stderr_file_in_stream.stream.readAllBuffer(&stderr_buf, max_stdout_size) catch unreachable; From 3eb89ee4db31eef9540273b25689c0515da6bf57 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 2 Sep 2018 19:23:30 -0400 Subject: [PATCH 47/56] fixups * zig fmt * use canonical parameter order. memcpy has dest first and the base64 code follows the pattern. * pass correct radix to charToDigit --- std/fmt/index.zig | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/std/fmt/index.zig b/std/fmt/index.zig index 13a0b525d7..80af750f3d 100644 --- a/std/fmt/index.zig +++ b/std/fmt/index.zig @@ -350,7 +350,7 @@ pub fn formatText( comptime var width = 0; if (fmt.len > 1) width = comptime (parseUnsigned(usize, fmt[1..], 10) catch unreachable); return formatBuf(bytes, width, context, Errors, output); - } else if ((fmt[0] == 'x') or (fmt[0] == 'X') ) { + } else if ((fmt[0] == 'x') or (fmt[0] == 'X')) { for (bytes) |c| { try formatInt(c, 16, fmt[0] == 'X', 2, context, Errors, output); } @@ -1332,22 +1332,21 @@ pub fn isWhiteSpace(byte: u8) bool { }; } -// depends on https://github.com/ziglang/zig/pull/1454 -pub fn hexToBytes(input: []const u8, out: []u8) !void { - if (out.len * 2 < input.len) - return error.InvalidLength; +pub fn hexToBytes(out: []u8, input: []const u8) !void { + if (out.len * 2 < input.len) + return error.InvalidLength; - var i: usize = 0; - while (i < input.len) : (i += 2) { - out[i/2] = (try charToDigit(input[i], 36)) << 4; - out[i/2] += try charToDigit(input[i+1], 36); - } + var in_i: usize = 0; + while (in_i != input.len) : (in_i += 2) { + const hi = try charToDigit(input[in_i], 16); + const lo = try charToDigit(input[in_i + 1], 16); + out[in_i / 2] = (hi << 4) | lo; + } } test "fmt.hexToBytes" { - const test_hex_str = "909A312BB12ED1F819B3521AC4C1E896F2160507FFC1C8381E3B07BB16BD1706"; - var pb: [32]u8 = undefined; - try hexToBytes(test_hex_str, pb[0..]); - try testFmt(test_hex_str, "{X}", pb); + const test_hex_str = "909A312BB12ED1F819B3521AC4C1E896F2160507FFC1C8381E3B07BB16BD1706"; + var pb: [32]u8 = undefined; + try hexToBytes(pb[0..], test_hex_str); + try testFmt(test_hex_str, "{X}", pb); } - From d5968086fe357aa5cf678327295677dba5102fc8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 2 Sep 2018 21:08:59 -0400 Subject: [PATCH 48/56] use the sret attribute at the callsite when appropriate Thanks to Shawn Landden for the original pull request. closes #1450 --- src/codegen.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index f25ecd8e08..388a1a371f 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3101,6 +3101,12 @@ static void gen_set_stack_pointer(CodeGen *g, LLVMValueRef aligned_end_addr) { LLVMBuildCall(g->builder, write_register_fn_val, params, 2, ""); } +static void set_call_instr_sret(CodeGen *g, LLVMValueRef call_instr) { + unsigned attr_kind_id = LLVMGetEnumAttributeKindForName("sret", 4); + LLVMAttributeRef sret_attr = LLVMCreateEnumAttribute(LLVMGetGlobalContext(), attr_kind_id, 1); + LLVMAddCallSiteAttribute(call_instr, 1, sret_attr); +} + static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstructionCall *instruction) { LLVMValueRef fn_val; TypeTableEntry *fn_type; @@ -3196,6 +3202,7 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr } else if (!ret_has_bits) { return nullptr; } else if (first_arg_ret) { + set_call_instr_sret(g, result); return instruction->tmp_ptr; } else if (handle_is_ptr(src_return_type)) { auto store_instr = LLVMBuildStore(g->builder, result, instruction->tmp_ptr); @@ -4662,8 +4669,9 @@ static LLVMValueRef get_coro_alloc_helper_fn_val(CodeGen *g, LLVMTypeRef alloc_f args.append(allocator_val); args.append(coro_size); args.append(alignment_val); - ZigLLVMBuildCall(g->builder, alloc_fn_val, args.items, args.length, + LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, alloc_fn_val, args.items, args.length, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); + set_call_instr_sret(g, call_instruction); LLVMValueRef err_val_ptr = LLVMBuildStructGEP(g->builder, sret_ptr, err_union_err_index, ""); LLVMValueRef err_val = LLVMBuildLoad(g->builder, err_val_ptr, ""); LLVMBuildStore(g->builder, err_val, err_code_ptr); From 92f747435930bc4d54114e414b372c7eafe7cc02 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 2 Sep 2018 23:25:04 -0400 Subject: [PATCH 49/56] switch most windows calls to use W versions instead of A See #534 --- CMakeLists.txt | 2 - std/os/child_process.zig | 58 ++++++++--- std/os/index.zig | 201 ++++++++++++++++++++---------------- std/os/windows/index.zig | 18 ++-- std/os/windows/kernel32.zig | 53 ++-------- std/os/windows/shlwapi.zig | 4 - std/os/windows/user32.zig | 4 - std/os/windows/util.zig | 84 ++++++++------- 8 files changed, 224 insertions(+), 200 deletions(-) delete mode 100644 std/os/windows/shlwapi.zig delete mode 100644 std/os/windows/user32.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 5664f1db19..d02de451f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -581,8 +581,6 @@ set(ZIG_STD_FILES "os/windows/ntdll.zig" "os/windows/ole32.zig" "os/windows/shell32.zig" - "os/windows/shlwapi.zig" - "os/windows/user32.zig" "os/windows/util.zig" "os/zen.zig" "pdb.zig" diff --git a/std/os/child_process.zig b/std/os/child_process.zig index decd8d04fa..a8f307f3b6 100644 --- a/std/os/child_process.zig +++ b/std/os/child_process.zig @@ -1,5 +1,6 @@ const std = @import("../index.zig"); const cstr = std.cstr; +const unicode = std.unicode; const io = std.io; const os = std.os; const posix = os.posix; @@ -12,6 +13,7 @@ const Buffer = std.Buffer; const builtin = @import("builtin"); const Os = builtin.Os; const LinkedList = std.LinkedList; +const windows_util = @import("windows/util.zig"); const is_windows = builtin.os == Os.windows; @@ -520,8 +522,8 @@ pub const ChildProcess = struct { const cmd_line = try windowsCreateCommandLine(self.allocator, self.argv); defer self.allocator.free(cmd_line); - var siStartInfo = windows.STARTUPINFOA{ - .cb = @sizeOf(windows.STARTUPINFOA), + var siStartInfo = windows.STARTUPINFOW{ + .cb = @sizeOf(windows.STARTUPINFOW), .hStdError = g_hChildStd_ERR_Wr, .hStdOutput = g_hChildStd_OUT_Wr, .hStdInput = g_hChildStd_IN_Rd, @@ -545,7 +547,9 @@ pub const ChildProcess = struct { const cwd_slice = if (self.cwd) |cwd| try cstr.addNullByte(self.allocator, cwd) else null; defer if (cwd_slice) |cwd| self.allocator.free(cwd); - const cwd_ptr = if (cwd_slice) |cwd| cwd.ptr else null; + const cwd_w = if (cwd_slice) |cwd| try unicode.utf8ToUtf16LeWithNull(self.allocator, cwd) else null; + defer if (cwd_w) |cwd| self.allocator.free(cwd); + const cwd_w_ptr = if (cwd_w) |cwd| cwd.ptr else null; const maybe_envp_buf = if (self.env_map) |env_map| try os.createWindowsEnvBlock(self.allocator, env_map) else null; defer if (maybe_envp_buf) |envp_buf| self.allocator.free(envp_buf); @@ -564,7 +568,13 @@ pub const ChildProcess = struct { }; defer self.allocator.free(app_name); - windowsCreateProcess(app_name.ptr, cmd_line.ptr, envp_ptr, cwd_ptr, &siStartInfo, &piProcInfo) catch |no_path_err| { + const app_name_w = try unicode.utf8ToUtf16LeWithNull(self.allocator, app_name); + defer self.allocator.free(app_name_w); + + const cmd_line_w = try unicode.utf8ToUtf16LeWithNull(self.allocator, cmd_line); + defer self.allocator.free(cmd_line_w); + + windowsCreateProcess(app_name_w.ptr, cmd_line_w.ptr, envp_ptr, cwd_w_ptr, &siStartInfo, &piProcInfo) catch |no_path_err| { if (no_path_err != error.FileNotFound) return no_path_err; const PATH = try os.getEnvVarOwned(self.allocator, "PATH"); @@ -575,7 +585,10 @@ pub const ChildProcess = struct { const joined_path = try os.path.join(self.allocator, search_path, app_name); defer self.allocator.free(joined_path); - if (windowsCreateProcess(joined_path.ptr, cmd_line.ptr, envp_ptr, cwd_ptr, &siStartInfo, &piProcInfo)) |_| { + const joined_path_w = try unicode.utf8ToUtf16LeWithNull(self.allocator, app_name); + defer self.allocator.free(joined_path_w); + + if (windowsCreateProcess(joined_path_w.ptr, cmd_line_w.ptr, envp_ptr, cwd_w_ptr, &siStartInfo, &piProcInfo)) |_| { break; } else |err| if (err == error.FileNotFound) { continue; @@ -626,15 +639,36 @@ pub const ChildProcess = struct { } }; -fn windowsCreateProcess(app_name: [*]u8, cmd_line: [*]u8, envp_ptr: ?[*]u8, cwd_ptr: ?[*]u8, lpStartupInfo: *windows.STARTUPINFOA, lpProcessInformation: *windows.PROCESS_INFORMATION) !void { - if (windows.CreateProcessA(app_name, cmd_line, null, null, windows.TRUE, 0, @ptrCast(?*c_void, envp_ptr), cwd_ptr, lpStartupInfo, lpProcessInformation) == 0) { +fn windowsCreateProcess(app_name: [*]u16, cmd_line: [*]u16, envp_ptr: ?[*]u16, cwd_ptr: ?[*]u16, lpStartupInfo: *windows.STARTUPINFOW, lpProcessInformation: *windows.PROCESS_INFORMATION) !void { + // TODO the docs for environment pointer say: + // > A pointer to the environment block for the new process. If this parameter + // > is NULL, the new process uses the environment of the calling process. + // > ... + // > An environment block can contain either Unicode or ANSI characters. If + // > the environment block pointed to by lpEnvironment contains Unicode + // > characters, be sure that dwCreationFlags includes CREATE_UNICODE_ENVIRONMENT. + // > If this parameter is NULL and the environment block of the parent process + // > contains Unicode characters, you must also ensure that dwCreationFlags + // > includes CREATE_UNICODE_ENVIRONMENT. + // This seems to imply that we have to somehow know whether our process parent passed + // CREATE_UNICODE_ENVIRONMENT if we want to pass NULL for the environment parameter. + // Since we do not know this information that would imply that we must not pass NULL + // for the parameter. + // However this would imply that programs compiled with -DUNICODE could not pass + // environment variables to programs that were not, which seems unlikely. + // More investigation is needed. + if (windows.CreateProcessW( + app_name, cmd_line, null, null, windows.TRUE, windows.CREATE_UNICODE_ENVIRONMENT, + @ptrCast(?*c_void, envp_ptr), cwd_ptr, lpStartupInfo, lpProcessInformation, + ) == 0) { const err = windows.GetLastError(); - return switch (err) { - windows.ERROR.FILE_NOT_FOUND, windows.ERROR.PATH_NOT_FOUND => error.FileNotFound, + switch (err) { + windows.ERROR.FILE_NOT_FOUND => return error.FileNotFound, + windows.ERROR.PATH_NOT_FOUND => return error.FileNotFound, windows.ERROR.INVALID_PARAMETER => unreachable, - windows.ERROR.INVALID_NAME => error.InvalidName, - else => os.unexpectedErrorWindows(err), - }; + windows.ERROR.INVALID_NAME => return error.InvalidName, + else => return os.unexpectedErrorWindows(err), + } } } diff --git a/std/os/index.zig b/std/os/index.zig index 9b49a05067..3ab73a79ac 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -819,37 +819,40 @@ test "os.getCwd" { pub const SymLinkError = PosixSymLinkError || WindowsSymLinkError; -pub fn symLink(allocator: *Allocator, existing_path: []const u8, new_path: []const u8) SymLinkError!void { +/// TODO add a symLinkC variant +pub fn symLink(existing_path: []const u8, new_path: []const u8) SymLinkError!void { if (is_windows) { - return symLinkWindows(allocator, existing_path, new_path); + return symLinkWindows(existing_path, new_path); } else { - return symLinkPosix(allocator, existing_path, new_path); + return symLinkPosix(existing_path, new_path); } } pub const WindowsSymLinkError = error{ - OutOfMemory, + NameTooLong, + InvalidUtf8, + BadPathName, /// See https://github.com/ziglang/zig/issues/1396 Unexpected, }; -pub fn symLinkWindows(allocator: *Allocator, existing_path: []const u8, new_path: []const u8) WindowsSymLinkError!void { - const existing_with_null = try cstr.addNullByte(allocator, existing_path); - defer allocator.free(existing_with_null); - const new_with_null = try cstr.addNullByte(allocator, new_path); - defer allocator.free(new_with_null); - - if (windows.CreateSymbolicLinkA(existing_with_null.ptr, new_with_null.ptr, 0) == 0) { +pub fn symLinkW(existing_path_w: [*]const u16, new_path_w: [*]const u16) WindowsSymLinkError!void { + if (windows.CreateSymbolicLinkW(existing_path_w, new_path_w, 0) == 0) { const err = windows.GetLastError(); - return switch (err) { - else => unexpectedErrorWindows(err), - }; + switch (err) { + else => return unexpectedErrorWindows(err), + } } } +pub fn symLinkWindows(existing_path: []const u8, new_path: []const u8) WindowsSymLinkError!void { + const existing_path_w = try windows_util.sliceToPrefixedFileW(existing_path); + const new_path_w = try windows_util.sliceToPrefixedFileW(new_path); + return symLinkW(&existing_path_w, &new_path_w); +} + pub const PosixSymLinkError = error{ - OutOfMemory, AccessDenied, DiskQuota, PathAlreadyExists, @@ -866,43 +869,40 @@ pub const PosixSymLinkError = error{ Unexpected, }; -pub fn symLinkPosix(allocator: *Allocator, existing_path: []const u8, new_path: []const u8) PosixSymLinkError!void { - const full_buf = try allocator.alloc(u8, existing_path.len + new_path.len + 2); - defer allocator.free(full_buf); - - const existing_buf = full_buf; - mem.copy(u8, existing_buf, existing_path); - existing_buf[existing_path.len] = 0; - - const new_buf = full_buf[existing_path.len + 1 ..]; - mem.copy(u8, new_buf, new_path); - new_buf[new_path.len] = 0; - - const err = posix.getErrno(posix.symlink(existing_buf.ptr, new_buf.ptr)); - if (err > 0) { - return switch (err) { - posix.EFAULT, posix.EINVAL => unreachable, - posix.EACCES, posix.EPERM => error.AccessDenied, - posix.EDQUOT => error.DiskQuota, - posix.EEXIST => error.PathAlreadyExists, - posix.EIO => error.FileSystem, - posix.ELOOP => error.SymLinkLoop, - posix.ENAMETOOLONG => error.NameTooLong, - posix.ENOENT => error.FileNotFound, - posix.ENOTDIR => error.NotDir, - posix.ENOMEM => error.SystemResources, - posix.ENOSPC => error.NoSpaceLeft, - posix.EROFS => error.ReadOnlyFileSystem, - else => unexpectedErrorPosix(err), - }; +pub fn symLinkPosixC(existing_path: [*]const u8, new_path: [*]const u8) PosixSymLinkError!void { + const err = posix.getErrno(posix.symlink(existing_path, new_path)); + switch (err) { + 0 => return, + posix.EFAULT => unreachable, + posix.EINVAL => unreachable, + posix.EACCES => return error.AccessDenied, + posix.EPERM => return error.AccessDenied, + posix.EDQUOT => return error.DiskQuota, + posix.EEXIST => return error.PathAlreadyExists, + posix.EIO => return error.FileSystem, + posix.ELOOP => return error.SymLinkLoop, + posix.ENAMETOOLONG => return error.NameTooLong, + posix.ENOENT => return error.FileNotFound, + posix.ENOTDIR => return error.NotDir, + posix.ENOMEM => return error.SystemResources, + posix.ENOSPC => return error.NoSpaceLeft, + posix.EROFS => return error.ReadOnlyFileSystem, + else => return unexpectedErrorPosix(err), } } +pub fn symLinkPosix(existing_path: []const u8, new_path: []const u8) PosixSymLinkError!void { + const existing_path_c = try toPosixPath(existing_path); + const new_path_c = try toPosixPath(new_path); + return symLinkPosixC(&existing_path_c, &new_path_c); +} + // here we replace the standard +/ with -_ so that it can be used in a file name const b64_fs_encoder = base64.Base64Encoder.init("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", base64.standard_pad_char); +/// TODO remove the allocator requirement from this API pub fn atomicSymLink(allocator: *Allocator, existing_path: []const u8, new_path: []const u8) !void { - if (symLink(allocator, existing_path, new_path)) { + if (symLink(existing_path, new_path)) { return; } else |err| switch (err) { error.PathAlreadyExists => {}, @@ -920,7 +920,7 @@ pub fn atomicSymLink(allocator: *Allocator, existing_path: []const u8, new_path: try getRandomBytes(rand_buf[0..]); b64_fs_encoder.encode(tmp_path[dirname.len + 1 ..], rand_buf); - if (symLink(allocator, existing_path, tmp_path)) { + if (symLink(existing_path, tmp_path)) { return rename(tmp_path, new_path); } else |err| switch (err) { error.PathAlreadyExists => continue, @@ -1252,54 +1252,70 @@ pub const DeleteDirError = error{ NotDir, DirNotEmpty, ReadOnlyFileSystem, - OutOfMemory, + InvalidUtf8, + BadPathName, /// See https://github.com/ziglang/zig/issues/1396 Unexpected, }; -/// Returns ::error.DirNotEmpty if the directory is not empty. -/// To delete a directory recursively, see ::deleteTree -pub fn deleteDir(allocator: *Allocator, dir_path: []const u8) DeleteDirError!void { - const path_buf = try allocator.alloc(u8, dir_path.len + 1); - defer allocator.free(path_buf); - - mem.copy(u8, path_buf, dir_path); - path_buf[dir_path.len] = 0; - +pub fn deleteDirC(dir_path: [*]const u8) DeleteDirError!void { switch (builtin.os) { Os.windows => { - if (windows.RemoveDirectoryA(path_buf.ptr) == 0) { - const err = windows.GetLastError(); - return switch (err) { - windows.ERROR.PATH_NOT_FOUND => error.FileNotFound, - windows.ERROR.DIR_NOT_EMPTY => error.DirNotEmpty, - else => unexpectedErrorWindows(err), - }; - } + const dir_path_w = try windows_util.cStrToPrefixedFileW(dir_path); + return deleteDirW(&dir_path_w); }, Os.linux, Os.macosx, Os.ios => { - const err = posix.getErrno(posix.rmdir(path_buf.ptr)); - if (err > 0) { - return switch (err) { - posix.EACCES, posix.EPERM => error.AccessDenied, - posix.EBUSY => error.FileBusy, - posix.EFAULT, posix.EINVAL => unreachable, - posix.ELOOP => error.SymLinkLoop, - posix.ENAMETOOLONG => error.NameTooLong, - posix.ENOENT => error.FileNotFound, - posix.ENOMEM => error.SystemResources, - posix.ENOTDIR => error.NotDir, - posix.EEXIST, posix.ENOTEMPTY => error.DirNotEmpty, - posix.EROFS => error.ReadOnlyFileSystem, - else => unexpectedErrorPosix(err), - }; + const err = posix.getErrno(posix.rmdir(dir_path)); + switch (err) { + 0 => return, + posix.EACCES => return error.AccessDenied, + posix.EPERM => return error.AccessDenied, + posix.EBUSY => return error.FileBusy, + posix.EFAULT => unreachable, + posix.EINVAL => unreachable, + posix.ELOOP => return error.SymLinkLoop, + posix.ENAMETOOLONG => return error.NameTooLong, + posix.ENOENT => return error.FileNotFound, + posix.ENOMEM => return error.SystemResources, + posix.ENOTDIR => return error.NotDir, + posix.EEXIST => return error.DirNotEmpty, + posix.ENOTEMPTY => return error.DirNotEmpty, + posix.EROFS => return error.ReadOnlyFileSystem, + else => return unexpectedErrorPosix(err), } }, else => @compileError("unimplemented"), } } +pub fn deleteDirW(dir_path_w: [*]const u16) DeleteDirError!void { + if (windows.RemoveDirectoryW(dir_path_w) == 0) { + const err = windows.GetLastError(); + switch (err) { + windows.ERROR.PATH_NOT_FOUND => return error.FileNotFound, + windows.ERROR.DIR_NOT_EMPTY => return error.DirNotEmpty, + else => return unexpectedErrorWindows(err), + } + } +} + +/// Returns ::error.DirNotEmpty if the directory is not empty. +/// To delete a directory recursively, see ::deleteTree +pub fn deleteDir(dir_path: []const u8) DeleteDirError!void { + switch (builtin.os) { + Os.windows => { + const dir_path_w = try windows_util.sliceToPrefixedFileW(dir_path); + return deleteDirW(&dir_path_w); + }, + Os.linux, Os.macosx, Os.ios => { + const dir_path_c = try toPosixPath(dir_path); + return deleteDirC(&dir_path_c); + }, + else => @compileError("unimplemented"), + } +} + /// Whether ::full_path describes a symlink, file, or directory, this function /// removes it. If it cannot be removed because it is a non-empty directory, /// this function recursively removes its entries and then tries again. @@ -1346,6 +1362,7 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! error.IsDir => {}, error.AccessDenied => got_access_denied = true, + error.InvalidUtf8, error.SymLinkLoop, error.NameTooLong, error.SystemResources, @@ -1353,7 +1370,6 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! error.NotDir, error.FileSystem, error.FileBusy, - error.InvalidUtf8, error.BadPathName, error.Unexpected, => return err, @@ -1381,6 +1397,8 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! error.NoSpaceLeft, error.PathAlreadyExists, error.Unexpected, + error.InvalidUtf8, + error.BadPathName, => return err, }; defer dir.close(); @@ -1398,7 +1416,7 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! try deleteTree(allocator, full_entry_path); } } - return deleteDir(allocator, full_path); + return deleteDir(full_path); } } @@ -1422,8 +1440,9 @@ pub const Dir = struct { }, Os.windows => struct { handle: windows.HANDLE, - find_file_data: windows.WIN32_FIND_DATAA, + find_file_data: windows.WIN32_FIND_DATAW, first: bool, + name_data: [256]u8, }, else => @compileError("unimplemented"), }; @@ -1460,6 +1479,8 @@ pub const Dir = struct { NoSpaceLeft, PathAlreadyExists, OutOfMemory, + InvalidUtf8, + BadPathName, /// See https://github.com/ziglang/zig/issues/1396 Unexpected, @@ -1471,12 +1492,13 @@ pub const Dir = struct { .allocator = allocator, .handle = switch (builtin.os) { Os.windows => blk: { - var find_file_data: windows.WIN32_FIND_DATAA = undefined; - const handle = try windows_util.windowsFindFirstFile(allocator, dir_path, &find_file_data); + var find_file_data: windows.WIN32_FIND_DATAW = undefined; + const handle = try windows_util.windowsFindFirstFile(dir_path, &find_file_data); break :blk Handle{ .handle = handle, .find_file_data = find_file_data, // TODO guaranteed copy elision .first = true, + .name_data = undefined, }; }, Os.macosx, Os.ios => Handle{ @@ -1591,9 +1613,12 @@ pub const Dir = struct { if (!try windows_util.windowsFindNextFile(self.handle.handle, &self.handle.find_file_data)) return null; } - const name = std.cstr.toSlice(self.handle.find_file_data.cFileName[0..].ptr); - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) + const name_utf16le = mem.toSlice(u16, self.handle.find_file_data.cFileName[0..].ptr); + if (mem.eql(u16, name_utf16le, []u16{'.'}) or mem.eql(u16, name_utf16le, []u16{'.', '.'})) continue; + // Trust that Windows gives us valid UTF-16LE + const name_utf8_len = std.unicode.utf16leToUtf8(self.handle.name_data[0..], name_utf16le) catch unreachable; + const name_utf8 = self.handle.name_data[0..name_utf8_len]; const kind = blk: { const attrs = self.handle.find_file_data.dwFileAttributes; if (attrs & windows.FILE_ATTRIBUTE_DIRECTORY != 0) break :blk Entry.Kind.Directory; @@ -1602,7 +1627,7 @@ pub const Dir = struct { break :blk Entry.Kind.Unknown; }; return Entry{ - .name = name, + .name = name_utf8, .kind = kind, }; } @@ -2070,7 +2095,7 @@ fn testWindowsCmdLine(input_cmd_line: [*]const u8, expected_args: []const []cons } // TODO make this a build variable that you can set -const unexpected_error_tracing = true; +const unexpected_error_tracing = false; const UnexpectedError = error{ /// The Operating System returned an undocumented error code. Unexpected, diff --git a/std/os/windows/index.zig b/std/os/windows/index.zig index 9286b7d090..fc64db7c37 100644 --- a/std/os/windows/index.zig +++ b/std/os/windows/index.zig @@ -6,8 +6,6 @@ pub use @import("kernel32.zig"); pub use @import("ntdll.zig"); pub use @import("ole32.zig"); pub use @import("shell32.zig"); -pub use @import("shlwapi.zig"); -pub use @import("user32.zig"); test "import" { _ = @import("util.zig"); @@ -174,11 +172,11 @@ pub const PROCESS_INFORMATION = extern struct { dwThreadId: DWORD, }; -pub const STARTUPINFOA = extern struct { +pub const STARTUPINFOW = extern struct { cb: DWORD, - lpReserved: ?LPSTR, - lpDesktop: ?LPSTR, - lpTitle: ?LPSTR, + lpReserved: ?LPWSTR, + lpDesktop: ?LPWSTR, + lpTitle: ?LPWSTR, dwX: DWORD, dwY: DWORD, dwXSize: DWORD, @@ -238,7 +236,7 @@ pub const HEAP_NO_SERIALIZE = 0x00000001; pub const PTHREAD_START_ROUTINE = extern fn (LPVOID) DWORD; pub const LPTHREAD_START_ROUTINE = PTHREAD_START_ROUTINE; -pub const WIN32_FIND_DATAA = extern struct { +pub const WIN32_FIND_DATAW = extern struct { dwFileAttributes: DWORD, ftCreationTime: FILETIME, ftLastAccessTime: FILETIME, @@ -247,8 +245,8 @@ pub const WIN32_FIND_DATAA = extern struct { nFileSizeLow: DWORD, dwReserved0: DWORD, dwReserved1: DWORD, - cFileName: [260]CHAR, - cAlternateFileName: [14]CHAR, + cFileName: [260]u16, + cAlternateFileName: [14]u16, }; pub const FILETIME = extern struct { @@ -377,3 +375,5 @@ pub const COORD = extern struct { X: SHORT, Y: SHORT, }; + +pub const CREATE_UNICODE_ENVIRONMENT = 1024; diff --git a/std/os/windows/kernel32.zig b/std/os/windows/kernel32.zig index ffa4422760..1a7b28eac2 100644 --- a/std/os/windows/kernel32.zig +++ b/std/os/windows/kernel32.zig @@ -4,19 +4,8 @@ pub extern "kernel32" stdcallcc fn CancelIoEx(hFile: HANDLE, lpOverlapped: LPOVE pub extern "kernel32" stdcallcc fn CloseHandle(hObject: HANDLE) BOOL; -pub extern "kernel32" stdcallcc fn CreateDirectoryA(lpPathName: [*]const u8, lpSecurityAttributes: ?*SECURITY_ATTRIBUTES) BOOL; pub extern "kernel32" stdcallcc fn CreateDirectoryW(lpPathName: [*]const u16, lpSecurityAttributes: ?*SECURITY_ATTRIBUTES) BOOL; -pub extern "kernel32" stdcallcc fn CreateFileA( - lpFileName: [*]const u8, // TODO null terminated pointer type - dwDesiredAccess: DWORD, - dwShareMode: DWORD, - lpSecurityAttributes: ?LPSECURITY_ATTRIBUTES, - dwCreationDisposition: DWORD, - dwFlagsAndAttributes: DWORD, - hTemplateFile: ?HANDLE, -) HANDLE; - pub extern "kernel32" stdcallcc fn CreateFileW( lpFileName: [*]const u16, // TODO null terminated pointer type dwDesiredAccess: DWORD, @@ -34,37 +23,32 @@ pub extern "kernel32" stdcallcc fn CreatePipe( nSize: DWORD, ) BOOL; -pub extern "kernel32" stdcallcc fn CreateProcessA( - lpApplicationName: ?LPCSTR, - lpCommandLine: LPSTR, +pub extern "kernel32" stdcallcc fn CreateProcessW( + lpApplicationName: ?LPWSTR, + lpCommandLine: LPWSTR, lpProcessAttributes: ?*SECURITY_ATTRIBUTES, lpThreadAttributes: ?*SECURITY_ATTRIBUTES, bInheritHandles: BOOL, dwCreationFlags: DWORD, lpEnvironment: ?*c_void, - lpCurrentDirectory: ?LPCSTR, - lpStartupInfo: *STARTUPINFOA, + lpCurrentDirectory: ?LPWSTR, + lpStartupInfo: *STARTUPINFOW, lpProcessInformation: *PROCESS_INFORMATION, ) BOOL; -pub extern "kernel32" stdcallcc fn CreateSymbolicLinkA( - lpSymlinkFileName: LPCSTR, - lpTargetFileName: LPCSTR, - dwFlags: DWORD, -) BOOLEAN; +pub extern "kernel32" stdcallcc fn CreateSymbolicLinkW(lpSymlinkFileName: [*]const u16, lpTargetFileName: [*]const u16, dwFlags: DWORD) BOOLEAN; pub extern "kernel32" stdcallcc fn CreateIoCompletionPort(FileHandle: HANDLE, ExistingCompletionPort: ?HANDLE, CompletionKey: ULONG_PTR, NumberOfConcurrentThreads: DWORD) ?HANDLE; pub extern "kernel32" stdcallcc fn CreateThread(lpThreadAttributes: ?LPSECURITY_ATTRIBUTES, dwStackSize: SIZE_T, lpStartAddress: LPTHREAD_START_ROUTINE, lpParameter: ?LPVOID, dwCreationFlags: DWORD, lpThreadId: ?LPDWORD) ?HANDLE; -pub extern "kernel32" stdcallcc fn DeleteFileA(lpFileName: [*]const u8) BOOL; pub extern "kernel32" stdcallcc fn DeleteFileW(lpFileName: [*]const u16) BOOL; pub extern "kernel32" stdcallcc fn ExitProcess(exit_code: UINT) noreturn; -pub extern "kernel32" stdcallcc fn FindFirstFileA(lpFileName: LPCSTR, lpFindFileData: *WIN32_FIND_DATAA) HANDLE; +pub extern "kernel32" stdcallcc fn FindFirstFileW(lpFileName: [*]const u16, lpFindFileData: *WIN32_FIND_DATAW) HANDLE; pub extern "kernel32" stdcallcc fn FindClose(hFindFile: HANDLE) BOOL; -pub extern "kernel32" stdcallcc fn FindNextFileA(hFindFile: HANDLE, lpFindFileData: *WIN32_FIND_DATAA) BOOL; +pub extern "kernel32" stdcallcc fn FindNextFileW(hFindFile: HANDLE, lpFindFileData: *WIN32_FIND_DATAW) BOOL; pub extern "kernel32" stdcallcc fn FreeEnvironmentStringsA(penv: [*]u8) BOOL; @@ -74,7 +58,6 @@ pub extern "kernel32" stdcallcc fn GetConsoleMode(in_hConsoleHandle: HANDLE, out pub extern "kernel32" stdcallcc fn GetConsoleScreenBufferInfo(hConsoleOutput: HANDLE, lpConsoleScreenBufferInfo: *CONSOLE_SCREEN_BUFFER_INFO) BOOL; -pub extern "kernel32" stdcallcc fn GetCurrentDirectoryA(nBufferLength: DWORD, lpBuffer: ?[*]CHAR) DWORD; pub extern "kernel32" stdcallcc fn GetCurrentDirectoryW(nBufferLength: DWORD, lpBuffer: ?[*]WCHAR) DWORD; pub extern "kernel32" stdcallcc fn GetCurrentThread() HANDLE; @@ -88,10 +71,8 @@ pub extern "kernel32" stdcallcc fn GetExitCodeProcess(hProcess: HANDLE, lpExitCo pub extern "kernel32" stdcallcc fn GetFileSizeEx(hFile: HANDLE, lpFileSize: *LARGE_INTEGER) BOOL; -pub extern "kernel32" stdcallcc fn GetFileAttributesA(lpFileName: [*]const CHAR) DWORD; pub extern "kernel32" stdcallcc fn GetFileAttributesW(lpFileName: [*]const WCHAR) DWORD; -pub extern "kernel32" stdcallcc fn GetModuleFileNameA(hModule: ?HMODULE, lpFilename: [*]u8, nSize: DWORD) DWORD; pub extern "kernel32" stdcallcc fn GetModuleFileNameW(hModule: ?HMODULE, lpFilename: [*]u16, nSize: DWORD) DWORD; pub extern "kernel32" stdcallcc fn GetModuleHandleW(lpModuleName: ?[*]const WCHAR) HMODULE; @@ -105,13 +86,6 @@ pub extern "kernel32" stdcallcc fn GetFileInformationByHandleEx( in_dwBufferSize: DWORD, ) BOOL; -pub extern "kernel32" stdcallcc fn GetFinalPathNameByHandleA( - hFile: HANDLE, - lpszFilePath: LPSTR, - cchFilePath: DWORD, - dwFlags: DWORD, -) DWORD; - pub extern "kernel32" stdcallcc fn GetFinalPathNameByHandleW( hFile: HANDLE, lpszFilePath: [*]u16, @@ -142,12 +116,6 @@ pub extern "kernel32" stdcallcc fn HeapFree(hHeap: HANDLE, dwFlags: DWORD, lpMem pub extern "kernel32" stdcallcc fn HeapValidate(hHeap: HANDLE, dwFlags: DWORD, lpMem: ?*const c_void) BOOL; -pub extern "kernel32" stdcallcc fn MoveFileExA( - lpExistingFileName: [*]const u8, - lpNewFileName: [*]const u8, - dwFlags: DWORD, -) BOOL; - pub extern "kernel32" stdcallcc fn MoveFileExW( lpExistingFileName: [*]const u16, lpNewFileName: [*]const u16, @@ -179,7 +147,7 @@ pub extern "kernel32" stdcallcc fn ReadFile( in_out_lpOverlapped: ?*OVERLAPPED, ) BOOL; -pub extern "kernel32" stdcallcc fn RemoveDirectoryA(lpPathName: LPCSTR) BOOL; +pub extern "kernel32" stdcallcc fn RemoveDirectoryW(lpPathName: [*]const u16) BOOL; pub extern "kernel32" stdcallcc fn SetConsoleTextAttribute(hConsoleOutput: HANDLE, wAttributes: WORD) BOOL; @@ -208,8 +176,7 @@ pub extern "kernel32" stdcallcc fn WriteFile( pub extern "kernel32" stdcallcc fn WriteFileEx(hFile: HANDLE, lpBuffer: [*]const u8, nNumberOfBytesToWrite: DWORD, lpOverlapped: LPOVERLAPPED, lpCompletionRoutine: LPOVERLAPPED_COMPLETION_ROUTINE) BOOL; -//TODO: call unicode versions instead of relying on ANSI code page -pub extern "kernel32" stdcallcc fn LoadLibraryA(lpLibFileName: LPCSTR) ?HMODULE; +pub extern "kernel32" stdcallcc fn LoadLibraryW(lpLibFileName: [*]const u16) ?HMODULE; pub extern "kernel32" stdcallcc fn FreeLibrary(hModule: HMODULE) BOOL; diff --git a/std/os/windows/shlwapi.zig b/std/os/windows/shlwapi.zig deleted file mode 100644 index 6bccefaf98..0000000000 --- a/std/os/windows/shlwapi.zig +++ /dev/null @@ -1,4 +0,0 @@ -use @import("index.zig"); - -pub extern "shlwapi" stdcallcc fn PathFileExistsA(pszPath: ?LPCTSTR) BOOL; - diff --git a/std/os/windows/user32.zig b/std/os/windows/user32.zig deleted file mode 100644 index 37f9f6f3b8..0000000000 --- a/std/os/windows/user32.zig +++ /dev/null @@ -1,4 +0,0 @@ -use @import("index.zig"); - -pub extern "user32" stdcallcc fn MessageBoxA(hWnd: ?HANDLE, lpText: ?LPCTSTR, lpCaption: ?LPCTSTR, uType: UINT) c_int; - diff --git a/std/os/windows/util.zig b/std/os/windows/util.zig index 168f6c3861..a489cca163 100644 --- a/std/os/windows/util.zig +++ b/std/os/windows/util.zig @@ -1,6 +1,7 @@ const std = @import("../../index.zig"); const builtin = @import("builtin"); const os = std.os; +const unicode = std.unicode; const windows = std.os.windows; const assert = std.debug.assert; const mem = std.mem; @@ -156,41 +157,51 @@ pub fn windowsOpen( } /// Caller must free result. -pub fn createWindowsEnvBlock(allocator: *mem.Allocator, env_map: *const BufMap) ![]u8 { +pub fn createWindowsEnvBlock(allocator: *mem.Allocator, env_map: *const BufMap) ![]u16 { // count bytes needed - const bytes_needed = x: { - var bytes_needed: usize = 1; // 1 for the final null byte + const max_chars_needed = x: { + var max_chars_needed: usize = 1; // 1 for the final null byte var it = env_map.iterator(); while (it.next()) |pair| { // +1 for '=' // +1 for null byte - bytes_needed += pair.key.len + pair.value.len + 2; + max_chars_needed += pair.key.len + pair.value.len + 2; } - break :x bytes_needed; + break :x max_chars_needed; }; - const result = try allocator.alloc(u8, bytes_needed); + const result = try allocator.alloc(u16, max_chars_needed); errdefer allocator.free(result); var it = env_map.iterator(); var i: usize = 0; while (it.next()) |pair| { - mem.copy(u8, result[i..], pair.key); - i += pair.key.len; + i += try unicode.utf8ToUtf16Le(result[i..], pair.key); result[i] = '='; i += 1; - mem.copy(u8, result[i..], pair.value); - i += pair.value.len; + i += try unicode.utf8ToUtf16Le(result[i..], pair.value); result[i] = 0; i += 1; } result[i] = 0; - return result; + i += 1; + return allocator.shrink(u16, result, i); } -pub fn windowsLoadDll(allocator: *mem.Allocator, dll_path: []const u8) !windows.HMODULE { - const padded_buff = try cstr.addNullByte(allocator, dll_path); - defer allocator.free(padded_buff); - return windows.LoadLibraryA(padded_buff.ptr) orelse error.DllNotFound; +pub fn windowsLoadDllW(dll_path_w: [*]const u16) !windows.HMODULE { + return windows.LoadLibraryW(dll_path_w) orelse { + const err = windows.GetLastError(); + switch (err) { + windows.ERROR.FILE_NOT_FOUND => return error.FileNotFound, + windows.ERROR.PATH_NOT_FOUND => return error.FileNotFound, + windows.ERROR.MOD_NOT_FOUND => return error.FileNotFound, + else => return os.unexpectedErrorWindows(err), + } + }; +} + +pub fn windowsLoadDll(dll_path: []const u8) !windows.HMODULE { + const dll_path_w = try sliceToPrefixedFileW(dll_path); + return windowsLoadDllW(&dll_path_w); } pub fn windowsUnloadDll(hModule: windows.HMODULE) void { @@ -200,27 +211,19 @@ pub fn windowsUnloadDll(hModule: windows.HMODULE) void { test "InvalidDll" { if (builtin.os != builtin.Os.windows) return error.SkipZigTest; - const DllName = "asdf.dll"; - const allocator = std.debug.global_allocator; - const handle = os.windowsLoadDll(allocator, DllName) catch |err| { - assert(err == error.DllNotFound); + const handle = os.windowsLoadDll("asdf.dll") catch |err| { + assert(err == error.FileNotFound); return; }; + @panic("Expected error from function"); } pub fn windowsFindFirstFile( - allocator: *mem.Allocator, dir_path: []const u8, - find_file_data: *windows.WIN32_FIND_DATAA, + find_file_data: *windows.WIN32_FIND_DATAW, ) !windows.HANDLE { - const wild_and_null = []u8{ '\\', '*', 0 }; - const path_with_wild_and_null = try allocator.alloc(u8, dir_path.len + wild_and_null.len); - defer allocator.free(path_with_wild_and_null); - - mem.copy(u8, path_with_wild_and_null, dir_path); - mem.copy(u8, path_with_wild_and_null[dir_path.len..], wild_and_null); - - const handle = windows.FindFirstFileA(path_with_wild_and_null.ptr, find_file_data); + const dir_path_w = try sliceToPrefixedSuffixedFileW(dir_path, []u16{'\\', '*', 0}); + const handle = windows.FindFirstFileW(&dir_path_w, find_file_data); if (handle == windows.INVALID_HANDLE_VALUE) { const err = windows.GetLastError(); @@ -235,8 +238,8 @@ pub fn windowsFindFirstFile( } /// Returns `true` if there was another file, `false` otherwise. -pub fn windowsFindNextFile(handle: windows.HANDLE, find_file_data: *windows.WIN32_FIND_DATAA) !bool { - if (windows.FindNextFileA(handle, find_file_data) == 0) { +pub fn windowsFindNextFile(handle: windows.HANDLE, find_file_data: *windows.WIN32_FIND_DATAW) !bool { + if (windows.FindNextFileW(handle, find_file_data) == 0) { const err = windows.GetLastError(); return switch (err) { windows.ERROR.NO_MORE_FILES => false, @@ -297,8 +300,12 @@ pub fn cStrToPrefixedFileW(s: [*]const u8) ![PATH_MAX_WIDE + 1]u16 { } pub fn sliceToPrefixedFileW(s: []const u8) ![PATH_MAX_WIDE + 1]u16 { + return sliceToPrefixedSuffixedFileW(s, []u16{0}); +} + +pub fn sliceToPrefixedSuffixedFileW(s: []const u8, comptime suffix: []const u16) ![PATH_MAX_WIDE + suffix.len]u16 { // TODO well defined copy elision - var result: [PATH_MAX_WIDE + 1]u16 = undefined; + var result: [PATH_MAX_WIDE + suffix.len]u16 = undefined; // > File I/O functions in the Windows API convert "/" to "\" as part of // > converting the name to an NT-style name, except when using the "\\?\" @@ -306,11 +313,12 @@ pub fn sliceToPrefixedFileW(s: []const u8) ![PATH_MAX_WIDE + 1]u16 { // from https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file#maximum-path-length-limitation // Because we want the larger maximum path length for absolute paths, we // disallow forward slashes in zig std lib file functions on Windows. - for (s) |byte| + for (s) |byte| { switch (byte) { - '/', '*', '?', '"', '<', '>', '|' => return error.BadPathName, - else => {}, - }; + '/', '*', '?', '"', '<', '>', '|' => return error.BadPathName, + else => {}, + } + } const start_index = if (mem.startsWith(u8, s, "\\\\") or !os.path.isAbsolute(s)) 0 else blk: { const prefix = []u16{ '\\', '\\', '?', '\\' }; mem.copy(u16, result[0..], prefix); @@ -318,7 +326,7 @@ pub fn sliceToPrefixedFileW(s: []const u8) ![PATH_MAX_WIDE + 1]u16 { }; const end_index = start_index + try std.unicode.utf8ToUtf16Le(result[start_index..], s); assert(end_index <= result.len); - if (end_index == result.len) return error.NameTooLong; - result[end_index] = 0; + if (end_index + suffix.len > result.len) return error.NameTooLong; + mem.copy(u16, result[end_index..], suffix); return result; } From 95636c7e5ff6ad0eeb768a2a0a1d7533b5872e20 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Sep 2018 00:04:12 -0400 Subject: [PATCH 50/56] ability to @ptrCast to *void fixes #960 --- src/codegen.cpp | 3 +++ src/ir.cpp | 10 ++++++++-- test/cases/cast.zig | 6 ++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 388a1a371f..298e89579f 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2655,6 +2655,9 @@ static LLVMValueRef ir_render_ptr_cast(CodeGen *g, IrExecutable *executable, IrInstructionPtrCast *instruction) { TypeTableEntry *wanted_type = instruction->base.value.type; + if (!type_has_bits(wanted_type)) { + return nullptr; + } LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr); return LLVMBuildBitCast(g->builder, ptr, wanted_type->type_ref, ""); } diff --git a/src/ir.cpp b/src/ir.cpp index 4e64b3e002..2c3fadfc2d 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -19757,6 +19757,8 @@ static IrInstruction *ir_align_cast(IrAnalyze *ira, IrInstruction *target, uint3 } static TypeTableEntry *ir_analyze_instruction_ptr_cast(IrAnalyze *ira, IrInstructionPtrCast *instruction) { + Error err; + IrInstruction *dest_type_value = instruction->dest_type->other; TypeTableEntry *dest_type = ir_resolve_type(ira, dest_type_value); if (type_is_invalid(dest_type)) @@ -19810,9 +19812,13 @@ static TypeTableEntry *ir_analyze_instruction_ptr_cast(IrAnalyze *ira, IrInstruc instruction->base.source_node, nullptr, ptr); casted_ptr->value.type = dest_type; - // keep the bigger alignment, it can only help + // Keep the bigger alignment, it can only help- + // unless the target is zero bits. + if ((err = type_ensure_zero_bits_known(ira->codegen, dest_type))) + return ira->codegen->builtin_types.entry_invalid; + IrInstruction *result; - if (src_align_bytes > dest_align_bytes) { + if (src_align_bytes > dest_align_bytes && type_has_bits(dest_type)) { result = ir_align_cast(ira, casted_ptr, src_align_bytes, false); if (type_is_invalid(result->value.type)) return ira->codegen->builtin_types.entry_invalid; diff --git a/test/cases/cast.zig b/test/cases/cast.zig index cc5e4b4394..2b455ccc43 100644 --- a/test/cases/cast.zig +++ b/test/cases/cast.zig @@ -519,3 +519,9 @@ fn incrementVoidPtrArray(array: ?*c_void, len: usize) void { @ptrCast([*]u8, array.?)[n] += 1; } } + +test "*usize to *void" { + var i = usize(0); + var v = @ptrCast(*void, &i); + v.* = {}; +} From 2a9329c9988430cfa11a8bb4b02da0dce8749028 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Sep 2018 11:32:39 -0400 Subject: [PATCH 51/56] better anonymous struct naming this makes anonymous structs inherit the name of the function they are in only when they are the return expression. also document the behavior and provide examples. closes #1243 --- doc/langref.html.in | 26 ++++++++++++++++++++++++++ src/all_types.hpp | 1 + src/ir.cpp | 27 ++++++++++++++------------- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 31d923e84b..97263347c7 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -1918,6 +1918,32 @@ test "linked list" { assert(list2.first.?.data == 1234); } {#code_end#} + {#header_open|struct naming#} +

Since all structs are anonymous, Zig infers the type name based on a few rules.

+
    +
  • If the struct is in the initialization expression of a variable, it gets named after + that variable.
    • +
  • If the struct is in the return expression, it gets named after + the function it is returning from, with the parameter values serialized.
    • +
  • Otherwise, the struct gets a same such as (anonymous struct at file.zig:7:38).
    • +
+ {#code_begin|exe|struct_name#} +const std = @import("std"); + +pub fn main() void { + const Foo = struct {}; + std.debug.warn("variable: {}\n", @typeName(Foo)); + std.debug.warn("anonymous: {}\n", @typeName(struct {})); + std.debug.warn("function: {}\n", @typeName(List(i32))); +} + +fn List(comptime T: type) type { + return struct { + x: T, + }; +} + {#code_end#} + {#header_close#} {#see_also|comptime|@fieldParentPtr#} {#header_close#} {#header_open|enum#} diff --git a/src/all_types.hpp b/src/all_types.hpp index 6f0dd9baac..d580191afe 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -43,6 +43,7 @@ struct IrAnalyze; struct IrExecutable { ZigList basic_block_list; Buf *name; + FnTableEntry *name_fn; size_t mem_slot_count; size_t next_debug_id; size_t *backward_branch_count; diff --git a/src/ir.cpp b/src/ir.cpp index 2c3fadfc2d..5d1bbe3c08 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -3186,7 +3186,11 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, { IrInstruction *return_value; if (expr_node) { + // Temporarily set this so that if we return a type it gets the name of the function + FnTableEntry *prev_name_fn = irb->exec->name_fn; + irb->exec->name_fn = exec_fn_entry(irb->exec); return_value = ir_gen_node(irb, expr_node, scope); + irb->exec->name_fn = prev_name_fn; if (return_value == irb->codegen->invalid_instruction) return irb->codegen->invalid_instruction; } else { @@ -6481,20 +6485,17 @@ static bool render_instance_name_recursive(CodeGen *codegen, Buf *name, Scope *o static Buf *get_anon_type_name(CodeGen *codegen, IrExecutable *exec, const char *kind_name, AstNode *source_node) { if (exec->name) { return exec->name; + } else if (exec->name_fn != nullptr) { + Buf *name = buf_alloc(); + buf_append_buf(name, &exec->name_fn->symbol_name); + buf_appendf(name, "("); + render_instance_name_recursive(codegen, name, &exec->name_fn->fndef_scope->base, exec->begin_scope); + buf_appendf(name, ")"); + return name; } else { - FnTableEntry *fn_entry = exec_fn_entry(exec); - if (fn_entry) { - Buf *name = buf_alloc(); - buf_append_buf(name, &fn_entry->symbol_name); - buf_appendf(name, "("); - render_instance_name_recursive(codegen, name, &fn_entry->fndef_scope->base, exec->begin_scope); - buf_appendf(name, ")"); - return name; - } else { - //Note: C-imports do not have valid location information - return buf_sprintf("(anonymous %s at %s:%" ZIG_PRI_usize ":%" ZIG_PRI_usize ")", kind_name, - (source_node->owner->path != nullptr) ? buf_ptr(source_node->owner->path) : "(null)", source_node->line + 1, source_node->column + 1); - } + //Note: C-imports do not have valid location information + return buf_sprintf("(anonymous %s at %s:%" ZIG_PRI_usize ":%" ZIG_PRI_usize ")", kind_name, + (source_node->owner->path != nullptr) ? buf_ptr(source_node->owner->path) : "(null)", source_node->line + 1, source_node->column + 1); } } From 3f273479f8f38fb408181b2ae2fff1acf4278ba1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Sep 2018 12:18:12 -0400 Subject: [PATCH 52/56] clarify const variables in docs closes #1200 --- doc/langref.html.in | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 97263347c7..bebbdaaa13 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -566,7 +566,7 @@ const c_string_literal = {#header_close#} {#header_close#} {#header_open|Assignment#} -

Use const to assign a value to an identifier:

+

Use the const keyword to assign a value to an identifier:

{#code_begin|test_err|cannot assign to constant#} const x = 1234; @@ -582,7 +582,8 @@ test "assignment" { foo(); } {#code_end#} -

If you need a variable that you can modify, use var:

+

const applies to all of the bytes that the identifier immediately addresses. {#link|Pointers#} have their own const-ness.

+

If you need a variable that you can modify, use the var keyword:

{#code_begin|test#} const assert = @import("std").debug.assert; @@ -1918,7 +1919,7 @@ test "linked list" { assert(list2.first.?.data == 1234); } {#code_end#} - {#header_open|struct naming#} + {#header_open|struct Naming#}

Since all structs are anonymous, Zig infers the type name based on a few rules.

  • If the struct is in the initialization expression of a variable, it gets named after From a11e73bee2266d4c568bdf9b23c891ac1365bb09 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Sep 2018 12:38:24 -0400 Subject: [PATCH 53/56] compile error instead of segfault for unimplemented feature closes #1103 --- src/ir.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ir.cpp b/src/ir.cpp index 5d1bbe3c08..98c749f623 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -6693,7 +6693,10 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo return irb->codegen->invalid_instruction; } } else { - return_type = nullptr; + add_node_error(irb->codegen, node, + buf_sprintf("TODO implement inferred return types https://github.com/ziglang/zig/issues/447")); + return irb->codegen->invalid_instruction; + //return_type = nullptr; } IrInstruction *async_allocator_type_value = nullptr; From e82cd53df483aa08354de935879a0f9935ba4a1b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Sep 2018 21:24:20 -0400 Subject: [PATCH 54/56] fix incorrect value for inline loop 09cc1dc66067f378 failed to handle mem_slot_index correctly closes #1436 --- src/ir.cpp | 30 +++++++++++++++++++----------- test/cases/eval.zig | 9 +++++++++ 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index 98c749f623..6eacb66f5c 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -17,8 +17,7 @@ #include "util.hpp" struct IrExecContext { - ConstExprValue *mem_slot_list; - size_t mem_slot_count; + ZigList mem_slot_list; }; struct IrBuilder { @@ -12496,13 +12495,20 @@ static TypeTableEntry *ir_analyze_instruction_decl_var(IrAnalyze *ira, IrInstruc } } - if (var->value->type != nullptr && var->value->type != result_type && !is_comptime_var) { + if (var->value->type != nullptr && !is_comptime_var) { // This is at least the second time we've seen this variable declaration during analysis. // This means that this is actually a different variable due to, e.g. an inline while loop. // We make a new variable so that it can hold a different type, and so the debug info can // be distinct. VariableTableEntry *new_var = create_local_var(ira->codegen, var->decl_node, var->child_scope, &var->name, var->src_is_const, var->gen_is_const, var->shadowable, var->is_comptime, true); + new_var->owner_exec = var->owner_exec; + if (var->mem_slot_index != SIZE_MAX) { + ConstExprValue *vals = create_const_vals(1); + new_var->mem_slot_index = ira->exec_context.mem_slot_list.length; + ira->exec_context.mem_slot_list.append(vals); + } + var->next_var = new_var; var = new_var; } @@ -12525,10 +12531,9 @@ static TypeTableEntry *ir_analyze_instruction_decl_var(IrAnalyze *ira, IrInstruc if (casted_init_value->value.special != ConstValSpecialRuntime) { if (var->mem_slot_index != SIZE_MAX) { - assert(var->mem_slot_index < ira->exec_context.mem_slot_count); - ConstExprValue *mem_slot = &ira->exec_context.mem_slot_list[var->mem_slot_index]; - copy_const_val(mem_slot, &casted_init_value->value, - !is_comptime_var || var->gen_is_const); + assert(var->mem_slot_index < ira->exec_context.mem_slot_list.length); + ConstExprValue *mem_slot = ira->exec_context.mem_slot_list.at(var->mem_slot_index); + copy_const_val(mem_slot, &casted_init_value->value, !is_comptime_var || var->gen_is_const); if (is_comptime_var || (var_class_requires_const && var->gen_is_const)) { ir_build_const_from(ira, &decl_var_instruction->base); @@ -13012,8 +13017,8 @@ static IrInstruction *ir_get_var_ptr(IrAnalyze *ira, IrInstruction *instruction, assert(var->owner_exec != nullptr); assert(var->owner_exec->analysis != nullptr); IrExecContext *exec_context = &var->owner_exec->analysis->exec_context; - assert(var->mem_slot_index < exec_context->mem_slot_count); - mem_slot = &exec_context->mem_slot_list[var->mem_slot_index]; + assert(var->mem_slot_index < exec_context->mem_slot_list.length); + mem_slot = exec_context->mem_slot_list.at(var->mem_slot_index); } } @@ -21228,8 +21233,11 @@ TypeTableEntry *ir_analyze(CodeGen *codegen, IrExecutable *old_exec, IrExecutabl ira->new_irb.codegen = codegen; ira->new_irb.exec = new_exec; - ira->exec_context.mem_slot_count = ira->old_irb.exec->mem_slot_count; - ira->exec_context.mem_slot_list = create_const_vals(ira->exec_context.mem_slot_count); + ConstExprValue *vals = create_const_vals(ira->old_irb.exec->mem_slot_count); + ira->exec_context.mem_slot_list.resize(ira->old_irb.exec->mem_slot_count); + for (size_t i = 0; i < ira->exec_context.mem_slot_list.length; i += 1) { + ira->exec_context.mem_slot_list.items[i] = &vals[i]; + } IrBasicBlock *old_entry_bb = ira->old_irb.exec->basic_block_list.at(0); IrBasicBlock *new_entry_bb = ir_get_new_bb(ira, old_entry_bb, nullptr); diff --git a/test/cases/eval.zig b/test/cases/eval.zig index 96b9e14cf5..695cd4df77 100644 --- a/test/cases/eval.zig +++ b/test/cases/eval.zig @@ -665,3 +665,12 @@ fn testVarInsideInlineLoop(args: ...) void { if (i == 1) assert(x == 42); } } + +test "inline for with same type but different values" { + var res: usize = 0; + inline for ([]type{ [2]u8, [1]u8, [2]u8 }) |T| { + var a: T = undefined; + res += a.len; + } + assert(res == 5); +} From bc88ef2dc331733e69ed895b7a815ba0a6996dc3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 3 Sep 2018 22:47:23 -0400 Subject: [PATCH 55/56] compile errors for unimplemented minValue/maxValue builtins --- src/analyze.cpp | 2 -- src/ir.cpp | 8 +------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/analyze.cpp b/src/analyze.cpp index 405e010ba4..85b34cb017 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -5624,8 +5624,6 @@ void eval_min_max_value(CodeGen *g, TypeTableEntry *type_entry, ConstExprValue * if (type_entry->id == TypeTableEntryIdInt) { const_val->special = ConstValSpecialStatic; eval_min_max_value_int(g, type_entry, &const_val->data.x_bigint, is_max); - } else if (type_entry->id == TypeTableEntryIdFloat) { - zig_panic("TODO analyze_min_max_value float"); } else if (type_entry->id == TypeTableEntryIdBool) { const_val->special = ConstValSpecialStatic; const_val->data.x_bool = is_max; diff --git a/src/ir.cpp b/src/ir.cpp index 6eacb66f5c..8bf0e710b3 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -16482,12 +16482,6 @@ static TypeTableEntry *ir_analyze_min_max(IrAnalyze *ira, IrInstruction *source_ eval_min_max_value(ira->codegen, target_type, out_val, is_max); return ira->codegen->builtin_types.entry_num_lit_int; } - case TypeTableEntryIdFloat: - { - ConstExprValue *out_val = ir_build_const_from(ira, source_instruction); - eval_min_max_value(ira->codegen, target_type, out_val, is_max); - return ira->codegen->builtin_types.entry_num_lit_float; - } case TypeTableEntryIdBool: case TypeTableEntryIdVoid: { @@ -16496,7 +16490,7 @@ static TypeTableEntry *ir_analyze_min_max(IrAnalyze *ira, IrInstruction *source_ return target_type; } case TypeTableEntryIdEnum: - zig_panic("TODO min/max value for enum type"); + case TypeTableEntryIdFloat: case TypeTableEntryIdMetaType: case TypeTableEntryIdUnreachable: case TypeTableEntryIdPointer: From 8b50d10a846005f221bf910b3d8928915c126c4c Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Tue, 4 Sep 2018 20:16:12 +1200 Subject: [PATCH 56/56] std/crypto: Clean up poly1305/x25519 --- std/crypto/index.zig | 8 +- std/crypto/poly1305.zig | 36 +- std/crypto/throughput_test.zig | 5 - std/crypto/x25519.zig | 936 ++++++++++++++++----------------- 4 files changed, 485 insertions(+), 500 deletions(-) diff --git a/std/crypto/index.zig b/std/crypto/index.zig index 9f20ce514b..3a88fe2b2c 100644 --- a/std/crypto/index.zig +++ b/std/crypto/index.zig @@ -32,13 +32,13 @@ pub const Poly1305 = @import("poly1305.zig").Poly1305; pub const X25519 = @import("x25519.zig").X25519; test "crypto" { + _ = @import("blake2.zig"); + _ = @import("chacha20.zig"); + _ = @import("hmac.zig"); _ = @import("md5.zig"); + _ = @import("poly1305.zig"); _ = @import("sha1.zig"); _ = @import("sha2.zig"); _ = @import("sha3.zig"); - _ = @import("blake2.zig"); - _ = @import("hmac.zig"); - _ = @import("chacha20.zig"); - _ = @import("poly1305.zig"); _ = @import("x25519.zig"); } diff --git a/std/crypto/poly1305.zig b/std/crypto/poly1305.zig index 1cec72890f..f5e11fc0a1 100644 --- a/std/crypto/poly1305.zig +++ b/std/crypto/poly1305.zig @@ -26,7 +26,7 @@ pub const Poly1305 = struct { // How many bytes are there in the chunk. c_idx: usize, - fn secure_zero(self: *Poly1305) void { + fn secureZero(self: *Self) void { std.mem.secureZero(u8, @ptrCast([*]u8, self)[0..@sizeOf(Poly1305)]); } @@ -53,7 +53,7 @@ pub const Poly1305 = struct { } // add 2^130 to every input block ctx.c[4] = 1; - poly_clear_c(&ctx); + polyClearC(&ctx); // load r and pad (r has some of its bits cleared) { @@ -85,7 +85,7 @@ pub const Poly1305 = struct { // ctx->r <= 0ffffffc_0ffffffc_0ffffffc_0fffffff // Postcondition: // ctx->h <= 4_ffffffff_ffffffff_ffffffff_ffffffff - fn poly_block(ctx: *Poly1305) void { + fn polyBlock(ctx: *Self) void { // s = h + c, without carry propagation const s0 = u64(ctx.h[0]) + ctx.c[0]; // s0 <= 1_fffffffe const s1 = u64(ctx.h[1]) + ctx.c[1]; // s1 <= 1_fffffffe @@ -127,7 +127,7 @@ pub const Poly1305 = struct { } // (re-)initializes the input counter and input buffer - fn poly_clear_c(ctx: *Poly1305) void { + fn polyClearC(ctx: *Self) void { ctx.c[0] = 0; ctx.c[1] = 0; ctx.c[2] = 0; @@ -135,32 +135,32 @@ pub const Poly1305 = struct { ctx.c_idx = 0; } - fn poly_take_input(ctx: *Poly1305, input: u8) void { + fn polyTakeInput(ctx: *Self, input: u8) void { const word = ctx.c_idx >> 2; const byte = ctx.c_idx & 3; ctx.c[word] |= std.math.shl(u32, input, byte * 8); ctx.c_idx += 1; } - fn poly_update(ctx: *Poly1305, msg: []const u8) void { + fn polyUpdate(ctx: *Self, msg: []const u8) void { for (msg) |b| { - poly_take_input(ctx, b); + polyTakeInput(ctx, b); if (ctx.c_idx == 16) { - poly_block(ctx); - poly_clear_c(ctx); + polyBlock(ctx); + polyClearC(ctx); } } } - inline fn alignto(x: usize, block_size: usize) usize { + fn alignTo(x: usize, block_size: usize) usize { return ((~x) +% 1) & (block_size - 1); } // Feed data into the MAC context. pub fn update(ctx: *Self, msg: []const u8) void { // Align ourselves with block boundaries - const alignm = std.math.min(alignto(ctx.c_idx, 16), msg.len); - poly_update(ctx, msg[0..alignm]); + const alignm = std.math.min(alignTo(ctx.c_idx, 16), msg.len); + polyUpdate(ctx, msg[0..alignm]); var nmsg = msg[alignm..]; @@ -172,15 +172,15 @@ pub const Poly1305 = struct { ctx.c[1] = readInt(nmsg[4..8], u32, Endian.Little); ctx.c[2] = readInt(nmsg[8..12], u32, Endian.Little); ctx.c[3] = readInt(nmsg[12..16], u32, Endian.Little); - poly_block(ctx); + polyBlock(ctx); nmsg = nmsg[16..]; } if (nb_blocks > 0) { - poly_clear_c(ctx); + polyClearC(ctx); } // remaining bytes - poly_update(ctx, nmsg[0..]); + polyUpdate(ctx, nmsg[0..]); } // Finalize the MAC and output into buffer provided by caller. @@ -190,9 +190,9 @@ pub const Poly1305 = struct { // move the final 1 according to remaining input length // (We may add less than 2^130 to the last input block) ctx.c[4] = 0; - poly_take_input(ctx, 1); + polyTakeInput(ctx, 1); // one last hash update - poly_block(ctx); + polyBlock(ctx); } // check if we should subtract 2^130-5 by performing the @@ -215,7 +215,7 @@ pub const Poly1305 = struct { writeInt(out[8..], @truncate(u32, uu2), Endian.Little); writeInt(out[12..], @truncate(u32, uu3), Endian.Little); - ctx.secure_zero(); + ctx.secureZero(); } }; diff --git a/std/crypto/throughput_test.zig b/std/crypto/throughput_test.zig index 5f25c0b885..294ef5df51 100644 --- a/std/crypto/throughput_test.zig +++ b/std/crypto/throughput_test.zig @@ -168,11 +168,6 @@ pub fn main() !void { } inline for (hashes) |H| { - // TODO: Inverted early continue case here segfaults compiler. Create reduced test case. - // - // if (filter != null and std.mem.indexOf(u8, H.name, filter.?) == null) { - // continue; - // } if (filter == null or std.mem.indexOf(u8, H.name, filter.?) != null) { const throughput = try benchmarkHash(H.ty, mode(32 * MiB)); try printPad(stdout, H.name); diff --git a/std/crypto/x25519.zig b/std/crypto/x25519.zig index cf88d6e1c0..16ec09f66e 100644 --- a/std/crypto/x25519.zig +++ b/std/crypto/x25519.zig @@ -14,13 +14,13 @@ pub const X25519 = struct { pub const secret_length = 32; pub const minimum_key_length = 32; - fn trim_scalar(s: []u8) void { + fn trimScalar(s: []u8) void { s[0] &= 248; s[31] &= 127; s[31] |= 64; } - fn scalar_bit(s: []const u8, i: usize) i32 { + fn scalarBit(s: []const u8, i: usize) i32 { return (s[i >> 3] >> @intCast(u3, i & 7)) & 1; } @@ -30,7 +30,6 @@ pub const X25519 = struct { std.debug.assert(public_key.len >= minimum_key_length); var storage: [7]Fe = undefined; - var x1 = &storage[0]; var x2 = &storage[1]; var z2 = &storage[2]; @@ -40,74 +39,74 @@ pub const X25519 = struct { var t1 = &storage[6]; // computes the scalar product - fe_frombytes(x1, public_key); + Fe.fromBytes(x1, public_key); // restrict the possible scalar values var e: [32]u8 = undefined; for (e[0..]) |_, i| { e[i] = private_key[i]; } - trim_scalar(e[0..]); + trimScalar(e[0..]); // computes the actual scalar product (the result is in x2 and z2) // Montgomery ladder // In projective coordinates, to avoid divisons: x = X / Z // We don't care about the y coordinate, it's only 1 bit of information - fe_1(x2); - fe_0(z2); // "zero" point - fe_copy(x3, x1); - fe_1(z3); + Fe.init1(x2); + Fe.init0(z2); // "zero" point + Fe.copy(x3, x1); + Fe.init1(z3); var swap: i32 = 0; var pos: isize = 254; while (pos >= 0) : (pos -= 1) { // constant time conditional swap before ladder step - const b = scalar_bit(e, @intCast(usize, pos)); + const b = scalarBit(e, @intCast(usize, pos)); swap ^= b; // xor trick avoids swapping at the end of the loop - fe_cswap(x2, x3, swap); - fe_cswap(z2, z3, swap); + Fe.cswap(x2, x3, swap); + Fe.cswap(z2, z3, swap); swap = b; // anticipates one last swap after the loop // Montgomery ladder step: replaces (P2, P3) by (P2*2, P2+P3) // with differential addition - fe_sub(t0, x3, z3); - fe_sub(t1, x2, z2); - fe_add(x2, x2, z2); - fe_add(z2, x3, z3); - fe_mul(z3, t0, x2); - fe_mul(z2, z2, t1); - fe_sq(t0, t1); - fe_sq(t1, x2); - fe_add(x3, z3, z2); - fe_sub(z2, z3, z2); - fe_mul(x2, t1, t0); - fe_sub(t1, t1, t0); - fe_sq(z2, z2); - fe_mul121666(z3, t1); - fe_sq(x3, x3); - fe_add(t0, t0, z3); - fe_mul(z3, x1, z2); - fe_mul(z2, t1, t0); + Fe.sub(t0, x3, z3); + Fe.sub(t1, x2, z2); + Fe.add(x2, x2, z2); + Fe.add(z2, x3, z3); + Fe.mul(z3, t0, x2); + Fe.mul(z2, z2, t1); + Fe.sq(t0, t1); + Fe.sq(t1, x2); + Fe.add(x3, z3, z2); + Fe.sub(z2, z3, z2); + Fe.mul(x2, t1, t0); + Fe.sub(t1, t1, t0); + Fe.sq(z2, z2); + Fe.mulSmall(z3, t1, 121666); + Fe.sq(x3, x3); + Fe.add(t0, t0, z3); + Fe.mul(z3, x1, z2); + Fe.mul(z2, t1, t0); } // last swap is necessary to compensate for the xor trick // Note: after this swap, P3 == P2 + P1. - fe_cswap(x2, x3, swap); - fe_cswap(z2, z3, swap); + Fe.cswap(x2, x3, swap); + Fe.cswap(z2, z3, swap); // normalises the coordinates: x == X / Z - fe_invert(z2, z2); - fe_mul(x2, x2, z2); - fe_tobytes(out, x2); + Fe.invert(z2, z2); + Fe.mul(x2, x2, z2); + Fe.toBytes(out, x2); - x1.secure_zero(); - x2.secure_zero(); - x3.secure_zero(); - t0.secure_zero(); - t1.secure_zero(); - z2.secure_zero(); - z3.secure_zero(); + x1.secureZero(); + x2.secureZero(); + x3.secureZero(); + t0.secureZero(); + t1.secureZero(); + z2.secureZero(); + z3.secureZero(); std.mem.secureZero(u8, e[0..]); // Returns false if the output is all zero @@ -140,448 +139,439 @@ fn zerocmp(comptime T: type, a: []const T) bool { const Fe = struct { b: [10]i32, - fn secure_zero(self: *Fe) void { + fn secureZero(self: *Fe) void { std.mem.secureZero(u8, @ptrCast([*]u8, self)[0..@sizeOf(Fe)]); } -}; -fn fe_0(h: *Fe) void { - for (h.b) |*e| { - e.* = 0; - } -} - -fn fe_1(h: *Fe) void { - for (h.b[1..]) |*e| { - e.* = 0; - } - h.b[0] = 1; -} - -fn fe_copy(h: *Fe, f: *const Fe) void { - for (h.b) |_, i| { - h.b[i] = f.b[i]; - } -} - -fn fe_neg(h: *Fe, f: *const Fe) void { - for (h.b) |_, i| { - h.b[i] = -f.b[i]; - } -} - -fn fe_add(h: *Fe, f: *const Fe, g: *const Fe) void { - for (h.b) |_, i| { - h.b[i] = f.b[i] + g.b[i]; - } -} - -fn fe_sub(h: *Fe, f: *const Fe, g: *const Fe) void { - for (h.b) |_, i| { - h.b[i] = f.b[i] - g.b[i]; - } -} - -fn fe_cswap(f: *Fe, g: *Fe, b: i32) void { - for (f.b) |_, i| { - const x = (f.b[i] ^ g.b[i]) & -b; - f.b[i] ^= x; - g.b[i] ^= x; - } -} - -fn fe_ccopy(f: *Fe, g: *const Fe, b: i32) void { - for (f.b) |_, i| { - const x = (f.b[i] ^ g.b[i]) & -b; - f.b[i] ^= x; - } -} - -inline fn carryround(c: []i64, t: []i64, comptime i: comptime_int, comptime shift: comptime_int, comptime mult: comptime_int) void { - const j = (i + 1) % 10; - - c[i] = (t[i] + (i64(1) << shift)) >> (shift + 1); - t[j] += c[i] * mult; - t[i] -= c[i] * (i64(1) << (shift + 1)); -} - -fn feCarry1(h: *Fe, t: []i64) void { - var c: [10]i64 = undefined; - - var sc = c[0..]; - var st = t[0..]; - - carryround(sc, st, 9, 24, 19); - carryround(sc, st, 1, 24, 1); - carryround(sc, st, 3, 24, 1); - carryround(sc, st, 5, 24, 1); - carryround(sc, st, 7, 24, 1); - carryround(sc, st, 0, 25, 1); - carryround(sc, st, 2, 25, 1); - carryround(sc, st, 4, 25, 1); - carryround(sc, st, 6, 25, 1); - carryround(sc, st, 8, 25, 1); - - for (h.b) |_, i| { - h.b[i] = @intCast(i32, t[i]); - } -} - -fn feCarry2(h: *Fe, t: []i64) void { - var c: [10]i64 = undefined; - - var sc = c[0..]; - var st = t[0..]; - - carryround(sc, st, 0, 25, 1); - carryround(sc, st, 4, 25, 1); - carryround(sc, st, 1, 24, 1); - carryround(sc, st, 5, 24, 1); - carryround(sc, st, 2, 25, 1); - carryround(sc, st, 6, 25, 1); - carryround(sc, st, 3, 24, 1); - carryround(sc, st, 7, 24, 1); - carryround(sc, st, 4, 25, 1); - carryround(sc, st, 8, 25, 1); - carryround(sc, st, 9, 24, 19); - carryround(sc, st, 0, 25, 1); - - for (h.b) |_, i| { - h.b[i] = @intCast(i32, t[i]); - } -} - -// TODO: Use readInt(u24) but double check alignment since currently it produces different values. -fn load24_le(s: []const u8) u32 { - return s[0] | (u32(s[1]) << 8) | (u32(s[2]) << 16); -} - -fn fe_frombytes(h: *Fe, s: []const u8) void { - std.debug.assert(s.len >= 32); - - var t: [10]i64 = undefined; - - t[0] = readInt(s[0..4], u32, Endian.Little); - t[1] = load24_le(s[4..7]) << 6; - t[2] = load24_le(s[7..10]) << 5; - t[3] = load24_le(s[10..13]) << 3; - t[4] = load24_le(s[13..16]) << 2; - t[5] = readInt(s[16..20], u32, Endian.Little); - t[6] = load24_le(s[20..23]) << 7; - t[7] = load24_le(s[23..26]) << 5; - t[8] = load24_le(s[26..29]) << 4; - t[9] = (load24_le(s[29..32]) & 0x7fffff) << 2; - - feCarry1(h, t[0..]); -} - -fn fe_mul_small(h: *Fe, f: *const Fe, comptime g: comptime_int) void { - var t: [10]i64 = undefined; - - for (t[0..]) |_, i| { - t[i] = i64(f.b[i]) * g; - } - - feCarry1(h, t[0..]); -} - -fn fe_mul121666(h: *Fe, f: *const Fe) void { - fe_mul_small(h, f, 121666); -} - -fn fe_mul(h: *Fe, f1: *const Fe, g1: *const Fe) void { - const f = f1.b; - const g = g1.b; - - var F: [10]i32 = undefined; - var G: [10]i32 = undefined; - - F[1] = f[1] * 2; - F[3] = f[3] * 2; - F[5] = f[5] * 2; - F[7] = f[7] * 2; - F[9] = f[9] * 2; - - G[1] = g[1] * 19; - G[2] = g[2] * 19; - G[3] = g[3] * 19; - G[4] = g[4] * 19; - G[5] = g[5] * 19; - G[6] = g[6] * 19; - G[7] = g[7] * 19; - G[8] = g[8] * 19; - G[9] = g[9] * 19; - - // t's become h - var t: [10]i64 = undefined; - - t[0] = f[0] * i64(g[0]) + F[1] * i64(G[9]) + f[2] * i64(G[8]) + F[3] * i64(G[7]) + f[4] * i64(G[6]) + F[5] * i64(G[5]) + f[6] * i64(G[4]) + F[7] * i64(G[3]) + f[8] * i64(G[2]) + F[9] * i64(G[1]); - t[1] = f[0] * i64(g[1]) + f[1] * i64(g[0]) + f[2] * i64(G[9]) + f[3] * i64(G[8]) + f[4] * i64(G[7]) + f[5] * i64(G[6]) + f[6] * i64(G[5]) + f[7] * i64(G[4]) + f[8] * i64(G[3]) + f[9] * i64(G[2]); - t[2] = f[0] * i64(g[2]) + F[1] * i64(g[1]) + f[2] * i64(g[0]) + F[3] * i64(G[9]) + f[4] * i64(G[8]) + F[5] * i64(G[7]) + f[6] * i64(G[6]) + F[7] * i64(G[5]) + f[8] * i64(G[4]) + F[9] * i64(G[3]); - t[3] = f[0] * i64(g[3]) + f[1] * i64(g[2]) + f[2] * i64(g[1]) + f[3] * i64(g[0]) + f[4] * i64(G[9]) + f[5] * i64(G[8]) + f[6] * i64(G[7]) + f[7] * i64(G[6]) + f[8] * i64(G[5]) + f[9] * i64(G[4]); - t[4] = f[0] * i64(g[4]) + F[1] * i64(g[3]) + f[2] * i64(g[2]) + F[3] * i64(g[1]) + f[4] * i64(g[0]) + F[5] * i64(G[9]) + f[6] * i64(G[8]) + F[7] * i64(G[7]) + f[8] * i64(G[6]) + F[9] * i64(G[5]); - t[5] = f[0] * i64(g[5]) + f[1] * i64(g[4]) + f[2] * i64(g[3]) + f[3] * i64(g[2]) + f[4] * i64(g[1]) + f[5] * i64(g[0]) + f[6] * i64(G[9]) + f[7] * i64(G[8]) + f[8] * i64(G[7]) + f[9] * i64(G[6]); - t[6] = f[0] * i64(g[6]) + F[1] * i64(g[5]) + f[2] * i64(g[4]) + F[3] * i64(g[3]) + f[4] * i64(g[2]) + F[5] * i64(g[1]) + f[6] * i64(g[0]) + F[7] * i64(G[9]) + f[8] * i64(G[8]) + F[9] * i64(G[7]); - t[7] = f[0] * i64(g[7]) + f[1] * i64(g[6]) + f[2] * i64(g[5]) + f[3] * i64(g[4]) + f[4] * i64(g[3]) + f[5] * i64(g[2]) + f[6] * i64(g[1]) + f[7] * i64(g[0]) + f[8] * i64(G[9]) + f[9] * i64(G[8]); - t[8] = f[0] * i64(g[8]) + F[1] * i64(g[7]) + f[2] * i64(g[6]) + F[3] * i64(g[5]) + f[4] * i64(g[4]) + F[5] * i64(g[3]) + f[6] * i64(g[2]) + F[7] * i64(g[1]) + f[8] * i64(g[0]) + F[9] * i64(G[9]); - t[9] = f[0] * i64(g[9]) + f[1] * i64(g[8]) + f[2] * i64(g[7]) + f[3] * i64(g[6]) + f[4] * i64(g[5]) + f[5] * i64(g[4]) + f[6] * i64(g[3]) + f[7] * i64(g[2]) + f[8] * i64(g[1]) + f[9] * i64(g[0]); - - feCarry2(h, t[0..]); -} - -// we could use fe_mul() for this, but this is significantly faster -fn fe_sq(h: *Fe, fz: *const Fe) void { - const f0 = fz.b[0]; - const f1 = fz.b[1]; - const f2 = fz.b[2]; - const f3 = fz.b[3]; - const f4 = fz.b[4]; - const f5 = fz.b[5]; - const f6 = fz.b[6]; - const f7 = fz.b[7]; - const f8 = fz.b[8]; - const f9 = fz.b[9]; - - const f0_2 = f0 * 2; - const f1_2 = f1 * 2; - const f2_2 = f2 * 2; - const f3_2 = f3 * 2; - const f4_2 = f4 * 2; - const f5_2 = f5 * 2; - const f6_2 = f6 * 2; - const f7_2 = f7 * 2; - const f5_38 = f5 * 38; - const f6_19 = f6 * 19; - const f7_38 = f7 * 38; - const f8_19 = f8 * 19; - const f9_38 = f9 * 38; - - var t: [10]i64 = undefined; - - t[0] = f0 * i64(f0) + f1_2 * i64(f9_38) + f2_2 * i64(f8_19) + f3_2 * i64(f7_38) + f4_2 * i64(f6_19) + f5 * i64(f5_38); - t[1] = f0_2 * i64(f1) + f2 * i64(f9_38) + f3_2 * i64(f8_19) + f4 * i64(f7_38) + f5_2 * i64(f6_19); - t[2] = f0_2 * i64(f2) + f1_2 * i64(f1) + f3_2 * i64(f9_38) + f4_2 * i64(f8_19) + f5_2 * i64(f7_38) + f6 * i64(f6_19); - t[3] = f0_2 * i64(f3) + f1_2 * i64(f2) + f4 * i64(f9_38) + f5_2 * i64(f8_19) + f6 * i64(f7_38); - t[4] = f0_2 * i64(f4) + f1_2 * i64(f3_2) + f2 * i64(f2) + f5_2 * i64(f9_38) + f6_2 * i64(f8_19) + f7 * i64(f7_38); - t[5] = f0_2 * i64(f5) + f1_2 * i64(f4) + f2_2 * i64(f3) + f6 * i64(f9_38) + f7_2 * i64(f8_19); - t[6] = f0_2 * i64(f6) + f1_2 * i64(f5_2) + f2_2 * i64(f4) + f3_2 * i64(f3) + f7_2 * i64(f9_38) + f8 * i64(f8_19); - t[7] = f0_2 * i64(f7) + f1_2 * i64(f6) + f2_2 * i64(f5) + f3_2 * i64(f4) + f8 * i64(f9_38); - t[8] = f0_2 * i64(f8) + f1_2 * i64(f7_2) + f2_2 * i64(f6) + f3_2 * i64(f5_2) + f4 * i64(f4) + f9 * i64(f9_38); - t[9] = f0_2 * i64(f9) + f1_2 * i64(f8) + f2_2 * i64(f7) + f3_2 * i64(f6) + f4 * i64(f5_2); - - feCarry2(h, t[0..]); -} - -fn fe_sq2(h: *Fe, f: *const Fe) void { - fe_sq(h, f); - fe_mul_small(h, h, 2); -} - -// This could be simplified, but it would be slower -fn fe_invert(out: *Fe, z: *const Fe) void { - var i: usize = undefined; - - var t: [4]Fe = undefined; - var t0 = &t[0]; - var t1 = &t[1]; - var t2 = &t[2]; - var t3 = &t[3]; - - fe_sq(t0, z); - fe_sq(t1, t0); - fe_sq(t1, t1); - fe_mul(t1, z, t1); - fe_mul(t0, t0, t1); - - fe_sq(t2, t0); - fe_mul(t1, t1, t2); - - fe_sq(t2, t1); - i = 1; - while (i < 5) : (i += 1) fe_sq(t2, t2); - fe_mul(t1, t2, t1); - - fe_sq(t2, t1); - i = 1; - while (i < 10) : (i += 1) fe_sq(t2, t2); - fe_mul(t2, t2, t1); - - fe_sq(t3, t2); - i = 1; - while (i < 20) : (i += 1) fe_sq(t3, t3); - fe_mul(t2, t3, t2); - - fe_sq(t2, t2); - i = 1; - while (i < 10) : (i += 1) fe_sq(t2, t2); - fe_mul(t1, t2, t1); - - fe_sq(t2, t1); - i = 1; - while (i < 50) : (i += 1) fe_sq(t2, t2); - fe_mul(t2, t2, t1); - - fe_sq(t3, t2); - i = 1; - while (i < 100) : (i += 1) fe_sq(t3, t3); - fe_mul(t2, t3, t2); - - fe_sq(t2, t2); - i = 1; - while (i < 50) : (i += 1) fe_sq(t2, t2); - fe_mul(t1, t2, t1); - - fe_sq(t1, t1); - i = 1; - while (i < 5) : (i += 1) fe_sq(t1, t1); - fe_mul(out, t1, t0); - - t0.secure_zero(); - t1.secure_zero(); - t2.secure_zero(); - t3.secure_zero(); -} - -// This could be simplified, but it would be slower -fn fe_pow22523(out: *Fe, z: *const Fe) void { - var i: usize = undefined; - - var t: [3]Fe = undefined; - var t0 = &t[0]; - var t1 = &t[1]; - var t2 = &t[2]; - - fe_sq(t0, z); - fe_sq(t1, t0); - fe_sq(t1, t1); - fe_mul(t1, z, t1); - fe_mul(t0, t0, t1); - - fe_sq(t0, t0); - fe_mul(t0, t1, t0); - - fe_sq(t1, t0); - i = 1; - while (i < 5) : (i += 1) fe_sq(t1, t1); - fe_mul(t0, t1, t0); - - fe_sq(t1, t0); - i = 1; - while (i < 10) : (i += 1) fe_sq(t1, t1); - fe_mul(t1, t1, t0); - - fe_sq(t2, t1); - i = 1; - while (i < 20) : (i += 1) fe_sq(t2, t2); - fe_mul(t1, t2, t1); - - fe_sq(t1, t1); - i = 1; - while (i < 10) : (i += 1) fe_sq(t1, t1); - fe_mul(t0, t1, t0); - - fe_sq(t1, t0); - i = 1; - while (i < 50) : (i += 1) fe_sq(t1, t1); - fe_mul(t1, t1, t0); - - fe_sq(t2, t1); - i = 1; - while (i < 100) : (i += 1) fe_sq(t2, t2); - fe_mul(t1, t2, t1); - - fe_sq(t1, t1); - i = 1; - while (i < 50) : (i += 1) fe_sq(t1, t1); - fe_mul(t0, t1, t0); - - fe_sq(t0, t0); - i = 1; - while (i < 2) : (i += 1) fe_sq(t0, t0); - fe_mul(out, t0, z); - - t0.secure_zero(); - t1.secure_zero(); - t2.secure_zero(); -} - -inline fn tobytesround(c: []i64, t: []i64, comptime i: comptime_int, comptime shift: comptime_int) void { - c[i] = t[i] >> shift; - if (i + 1 < 10) { - t[i + 1] += c[i]; - } - t[i] -= c[i] * (i32(1) << shift); -} - -fn fe_tobytes(s: []u8, h: *const Fe) void { - std.debug.assert(s.len >= 32); - - var t: [10]i64 = undefined; - for (h.b[0..]) |_, i| { - t[i] = h.b[i]; - } - - var q = (19 * t[9] + ((i32(1) << 24))) >> 25; - { - var i: usize = 0; - while (i < 5) : (i += 1) { - q += t[2 * i]; - q >>= 26; - q += t[2 * i + 1]; - q >>= 25; + fn init0(h: *Fe) void { + for (h.b) |*e| { + e.* = 0; } } - t[0] += 19 * q; - var c: [10]i64 = undefined; - - var st = t[0..]; - var sc = c[0..]; - - tobytesround(sc, st, 0, 26); - tobytesround(sc, st, 1, 25); - tobytesround(sc, st, 2, 26); - tobytesround(sc, st, 3, 25); - tobytesround(sc, st, 4, 26); - tobytesround(sc, st, 5, 25); - tobytesround(sc, st, 6, 26); - tobytesround(sc, st, 7, 25); - tobytesround(sc, st, 8, 26); - tobytesround(sc, st, 9, 25); - - var ut: [10]u32 = undefined; - for (ut[0..]) |_, i| { - ut[i] = @bitCast(u32, @intCast(i32, t[i])); + fn init1(h: *Fe) void { + for (h.b[1..]) |*e| { + e.* = 0; + } + h.b[0] = 1; } - writeInt(s[0..], (ut[0] >> 0) | (ut[1] << 26), Endian.Little); - writeInt(s[4..], (ut[1] >> 6) | (ut[2] << 19), Endian.Little); - writeInt(s[8..], (ut[2] >> 13) | (ut[3] << 13), Endian.Little); - writeInt(s[12..], (ut[3] >> 19) | (ut[4] << 6), Endian.Little); - writeInt(s[16..], (ut[5] >> 0) | (ut[6] << 25), Endian.Little); - writeInt(s[20..], (ut[6] >> 7) | (ut[7] << 19), Endian.Little); - writeInt(s[24..], (ut[7] >> 13) | (ut[8] << 12), Endian.Little); - writeInt(s[28..], (ut[8] >> 20) | (ut[9] << 6), Endian.Little); + fn copy(h: *Fe, f: *const Fe) void { + for (h.b) |_, i| { + h.b[i] = f.b[i]; + } + } - std.mem.secureZero(i64, t[0..]); -} + fn neg(h: *Fe, f: *const Fe) void { + for (h.b) |_, i| { + h.b[i] = -f.b[i]; + } + } -// Parity check. Returns 0 if even, 1 if odd -fn fe_isnegative(f: *const Fe) bool { - var s: [32]u8 = undefined; - fe_tobytes(s[0..], f); - const isneg = s[0] & 1; - s.secure_zero(); - return isneg; -} + fn add(h: *Fe, f: *const Fe, g: *const Fe) void { + for (h.b) |_, i| { + h.b[i] = f.b[i] + g.b[i]; + } + } -fn fe_isnonzero(f: *const Fe) bool { - var s: [32]u8 = undefined; - fe_tobytes(s[0..], f); - const isnonzero = zerocmp(u8, s[0..]); - s.secure_zero(); - return isneg; -} + fn sub(h: *Fe, f: *const Fe, g: *const Fe) void { + for (h.b) |_, i| { + h.b[i] = f.b[i] - g.b[i]; + } + } + + fn cswap(f: *Fe, g: *Fe, b: i32) void { + for (f.b) |_, i| { + const x = (f.b[i] ^ g.b[i]) & -b; + f.b[i] ^= x; + g.b[i] ^= x; + } + } + + fn ccopy(f: *Fe, g: *const Fe, b: i32) void { + for (f.b) |_, i| { + const x = (f.b[i] ^ g.b[i]) & -b; + f.b[i] ^= x; + } + } + + inline fn carryRound(c: []i64, t: []i64, comptime i: comptime_int, comptime shift: comptime_int, comptime mult: comptime_int) void { + const j = (i + 1) % 10; + + c[i] = (t[i] + (i64(1) << shift)) >> (shift + 1); + t[j] += c[i] * mult; + t[i] -= c[i] * (i64(1) << (shift + 1)); + } + + fn carry1(h: *Fe, t: []i64) void { + var c: [10]i64 = undefined; + + var sc = c[0..]; + var st = t[0..]; + + carryRound(sc, st, 9, 24, 19); + carryRound(sc, st, 1, 24, 1); + carryRound(sc, st, 3, 24, 1); + carryRound(sc, st, 5, 24, 1); + carryRound(sc, st, 7, 24, 1); + carryRound(sc, st, 0, 25, 1); + carryRound(sc, st, 2, 25, 1); + carryRound(sc, st, 4, 25, 1); + carryRound(sc, st, 6, 25, 1); + carryRound(sc, st, 8, 25, 1); + + for (h.b) |_, i| { + h.b[i] = @intCast(i32, t[i]); + } + } + + fn carry2(h: *Fe, t: []i64) void { + var c: [10]i64 = undefined; + + var sc = c[0..]; + var st = t[0..]; + + carryRound(sc, st, 0, 25, 1); + carryRound(sc, st, 4, 25, 1); + carryRound(sc, st, 1, 24, 1); + carryRound(sc, st, 5, 24, 1); + carryRound(sc, st, 2, 25, 1); + carryRound(sc, st, 6, 25, 1); + carryRound(sc, st, 3, 24, 1); + carryRound(sc, st, 7, 24, 1); + carryRound(sc, st, 4, 25, 1); + carryRound(sc, st, 8, 25, 1); + carryRound(sc, st, 9, 24, 19); + carryRound(sc, st, 0, 25, 1); + + for (h.b) |_, i| { + h.b[i] = @intCast(i32, t[i]); + } + } + + fn fromBytes(h: *Fe, s: []const u8) void { + std.debug.assert(s.len >= 32); + + var t: [10]i64 = undefined; + + t[0] = readInt(s[0..4], u32, Endian.Little); + t[1] = readInt(s[4..7], u32, Endian.Little) << 6; + t[2] = readInt(s[7..10], u32, Endian.Little) << 5; + t[3] = readInt(s[10..13], u32, Endian.Little) << 3; + t[4] = readInt(s[13..16], u32, Endian.Little) << 2; + t[5] = readInt(s[16..20], u32, Endian.Little); + t[6] = readInt(s[20..23], u32, Endian.Little) << 7; + t[7] = readInt(s[23..26], u32, Endian.Little) << 5; + t[8] = readInt(s[26..29], u32, Endian.Little) << 4; + t[9] = (readInt(s[29..32], u32, Endian.Little) & 0x7fffff) << 2; + + carry1(h, t[0..]); + } + + fn mulSmall(h: *Fe, f: *const Fe, comptime g: comptime_int) void { + var t: [10]i64 = undefined; + + for (t[0..]) |_, i| { + t[i] = i64(f.b[i]) * g; + } + + carry1(h, t[0..]); + } + + fn mul(h: *Fe, f1: *const Fe, g1: *const Fe) void { + const f = f1.b; + const g = g1.b; + + var F: [10]i32 = undefined; + var G: [10]i32 = undefined; + + F[1] = f[1] * 2; + F[3] = f[3] * 2; + F[5] = f[5] * 2; + F[7] = f[7] * 2; + F[9] = f[9] * 2; + + G[1] = g[1] * 19; + G[2] = g[2] * 19; + G[3] = g[3] * 19; + G[4] = g[4] * 19; + G[5] = g[5] * 19; + G[6] = g[6] * 19; + G[7] = g[7] * 19; + G[8] = g[8] * 19; + G[9] = g[9] * 19; + + // t's become h + var t: [10]i64 = undefined; + + t[0] = f[0] * i64(g[0]) + F[1] * i64(G[9]) + f[2] * i64(G[8]) + F[3] * i64(G[7]) + f[4] * i64(G[6]) + F[5] * i64(G[5]) + f[6] * i64(G[4]) + F[7] * i64(G[3]) + f[8] * i64(G[2]) + F[9] * i64(G[1]); + t[1] = f[0] * i64(g[1]) + f[1] * i64(g[0]) + f[2] * i64(G[9]) + f[3] * i64(G[8]) + f[4] * i64(G[7]) + f[5] * i64(G[6]) + f[6] * i64(G[5]) + f[7] * i64(G[4]) + f[8] * i64(G[3]) + f[9] * i64(G[2]); + t[2] = f[0] * i64(g[2]) + F[1] * i64(g[1]) + f[2] * i64(g[0]) + F[3] * i64(G[9]) + f[4] * i64(G[8]) + F[5] * i64(G[7]) + f[6] * i64(G[6]) + F[7] * i64(G[5]) + f[8] * i64(G[4]) + F[9] * i64(G[3]); + t[3] = f[0] * i64(g[3]) + f[1] * i64(g[2]) + f[2] * i64(g[1]) + f[3] * i64(g[0]) + f[4] * i64(G[9]) + f[5] * i64(G[8]) + f[6] * i64(G[7]) + f[7] * i64(G[6]) + f[8] * i64(G[5]) + f[9] * i64(G[4]); + t[4] = f[0] * i64(g[4]) + F[1] * i64(g[3]) + f[2] * i64(g[2]) + F[3] * i64(g[1]) + f[4] * i64(g[0]) + F[5] * i64(G[9]) + f[6] * i64(G[8]) + F[7] * i64(G[7]) + f[8] * i64(G[6]) + F[9] * i64(G[5]); + t[5] = f[0] * i64(g[5]) + f[1] * i64(g[4]) + f[2] * i64(g[3]) + f[3] * i64(g[2]) + f[4] * i64(g[1]) + f[5] * i64(g[0]) + f[6] * i64(G[9]) + f[7] * i64(G[8]) + f[8] * i64(G[7]) + f[9] * i64(G[6]); + t[6] = f[0] * i64(g[6]) + F[1] * i64(g[5]) + f[2] * i64(g[4]) + F[3] * i64(g[3]) + f[4] * i64(g[2]) + F[5] * i64(g[1]) + f[6] * i64(g[0]) + F[7] * i64(G[9]) + f[8] * i64(G[8]) + F[9] * i64(G[7]); + t[7] = f[0] * i64(g[7]) + f[1] * i64(g[6]) + f[2] * i64(g[5]) + f[3] * i64(g[4]) + f[4] * i64(g[3]) + f[5] * i64(g[2]) + f[6] * i64(g[1]) + f[7] * i64(g[0]) + f[8] * i64(G[9]) + f[9] * i64(G[8]); + t[8] = f[0] * i64(g[8]) + F[1] * i64(g[7]) + f[2] * i64(g[6]) + F[3] * i64(g[5]) + f[4] * i64(g[4]) + F[5] * i64(g[3]) + f[6] * i64(g[2]) + F[7] * i64(g[1]) + f[8] * i64(g[0]) + F[9] * i64(G[9]); + t[9] = f[0] * i64(g[9]) + f[1] * i64(g[8]) + f[2] * i64(g[7]) + f[3] * i64(g[6]) + f[4] * i64(g[5]) + f[5] * i64(g[4]) + f[6] * i64(g[3]) + f[7] * i64(g[2]) + f[8] * i64(g[1]) + f[9] * i64(g[0]); + + carry2(h, t[0..]); + } + + // we could use Fe.mul() for this, but this is significantly faster + fn sq(h: *Fe, fz: *const Fe) void { + const f0 = fz.b[0]; + const f1 = fz.b[1]; + const f2 = fz.b[2]; + const f3 = fz.b[3]; + const f4 = fz.b[4]; + const f5 = fz.b[5]; + const f6 = fz.b[6]; + const f7 = fz.b[7]; + const f8 = fz.b[8]; + const f9 = fz.b[9]; + + const f0_2 = f0 * 2; + const f1_2 = f1 * 2; + const f2_2 = f2 * 2; + const f3_2 = f3 * 2; + const f4_2 = f4 * 2; + const f5_2 = f5 * 2; + const f6_2 = f6 * 2; + const f7_2 = f7 * 2; + const f5_38 = f5 * 38; + const f6_19 = f6 * 19; + const f7_38 = f7 * 38; + const f8_19 = f8 * 19; + const f9_38 = f9 * 38; + + var t: [10]i64 = undefined; + + t[0] = f0 * i64(f0) + f1_2 * i64(f9_38) + f2_2 * i64(f8_19) + f3_2 * i64(f7_38) + f4_2 * i64(f6_19) + f5 * i64(f5_38); + t[1] = f0_2 * i64(f1) + f2 * i64(f9_38) + f3_2 * i64(f8_19) + f4 * i64(f7_38) + f5_2 * i64(f6_19); + t[2] = f0_2 * i64(f2) + f1_2 * i64(f1) + f3_2 * i64(f9_38) + f4_2 * i64(f8_19) + f5_2 * i64(f7_38) + f6 * i64(f6_19); + t[3] = f0_2 * i64(f3) + f1_2 * i64(f2) + f4 * i64(f9_38) + f5_2 * i64(f8_19) + f6 * i64(f7_38); + t[4] = f0_2 * i64(f4) + f1_2 * i64(f3_2) + f2 * i64(f2) + f5_2 * i64(f9_38) + f6_2 * i64(f8_19) + f7 * i64(f7_38); + t[5] = f0_2 * i64(f5) + f1_2 * i64(f4) + f2_2 * i64(f3) + f6 * i64(f9_38) + f7_2 * i64(f8_19); + t[6] = f0_2 * i64(f6) + f1_2 * i64(f5_2) + f2_2 * i64(f4) + f3_2 * i64(f3) + f7_2 * i64(f9_38) + f8 * i64(f8_19); + t[7] = f0_2 * i64(f7) + f1_2 * i64(f6) + f2_2 * i64(f5) + f3_2 * i64(f4) + f8 * i64(f9_38); + t[8] = f0_2 * i64(f8) + f1_2 * i64(f7_2) + f2_2 * i64(f6) + f3_2 * i64(f5_2) + f4 * i64(f4) + f9 * i64(f9_38); + t[9] = f0_2 * i64(f9) + f1_2 * i64(f8) + f2_2 * i64(f7) + f3_2 * i64(f6) + f4 * i64(f5_2); + + carry2(h, t[0..]); + } + + fn sq2(h: *Fe, f: *const Fe) void { + Fe.sq(h, f); + Fe.mul_small(h, h, 2); + } + + // This could be simplified, but it would be slower + fn invert(out: *Fe, z: *const Fe) void { + var i: usize = undefined; + + var t: [4]Fe = undefined; + var t0 = &t[0]; + var t1 = &t[1]; + var t2 = &t[2]; + var t3 = &t[3]; + + Fe.sq(t0, z); + Fe.sq(t1, t0); + Fe.sq(t1, t1); + Fe.mul(t1, z, t1); + Fe.mul(t0, t0, t1); + + Fe.sq(t2, t0); + Fe.mul(t1, t1, t2); + + Fe.sq(t2, t1); + i = 1; + while (i < 5) : (i += 1) Fe.sq(t2, t2); + Fe.mul(t1, t2, t1); + + Fe.sq(t2, t1); + i = 1; + while (i < 10) : (i += 1) Fe.sq(t2, t2); + Fe.mul(t2, t2, t1); + + Fe.sq(t3, t2); + i = 1; + while (i < 20) : (i += 1) Fe.sq(t3, t3); + Fe.mul(t2, t3, t2); + + Fe.sq(t2, t2); + i = 1; + while (i < 10) : (i += 1) Fe.sq(t2, t2); + Fe.mul(t1, t2, t1); + + Fe.sq(t2, t1); + i = 1; + while (i < 50) : (i += 1) Fe.sq(t2, t2); + Fe.mul(t2, t2, t1); + + Fe.sq(t3, t2); + i = 1; + while (i < 100) : (i += 1) Fe.sq(t3, t3); + Fe.mul(t2, t3, t2); + + Fe.sq(t2, t2); + i = 1; + while (i < 50) : (i += 1) Fe.sq(t2, t2); + Fe.mul(t1, t2, t1); + + Fe.sq(t1, t1); + i = 1; + while (i < 5) : (i += 1) Fe.sq(t1, t1); + Fe.mul(out, t1, t0); + + t0.secureZero(); + t1.secureZero(); + t2.secureZero(); + t3.secureZero(); + } + + // This could be simplified, but it would be slower + fn pow22523(out: *Fe, z: *const Fe) void { + var i: usize = undefined; + + var t: [3]Fe = undefined; + var t0 = &t[0]; + var t1 = &t[1]; + var t2 = &t[2]; + + Fe.sq(t0, z); + Fe.sq(t1, t0); + Fe.sq(t1, t1); + Fe.mul(t1, z, t1); + Fe.mul(t0, t0, t1); + + Fe.sq(t0, t0); + Fe.mul(t0, t1, t0); + + Fe.sq(t1, t0); + i = 1; + while (i < 5) : (i += 1) Fe.sq(t1, t1); + Fe.mul(t0, t1, t0); + + Fe.sq(t1, t0); + i = 1; + while (i < 10) : (i += 1) Fe.sq(t1, t1); + Fe.mul(t1, t1, t0); + + Fe.sq(t2, t1); + i = 1; + while (i < 20) : (i += 1) Fe.sq(t2, t2); + Fe.mul(t1, t2, t1); + + Fe.sq(t1, t1); + i = 1; + while (i < 10) : (i += 1) Fe.sq(t1, t1); + Fe.mul(t0, t1, t0); + + Fe.sq(t1, t0); + i = 1; + while (i < 50) : (i += 1) Fe.sq(t1, t1); + Fe.mul(t1, t1, t0); + + Fe.sq(t2, t1); + i = 1; + while (i < 100) : (i += 1) Fe.sq(t2, t2); + Fe.mul(t1, t2, t1); + + Fe.sq(t1, t1); + i = 1; + while (i < 50) : (i += 1) Fe.sq(t1, t1); + Fe.mul(t0, t1, t0); + + Fe.sq(t0, t0); + i = 1; + while (i < 2) : (i += 1) Fe.sq(t0, t0); + Fe.mul(out, t0, z); + + t0.secureZero(); + t1.secureZero(); + t2.secureZero(); + } + + inline fn toBytesRound(c: []i64, t: []i64, comptime i: comptime_int, comptime shift: comptime_int) void { + c[i] = t[i] >> shift; + if (i + 1 < 10) { + t[i + 1] += c[i]; + } + t[i] -= c[i] * (i32(1) << shift); + } + + fn toBytes(s: []u8, h: *const Fe) void { + std.debug.assert(s.len >= 32); + + var t: [10]i64 = undefined; + for (h.b[0..]) |_, i| { + t[i] = h.b[i]; + } + + var q = (19 * t[9] + ((i32(1) << 24))) >> 25; + { + var i: usize = 0; + while (i < 5) : (i += 1) { + q += t[2 * i]; + q >>= 26; + q += t[2 * i + 1]; + q >>= 25; + } + } + t[0] += 19 * q; + + var c: [10]i64 = undefined; + + var st = t[0..]; + var sc = c[0..]; + + toBytesRound(sc, st, 0, 26); + toBytesRound(sc, st, 1, 25); + toBytesRound(sc, st, 2, 26); + toBytesRound(sc, st, 3, 25); + toBytesRound(sc, st, 4, 26); + toBytesRound(sc, st, 5, 25); + toBytesRound(sc, st, 6, 26); + toBytesRound(sc, st, 7, 25); + toBytesRound(sc, st, 8, 26); + toBytesRound(sc, st, 9, 25); + + var ut: [10]u32 = undefined; + for (ut[0..]) |_, i| { + ut[i] = @bitCast(u32, @intCast(i32, t[i])); + } + + writeInt(s[0..], (ut[0] >> 0) | (ut[1] << 26), Endian.Little); + writeInt(s[4..], (ut[1] >> 6) | (ut[2] << 19), Endian.Little); + writeInt(s[8..], (ut[2] >> 13) | (ut[3] << 13), Endian.Little); + writeInt(s[12..], (ut[3] >> 19) | (ut[4] << 6), Endian.Little); + writeInt(s[16..], (ut[5] >> 0) | (ut[6] << 25), Endian.Little); + writeInt(s[20..], (ut[6] >> 7) | (ut[7] << 19), Endian.Little); + writeInt(s[24..], (ut[7] >> 13) | (ut[8] << 12), Endian.Little); + writeInt(s[28..], (ut[8] >> 20) | (ut[9] << 6), Endian.Little); + + std.mem.secureZero(i64, t[0..]); + } + + // Parity check. Returns 0 if even, 1 if odd + fn isNegative(f: *const Fe) bool { + var s: [32]u8 = undefined; + Fe.toBytes(s[0..], f); + const isneg = s[0] & 1; + s.secureZero(); + return isneg; + } + + fn isNonZero(f: *const Fe) bool { + var s: [32]u8 = undefined; + Fe.toBytes(s[0..], f); + const isnonzero = zerocmp(u8, s[0..]); + s.secureZero(); + return isneg; + } +}; test "x25519 rfc7748 vector1" { const secret_key = "\xa5\x46\xe3\x6b\xf0\x52\x7c\x9d\x3b\x16\x15\x4b\x82\x46\x5e\xdd\x62\x14\x4c\x0a\xc1\xfc\x5a\x18\x50\x6a\x22\x44\xba\x44\x9a\xc4";