From 83513ade3591de673e9ac4824fe974cd8f90c847 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 25 Jul 2025 22:10:29 -0700 Subject: [PATCH 01/47] std.compress: rework flate to new I/O API --- lib/std/compress.zig | 5 +- lib/std/compress/flate.zig | 515 +++++--- lib/std/compress/flate/BlockWriter.zig | 696 +++++++++++ lib/std/compress/flate/CircularBuffer.zig | 240 ---- lib/std/compress/flate/Compress.zig | 1264 ++++++++++++++++++++ lib/std/compress/flate/Decompress.zig | 894 ++++++++++++++ lib/std/compress/flate/Lookup.zig | 30 +- lib/std/compress/flate/SlidingWindow.zig | 160 --- lib/std/compress/flate/Token.zig | 14 +- lib/std/compress/flate/bit_reader.zig | 422 ------- lib/std/compress/flate/bit_writer.zig | 99 -- lib/std/compress/flate/block_writer.zig | 706 ----------- lib/std/compress/flate/consts.zig | 49 - lib/std/compress/flate/container.zig | 208 ---- lib/std/compress/flate/deflate.zig | 744 ------------ lib/std/compress/flate/huffman_decoder.zig | 302 ----- lib/std/compress/flate/huffman_encoder.zig | 536 --------- lib/std/compress/flate/inflate.zig | 570 --------- lib/std/compress/gzip.zig | 66 - lib/std/compress/zlib.zig | 101 -- lib/std/debug/Dwarf.zig | 14 +- lib/std/http/Client.zig | 9 +- lib/std/http/Server.zig | 4 +- lib/std/zip.zig | 989 +++++++-------- 24 files changed, 3647 insertions(+), 4990 deletions(-) create mode 100644 lib/std/compress/flate/BlockWriter.zig delete mode 100644 lib/std/compress/flate/CircularBuffer.zig create mode 100644 lib/std/compress/flate/Compress.zig create mode 100644 lib/std/compress/flate/Decompress.zig delete mode 100644 lib/std/compress/flate/SlidingWindow.zig delete mode 100644 lib/std/compress/flate/bit_reader.zig delete mode 100644 lib/std/compress/flate/bit_writer.zig delete mode 100644 lib/std/compress/flate/block_writer.zig delete mode 100644 lib/std/compress/flate/consts.zig delete mode 100644 lib/std/compress/flate/container.zig delete mode 100644 lib/std/compress/flate/deflate.zig delete mode 100644 lib/std/compress/flate/huffman_decoder.zig delete mode 100644 lib/std/compress/flate/huffman_encoder.zig delete mode 100644 lib/std/compress/flate/inflate.zig delete mode 100644 lib/std/compress/gzip.zig delete mode 100644 lib/std/compress/zlib.zig diff --git a/lib/std/compress.zig b/lib/std/compress.zig index 018de51001..199d046af6 100644 --- a/lib/std/compress.zig +++ b/lib/std/compress.zig @@ -1,8 +1,7 @@ //! Compression algorithms. +/// gzip and zlib are here. pub const flate = @import("compress/flate.zig"); -pub const gzip = @import("compress/gzip.zig"); -pub const zlib = @import("compress/zlib.zig"); pub const lzma = @import("compress/lzma.zig"); pub const lzma2 = @import("compress/lzma2.zig"); pub const xz = @import("compress/xz.zig"); @@ -14,6 +13,4 @@ test { _ = lzma2; _ = xz; _ = zstd; - _ = gzip; - _ = zlib; } diff --git a/lib/std/compress/flate.zig b/lib/std/compress/flate.zig index 6a111ac0fc..5a54643f45 100644 --- a/lib/std/compress/flate.zig +++ b/lib/std/compress/flate.zig @@ -1,94 +1,189 @@ +const builtin = @import("builtin"); +const std = @import("../std.zig"); +const testing = std.testing; +const Writer = std.io.Writer; + +/// Container of the deflate bit stream body. Container adds header before +/// deflate bit stream and footer after. It can bi gzip, zlib or raw (no header, +/// no footer, raw bit stream). +/// +/// Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes +/// addler 32 checksum. +/// +/// Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes +/// crc32 checksum and 4 bytes of uncompressed data length. +/// +/// +/// rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4 +/// rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5 +pub const Container = enum { + raw, // no header or footer + gzip, // gzip header and footer + zlib, // zlib header and footer + + pub fn size(w: Container) usize { + return headerSize(w) + footerSize(w); + } + + pub fn headerSize(w: Container) usize { + return header(w).len; + } + + pub fn footerSize(w: Container) usize { + return switch (w) { + .gzip => 8, + .zlib => 4, + .raw => 0, + }; + } + + pub const list = [_]Container{ .raw, .gzip, .zlib }; + + pub const Error = error{ + BadGzipHeader, + BadZlibHeader, + WrongGzipChecksum, + WrongGzipSize, + WrongZlibChecksum, + }; + + pub fn header(container: Container) []const u8 { + return switch (container) { + // GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5): + // - ID1 (IDentification 1), always 0x1f + // - ID2 (IDentification 2), always 0x8b + // - CM (Compression Method), always 8 = deflate + // - FLG (Flags), all set to 0 + // - 4 bytes, MTIME (Modification time), not used, all set to zero + // - XFL (eXtra FLags), all set to zero + // - OS (Operating System), 03 = Unix + .gzip => &[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 }, + // ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4): + // 1st byte: + // - First four bits is the CINFO (compression info), which is 7 for the default deflate window size. + // - The next four bits is the CM (compression method), which is 8 for deflate. + // 2nd byte: + // - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best. + // - The next bit, FDICT, is set if a dictionary is given. + // - The final five FCHECK bits form a mod-31 checksum. + // + // CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100 + .zlib => &[_]u8{ 0x78, 0b10_0_11100 }, + .raw => &.{}, + }; + } + + pub const Hasher = union(Container) { + raw: void, + gzip: struct { + crc: std.hash.Crc32 = .init(), + count: usize = 0, + }, + zlib: std.hash.Adler32, + + pub fn init(containter: Container) Hasher { + return switch (containter) { + .gzip => .{ .gzip = .{} }, + .zlib => .{ .zlib = .init() }, + .raw => .raw, + }; + } + + pub fn container(h: Hasher) Container { + return h; + } + + pub fn update(h: *Hasher, buf: []const u8) void { + switch (h.*) { + .raw => {}, + .gzip => |*gzip| { + gzip.update(buf); + gzip.count += buf.len; + }, + .zlib => |*zlib| { + zlib.update(buf); + }, + inline .gzip, .zlib => |*x| x.update(buf), + } + } + + pub fn writeFooter(hasher: *Hasher, writer: *Writer) Writer.Error!void { + var bits: [4]u8 = undefined; + switch (hasher.*) { + .gzip => |*gzip| { + // GZIP 8 bytes footer + // - 4 bytes, CRC32 (CRC-32) + // - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32 + std.mem.writeInt(u32, &bits, gzip.final(), .little); + try writer.writeAll(&bits); + + std.mem.writeInt(u32, &bits, gzip.bytes_read, .little); + try writer.writeAll(&bits); + }, + .zlib => |*zlib| { + // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). + // 4 bytes of ADLER32 (Adler-32 checksum) + // Checksum value of the uncompressed data (excluding any + // dictionary data) computed according to Adler-32 + // algorithm. + std.mem.writeInt(u32, &bits, zlib.final, .big); + try writer.writeAll(&bits); + }, + .raw => {}, + } + } + }; +}; + +/// When decompressing, the output buffer is used as the history window, so +/// less than this may result in failure to decompress streams that were +/// compressed with a larger window. +pub const max_window_len = 1 << 16; + /// Deflate is a lossless data compression file format that uses a combination /// of LZ77 and Huffman coding. -pub const deflate = @import("flate/deflate.zig"); +pub const Compress = @import("flate/Compress.zig"); /// Inflate is the decoding process that takes a Deflate bitstream for /// decompression and correctly produces the original full-size data or file. -pub const inflate = @import("flate/inflate.zig"); - -/// Decompress compressed data from reader and write plain data to the writer. -pub fn decompress(reader: anytype, writer: anytype) !void { - try inflate.decompress(.raw, reader, writer); -} - -/// Decompressor type -pub fn Decompressor(comptime ReaderType: type) type { - return inflate.Decompressor(.raw, ReaderType); -} - -/// Create Decompressor which will read compressed data from reader. -pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) { - return inflate.decompressor(.raw, reader); -} - -/// Compression level, trades between speed and compression size. -pub const Options = deflate.Options; - -/// Compress plain data from reader and write compressed data to the writer. -pub fn compress(reader: anytype, writer: anytype, options: Options) !void { - try deflate.compress(.raw, reader, writer, options); -} - -/// Compressor type -pub fn Compressor(comptime WriterType: type) type { - return deflate.Compressor(.raw, WriterType); -} - -/// Create Compressor which outputs compressed data to the writer. -pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) { - return try deflate.compressor(.raw, writer, options); -} +pub const Decompress = @import("flate/Decompress.zig"); /// Huffman only compression. Without Lempel-Ziv match searching. Faster /// compression, less memory requirements but bigger compressed sizes. pub const huffman = struct { - pub fn compress(reader: anytype, writer: anytype) !void { - try deflate.huffman.compress(.raw, reader, writer); - } + // The odd order in which the codegen code sizes are written. + pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; + // The number of codegen codes. + pub const codegen_code_count = 19; - pub fn Compressor(comptime WriterType: type) type { - return deflate.huffman.Compressor(.raw, WriterType); - } + // The largest distance code. + pub const distance_code_count = 30; - pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) { - return deflate.huffman.compressor(.raw, writer); - } + // Maximum number of literals. + pub const max_num_lit = 286; + + // Max number of frequencies used for a Huffman Code + // Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286). + // The largest of these is max_num_lit. + pub const max_num_frequencies = max_num_lit; + + // Biggest block size for uncompressed block. + pub const max_store_block_size = 65535; + // The special code used to mark the end of a block. + pub const end_block_marker = 256; }; -// No compression store only. Compressed size is slightly bigger than plain. -pub const store = struct { - pub fn compress(reader: anytype, writer: anytype) !void { - try deflate.store.compress(.raw, reader, writer); - } - - pub fn Compressor(comptime WriterType: type) type { - return deflate.store.Compressor(.raw, WriterType); - } - - pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) { - return deflate.store.compressor(.raw, writer); - } -}; - -/// Container defines header/footer around deflate bit stream. Gzip and zlib -/// compression algorithms are containers around deflate bit stream body. -const Container = @import("flate/container.zig").Container; -const std = @import("std"); -const testing = std.testing; -const fixedBufferStream = std.io.fixedBufferStream; -const print = std.debug.print; -const builtin = @import("builtin"); - test { - _ = deflate; - _ = inflate; + _ = Compress; + _ = Decompress; } test "compress/decompress" { + const print = std.debug.print; var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer - const levels = [_]deflate.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 }; + const levels = [_]Compress.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 }; const cases = [_]struct { data: []const u8, // uncompressed content // compressed data sizes per level 4-9 @@ -135,28 +230,34 @@ test "compress/decompress" { // compress original stream to compressed stream { - var original = fixedBufferStream(data); - var compressed = fixedBufferStream(&cmp_buf); - try deflate.compress(container, original.reader(), compressed.writer(), .{ .level = level }); + var original: std.io.Reader = .fixed(data); + var compressed: Writer = .fixed(&cmp_buf); + var compress: Compress = .init(&original, &.{}, .{ .container = .raw, .level = level }); + const n = try compress.reader.streamRemaining(&compressed); if (compressed_size == 0) { if (container == .gzip) print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos }); - compressed_size = compressed.pos; + compressed_size = compressed.end; } - try testing.expectEqual(compressed_size, compressed.pos); + try testing.expectEqual(compressed_size, n); + try testing.expectEqual(compressed_size, compressed.end); } // decompress compressed stream to decompressed stream { - var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); - var decompressed = fixedBufferStream(&dcm_buf); - try inflate.decompress(container, compressed.reader(), decompressed.writer()); - try testing.expectEqualSlices(u8, data, decompressed.getWritten()); + var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]); + var decompressed: Writer = .fixed(&dcm_buf); + var decompress: Decompress = .init(&compressed, container, &.{}); + _ = try decompress.reader.streamRemaining(&decompressed); + try testing.expectEqualSlices(u8, data, decompressed.buffered()); } // compressor writer interface { - var compressed = fixedBufferStream(&cmp_buf); - var cmp = try deflate.compressor(container, compressed.writer(), .{ .level = level }); + var compressed: Writer = .fixed(&cmp_buf); + var cmp = try Compress.init(&compressed, &.{}, .{ + .level = level, + .container = container, + }); var cmp_wrt = cmp.writer(); try cmp_wrt.writeAll(data); try cmp.finish(); @@ -165,10 +266,9 @@ test "compress/decompress" { } // decompressor reader interface { - var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); - var dcm = inflate.decompressor(container, compressed.reader()); - var dcm_rdr = dcm.reader(); - const n = try dcm_rdr.readAll(&dcm_buf); + var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]); + var decompress: Decompress = .init(&compressed, container, &.{}); + const n = try decompress.reader.readSliceShort(&dcm_buf); try testing.expectEqual(data.len, n); try testing.expectEqualSlices(u8, data, dcm_buf[0..n]); } @@ -184,9 +284,9 @@ test "compress/decompress" { // compress original stream to compressed stream { - var original = fixedBufferStream(data); - var compressed = fixedBufferStream(&cmp_buf); - var cmp = try deflate.huffman.compressor(container, compressed.writer()); + var original: std.io.Reader = .fixed(data); + var compressed: Writer = .fixed(&cmp_buf); + var cmp = try Compress.Huffman.init(container, &compressed); try cmp.compress(original.reader()); try cmp.finish(); if (compressed_size == 0) { @@ -198,10 +298,11 @@ test "compress/decompress" { } // decompress compressed stream to decompressed stream { - var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); - var decompressed = fixedBufferStream(&dcm_buf); - try inflate.decompress(container, compressed.reader(), decompressed.writer()); - try testing.expectEqualSlices(u8, data, decompressed.getWritten()); + var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]); + var decompress: Decompress = .init(&compressed, container, &.{}); + var decompressed: Writer = .fixed(&dcm_buf); + _ = try decompress.reader.streamRemaining(&decompressed); + try testing.expectEqualSlices(u8, data, decompressed.buffered()); } } } @@ -216,9 +317,9 @@ test "compress/decompress" { // compress original stream to compressed stream { - var original = fixedBufferStream(data); - var compressed = fixedBufferStream(&cmp_buf); - var cmp = try deflate.store.compressor(container, compressed.writer()); + var original: std.io.Reader = .fixed(data); + var compressed: Writer = .fixed(&cmp_buf); + var cmp = try Compress.SimpleCompressor(.store, container).init(&compressed); try cmp.compress(original.reader()); try cmp.finish(); if (compressed_size == 0) { @@ -231,23 +332,25 @@ test "compress/decompress" { } // decompress compressed stream to decompressed stream { - var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); - var decompressed = fixedBufferStream(&dcm_buf); - try inflate.decompress(container, compressed.reader(), decompressed.writer()); - try testing.expectEqualSlices(u8, data, decompressed.getWritten()); + var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]); + var decompress: Decompress = .init(&compressed, container, &.{}); + var decompressed: Writer = .fixed(&dcm_buf); + _ = try decompress.reader.streamRemaining(&decompressed); + try testing.expectEqualSlices(u8, data, decompressed.buffered()); } } } } } -fn testDecompress(comptime container: Container, compressed: []const u8, expected_plain: []const u8) !void { - var in = fixedBufferStream(compressed); - var out = std.ArrayList(u8).init(testing.allocator); - defer out.deinit(); +fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void { + var in: std.io.Reader = .fixed(compressed); + var aw: std.io.Writer.Allocating = .init(testing.allocator); + defer aw.deinit(); - try inflate.decompress(container, in.reader(), out.writer()); - try testing.expectEqualSlices(u8, expected_plain, out.items); + var decompress: Decompress = .init(&in, container, &.{}); + _ = try decompress.reader.streamRemaining(&aw.writer); + try testing.expectEqualSlices(u8, expected_plain, aw.items); } test "don't read past deflate stream's end" { @@ -352,126 +455,186 @@ test "gzip header" { } test "public interface" { - const plain_data = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a }; + const plain_data_buf = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a }; // deflate final stored block, header + plain (stored) data const deflate_block = [_]u8{ 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen - } ++ plain_data; + } ++ plain_data_buf; - // gzip header/footer + deflate block - const gzip_data = - [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes) - deflate_block ++ - [_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes) + const plain_data: []const u8 = &plain_data_buf; + const gzip_data: []const u8 = &deflate_block; - // zlib header/footer + deflate block - const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)} - deflate_block ++ - [_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum + //// gzip header/footer + deflate block + //const gzip_data = + // [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes) + // deflate_block ++ + // [_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes) - const gzip = @import("gzip.zig"); - const zlib = @import("zlib.zig"); - const flate = @This(); + //// zlib header/footer + deflate block + //const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)} + // deflate_block ++ + // [_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum - try testInterface(gzip, &gzip_data, &plain_data); - try testInterface(zlib, &zlib_data, &plain_data); - try testInterface(flate, &deflate_block, &plain_data); -} + // TODO + //const gzip = @import("gzip.zig"); + //const zlib = @import("zlib.zig"); -fn testInterface(comptime pkg: type, gzip_data: []const u8, plain_data: []const u8) !void { var buffer1: [64]u8 = undefined; var buffer2: [64]u8 = undefined; - var compressed = fixedBufferStream(&buffer1); - var plain = fixedBufferStream(&buffer2); + // TODO These used to be functions, need to migrate the tests + const decompress = void; + const compress = void; + const store = void; // decompress { - var in = fixedBufferStream(gzip_data); - try pkg.decompress(in.reader(), plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + var plain: Writer = .fixed(&buffer2); + + var in: std.io.Reader = .fixed(gzip_data); + try decompress(&in, &plain); + try testing.expectEqualSlices(u8, plain_data, plain.buffered()); } - plain.reset(); - compressed.reset(); // compress/decompress { - var in = fixedBufferStream(plain_data); - try pkg.compress(in.reader(), compressed.writer(), .{}); - compressed.reset(); - try pkg.decompress(compressed.reader(), plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + var plain: Writer = .fixed(&buffer2); + var compressed: Writer = .fixed(&buffer1); + + var in: std.io.Reader = .fixed(plain_data); + try compress(&in, &compressed, .{}); + + var r: std.io.Reader = .fixed(&buffer1); + try decompress(&r, &plain); + try testing.expectEqualSlices(u8, plain_data, plain.buffered()); } - plain.reset(); - compressed.reset(); // compressor/decompressor { - var in = fixedBufferStream(plain_data); - var cmp = try pkg.compressor(compressed.writer(), .{}); - try cmp.compress(in.reader()); + var plain: Writer = .fixed(&buffer2); + var compressed: Writer = .fixed(&buffer1); + + var in: std.io.Reader = .fixed(plain_data); + var cmp = try Compress(&compressed, .{}); + try cmp.compress(&in); try cmp.finish(); - compressed.reset(); - var dcp = pkg.decompressor(compressed.reader()); - try dcp.decompress(plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + var r: std.io.Reader = .fixed(&buffer1); + var dcp = Decompress(&r); + try dcp.decompress(&plain); + try testing.expectEqualSlices(u8, plain_data, plain.buffered()); } - plain.reset(); - compressed.reset(); // huffman { // huffman compress/decompress { - var in = fixedBufferStream(plain_data); - try pkg.huffman.compress(in.reader(), compressed.writer()); - compressed.reset(); - try pkg.decompress(compressed.reader(), plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + var plain: Writer = .fixed(&buffer2); + var compressed: Writer = .fixed(&buffer1); + + var in: std.io.Reader = .fixed(plain_data); + try huffman.compress(&in, &compressed); + + var r: std.io.Reader = .fixed(&buffer1); + try decompress(&r, &plain); + try testing.expectEqualSlices(u8, plain_data, plain.buffered()); } - plain.reset(); - compressed.reset(); // huffman compressor/decompressor { - var in = fixedBufferStream(plain_data); - var cmp = try pkg.huffman.compressor(compressed.writer()); - try cmp.compress(in.reader()); + var plain: Writer = .fixed(&buffer2); + var compressed: Writer = .fixed(&buffer1); + + var in: std.io.Reader = .fixed(plain_data); + var cmp = try huffman.Compressor(&compressed); + try cmp.compress(&in); try cmp.finish(); - compressed.reset(); - try pkg.decompress(compressed.reader(), plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + var r: std.io.Reader = .fixed(&buffer1); + try decompress(&r, &plain); + try testing.expectEqualSlices(u8, plain_data, plain.buffered()); } } - plain.reset(); - compressed.reset(); // store { // store compress/decompress { - var in = fixedBufferStream(plain_data); - try pkg.store.compress(in.reader(), compressed.writer()); - compressed.reset(); - try pkg.decompress(compressed.reader(), plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + var plain: Writer = .fixed(&buffer2); + var compressed: Writer = .fixed(&buffer1); + + var in: std.io.Reader = .fixed(plain_data); + try store.compress(&in, &compressed); + + var r: std.io.Reader = .fixed(&buffer1); + try decompress(&r, &plain); + try testing.expectEqualSlices(u8, plain_data, plain.buffered()); } - plain.reset(); - compressed.reset(); // store compressor/decompressor { - var in = fixedBufferStream(plain_data); - var cmp = try pkg.store.compressor(compressed.writer()); - try cmp.compress(in.reader()); + var plain: Writer = .fixed(&buffer2); + var compressed: Writer = .fixed(&buffer1); + + var in: std.io.Reader = .fixed(plain_data); + var cmp = try store.compressor(&compressed); + try cmp.compress(&in); try cmp.finish(); - compressed.reset(); - try pkg.decompress(compressed.reader(), plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + var r: std.io.Reader = .fixed(&buffer1); + try decompress(&r, &plain); + try testing.expectEqualSlices(u8, plain_data, plain.buffered()); } } } + +pub const match = struct { + pub const base_length = 3; // smallest match length per the RFC section 3.2.5 + pub const min_length = 4; // min length used in this algorithm + pub const max_length = 258; + + pub const min_distance = 1; + pub const max_distance = 32768; +}; + +pub const history_len = match.max_distance; + +pub const lookup = struct { + pub const bits = 15; + pub const len = 1 << bits; + pub const shift = 32 - bits; +}; + +test "zlib should not overshoot" { + // Compressed zlib data with extra 4 bytes at the end. + const data = [_]u8{ + 0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9, + 0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08, + 0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34, + 0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12, + }; + + var stream: std.io.Reader = .fixed(&data); + const reader = stream.reader(); + + var dcp = Decompress.init(reader); + var out: [128]u8 = undefined; + + // Decompress + var n = try dcp.reader().readAll(out[0..]); + + // Expected decompressed data + try std.testing.expectEqual(46, n); + try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]); + + // Decompressor don't overshoot underlying reader. + // It is leaving it at the end of compressed data chunk. + try std.testing.expectEqual(data.len - 4, stream.getPos()); + try std.testing.expectEqual(0, dcp.unreadBytes()); + + // 4 bytes after compressed chunk are available in reader. + n = try reader.readAll(out[0..]); + try std.testing.expectEqual(n, 4); + try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]); +} diff --git a/lib/std/compress/flate/BlockWriter.zig b/lib/std/compress/flate/BlockWriter.zig new file mode 100644 index 0000000000..d1eb3a068e --- /dev/null +++ b/lib/std/compress/flate/BlockWriter.zig @@ -0,0 +1,696 @@ +//! Accepts list of tokens, decides what is best block type to write. What block +//! type will provide best compression. Writes header and body of the block. +const std = @import("std"); +const io = std.io; +const assert = std.debug.assert; +const Writer = std.io.Writer; + +const BlockWriter = @This(); +const flate = @import("../flate.zig"); +const Compress = flate.Compress; +const huffman = flate.huffman; +const Token = @import("Token.zig"); + +const codegen_order = huffman.codegen_order; +const end_code_mark = 255; + +output: *Writer, + +codegen_freq: [huffman.codegen_code_count]u16 = undefined, +literal_freq: [huffman.max_num_lit]u16 = undefined, +distance_freq: [huffman.distance_code_count]u16 = undefined, +codegen: [huffman.max_num_lit + huffman.distance_code_count + 1]u8 = undefined, +literal_encoding: Compress.LiteralEncoder = .{}, +distance_encoding: Compress.DistanceEncoder = .{}, +codegen_encoding: Compress.CodegenEncoder = .{}, +fixed_literal_encoding: Compress.LiteralEncoder, +fixed_distance_encoding: Compress.DistanceEncoder, +huff_distance: Compress.DistanceEncoder, + +pub fn init(output: *Writer) BlockWriter { + return .{ + .output = output, + .fixed_literal_encoding = Compress.fixedLiteralEncoder(), + .fixed_distance_encoding = Compress.fixedDistanceEncoder(), + .huff_distance = Compress.huffmanDistanceEncoder(), + }; +} + +/// Flush intrenal bit buffer to the writer. +/// Should be called only when bit stream is at byte boundary. +/// +/// That is after final block; when last byte could be incomplete or +/// after stored block; which is aligned to the byte boundary (it has x +/// padding bits after first 3 bits). +pub fn flush(self: *BlockWriter) Writer.Error!void { + try self.bit_writer.flush(); +} + +pub fn setWriter(self: *BlockWriter, new_writer: *Writer) void { + self.bit_writer.setWriter(new_writer); +} + +fn writeCode(self: *BlockWriter, c: Compress.HuffCode) Writer.Error!void { + try self.bit_writer.writeBits(c.code, c.len); +} + +// RFC 1951 3.2.7 specifies a special run-length encoding for specifying +// the literal and distance lengths arrays (which are concatenated into a single +// array). This method generates that run-length encoding. +// +// The result is written into the codegen array, and the frequencies +// of each code is written into the codegen_freq array. +// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional +// information. Code bad_code is an end marker +// +// num_literals: The number of literals in literal_encoding +// num_distances: The number of distances in distance_encoding +// lit_enc: The literal encoder to use +// dist_enc: The distance encoder to use +fn generateCodegen( + self: *BlockWriter, + num_literals: u32, + num_distances: u32, + lit_enc: *Compress.LiteralEncoder, + dist_enc: *Compress.DistanceEncoder, +) void { + for (self.codegen_freq, 0..) |_, i| { + self.codegen_freq[i] = 0; + } + + // Note that we are using codegen both as a temporary variable for holding + // a copy of the frequencies, and as the place where we put the result. + // This is fine because the output is always shorter than the input used + // so far. + var codegen = &self.codegen; // cache + // Copy the concatenated code sizes to codegen. Put a marker at the end. + var cgnl = codegen[0..num_literals]; + for (cgnl, 0..) |_, i| { + cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len)); + } + + cgnl = codegen[num_literals .. num_literals + num_distances]; + for (cgnl, 0..) |_, i| { + cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len)); + } + codegen[num_literals + num_distances] = end_code_mark; + + var size = codegen[0]; + var count: i32 = 1; + var out_index: u32 = 0; + var in_index: u32 = 1; + while (size != end_code_mark) : (in_index += 1) { + // INVARIANT: We have seen "count" copies of size that have not yet + // had output generated for them. + const next_size = codegen[in_index]; + if (next_size == size) { + count += 1; + continue; + } + // We need to generate codegen indicating "count" of size. + if (size != 0) { + codegen[out_index] = size; + out_index += 1; + self.codegen_freq[size] += 1; + count -= 1; + while (count >= 3) { + var n: i32 = 6; + if (n > count) { + n = count; + } + codegen[out_index] = 16; + out_index += 1; + codegen[out_index] = @as(u8, @intCast(n - 3)); + out_index += 1; + self.codegen_freq[16] += 1; + count -= n; + } + } else { + while (count >= 11) { + var n: i32 = 138; + if (n > count) { + n = count; + } + codegen[out_index] = 18; + out_index += 1; + codegen[out_index] = @as(u8, @intCast(n - 11)); + out_index += 1; + self.codegen_freq[18] += 1; + count -= n; + } + if (count >= 3) { + // 3 <= count <= 10 + codegen[out_index] = 17; + out_index += 1; + codegen[out_index] = @as(u8, @intCast(count - 3)); + out_index += 1; + self.codegen_freq[17] += 1; + count = 0; + } + } + count -= 1; + while (count >= 0) : (count -= 1) { + codegen[out_index] = size; + out_index += 1; + self.codegen_freq[size] += 1; + } + // Set up invariant for next time through the loop. + size = next_size; + count = 1; + } + // Marker indicating the end of the codegen. + codegen[out_index] = end_code_mark; +} + +const DynamicSize = struct { + size: u32, + num_codegens: u32, +}; + +// dynamicSize returns the size of dynamically encoded data in bits. +fn dynamicSize( + self: *BlockWriter, + lit_enc: *Compress.LiteralEncoder, // literal encoder + dist_enc: *Compress.DistanceEncoder, // distance encoder + extra_bits: u32, +) DynamicSize { + var num_codegens = self.codegen_freq.len; + while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) { + num_codegens -= 1; + } + const header = 3 + 5 + 5 + 4 + (3 * num_codegens) + + self.codegen_encoding.bitLength(self.codegen_freq[0..]) + + self.codegen_freq[16] * 2 + + self.codegen_freq[17] * 3 + + self.codegen_freq[18] * 7; + const size = header + + lit_enc.bitLength(&self.literal_freq) + + dist_enc.bitLength(&self.distance_freq) + + extra_bits; + + return DynamicSize{ + .size = @as(u32, @intCast(size)), + .num_codegens = @as(u32, @intCast(num_codegens)), + }; +} + +// fixedSize returns the size of dynamically encoded data in bits. +fn fixedSize(self: *BlockWriter, extra_bits: u32) u32 { + return 3 + + self.fixed_literal_encoding.bitLength(&self.literal_freq) + + self.fixed_distance_encoding.bitLength(&self.distance_freq) + + extra_bits; +} + +const StoredSize = struct { + size: u32, + storable: bool, +}; + +// storedSizeFits calculates the stored size, including header. +// The function returns the size in bits and whether the block +// fits inside a single block. +fn storedSizeFits(in: ?[]const u8) StoredSize { + if (in == null) { + return .{ .size = 0, .storable = false }; + } + if (in.?.len <= huffman.max_store_block_size) { + return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true }; + } + return .{ .size = 0, .storable = false }; +} + +// Write the header of a dynamic Huffman block to the output stream. +// +// num_literals: The number of literals specified in codegen +// num_distances: The number of distances specified in codegen +// num_codegens: The number of codegens used in codegen +// eof: Is it the end-of-file? (end of stream) +fn dynamicHeader( + self: *BlockWriter, + num_literals: u32, + num_distances: u32, + num_codegens: u32, + eof: bool, +) Writer.Error!void { + const first_bits: u32 = if (eof) 5 else 4; + try self.bit_writer.writeBits(first_bits, 3); + try self.bit_writer.writeBits(num_literals - 257, 5); + try self.bit_writer.writeBits(num_distances - 1, 5); + try self.bit_writer.writeBits(num_codegens - 4, 4); + + var i: u32 = 0; + while (i < num_codegens) : (i += 1) { + const value = self.codegen_encoding.codes[codegen_order[i]].len; + try self.bit_writer.writeBits(value, 3); + } + + i = 0; + while (true) { + const code_word: u32 = @as(u32, @intCast(self.codegen[i])); + i += 1; + if (code_word == end_code_mark) { + break; + } + try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]); + + switch (code_word) { + 16 => { + try self.bit_writer.writeBits(self.codegen[i], 2); + i += 1; + }, + 17 => { + try self.bit_writer.writeBits(self.codegen[i], 3); + i += 1; + }, + 18 => { + try self.bit_writer.writeBits(self.codegen[i], 7); + i += 1; + }, + else => {}, + } + } +} + +fn storedHeader(self: *BlockWriter, length: usize, eof: bool) Writer.Error!void { + assert(length <= 65535); + const flag: u32 = if (eof) 1 else 0; + try self.bit_writer.writeBits(flag, 3); + try self.flush(); + const l: u16 = @intCast(length); + try self.bit_writer.writeBits(l, 16); + try self.bit_writer.writeBits(~l, 16); +} + +fn fixedHeader(self: *BlockWriter, eof: bool) Writer.Error!void { + // Indicate that we are a fixed Huffman block + var value: u32 = 2; + if (eof) { + value = 3; + } + try self.bit_writer.writeBits(value, 3); +} + +// Write a block of tokens with the smallest encoding. Will choose block type. +// The original input can be supplied, and if the huffman encoded data +// is larger than the original bytes, the data will be written as a +// stored block. +// If the input is null, the tokens will always be Huffman encoded. +pub fn write(self: *BlockWriter, tokens: []const Token, eof: bool, input: ?[]const u8) Writer.Error!void { + const lit_and_dist = self.indexTokens(tokens); + const num_literals = lit_and_dist.num_literals; + const num_distances = lit_and_dist.num_distances; + + var extra_bits: u32 = 0; + const ret = storedSizeFits(input); + const stored_size = ret.size; + const storable = ret.storable; + + if (storable) { + // We only bother calculating the costs of the extra bits required by + // the length of distance fields (which will be the same for both fixed + // and dynamic encoding), if we need to compare those two encodings + // against stored encoding. + var length_code: u16 = Token.length_codes_start + 8; + while (length_code < num_literals) : (length_code += 1) { + // First eight length codes have extra size = 0. + extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) * + @as(u32, @intCast(Token.lengthExtraBits(length_code))); + } + var distance_code: u16 = 4; + while (distance_code < num_distances) : (distance_code += 1) { + // First four distance codes have extra size = 0. + extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) * + @as(u32, @intCast(Token.distanceExtraBits(distance_code))); + } + } + + // Figure out smallest code. + // Fixed Huffman baseline. + var literal_encoding = &self.fixed_literal_encoding; + var distance_encoding = &self.fixed_distance_encoding; + var size = self.fixedSize(extra_bits); + + // Dynamic Huffman? + var num_codegens: u32 = 0; + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literal_encoding and the distance_encoding. + self.generateCodegen( + num_literals, + num_distances, + &self.literal_encoding, + &self.distance_encoding, + ); + self.codegen_encoding.generate(self.codegen_freq[0..], 7); + const dynamic_size = self.dynamicSize( + &self.literal_encoding, + &self.distance_encoding, + extra_bits, + ); + const dyn_size = dynamic_size.size; + num_codegens = dynamic_size.num_codegens; + + if (dyn_size < size) { + size = dyn_size; + literal_encoding = &self.literal_encoding; + distance_encoding = &self.distance_encoding; + } + + // Stored bytes? + if (storable and stored_size < size) { + try self.storedBlock(input.?, eof); + return; + } + + // Huffman. + if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) { + try self.fixedHeader(eof); + } else { + try self.dynamicHeader(num_literals, num_distances, num_codegens, eof); + } + + // Write the tokens. + try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes); +} + +pub fn storedBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Error!void { + try self.storedHeader(input.len, eof); + try self.bit_writer.writeBytes(input); +} + +// writeBlockDynamic encodes a block using a dynamic Huffman table. +// This should be used if the symbols used have a disproportionate +// histogram distribution. +// If input is supplied and the compression savings are below 1/16th of the +// input size the block is stored. +fn dynamicBlock( + self: *BlockWriter, + tokens: []const Token, + eof: bool, + input: ?[]const u8, +) Writer.Error!void { + const total_tokens = self.indexTokens(tokens); + const num_literals = total_tokens.num_literals; + const num_distances = total_tokens.num_distances; + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literal_encoding and the distance_encoding. + self.generateCodegen( + num_literals, + num_distances, + &self.literal_encoding, + &self.distance_encoding, + ); + self.codegen_encoding.generate(self.codegen_freq[0..], 7); + const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0); + const size = dynamic_size.size; + const num_codegens = dynamic_size.num_codegens; + + // Store bytes, if we don't get a reasonable improvement. + + const stored_size = storedSizeFits(input); + const ssize = stored_size.size; + const storable = stored_size.storable; + if (storable and ssize < (size + (size >> 4))) { + try self.storedBlock(input.?, eof); + return; + } + + // Write Huffman table. + try self.dynamicHeader(num_literals, num_distances, num_codegens, eof); + + // Write the tokens. + try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes); +} + +const TotalIndexedTokens = struct { + num_literals: u32, + num_distances: u32, +}; + +// Indexes a slice of tokens followed by an end_block_marker, and updates +// literal_freq and distance_freq, and generates literal_encoding +// and distance_encoding. +// The number of literal and distance tokens is returned. +fn indexTokens(self: *BlockWriter, tokens: []const Token) TotalIndexedTokens { + var num_literals: u32 = 0; + var num_distances: u32 = 0; + + for (self.literal_freq, 0..) |_, i| { + self.literal_freq[i] = 0; + } + for (self.distance_freq, 0..) |_, i| { + self.distance_freq[i] = 0; + } + + for (tokens) |t| { + if (t.kind == Token.Kind.literal) { + self.literal_freq[t.literal()] += 1; + continue; + } + self.literal_freq[t.lengthCode()] += 1; + self.distance_freq[t.distanceCode()] += 1; + } + // add end_block_marker token at the end + self.literal_freq[huffman.end_block_marker] += 1; + + // get the number of literals + num_literals = @as(u32, @intCast(self.literal_freq.len)); + while (self.literal_freq[num_literals - 1] == 0) { + num_literals -= 1; + } + // get the number of distances + num_distances = @as(u32, @intCast(self.distance_freq.len)); + while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) { + num_distances -= 1; + } + if (num_distances == 0) { + // We haven't found a single match. If we want to go with the dynamic encoding, + // we should count at least one distance to be sure that the distance huffman tree could be encoded. + self.distance_freq[0] = 1; + num_distances = 1; + } + self.literal_encoding.generate(&self.literal_freq, 15); + self.distance_encoding.generate(&self.distance_freq, 15); + return TotalIndexedTokens{ + .num_literals = num_literals, + .num_distances = num_distances, + }; +} + +// Writes a slice of tokens to the output followed by and end_block_marker. +// codes for literal and distance encoding must be supplied. +fn writeTokens( + self: *BlockWriter, + tokens: []const Token, + le_codes: []Compress.HuffCode, + oe_codes: []Compress.HuffCode, +) Writer.Error!void { + for (tokens) |t| { + if (t.kind == Token.Kind.literal) { + try self.writeCode(le_codes[t.literal()]); + continue; + } + + // Write the length + const le = t.lengthEncoding(); + try self.writeCode(le_codes[le.code]); + if (le.extra_bits > 0) { + try self.bit_writer.writeBits(le.extra_length, le.extra_bits); + } + + // Write the distance + const oe = t.distanceEncoding(); + try self.writeCode(oe_codes[oe.code]); + if (oe.extra_bits > 0) { + try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits); + } + } + // add end_block_marker at the end + try self.writeCode(le_codes[huffman.end_block_marker]); +} + +// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes +// if the results only gains very little from compression. +pub fn huffmanBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Error!void { + // Add everything as literals + histogram(input, &self.literal_freq); + + self.literal_freq[huffman.end_block_marker] = 1; + + const num_literals = huffman.end_block_marker + 1; + self.distance_freq[0] = 1; + const num_distances = 1; + + self.literal_encoding.generate(&self.literal_freq, 15); + + // Figure out smallest code. + // Always use dynamic Huffman or Store + var num_codegens: u32 = 0; + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literal_encoding and the distance_encoding. + self.generateCodegen( + num_literals, + num_distances, + &self.literal_encoding, + &self.huff_distance, + ); + self.codegen_encoding.generate(self.codegen_freq[0..], 7); + const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0); + const size = dynamic_size.size; + num_codegens = dynamic_size.num_codegens; + + // Store bytes, if we don't get a reasonable improvement. + const stored_size_ret = storedSizeFits(input); + const ssize = stored_size_ret.size; + const storable = stored_size_ret.storable; + + if (storable and ssize < (size + (size >> 4))) { + try self.storedBlock(input, eof); + return; + } + + // Huffman. + try self.dynamicHeader(num_literals, num_distances, num_codegens, eof); + const encoding = self.literal_encoding.codes[0..257]; + + for (input) |t| { + const c = encoding[t]; + try self.bit_writer.writeBits(c.code, c.len); + } + try self.writeCode(encoding[huffman.end_block_marker]); +} + +// histogram accumulates a histogram of b in h. +fn histogram(b: []const u8, h: *[286]u16) void { + // Clear histogram + for (h, 0..) |_, i| { + h[i] = 0; + } + + var lh = h.*[0..256]; + for (b) |t| { + lh[t] += 1; + } +} + +// tests +const expect = std.testing.expect; +const fmt = std.fmt; +const testing = std.testing; +const ArrayList = std.ArrayList; + +const TestCase = @import("testdata/block_writer.zig").TestCase; +const testCases = @import("testdata/block_writer.zig").testCases; + +// tests if the writeBlock encoding has changed. +test "write" { + inline for (0..testCases.len) |i| { + try testBlock(testCases[i], .write_block); + } +} + +// tests if the writeBlockDynamic encoding has changed. +test "dynamicBlock" { + inline for (0..testCases.len) |i| { + try testBlock(testCases[i], .write_dyn_block); + } +} + +test "huffmanBlock" { + inline for (0..testCases.len) |i| { + try testBlock(testCases[i], .write_huffman_block); + } + try testBlock(.{ + .tokens = &[_]Token{}, + .input = "huffman-rand-max.input", + .want = "huffman-rand-max.{s}.expect", + }, .write_huffman_block); +} + +const TestFn = enum { + write_block, + write_dyn_block, // write dynamic block + write_huffman_block, + + fn to_s(self: TestFn) []const u8 { + return switch (self) { + .write_block => "wb", + .write_dyn_block => "dyn", + .write_huffman_block => "huff", + }; + } + + fn write( + comptime self: TestFn, + bw: anytype, + tok: []const Token, + input: ?[]const u8, + final: bool, + ) !void { + switch (self) { + .write_block => try bw.write(tok, final, input), + .write_dyn_block => try bw.dynamicBlock(tok, final, input), + .write_huffman_block => try bw.huffmanBlock(input.?, final), + } + try bw.flush(); + } +}; + +// testBlock tests a block against its references +// +// size +// 64K [file-name].input - input non compressed file +// 8.1K [file-name].golden - +// 78 [file-name].dyn.expect - output with writeBlockDynamic +// 78 [file-name].wb.expect - output with writeBlock +// 8.1K [file-name].huff.expect - output with writeBlockHuff +// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null +// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null +// +// wb - writeBlock +// dyn - writeBlockDynamic +// huff - writeBlockHuff +// +fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void { + if (tc.input.len != 0 and tc.want.len != 0) { + const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()}); + const input = @embedFile("testdata/block_writer/" ++ tc.input); + const want = @embedFile("testdata/block_writer/" ++ want_name); + try testWriteBlock(tfn, input, want, tc.tokens); + } + + if (tfn == .write_huffman_block) { + return; + } + + const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()}); + const want = @embedFile("testdata/block_writer/" ++ want_name_no_input); + try testWriteBlock(tfn, null, want, tc.tokens); +} + +// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output. +fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void { + var buf = ArrayList(u8).init(testing.allocator); + var bw: BlockWriter = .init(buf.writer()); + try tfn.write(&bw, tokens, input, false); + var got = buf.items; + try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result + try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set + // + // Test if the writer produces the same output after reset. + buf.deinit(); + buf = ArrayList(u8).init(testing.allocator); + defer buf.deinit(); + bw.setWriter(buf.writer()); + + try tfn.write(&bw, tokens, input, true); + try bw.flush(); + got = buf.items; + + try expect(got[0] & 1 == 1); // bfinal is set + buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices + try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result +} diff --git a/lib/std/compress/flate/CircularBuffer.zig b/lib/std/compress/flate/CircularBuffer.zig deleted file mode 100644 index 552d364894..0000000000 --- a/lib/std/compress/flate/CircularBuffer.zig +++ /dev/null @@ -1,240 +0,0 @@ -//! 64K buffer of uncompressed data created in inflate (decompression). Has enough -//! history to support writing match; copying length of bytes -//! from the position distance backward from current. -//! -//! Reads can return less than available bytes if they are spread across -//! different circles. So reads should repeat until get required number of bytes -//! or until returned slice is zero length. -//! -//! Note on deflate limits: -//! * non-compressible block is limited to 65,535 bytes. -//! * backward pointer is limited in distance to 32K bytes and in length to 258 bytes. -//! -//! Whole non-compressed block can be written without overlap. We always have -//! history of up to 64K, more then 32K needed. -//! -const std = @import("std"); -const assert = std.debug.assert; -const testing = std.testing; - -const consts = @import("consts.zig").match; - -const mask = 0xffff; // 64K - 1 -const buffer_len = mask + 1; // 64K buffer - -const Self = @This(); - -buffer: [buffer_len]u8 = undefined, -wp: usize = 0, // write position -rp: usize = 0, // read position - -fn writeAll(self: *Self, buf: []const u8) void { - for (buf) |c| self.write(c); -} - -/// Write literal. -pub fn write(self: *Self, b: u8) void { - assert(self.wp - self.rp < mask); - self.buffer[self.wp & mask] = b; - self.wp += 1; -} - -/// Write match (back-reference to the same data slice) starting at `distance` -/// back from current write position, and `length` of bytes. -pub fn writeMatch(self: *Self, length: u16, distance: u16) !void { - if (self.wp < distance or - length < consts.base_length or length > consts.max_length or - distance < consts.min_distance or distance > consts.max_distance) - { - return error.InvalidMatch; - } - assert(self.wp - self.rp < mask); - - var from: usize = self.wp - distance & mask; - const from_end: usize = from + length; - var to: usize = self.wp & mask; - const to_end: usize = to + length; - - self.wp += length; - - // Fast path using memcpy - if (from_end < buffer_len and to_end < buffer_len) // start and end at the same circle - { - var cur_len = distance; - var remaining_len = length; - while (cur_len < remaining_len) { - @memcpy(self.buffer[to..][0..cur_len], self.buffer[from..][0..cur_len]); - to += cur_len; - remaining_len -= cur_len; - cur_len = cur_len * 2; - } - @memcpy(self.buffer[to..][0..remaining_len], self.buffer[from..][0..remaining_len]); - return; - } - - // Slow byte by byte - while (to < to_end) { - self.buffer[to & mask] = self.buffer[from & mask]; - to += 1; - from += 1; - } -} - -/// Returns writable part of the internal buffer of size `n` at most. Advances -/// write pointer, assumes that returned buffer will be filled with data. -pub fn getWritable(self: *Self, n: usize) []u8 { - const wp = self.wp & mask; - const len = @min(n, buffer_len - wp); - self.wp += len; - return self.buffer[wp .. wp + len]; -} - -/// Read available data. Can return part of the available data if it is -/// spread across two circles. So read until this returns zero length. -pub fn read(self: *Self) []const u8 { - return self.readAtMost(buffer_len); -} - -/// Read part of available data. Can return less than max even if there are -/// more than max decoded data. -pub fn readAtMost(self: *Self, limit: usize) []const u8 { - const rb = self.readBlock(if (limit == 0) buffer_len else limit); - defer self.rp += rb.len; - return self.buffer[rb.head..rb.tail]; -} - -const ReadBlock = struct { - head: usize, - tail: usize, - len: usize, -}; - -/// Returns position of continuous read block data. -fn readBlock(self: *Self, max: usize) ReadBlock { - const r = self.rp & mask; - const w = self.wp & mask; - const n = @min( - max, - if (w >= r) w - r else buffer_len - r, - ); - return .{ - .head = r, - .tail = r + n, - .len = n, - }; -} - -/// Number of free bytes for write. -pub fn free(self: *Self) usize { - return buffer_len - (self.wp - self.rp); -} - -/// Full if largest match can't fit. 258 is largest match length. That much -/// bytes can be produced in single decode step. -pub fn full(self: *Self) bool { - return self.free() < 258 + 1; -} - -// example from: https://youtu.be/SJPvNi4HrWQ?t=3558 -test writeMatch { - var cb: Self = .{}; - - cb.writeAll("a salad; "); - try cb.writeMatch(5, 9); - try cb.writeMatch(3, 3); - - try testing.expectEqualStrings("a salad; a salsal", cb.read()); -} - -test "writeMatch overlap" { - var cb: Self = .{}; - - cb.writeAll("a b c "); - try cb.writeMatch(8, 4); - cb.write('d'); - - try testing.expectEqualStrings("a b c b c b c d", cb.read()); -} - -test readAtMost { - var cb: Self = .{}; - - cb.writeAll("0123456789"); - try cb.writeMatch(50, 10); - - try testing.expectEqualStrings("0123456789" ** 6, cb.buffer[cb.rp..cb.wp]); - for (0..6) |i| { - try testing.expectEqual(i * 10, cb.rp); - try testing.expectEqualStrings("0123456789", cb.readAtMost(10)); - } - try testing.expectEqualStrings("", cb.readAtMost(10)); - try testing.expectEqualStrings("", cb.read()); -} - -test Self { - var cb: Self = .{}; - - const data = "0123456789abcdef" ** (1024 / 16); - cb.writeAll(data); - try testing.expectEqual(@as(usize, 0), cb.rp); - try testing.expectEqual(@as(usize, 1024), cb.wp); - try testing.expectEqual(@as(usize, 1024 * 63), cb.free()); - - for (0..62 * 4) |_| - try cb.writeMatch(256, 1024); // write 62K - - try testing.expectEqual(@as(usize, 0), cb.rp); - try testing.expectEqual(@as(usize, 63 * 1024), cb.wp); - try testing.expectEqual(@as(usize, 1024), cb.free()); - - cb.writeAll(data[0..200]); - _ = cb.readAtMost(1024); // make some space - cb.writeAll(data); // overflows write position - try testing.expectEqual(@as(usize, 200 + 65536), cb.wp); - try testing.expectEqual(@as(usize, 1024), cb.rp); - try testing.expectEqual(@as(usize, 1024 - 200), cb.free()); - - const rb = cb.readBlock(Self.buffer_len); - try testing.expectEqual(@as(usize, 65536 - 1024), rb.len); - try testing.expectEqual(@as(usize, 1024), rb.head); - try testing.expectEqual(@as(usize, 65536), rb.tail); - - try testing.expectEqual(@as(usize, 65536 - 1024), cb.read().len); // read to the end of the buffer - try testing.expectEqual(@as(usize, 200 + 65536), cb.wp); - try testing.expectEqual(@as(usize, 65536), cb.rp); - try testing.expectEqual(@as(usize, 65536 - 200), cb.free()); - - try testing.expectEqual(@as(usize, 200), cb.read().len); // read the rest -} - -test "write overlap" { - var cb: Self = .{}; - cb.wp = cb.buffer.len - 15; - cb.rp = cb.wp; - - cb.writeAll("0123456789"); - cb.writeAll("abcdefghij"); - - try testing.expectEqual(cb.buffer.len + 5, cb.wp); - try testing.expectEqual(cb.buffer.len - 15, cb.rp); - - try testing.expectEqualStrings("0123456789abcde", cb.read()); - try testing.expectEqualStrings("fghij", cb.read()); - - try testing.expect(cb.wp == cb.rp); -} - -test "writeMatch/read overlap" { - var cb: Self = .{}; - cb.wp = cb.buffer.len - 15; - cb.rp = cb.wp; - - cb.writeAll("0123456789"); - try cb.writeMatch(15, 5); - - try testing.expectEqualStrings("012345678956789", cb.read()); - try testing.expectEqualStrings("5678956789", cb.read()); - - try cb.writeMatch(20, 25); - try testing.expectEqualStrings("01234567895678956789", cb.read()); -} diff --git a/lib/std/compress/flate/Compress.zig b/lib/std/compress/flate/Compress.zig new file mode 100644 index 0000000000..4d827fd590 --- /dev/null +++ b/lib/std/compress/flate/Compress.zig @@ -0,0 +1,1264 @@ +//! Default compression algorithm. Has two steps: tokenization and token +//! encoding. +//! +//! Tokenization takes uncompressed input stream and produces list of tokens. +//! Each token can be literal (byte of data) or match (backrefernce to previous +//! data with length and distance). Tokenization accumulators 32K tokens, when +//! full or `flush` is called tokens are passed to the `block_writer`. Level +//! defines how hard (how slow) it tries to find match. +//! +//! Block writer will decide which type of deflate block to write (stored, fixed, +//! dynamic) and encode tokens to the output byte stream. Client has to call +//! `finish` to write block with the final bit set. +//! +//! Container defines type of header and footer which can be gzip, zlib or raw. +//! They all share same deflate body. Raw has no header or footer just deflate +//! body. +//! +//! Compression algorithm explained in rfc-1951 (slightly edited for this case): +//! +//! The compressor uses a chained hash table `lookup` to find duplicated +//! strings, using a hash function that operates on 4-byte sequences. At any +//! given point during compression, let XYZW be the next 4 input bytes +//! (lookahead) to be examined (not necessarily all different, of course). +//! First, the compressor examines the hash chain for XYZW. If the chain is +//! empty, the compressor simply writes out X as a literal byte and advances +//! one byte in the input. If the hash chain is not empty, indicating that the +//! sequence XYZW (or, if we are unlucky, some other 4 bytes with the same +//! hash function value) has occurred recently, the compressor compares all +//! strings on the XYZW hash chain with the actual input data sequence +//! starting at the current point, and selects the longest match. +//! +//! To improve overall compression, the compressor defers the selection of +//! matches ("lazy matching"): after a match of length N has been found, the +//! compressor searches for a longer match starting at the next input byte. If +//! it finds a longer match, it truncates the previous match to a length of +//! one (thus producing a single literal byte) and then emits the longer +//! match. Otherwise, it emits the original match, and, as described above, +//! advances N bytes before continuing. +//! +//! +//! Allocates statically ~400K (192K lookup, 128K tokens, 64K window). +const builtin = @import("builtin"); +const std = @import("std"); +const assert = std.debug.assert; +const testing = std.testing; +const expect = testing.expect; +const mem = std.mem; +const math = std.math; +const Writer = std.Io.Writer; +const Reader = std.Io.Reader; + +const Compress = @This(); +const Token = @import("Token.zig"); +const BlockWriter = @import("BlockWriter.zig"); +const flate = @import("../flate.zig"); +const Container = flate.Container; +const Lookup = @import("Lookup.zig"); +const huffman = flate.huffman; + +lookup: Lookup = .{}, +tokens: Tokens = .{}, +/// Asserted to have a buffer capacity of at least `flate.max_window_len`. +input: *Reader, +block_writer: BlockWriter, +level: LevelArgs, +hasher: Container.Hasher, +reader: Reader, + +// Match and literal at the previous position. +// Used for lazy match finding in processWindow. +prev_match: ?Token = null, +prev_literal: ?u8 = null, + +/// Trades between speed and compression size. +/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43) +/// levels 1-3 are using different algorithm to perform faster but with less +/// compression. That is not implemented here. +pub const Level = enum(u4) { + level_4 = 4, + level_5 = 5, + level_6 = 6, + level_7 = 7, + level_8 = 8, + level_9 = 9, + + fast = 0xb, + default = 0xc, + best = 0xd, +}; + +/// Number of tokens to accumulate in deflate before starting block encoding. +/// +/// In zlib this depends on memlevel: 6 + memlevel, where default memlevel is +/// 8 and max 9 that gives 14 or 15 bits. +pub const n_tokens = 1 << 15; + +/// Algorithm knobs for each level. +const LevelArgs = struct { + good: u16, // Do less lookups if we already have match of this length. + nice: u16, // Stop looking for better match if we found match with at least this length. + lazy: u16, // Don't do lazy match find if got match with at least this length. + chain: u16, // How many lookups for previous match to perform. + + pub fn get(level: Level) LevelArgs { + return switch (level) { + .fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 }, + .level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 }, + .default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 }, + .level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 }, + .level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 }, + .best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 }, + }; + } +}; + +pub const Options = struct { + level: Level = .default, + container: Container = .raw, +}; + +pub fn init(input: *Reader, buffer: []u8, options: Options) Compress { + return .{ + .input = input, + .block_writer = undefined, + .level = .get(options.level), + .hasher = .init(options.container), + .state = .header, + .reader = .{ + .buffer = buffer, + .stream = stream, + }, + }; +} + +const FlushOption = enum { none, flush, final }; + +/// Process data in window and create tokens. If token buffer is full +/// flush tokens to the token writer. +/// +/// Returns number of bytes consumed from `lh`. +fn tokenizeSlice(c: *Compress, bw: *Writer, limit: std.Io.Limit, lh: []const u8) !usize { + _ = bw; + _ = limit; + if (true) @panic("TODO"); + var step: u16 = 1; // 1 in the case of literal, match length otherwise + const pos: u16 = c.win.pos(); + const literal = lh[0]; // literal at current position + const min_len: u16 = if (c.prev_match) |m| m.length() else 0; + + // Try to find match at least min_len long. + if (c.findMatch(pos, lh, min_len)) |match| { + // Found better match than previous. + try c.addPrevLiteral(); + + // Is found match length good enough? + if (match.length() >= c.level.lazy) { + // Don't try to lazy find better match, use this. + step = try c.addMatch(match); + } else { + // Store this match. + c.prev_literal = literal; + c.prev_match = match; + } + } else { + // There is no better match at current pos then it was previous. + // Write previous match or literal. + if (c.prev_match) |m| { + // Write match from previous position. + step = try c.addMatch(m) - 1; // we already advanced 1 from previous position + } else { + // No match at previous position. + // Write previous literal if any, and remember this literal. + try c.addPrevLiteral(); + c.prev_literal = literal; + } + } + // Advance window and add hashes. + c.windowAdvance(step, lh, pos); +} + +fn windowAdvance(self: *Compress, step: u16, lh: []const u8, pos: u16) void { + // current position is already added in findMatch + self.lookup.bulkAdd(lh[1..], step - 1, pos + 1); + self.win.advance(step); +} + +// Add previous literal (if any) to the tokens list. +fn addPrevLiteral(self: *Compress) !void { + if (self.prev_literal) |l| try self.addToken(Token.initLiteral(l)); +} + +// Add match to the tokens list, reset prev pointers. +// Returns length of the added match. +fn addMatch(self: *Compress, m: Token) !u16 { + try self.addToken(m); + self.prev_literal = null; + self.prev_match = null; + return m.length(); +} + +fn addToken(self: *Compress, token: Token) !void { + self.tokens.add(token); + if (self.tokens.full()) try self.flushTokens(.none); +} + +// Finds largest match in the history window with the data at current pos. +fn findMatch(self: *Compress, pos: u16, lh: []const u8, min_len: u16) ?Token { + var len: u16 = min_len; + // Previous location with the same hash (same 4 bytes). + var prev_pos = self.lookup.add(lh, pos); + // Last found match. + var match: ?Token = null; + + // How much back-references to try, performance knob. + var chain: usize = self.level.chain; + if (len >= self.level.good) { + // If we've got a match that's good enough, only look in 1/4 the chain. + chain >>= 2; + } + + // Hot path loop! + while (prev_pos > 0 and chain > 0) : (chain -= 1) { + const distance = pos - prev_pos; + if (distance > flate.match.max_distance) + break; + + const new_len = self.win.match(prev_pos, pos, len); + if (new_len > len) { + match = Token.initMatch(@intCast(distance), new_len); + if (new_len >= self.level.nice) { + // The match is good enough that we don't try to find a better one. + return match; + } + len = new_len; + } + prev_pos = self.lookup.prev(prev_pos); + } + + return match; +} + +fn flushTokens(self: *Compress, flush_opt: FlushOption) !void { + // Pass tokens to the token writer + try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer()); + // Stored block ensures byte alignment. + // It has 3 bits (final, block_type) and then padding until byte boundary. + // After that everything is aligned to the boundary in the stored block. + // Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF. + // Last 4 bytes are byte aligned. + if (flush_opt == .flush) { + try self.block_writer.storedBlock("", false); + } + if (flush_opt != .none) { + // Safe to call only when byte aligned or it is OK to add + // padding bits (on last byte of the final block). + try self.block_writer.flush(); + } + // Reset internal tokens store. + self.tokens.reset(); + // Notify win that tokens are flushed. + self.win.flush(); +} + +// Slide win and if needed lookup tables. +fn slide(self: *Compress) void { + const n = self.win.slide(); + self.lookup.slide(n); +} + +/// Flushes internal buffers to the output writer. Outputs empty stored +/// block to sync bit stream to the byte boundary, so that the +/// decompressor can get all input data available so far. +/// +/// It is useful mainly in compressed network protocols, to ensure that +/// deflate bit stream can be used as byte stream. May degrade +/// compression so it should be used only when necessary. +/// +/// Completes the current deflate block and follows it with an empty +/// stored block that is three zero bits plus filler bits to the next +/// byte, followed by four bytes (00 00 ff ff). +/// +pub fn flush(c: *Compress) !void { + try c.tokenize(.flush); +} + +/// Completes deflate bit stream by writing any pending data as deflate +/// final deflate block. HAS to be called once all data are written to +/// the compressor as a signal that next block has to have final bit +/// set. +/// +pub fn finish(c: *Compress) !void { + _ = c; + @panic("TODO"); +} + +/// Use another writer while preserving history. Most probably flush +/// should be called on old writer before setting new. +pub fn setWriter(self: *Compress, new_writer: *Writer) void { + self.block_writer.setWriter(new_writer); + self.output = new_writer; +} + +// Tokens store +const Tokens = struct { + list: [n_tokens]Token = undefined, + pos: usize = 0, + + fn add(self: *Tokens, t: Token) void { + self.list[self.pos] = t; + self.pos += 1; + } + + fn full(self: *Tokens) bool { + return self.pos == self.list.len; + } + + fn reset(self: *Tokens) void { + self.pos = 0; + } + + fn tokens(self: *Tokens) []const Token { + return self.list[0..self.pos]; + } +}; + +/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and +/// only performs Huffman entropy encoding. Results in faster compression, much +/// less memory requirements during compression but bigger compressed sizes. +pub const Huffman = SimpleCompressor(.huffman, .raw); + +/// Creates store blocks only. Data are not compressed only packed into deflate +/// store blocks. That adds 9 bytes of header for each block. Max stored block +/// size is 64K. Block is emitted when flush is called on on finish. +pub const store = struct { + pub fn Compressor(comptime container: Container, comptime WriterType: type) type { + return SimpleCompressor(.store, container, WriterType); + } + + pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) { + return try store.Compressor(container, @TypeOf(writer)).init(writer); + } +}; + +const SimpleCompressorKind = enum { + huffman, + store, +}; + +fn simpleCompressor( + comptime kind: SimpleCompressorKind, + comptime container: Container, + writer: anytype, +) !SimpleCompressor(kind, container, @TypeOf(writer)) { + return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer); +} + +fn SimpleCompressor( + comptime kind: SimpleCompressorKind, + comptime container: Container, + comptime WriterType: type, +) type { + const BlockWriterType = BlockWriter(WriterType); + return struct { + buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes + wp: usize = 0, + + output: WriterType, + block_writer: BlockWriterType, + hasher: container.Hasher() = .{}, + + const Self = @This(); + + pub fn init(output: WriterType) !Self { + const self = Self{ + .output = output, + .block_writer = BlockWriterType.init(output), + }; + try container.writeHeader(self.output); + return self; + } + + pub fn flush(self: *Self) !void { + try self.flushBuffer(false); + try self.block_writer.storedBlock("", false); + try self.block_writer.flush(); + } + + pub fn finish(self: *Self) !void { + try self.flushBuffer(true); + try self.block_writer.flush(); + try container.writeFooter(&self.hasher, self.output); + } + + fn flushBuffer(self: *Self, final: bool) !void { + const buf = self.buffer[0..self.wp]; + switch (kind) { + .huffman => try self.block_writer.huffmanBlock(buf, final), + .store => try self.block_writer.storedBlock(buf, final), + } + self.wp = 0; + } + }; +} + +const LiteralNode = struct { + literal: u16, + freq: u16, +}; + +// Describes the state of the constructed tree for a given depth. +const LevelInfo = struct { + // Our level. for better printing + level: u32, + + // The frequency of the last node at this level + last_freq: u32, + + // The frequency of the next character to add to this level + next_char_freq: u32, + + // The frequency of the next pair (from level below) to add to this level. + // Only valid if the "needed" value of the next lower level is 0. + next_pair_freq: u32, + + // The number of chains remaining to generate for this level before moving + // up to the next level + needed: u32, +}; + +// hcode is a huffman code with a bit code and bit length. +pub const HuffCode = struct { + code: u16 = 0, + len: u16 = 0, + + // set sets the code and length of an hcode. + fn set(self: *HuffCode, code: u16, length: u16) void { + self.len = length; + self.code = code; + } +}; + +pub fn HuffmanEncoder(comptime size: usize) type { + return struct { + codes: [size]HuffCode = undefined, + // Reusable buffer with the longest possible frequency table. + freq_cache: [huffman.max_num_frequencies + 1]LiteralNode = undefined, + bit_count: [17]u32 = undefined, + lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate + lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate + + const Self = @This(); + + // Update this Huffman Code object to be the minimum code for the specified frequency count. + // + // freq An array of frequencies, in which frequency[i] gives the frequency of literal i. + // max_bits The maximum number of bits to use for any literal. + pub fn generate(self: *Self, freq: []u16, max_bits: u32) void { + var list = self.freq_cache[0 .. freq.len + 1]; + // Number of non-zero literals + var count: u32 = 0; + // Set list to be the set of all non-zero literals and their frequencies + for (freq, 0..) |f, i| { + if (f != 0) { + list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f }; + count += 1; + } else { + list[count] = LiteralNode{ .literal = 0x00, .freq = 0 }; + self.codes[i].len = 0; + } + } + list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 }; + + list = list[0..count]; + if (count <= 2) { + // Handle the small cases here, because they are awkward for the general case code. With + // two or fewer literals, everything has bit length 1. + for (list, 0..) |node, i| { + // "list" is in order of increasing literal value. + self.codes[node.literal].set(@as(u16, @intCast(i)), 1); + } + return; + } + self.lfs = list; + mem.sort(LiteralNode, self.lfs, {}, byFreq); + + // Get the number of literals for each bit count + const bit_count = self.bitCounts(list, max_bits); + // And do the assignment + self.assignEncodingAndSize(bit_count, list); + } + + pub fn bitLength(self: *Self, freq: []u16) u32 { + var total: u32 = 0; + for (freq, 0..) |f, i| { + if (f != 0) { + total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len)); + } + } + return total; + } + + // Return the number of literals assigned to each bit size in the Huffman encoding + // + // This method is only called when list.len >= 3 + // The cases of 0, 1, and 2 literals are handled by special case code. + // + // list: An array of the literals with non-zero frequencies + // and their associated frequencies. The array is in order of increasing + // frequency, and has as its last element a special element with frequency + // `math.maxInt(i32)` + // + // max_bits: The maximum number of bits that should be used to encode any literal. + // Must be less than 16. + // + // Returns an integer array in which array[i] indicates the number of literals + // that should be encoded in i bits. + fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 { + var max_bits = max_bits_to_use; + const n = list.len; + const max_bits_limit = 16; + + assert(max_bits < max_bits_limit); + + // The tree can't have greater depth than n - 1, no matter what. This + // saves a little bit of work in some small cases + max_bits = @min(max_bits, n - 1); + + // Create information about each of the levels. + // A bogus "Level 0" whose sole purpose is so that + // level1.prev.needed == 0. This makes level1.next_pair_freq + // be a legitimate value that never gets chosen. + var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo); + // leaf_counts[i] counts the number of literals at the left + // of ancestors of the rightmost node at level i. + // leaf_counts[i][j] is the number of literals at the left + // of the level j ancestor. + var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32); + + { + var level = @as(u32, 1); + while (level <= max_bits) : (level += 1) { + // For every level, the first two items are the first two characters. + // We initialize the levels as if we had already figured this out. + levels[level] = LevelInfo{ + .level = level, + .last_freq = list[1].freq, + .next_char_freq = list[2].freq, + .next_pair_freq = list[0].freq + list[1].freq, + .needed = 0, + }; + leaf_counts[level][level] = 2; + if (level == 1) { + levels[level].next_pair_freq = math.maxInt(i32); + } + } + } + + // We need a total of 2*n - 2 items at top level and have already generated 2. + levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4; + + { + var level = max_bits; + while (true) { + var l = &levels[level]; + if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) { + // We've run out of both leaves and pairs. + // End all calculations for this level. + // To make sure we never come back to this level or any lower level, + // set next_pair_freq impossibly large. + l.needed = 0; + levels[level + 1].next_pair_freq = math.maxInt(i32); + level += 1; + continue; + } + + const prev_freq = l.last_freq; + if (l.next_char_freq < l.next_pair_freq) { + // The next item on this row is a leaf node. + const next = leaf_counts[level][level] + 1; + l.last_freq = l.next_char_freq; + // Lower leaf_counts are the same of the previous node. + leaf_counts[level][level] = next; + if (next >= list.len) { + l.next_char_freq = maxNode().freq; + } else { + l.next_char_freq = list[next].freq; + } + } else { + // The next item on this row is a pair from the previous row. + // next_pair_freq isn't valid until we generate two + // more values in the level below + l.last_freq = l.next_pair_freq; + // Take leaf counts from the lower level, except counts[level] remains the same. + @memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]); + levels[l.level - 1].needed = 2; + } + + l.needed -= 1; + if (l.needed == 0) { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in next_pair_freq + // of that level with the sum of the two nodes we've just calculated on + // this level. + if (l.level == max_bits) { + // All done! + break; + } + levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq; + level += 1; + } else { + // If we stole from below, move down temporarily to replenish it. + while (levels[level - 1].needed > 0) { + level -= 1; + if (level == 0) { + break; + } + } + } + } + } + + // Somethings is wrong if at the end, the top level is null or hasn't used + // all of the leaves. + assert(leaf_counts[max_bits][max_bits] == n); + + var bit_count = self.bit_count[0 .. max_bits + 1]; + var bits: u32 = 1; + const counts = &leaf_counts[max_bits]; + { + var level = max_bits; + while (level > 0) : (level -= 1) { + // counts[level] gives the number of literals requiring at least "bits" + // bits to encode. + bit_count[bits] = counts[level] - counts[level - 1]; + bits += 1; + if (level == 0) { + break; + } + } + } + return bit_count; + } + + // Look at the leaves and assign them a bit count and an encoding as specified + // in RFC 1951 3.2.2 + fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void { + var code = @as(u16, 0); + var list = list_arg; + + for (bit_count, 0..) |bits, n| { + code <<= 1; + if (n == 0 or bits == 0) { + continue; + } + // The literals list[list.len-bits] .. list[list.len-bits] + // are encoded using "bits" bits, and get the values + // code, code + 1, .... The code values are + // assigned in literal order (not frequency order). + const chunk = list[list.len - @as(u32, @intCast(bits)) ..]; + + self.lns = chunk; + mem.sort(LiteralNode, self.lns, {}, byLiteral); + + for (chunk) |node| { + self.codes[node.literal] = HuffCode{ + .code = bitReverse(u16, code, @as(u5, @intCast(n))), + .len = @as(u16, @intCast(n)), + }; + code += 1; + } + list = list[0 .. list.len - @as(u32, @intCast(bits))]; + } + } + }; +} + +fn maxNode() LiteralNode { + return LiteralNode{ + .literal = math.maxInt(u16), + .freq = math.maxInt(u16), + }; +} + +pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) { + return .{}; +} + +pub const LiteralEncoder = HuffmanEncoder(huffman.max_num_frequencies); +pub const DistanceEncoder = HuffmanEncoder(huffman.distance_code_count); +pub const CodegenEncoder = HuffmanEncoder(19); + +// Generates a HuffmanCode corresponding to the fixed literal table +pub fn fixedLiteralEncoder() LiteralEncoder { + var h: LiteralEncoder = undefined; + var ch: u16 = 0; + + while (ch < huffman.max_num_frequencies) : (ch += 1) { + var bits: u16 = undefined; + var size: u16 = undefined; + switch (ch) { + 0...143 => { + // size 8, 000110000 .. 10111111 + bits = ch + 48; + size = 8; + }, + 144...255 => { + // size 9, 110010000 .. 111111111 + bits = ch + 400 - 144; + size = 9; + }, + 256...279 => { + // size 7, 0000000 .. 0010111 + bits = ch - 256; + size = 7; + }, + else => { + // size 8, 11000000 .. 11000111 + bits = ch + 192 - 280; + size = 8; + }, + } + h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size }; + } + return h; +} + +pub fn fixedDistanceEncoder() DistanceEncoder { + var h: DistanceEncoder = undefined; + for (h.codes, 0..) |_, ch| { + h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 }; + } + return h; +} + +pub fn huffmanDistanceEncoder() DistanceEncoder { + var distance_freq = [1]u16{0} ** huffman.distance_code_count; + distance_freq[0] = 1; + // huff_distance is a static distance encoder used for huffman only encoding. + // It can be reused since we will not be encoding distance values. + var h: DistanceEncoder = .{}; + h.generate(distance_freq[0..], 15); + return h; +} + +fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool { + _ = context; + return a.literal < b.literal; +} + +fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool { + _ = context; + if (a.freq == b.freq) { + return a.literal < b.literal; + } + return a.freq < b.freq; +} + +fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize { + const c: *Compress = @fieldParentPtr("reader", r); + switch (c.state) { + .header => |i| { + const header = c.hasher.container().header(); + const n = try w.write(header[i..]); + if (header.len - i - n == 0) { + c.state = .middle; + } else { + c.state.header += n; + } + return n; + }, + .middle => { + c.input.fillMore() catch |err| switch (err) { + error.EndOfStream => { + c.state = .final; + return 0; + }, + else => |e| return e, + }; + const buffer_contents = c.input.buffered(); + const min_lookahead = flate.match.min_length + flate.match.max_length; + const history_plus_lookahead_len = flate.history_len + min_lookahead; + if (buffer_contents.len < history_plus_lookahead_len) return 0; + const lookahead = buffer_contents[flate.history_len..]; + const start = w.count; + const n = try c.tokenizeSlice(w, limit, lookahead) catch |err| switch (err) { + error.WriteFailed => return error.WriteFailed, + }; + c.hasher.update(lookahead[0..n]); + c.input.toss(n); + return w.count - start; + }, + .final => { + const buffer_contents = c.input.buffered(); + const start = w.count; + const n = c.tokenizeSlice(w, limit, buffer_contents) catch |err| switch (err) { + error.WriteFailed => return error.WriteFailed, + }; + if (buffer_contents.len - n == 0) { + c.hasher.update(buffer_contents); + c.input.tossAll(); + { + // In the case of flushing, last few lookahead buffers were + // smaller than min match len, so only last literal can be + // unwritten. + assert(c.prev_match == null); + try c.addPrevLiteral(); + c.prev_literal = null; + + try c.flushTokens(.final); + } + switch (c.hasher) { + .gzip => |*gzip| { + // GZIP 8 bytes footer + // - 4 bytes, CRC32 (CRC-32) + // - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32 + comptime assert(c.footer_buffer.len == 8); + std.mem.writeInt(u32, c.footer_buffer[0..4], gzip.final(), .little); + std.mem.writeInt(u32, c.footer_buffer[4..8], gzip.bytes_read, .little); + c.state = .{ .footer = 0 }; + }, + .zlib => |*zlib| { + // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). + // 4 bytes of ADLER32 (Adler-32 checksum) + // Checksum value of the uncompressed data (excluding any + // dictionary data) computed according to Adler-32 + // algorithm. + comptime assert(c.footer_buffer.len == 8); + std.mem.writeInt(u32, c.footer_buffer[4..8], zlib.final, .big); + c.state = .{ .footer = 4 }; + }, + .raw => { + c.state = .ended; + }, + } + } + return w.count - start; + }, + .ended => return error.EndOfStream, + .footer => |i| { + const remaining = c.footer_buffer[i..]; + const n = try w.write(limit.slice(remaining)); + c.state = if (n == remaining) .ended else .{ .footer = i - n }; + return n; + }, + } +} + +test "generate a Huffman code from an array of frequencies" { + var freqs: [19]u16 = [_]u16{ + 8, // 0 + 1, // 1 + 1, // 2 + 2, // 3 + 5, // 4 + 10, // 5 + 9, // 6 + 1, // 7 + 0, // 8 + 0, // 9 + 0, // 10 + 0, // 11 + 0, // 12 + 0, // 13 + 0, // 14 + 0, // 15 + 1, // 16 + 3, // 17 + 5, // 18 + }; + + var enc = huffmanEncoder(19); + enc.generate(freqs[0..], 7); + + try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..])); + + try testing.expectEqual(@as(usize, 3), enc.codes[0].len); + try testing.expectEqual(@as(usize, 6), enc.codes[1].len); + try testing.expectEqual(@as(usize, 6), enc.codes[2].len); + try testing.expectEqual(@as(usize, 5), enc.codes[3].len); + try testing.expectEqual(@as(usize, 3), enc.codes[4].len); + try testing.expectEqual(@as(usize, 2), enc.codes[5].len); + try testing.expectEqual(@as(usize, 2), enc.codes[6].len); + try testing.expectEqual(@as(usize, 6), enc.codes[7].len); + try testing.expectEqual(@as(usize, 0), enc.codes[8].len); + try testing.expectEqual(@as(usize, 0), enc.codes[9].len); + try testing.expectEqual(@as(usize, 0), enc.codes[10].len); + try testing.expectEqual(@as(usize, 0), enc.codes[11].len); + try testing.expectEqual(@as(usize, 0), enc.codes[12].len); + try testing.expectEqual(@as(usize, 0), enc.codes[13].len); + try testing.expectEqual(@as(usize, 0), enc.codes[14].len); + try testing.expectEqual(@as(usize, 0), enc.codes[15].len); + try testing.expectEqual(@as(usize, 6), enc.codes[16].len); + try testing.expectEqual(@as(usize, 5), enc.codes[17].len); + try testing.expectEqual(@as(usize, 3), enc.codes[18].len); + + try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code); + try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code); + try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code); + try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code); + try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code); + try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code); + try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code); + try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code); + try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code); + try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code); + try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code); +} + +test "generate a Huffman code for the fixed literal table specific to Deflate" { + const enc = fixedLiteralEncoder(); + for (enc.codes) |c| { + switch (c.len) { + 7 => { + const v = @bitReverse(@as(u7, @intCast(c.code))); + try testing.expect(v <= 0b0010111); + }, + 8 => { + const v = @bitReverse(@as(u8, @intCast(c.code))); + try testing.expect((v >= 0b000110000 and v <= 0b10111111) or + (v >= 0b11000000 and v <= 11000111)); + }, + 9 => { + const v = @bitReverse(@as(u9, @intCast(c.code))); + try testing.expect(v >= 0b110010000 and v <= 0b111111111); + }, + else => unreachable, + } + } +} + +test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" { + const enc = fixedDistanceEncoder(); + for (enc.codes) |c| { + const v = @bitReverse(@as(u5, @intCast(c.code))); + try testing.expect(v <= 29); + try testing.expect(c.len == 5); + } +} + +// Reverse bit-by-bit a N-bit code. +fn bitReverse(comptime T: type, value: T, n: usize) T { + const r = @bitReverse(value); + return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n)); +} + +test bitReverse { + const ReverseBitsTest = struct { + in: u16, + bit_count: u5, + out: u16, + }; + + const reverse_bits_tests = [_]ReverseBitsTest{ + .{ .in = 1, .bit_count = 1, .out = 1 }, + .{ .in = 1, .bit_count = 2, .out = 2 }, + .{ .in = 1, .bit_count = 3, .out = 4 }, + .{ .in = 1, .bit_count = 4, .out = 8 }, + .{ .in = 1, .bit_count = 5, .out = 16 }, + .{ .in = 17, .bit_count = 5, .out = 17 }, + .{ .in = 257, .bit_count = 9, .out = 257 }, + .{ .in = 29, .bit_count = 5, .out = 23 }, + }; + + for (reverse_bits_tests) |h| { + const v = bitReverse(u16, h.in, h.bit_count); + try std.testing.expectEqual(h.out, v); + } +} + +test "fixedLiteralEncoder codes" { + var al = std.ArrayList(u8).init(testing.allocator); + defer al.deinit(); + var bw = std.Io.bitWriter(.little, al.writer()); + + const f = fixedLiteralEncoder(); + for (f.codes) |c| { + try bw.writeBits(c.code, c.len); + } + try testing.expectEqualSlices(u8, &fixed_codes, al.items); +} + +pub const fixed_codes = [_]u8{ + 0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100, + 0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100, + 0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010, + 0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010, + 0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010, + 0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010, + 0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110, + 0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110, + 0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110, + 0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110, + 0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001, + 0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001, + 0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001, + 0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001, + 0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101, + 0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101, + 0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101, + 0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101, + 0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100, + 0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110, + 0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100, + 0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101, + 0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010, + 0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110, + 0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110, + 0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111, + 0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011, + 0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101, + 0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110, + 0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100, + 0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101, + 0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010, + 0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110, + 0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000, + 0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010, + 0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100, + 0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011, + 0b10100011, +}; + +test "tokenization" { + const L = Token.initLiteral; + const M = Token.initMatch; + + const cases = [_]struct { + data: []const u8, + tokens: []const Token, + }{ + .{ + .data = "Blah blah blah blah blah!", + .tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') }, + }, + .{ + .data = "ABCDEABCD ABCDEABCD", + .tokens = &[_]Token{ + L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '), + L('A'), M(10, 8), + }, + }, + }; + + for (cases) |c| { + inline for (Container.list) |container| { // for each wrapping + + var cw = std.Io.countingWriter(std.Io.null_writer); + const cww = cw.writer(); + var df = try Compress(container, @TypeOf(cww), TestTokenWriter).init(cww, .{}); + + _ = try df.write(c.data); + try df.flush(); + + // df.token_writer.show(); + try expect(df.block_writer.pos == c.tokens.len); // number of tokens written + try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match + + try testing.expectEqual(container.headerSize(), cw.bytes_written); + try df.finish(); + try testing.expectEqual(container.size(), cw.bytes_written); + } + } +} + +// Tests that tokens written are equal to expected token list. +const TestTokenWriter = struct { + const Self = @This(); + + pos: usize = 0, + actual: [128]Token = undefined, + + pub fn init(_: anytype) Self { + return .{}; + } + pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void { + for (tokens) |t| { + self.actual[self.pos] = t; + self.pos += 1; + } + } + + pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {} + + pub fn get(self: *Self) []Token { + return self.actual[0..self.pos]; + } + + pub fn show(self: *Self) void { + std.debug.print("\n", .{}); + for (self.get()) |t| { + t.show(); + } + } + + pub fn flush(_: *Self) !void {} +}; + +test "file tokenization" { + const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 }; + const cases = [_]struct { + data: []const u8, // uncompressed content + // expected number of tokens producet in deflate tokenization + tokens_count: [levels.len]usize = .{0} ** levels.len, + }{ + .{ + .data = @embedFile("testdata/rfc1951.txt"), + .tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 }, + }, + + .{ + .data = @embedFile("testdata/block_writer/huffman-null-max.input"), + .tokens_count = .{ 257, 257, 257, 257, 257, 257 }, + }, + .{ + .data = @embedFile("testdata/block_writer/huffman-pi.input"), + .tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 }, + }, + .{ + .data = @embedFile("testdata/block_writer/huffman-text.input"), + .tokens_count = .{ 235, 234, 234, 234, 234, 234 }, + }, + .{ + .data = @embedFile("testdata/fuzz/roundtrip1.input"), + .tokens_count = .{ 333, 331, 331, 331, 331, 331 }, + }, + .{ + .data = @embedFile("testdata/fuzz/roundtrip2.input"), + .tokens_count = .{ 334, 334, 334, 334, 334, 334 }, + }, + }; + + for (cases) |case| { // for each case + const data = case.data; + + for (levels, 0..) |level, i| { // for each compression level + var original: Reader = .fixed(data); + + // buffer for decompressed data + var al = std.ArrayList(u8).init(testing.allocator); + defer al.deinit(); + const writer = al.writer(); + + // create compressor + const WriterType = @TypeOf(writer); + const TokenWriter = TokenDecoder(@TypeOf(writer)); + var cmp = try Compress(.raw, WriterType, TokenWriter).init(writer, .{ .level = level }); + + // Stream uncompressed `original` data to the compressor. It will + // produce tokens list and pass that list to the TokenDecoder. This + // TokenDecoder uses CircularBuffer from inflate to convert list of + // tokens back to the uncompressed stream. + try cmp.compress(original.reader()); + try cmp.flush(); + const expected_count = case.tokens_count[i]; + const actual = cmp.block_writer.tokens_count; + if (expected_count == 0) { + std.debug.print("actual token count {d}\n", .{actual}); + } else { + try testing.expectEqual(expected_count, actual); + } + + try testing.expectEqual(data.len, al.items.len); + try testing.expectEqualSlices(u8, data, al.items); + } + } +} + +const TokenDecoder = struct { + output: *Writer, + tokens_count: usize, + + pub fn init(output: *Writer) TokenDecoder { + return .{ + .output = output, + .tokens_count = 0, + }; + } + + pub fn write(self: *TokenDecoder, tokens: []const Token, _: bool, _: ?[]const u8) !void { + self.tokens_count += tokens.len; + for (tokens) |t| { + switch (t.kind) { + .literal => self.hist.write(t.literal()), + .match => try self.hist.writeMatch(t.length(), t.distance()), + } + if (self.hist.free() < 285) try self.flushWin(); + } + try self.flushWin(); + } + + fn flushWin(self: *TokenDecoder) !void { + while (true) { + const buf = self.hist.read(); + if (buf.len == 0) break; + try self.output.writeAll(buf); + } + } +}; + +test "store simple compressor" { + const data = "Hello world!"; + const expected = [_]u8{ + 0x1, // block type 0, final bit set + 0xc, 0x0, // len = 12 + 0xf3, 0xff, // ~len + 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', // + //0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21, + }; + + var fbs: Reader = .fixed(data); + var al = std.ArrayList(u8).init(testing.allocator); + defer al.deinit(); + + var cmp = try store.compressor(.raw, al.writer()); + try cmp.compress(&fbs); + try cmp.finish(); + try testing.expectEqualSlices(u8, &expected, al.items); + + fbs = .fixed(data); + try al.resize(0); + + // huffman only compresoor will also emit store block for this small sample + var hc = try huffman.compressor(.raw, al.writer()); + try hc.compress(&fbs); + try hc.finish(); + try testing.expectEqualSlices(u8, &expected, al.items); +} + +test "sliding window match" { + const data = "Blah blah blah blah blah!"; + var win: Writer = .{}; + try expect(win.write(data) == data.len); + try expect(win.wp == data.len); + try expect(win.rp == 0); + + // length between l symbols + try expect(win.match(1, 6, 0) == 18); + try expect(win.match(1, 11, 0) == 13); + try expect(win.match(1, 16, 0) == 8); + try expect(win.match(1, 21, 0) == 0); + + // position 15 = "blah blah!" + // position 20 = "blah!" + try expect(win.match(15, 20, 0) == 4); + try expect(win.match(15, 20, 3) == 4); + try expect(win.match(15, 20, 4) == 0); +} + +test "sliding window slide" { + var win: Writer = .{}; + win.wp = Writer.buffer_len - 11; + win.rp = Writer.buffer_len - 111; + win.buffer[win.rp] = 0xab; + try expect(win.lookahead().len == 100); + try expect(win.tokensBuffer().?.len == win.rp); + + const n = win.slide(); + try expect(n == 32757); + try expect(win.buffer[win.rp] == 0xab); + try expect(win.rp == Writer.hist_len - 111); + try expect(win.wp == Writer.hist_len - 11); + try expect(win.lookahead().len == 100); + try expect(win.tokensBuffer() == null); +} diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig new file mode 100644 index 0000000000..6cb5953763 --- /dev/null +++ b/lib/std/compress/flate/Decompress.zig @@ -0,0 +1,894 @@ +const std = @import("../../std.zig"); +const flate = std.compress.flate; +const Container = flate.Container; +const Token = @import("Token.zig"); +const testing = std.testing; +const Decompress = @This(); +const Writer = std.io.Writer; +const Reader = std.io.Reader; + +input: *Reader, +reader: Reader, +/// Hashes, produces checksum, of uncompressed data for gzip/zlib footer. +hasher: Container.Hasher, + +lit_dec: LiteralDecoder, +dst_dec: DistanceDecoder, + +final_block: bool, +state: State, + +read_err: ?Error, + +const BlockType = enum(u2) { + stored = 0, + fixed = 1, + dynamic = 2, +}; + +const State = union(enum) { + protocol_header, + block_header, + stored_block: u16, + fixed_block, + dynamic_block, + protocol_footer, + end, +}; + +pub const Error = Container.Error || error{ + InvalidCode, + InvalidMatch, + InvalidBlockType, + WrongStoredBlockNlen, + InvalidDynamicBlockHeader, + EndOfStream, + ReadFailed, + OversubscribedHuffmanTree, + IncompleteHuffmanTree, + MissingEndOfBlockCode, +}; + +pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { + return .{ + .reader = .{ + // TODO populate discard so that when an amount is discarded that + // includes an entire frame, skip decoding that frame. + .vtable = &.{ .stream = stream }, + .buffer = buffer, + .seek = 0, + .end = 0, + }, + .input = input, + .hasher = .init(container), + .lit_dec = .{}, + .dst_dec = .{}, + .final_block = false, + .state = .protocol_header, + .read_err = null, + }; +} + +fn decodeLength(self: *Decompress, code: u8) !u16 { + if (code > 28) return error.InvalidCode; + const ml = Token.matchLength(code); + return if (ml.extra_bits == 0) // 0 - 5 extra bits + ml.base + else + ml.base + try self.takeNBitsBuffered(ml.extra_bits); +} + +fn decodeDistance(self: *Decompress, code: u8) !u16 { + if (code > 29) return error.InvalidCode; + const md = Token.matchDistance(code); + return if (md.extra_bits == 0) // 0 - 13 extra bits + md.base + else + md.base + try self.takeNBitsBuffered(md.extra_bits); +} + +// Decode code length symbol to code length. Writes decoded length into +// lens slice starting at position pos. Returns number of positions +// advanced. +fn dynamicCodeLength(self: *Decompress, code: u16, lens: []u4, pos: usize) !usize { + if (pos >= lens.len) + return error.InvalidDynamicBlockHeader; + + switch (code) { + 0...15 => { + // Represent code lengths of 0 - 15 + lens[pos] = @intCast(code); + return 1; + }, + 16 => { + // Copy the previous code length 3 - 6 times. + // The next 2 bits indicate repeat length + const n: u8 = @as(u8, try self.takeBits(u2)) + 3; + if (pos == 0 or pos + n > lens.len) + return error.InvalidDynamicBlockHeader; + for (0..n) |i| { + lens[pos + i] = lens[pos + i - 1]; + } + return n; + }, + // Repeat a code length of 0 for 3 - 10 times. (3 bits of length) + 17 => return @as(u8, try self.takeBits(u3)) + 3, + // Repeat a code length of 0 for 11 - 138 times (7 bits of length) + 18 => return @as(u8, try self.takeBits(u7)) + 11, + else => return error.InvalidDynamicBlockHeader, + } +} + +// Peek 15 bits from bits reader (maximum code len is 15 bits). Use +// decoder to find symbol for that code. We then know how many bits is +// used. Shift bit reader for that much bits, those bits are used. And +// return symbol. +fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol { + const sym = try decoder.find(try self.peekBitsReverseBuffered(u15)); + try self.shiftBits(sym.code_bits); + return sym; +} + +pub fn stream(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!usize { + const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); + return readInner(d, w, limit) catch |err| switch (err) { + error.EndOfStream => return error.EndOfStream, + error.WriteFailed => return error.WriteFailed, + else => |e| { + // In the event of an error, state is unmodified so that it can be + // better used to diagnose the failure. + d.read_err = e; + return error.ReadFailed; + }, + }; +} + +fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.StreamError)!usize { + const in = d.input; + sw: switch (d.state) { + .protocol_header => switch (d.hasher.container()) { + .gzip => { + const Header = extern struct { + magic: u16 align(1), + method: u8, + flags: packed struct(u8) { + text: bool, + hcrc: bool, + extra: bool, + name: bool, + comment: bool, + reserved: u3, + }, + mtime: u32 align(1), + xfl: u8, + os: u8, + }; + const header = try in.takeStruct(Header, .little); + if (header.magic != 0x8b1f or header.method != 0x08) + return error.BadGzipHeader; + if (header.flags.extra) { + const extra_len = try in.takeInt(u16, .little); + try in.discardAll(extra_len); + } + if (header.flags.name) { + _ = try in.discardDelimiterInclusive(0); + } + if (header.flags.comment) { + _ = try in.discardDelimiterInclusive(0); + } + if (header.flags.hcrc) { + try in.discardAll(2); + } + continue :sw .block_header; + }, + .zlib => { + const Header = extern struct { + cmf: packed struct(u8) { + cm: u4, + cinfo: u4, + }, + flg: u8, + }; + const header = try in.takeStruct(Header); + if (header.cmf.cm != 8 or header.cmf.cinfo > 7) return error.BadZlibHeader; + continue :sw .block_header; + }, + .raw => continue :sw .block_header, + }, + .block_header => { + d.final_block = (try d.takeBits(u1)) != 0; + const block_type = try d.takeBits(BlockType); + switch (block_type) { + .stored => { + d.alignBitsToByte(); // skip padding until byte boundary + // everything after this is byte aligned in stored block + const len = try in.takeInt(u16, .little); + const nlen = try in.takeInt(u16, .little); + if (len != ~nlen) return error.WrongStoredBlockNlen; + continue :sw .{ .stored_block = len }; + }, + .fixed => continue :sw .fixed_block, + .dynamic => { + const hlit: u16 = @as(u16, try d.takeBits(u5)) + 257; // number of ll code entries present - 257 + const hdist: u16 = @as(u16, try d.takeBits(u5)) + 1; // number of distance code entries - 1 + const hclen: u8 = @as(u8, try d.takeBits(u4)) + 4; // hclen + 4 code lengths are encoded + + if (hlit > 286 or hdist > 30) + return error.InvalidDynamicBlockHeader; + + // lengths for code lengths + var cl_lens = [_]u4{0} ** 19; + for (0..hclen) |i| { + cl_lens[flate.huffman.codegen_order[i]] = try d.takeBits(u3); + } + var cl_dec: CodegenDecoder = .{}; + try cl_dec.generate(&cl_lens); + + // decoded code lengths + var dec_lens = [_]u4{0} ** (286 + 30); + var pos: usize = 0; + while (pos < hlit + hdist) { + const sym = try cl_dec.find(try d.peekBitsReverse(u7)); + try d.shiftBits(sym.code_bits); + pos += try d.dynamicCodeLength(sym.symbol, &dec_lens, pos); + } + if (pos > hlit + hdist) { + return error.InvalidDynamicBlockHeader; + } + + // literal code lengths to literal decoder + try d.lit_dec.generate(dec_lens[0..hlit]); + + // distance code lengths to distance decoder + try d.dst_dec.generate(dec_lens[hlit .. hlit + hdist]); + + continue :sw .dynamic_block; + }, + } + }, + .stored_block => |remaining_len| { + const out = try w.writableSliceGreedyPreserve(flate.history_len, 1); + const limited_out = limit.min(.limited(remaining_len)).slice(out); + const n = try d.input.readVec(&.{limited_out}); + if (remaining_len - n == 0) { + d.state = if (d.final_block) .protocol_footer else .block_header; + } else { + d.state = .{ .stored_block = @intCast(remaining_len - n) }; + } + w.advance(n); + return n; + }, + .fixed_block => { + const start = w.count; + while (@intFromEnum(limit) > w.count - start) { + const code = try d.readFixedCode(); + switch (code) { + 0...255 => try w.writeBytePreserve(flate.history_len, @intCast(code)), + 256 => { + d.state = if (d.final_block) .protocol_footer else .block_header; + return w.count - start; + }, + 257...285 => { + // Handles fixed block non literal (length) code. + // Length code is followed by 5 bits of distance code. + const length = try d.decodeLength(@intCast(code - 257)); + const distance = try d.decodeDistance(try d.takeBitsReverseBuffered(u5)); + try writeMatch(w, length, distance); + }, + else => return error.InvalidCode, + } + } + d.state = .fixed_block; + return w.count - start; + }, + .dynamic_block => { + // In larger archives most blocks are usually dynamic, so decompression + // performance depends on this logic. + const start = w.count; + while (@intFromEnum(limit) > w.count - start) { + const sym = try d.decodeSymbol(&d.lit_dec); + + switch (sym.kind) { + .literal => try w.writeBytePreserve(flate.history_len, sym.symbol), + .match => { + // Decode match backreference + const length = try d.decodeLength(sym.symbol); + const dsm = try d.decodeSymbol(&d.dst_dec); + const distance = try d.decodeDistance(dsm.symbol); + try writeMatch(w, length, distance); + }, + .end_of_block => { + d.state = if (d.final_block) .protocol_footer else .block_header; + return w.count - start; + }, + } + } + d.state = .dynamic_block; + return w.count - start; + }, + .protocol_footer => { + d.alignBitsToByte(); + switch (d.hasher) { + .gzip => |*gzip| { + if (try in.takeInt(u32, .little) != gzip.crc.final()) return error.WrongGzipChecksum; + if (try in.takeInt(u32, .little) != gzip.count) return error.WrongGzipSize; + }, + .zlib => |*zlib| { + const chksum: u32 = @byteSwap(zlib.final()); + if (try in.takeInt(u32, .big) != chksum) return error.WrongZlibChecksum; + }, + .raw => {}, + } + d.state = .end; + return 0; + }, + .end => return error.EndOfStream, + } +} + +/// Write match (back-reference to the same data slice) starting at `distance` +/// back from current write position, and `length` of bytes. +fn writeMatch(bw: *Writer, length: u16, distance: u16) !void { + _ = bw; + _ = length; + _ = distance; + @panic("TODO"); +} + +fn takeBits(d: *Decompress, comptime T: type) !T { + _ = d; + @panic("TODO"); +} + +fn takeBitsReverseBuffered(d: *Decompress, comptime T: type) !T { + _ = d; + @panic("TODO"); +} + +fn takeNBitsBuffered(d: *Decompress, n: u4) !u16 { + _ = d; + _ = n; + @panic("TODO"); +} + +fn peekBitsReverse(d: *Decompress, comptime T: type) !T { + _ = d; + @panic("TODO"); +} + +fn peekBitsReverseBuffered(d: *Decompress, comptime T: type) !T { + _ = d; + @panic("TODO"); +} + +fn alignBitsToByte(d: *Decompress) void { + _ = d; + @panic("TODO"); +} + +fn shiftBits(d: *Decompress, n: u6) !void { + _ = d; + _ = n; + @panic("TODO"); +} + +fn readFixedCode(d: *Decompress) !u16 { + _ = d; + @panic("TODO"); +} + +pub const Symbol = packed struct { + pub const Kind = enum(u2) { + literal, + end_of_block, + match, + }; + + symbol: u8 = 0, // symbol from alphabet + code_bits: u4 = 0, // number of bits in code 0-15 + kind: Kind = .literal, + + code: u16 = 0, // huffman code of the symbol + next: u16 = 0, // pointer to the next symbol in linked list + // it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup + + // Sorting less than function. + pub fn asc(_: void, a: Symbol, b: Symbol) bool { + if (a.code_bits == b.code_bits) { + if (a.kind == b.kind) { + return a.symbol < b.symbol; + } + return @intFromEnum(a.kind) < @intFromEnum(b.kind); + } + return a.code_bits < b.code_bits; + } +}; + +pub const LiteralDecoder = HuffmanDecoder(286, 15, 9); +pub const DistanceDecoder = HuffmanDecoder(30, 15, 9); +pub const CodegenDecoder = HuffmanDecoder(19, 7, 7); + +/// Creates huffman tree codes from list of code lengths (in `build`). +/// +/// `find` then finds symbol for code bits. Code can be any length between 1 and +/// 15 bits. When calling `find` we don't know how many bits will be used to +/// find symbol. When symbol is returned it has code_bits field which defines +/// how much we should advance in bit stream. +/// +/// Lookup table is used to map 15 bit int to symbol. Same symbol is written +/// many times in this table; 32K places for 286 (at most) symbols. +/// Small lookup table is optimization for faster search. +/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92) +/// with difference that we here use statically allocated arrays. +/// +fn HuffmanDecoder( + comptime alphabet_size: u16, + comptime max_code_bits: u4, + comptime lookup_bits: u4, +) type { + const lookup_shift = max_code_bits - lookup_bits; + + return struct { + // all symbols in alaphabet, sorted by code_len, symbol + symbols: [alphabet_size]Symbol = undefined, + // lookup table code -> symbol + lookup: [1 << lookup_bits]Symbol = undefined, + + const Self = @This(); + + /// Generates symbols and lookup tables from list of code lens for each symbol. + pub fn generate(self: *Self, lens: []const u4) !void { + try checkCompleteness(lens); + + // init alphabet with code_bits + for (self.symbols, 0..) |_, i| { + const cb: u4 = if (i < lens.len) lens[i] else 0; + self.symbols[i] = if (i < 256) + .{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb } + else if (i == 256) + .{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb } + else + .{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb }; + } + std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc); + + // reset lookup table + for (0..self.lookup.len) |i| { + self.lookup[i] = .{}; + } + + // assign code to symbols + // reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639 + var code: u16 = 0; + var idx: u16 = 0; + for (&self.symbols, 0..) |*sym, pos| { + if (sym.code_bits == 0) continue; // skip unused + sym.code = code; + + const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits)); + const next_idx = next_code >> lookup_shift; + + if (next_idx > self.lookup.len or idx >= self.lookup.len) break; + if (sym.code_bits <= lookup_bits) { + // fill small lookup table + for (idx..next_idx) |j| + self.lookup[j] = sym.*; + } else { + // insert into linked table starting at root + const root = &self.lookup[idx]; + const root_next = root.next; + root.next = @intCast(pos); + sym.next = root_next; + } + + idx = next_idx; + code = next_code; + } + } + + /// Given the list of code lengths check that it represents a canonical + /// Huffman code for n symbols. + /// + /// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340 + fn checkCompleteness(lens: []const u4) !void { + if (alphabet_size == 286) + if (lens[256] == 0) return error.MissingEndOfBlockCode; + + var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1); + var max: usize = 0; + for (lens) |n| { + if (n == 0) continue; + if (n > max) max = n; + count[n] += 1; + } + if (max == 0) // empty tree + return; + + // check for an over-subscribed or incomplete set of lengths + var left: usize = 1; // one possible code of zero length + for (1..count.len) |len| { + left <<= 1; // one more bit, double codes left + if (count[len] > left) + return error.OversubscribedHuffmanTree; + left -= count[len]; // deduct count from possible codes + } + if (left > 0) { // left > 0 means incomplete + // incomplete code ok only for single length 1 code + if (max_code_bits > 7 and max == count[0] + count[1]) return; + return error.IncompleteHuffmanTree; + } + } + + /// Finds symbol for lookup table code. + pub fn find(self: *Self, code: u16) !Symbol { + // try to find in lookup table + const idx = code >> lookup_shift; + const sym = self.lookup[idx]; + if (sym.code_bits != 0) return sym; + // if not use linked list of symbols with same prefix + return self.findLinked(code, sym.next); + } + + inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol { + var pos = start; + while (pos > 0) { + const sym = self.symbols[pos]; + const shift = max_code_bits - sym.code_bits; + // compare code_bits number of upper bits + if ((code ^ sym.code) >> shift == 0) return sym; + pos = sym.next; + } + return error.InvalidCode; + } + }; +} + +test "init/find" { + // example data from: https://youtu.be/SJPvNi4HrWQ?t=8423 + const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 }; + var h: CodegenDecoder = .{}; + try h.generate(&code_lens); + + const expected = [_]struct { + sym: Symbol, + code: u16, + }{ + .{ + .code = 0b00_00000, + .sym = .{ .symbol = 3, .code_bits = 2 }, + }, + .{ + .code = 0b01_00000, + .sym = .{ .symbol = 18, .code_bits = 2 }, + }, + .{ + .code = 0b100_0000, + .sym = .{ .symbol = 1, .code_bits = 3 }, + }, + .{ + .code = 0b101_0000, + .sym = .{ .symbol = 4, .code_bits = 3 }, + }, + .{ + .code = 0b110_0000, + .sym = .{ .symbol = 17, .code_bits = 3 }, + }, + .{ + .code = 0b1110_000, + .sym = .{ .symbol = 0, .code_bits = 4 }, + }, + .{ + .code = 0b1111_000, + .sym = .{ .symbol = 16, .code_bits = 4 }, + }, + }; + + // unused symbols + for (0..12) |i| { + try testing.expectEqual(0, h.symbols[i].code_bits); + } + // used, from index 12 + for (expected, 12..) |e, i| { + try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol); + try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits); + const sym_from_code = try h.find(e.code); + try testing.expectEqual(e.sym.symbol, sym_from_code.symbol); + } + + // All possible codes for each symbol. + // Lookup table has 126 elements, to cover all possible 7 bit codes. + for (0b0000_000..0b0100_000) |c| // 0..32 (32) + try testing.expectEqual(3, (try h.find(@intCast(c))).symbol); + + for (0b0100_000..0b1000_000) |c| // 32..64 (32) + try testing.expectEqual(18, (try h.find(@intCast(c))).symbol); + + for (0b1000_000..0b1010_000) |c| // 64..80 (16) + try testing.expectEqual(1, (try h.find(@intCast(c))).symbol); + + for (0b1010_000..0b1100_000) |c| // 80..96 (16) + try testing.expectEqual(4, (try h.find(@intCast(c))).symbol); + + for (0b1100_000..0b1110_000) |c| // 96..112 (16) + try testing.expectEqual(17, (try h.find(@intCast(c))).symbol); + + for (0b1110_000..0b1111_000) |c| // 112..120 (8) + try testing.expectEqual(0, (try h.find(@intCast(c))).symbol); + + for (0b1111_000..0b1_0000_000) |c| // 120...128 (8) + try testing.expectEqual(16, (try h.find(@intCast(c))).symbol); +} + +test "encode/decode literals" { + const LiteralEncoder = std.compress.flate.Compress.LiteralEncoder; + + for (1..286) |j| { // for all different number of codes + var enc: LiteralEncoder = .{}; + // create frequencies + var freq = [_]u16{0} ** 286; + freq[256] = 1; // ensure we have end of block code + for (&freq, 1..) |*f, i| { + if (i % j == 0) + f.* = @intCast(i); + } + + // encoder from frequencies + enc.generate(&freq, 15); + + // get code_lens from encoder + var code_lens = [_]u4{0} ** 286; + for (code_lens, 0..) |_, i| { + code_lens[i] = @intCast(enc.codes[i].len); + } + // generate decoder from code lens + var dec: LiteralDecoder = .{}; + try dec.generate(&code_lens); + + // expect decoder code to match original encoder code + for (dec.symbols) |s| { + if (s.code_bits == 0) continue; + const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code))); + const symbol: u16 = switch (s.kind) { + .literal => s.symbol, + .end_of_block => 256, + .match => @as(u16, s.symbol) + 257, + }; + + const c = enc.codes[symbol]; + try testing.expect(c.code == c_code); + } + + // find each symbol by code + for (enc.codes) |c| { + if (c.len == 0) continue; + + const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code))); + const s = try dec.find(s_code); + try testing.expect(s.code == s_code); + try testing.expect(s.code_bits == c.len); + } + } +} + +test "decompress" { + const cases = [_]struct { + in: []const u8, + out: []const u8, + }{ + // non compressed block (type 0) + .{ + .in = &[_]u8{ + 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen + 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data + }, + .out = "Hello world\n", + }, + // fixed code block (type 1) + .{ + .in = &[_]u8{ + 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 + 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, + }, + .out = "Hello world\n", + }, + // dynamic block (type 2) + .{ + .in = &[_]u8{ + 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 + 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, + 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01, + }, + .out = "ABCDEABCD ABCDEABCD", + }, + }; + for (cases) |c| { + var fb: Reader = .fixed(c.in); + var aw: Writer.Allocating = .init(testing.allocator); + defer aw.deinit(); + + var decompress: Decompress = .init(&fb, .raw, &.{}); + const r = &decompress.reader; + _ = try r.streamRemaining(&aw.writer); + try testing.expectEqualStrings(c.out, aw.getWritten()); + } +} + +test "gzip decompress" { + const cases = [_]struct { + in: []const u8, + out: []const u8, + }{ + // non compressed block (type 0) + .{ + .in = &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes) + 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen + 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data + 0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum + 0x0c, 0x00, 0x00, 0x00, // gzip footer: size + }, + .out = "Hello world\n", + }, + // fixed code block (type 1) + .{ + .in = &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes) + 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 + 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, + 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len) + }, + .out = "Hello world\n", + }, + // dynamic block (type 2) + .{ + .in = &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes) + 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 + 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, + 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01, + 0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len) + }, + .out = "ABCDEABCD ABCDEABCD", + }, + // gzip header with name + .{ + .in = &[_]u8{ + 0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, + }, + .out = "Hello world\n", + }, + }; + for (cases) |c| { + var fb: Reader = .fixed(c.in); + var aw: Writer.Allocating = .init(testing.allocator); + defer aw.deinit(); + + var decompress: Decompress = .init(&fb, .gzip, &.{}); + const r = &decompress.reader; + _ = try r.streamRemaining(&aw.writer); + try testing.expectEqualStrings(c.out, aw.getWritten()); + } +} + +test "zlib decompress" { + const cases = [_]struct { + in: []const u8, + out: []const u8, + }{ + // non compressed block (type 0) + .{ + .in = &[_]u8{ + 0x78, 0b10_0_11100, // zlib header (2 bytes) + 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen + 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data + 0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum + }, + .out = "Hello world\n", + }, + }; + for (cases) |c| { + var fb: Reader = .fixed(c.in); + var aw: Writer.Allocating = .init(testing.allocator); + defer aw.deinit(); + + var decompress: Decompress = .init(&fb, .zlib, &.{}); + const r = &decompress.reader; + _ = try r.streamRemaining(&aw.writer); + try testing.expectEqualStrings(c.out, aw.getWritten()); + } +} + +test "fuzzing tests" { + const cases = [_]struct { + input: []const u8, + out: []const u8 = "", + err: ?anyerror = null, + }{ + .{ .input = "deflate-stream", .out = @embedFile("testdata/fuzz/deflate-stream.expect") }, // 0 + .{ .input = "empty-distance-alphabet01" }, + .{ .input = "empty-distance-alphabet02" }, + .{ .input = "end-of-stream", .err = error.EndOfStream }, + .{ .input = "invalid-distance", .err = error.InvalidMatch }, + .{ .input = "invalid-tree01", .err = error.IncompleteHuffmanTree }, // 5 + .{ .input = "invalid-tree02", .err = error.IncompleteHuffmanTree }, + .{ .input = "invalid-tree03", .err = error.IncompleteHuffmanTree }, + .{ .input = "lengths-overflow", .err = error.InvalidDynamicBlockHeader }, + .{ .input = "out-of-codes", .err = error.InvalidCode }, + .{ .input = "puff01", .err = error.WrongStoredBlockNlen }, // 10 + .{ .input = "puff02", .err = error.EndOfStream }, + .{ .input = "puff03", .out = &[_]u8{0xa} }, + .{ .input = "puff04", .err = error.InvalidCode }, + .{ .input = "puff05", .err = error.EndOfStream }, + .{ .input = "puff06", .err = error.EndOfStream }, + .{ .input = "puff08", .err = error.InvalidCode }, + .{ .input = "puff09", .out = "P" }, + .{ .input = "puff10", .err = error.InvalidCode }, + .{ .input = "puff11", .err = error.InvalidMatch }, + .{ .input = "puff12", .err = error.InvalidDynamicBlockHeader }, // 20 + .{ .input = "puff13", .err = error.IncompleteHuffmanTree }, + .{ .input = "puff14", .err = error.EndOfStream }, + .{ .input = "puff15", .err = error.IncompleteHuffmanTree }, + .{ .input = "puff16", .err = error.InvalidDynamicBlockHeader }, + .{ .input = "puff17", .err = error.MissingEndOfBlockCode }, // 25 + .{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader }, + .{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader }, + .{ .input = "fuzz3", .err = error.InvalidMatch }, + .{ .input = "fuzz4", .err = error.OversubscribedHuffmanTree }, + .{ .input = "puff18", .err = error.OversubscribedHuffmanTree }, // 30 + .{ .input = "puff19", .err = error.OversubscribedHuffmanTree }, + .{ .input = "puff20", .err = error.OversubscribedHuffmanTree }, + .{ .input = "puff21", .err = error.OversubscribedHuffmanTree }, + .{ .input = "puff22", .err = error.OversubscribedHuffmanTree }, + .{ .input = "puff23", .err = error.OversubscribedHuffmanTree }, // 35 + .{ .input = "puff24", .err = error.IncompleteHuffmanTree }, + .{ .input = "puff25", .err = error.OversubscribedHuffmanTree }, + .{ .input = "puff26", .err = error.InvalidDynamicBlockHeader }, + .{ .input = "puff27", .err = error.InvalidDynamicBlockHeader }, + }; + + inline for (cases, 0..) |c, case_no| { + var in: Reader = .fixed(@embedFile("testdata/fuzz/" ++ c.input ++ ".input")); + var aw: Writer.Allocating = .init(testing.allocator); + defer aw.deinit(); + errdefer std.debug.print("test case failed {}\n", .{case_no}); + + var decompress: Decompress = .init(&in, .raw, &.{}); + const r = &decompress.reader; + if (c.err) |expected_err| { + try testing.expectError(error.ReadFailed, r.streamRemaining(&aw.writer)); + try testing.expectError(expected_err, decompress.read_err.?); + } else { + _ = try r.streamRemaining(&aw.writer); + try testing.expectEqualStrings(c.out, aw.getWritten()); + } + } +} + +test "bug 18966" { + const input = @embedFile("testdata/fuzz/bug_18966.input"); + const expect = @embedFile("testdata/fuzz/bug_18966.expect"); + + var in: Reader = .fixed(input); + var aw: Writer.Allocating = .init(testing.allocator); + defer aw.deinit(); + + var decompress: Decompress = .init(&in, .gzip, &.{}); + const r = &decompress.reader; + _ = try r.streamRemaining(&aw.writer); + try testing.expectEqualStrings(expect, aw.getWritten()); +} + +test "reading into empty buffer" { + // Inspired by https://github.com/ziglang/zig/issues/19895 + const input = &[_]u8{ + 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen + 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data + }; + var in: Reader = .fixed(input); + var decomp: Decompress = .init(&in, .raw, &.{}); + const r = &decomp.reader; + var buf: [0]u8 = undefined; + try testing.expectEqual(0, try r.readVec(&.{&buf})); +} diff --git a/lib/std/compress/flate/Lookup.zig b/lib/std/compress/flate/Lookup.zig index 90d0341bca..722e175c8a 100644 --- a/lib/std/compress/flate/Lookup.zig +++ b/lib/std/compress/flate/Lookup.zig @@ -5,22 +5,22 @@ const std = @import("std"); const testing = std.testing; const expect = testing.expect; -const consts = @import("consts.zig"); +const flate = @import("../flate.zig"); -const Self = @This(); +const Lookup = @This(); const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761 -const chain_len = 2 * consts.history.len; +const chain_len = 2 * flate.history_len; // Maps hash => first position -head: [consts.lookup.len]u16 = [_]u16{0} ** consts.lookup.len, +head: [flate.lookup.len]u16 = [_]u16{0} ** flate.lookup.len, // Maps position => previous positions for the same hash value chain: [chain_len]u16 = [_]u16{0} ** (chain_len), // Calculates hash of the 4 bytes from data. // Inserts `pos` position of that hash in the lookup tables. // Returns previous location with the same hash value. -pub fn add(self: *Self, data: []const u8, pos: u16) u16 { +pub fn add(self: *Lookup, data: []const u8, pos: u16) u16 { if (data.len < 4) return 0; const h = hash(data[0..4]); return self.set(h, pos); @@ -28,11 +28,11 @@ pub fn add(self: *Self, data: []const u8, pos: u16) u16 { // Returns previous location with the same hash value given the current // position. -pub fn prev(self: *Self, pos: u16) u16 { +pub fn prev(self: *Lookup, pos: u16) u16 { return self.chain[pos]; } -fn set(self: *Self, h: u32, pos: u16) u16 { +fn set(self: *Lookup, h: u32, pos: u16) u16 { const p = self.head[h]; self.head[h] = pos; self.chain[pos] = p; @@ -40,7 +40,7 @@ fn set(self: *Self, h: u32, pos: u16) u16 { } // Slide all positions in head and chain for `n` -pub fn slide(self: *Self, n: u16) void { +pub fn slide(self: *Lookup, n: u16) void { for (&self.head) |*v| { v.* -|= n; } @@ -52,8 +52,8 @@ pub fn slide(self: *Self, n: u16) void { // Add `len` 4 bytes hashes from `data` into lookup. // Position of the first byte is `pos`. -pub fn bulkAdd(self: *Self, data: []const u8, len: u16, pos: u16) void { - if (len == 0 or data.len < consts.match.min_length) { +pub fn bulkAdd(self: *Lookup, data: []const u8, len: u16, pos: u16) void { + if (len == 0 or data.len < flate.match.min_length) { return; } var hb = @@ -80,7 +80,7 @@ fn hash(b: *const [4]u8) u32 { } fn hashu(v: u32) u32 { - return @intCast((v *% prime4) >> consts.lookup.shift); + return @intCast((v *% prime4) >> flate.lookup.shift); } test add { @@ -91,7 +91,7 @@ test add { 0x01, 0x02, 0x03, }; - var h: Self = .{}; + var h: Lookup = .{}; for (data, 0..) |_, i| { const p = h.add(data[i..], @intCast(i)); if (i >= 8 and i < 24) { @@ -101,7 +101,7 @@ test add { } } - const v = Self.hash(data[2 .. 2 + 4]); + const v = Lookup.hash(data[2 .. 2 + 4]); try expect(h.head[v] == 2 + 16); try expect(h.chain[2 + 16] == 2 + 8); try expect(h.chain[2 + 8] == 2); @@ -111,13 +111,13 @@ test bulkAdd { const data = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; // one by one - var h: Self = .{}; + var h: Lookup = .{}; for (data, 0..) |_, i| { _ = h.add(data[i..], @intCast(i)); } // in bulk - var bh: Self = .{}; + var bh: Lookup = .{}; bh.bulkAdd(data, data.len, 0); try testing.expectEqualSlices(u16, &h.head, &bh.head); diff --git a/lib/std/compress/flate/SlidingWindow.zig b/lib/std/compress/flate/SlidingWindow.zig deleted file mode 100644 index ece907c32f..0000000000 --- a/lib/std/compress/flate/SlidingWindow.zig +++ /dev/null @@ -1,160 +0,0 @@ -//! Used in deflate (compression), holds uncompressed data form which Tokens are -//! produces. In combination with Lookup it is used to find matches in history data. -//! -const std = @import("std"); -const consts = @import("consts.zig"); - -const expect = testing.expect; -const assert = std.debug.assert; -const testing = std.testing; - -const hist_len = consts.history.len; -const buffer_len = 2 * hist_len; -const min_lookahead = consts.match.min_length + consts.match.max_length; -const max_rp = buffer_len - min_lookahead; - -const Self = @This(); - -buffer: [buffer_len]u8 = undefined, -wp: usize = 0, // write position -rp: usize = 0, // read position -fp: isize = 0, // last flush position, tokens are build from fp..rp - -/// Returns number of bytes written, or 0 if buffer is full and need to slide. -pub fn write(self: *Self, buf: []const u8) usize { - if (self.rp >= max_rp) return 0; // need to slide - - const n = @min(buf.len, buffer_len - self.wp); - @memcpy(self.buffer[self.wp .. self.wp + n], buf[0..n]); - self.wp += n; - return n; -} - -/// Slide buffer for hist_len. -/// Drops old history, preserves between hist_len and hist_len - min_lookahead. -/// Returns number of bytes removed. -pub fn slide(self: *Self) u16 { - assert(self.rp >= max_rp and self.wp >= self.rp); - const n = self.wp - hist_len; - @memcpy(self.buffer[0..n], self.buffer[hist_len..self.wp]); - self.rp -= hist_len; - self.wp -= hist_len; - self.fp -= hist_len; - return @intCast(n); -} - -/// Data from the current position (read position). Those part of the buffer is -/// not converted to tokens yet. -fn lookahead(self: *Self) []const u8 { - assert(self.wp >= self.rp); - return self.buffer[self.rp..self.wp]; -} - -/// Returns part of the lookahead buffer. If should_flush is set no lookahead is -/// preserved otherwise preserves enough data for the longest match. Returns -/// null if there is not enough data. -pub fn activeLookahead(self: *Self, should_flush: bool) ?[]const u8 { - const min: usize = if (should_flush) 0 else min_lookahead; - const lh = self.lookahead(); - return if (lh.len > min) lh else null; -} - -/// Advances read position, shrinks lookahead. -pub fn advance(self: *Self, n: u16) void { - assert(self.wp >= self.rp + n); - self.rp += n; -} - -/// Returns writable part of the buffer, where new uncompressed data can be -/// written. -pub fn writable(self: *Self) []u8 { - return self.buffer[self.wp..]; -} - -/// Notification of what part of writable buffer is filled with data. -pub fn written(self: *Self, n: usize) void { - self.wp += n; -} - -/// Finds match length between previous and current position. -/// Used in hot path! -pub fn match(self: *Self, prev_pos: u16, curr_pos: u16, min_len: u16) u16 { - const max_len: usize = @min(self.wp - curr_pos, consts.match.max_length); - // lookahead buffers from previous and current positions - const prev_lh = self.buffer[prev_pos..][0..max_len]; - const curr_lh = self.buffer[curr_pos..][0..max_len]; - - // If we already have match (min_len > 0), - // test the first byte above previous len a[min_len] != b[min_len] - // and then all the bytes from that position to zero. - // That is likely positions to find difference than looping from first bytes. - var i: usize = min_len; - if (i > 0) { - if (max_len <= i) return 0; - while (true) { - if (prev_lh[i] != curr_lh[i]) return 0; - if (i == 0) break; - i -= 1; - } - i = min_len; - } - while (i < max_len) : (i += 1) - if (prev_lh[i] != curr_lh[i]) break; - return if (i >= consts.match.min_length) @intCast(i) else 0; -} - -/// Current position of non-compressed data. Data before rp are already converted -/// to tokens. -pub fn pos(self: *Self) u16 { - return @intCast(self.rp); -} - -/// Notification that token list is cleared. -pub fn flush(self: *Self) void { - self.fp = @intCast(self.rp); -} - -/// Part of the buffer since last flush or null if there was slide in between (so -/// fp becomes negative). -pub fn tokensBuffer(self: *Self) ?[]const u8 { - assert(self.fp <= self.rp); - if (self.fp < 0) return null; - return self.buffer[@intCast(self.fp)..self.rp]; -} - -test match { - const data = "Blah blah blah blah blah!"; - var win: Self = .{}; - try expect(win.write(data) == data.len); - try expect(win.wp == data.len); - try expect(win.rp == 0); - - // length between l symbols - try expect(win.match(1, 6, 0) == 18); - try expect(win.match(1, 11, 0) == 13); - try expect(win.match(1, 16, 0) == 8); - try expect(win.match(1, 21, 0) == 0); - - // position 15 = "blah blah!" - // position 20 = "blah!" - try expect(win.match(15, 20, 0) == 4); - try expect(win.match(15, 20, 3) == 4); - try expect(win.match(15, 20, 4) == 0); -} - -test slide { - var win: Self = .{}; - win.wp = Self.buffer_len - 11; - win.rp = Self.buffer_len - 111; - win.buffer[win.rp] = 0xab; - try expect(win.lookahead().len == 100); - try expect(win.tokensBuffer().?.len == win.rp); - - const n = win.slide(); - try expect(n == 32757); - try expect(win.buffer[win.rp] == 0xab); - try expect(win.rp == Self.hist_len - 111); - try expect(win.wp == Self.hist_len - 11); - try expect(win.lookahead().len == 100); - try expect(win.tokensBuffer() == null); -} diff --git a/lib/std/compress/flate/Token.zig b/lib/std/compress/flate/Token.zig index a9641f6adc..293a786cef 100644 --- a/lib/std/compress/flate/Token.zig +++ b/lib/std/compress/flate/Token.zig @@ -6,7 +6,7 @@ const std = @import("std"); const assert = std.debug.assert; const print = std.debug.print; const expect = std.testing.expect; -const consts = @import("consts.zig").match; +const match = std.compress.flate.match; const Token = @This(); @@ -26,11 +26,11 @@ pub fn literal(t: Token) u8 { } pub fn distance(t: Token) u16 { - return @as(u16, t.dist) + consts.min_distance; + return @as(u16, t.dist) + match.min_distance; } pub fn length(t: Token) u16 { - return @as(u16, t.len_lit) + consts.base_length; + return @as(u16, t.len_lit) + match.base_length; } pub fn initLiteral(lit: u8) Token { @@ -40,12 +40,12 @@ pub fn initLiteral(lit: u8) Token { // distance range 1 - 32768, stored in dist as 0 - 32767 (u15) // length range 3 - 258, stored in len_lit as 0 - 255 (u8) pub fn initMatch(dist: u16, len: u16) Token { - assert(len >= consts.min_length and len <= consts.max_length); - assert(dist >= consts.min_distance and dist <= consts.max_distance); + assert(len >= match.min_length and len <= match.max_length); + assert(dist >= match.min_distance and dist <= match.max_distance); return .{ .kind = .match, - .dist = @intCast(dist - consts.min_distance), - .len_lit = @intCast(len - consts.base_length), + .dist = @intCast(dist - match.min_distance), + .len_lit = @intCast(len - match.base_length), }; } diff --git a/lib/std/compress/flate/bit_reader.zig b/lib/std/compress/flate/bit_reader.zig deleted file mode 100644 index 1e41f081c1..0000000000 --- a/lib/std/compress/flate/bit_reader.zig +++ /dev/null @@ -1,422 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const testing = std.testing; - -pub fn bitReader(comptime T: type, reader: anytype) BitReader(T, @TypeOf(reader)) { - return BitReader(T, @TypeOf(reader)).init(reader); -} - -pub fn BitReader64(comptime ReaderType: type) type { - return BitReader(u64, ReaderType); -} - -pub fn BitReader32(comptime ReaderType: type) type { - return BitReader(u32, ReaderType); -} - -/// Bit reader used during inflate (decompression). Has internal buffer of 64 -/// bits which shifts right after bits are consumed. Uses forward_reader to fill -/// that internal buffer when needed. -/// -/// readF is the core function. Supports few different ways of getting bits -/// controlled by flags. In hot path we try to avoid checking whether we need to -/// fill buffer from forward_reader by calling fill in advance and readF with -/// buffered flag set. -/// -pub fn BitReader(comptime T: type, comptime ReaderType: type) type { - assert(T == u32 or T == u64); - const t_bytes: usize = @sizeOf(T); - const Tshift = if (T == u64) u6 else u5; - - return struct { - // Underlying reader used for filling internal bits buffer - forward_reader: ReaderType = undefined, - // Internal buffer of 64 bits - bits: T = 0, - // Number of bits in the buffer - nbits: u32 = 0, - - const Self = @This(); - - pub const Error = ReaderType.Error || error{EndOfStream}; - - pub fn init(rdr: ReaderType) Self { - var self = Self{ .forward_reader = rdr }; - self.fill(1) catch {}; - return self; - } - - /// Try to have `nice` bits are available in buffer. Reads from - /// forward reader if there is no `nice` bits in buffer. Returns error - /// if end of forward stream is reached and internal buffer is empty. - /// It will not error if less than `nice` bits are in buffer, only when - /// all bits are exhausted. During inflate we usually know what is the - /// maximum bits for the next step but usually that step will need less - /// bits to decode. So `nice` is not hard limit, it will just try to have - /// that number of bits available. If end of forward stream is reached - /// it may be some extra zero bits in buffer. - pub inline fn fill(self: *Self, nice: u6) !void { - if (self.nbits >= nice and nice != 0) { - return; // We have enough bits - } - // Read more bits from forward reader - - // Number of empty bytes in bits, round nbits to whole bytes. - const empty_bytes = - @as(u8, if (self.nbits & 0x7 == 0) t_bytes else t_bytes - 1) - // 8 for 8, 16, 24..., 7 otherwise - (self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8 - - var buf: [t_bytes]u8 = [_]u8{0} ** t_bytes; - const bytes_read = self.forward_reader.readAll(buf[0..empty_bytes]) catch 0; - if (bytes_read > 0) { - const u: T = std.mem.readInt(T, buf[0..t_bytes], .little); - self.bits |= u << @as(Tshift, @intCast(self.nbits)); - self.nbits += 8 * @as(u8, @intCast(bytes_read)); - return; - } - - if (self.nbits == 0) - return error.EndOfStream; - } - - /// Read exactly buf.len bytes into buf. - pub fn readAll(self: *Self, buf: []u8) !void { - assert(self.alignBits() == 0); // internal bits must be at byte boundary - - // First read from internal bits buffer. - var n: usize = 0; - while (self.nbits > 0 and n < buf.len) { - buf[n] = try self.readF(u8, flag.buffered); - n += 1; - } - // Then use forward reader for all other bytes. - try self.forward_reader.readNoEof(buf[n..]); - } - - pub const flag = struct { - pub const peek: u3 = 0b001; // dont advance internal buffer, just get bits, leave them in buffer - pub const buffered: u3 = 0b010; // assume that there is no need to fill, fill should be called before - pub const reverse: u3 = 0b100; // bit reverse read bits - }; - - /// Alias for readF(U, 0). - pub fn read(self: *Self, comptime U: type) !U { - return self.readF(U, 0); - } - - /// Alias for readF with flag.peak set. - pub inline fn peekF(self: *Self, comptime U: type, comptime how: u3) !U { - return self.readF(U, how | flag.peek); - } - - /// Read with flags provided. - pub fn readF(self: *Self, comptime U: type, comptime how: u3) !U { - if (U == T) { - assert(how == 0); - assert(self.alignBits() == 0); - try self.fill(@bitSizeOf(T)); - if (self.nbits != @bitSizeOf(T)) return error.EndOfStream; - const v = self.bits; - self.nbits = 0; - self.bits = 0; - return v; - } - const n: Tshift = @bitSizeOf(U); - switch (how) { - 0 => { // `normal` read - try self.fill(n); // ensure that there are n bits in the buffer - const u: U = @truncate(self.bits); // get n bits - try self.shift(n); // advance buffer for n - return u; - }, - (flag.peek) => { // no shift, leave bits in the buffer - try self.fill(n); - return @truncate(self.bits); - }, - flag.buffered => { // no fill, assume that buffer has enough bits - const u: U = @truncate(self.bits); - try self.shift(n); - return u; - }, - (flag.reverse) => { // same as 0 with bit reverse - try self.fill(n); - const u: U = @truncate(self.bits); - try self.shift(n); - return @bitReverse(u); - }, - (flag.peek | flag.reverse) => { - try self.fill(n); - return @bitReverse(@as(U, @truncate(self.bits))); - }, - (flag.buffered | flag.reverse) => { - const u: U = @truncate(self.bits); - try self.shift(n); - return @bitReverse(u); - }, - (flag.peek | flag.buffered) => { - return @truncate(self.bits); - }, - (flag.peek | flag.buffered | flag.reverse) => { - return @bitReverse(@as(U, @truncate(self.bits))); - }, - } - } - - /// Read n number of bits. - /// Only buffered flag can be used in how. - pub fn readN(self: *Self, n: u4, comptime how: u3) !u16 { - switch (how) { - 0 => { - try self.fill(n); - }, - flag.buffered => {}, - else => unreachable, - } - const mask: u16 = (@as(u16, 1) << n) - 1; - const u: u16 = @as(u16, @truncate(self.bits)) & mask; - try self.shift(n); - return u; - } - - /// Advance buffer for n bits. - pub fn shift(self: *Self, n: Tshift) !void { - if (n > self.nbits) return error.EndOfStream; - self.bits >>= n; - self.nbits -= n; - } - - /// Skip n bytes. - pub fn skipBytes(self: *Self, n: u16) !void { - for (0..n) |_| { - try self.fill(8); - try self.shift(8); - } - } - - // Number of bits to align stream to the byte boundary. - fn alignBits(self: *Self) u3 { - return @intCast(self.nbits & 0x7); - } - - /// Align stream to the byte boundary. - pub fn alignToByte(self: *Self) void { - const ab = self.alignBits(); - if (ab > 0) self.shift(ab) catch unreachable; - } - - /// Skip zero terminated string. - pub fn skipStringZ(self: *Self) !void { - while (true) { - if (try self.readF(u8, 0) == 0) break; - } - } - - /// Read deflate fixed fixed code. - /// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code. - /// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12 - /// Lit Value Bits Codes - /// --------- ---- ----- - /// 0 - 143 8 00110000 through - /// 10111111 - /// 144 - 255 9 110010000 through - /// 111111111 - /// 256 - 279 7 0000000 through - /// 0010111 - /// 280 - 287 8 11000000 through - /// 11000111 - pub fn readFixedCode(self: *Self) !u16 { - try self.fill(7 + 2); - const code7 = try self.readF(u7, flag.buffered | flag.reverse); - if (code7 <= 0b0010_111) { // 7 bits, 256-279, codes 0000_000 - 0010_111 - return @as(u16, code7) + 256; - } else if (code7 <= 0b1011_111) { // 8 bits, 0-143, codes 0011_0000 through 1011_1111 - return (@as(u16, code7) << 1) + @as(u16, try self.readF(u1, flag.buffered)) - 0b0011_0000; - } else if (code7 <= 0b1100_011) { // 8 bit, 280-287, codes 1100_0000 - 1100_0111 - return (@as(u16, code7 - 0b1100000) << 1) + try self.readF(u1, flag.buffered) + 280; - } else { // 9 bit, 144-255, codes 1_1001_0000 - 1_1111_1111 - return (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, try self.readF(u2, flag.buffered | flag.reverse)) + 144; - } - } - }; -} - -test "readF" { - var fbs = std.io.fixedBufferStream(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 }); - var br = bitReader(u64, fbs.reader()); - const F = BitReader64(@TypeOf(fbs.reader())).flag; - - try testing.expectEqual(@as(u8, 48), br.nbits); - try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits); - - try testing.expect(try br.readF(u1, 0) == 0b0000_0001); - try testing.expect(try br.readF(u2, 0) == 0b0000_0001); - try testing.expectEqual(@as(u8, 48 - 3), br.nbits); - try testing.expectEqual(@as(u3, 5), br.alignBits()); - - try testing.expect(try br.readF(u8, F.peek) == 0b0001_1110); - try testing.expect(try br.readF(u9, F.peek) == 0b1_0001_1110); - try br.shift(9); - try testing.expectEqual(@as(u8, 36), br.nbits); - try testing.expectEqual(@as(u3, 4), br.alignBits()); - - try testing.expect(try br.readF(u4, 0) == 0b0100); - try testing.expectEqual(@as(u8, 32), br.nbits); - try testing.expectEqual(@as(u3, 0), br.alignBits()); - - try br.shift(1); - try testing.expectEqual(@as(u3, 7), br.alignBits()); - try br.shift(1); - try testing.expectEqual(@as(u3, 6), br.alignBits()); - br.alignToByte(); - try testing.expectEqual(@as(u3, 0), br.alignBits()); - - try testing.expectEqual(@as(u64, 0xc9), br.bits); - try testing.expectEqual(@as(u16, 0x9), try br.readN(4, 0)); - try testing.expectEqual(@as(u16, 0xc), try br.readN(4, 0)); -} - -test "read block type 1 data" { - inline for ([_]type{ u64, u32 }) |T| { - const data = [_]u8{ - 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 - 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, - 0x0c, 0x01, 0x02, 0x03, // - 0xaa, 0xbb, 0xcc, 0xdd, - }; - var fbs = std.io.fixedBufferStream(&data); - var br = bitReader(T, fbs.reader()); - const F = BitReader(T, @TypeOf(fbs.reader())).flag; - - try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal - try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type - - for ("Hello world\n") |c| { - try testing.expectEqual(@as(u8, c), try br.readF(u8, F.reverse) - 0x30); - } - try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block - br.alignToByte(); - try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0)); - try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0)); - try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0)); - } -} - -test "shift/fill" { - const data = [_]u8{ - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - }; - var fbs = std.io.fixedBufferStream(&data); - var br = bitReader(u64, fbs.reader()); - - try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits); - try br.shift(8); - try testing.expectEqual(@as(u64, 0x00_08_07_06_05_04_03_02), br.bits); - try br.fill(60); // fill with 1 byte - try testing.expectEqual(@as(u64, 0x01_08_07_06_05_04_03_02), br.bits); - try br.shift(8 * 4 + 4); - try testing.expectEqual(@as(u64, 0x00_00_00_00_00_10_80_70), br.bits); - - try br.fill(60); // fill with 4 bytes (shift by 4) - try testing.expectEqual(@as(u64, 0x00_50_40_30_20_10_80_70), br.bits); - try testing.expectEqual(@as(u8, 8 * 7 + 4), br.nbits); - - try br.shift(@intCast(br.nbits)); // clear buffer - try br.fill(8); // refill with the rest of the bytes - try testing.expectEqual(@as(u64, 0x00_00_00_00_00_08_07_06), br.bits); -} - -test "readAll" { - inline for ([_]type{ u64, u32 }) |T| { - const data = [_]u8{ - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - }; - var fbs = std.io.fixedBufferStream(&data); - var br = bitReader(T, fbs.reader()); - - switch (T) { - u64 => try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits), - u32 => try testing.expectEqual(@as(u32, 0x04_03_02_01), br.bits), - else => unreachable, - } - - var out: [16]u8 = undefined; - try br.readAll(out[0..]); - try testing.expect(br.nbits == 0); - try testing.expect(br.bits == 0); - - try testing.expectEqualSlices(u8, data[0..16], &out); - } -} - -test "readFixedCode" { - inline for ([_]type{ u64, u32 }) |T| { - const fixed_codes = @import("huffman_encoder.zig").fixed_codes; - - var fbs = std.io.fixedBufferStream(&fixed_codes); - var rdr = bitReader(T, fbs.reader()); - - for (0..286) |c| { - try testing.expectEqual(c, try rdr.readFixedCode()); - } - try testing.expect(rdr.nbits == 0); - } -} - -test "u32 leaves no bits on u32 reads" { - const data = [_]u8{ - 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - }; - var fbs = std.io.fixedBufferStream(&data); - var br = bitReader(u32, fbs.reader()); - - _ = try br.read(u3); - try testing.expectEqual(29, br.nbits); - br.alignToByte(); - try testing.expectEqual(24, br.nbits); - try testing.expectEqual(0x04_03_02_01, try br.read(u32)); - try testing.expectEqual(0, br.nbits); - try testing.expectEqual(0x08_07_06_05, try br.read(u32)); - try testing.expectEqual(0, br.nbits); - - _ = try br.read(u9); - try testing.expectEqual(23, br.nbits); - br.alignToByte(); - try testing.expectEqual(16, br.nbits); - try testing.expectEqual(0x0e_0d_0c_0b, try br.read(u32)); - try testing.expectEqual(0, br.nbits); -} - -test "u64 need fill after alignToByte" { - const data = [_]u8{ - 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - }; - - // without fill - var fbs = std.io.fixedBufferStream(&data); - var br = bitReader(u64, fbs.reader()); - _ = try br.read(u23); - try testing.expectEqual(41, br.nbits); - br.alignToByte(); - try testing.expectEqual(40, br.nbits); - try testing.expectEqual(0x06_05_04_03, try br.read(u32)); - try testing.expectEqual(8, br.nbits); - try testing.expectEqual(0x0a_09_08_07, try br.read(u32)); - try testing.expectEqual(32, br.nbits); - - // fill after align ensures all bits filled - fbs.reset(); - br = bitReader(u64, fbs.reader()); - _ = try br.read(u23); - try testing.expectEqual(41, br.nbits); - br.alignToByte(); - try br.fill(0); - try testing.expectEqual(64, br.nbits); - try testing.expectEqual(0x06_05_04_03, try br.read(u32)); - try testing.expectEqual(32, br.nbits); - try testing.expectEqual(0x0a_09_08_07, try br.read(u32)); - try testing.expectEqual(0, br.nbits); -} diff --git a/lib/std/compress/flate/bit_writer.zig b/lib/std/compress/flate/bit_writer.zig deleted file mode 100644 index b5d84c7e2a..0000000000 --- a/lib/std/compress/flate/bit_writer.zig +++ /dev/null @@ -1,99 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; - -/// Bit writer for use in deflate (compression). -/// -/// Has internal bits buffer of 64 bits and internal bytes buffer of 248 bytes. -/// When we accumulate 48 bits 6 bytes are moved to the bytes buffer. When we -/// accumulate 240 bytes they are flushed to the underlying inner_writer. -/// -pub fn BitWriter(comptime WriterType: type) type { - // buffer_flush_size indicates the buffer size - // after which bytes are flushed to the writer. - // Should preferably be a multiple of 6, since - // we accumulate 6 bytes between writes to the buffer. - const buffer_flush_size = 240; - - // buffer_size is the actual output byte buffer size. - // It must have additional headroom for a flush - // which can contain up to 8 bytes. - const buffer_size = buffer_flush_size + 8; - - return struct { - inner_writer: WriterType, - - // Data waiting to be written is bytes[0 .. nbytes] - // and then the low nbits of bits. Data is always written - // sequentially into the bytes array. - bits: u64 = 0, - nbits: u32 = 0, // number of bits - bytes: [buffer_size]u8 = undefined, - nbytes: u32 = 0, // number of bytes - - const Self = @This(); - - pub const Error = WriterType.Error || error{UnfinishedBits}; - - pub fn init(writer: WriterType) Self { - return .{ .inner_writer = writer }; - } - - pub fn setWriter(self: *Self, new_writer: WriterType) void { - //assert(self.bits == 0 and self.nbits == 0 and self.nbytes == 0); - self.inner_writer = new_writer; - } - - pub fn flush(self: *Self) Error!void { - var n = self.nbytes; - while (self.nbits != 0) { - self.bytes[n] = @as(u8, @truncate(self.bits)); - self.bits >>= 8; - if (self.nbits > 8) { // Avoid underflow - self.nbits -= 8; - } else { - self.nbits = 0; - } - n += 1; - } - self.bits = 0; - _ = try self.inner_writer.write(self.bytes[0..n]); - self.nbytes = 0; - } - - pub fn writeBits(self: *Self, b: u32, nb: u32) Error!void { - self.bits |= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits)); - self.nbits += nb; - if (self.nbits < 48) - return; - - var n = self.nbytes; - std.mem.writeInt(u64, self.bytes[n..][0..8], self.bits, .little); - n += 6; - if (n >= buffer_flush_size) { - _ = try self.inner_writer.write(self.bytes[0..n]); - n = 0; - } - self.nbytes = n; - self.bits >>= 48; - self.nbits -= 48; - } - - pub fn writeBytes(self: *Self, bytes: []const u8) Error!void { - var n = self.nbytes; - if (self.nbits & 7 != 0) { - return error.UnfinishedBits; - } - while (self.nbits != 0) { - self.bytes[n] = @as(u8, @truncate(self.bits)); - self.bits >>= 8; - self.nbits -= 8; - n += 1; - } - if (n != 0) { - _ = try self.inner_writer.write(self.bytes[0..n]); - } - self.nbytes = 0; - _ = try self.inner_writer.write(bytes); - } - }; -} diff --git a/lib/std/compress/flate/block_writer.zig b/lib/std/compress/flate/block_writer.zig deleted file mode 100644 index fa0d299e84..0000000000 --- a/lib/std/compress/flate/block_writer.zig +++ /dev/null @@ -1,706 +0,0 @@ -const std = @import("std"); -const io = std.io; -const assert = std.debug.assert; - -const hc = @import("huffman_encoder.zig"); -const consts = @import("consts.zig").huffman; -const Token = @import("Token.zig"); -const BitWriter = @import("bit_writer.zig").BitWriter; - -pub fn blockWriter(writer: anytype) BlockWriter(@TypeOf(writer)) { - return BlockWriter(@TypeOf(writer)).init(writer); -} - -/// Accepts list of tokens, decides what is best block type to write. What block -/// type will provide best compression. Writes header and body of the block. -/// -pub fn BlockWriter(comptime WriterType: type) type { - const BitWriterType = BitWriter(WriterType); - return struct { - const codegen_order = consts.codegen_order; - const end_code_mark = 255; - const Self = @This(); - - pub const Error = BitWriterType.Error; - bit_writer: BitWriterType, - - codegen_freq: [consts.codegen_code_count]u16 = undefined, - literal_freq: [consts.max_num_lit]u16 = undefined, - distance_freq: [consts.distance_code_count]u16 = undefined, - codegen: [consts.max_num_lit + consts.distance_code_count + 1]u8 = undefined, - literal_encoding: hc.LiteralEncoder = .{}, - distance_encoding: hc.DistanceEncoder = .{}, - codegen_encoding: hc.CodegenEncoder = .{}, - fixed_literal_encoding: hc.LiteralEncoder, - fixed_distance_encoding: hc.DistanceEncoder, - huff_distance: hc.DistanceEncoder, - - pub fn init(writer: WriterType) Self { - return .{ - .bit_writer = BitWriterType.init(writer), - .fixed_literal_encoding = hc.fixedLiteralEncoder(), - .fixed_distance_encoding = hc.fixedDistanceEncoder(), - .huff_distance = hc.huffmanDistanceEncoder(), - }; - } - - /// Flush intrenal bit buffer to the writer. - /// Should be called only when bit stream is at byte boundary. - /// - /// That is after final block; when last byte could be incomplete or - /// after stored block; which is aligned to the byte boundary (it has x - /// padding bits after first 3 bits). - pub fn flush(self: *Self) Error!void { - try self.bit_writer.flush(); - } - - pub fn setWriter(self: *Self, new_writer: WriterType) void { - self.bit_writer.setWriter(new_writer); - } - - fn writeCode(self: *Self, c: hc.HuffCode) Error!void { - try self.bit_writer.writeBits(c.code, c.len); - } - - // RFC 1951 3.2.7 specifies a special run-length encoding for specifying - // the literal and distance lengths arrays (which are concatenated into a single - // array). This method generates that run-length encoding. - // - // The result is written into the codegen array, and the frequencies - // of each code is written into the codegen_freq array. - // Codes 0-15 are single byte codes. Codes 16-18 are followed by additional - // information. Code bad_code is an end marker - // - // num_literals: The number of literals in literal_encoding - // num_distances: The number of distances in distance_encoding - // lit_enc: The literal encoder to use - // dist_enc: The distance encoder to use - fn generateCodegen( - self: *Self, - num_literals: u32, - num_distances: u32, - lit_enc: *hc.LiteralEncoder, - dist_enc: *hc.DistanceEncoder, - ) void { - for (self.codegen_freq, 0..) |_, i| { - self.codegen_freq[i] = 0; - } - - // Note that we are using codegen both as a temporary variable for holding - // a copy of the frequencies, and as the place where we put the result. - // This is fine because the output is always shorter than the input used - // so far. - var codegen = &self.codegen; // cache - // Copy the concatenated code sizes to codegen. Put a marker at the end. - var cgnl = codegen[0..num_literals]; - for (cgnl, 0..) |_, i| { - cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len)); - } - - cgnl = codegen[num_literals .. num_literals + num_distances]; - for (cgnl, 0..) |_, i| { - cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len)); - } - codegen[num_literals + num_distances] = end_code_mark; - - var size = codegen[0]; - var count: i32 = 1; - var out_index: u32 = 0; - var in_index: u32 = 1; - while (size != end_code_mark) : (in_index += 1) { - // INVARIANT: We have seen "count" copies of size that have not yet - // had output generated for them. - const next_size = codegen[in_index]; - if (next_size == size) { - count += 1; - continue; - } - // We need to generate codegen indicating "count" of size. - if (size != 0) { - codegen[out_index] = size; - out_index += 1; - self.codegen_freq[size] += 1; - count -= 1; - while (count >= 3) { - var n: i32 = 6; - if (n > count) { - n = count; - } - codegen[out_index] = 16; - out_index += 1; - codegen[out_index] = @as(u8, @intCast(n - 3)); - out_index += 1; - self.codegen_freq[16] += 1; - count -= n; - } - } else { - while (count >= 11) { - var n: i32 = 138; - if (n > count) { - n = count; - } - codegen[out_index] = 18; - out_index += 1; - codegen[out_index] = @as(u8, @intCast(n - 11)); - out_index += 1; - self.codegen_freq[18] += 1; - count -= n; - } - if (count >= 3) { - // 3 <= count <= 10 - codegen[out_index] = 17; - out_index += 1; - codegen[out_index] = @as(u8, @intCast(count - 3)); - out_index += 1; - self.codegen_freq[17] += 1; - count = 0; - } - } - count -= 1; - while (count >= 0) : (count -= 1) { - codegen[out_index] = size; - out_index += 1; - self.codegen_freq[size] += 1; - } - // Set up invariant for next time through the loop. - size = next_size; - count = 1; - } - // Marker indicating the end of the codegen. - codegen[out_index] = end_code_mark; - } - - const DynamicSize = struct { - size: u32, - num_codegens: u32, - }; - - // dynamicSize returns the size of dynamically encoded data in bits. - fn dynamicSize( - self: *Self, - lit_enc: *hc.LiteralEncoder, // literal encoder - dist_enc: *hc.DistanceEncoder, // distance encoder - extra_bits: u32, - ) DynamicSize { - var num_codegens = self.codegen_freq.len; - while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) { - num_codegens -= 1; - } - const header = 3 + 5 + 5 + 4 + (3 * num_codegens) + - self.codegen_encoding.bitLength(self.codegen_freq[0..]) + - self.codegen_freq[16] * 2 + - self.codegen_freq[17] * 3 + - self.codegen_freq[18] * 7; - const size = header + - lit_enc.bitLength(&self.literal_freq) + - dist_enc.bitLength(&self.distance_freq) + - extra_bits; - - return DynamicSize{ - .size = @as(u32, @intCast(size)), - .num_codegens = @as(u32, @intCast(num_codegens)), - }; - } - - // fixedSize returns the size of dynamically encoded data in bits. - fn fixedSize(self: *Self, extra_bits: u32) u32 { - return 3 + - self.fixed_literal_encoding.bitLength(&self.literal_freq) + - self.fixed_distance_encoding.bitLength(&self.distance_freq) + - extra_bits; - } - - const StoredSize = struct { - size: u32, - storable: bool, - }; - - // storedSizeFits calculates the stored size, including header. - // The function returns the size in bits and whether the block - // fits inside a single block. - fn storedSizeFits(in: ?[]const u8) StoredSize { - if (in == null) { - return .{ .size = 0, .storable = false }; - } - if (in.?.len <= consts.max_store_block_size) { - return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true }; - } - return .{ .size = 0, .storable = false }; - } - - // Write the header of a dynamic Huffman block to the output stream. - // - // num_literals: The number of literals specified in codegen - // num_distances: The number of distances specified in codegen - // num_codegens: The number of codegens used in codegen - // eof: Is it the end-of-file? (end of stream) - fn dynamicHeader( - self: *Self, - num_literals: u32, - num_distances: u32, - num_codegens: u32, - eof: bool, - ) Error!void { - const first_bits: u32 = if (eof) 5 else 4; - try self.bit_writer.writeBits(first_bits, 3); - try self.bit_writer.writeBits(num_literals - 257, 5); - try self.bit_writer.writeBits(num_distances - 1, 5); - try self.bit_writer.writeBits(num_codegens - 4, 4); - - var i: u32 = 0; - while (i < num_codegens) : (i += 1) { - const value = self.codegen_encoding.codes[codegen_order[i]].len; - try self.bit_writer.writeBits(value, 3); - } - - i = 0; - while (true) { - const code_word: u32 = @as(u32, @intCast(self.codegen[i])); - i += 1; - if (code_word == end_code_mark) { - break; - } - try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]); - - switch (code_word) { - 16 => { - try self.bit_writer.writeBits(self.codegen[i], 2); - i += 1; - }, - 17 => { - try self.bit_writer.writeBits(self.codegen[i], 3); - i += 1; - }, - 18 => { - try self.bit_writer.writeBits(self.codegen[i], 7); - i += 1; - }, - else => {}, - } - } - } - - fn storedHeader(self: *Self, length: usize, eof: bool) Error!void { - assert(length <= 65535); - const flag: u32 = if (eof) 1 else 0; - try self.bit_writer.writeBits(flag, 3); - try self.flush(); - const l: u16 = @intCast(length); - try self.bit_writer.writeBits(l, 16); - try self.bit_writer.writeBits(~l, 16); - } - - fn fixedHeader(self: *Self, eof: bool) Error!void { - // Indicate that we are a fixed Huffman block - var value: u32 = 2; - if (eof) { - value = 3; - } - try self.bit_writer.writeBits(value, 3); - } - - // Write a block of tokens with the smallest encoding. Will choose block type. - // The original input can be supplied, and if the huffman encoded data - // is larger than the original bytes, the data will be written as a - // stored block. - // If the input is null, the tokens will always be Huffman encoded. - pub fn write(self: *Self, tokens: []const Token, eof: bool, input: ?[]const u8) Error!void { - const lit_and_dist = self.indexTokens(tokens); - const num_literals = lit_and_dist.num_literals; - const num_distances = lit_and_dist.num_distances; - - var extra_bits: u32 = 0; - const ret = storedSizeFits(input); - const stored_size = ret.size; - const storable = ret.storable; - - if (storable) { - // We only bother calculating the costs of the extra bits required by - // the length of distance fields (which will be the same for both fixed - // and dynamic encoding), if we need to compare those two encodings - // against stored encoding. - var length_code: u16 = Token.length_codes_start + 8; - while (length_code < num_literals) : (length_code += 1) { - // First eight length codes have extra size = 0. - extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) * - @as(u32, @intCast(Token.lengthExtraBits(length_code))); - } - var distance_code: u16 = 4; - while (distance_code < num_distances) : (distance_code += 1) { - // First four distance codes have extra size = 0. - extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) * - @as(u32, @intCast(Token.distanceExtraBits(distance_code))); - } - } - - // Figure out smallest code. - // Fixed Huffman baseline. - var literal_encoding = &self.fixed_literal_encoding; - var distance_encoding = &self.fixed_distance_encoding; - var size = self.fixedSize(extra_bits); - - // Dynamic Huffman? - var num_codegens: u32 = 0; - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literal_encoding and the distance_encoding. - self.generateCodegen( - num_literals, - num_distances, - &self.literal_encoding, - &self.distance_encoding, - ); - self.codegen_encoding.generate(self.codegen_freq[0..], 7); - const dynamic_size = self.dynamicSize( - &self.literal_encoding, - &self.distance_encoding, - extra_bits, - ); - const dyn_size = dynamic_size.size; - num_codegens = dynamic_size.num_codegens; - - if (dyn_size < size) { - size = dyn_size; - literal_encoding = &self.literal_encoding; - distance_encoding = &self.distance_encoding; - } - - // Stored bytes? - if (storable and stored_size < size) { - try self.storedBlock(input.?, eof); - return; - } - - // Huffman. - if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) { - try self.fixedHeader(eof); - } else { - try self.dynamicHeader(num_literals, num_distances, num_codegens, eof); - } - - // Write the tokens. - try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes); - } - - pub fn storedBlock(self: *Self, input: []const u8, eof: bool) Error!void { - try self.storedHeader(input.len, eof); - try self.bit_writer.writeBytes(input); - } - - // writeBlockDynamic encodes a block using a dynamic Huffman table. - // This should be used if the symbols used have a disproportionate - // histogram distribution. - // If input is supplied and the compression savings are below 1/16th of the - // input size the block is stored. - fn dynamicBlock( - self: *Self, - tokens: []const Token, - eof: bool, - input: ?[]const u8, - ) Error!void { - const total_tokens = self.indexTokens(tokens); - const num_literals = total_tokens.num_literals; - const num_distances = total_tokens.num_distances; - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literal_encoding and the distance_encoding. - self.generateCodegen( - num_literals, - num_distances, - &self.literal_encoding, - &self.distance_encoding, - ); - self.codegen_encoding.generate(self.codegen_freq[0..], 7); - const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0); - const size = dynamic_size.size; - const num_codegens = dynamic_size.num_codegens; - - // Store bytes, if we don't get a reasonable improvement. - - const stored_size = storedSizeFits(input); - const ssize = stored_size.size; - const storable = stored_size.storable; - if (storable and ssize < (size + (size >> 4))) { - try self.storedBlock(input.?, eof); - return; - } - - // Write Huffman table. - try self.dynamicHeader(num_literals, num_distances, num_codegens, eof); - - // Write the tokens. - try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes); - } - - const TotalIndexedTokens = struct { - num_literals: u32, - num_distances: u32, - }; - - // Indexes a slice of tokens followed by an end_block_marker, and updates - // literal_freq and distance_freq, and generates literal_encoding - // and distance_encoding. - // The number of literal and distance tokens is returned. - fn indexTokens(self: *Self, tokens: []const Token) TotalIndexedTokens { - var num_literals: u32 = 0; - var num_distances: u32 = 0; - - for (self.literal_freq, 0..) |_, i| { - self.literal_freq[i] = 0; - } - for (self.distance_freq, 0..) |_, i| { - self.distance_freq[i] = 0; - } - - for (tokens) |t| { - if (t.kind == Token.Kind.literal) { - self.literal_freq[t.literal()] += 1; - continue; - } - self.literal_freq[t.lengthCode()] += 1; - self.distance_freq[t.distanceCode()] += 1; - } - // add end_block_marker token at the end - self.literal_freq[consts.end_block_marker] += 1; - - // get the number of literals - num_literals = @as(u32, @intCast(self.literal_freq.len)); - while (self.literal_freq[num_literals - 1] == 0) { - num_literals -= 1; - } - // get the number of distances - num_distances = @as(u32, @intCast(self.distance_freq.len)); - while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) { - num_distances -= 1; - } - if (num_distances == 0) { - // We haven't found a single match. If we want to go with the dynamic encoding, - // we should count at least one distance to be sure that the distance huffman tree could be encoded. - self.distance_freq[0] = 1; - num_distances = 1; - } - self.literal_encoding.generate(&self.literal_freq, 15); - self.distance_encoding.generate(&self.distance_freq, 15); - return TotalIndexedTokens{ - .num_literals = num_literals, - .num_distances = num_distances, - }; - } - - // Writes a slice of tokens to the output followed by and end_block_marker. - // codes for literal and distance encoding must be supplied. - fn writeTokens( - self: *Self, - tokens: []const Token, - le_codes: []hc.HuffCode, - oe_codes: []hc.HuffCode, - ) Error!void { - for (tokens) |t| { - if (t.kind == Token.Kind.literal) { - try self.writeCode(le_codes[t.literal()]); - continue; - } - - // Write the length - const le = t.lengthEncoding(); - try self.writeCode(le_codes[le.code]); - if (le.extra_bits > 0) { - try self.bit_writer.writeBits(le.extra_length, le.extra_bits); - } - - // Write the distance - const oe = t.distanceEncoding(); - try self.writeCode(oe_codes[oe.code]); - if (oe.extra_bits > 0) { - try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits); - } - } - // add end_block_marker at the end - try self.writeCode(le_codes[consts.end_block_marker]); - } - - // Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes - // if the results only gains very little from compression. - pub fn huffmanBlock(self: *Self, input: []const u8, eof: bool) Error!void { - // Add everything as literals - histogram(input, &self.literal_freq); - - self.literal_freq[consts.end_block_marker] = 1; - - const num_literals = consts.end_block_marker + 1; - self.distance_freq[0] = 1; - const num_distances = 1; - - self.literal_encoding.generate(&self.literal_freq, 15); - - // Figure out smallest code. - // Always use dynamic Huffman or Store - var num_codegens: u32 = 0; - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literal_encoding and the distance_encoding. - self.generateCodegen( - num_literals, - num_distances, - &self.literal_encoding, - &self.huff_distance, - ); - self.codegen_encoding.generate(self.codegen_freq[0..], 7); - const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0); - const size = dynamic_size.size; - num_codegens = dynamic_size.num_codegens; - - // Store bytes, if we don't get a reasonable improvement. - const stored_size_ret = storedSizeFits(input); - const ssize = stored_size_ret.size; - const storable = stored_size_ret.storable; - - if (storable and ssize < (size + (size >> 4))) { - try self.storedBlock(input, eof); - return; - } - - // Huffman. - try self.dynamicHeader(num_literals, num_distances, num_codegens, eof); - const encoding = self.literal_encoding.codes[0..257]; - - for (input) |t| { - const c = encoding[t]; - try self.bit_writer.writeBits(c.code, c.len); - } - try self.writeCode(encoding[consts.end_block_marker]); - } - - // histogram accumulates a histogram of b in h. - fn histogram(b: []const u8, h: *[286]u16) void { - // Clear histogram - for (h, 0..) |_, i| { - h[i] = 0; - } - - var lh = h.*[0..256]; - for (b) |t| { - lh[t] += 1; - } - } - }; -} - -// tests -const expect = std.testing.expect; -const fmt = std.fmt; -const testing = std.testing; -const ArrayList = std.ArrayList; - -const TestCase = @import("testdata/block_writer.zig").TestCase; -const testCases = @import("testdata/block_writer.zig").testCases; - -// tests if the writeBlock encoding has changed. -test "write" { - inline for (0..testCases.len) |i| { - try testBlock(testCases[i], .write_block); - } -} - -// tests if the writeBlockDynamic encoding has changed. -test "dynamicBlock" { - inline for (0..testCases.len) |i| { - try testBlock(testCases[i], .write_dyn_block); - } -} - -test "huffmanBlock" { - inline for (0..testCases.len) |i| { - try testBlock(testCases[i], .write_huffman_block); - } - try testBlock(.{ - .tokens = &[_]Token{}, - .input = "huffman-rand-max.input", - .want = "huffman-rand-max.{s}.expect", - }, .write_huffman_block); -} - -const TestFn = enum { - write_block, - write_dyn_block, // write dynamic block - write_huffman_block, - - fn to_s(self: TestFn) []const u8 { - return switch (self) { - .write_block => "wb", - .write_dyn_block => "dyn", - .write_huffman_block => "huff", - }; - } - - fn write( - comptime self: TestFn, - bw: anytype, - tok: []const Token, - input: ?[]const u8, - final: bool, - ) !void { - switch (self) { - .write_block => try bw.write(tok, final, input), - .write_dyn_block => try bw.dynamicBlock(tok, final, input), - .write_huffman_block => try bw.huffmanBlock(input.?, final), - } - try bw.flush(); - } -}; - -// testBlock tests a block against its references -// -// size -// 64K [file-name].input - input non compressed file -// 8.1K [file-name].golden - -// 78 [file-name].dyn.expect - output with writeBlockDynamic -// 78 [file-name].wb.expect - output with writeBlock -// 8.1K [file-name].huff.expect - output with writeBlockHuff -// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null -// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null -// -// wb - writeBlock -// dyn - writeBlockDynamic -// huff - writeBlockHuff -// -fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void { - if (tc.input.len != 0 and tc.want.len != 0) { - const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()}); - const input = @embedFile("testdata/block_writer/" ++ tc.input); - const want = @embedFile("testdata/block_writer/" ++ want_name); - try testWriteBlock(tfn, input, want, tc.tokens); - } - - if (tfn == .write_huffman_block) { - return; - } - - const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()}); - const want = @embedFile("testdata/block_writer/" ++ want_name_no_input); - try testWriteBlock(tfn, null, want, tc.tokens); -} - -// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output. -fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void { - var buf = ArrayList(u8).init(testing.allocator); - var bw = blockWriter(buf.writer()); - try tfn.write(&bw, tokens, input, false); - var got = buf.items; - try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result - try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set - // - // Test if the writer produces the same output after reset. - buf.deinit(); - buf = ArrayList(u8).init(testing.allocator); - defer buf.deinit(); - bw.setWriter(buf.writer()); - - try tfn.write(&bw, tokens, input, true); - try bw.flush(); - got = buf.items; - - try expect(got[0] & 1 == 1); // bfinal is set - buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices - try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result -} diff --git a/lib/std/compress/flate/consts.zig b/lib/std/compress/flate/consts.zig deleted file mode 100644 index b17083461b..0000000000 --- a/lib/std/compress/flate/consts.zig +++ /dev/null @@ -1,49 +0,0 @@ -pub const deflate = struct { - // Number of tokens to accumulate in deflate before starting block encoding. - // - // In zlib this depends on memlevel: 6 + memlevel, where default memlevel is - // 8 and max 9 that gives 14 or 15 bits. - pub const tokens = 1 << 15; -}; - -pub const match = struct { - pub const base_length = 3; // smallest match length per the RFC section 3.2.5 - pub const min_length = 4; // min length used in this algorithm - pub const max_length = 258; - - pub const min_distance = 1; - pub const max_distance = 32768; -}; - -pub const history = struct { - pub const len = match.max_distance; -}; - -pub const lookup = struct { - pub const bits = 15; - pub const len = 1 << bits; - pub const shift = 32 - bits; -}; - -pub const huffman = struct { - // The odd order in which the codegen code sizes are written. - pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; - // The number of codegen codes. - pub const codegen_code_count = 19; - - // The largest distance code. - pub const distance_code_count = 30; - - // Maximum number of literals. - pub const max_num_lit = 286; - - // Max number of frequencies used for a Huffman Code - // Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286). - // The largest of these is max_num_lit. - pub const max_num_frequencies = max_num_lit; - - // Biggest block size for uncompressed block. - pub const max_store_block_size = 65535; - // The special code used to mark the end of a block. - pub const end_block_marker = 256; -}; diff --git a/lib/std/compress/flate/container.zig b/lib/std/compress/flate/container.zig deleted file mode 100644 index fe6dec446d..0000000000 --- a/lib/std/compress/flate/container.zig +++ /dev/null @@ -1,208 +0,0 @@ -//! Container of the deflate bit stream body. Container adds header before -//! deflate bit stream and footer after. It can bi gzip, zlib or raw (no header, -//! no footer, raw bit stream). -//! -//! Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes -//! addler 32 checksum. -//! -//! Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes -//! crc32 checksum and 4 bytes of uncompressed data length. -//! -//! -//! rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4 -//! rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5 -//! - -const std = @import("std"); - -pub const Container = enum { - raw, // no header or footer - gzip, // gzip header and footer - zlib, // zlib header and footer - - pub fn size(w: Container) usize { - return headerSize(w) + footerSize(w); - } - - pub fn headerSize(w: Container) usize { - return switch (w) { - .gzip => 10, - .zlib => 2, - .raw => 0, - }; - } - - pub fn footerSize(w: Container) usize { - return switch (w) { - .gzip => 8, - .zlib => 4, - .raw => 0, - }; - } - - pub const list = [_]Container{ .raw, .gzip, .zlib }; - - pub const Error = error{ - BadGzipHeader, - BadZlibHeader, - WrongGzipChecksum, - WrongGzipSize, - WrongZlibChecksum, - }; - - pub fn writeHeader(comptime wrap: Container, writer: anytype) !void { - switch (wrap) { - .gzip => { - // GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5): - // - ID1 (IDentification 1), always 0x1f - // - ID2 (IDentification 2), always 0x8b - // - CM (Compression Method), always 8 = deflate - // - FLG (Flags), all set to 0 - // - 4 bytes, MTIME (Modification time), not used, all set to zero - // - XFL (eXtra FLags), all set to zero - // - OS (Operating System), 03 = Unix - const gzipHeader = [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 }; - try writer.writeAll(&gzipHeader); - }, - .zlib => { - // ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4): - // 1st byte: - // - First four bits is the CINFO (compression info), which is 7 for the default deflate window size. - // - The next four bits is the CM (compression method), which is 8 for deflate. - // 2nd byte: - // - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best. - // - The next bit, FDICT, is set if a dictionary is given. - // - The final five FCHECK bits form a mod-31 checksum. - // - // CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100 - const zlibHeader = [_]u8{ 0x78, 0b10_0_11100 }; - try writer.writeAll(&zlibHeader); - }, - .raw => {}, - } - } - - pub fn writeFooter(comptime wrap: Container, hasher: *Hasher(wrap), writer: anytype) !void { - var bits: [4]u8 = undefined; - switch (wrap) { - .gzip => { - // GZIP 8 bytes footer - // - 4 bytes, CRC32 (CRC-32) - // - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32 - std.mem.writeInt(u32, &bits, hasher.chksum(), .little); - try writer.writeAll(&bits); - - std.mem.writeInt(u32, &bits, hasher.bytesRead(), .little); - try writer.writeAll(&bits); - }, - .zlib => { - // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). - // 4 bytes of ADLER32 (Adler-32 checksum) - // Checksum value of the uncompressed data (excluding any - // dictionary data) computed according to Adler-32 - // algorithm. - std.mem.writeInt(u32, &bits, hasher.chksum(), .big); - try writer.writeAll(&bits); - }, - .raw => {}, - } - } - - pub fn parseHeader(comptime wrap: Container, reader: anytype) !void { - switch (wrap) { - .gzip => try parseGzipHeader(reader), - .zlib => try parseZlibHeader(reader), - .raw => {}, - } - } - - fn parseGzipHeader(reader: anytype) !void { - const magic1 = try reader.read(u8); - const magic2 = try reader.read(u8); - const method = try reader.read(u8); - const flags = try reader.read(u8); - try reader.skipBytes(6); // mtime(4), xflags, os - if (magic1 != 0x1f or magic2 != 0x8b or method != 0x08) - return error.BadGzipHeader; - // Flags description: https://www.rfc-editor.org/rfc/rfc1952.html#page-5 - if (flags != 0) { - if (flags & 0b0000_0100 != 0) { // FEXTRA - const extra_len = try reader.read(u16); - try reader.skipBytes(extra_len); - } - if (flags & 0b0000_1000 != 0) { // FNAME - try reader.skipStringZ(); - } - if (flags & 0b0001_0000 != 0) { // FCOMMENT - try reader.skipStringZ(); - } - if (flags & 0b0000_0010 != 0) { // FHCRC - try reader.skipBytes(2); - } - } - } - - fn parseZlibHeader(reader: anytype) !void { - const cm = try reader.read(u4); - const cinfo = try reader.read(u4); - _ = try reader.read(u8); - if (cm != 8 or cinfo > 7) { - return error.BadZlibHeader; - } - } - - pub fn parseFooter(comptime wrap: Container, hasher: *Hasher(wrap), reader: anytype) !void { - switch (wrap) { - .gzip => { - try reader.fill(0); - if (try reader.read(u32) != hasher.chksum()) return error.WrongGzipChecksum; - if (try reader.read(u32) != hasher.bytesRead()) return error.WrongGzipSize; - }, - .zlib => { - const chksum: u32 = @byteSwap(hasher.chksum()); - if (try reader.read(u32) != chksum) return error.WrongZlibChecksum; - }, - .raw => {}, - } - } - - pub fn Hasher(comptime wrap: Container) type { - const HasherType = switch (wrap) { - .gzip => std.hash.Crc32, - .zlib => std.hash.Adler32, - .raw => struct { - pub fn init() @This() { - return .{}; - } - }, - }; - - return struct { - hasher: HasherType = HasherType.init(), - bytes: usize = 0, - - const Self = @This(); - - pub fn update(self: *Self, buf: []const u8) void { - switch (wrap) { - .raw => {}, - else => { - self.hasher.update(buf); - self.bytes += buf.len; - }, - } - } - - pub fn chksum(self: *Self) u32 { - switch (wrap) { - .raw => return 0, - else => return self.hasher.final(), - } - } - - pub fn bytesRead(self: *Self) u32 { - return @truncate(self.bytes); - } - }; - } -}; diff --git a/lib/std/compress/flate/deflate.zig b/lib/std/compress/flate/deflate.zig deleted file mode 100644 index fd93236000..0000000000 --- a/lib/std/compress/flate/deflate.zig +++ /dev/null @@ -1,744 +0,0 @@ -const std = @import("std"); -const io = std.io; -const assert = std.debug.assert; -const testing = std.testing; -const expect = testing.expect; -const print = std.debug.print; - -const Token = @import("Token.zig"); -const consts = @import("consts.zig"); -const BlockWriter = @import("block_writer.zig").BlockWriter; -const Container = @import("container.zig").Container; -const SlidingWindow = @import("SlidingWindow.zig"); -const Lookup = @import("Lookup.zig"); - -pub const Options = struct { - level: Level = .default, -}; - -/// Trades between speed and compression size. -/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43) -/// levels 1-3 are using different algorithm to perform faster but with less -/// compression. That is not implemented here. -pub const Level = enum(u4) { - // zig fmt: off - fast = 0xb, level_4 = 4, - level_5 = 5, - default = 0xc, level_6 = 6, - level_7 = 7, - level_8 = 8, - best = 0xd, level_9 = 9, - // zig fmt: on -}; - -/// Algorithm knobs for each level. -const LevelArgs = struct { - good: u16, // Do less lookups if we already have match of this length. - nice: u16, // Stop looking for better match if we found match with at least this length. - lazy: u16, // Don't do lazy match find if got match with at least this length. - chain: u16, // How many lookups for previous match to perform. - - pub fn get(level: Level) LevelArgs { - // zig fmt: off - return switch (level) { - .fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 }, - .level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 }, - .default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 }, - .level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 }, - .level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 }, - .best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 }, - }; - // zig fmt: on - } -}; - -/// Compress plain data from reader into compressed stream written to writer. -pub fn compress(comptime container: Container, reader: anytype, writer: anytype, options: Options) !void { - var c = try compressor(container, writer, options); - try c.compress(reader); - try c.finish(); -} - -/// Create compressor for writer type. -pub fn compressor(comptime container: Container, writer: anytype, options: Options) !Compressor( - container, - @TypeOf(writer), -) { - return try Compressor(container, @TypeOf(writer)).init(writer, options); -} - -/// Compressor type. -pub fn Compressor(comptime container: Container, comptime WriterType: type) type { - const TokenWriterType = BlockWriter(WriterType); - return Deflate(container, WriterType, TokenWriterType); -} - -/// Default compression algorithm. Has two steps: tokenization and token -/// encoding. -/// -/// Tokenization takes uncompressed input stream and produces list of tokens. -/// Each token can be literal (byte of data) or match (backrefernce to previous -/// data with length and distance). Tokenization accumulators 32K tokens, when -/// full or `flush` is called tokens are passed to the `block_writer`. Level -/// defines how hard (how slow) it tries to find match. -/// -/// Block writer will decide which type of deflate block to write (stored, fixed, -/// dynamic) and encode tokens to the output byte stream. Client has to call -/// `finish` to write block with the final bit set. -/// -/// Container defines type of header and footer which can be gzip, zlib or raw. -/// They all share same deflate body. Raw has no header or footer just deflate -/// body. -/// -/// Compression algorithm explained in rfc-1951 (slightly edited for this case): -/// -/// The compressor uses a chained hash table `lookup` to find duplicated -/// strings, using a hash function that operates on 4-byte sequences. At any -/// given point during compression, let XYZW be the next 4 input bytes -/// (lookahead) to be examined (not necessarily all different, of course). -/// First, the compressor examines the hash chain for XYZW. If the chain is -/// empty, the compressor simply writes out X as a literal byte and advances -/// one byte in the input. If the hash chain is not empty, indicating that the -/// sequence XYZW (or, if we are unlucky, some other 4 bytes with the same -/// hash function value) has occurred recently, the compressor compares all -/// strings on the XYZW hash chain with the actual input data sequence -/// starting at the current point, and selects the longest match. -/// -/// To improve overall compression, the compressor defers the selection of -/// matches ("lazy matching"): after a match of length N has been found, the -/// compressor searches for a longer match starting at the next input byte. If -/// it finds a longer match, it truncates the previous match to a length of -/// one (thus producing a single literal byte) and then emits the longer -/// match. Otherwise, it emits the original match, and, as described above, -/// advances N bytes before continuing. -/// -/// -/// Allocates statically ~400K (192K lookup, 128K tokens, 64K window). -/// -/// Deflate function accepts BlockWriterType so we can change that in test to test -/// just tokenization part. -/// -fn Deflate(comptime container: Container, comptime WriterType: type, comptime BlockWriterType: type) type { - return struct { - lookup: Lookup = .{}, - win: SlidingWindow = .{}, - tokens: Tokens = .{}, - wrt: WriterType, - block_writer: BlockWriterType, - level: LevelArgs, - hasher: container.Hasher() = .{}, - - // Match and literal at the previous position. - // Used for lazy match finding in processWindow. - prev_match: ?Token = null, - prev_literal: ?u8 = null, - - const Self = @This(); - - pub fn init(wrt: WriterType, options: Options) !Self { - const self = Self{ - .wrt = wrt, - .block_writer = BlockWriterType.init(wrt), - .level = LevelArgs.get(options.level), - }; - try container.writeHeader(self.wrt); - return self; - } - - const FlushOption = enum { none, flush, final }; - - // Process data in window and create tokens. If token buffer is full - // flush tokens to the token writer. In the case of `flush` or `final` - // option it will process all data from the window. In the `none` case - // it will preserve some data for the next match. - fn tokenize(self: *Self, flush_opt: FlushOption) !void { - // flush - process all data from window - const should_flush = (flush_opt != .none); - - // While there is data in active lookahead buffer. - while (self.win.activeLookahead(should_flush)) |lh| { - var step: u16 = 1; // 1 in the case of literal, match length otherwise - const pos: u16 = self.win.pos(); - const literal = lh[0]; // literal at current position - const min_len: u16 = if (self.prev_match) |m| m.length() else 0; - - // Try to find match at least min_len long. - if (self.findMatch(pos, lh, min_len)) |match| { - // Found better match than previous. - try self.addPrevLiteral(); - - // Is found match length good enough? - if (match.length() >= self.level.lazy) { - // Don't try to lazy find better match, use this. - step = try self.addMatch(match); - } else { - // Store this match. - self.prev_literal = literal; - self.prev_match = match; - } - } else { - // There is no better match at current pos then it was previous. - // Write previous match or literal. - if (self.prev_match) |m| { - // Write match from previous position. - step = try self.addMatch(m) - 1; // we already advanced 1 from previous position - } else { - // No match at previous position. - // Write previous literal if any, and remember this literal. - try self.addPrevLiteral(); - self.prev_literal = literal; - } - } - // Advance window and add hashes. - self.windowAdvance(step, lh, pos); - } - - if (should_flush) { - // In the case of flushing, last few lookahead buffers were smaller then min match len. - // So only last literal can be unwritten. - assert(self.prev_match == null); - try self.addPrevLiteral(); - self.prev_literal = null; - - try self.flushTokens(flush_opt); - } - } - - fn windowAdvance(self: *Self, step: u16, lh: []const u8, pos: u16) void { - // current position is already added in findMatch - self.lookup.bulkAdd(lh[1..], step - 1, pos + 1); - self.win.advance(step); - } - - // Add previous literal (if any) to the tokens list. - fn addPrevLiteral(self: *Self) !void { - if (self.prev_literal) |l| try self.addToken(Token.initLiteral(l)); - } - - // Add match to the tokens list, reset prev pointers. - // Returns length of the added match. - fn addMatch(self: *Self, m: Token) !u16 { - try self.addToken(m); - self.prev_literal = null; - self.prev_match = null; - return m.length(); - } - - fn addToken(self: *Self, token: Token) !void { - self.tokens.add(token); - if (self.tokens.full()) try self.flushTokens(.none); - } - - // Finds largest match in the history window with the data at current pos. - fn findMatch(self: *Self, pos: u16, lh: []const u8, min_len: u16) ?Token { - var len: u16 = min_len; - // Previous location with the same hash (same 4 bytes). - var prev_pos = self.lookup.add(lh, pos); - // Last found match. - var match: ?Token = null; - - // How much back-references to try, performance knob. - var chain: usize = self.level.chain; - if (len >= self.level.good) { - // If we've got a match that's good enough, only look in 1/4 the chain. - chain >>= 2; - } - - // Hot path loop! - while (prev_pos > 0 and chain > 0) : (chain -= 1) { - const distance = pos - prev_pos; - if (distance > consts.match.max_distance) - break; - - const new_len = self.win.match(prev_pos, pos, len); - if (new_len > len) { - match = Token.initMatch(@intCast(distance), new_len); - if (new_len >= self.level.nice) { - // The match is good enough that we don't try to find a better one. - return match; - } - len = new_len; - } - prev_pos = self.lookup.prev(prev_pos); - } - - return match; - } - - fn flushTokens(self: *Self, flush_opt: FlushOption) !void { - // Pass tokens to the token writer - try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer()); - // Stored block ensures byte alignment. - // It has 3 bits (final, block_type) and then padding until byte boundary. - // After that everything is aligned to the boundary in the stored block. - // Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF. - // Last 4 bytes are byte aligned. - if (flush_opt == .flush) { - try self.block_writer.storedBlock("", false); - } - if (flush_opt != .none) { - // Safe to call only when byte aligned or it is OK to add - // padding bits (on last byte of the final block). - try self.block_writer.flush(); - } - // Reset internal tokens store. - self.tokens.reset(); - // Notify win that tokens are flushed. - self.win.flush(); - } - - // Slide win and if needed lookup tables. - fn slide(self: *Self) void { - const n = self.win.slide(); - self.lookup.slide(n); - } - - /// Compresses as much data as possible, stops when the reader becomes - /// empty. It will introduce some output latency (reading input without - /// producing all output) because some data are still in internal - /// buffers. - /// - /// It is up to the caller to call flush (if needed) or finish (required) - /// when is need to output any pending data or complete stream. - /// - pub fn compress(self: *Self, reader: anytype) !void { - while (true) { - // Fill window from reader - const buf = self.win.writable(); - if (buf.len == 0) { - try self.tokenize(.none); - self.slide(); - continue; - } - const n = try reader.readAll(buf); - self.hasher.update(buf[0..n]); - self.win.written(n); - // Process window - try self.tokenize(.none); - // Exit when no more data in reader - if (n < buf.len) break; - } - } - - /// Flushes internal buffers to the output writer. Outputs empty stored - /// block to sync bit stream to the byte boundary, so that the - /// decompressor can get all input data available so far. - /// - /// It is useful mainly in compressed network protocols, to ensure that - /// deflate bit stream can be used as byte stream. May degrade - /// compression so it should be used only when necessary. - /// - /// Completes the current deflate block and follows it with an empty - /// stored block that is three zero bits plus filler bits to the next - /// byte, followed by four bytes (00 00 ff ff). - /// - pub fn flush(self: *Self) !void { - try self.tokenize(.flush); - } - - /// Completes deflate bit stream by writing any pending data as deflate - /// final deflate block. HAS to be called once all data are written to - /// the compressor as a signal that next block has to have final bit - /// set. - /// - pub fn finish(self: *Self) !void { - try self.tokenize(.final); - try container.writeFooter(&self.hasher, self.wrt); - } - - /// Use another writer while preserving history. Most probably flush - /// should be called on old writer before setting new. - pub fn setWriter(self: *Self, new_writer: WriterType) void { - self.block_writer.setWriter(new_writer); - self.wrt = new_writer; - } - - // Writer interface - - pub const Writer = io.GenericWriter(*Self, Error, write); - pub const Error = BlockWriterType.Error; - - /// Write `input` of uncompressed data. - /// See compress. - pub fn write(self: *Self, input: []const u8) !usize { - var fbs = io.fixedBufferStream(input); - try self.compress(fbs.reader()); - return input.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } - }; -} - -// Tokens store -const Tokens = struct { - list: [consts.deflate.tokens]Token = undefined, - pos: usize = 0, - - fn add(self: *Tokens, t: Token) void { - self.list[self.pos] = t; - self.pos += 1; - } - - fn full(self: *Tokens) bool { - return self.pos == self.list.len; - } - - fn reset(self: *Tokens) void { - self.pos = 0; - } - - fn tokens(self: *Tokens) []const Token { - return self.list[0..self.pos]; - } -}; - -/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and -/// only performs Huffman entropy encoding. Results in faster compression, much -/// less memory requirements during compression but bigger compressed sizes. -pub const huffman = struct { - pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void { - var c = try huffman.compressor(container, writer); - try c.compress(reader); - try c.finish(); - } - - pub fn Compressor(comptime container: Container, comptime WriterType: type) type { - return SimpleCompressor(.huffman, container, WriterType); - } - - pub fn compressor(comptime container: Container, writer: anytype) !huffman.Compressor(container, @TypeOf(writer)) { - return try huffman.Compressor(container, @TypeOf(writer)).init(writer); - } -}; - -/// Creates store blocks only. Data are not compressed only packed into deflate -/// store blocks. That adds 9 bytes of header for each block. Max stored block -/// size is 64K. Block is emitted when flush is called on on finish. -pub const store = struct { - pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void { - var c = try store.compressor(container, writer); - try c.compress(reader); - try c.finish(); - } - - pub fn Compressor(comptime container: Container, comptime WriterType: type) type { - return SimpleCompressor(.store, container, WriterType); - } - - pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) { - return try store.Compressor(container, @TypeOf(writer)).init(writer); - } -}; - -const SimpleCompressorKind = enum { - huffman, - store, -}; - -fn simpleCompressor( - comptime kind: SimpleCompressorKind, - comptime container: Container, - writer: anytype, -) !SimpleCompressor(kind, container, @TypeOf(writer)) { - return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer); -} - -fn SimpleCompressor( - comptime kind: SimpleCompressorKind, - comptime container: Container, - comptime WriterType: type, -) type { - const BlockWriterType = BlockWriter(WriterType); - return struct { - buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes - wp: usize = 0, - - wrt: WriterType, - block_writer: BlockWriterType, - hasher: container.Hasher() = .{}, - - const Self = @This(); - - pub fn init(wrt: WriterType) !Self { - const self = Self{ - .wrt = wrt, - .block_writer = BlockWriterType.init(wrt), - }; - try container.writeHeader(self.wrt); - return self; - } - - pub fn flush(self: *Self) !void { - try self.flushBuffer(false); - try self.block_writer.storedBlock("", false); - try self.block_writer.flush(); - } - - pub fn finish(self: *Self) !void { - try self.flushBuffer(true); - try self.block_writer.flush(); - try container.writeFooter(&self.hasher, self.wrt); - } - - fn flushBuffer(self: *Self, final: bool) !void { - const buf = self.buffer[0..self.wp]; - switch (kind) { - .huffman => try self.block_writer.huffmanBlock(buf, final), - .store => try self.block_writer.storedBlock(buf, final), - } - self.wp = 0; - } - - // Writes all data from the input reader of uncompressed data. - // It is up to the caller to call flush or finish if there is need to - // output compressed blocks. - pub fn compress(self: *Self, reader: anytype) !void { - while (true) { - // read from rdr into buffer - const buf = self.buffer[self.wp..]; - if (buf.len == 0) { - try self.flushBuffer(false); - continue; - } - const n = try reader.readAll(buf); - self.hasher.update(buf[0..n]); - self.wp += n; - if (n < buf.len) break; // no more data in reader - } - } - - // Writer interface - - pub const Writer = io.GenericWriter(*Self, Error, write); - pub const Error = BlockWriterType.Error; - - // Write `input` of uncompressed data. - pub fn write(self: *Self, input: []const u8) !usize { - var fbs = io.fixedBufferStream(input); - try self.compress(fbs.reader()); - return input.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } - }; -} - -const builtin = @import("builtin"); - -test "tokenization" { - const L = Token.initLiteral; - const M = Token.initMatch; - - const cases = [_]struct { - data: []const u8, - tokens: []const Token, - }{ - .{ - .data = "Blah blah blah blah blah!", - .tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') }, - }, - .{ - .data = "ABCDEABCD ABCDEABCD", - .tokens = &[_]Token{ - L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '), - L('A'), M(10, 8), - }, - }, - }; - - for (cases) |c| { - inline for (Container.list) |container| { // for each wrapping - - var cw = io.countingWriter(io.null_writer); - const cww = cw.writer(); - var df = try Deflate(container, @TypeOf(cww), TestTokenWriter).init(cww, .{}); - - _ = try df.write(c.data); - try df.flush(); - - // df.token_writer.show(); - try expect(df.block_writer.pos == c.tokens.len); // number of tokens written - try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match - - try testing.expectEqual(container.headerSize(), cw.bytes_written); - try df.finish(); - try testing.expectEqual(container.size(), cw.bytes_written); - } - } -} - -// Tests that tokens written are equal to expected token list. -const TestTokenWriter = struct { - const Self = @This(); - - pos: usize = 0, - actual: [128]Token = undefined, - - pub fn init(_: anytype) Self { - return .{}; - } - pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void { - for (tokens) |t| { - self.actual[self.pos] = t; - self.pos += 1; - } - } - - pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {} - - pub fn get(self: *Self) []Token { - return self.actual[0..self.pos]; - } - - pub fn show(self: *Self) void { - print("\n", .{}); - for (self.get()) |t| { - t.show(); - } - } - - pub fn flush(_: *Self) !void {} -}; - -test "file tokenization" { - const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 }; - const cases = [_]struct { - data: []const u8, // uncompressed content - // expected number of tokens producet in deflate tokenization - tokens_count: [levels.len]usize = .{0} ** levels.len, - }{ - .{ - .data = @embedFile("testdata/rfc1951.txt"), - .tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 }, - }, - - .{ - .data = @embedFile("testdata/block_writer/huffman-null-max.input"), - .tokens_count = .{ 257, 257, 257, 257, 257, 257 }, - }, - .{ - .data = @embedFile("testdata/block_writer/huffman-pi.input"), - .tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 }, - }, - .{ - .data = @embedFile("testdata/block_writer/huffman-text.input"), - .tokens_count = .{ 235, 234, 234, 234, 234, 234 }, - }, - .{ - .data = @embedFile("testdata/fuzz/roundtrip1.input"), - .tokens_count = .{ 333, 331, 331, 331, 331, 331 }, - }, - .{ - .data = @embedFile("testdata/fuzz/roundtrip2.input"), - .tokens_count = .{ 334, 334, 334, 334, 334, 334 }, - }, - }; - - for (cases) |case| { // for each case - const data = case.data; - - for (levels, 0..) |level, i| { // for each compression level - var original = io.fixedBufferStream(data); - - // buffer for decompressed data - var al = std.ArrayList(u8).init(testing.allocator); - defer al.deinit(); - const writer = al.writer(); - - // create compressor - const WriterType = @TypeOf(writer); - const TokenWriter = TokenDecoder(@TypeOf(writer)); - var cmp = try Deflate(.raw, WriterType, TokenWriter).init(writer, .{ .level = level }); - - // Stream uncompressed `original` data to the compressor. It will - // produce tokens list and pass that list to the TokenDecoder. This - // TokenDecoder uses CircularBuffer from inflate to convert list of - // tokens back to the uncompressed stream. - try cmp.compress(original.reader()); - try cmp.flush(); - const expected_count = case.tokens_count[i]; - const actual = cmp.block_writer.tokens_count; - if (expected_count == 0) { - print("actual token count {d}\n", .{actual}); - } else { - try testing.expectEqual(expected_count, actual); - } - - try testing.expectEqual(data.len, al.items.len); - try testing.expectEqualSlices(u8, data, al.items); - } - } -} - -fn TokenDecoder(comptime WriterType: type) type { - return struct { - const CircularBuffer = @import("CircularBuffer.zig"); - hist: CircularBuffer = .{}, - wrt: WriterType, - tokens_count: usize = 0, - - const Self = @This(); - - pub fn init(wrt: WriterType) Self { - return .{ .wrt = wrt }; - } - - pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void { - self.tokens_count += tokens.len; - for (tokens) |t| { - switch (t.kind) { - .literal => self.hist.write(t.literal()), - .match => try self.hist.writeMatch(t.length(), t.distance()), - } - if (self.hist.free() < 285) try self.flushWin(); - } - try self.flushWin(); - } - - pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {} - - fn flushWin(self: *Self) !void { - while (true) { - const buf = self.hist.read(); - if (buf.len == 0) break; - try self.wrt.writeAll(buf); - } - } - - pub fn flush(_: *Self) !void {} - }; -} - -test "store simple compressor" { - const data = "Hello world!"; - const expected = [_]u8{ - 0x1, // block type 0, final bit set - 0xc, 0x0, // len = 12 - 0xf3, 0xff, // ~len - 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', // - //0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21, - }; - - var fbs = std.io.fixedBufferStream(data); - var al = std.ArrayList(u8).init(testing.allocator); - defer al.deinit(); - - var cmp = try store.compressor(.raw, al.writer()); - try cmp.compress(fbs.reader()); - try cmp.finish(); - try testing.expectEqualSlices(u8, &expected, al.items); - - fbs.reset(); - try al.resize(0); - - // huffman only compresoor will also emit store block for this small sample - var hc = try huffman.compressor(.raw, al.writer()); - try hc.compress(fbs.reader()); - try hc.finish(); - try testing.expectEqualSlices(u8, &expected, al.items); -} diff --git a/lib/std/compress/flate/huffman_decoder.zig b/lib/std/compress/flate/huffman_decoder.zig deleted file mode 100644 index abff915f76..0000000000 --- a/lib/std/compress/flate/huffman_decoder.zig +++ /dev/null @@ -1,302 +0,0 @@ -const std = @import("std"); -const testing = std.testing; - -pub const Symbol = packed struct { - pub const Kind = enum(u2) { - literal, - end_of_block, - match, - }; - - symbol: u8 = 0, // symbol from alphabet - code_bits: u4 = 0, // number of bits in code 0-15 - kind: Kind = .literal, - - code: u16 = 0, // huffman code of the symbol - next: u16 = 0, // pointer to the next symbol in linked list - // it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup - - // Sorting less than function. - pub fn asc(_: void, a: Symbol, b: Symbol) bool { - if (a.code_bits == b.code_bits) { - if (a.kind == b.kind) { - return a.symbol < b.symbol; - } - return @intFromEnum(a.kind) < @intFromEnum(b.kind); - } - return a.code_bits < b.code_bits; - } -}; - -pub const LiteralDecoder = HuffmanDecoder(286, 15, 9); -pub const DistanceDecoder = HuffmanDecoder(30, 15, 9); -pub const CodegenDecoder = HuffmanDecoder(19, 7, 7); - -pub const Error = error{ - InvalidCode, - OversubscribedHuffmanTree, - IncompleteHuffmanTree, - MissingEndOfBlockCode, -}; - -/// Creates huffman tree codes from list of code lengths (in `build`). -/// -/// `find` then finds symbol for code bits. Code can be any length between 1 and -/// 15 bits. When calling `find` we don't know how many bits will be used to -/// find symbol. When symbol is returned it has code_bits field which defines -/// how much we should advance in bit stream. -/// -/// Lookup table is used to map 15 bit int to symbol. Same symbol is written -/// many times in this table; 32K places for 286 (at most) symbols. -/// Small lookup table is optimization for faster search. -/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92) -/// with difference that we here use statically allocated arrays. -/// -fn HuffmanDecoder( - comptime alphabet_size: u16, - comptime max_code_bits: u4, - comptime lookup_bits: u4, -) type { - const lookup_shift = max_code_bits - lookup_bits; - - return struct { - // all symbols in alaphabet, sorted by code_len, symbol - symbols: [alphabet_size]Symbol = undefined, - // lookup table code -> symbol - lookup: [1 << lookup_bits]Symbol = undefined, - - const Self = @This(); - - /// Generates symbols and lookup tables from list of code lens for each symbol. - pub fn generate(self: *Self, lens: []const u4) !void { - try checkCompleteness(lens); - - // init alphabet with code_bits - for (self.symbols, 0..) |_, i| { - const cb: u4 = if (i < lens.len) lens[i] else 0; - self.symbols[i] = if (i < 256) - .{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb } - else if (i == 256) - .{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb } - else - .{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb }; - } - std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc); - - // reset lookup table - for (0..self.lookup.len) |i| { - self.lookup[i] = .{}; - } - - // assign code to symbols - // reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639 - var code: u16 = 0; - var idx: u16 = 0; - for (&self.symbols, 0..) |*sym, pos| { - if (sym.code_bits == 0) continue; // skip unused - sym.code = code; - - const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits)); - const next_idx = next_code >> lookup_shift; - - if (next_idx > self.lookup.len or idx >= self.lookup.len) break; - if (sym.code_bits <= lookup_bits) { - // fill small lookup table - for (idx..next_idx) |j| - self.lookup[j] = sym.*; - } else { - // insert into linked table starting at root - const root = &self.lookup[idx]; - const root_next = root.next; - root.next = @intCast(pos); - sym.next = root_next; - } - - idx = next_idx; - code = next_code; - } - } - - /// Given the list of code lengths check that it represents a canonical - /// Huffman code for n symbols. - /// - /// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340 - fn checkCompleteness(lens: []const u4) !void { - if (alphabet_size == 286) - if (lens[256] == 0) return error.MissingEndOfBlockCode; - - var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1); - var max: usize = 0; - for (lens) |n| { - if (n == 0) continue; - if (n > max) max = n; - count[n] += 1; - } - if (max == 0) // empty tree - return; - - // check for an over-subscribed or incomplete set of lengths - var left: usize = 1; // one possible code of zero length - for (1..count.len) |len| { - left <<= 1; // one more bit, double codes left - if (count[len] > left) - return error.OversubscribedHuffmanTree; - left -= count[len]; // deduct count from possible codes - } - if (left > 0) { // left > 0 means incomplete - // incomplete code ok only for single length 1 code - if (max_code_bits > 7 and max == count[0] + count[1]) return; - return error.IncompleteHuffmanTree; - } - } - - /// Finds symbol for lookup table code. - pub fn find(self: *Self, code: u16) !Symbol { - // try to find in lookup table - const idx = code >> lookup_shift; - const sym = self.lookup[idx]; - if (sym.code_bits != 0) return sym; - // if not use linked list of symbols with same prefix - return self.findLinked(code, sym.next); - } - - inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol { - var pos = start; - while (pos > 0) { - const sym = self.symbols[pos]; - const shift = max_code_bits - sym.code_bits; - // compare code_bits number of upper bits - if ((code ^ sym.code) >> shift == 0) return sym; - pos = sym.next; - } - return error.InvalidCode; - } - }; -} - -test "init/find" { - // example data from: https://youtu.be/SJPvNi4HrWQ?t=8423 - const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 }; - var h: CodegenDecoder = .{}; - try h.generate(&code_lens); - - const expected = [_]struct { - sym: Symbol, - code: u16, - }{ - .{ - .code = 0b00_00000, - .sym = .{ .symbol = 3, .code_bits = 2 }, - }, - .{ - .code = 0b01_00000, - .sym = .{ .symbol = 18, .code_bits = 2 }, - }, - .{ - .code = 0b100_0000, - .sym = .{ .symbol = 1, .code_bits = 3 }, - }, - .{ - .code = 0b101_0000, - .sym = .{ .symbol = 4, .code_bits = 3 }, - }, - .{ - .code = 0b110_0000, - .sym = .{ .symbol = 17, .code_bits = 3 }, - }, - .{ - .code = 0b1110_000, - .sym = .{ .symbol = 0, .code_bits = 4 }, - }, - .{ - .code = 0b1111_000, - .sym = .{ .symbol = 16, .code_bits = 4 }, - }, - }; - - // unused symbols - for (0..12) |i| { - try testing.expectEqual(0, h.symbols[i].code_bits); - } - // used, from index 12 - for (expected, 12..) |e, i| { - try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol); - try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits); - const sym_from_code = try h.find(e.code); - try testing.expectEqual(e.sym.symbol, sym_from_code.symbol); - } - - // All possible codes for each symbol. - // Lookup table has 126 elements, to cover all possible 7 bit codes. - for (0b0000_000..0b0100_000) |c| // 0..32 (32) - try testing.expectEqual(3, (try h.find(@intCast(c))).symbol); - - for (0b0100_000..0b1000_000) |c| // 32..64 (32) - try testing.expectEqual(18, (try h.find(@intCast(c))).symbol); - - for (0b1000_000..0b1010_000) |c| // 64..80 (16) - try testing.expectEqual(1, (try h.find(@intCast(c))).symbol); - - for (0b1010_000..0b1100_000) |c| // 80..96 (16) - try testing.expectEqual(4, (try h.find(@intCast(c))).symbol); - - for (0b1100_000..0b1110_000) |c| // 96..112 (16) - try testing.expectEqual(17, (try h.find(@intCast(c))).symbol); - - for (0b1110_000..0b1111_000) |c| // 112..120 (8) - try testing.expectEqual(0, (try h.find(@intCast(c))).symbol); - - for (0b1111_000..0b1_0000_000) |c| // 120...128 (8) - try testing.expectEqual(16, (try h.find(@intCast(c))).symbol); -} - -test "encode/decode literals" { - const LiteralEncoder = @import("huffman_encoder.zig").LiteralEncoder; - - for (1..286) |j| { // for all different number of codes - var enc: LiteralEncoder = .{}; - // create frequencies - var freq = [_]u16{0} ** 286; - freq[256] = 1; // ensure we have end of block code - for (&freq, 1..) |*f, i| { - if (i % j == 0) - f.* = @intCast(i); - } - - // encoder from frequencies - enc.generate(&freq, 15); - - // get code_lens from encoder - var code_lens = [_]u4{0} ** 286; - for (code_lens, 0..) |_, i| { - code_lens[i] = @intCast(enc.codes[i].len); - } - // generate decoder from code lens - var dec: LiteralDecoder = .{}; - try dec.generate(&code_lens); - - // expect decoder code to match original encoder code - for (dec.symbols) |s| { - if (s.code_bits == 0) continue; - const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code))); - const symbol: u16 = switch (s.kind) { - .literal => s.symbol, - .end_of_block => 256, - .match => @as(u16, s.symbol) + 257, - }; - - const c = enc.codes[symbol]; - try testing.expect(c.code == c_code); - } - - // find each symbol by code - for (enc.codes) |c| { - if (c.len == 0) continue; - - const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code))); - const s = try dec.find(s_code); - try testing.expect(s.code == s_code); - try testing.expect(s.code_bits == c.len); - } - } -} diff --git a/lib/std/compress/flate/huffman_encoder.zig b/lib/std/compress/flate/huffman_encoder.zig deleted file mode 100644 index 3e92e55a63..0000000000 --- a/lib/std/compress/flate/huffman_encoder.zig +++ /dev/null @@ -1,536 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const math = std.math; -const mem = std.mem; -const sort = std.sort; -const testing = std.testing; - -const consts = @import("consts.zig").huffman; - -const LiteralNode = struct { - literal: u16, - freq: u16, -}; - -// Describes the state of the constructed tree for a given depth. -const LevelInfo = struct { - // Our level. for better printing - level: u32, - - // The frequency of the last node at this level - last_freq: u32, - - // The frequency of the next character to add to this level - next_char_freq: u32, - - // The frequency of the next pair (from level below) to add to this level. - // Only valid if the "needed" value of the next lower level is 0. - next_pair_freq: u32, - - // The number of chains remaining to generate for this level before moving - // up to the next level - needed: u32, -}; - -// hcode is a huffman code with a bit code and bit length. -pub const HuffCode = struct { - code: u16 = 0, - len: u16 = 0, - - // set sets the code and length of an hcode. - fn set(self: *HuffCode, code: u16, length: u16) void { - self.len = length; - self.code = code; - } -}; - -pub fn HuffmanEncoder(comptime size: usize) type { - return struct { - codes: [size]HuffCode = undefined, - // Reusable buffer with the longest possible frequency table. - freq_cache: [consts.max_num_frequencies + 1]LiteralNode = undefined, - bit_count: [17]u32 = undefined, - lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate - lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate - - const Self = @This(); - - // Update this Huffman Code object to be the minimum code for the specified frequency count. - // - // freq An array of frequencies, in which frequency[i] gives the frequency of literal i. - // max_bits The maximum number of bits to use for any literal. - pub fn generate(self: *Self, freq: []u16, max_bits: u32) void { - var list = self.freq_cache[0 .. freq.len + 1]; - // Number of non-zero literals - var count: u32 = 0; - // Set list to be the set of all non-zero literals and their frequencies - for (freq, 0..) |f, i| { - if (f != 0) { - list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f }; - count += 1; - } else { - list[count] = LiteralNode{ .literal = 0x00, .freq = 0 }; - self.codes[i].len = 0; - } - } - list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 }; - - list = list[0..count]; - if (count <= 2) { - // Handle the small cases here, because they are awkward for the general case code. With - // two or fewer literals, everything has bit length 1. - for (list, 0..) |node, i| { - // "list" is in order of increasing literal value. - self.codes[node.literal].set(@as(u16, @intCast(i)), 1); - } - return; - } - self.lfs = list; - mem.sort(LiteralNode, self.lfs, {}, byFreq); - - // Get the number of literals for each bit count - const bit_count = self.bitCounts(list, max_bits); - // And do the assignment - self.assignEncodingAndSize(bit_count, list); - } - - pub fn bitLength(self: *Self, freq: []u16) u32 { - var total: u32 = 0; - for (freq, 0..) |f, i| { - if (f != 0) { - total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len)); - } - } - return total; - } - - // Return the number of literals assigned to each bit size in the Huffman encoding - // - // This method is only called when list.len >= 3 - // The cases of 0, 1, and 2 literals are handled by special case code. - // - // list: An array of the literals with non-zero frequencies - // and their associated frequencies. The array is in order of increasing - // frequency, and has as its last element a special element with frequency - // std.math.maxInt(i32) - // - // max_bits: The maximum number of bits that should be used to encode any literal. - // Must be less than 16. - // - // Returns an integer array in which array[i] indicates the number of literals - // that should be encoded in i bits. - fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 { - var max_bits = max_bits_to_use; - const n = list.len; - const max_bits_limit = 16; - - assert(max_bits < max_bits_limit); - - // The tree can't have greater depth than n - 1, no matter what. This - // saves a little bit of work in some small cases - max_bits = @min(max_bits, n - 1); - - // Create information about each of the levels. - // A bogus "Level 0" whose sole purpose is so that - // level1.prev.needed == 0. This makes level1.next_pair_freq - // be a legitimate value that never gets chosen. - var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo); - // leaf_counts[i] counts the number of literals at the left - // of ancestors of the rightmost node at level i. - // leaf_counts[i][j] is the number of literals at the left - // of the level j ancestor. - var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32); - - { - var level = @as(u32, 1); - while (level <= max_bits) : (level += 1) { - // For every level, the first two items are the first two characters. - // We initialize the levels as if we had already figured this out. - levels[level] = LevelInfo{ - .level = level, - .last_freq = list[1].freq, - .next_char_freq = list[2].freq, - .next_pair_freq = list[0].freq + list[1].freq, - .needed = 0, - }; - leaf_counts[level][level] = 2; - if (level == 1) { - levels[level].next_pair_freq = math.maxInt(i32); - } - } - } - - // We need a total of 2*n - 2 items at top level and have already generated 2. - levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4; - - { - var level = max_bits; - while (true) { - var l = &levels[level]; - if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) { - // We've run out of both leaves and pairs. - // End all calculations for this level. - // To make sure we never come back to this level or any lower level, - // set next_pair_freq impossibly large. - l.needed = 0; - levels[level + 1].next_pair_freq = math.maxInt(i32); - level += 1; - continue; - } - - const prev_freq = l.last_freq; - if (l.next_char_freq < l.next_pair_freq) { - // The next item on this row is a leaf node. - const next = leaf_counts[level][level] + 1; - l.last_freq = l.next_char_freq; - // Lower leaf_counts are the same of the previous node. - leaf_counts[level][level] = next; - if (next >= list.len) { - l.next_char_freq = maxNode().freq; - } else { - l.next_char_freq = list[next].freq; - } - } else { - // The next item on this row is a pair from the previous row. - // next_pair_freq isn't valid until we generate two - // more values in the level below - l.last_freq = l.next_pair_freq; - // Take leaf counts from the lower level, except counts[level] remains the same. - @memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]); - levels[l.level - 1].needed = 2; - } - - l.needed -= 1; - if (l.needed == 0) { - // We've done everything we need to do for this level. - // Continue calculating one level up. Fill in next_pair_freq - // of that level with the sum of the two nodes we've just calculated on - // this level. - if (l.level == max_bits) { - // All done! - break; - } - levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq; - level += 1; - } else { - // If we stole from below, move down temporarily to replenish it. - while (levels[level - 1].needed > 0) { - level -= 1; - if (level == 0) { - break; - } - } - } - } - } - - // Somethings is wrong if at the end, the top level is null or hasn't used - // all of the leaves. - assert(leaf_counts[max_bits][max_bits] == n); - - var bit_count = self.bit_count[0 .. max_bits + 1]; - var bits: u32 = 1; - const counts = &leaf_counts[max_bits]; - { - var level = max_bits; - while (level > 0) : (level -= 1) { - // counts[level] gives the number of literals requiring at least "bits" - // bits to encode. - bit_count[bits] = counts[level] - counts[level - 1]; - bits += 1; - if (level == 0) { - break; - } - } - } - return bit_count; - } - - // Look at the leaves and assign them a bit count and an encoding as specified - // in RFC 1951 3.2.2 - fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void { - var code = @as(u16, 0); - var list = list_arg; - - for (bit_count, 0..) |bits, n| { - code <<= 1; - if (n == 0 or bits == 0) { - continue; - } - // The literals list[list.len-bits] .. list[list.len-bits] - // are encoded using "bits" bits, and get the values - // code, code + 1, .... The code values are - // assigned in literal order (not frequency order). - const chunk = list[list.len - @as(u32, @intCast(bits)) ..]; - - self.lns = chunk; - mem.sort(LiteralNode, self.lns, {}, byLiteral); - - for (chunk) |node| { - self.codes[node.literal] = HuffCode{ - .code = bitReverse(u16, code, @as(u5, @intCast(n))), - .len = @as(u16, @intCast(n)), - }; - code += 1; - } - list = list[0 .. list.len - @as(u32, @intCast(bits))]; - } - } - }; -} - -fn maxNode() LiteralNode { - return LiteralNode{ - .literal = math.maxInt(u16), - .freq = math.maxInt(u16), - }; -} - -pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) { - return .{}; -} - -pub const LiteralEncoder = HuffmanEncoder(consts.max_num_frequencies); -pub const DistanceEncoder = HuffmanEncoder(consts.distance_code_count); -pub const CodegenEncoder = HuffmanEncoder(19); - -// Generates a HuffmanCode corresponding to the fixed literal table -pub fn fixedLiteralEncoder() LiteralEncoder { - var h: LiteralEncoder = undefined; - var ch: u16 = 0; - - while (ch < consts.max_num_frequencies) : (ch += 1) { - var bits: u16 = undefined; - var size: u16 = undefined; - switch (ch) { - 0...143 => { - // size 8, 000110000 .. 10111111 - bits = ch + 48; - size = 8; - }, - 144...255 => { - // size 9, 110010000 .. 111111111 - bits = ch + 400 - 144; - size = 9; - }, - 256...279 => { - // size 7, 0000000 .. 0010111 - bits = ch - 256; - size = 7; - }, - else => { - // size 8, 11000000 .. 11000111 - bits = ch + 192 - 280; - size = 8; - }, - } - h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size }; - } - return h; -} - -pub fn fixedDistanceEncoder() DistanceEncoder { - var h: DistanceEncoder = undefined; - for (h.codes, 0..) |_, ch| { - h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 }; - } - return h; -} - -pub fn huffmanDistanceEncoder() DistanceEncoder { - var distance_freq = [1]u16{0} ** consts.distance_code_count; - distance_freq[0] = 1; - // huff_distance is a static distance encoder used for huffman only encoding. - // It can be reused since we will not be encoding distance values. - var h: DistanceEncoder = .{}; - h.generate(distance_freq[0..], 15); - return h; -} - -fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool { - _ = context; - return a.literal < b.literal; -} - -fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool { - _ = context; - if (a.freq == b.freq) { - return a.literal < b.literal; - } - return a.freq < b.freq; -} - -test "generate a Huffman code from an array of frequencies" { - var freqs: [19]u16 = [_]u16{ - 8, // 0 - 1, // 1 - 1, // 2 - 2, // 3 - 5, // 4 - 10, // 5 - 9, // 6 - 1, // 7 - 0, // 8 - 0, // 9 - 0, // 10 - 0, // 11 - 0, // 12 - 0, // 13 - 0, // 14 - 0, // 15 - 1, // 16 - 3, // 17 - 5, // 18 - }; - - var enc = huffmanEncoder(19); - enc.generate(freqs[0..], 7); - - try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..])); - - try testing.expectEqual(@as(usize, 3), enc.codes[0].len); - try testing.expectEqual(@as(usize, 6), enc.codes[1].len); - try testing.expectEqual(@as(usize, 6), enc.codes[2].len); - try testing.expectEqual(@as(usize, 5), enc.codes[3].len); - try testing.expectEqual(@as(usize, 3), enc.codes[4].len); - try testing.expectEqual(@as(usize, 2), enc.codes[5].len); - try testing.expectEqual(@as(usize, 2), enc.codes[6].len); - try testing.expectEqual(@as(usize, 6), enc.codes[7].len); - try testing.expectEqual(@as(usize, 0), enc.codes[8].len); - try testing.expectEqual(@as(usize, 0), enc.codes[9].len); - try testing.expectEqual(@as(usize, 0), enc.codes[10].len); - try testing.expectEqual(@as(usize, 0), enc.codes[11].len); - try testing.expectEqual(@as(usize, 0), enc.codes[12].len); - try testing.expectEqual(@as(usize, 0), enc.codes[13].len); - try testing.expectEqual(@as(usize, 0), enc.codes[14].len); - try testing.expectEqual(@as(usize, 0), enc.codes[15].len); - try testing.expectEqual(@as(usize, 6), enc.codes[16].len); - try testing.expectEqual(@as(usize, 5), enc.codes[17].len); - try testing.expectEqual(@as(usize, 3), enc.codes[18].len); - - try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code); - try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code); - try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code); - try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code); - try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code); - try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code); - try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code); - try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code); - try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code); - try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code); - try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code); -} - -test "generate a Huffman code for the fixed literal table specific to Deflate" { - const enc = fixedLiteralEncoder(); - for (enc.codes) |c| { - switch (c.len) { - 7 => { - const v = @bitReverse(@as(u7, @intCast(c.code))); - try testing.expect(v <= 0b0010111); - }, - 8 => { - const v = @bitReverse(@as(u8, @intCast(c.code))); - try testing.expect((v >= 0b000110000 and v <= 0b10111111) or - (v >= 0b11000000 and v <= 11000111)); - }, - 9 => { - const v = @bitReverse(@as(u9, @intCast(c.code))); - try testing.expect(v >= 0b110010000 and v <= 0b111111111); - }, - else => unreachable, - } - } -} - -test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" { - const enc = fixedDistanceEncoder(); - for (enc.codes) |c| { - const v = @bitReverse(@as(u5, @intCast(c.code))); - try testing.expect(v <= 29); - try testing.expect(c.len == 5); - } -} - -// Reverse bit-by-bit a N-bit code. -fn bitReverse(comptime T: type, value: T, n: usize) T { - const r = @bitReverse(value); - return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n)); -} - -test bitReverse { - const ReverseBitsTest = struct { - in: u16, - bit_count: u5, - out: u16, - }; - - const reverse_bits_tests = [_]ReverseBitsTest{ - .{ .in = 1, .bit_count = 1, .out = 1 }, - .{ .in = 1, .bit_count = 2, .out = 2 }, - .{ .in = 1, .bit_count = 3, .out = 4 }, - .{ .in = 1, .bit_count = 4, .out = 8 }, - .{ .in = 1, .bit_count = 5, .out = 16 }, - .{ .in = 17, .bit_count = 5, .out = 17 }, - .{ .in = 257, .bit_count = 9, .out = 257 }, - .{ .in = 29, .bit_count = 5, .out = 23 }, - }; - - for (reverse_bits_tests) |h| { - const v = bitReverse(u16, h.in, h.bit_count); - try std.testing.expectEqual(h.out, v); - } -} - -test "fixedLiteralEncoder codes" { - var al = std.ArrayList(u8).init(testing.allocator); - defer al.deinit(); - var bw = std.io.bitWriter(.little, al.writer()); - - const f = fixedLiteralEncoder(); - for (f.codes) |c| { - try bw.writeBits(c.code, c.len); - } - try testing.expectEqualSlices(u8, &fixed_codes, al.items); -} - -pub const fixed_codes = [_]u8{ - 0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100, - 0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100, - 0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010, - 0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010, - 0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010, - 0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010, - 0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110, - 0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110, - 0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110, - 0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110, - 0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001, - 0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001, - 0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001, - 0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001, - 0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101, - 0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101, - 0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101, - 0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101, - 0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100, - 0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110, - 0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100, - 0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101, - 0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010, - 0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110, - 0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110, - 0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111, - 0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011, - 0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101, - 0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110, - 0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100, - 0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101, - 0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010, - 0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110, - 0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000, - 0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010, - 0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100, - 0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011, - 0b10100011, -}; diff --git a/lib/std/compress/flate/inflate.zig b/lib/std/compress/flate/inflate.zig deleted file mode 100644 index 2fcf3cafd4..0000000000 --- a/lib/std/compress/flate/inflate.zig +++ /dev/null @@ -1,570 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const testing = std.testing; - -const hfd = @import("huffman_decoder.zig"); -const BitReader = @import("bit_reader.zig").BitReader; -const CircularBuffer = @import("CircularBuffer.zig"); -const Container = @import("container.zig").Container; -const Token = @import("Token.zig"); -const codegen_order = @import("consts.zig").huffman.codegen_order; - -/// Decompresses deflate bit stream `reader` and writes uncompressed data to the -/// `writer` stream. -pub fn decompress(comptime container: Container, reader: anytype, writer: anytype) !void { - var d = decompressor(container, reader); - try d.decompress(writer); -} - -/// Inflate decompressor for the reader type. -pub fn decompressor(comptime container: Container, reader: anytype) Decompressor(container, @TypeOf(reader)) { - return Decompressor(container, @TypeOf(reader)).init(reader); -} - -pub fn Decompressor(comptime container: Container, comptime ReaderType: type) type { - // zlib has 4 bytes footer, lookahead of 4 bytes ensures that we will not overshoot. - // gzip has 8 bytes footer so we will not overshoot even with 8 bytes of lookahead. - // For raw deflate there is always possibility of overshot so we use 8 bytes lookahead. - const lookahead: type = if (container == .zlib) u32 else u64; - return Inflate(container, lookahead, ReaderType); -} - -/// Inflate decompresses deflate bit stream. Reads compressed data from reader -/// provided in init. Decompressed data are stored in internal hist buffer and -/// can be accesses iterable `next` or reader interface. -/// -/// Container defines header/footer wrapper around deflate bit stream. Can be -/// gzip or zlib. -/// -/// Deflate bit stream consists of multiple blocks. Block can be one of three types: -/// * stored, non compressed, max 64k in size -/// * fixed, huffman codes are predefined -/// * dynamic, huffman code tables are encoded at the block start -/// -/// `step` function runs decoder until internal `hist` buffer is full. Client -/// than needs to read that data in order to proceed with decoding. -/// -/// Allocates 74.5K of internal buffers, most important are: -/// * 64K for history (CircularBuffer) -/// * ~10K huffman decoders (Literal and DistanceDecoder) -/// -pub fn Inflate(comptime container: Container, comptime LookaheadType: type, comptime ReaderType: type) type { - assert(LookaheadType == u32 or LookaheadType == u64); - const BitReaderType = BitReader(LookaheadType, ReaderType); - - return struct { - //const BitReaderType = BitReader(ReaderType); - const F = BitReaderType.flag; - - bits: BitReaderType = .{}, - hist: CircularBuffer = .{}, - // Hashes, produces checkusm, of uncompressed data for gzip/zlib footer. - hasher: container.Hasher() = .{}, - - // dynamic block huffman code decoders - lit_dec: hfd.LiteralDecoder = .{}, // literals - dst_dec: hfd.DistanceDecoder = .{}, // distances - - // current read state - bfinal: u1 = 0, - block_type: u2 = 0b11, - state: ReadState = .protocol_header, - - const ReadState = enum { - protocol_header, - block_header, - block, - protocol_footer, - end, - }; - - const Self = @This(); - - pub const Error = BitReaderType.Error || Container.Error || hfd.Error || error{ - InvalidCode, - InvalidMatch, - InvalidBlockType, - WrongStoredBlockNlen, - InvalidDynamicBlockHeader, - }; - - pub fn init(rt: ReaderType) Self { - return .{ .bits = BitReaderType.init(rt) }; - } - - fn blockHeader(self: *Self) !void { - self.bfinal = try self.bits.read(u1); - self.block_type = try self.bits.read(u2); - } - - fn storedBlock(self: *Self) !bool { - self.bits.alignToByte(); // skip padding until byte boundary - // everything after this is byte aligned in stored block - var len = try self.bits.read(u16); - const nlen = try self.bits.read(u16); - if (len != ~nlen) return error.WrongStoredBlockNlen; - - while (len > 0) { - const buf = self.hist.getWritable(len); - try self.bits.readAll(buf); - len -= @intCast(buf.len); - } - return true; - } - - fn fixedBlock(self: *Self) !bool { - while (!self.hist.full()) { - const code = try self.bits.readFixedCode(); - switch (code) { - 0...255 => self.hist.write(@intCast(code)), - 256 => return true, // end of block - 257...285 => try self.fixedDistanceCode(@intCast(code - 257)), - else => return error.InvalidCode, - } - } - return false; - } - - // Handles fixed block non literal (length) code. - // Length code is followed by 5 bits of distance code. - fn fixedDistanceCode(self: *Self, code: u8) !void { - try self.bits.fill(5 + 5 + 13); - const length = try self.decodeLength(code); - const distance = try self.decodeDistance(try self.bits.readF(u5, F.buffered | F.reverse)); - try self.hist.writeMatch(length, distance); - } - - inline fn decodeLength(self: *Self, code: u8) !u16 { - if (code > 28) return error.InvalidCode; - const ml = Token.matchLength(code); - return if (ml.extra_bits == 0) // 0 - 5 extra bits - ml.base - else - ml.base + try self.bits.readN(ml.extra_bits, F.buffered); - } - - fn decodeDistance(self: *Self, code: u8) !u16 { - if (code > 29) return error.InvalidCode; - const md = Token.matchDistance(code); - return if (md.extra_bits == 0) // 0 - 13 extra bits - md.base - else - md.base + try self.bits.readN(md.extra_bits, F.buffered); - } - - fn dynamicBlockHeader(self: *Self) !void { - const hlit: u16 = @as(u16, try self.bits.read(u5)) + 257; // number of ll code entries present - 257 - const hdist: u16 = @as(u16, try self.bits.read(u5)) + 1; // number of distance code entries - 1 - const hclen: u8 = @as(u8, try self.bits.read(u4)) + 4; // hclen + 4 code lengths are encoded - - if (hlit > 286 or hdist > 30) - return error.InvalidDynamicBlockHeader; - - // lengths for code lengths - var cl_lens = [_]u4{0} ** 19; - for (0..hclen) |i| { - cl_lens[codegen_order[i]] = try self.bits.read(u3); - } - var cl_dec: hfd.CodegenDecoder = .{}; - try cl_dec.generate(&cl_lens); - - // decoded code lengths - var dec_lens = [_]u4{0} ** (286 + 30); - var pos: usize = 0; - while (pos < hlit + hdist) { - const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse)); - try self.bits.shift(sym.code_bits); - pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos); - } - if (pos > hlit + hdist) { - return error.InvalidDynamicBlockHeader; - } - - // literal code lengths to literal decoder - try self.lit_dec.generate(dec_lens[0..hlit]); - - // distance code lengths to distance decoder - try self.dst_dec.generate(dec_lens[hlit .. hlit + hdist]); - } - - // Decode code length symbol to code length. Writes decoded length into - // lens slice starting at position pos. Returns number of positions - // advanced. - fn dynamicCodeLength(self: *Self, code: u16, lens: []u4, pos: usize) !usize { - if (pos >= lens.len) - return error.InvalidDynamicBlockHeader; - - switch (code) { - 0...15 => { - // Represent code lengths of 0 - 15 - lens[pos] = @intCast(code); - return 1; - }, - 16 => { - // Copy the previous code length 3 - 6 times. - // The next 2 bits indicate repeat length - const n: u8 = @as(u8, try self.bits.read(u2)) + 3; - if (pos == 0 or pos + n > lens.len) - return error.InvalidDynamicBlockHeader; - for (0..n) |i| { - lens[pos + i] = lens[pos + i - 1]; - } - return n; - }, - // Repeat a code length of 0 for 3 - 10 times. (3 bits of length) - 17 => return @as(u8, try self.bits.read(u3)) + 3, - // Repeat a code length of 0 for 11 - 138 times (7 bits of length) - 18 => return @as(u8, try self.bits.read(u7)) + 11, - else => return error.InvalidDynamicBlockHeader, - } - } - - // In larger archives most blocks are usually dynamic, so decompression - // performance depends on this function. - fn dynamicBlock(self: *Self) !bool { - // Hot path loop! - while (!self.hist.full()) { - try self.bits.fill(15); // optimization so other bit reads can be buffered (avoiding one `if` in hot path) - const sym = try self.decodeSymbol(&self.lit_dec); - - switch (sym.kind) { - .literal => self.hist.write(sym.symbol), - .match => { // Decode match backreference - // fill so we can use buffered reads - if (LookaheadType == u32) - try self.bits.fill(5 + 15) - else - try self.bits.fill(5 + 15 + 13); - const length = try self.decodeLength(sym.symbol); - const dsm = try self.decodeSymbol(&self.dst_dec); - if (LookaheadType == u32) try self.bits.fill(13); - const distance = try self.decodeDistance(dsm.symbol); - try self.hist.writeMatch(length, distance); - }, - .end_of_block => return true, - } - } - return false; - } - - // Peek 15 bits from bits reader (maximum code len is 15 bits). Use - // decoder to find symbol for that code. We then know how many bits is - // used. Shift bit reader for that much bits, those bits are used. And - // return symbol. - fn decodeSymbol(self: *Self, decoder: anytype) !hfd.Symbol { - const sym = try decoder.find(try self.bits.peekF(u15, F.buffered | F.reverse)); - try self.bits.shift(sym.code_bits); - return sym; - } - - fn step(self: *Self) !void { - switch (self.state) { - .protocol_header => { - try container.parseHeader(&self.bits); - self.state = .block_header; - }, - .block_header => { - try self.blockHeader(); - self.state = .block; - if (self.block_type == 2) try self.dynamicBlockHeader(); - }, - .block => { - const done = switch (self.block_type) { - 0 => try self.storedBlock(), - 1 => try self.fixedBlock(), - 2 => try self.dynamicBlock(), - else => return error.InvalidBlockType, - }; - if (done) { - self.state = if (self.bfinal == 1) .protocol_footer else .block_header; - } - }, - .protocol_footer => { - self.bits.alignToByte(); - try container.parseFooter(&self.hasher, &self.bits); - self.state = .end; - }, - .end => {}, - } - } - - /// Replaces the inner reader with new reader. - pub fn setReader(self: *Self, new_reader: ReaderType) void { - self.bits.forward_reader = new_reader; - if (self.state == .end or self.state == .protocol_footer) { - self.state = .protocol_header; - } - } - - // Reads all compressed data from the internal reader and outputs plain - // (uncompressed) data to the provided writer. - pub fn decompress(self: *Self, writer: anytype) !void { - while (try self.next()) |buf| { - try writer.writeAll(buf); - } - } - - /// Returns the number of bytes that have been read from the internal - /// reader but not yet consumed by the decompressor. - pub fn unreadBytes(self: Self) usize { - // There can be no error here: the denominator is not zero, and - // overflow is not possible since the type is unsigned. - return std.math.divCeil(usize, self.bits.nbits, 8) catch unreachable; - } - - // Iterator interface - - /// Can be used in iterator like loop without memcpy to another buffer: - /// while (try inflate.next()) |buf| { ... } - pub fn next(self: *Self) Error!?[]const u8 { - const out = try self.get(0); - if (out.len == 0) return null; - return out; - } - - /// Returns decompressed data from internal sliding window buffer. - /// Returned buffer can be any length between 0 and `limit` bytes. 0 - /// returned bytes means end of stream reached. With limit=0 returns as - /// much data it can. It newer will be more than 65536 bytes, which is - /// size of internal buffer. - pub fn get(self: *Self, limit: usize) Error![]const u8 { - while (true) { - const out = self.hist.readAtMost(limit); - if (out.len > 0) { - self.hasher.update(out); - return out; - } - if (self.state == .end) return out; - try self.step(); - } - } - - // Reader interface - - pub const Reader = std.io.GenericReader(*Self, Error, read); - - /// Returns the number of bytes read. It may be less than buffer.len. - /// If the number of bytes read is 0, it means end of stream. - /// End of stream is not an error condition. - pub fn read(self: *Self, buffer: []u8) Error!usize { - if (buffer.len == 0) return 0; - const out = try self.get(buffer.len); - @memcpy(buffer[0..out.len], out); - return out.len; - } - - pub fn reader(self: *Self) Reader { - return .{ .context = self }; - } - }; -} - -test "decompress" { - const cases = [_]struct { - in: []const u8, - out: []const u8, - }{ - // non compressed block (type 0) - .{ - .in = &[_]u8{ - 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen - 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data - }, - .out = "Hello world\n", - }, - // fixed code block (type 1) - .{ - .in = &[_]u8{ - 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 - 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, - }, - .out = "Hello world\n", - }, - // dynamic block (type 2) - .{ - .in = &[_]u8{ - 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 - 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, - 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01, - }, - .out = "ABCDEABCD ABCDEABCD", - }, - }; - for (cases) |c| { - var fb = std.io.fixedBufferStream(c.in); - var al = std.ArrayList(u8).init(testing.allocator); - defer al.deinit(); - - try decompress(.raw, fb.reader(), al.writer()); - try testing.expectEqualStrings(c.out, al.items); - } -} - -test "gzip decompress" { - const cases = [_]struct { - in: []const u8, - out: []const u8, - }{ - // non compressed block (type 0) - .{ - .in = &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes) - 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen - 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data - 0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum - 0x0c, 0x00, 0x00, 0x00, // gzip footer: size - }, - .out = "Hello world\n", - }, - // fixed code block (type 1) - .{ - .in = &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes) - 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 - 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, - 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len) - }, - .out = "Hello world\n", - }, - // dynamic block (type 2) - .{ - .in = &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes) - 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 - 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, - 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01, - 0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len) - }, - .out = "ABCDEABCD ABCDEABCD", - }, - // gzip header with name - .{ - .in = &[_]u8{ - 0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, - 0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, - 0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, - }, - .out = "Hello world\n", - }, - }; - for (cases) |c| { - var fb = std.io.fixedBufferStream(c.in); - var al = std.ArrayList(u8).init(testing.allocator); - defer al.deinit(); - - try decompress(.gzip, fb.reader(), al.writer()); - try testing.expectEqualStrings(c.out, al.items); - } -} - -test "zlib decompress" { - const cases = [_]struct { - in: []const u8, - out: []const u8, - }{ - // non compressed block (type 0) - .{ - .in = &[_]u8{ - 0x78, 0b10_0_11100, // zlib header (2 bytes) - 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen - 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data - 0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum - }, - .out = "Hello world\n", - }, - }; - for (cases) |c| { - var fb = std.io.fixedBufferStream(c.in); - var al = std.ArrayList(u8).init(testing.allocator); - defer al.deinit(); - - try decompress(.zlib, fb.reader(), al.writer()); - try testing.expectEqualStrings(c.out, al.items); - } -} - -test "fuzzing tests" { - const cases = [_]struct { - input: []const u8, - out: []const u8 = "", - err: ?anyerror = null, - }{ - .{ .input = "deflate-stream", .out = @embedFile("testdata/fuzz/deflate-stream.expect") }, // 0 - .{ .input = "empty-distance-alphabet01" }, - .{ .input = "empty-distance-alphabet02" }, - .{ .input = "end-of-stream", .err = error.EndOfStream }, - .{ .input = "invalid-distance", .err = error.InvalidMatch }, - .{ .input = "invalid-tree01", .err = error.IncompleteHuffmanTree }, // 5 - .{ .input = "invalid-tree02", .err = error.IncompleteHuffmanTree }, - .{ .input = "invalid-tree03", .err = error.IncompleteHuffmanTree }, - .{ .input = "lengths-overflow", .err = error.InvalidDynamicBlockHeader }, - .{ .input = "out-of-codes", .err = error.InvalidCode }, - .{ .input = "puff01", .err = error.WrongStoredBlockNlen }, // 10 - .{ .input = "puff02", .err = error.EndOfStream }, - .{ .input = "puff03", .out = &[_]u8{0xa} }, - .{ .input = "puff04", .err = error.InvalidCode }, - .{ .input = "puff05", .err = error.EndOfStream }, - .{ .input = "puff06", .err = error.EndOfStream }, - .{ .input = "puff08", .err = error.InvalidCode }, - .{ .input = "puff09", .out = "P" }, - .{ .input = "puff10", .err = error.InvalidCode }, - .{ .input = "puff11", .err = error.InvalidMatch }, - .{ .input = "puff12", .err = error.InvalidDynamicBlockHeader }, // 20 - .{ .input = "puff13", .err = error.IncompleteHuffmanTree }, - .{ .input = "puff14", .err = error.EndOfStream }, - .{ .input = "puff15", .err = error.IncompleteHuffmanTree }, - .{ .input = "puff16", .err = error.InvalidDynamicBlockHeader }, - .{ .input = "puff17", .err = error.MissingEndOfBlockCode }, // 25 - .{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader }, - .{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader }, - .{ .input = "fuzz3", .err = error.InvalidMatch }, - .{ .input = "fuzz4", .err = error.OversubscribedHuffmanTree }, - .{ .input = "puff18", .err = error.OversubscribedHuffmanTree }, // 30 - .{ .input = "puff19", .err = error.OversubscribedHuffmanTree }, - .{ .input = "puff20", .err = error.OversubscribedHuffmanTree }, - .{ .input = "puff21", .err = error.OversubscribedHuffmanTree }, - .{ .input = "puff22", .err = error.OversubscribedHuffmanTree }, - .{ .input = "puff23", .err = error.OversubscribedHuffmanTree }, // 35 - .{ .input = "puff24", .err = error.IncompleteHuffmanTree }, - .{ .input = "puff25", .err = error.OversubscribedHuffmanTree }, - .{ .input = "puff26", .err = error.InvalidDynamicBlockHeader }, - .{ .input = "puff27", .err = error.InvalidDynamicBlockHeader }, - }; - - inline for (cases, 0..) |c, case_no| { - var in = std.io.fixedBufferStream(@embedFile("testdata/fuzz/" ++ c.input ++ ".input")); - var out = std.ArrayList(u8).init(testing.allocator); - defer out.deinit(); - errdefer std.debug.print("test case failed {}\n", .{case_no}); - - if (c.err) |expected_err| { - try testing.expectError(expected_err, decompress(.raw, in.reader(), out.writer())); - } else { - try decompress(.raw, in.reader(), out.writer()); - try testing.expectEqualStrings(c.out, out.items); - } - } -} - -test "bug 18966" { - const input = @embedFile("testdata/fuzz/bug_18966.input"); - const expect = @embedFile("testdata/fuzz/bug_18966.expect"); - - var in = std.io.fixedBufferStream(input); - var out = std.ArrayList(u8).init(testing.allocator); - defer out.deinit(); - - try decompress(.gzip, in.reader(), out.writer()); - try testing.expectEqualStrings(expect, out.items); -} - -test "bug 19895" { - const input = &[_]u8{ - 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen - 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data - }; - var in = std.io.fixedBufferStream(input); - var decomp = decompressor(.raw, in.reader()); - var buf: [0]u8 = undefined; - try testing.expectEqual(0, try decomp.read(&buf)); -} diff --git a/lib/std/compress/gzip.zig b/lib/std/compress/gzip.zig deleted file mode 100644 index e619b575de..0000000000 --- a/lib/std/compress/gzip.zig +++ /dev/null @@ -1,66 +0,0 @@ -const deflate = @import("flate/deflate.zig"); -const inflate = @import("flate/inflate.zig"); - -/// Decompress compressed data from reader and write plain data to the writer. -pub fn decompress(reader: anytype, writer: anytype) !void { - try inflate.decompress(.gzip, reader, writer); -} - -/// Decompressor type -pub fn Decompressor(comptime ReaderType: type) type { - return inflate.Decompressor(.gzip, ReaderType); -} - -/// Create Decompressor which will read compressed data from reader. -pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) { - return inflate.decompressor(.gzip, reader); -} - -/// Compression level, trades between speed and compression size. -pub const Options = deflate.Options; - -/// Compress plain data from reader and write compressed data to the writer. -pub fn compress(reader: anytype, writer: anytype, options: Options) !void { - try deflate.compress(.gzip, reader, writer, options); -} - -/// Compressor type -pub fn Compressor(comptime WriterType: type) type { - return deflate.Compressor(.gzip, WriterType); -} - -/// Create Compressor which outputs compressed data to the writer. -pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) { - return try deflate.compressor(.gzip, writer, options); -} - -/// Huffman only compression. Without Lempel-Ziv match searching. Faster -/// compression, less memory requirements but bigger compressed sizes. -pub const huffman = struct { - pub fn compress(reader: anytype, writer: anytype) !void { - try deflate.huffman.compress(.gzip, reader, writer); - } - - pub fn Compressor(comptime WriterType: type) type { - return deflate.huffman.Compressor(.gzip, WriterType); - } - - pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) { - return deflate.huffman.compressor(.gzip, writer); - } -}; - -// No compression store only. Compressed size is slightly bigger than plain. -pub const store = struct { - pub fn compress(reader: anytype, writer: anytype) !void { - try deflate.store.compress(.gzip, reader, writer); - } - - pub fn Compressor(comptime WriterType: type) type { - return deflate.store.Compressor(.gzip, WriterType); - } - - pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) { - return deflate.store.compressor(.gzip, writer); - } -}; diff --git a/lib/std/compress/zlib.zig b/lib/std/compress/zlib.zig deleted file mode 100644 index 554f6f894b..0000000000 --- a/lib/std/compress/zlib.zig +++ /dev/null @@ -1,101 +0,0 @@ -const deflate = @import("flate/deflate.zig"); -const inflate = @import("flate/inflate.zig"); - -/// Decompress compressed data from reader and write plain data to the writer. -pub fn decompress(reader: anytype, writer: anytype) !void { - try inflate.decompress(.zlib, reader, writer); -} - -/// Decompressor type -pub fn Decompressor(comptime ReaderType: type) type { - return inflate.Decompressor(.zlib, ReaderType); -} - -/// Create Decompressor which will read compressed data from reader. -pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) { - return inflate.decompressor(.zlib, reader); -} - -/// Compression level, trades between speed and compression size. -pub const Options = deflate.Options; - -/// Compress plain data from reader and write compressed data to the writer. -pub fn compress(reader: anytype, writer: anytype, options: Options) !void { - try deflate.compress(.zlib, reader, writer, options); -} - -/// Compressor type -pub fn Compressor(comptime WriterType: type) type { - return deflate.Compressor(.zlib, WriterType); -} - -/// Create Compressor which outputs compressed data to the writer. -pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) { - return try deflate.compressor(.zlib, writer, options); -} - -/// Huffman only compression. Without Lempel-Ziv match searching. Faster -/// compression, less memory requirements but bigger compressed sizes. -pub const huffman = struct { - pub fn compress(reader: anytype, writer: anytype) !void { - try deflate.huffman.compress(.zlib, reader, writer); - } - - pub fn Compressor(comptime WriterType: type) type { - return deflate.huffman.Compressor(.zlib, WriterType); - } - - pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) { - return deflate.huffman.compressor(.zlib, writer); - } -}; - -// No compression store only. Compressed size is slightly bigger than plain. -pub const store = struct { - pub fn compress(reader: anytype, writer: anytype) !void { - try deflate.store.compress(.zlib, reader, writer); - } - - pub fn Compressor(comptime WriterType: type) type { - return deflate.store.Compressor(.zlib, WriterType); - } - - pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) { - return deflate.store.compressor(.zlib, writer); - } -}; - -test "should not overshoot" { - const std = @import("std"); - - // Compressed zlib data with extra 4 bytes at the end. - const data = [_]u8{ - 0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9, - 0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08, - 0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34, - 0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12, - }; - - var stream = std.io.fixedBufferStream(data[0..]); - const reader = stream.reader(); - - var dcp = decompressor(reader); - var out: [128]u8 = undefined; - - // Decompress - var n = try dcp.reader().readAll(out[0..]); - - // Expected decompressed data - try std.testing.expectEqual(46, n); - try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]); - - // Decompressor don't overshoot underlying reader. - // It is leaving it at the end of compressed data chunk. - try std.testing.expectEqual(data.len - 4, stream.getPos()); - try std.testing.expectEqual(0, dcp.unreadBytes()); - - // 4 bytes after compressed chunk are available in reader. - n = try reader.readAll(out[0..]); - try std.testing.expectEqual(n, 4); - try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]); -} diff --git a/lib/std/debug/Dwarf.zig b/lib/std/debug/Dwarf.zig index 3f1fc41feb..5fa5dd002a 100644 --- a/lib/std/debug/Dwarf.zig +++ b/lib/std/debug/Dwarf.zig @@ -2235,18 +2235,14 @@ pub const ElfModule = struct { const section_bytes = try chopSlice(mapped_mem, shdr.sh_offset, shdr.sh_size); sections[section_index.?] = if ((shdr.sh_flags & elf.SHF_COMPRESSED) > 0) blk: { - var section_stream = std.io.fixedBufferStream(section_bytes); - const section_reader = section_stream.reader(); - const chdr = section_reader.readStruct(elf.Chdr) catch continue; + var section_reader: std.Io.Reader = .fixed(section_bytes); + const chdr = section_reader.takeStruct(elf.Chdr, endian) catch continue; if (chdr.ch_type != .ZLIB) continue; - var zlib_stream = std.compress.zlib.decompressor(section_reader); - - const decompressed_section = try gpa.alloc(u8, chdr.ch_size); + var zlib_stream: std.compress.flate.Decompress = .init(§ion_reader, .zlib, &.{}); + const decompressed_section = zlib_stream.reader.allocRemaining(gpa, .unlimited) catch continue; errdefer gpa.free(decompressed_section); - - const read = zlib_stream.reader().readAll(decompressed_section) catch continue; - assert(read == decompressed_section.len); + assert(chdr.ch_size == decompressed_section.len); break :blk .{ .data = decompressed_section, diff --git a/lib/std/http/Client.zig b/lib/std/http/Client.zig index 838411bebc..83c1c8b50b 100644 --- a/lib/std/http/Client.zig +++ b/lib/std/http/Client.zig @@ -405,13 +405,8 @@ pub const RequestTransfer = union(enum) { /// The decompressor for response messages. pub const Compression = union(enum) { - pub const DeflateDecompressor = std.compress.zlib.Decompressor(Request.TransferReader); - pub const GzipDecompressor = std.compress.gzip.Decompressor(Request.TransferReader); - // https://github.com/ziglang/zig/issues/18937 - //pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{}); - - deflate: DeflateDecompressor, - gzip: GzipDecompressor, + deflate: std.compress.flate.Decompress, + gzip: std.compress.flate.Decompress, // https://github.com/ziglang/zig/issues/18937 //zstd: ZstdDecompressor, none: void, diff --git a/lib/std/http/Server.zig b/lib/std/http/Server.zig index 886aed72dc..7ec5d5c11f 100644 --- a/lib/std/http/Server.zig +++ b/lib/std/http/Server.zig @@ -130,8 +130,8 @@ pub const Request = struct { pub const DeflateDecompressor = std.compress.zlib.Decompressor(std.io.AnyReader); pub const GzipDecompressor = std.compress.gzip.Decompressor(std.io.AnyReader); - deflate: DeflateDecompressor, - gzip: GzipDecompressor, + deflate: std.compress.flate.Decompress, + gzip: std.compress.flate.Decompress, zstd: std.compress.zstd.Decompress, none: void, }; diff --git a/lib/std/zip.zig b/lib/std/zip.zig index e181bc1f65..b13c1d5010 100644 --- a/lib/std/zip.zig +++ b/lib/std/zip.zig @@ -5,11 +5,10 @@ const builtin = @import("builtin"); const std = @import("std"); -const testing = std.testing; - -pub const testutil = @import("zip/test.zig"); -const File = testutil.File; -const FileStore = testutil.FileStore; +const File = std.fs.File; +const is_le = builtin.target.cpu.arch.endian() == .little; +const Writer = std.io.Writer; +const Reader = std.io.Reader; pub const CompressionMethod = enum(u16) { store = 0, @@ -95,102 +94,116 @@ pub const EndRecord = extern struct { central_directory_size: u32 align(1), central_directory_offset: u32 align(1), comment_len: u16 align(1), + pub fn need_zip64(self: EndRecord) bool { return isMaxInt(self.record_count_disk) or isMaxInt(self.record_count_total) or isMaxInt(self.central_directory_size) or isMaxInt(self.central_directory_offset); } + + pub const FindBufferError = error{ ZipNoEndRecord, ZipTruncated }; + + /// TODO audit this logic + pub fn findBuffer(buffer: []const u8) FindBufferError!EndRecord { + const pos = std.mem.lastIndexOf(u8, buffer, &end_record_sig) orelse return error.ZipNoEndRecord; + if (pos + @sizeOf(EndRecord) > buffer.len) return error.EndOfStream; + const record_ptr: *EndRecord = @ptrCast(buffer[pos..][0..@sizeOf(EndRecord)]); + var record = record_ptr.*; + if (!is_le) std.mem.byteSwapAllFields(EndRecord, &record); + return record; + } + + pub const FindFileError = File.GetEndPosError || File.SeekError || File.ReadError || error{ + ZipNoEndRecord, + EndOfStream, + }; + + pub fn findFile(fr: *File.Reader) FindFileError!EndRecord { + const end_pos = try fr.getSize(); + + var buf: [@sizeOf(EndRecord) + std.math.maxInt(u16)]u8 = undefined; + const record_len_max = @min(end_pos, buf.len); + var loaded_len: u32 = 0; + var comment_len: u16 = 0; + while (true) { + const record_len: u32 = @as(u32, comment_len) + @sizeOf(EndRecord); + if (record_len > record_len_max) + return error.ZipNoEndRecord; + + if (record_len > loaded_len) { + const new_loaded_len = @min(loaded_len + 300, record_len_max); + const read_len = new_loaded_len - loaded_len; + + try fr.seekTo(end_pos - @as(u64, new_loaded_len)); + const read_buf: []u8 = buf[buf.len - new_loaded_len ..][0..read_len]; + var br = fr.interface().unbuffered(); + br.readSlice(read_buf) catch |err| switch (err) { + error.ReadFailed => return fr.err.?, + error.EndOfStream => return error.EndOfStream, + }; + loaded_len = new_loaded_len; + } + + const record_bytes = buf[buf.len - record_len ..][0..@sizeOf(EndRecord)]; + if (std.mem.eql(u8, record_bytes[0..4], &end_record_sig) and + std.mem.readInt(u16, record_bytes[20..22], .little) == comment_len) + { + const record: *align(1) EndRecord = @ptrCast(record_bytes.ptr); + if (!is_le) std.mem.byteSwapAllFields(EndRecord, record); + return record.*; + } + + if (comment_len == std.math.maxInt(u16)) + return error.ZipNoEndRecord; + comment_len += 1; + } + } }; -/// Find and return the end record for the given seekable zip stream. -/// Note that `seekable_stream` must be an instance of `std.io.SeekableStream` and -/// its context must also have a `.reader()` method that returns an instance of -/// `std.io.GenericReader`. -pub fn findEndRecord(seekable_stream: anytype, stream_len: u64) !EndRecord { - var buf: [@sizeOf(EndRecord) + std.math.maxInt(u16)]u8 = undefined; - const record_len_max = @min(stream_len, buf.len); - var loaded_len: u32 = 0; +pub const Decompress = struct { + interface: Reader, + state: union { + inflate: std.compress.flate.Decompress, + store: *Reader, + }, - var comment_len: u16 = 0; - while (true) { - const record_len: u32 = @as(u32, comment_len) + @sizeOf(EndRecord); - if (record_len > record_len_max) - return error.ZipNoEndRecord; - - if (record_len > loaded_len) { - const new_loaded_len = @min(loaded_len + 300, record_len_max); - const read_len = new_loaded_len - loaded_len; - - try seekable_stream.seekTo(stream_len - @as(u64, new_loaded_len)); - const read_buf: []u8 = buf[buf.len - new_loaded_len ..][0..read_len]; - const len = try (if (@TypeOf(seekable_stream.context) == std.fs.File) seekable_stream.context.deprecatedReader() else seekable_stream.context.reader()).readAll(read_buf); - if (len != read_len) - return error.ZipTruncated; - loaded_len = new_loaded_len; - } - - const record_bytes = buf[buf.len - record_len ..][0..@sizeOf(EndRecord)]; - if (std.mem.eql(u8, record_bytes[0..4], &end_record_sig) and - std.mem.readInt(u16, record_bytes[20..22], .little) == comment_len) - { - const record: *align(1) EndRecord = @ptrCast(record_bytes.ptr); - if (builtin.target.cpu.arch.endian() != .little) { - std.mem.byteSwapAllFields(@TypeOf(record.*), record); - } - return record.*; - } - - if (comment_len == std.math.maxInt(u16)) - return error.ZipNoEndRecord; - comment_len += 1; + pub fn init(reader: *Reader, method: CompressionMethod, buffer: []u8) Reader { + return switch (method) { + .store => .{ + .state = .{ .store = reader }, + .interface = .{ + .context = undefined, + .vtable = &.{ .stream = streamStore }, + .buffer = buffer, + .end = 0, + .seek = 0, + }, + }, + .deflate => .{ + .state = .{ .inflate = .init(reader, .raw) }, + .interface = .{ + .context = undefined, + .vtable = &.{ .stream = streamDeflate }, + .buffer = buffer, + .end = 0, + .seek = 0, + }, + }, + else => unreachable, + }; } -} -/// Decompresses the given data from `reader` into `writer`. Stops early if more -/// than `uncompressed_size` bytes are processed and verifies that exactly that -/// number of bytes are decompressed. Returns the CRC-32 of the uncompressed data. -/// `writer` can be anything with a `writeAll(self: *Self, chunk: []const u8) anyerror!void` method. -pub fn decompress( - method: CompressionMethod, - uncompressed_size: u64, - reader: anytype, - writer: anytype, -) !u32 { - var hash = std.hash.Crc32.init(); - - var total_uncompressed: u64 = 0; - switch (method) { - .store => { - var buf: [4096]u8 = undefined; - while (true) { - const len = try reader.read(&buf); - if (len == 0) break; - try writer.writeAll(buf[0..len]); - hash.update(buf[0..len]); - total_uncompressed += @intCast(len); - } - }, - .deflate => { - var br = std.io.bufferedReader(reader); - var decompressor = std.compress.flate.decompressor(br.reader()); - while (try decompressor.next()) |chunk| { - try writer.writeAll(chunk); - hash.update(chunk); - total_uncompressed += @intCast(chunk.len); - if (total_uncompressed > uncompressed_size) - return error.ZipUncompressSizeTooSmall; - } - if (br.end != br.start) - return error.ZipDeflateTruncated; - }, - _ => return error.UnsupportedCompressionMethod, + fn streamStore(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!usize { + const d: *Decompress = @fieldParentPtr("interface", r); + return d.store.read(w, limit); } - if (total_uncompressed != uncompressed_size) - return error.ZipUncompressSizeMismatch; - return hash.final(); -} + fn streamDeflate(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!usize { + const d: *Decompress = @fieldParentPtr("interface", r); + return std.compress.flate.Decompress.read(&d.inflate, w, limit); + } +}; fn isBadFilename(filename: []const u8) bool { if (filename.len == 0 or filename[0] == '/') @@ -253,319 +266,332 @@ fn readZip64FileExtents(comptime T: type, header: T, extents: *FileExtents, data } } -pub fn Iterator(comptime SeekableStream: type) type { - return struct { - stream: SeekableStream, +pub const Iterator = struct { + input: *File.Reader, - cd_record_count: u64, - cd_zip_offset: u64, - cd_size: u64, + cd_record_count: u64, + cd_zip_offset: u64, + cd_size: u64, - cd_record_index: u64 = 0, - cd_record_offset: u64 = 0, + cd_record_index: u64 = 0, + cd_record_offset: u64 = 0, - const Self = @This(); + pub fn init(input: *File.Reader) !Iterator { + const end_record = try EndRecord.findFile(input); - pub fn init(stream: SeekableStream) !Self { - const stream_len = try stream.getEndPos(); + if (!isMaxInt(end_record.record_count_disk) and end_record.record_count_disk > end_record.record_count_total) + return error.ZipDiskRecordCountTooLarge; - const end_record = try findEndRecord(stream, stream_len); + if (end_record.disk_number != 0 or end_record.central_directory_disk_number != 0) + return error.ZipMultiDiskUnsupported; - if (!isMaxInt(end_record.record_count_disk) and end_record.record_count_disk > end_record.record_count_total) - return error.ZipDiskRecordCountTooLarge; - - if (end_record.disk_number != 0 or end_record.central_directory_disk_number != 0) + { + const counts_valid = !isMaxInt(end_record.record_count_disk) and !isMaxInt(end_record.record_count_total); + if (counts_valid and end_record.record_count_disk != end_record.record_count_total) return error.ZipMultiDiskUnsupported; - - { - const counts_valid = !isMaxInt(end_record.record_count_disk) and !isMaxInt(end_record.record_count_total); - if (counts_valid and end_record.record_count_disk != end_record.record_count_total) - return error.ZipMultiDiskUnsupported; - } - - var result = Self{ - .stream = stream, - .cd_record_count = end_record.record_count_total, - .cd_zip_offset = end_record.central_directory_offset, - .cd_size = end_record.central_directory_size, - }; - if (!end_record.need_zip64()) return result; - - const locator_end_offset: u64 = @as(u64, end_record.comment_len) + @sizeOf(EndRecord) + @sizeOf(EndLocator64); - if (locator_end_offset > stream_len) - return error.ZipTruncated; - try stream.seekTo(stream_len - locator_end_offset); - const locator = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readStructEndian(EndLocator64, .little); - if (!std.mem.eql(u8, &locator.signature, &end_locator64_sig)) - return error.ZipBadLocatorSig; - if (locator.zip64_disk_count != 0) - return error.ZipUnsupportedZip64DiskCount; - if (locator.total_disk_count != 1) - return error.ZipMultiDiskUnsupported; - - try stream.seekTo(locator.record_file_offset); - - const record64 = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readStructEndian(EndRecord64, .little); - - if (!std.mem.eql(u8, &record64.signature, &end_record64_sig)) - return error.ZipBadEndRecord64Sig; - - if (record64.end_record_size < @sizeOf(EndRecord64) - 12) - return error.ZipEndRecord64SizeTooSmall; - if (record64.end_record_size > @sizeOf(EndRecord64) - 12) - return error.ZipEndRecord64UnhandledExtraData; - - if (record64.version_needed_to_extract > 45) - return error.ZipUnsupportedVersion; - - { - const is_multidisk = record64.disk_number != 0 or - record64.central_directory_disk_number != 0 or - record64.record_count_disk != record64.record_count_total; - if (is_multidisk) - return error.ZipMultiDiskUnsupported; - } - - if (isMaxInt(end_record.record_count_total)) { - result.cd_record_count = record64.record_count_total; - } else if (end_record.record_count_total != record64.record_count_total) - return error.Zip64RecordCountTotalMismatch; - - if (isMaxInt(end_record.central_directory_offset)) { - result.cd_zip_offset = record64.central_directory_offset; - } else if (end_record.central_directory_offset != record64.central_directory_offset) - return error.Zip64CentralDirectoryOffsetMismatch; - - if (isMaxInt(end_record.central_directory_size)) { - result.cd_size = record64.central_directory_size; - } else if (end_record.central_directory_size != record64.central_directory_size) - return error.Zip64CentralDirectorySizeMismatch; - - return result; } - pub fn next(self: *Self) !?Entry { - if (self.cd_record_index == self.cd_record_count) { - if (self.cd_record_offset != self.cd_size) - return if (self.cd_size > self.cd_record_offset) - error.ZipCdOversized - else - error.ZipCdUndersized; - - return null; - } - - const header_zip_offset = self.cd_zip_offset + self.cd_record_offset; - try self.stream.seekTo(header_zip_offset); - const header = try (if (@TypeOf(self.stream.context) == std.fs.File) self.stream.context.deprecatedReader() else self.stream.context.reader()).readStructEndian(CentralDirectoryFileHeader, .little); - if (!std.mem.eql(u8, &header.signature, ¢ral_file_header_sig)) - return error.ZipBadCdOffset; - - self.cd_record_index += 1; - self.cd_record_offset += @sizeOf(CentralDirectoryFileHeader) + header.filename_len + header.extra_len + header.comment_len; - - // Note: checking the version_needed_to_extract doesn't seem to be helpful, i.e. the zip file - // at https://github.com/ninja-build/ninja/releases/download/v1.12.0/ninja-linux.zip - // has an undocumented version 788 but extracts just fine. - - if (header.flags.encrypted) - return error.ZipEncryptionUnsupported; - // TODO: check/verify more flags - if (header.disk_number != 0) - return error.ZipMultiDiskUnsupported; - - var extents: FileExtents = .{ - .uncompressed_size = header.uncompressed_size, - .compressed_size = header.compressed_size, - .local_file_header_offset = header.local_file_header_offset, - }; - - if (header.extra_len > 0) { - var extra_buf: [std.math.maxInt(u16)]u8 = undefined; - const extra = extra_buf[0..header.extra_len]; - - { - try self.stream.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len); - const len = try (if (@TypeOf(self.stream.context) == std.fs.File) self.stream.context.deprecatedReader() else self.stream.context.reader()).readAll(extra); - if (len != extra.len) - return error.ZipTruncated; - } - - var extra_offset: usize = 0; - while (extra_offset + 4 <= extra.len) { - const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little); - const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little); - const end = extra_offset + 4 + data_size; - if (end > extra.len) - return error.ZipBadExtraFieldSize; - const data = extra[extra_offset + 4 .. end]; - switch (@as(ExtraHeader, @enumFromInt(header_id))) { - .zip64_info => try readZip64FileExtents(CentralDirectoryFileHeader, header, &extents, data), - else => {}, // ignore - } - extra_offset = end; - } - } - - return .{ - .version_needed_to_extract = header.version_needed_to_extract, - .flags = header.flags, - .compression_method = header.compression_method, - .last_modification_time = header.last_modification_time, - .last_modification_date = header.last_modification_date, - .header_zip_offset = header_zip_offset, - .crc32 = header.crc32, - .filename_len = header.filename_len, - .compressed_size = extents.compressed_size, - .uncompressed_size = extents.uncompressed_size, - .file_offset = extents.local_file_header_offset, - }; - } - - pub const Entry = struct { - version_needed_to_extract: u16, - flags: GeneralPurposeFlags, - compression_method: CompressionMethod, - last_modification_time: u16, - last_modification_date: u16, - header_zip_offset: u64, - crc32: u32, - filename_len: u32, - compressed_size: u64, - uncompressed_size: u64, - file_offset: u64, - - pub fn extract( - self: Entry, - stream: SeekableStream, - options: ExtractOptions, - filename_buf: []u8, - dest: std.fs.Dir, - ) !u32 { - if (filename_buf.len < self.filename_len) - return error.ZipInsufficientBuffer; - const filename = filename_buf[0..self.filename_len]; - - try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader)); - - { - const len = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readAll(filename); - if (len != filename.len) - return error.ZipBadFileOffset; - } - - const local_data_header_offset: u64 = local_data_header_offset: { - const local_header = blk: { - try stream.seekTo(self.file_offset); - break :blk try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readStructEndian(LocalFileHeader, .little); - }; - if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig)) - return error.ZipBadFileOffset; - if (local_header.version_needed_to_extract != self.version_needed_to_extract) - return error.ZipMismatchVersionNeeded; - if (local_header.last_modification_time != self.last_modification_time) - return error.ZipMismatchModTime; - if (local_header.last_modification_date != self.last_modification_date) - return error.ZipMismatchModDate; - - if (@as(u16, @bitCast(local_header.flags)) != @as(u16, @bitCast(self.flags))) - return error.ZipMismatchFlags; - if (local_header.crc32 != 0 and local_header.crc32 != self.crc32) - return error.ZipMismatchCrc32; - var extents: FileExtents = .{ - .uncompressed_size = local_header.uncompressed_size, - .compressed_size = local_header.compressed_size, - .local_file_header_offset = 0, - }; - if (local_header.extra_len > 0) { - var extra_buf: [std.math.maxInt(u16)]u8 = undefined; - const extra = extra_buf[0..local_header.extra_len]; - - { - try stream.seekTo(self.file_offset + @sizeOf(LocalFileHeader) + local_header.filename_len); - const len = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readAll(extra); - if (len != extra.len) - return error.ZipTruncated; - } - - var extra_offset: usize = 0; - while (extra_offset + 4 <= local_header.extra_len) { - const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little); - const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little); - const end = extra_offset + 4 + data_size; - if (end > local_header.extra_len) - return error.ZipBadExtraFieldSize; - const data = extra[extra_offset + 4 .. end]; - switch (@as(ExtraHeader, @enumFromInt(header_id))) { - .zip64_info => try readZip64FileExtents(LocalFileHeader, local_header, &extents, data), - else => {}, // ignore - } - extra_offset = end; - } - } - - if (extents.compressed_size != 0 and - extents.compressed_size != self.compressed_size) - return error.ZipMismatchCompLen; - if (extents.uncompressed_size != 0 and - extents.uncompressed_size != self.uncompressed_size) - return error.ZipMismatchUncompLen; - - if (local_header.filename_len != self.filename_len) - return error.ZipMismatchFilenameLen; - - break :local_data_header_offset @as(u64, local_header.filename_len) + - @as(u64, local_header.extra_len); - }; - - if (isBadFilename(filename)) - return error.ZipBadFilename; - - if (options.allow_backslashes) { - std.mem.replaceScalar(u8, filename, '\\', '/'); - } else { - if (std.mem.indexOfScalar(u8, filename, '\\')) |_| - return error.ZipFilenameHasBackslash; - } - - // All entries that end in '/' are directories - if (filename[filename.len - 1] == '/') { - if (self.uncompressed_size != 0) - return error.ZipBadDirectorySize; - try dest.makePath(filename[0 .. filename.len - 1]); - return std.hash.Crc32.hash(&.{}); - } - - const out_file = blk: { - if (std.fs.path.dirname(filename)) |dirname| { - var parent_dir = try dest.makeOpenPath(dirname, .{}); - defer parent_dir.close(); - - const basename = std.fs.path.basename(filename); - break :blk try parent_dir.createFile(basename, .{ .exclusive = true }); - } - break :blk try dest.createFile(filename, .{ .exclusive = true }); - }; - defer out_file.close(); - const local_data_file_offset: u64 = - @as(u64, self.file_offset) + - @as(u64, @sizeOf(LocalFileHeader)) + - local_data_header_offset; - try stream.seekTo(local_data_file_offset); - var limited_reader = std.io.limitedReader((if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()), self.compressed_size); - const crc = try decompress( - self.compression_method, - self.uncompressed_size, - limited_reader.reader(), - out_file.deprecatedWriter(), - ); - if (limited_reader.bytes_left != 0) - return error.ZipDecompressTruncated; - return crc; - } + var result: Iterator = .{ + .input = input, + .cd_record_count = end_record.record_count_total, + .cd_zip_offset = end_record.central_directory_offset, + .cd_size = end_record.central_directory_size, }; + if (!end_record.need_zip64()) return result; + + const locator_end_offset: u64 = @as(u64, end_record.comment_len) + @sizeOf(EndRecord) + @sizeOf(EndLocator64); + const stream_len = try input.getSize(); + + if (locator_end_offset > stream_len) + return error.ZipTruncated; + try input.seekTo(stream_len - locator_end_offset); + const locator = input.interface.takeStructEndian(EndLocator64, .little) catch |err| switch (err) { + error.ReadFailed => return input.err.?, + error.EndOfStream => return error.EndOfStream, + }; + if (!std.mem.eql(u8, &locator.signature, &end_locator64_sig)) + return error.ZipBadLocatorSig; + if (locator.zip64_disk_count != 0) + return error.ZipUnsupportedZip64DiskCount; + if (locator.total_disk_count != 1) + return error.ZipMultiDiskUnsupported; + + try input.seekTo(locator.record_file_offset); + + const record64 = input.interface.takeStructEndian(EndRecord64, .little) catch |err| switch (err) { + error.ReadFailed => return input.err.?, + error.EndOfStream => return error.EndOfStream, + }; + + if (!std.mem.eql(u8, &record64.signature, &end_record64_sig)) + return error.ZipBadEndRecord64Sig; + + if (record64.end_record_size < @sizeOf(EndRecord64) - 12) + return error.ZipEndRecord64SizeTooSmall; + if (record64.end_record_size > @sizeOf(EndRecord64) - 12) + return error.ZipEndRecord64UnhandledExtraData; + + if (record64.version_needed_to_extract > 45) + return error.ZipUnsupportedVersion; + + { + const is_multidisk = record64.disk_number != 0 or + record64.central_directory_disk_number != 0 or + record64.record_count_disk != record64.record_count_total; + if (is_multidisk) + return error.ZipMultiDiskUnsupported; + } + + if (isMaxInt(end_record.record_count_total)) { + result.cd_record_count = record64.record_count_total; + } else if (end_record.record_count_total != record64.record_count_total) + return error.Zip64RecordCountTotalMismatch; + + if (isMaxInt(end_record.central_directory_offset)) { + result.cd_zip_offset = record64.central_directory_offset; + } else if (end_record.central_directory_offset != record64.central_directory_offset) + return error.Zip64CentralDirectoryOffsetMismatch; + + if (isMaxInt(end_record.central_directory_size)) { + result.cd_size = record64.central_directory_size; + } else if (end_record.central_directory_size != record64.central_directory_size) + return error.Zip64CentralDirectorySizeMismatch; + + return result; + } + + pub fn next(self: *Iterator) !?Entry { + if (self.cd_record_index == self.cd_record_count) { + if (self.cd_record_offset != self.cd_size) + return if (self.cd_size > self.cd_record_offset) + error.ZipCdOversized + else + error.ZipCdUndersized; + + return null; + } + + const header_zip_offset = self.cd_zip_offset + self.cd_record_offset; + const input = self.input; + try input.seekTo(header_zip_offset); + const header = input.interface.takeStructEndian(CentralDirectoryFileHeader, .little) catch |err| switch (err) { + error.ReadFailed => return input.err.?, + error.EndOfStream => return error.EndOfStream, + }; + if (!std.mem.eql(u8, &header.signature, ¢ral_file_header_sig)) + return error.ZipBadCdOffset; + + self.cd_record_index += 1; + self.cd_record_offset += @sizeOf(CentralDirectoryFileHeader) + header.filename_len + header.extra_len + header.comment_len; + + // Note: checking the version_needed_to_extract doesn't seem to be helpful, i.e. the zip file + // at https://github.com/ninja-build/ninja/releases/download/v1.12.0/ninja-linux.zip + // has an undocumented version 788 but extracts just fine. + + if (header.flags.encrypted) + return error.ZipEncryptionUnsupported; + // TODO: check/verify more flags + if (header.disk_number != 0) + return error.ZipMultiDiskUnsupported; + + var extents: FileExtents = .{ + .uncompressed_size = header.uncompressed_size, + .compressed_size = header.compressed_size, + .local_file_header_offset = header.local_file_header_offset, + }; + + if (header.extra_len > 0) { + var extra_buf: [std.math.maxInt(u16)]u8 = undefined; + const extra = extra_buf[0..header.extra_len]; + + try input.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len); + input.interface.readSlice(extra) catch |err| switch (err) { + error.ReadFailed => return input.err.?, + error.EndOfStream => return error.EndOfStream, + }; + + var extra_offset: usize = 0; + while (extra_offset + 4 <= extra.len) { + const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little); + const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little); + const end = extra_offset + 4 + data_size; + if (end > extra.len) + return error.ZipBadExtraFieldSize; + const data = extra[extra_offset + 4 .. end]; + switch (@as(ExtraHeader, @enumFromInt(header_id))) { + .zip64_info => try readZip64FileExtents(CentralDirectoryFileHeader, header, &extents, data), + else => {}, // ignore + } + extra_offset = end; + } + } + + return .{ + .version_needed_to_extract = header.version_needed_to_extract, + .flags = header.flags, + .compression_method = header.compression_method, + .last_modification_time = header.last_modification_time, + .last_modification_date = header.last_modification_date, + .header_zip_offset = header_zip_offset, + .crc32 = header.crc32, + .filename_len = header.filename_len, + .compressed_size = extents.compressed_size, + .uncompressed_size = extents.uncompressed_size, + .file_offset = extents.local_file_header_offset, + }; + } + + pub const Entry = struct { + version_needed_to_extract: u16, + flags: GeneralPurposeFlags, + compression_method: CompressionMethod, + last_modification_time: u16, + last_modification_date: u16, + header_zip_offset: u64, + crc32: u32, + filename_len: u32, + compressed_size: u64, + uncompressed_size: u64, + file_offset: u64, + + pub fn extract( + self: Entry, + stream: *File.Reader, + options: ExtractOptions, + filename_buf: []u8, + dest: std.fs.Dir, + ) !u32 { + if (filename_buf.len < self.filename_len) + return error.ZipInsufficientBuffer; + switch (self.compression_method) { + .store, .deflate => {}, + else => return error.UnsupportedCompressionMethod, + } + const filename = filename_buf[0..self.filename_len]; + { + try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader)); + try stream.interface.readSlice(filename); + } + + const local_data_header_offset: u64 = local_data_header_offset: { + const local_header = blk: { + try stream.seekTo(self.file_offset); + break :blk try stream.interface.takeStructEndian(LocalFileHeader, .little); + }; + if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig)) + return error.ZipBadFileOffset; + if (local_header.version_needed_to_extract != self.version_needed_to_extract) + return error.ZipMismatchVersionNeeded; + if (local_header.last_modification_time != self.last_modification_time) + return error.ZipMismatchModTime; + if (local_header.last_modification_date != self.last_modification_date) + return error.ZipMismatchModDate; + + if (@as(u16, @bitCast(local_header.flags)) != @as(u16, @bitCast(self.flags))) + return error.ZipMismatchFlags; + if (local_header.crc32 != 0 and local_header.crc32 != self.crc32) + return error.ZipMismatchCrc32; + var extents: FileExtents = .{ + .uncompressed_size = local_header.uncompressed_size, + .compressed_size = local_header.compressed_size, + .local_file_header_offset = 0, + }; + if (local_header.extra_len > 0) { + var extra_buf: [std.math.maxInt(u16)]u8 = undefined; + const extra = extra_buf[0..local_header.extra_len]; + + { + try stream.seekTo(self.file_offset + @sizeOf(LocalFileHeader) + local_header.filename_len); + try stream.interface.readSlice(extra); + } + + var extra_offset: usize = 0; + while (extra_offset + 4 <= local_header.extra_len) { + const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little); + const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little); + const end = extra_offset + 4 + data_size; + if (end > local_header.extra_len) + return error.ZipBadExtraFieldSize; + const data = extra[extra_offset + 4 .. end]; + switch (@as(ExtraHeader, @enumFromInt(header_id))) { + .zip64_info => try readZip64FileExtents(LocalFileHeader, local_header, &extents, data), + else => {}, // ignore + } + extra_offset = end; + } + } + + if (extents.compressed_size != 0 and + extents.compressed_size != self.compressed_size) + return error.ZipMismatchCompLen; + if (extents.uncompressed_size != 0 and + extents.uncompressed_size != self.uncompressed_size) + return error.ZipMismatchUncompLen; + + if (local_header.filename_len != self.filename_len) + return error.ZipMismatchFilenameLen; + + break :local_data_header_offset @as(u64, local_header.filename_len) + + @as(u64, local_header.extra_len); + }; + + if (isBadFilename(filename)) + return error.ZipBadFilename; + + if (options.allow_backslashes) { + std.mem.replaceScalar(u8, filename, '\\', '/'); + } else { + if (std.mem.indexOfScalar(u8, filename, '\\')) |_| + return error.ZipFilenameHasBackslash; + } + + // All entries that end in '/' are directories + if (filename[filename.len - 1] == '/') { + if (self.uncompressed_size != 0) + return error.ZipBadDirectorySize; + try dest.makePath(filename[0 .. filename.len - 1]); + return std.hash.Crc32.hash(&.{}); + } + + const out_file = blk: { + if (std.fs.path.dirname(filename)) |dirname| { + var parent_dir = try dest.makeOpenPath(dirname, .{}); + defer parent_dir.close(); + + const basename = std.fs.path.basename(filename); + break :blk try parent_dir.createFile(basename, .{ .exclusive = true }); + } + break :blk try dest.createFile(filename, .{ .exclusive = true }); + }; + defer out_file.close(); + var file_writer = out_file.writer(); + var file_bw = file_writer.writer(&.{}); + const local_data_file_offset: u64 = + @as(u64, self.file_offset) + + @as(u64, @sizeOf(LocalFileHeader)) + + local_data_header_offset; + try stream.seekTo(local_data_file_offset); + var limited_file_reader = stream.interface.limited(.limited(self.compressed_size)); + var file_read_buffer: [1000]u8 = undefined; + var decompress_read_buffer: [1000]u8 = undefined; + var limited_br = limited_file_reader.reader().buffered(&file_read_buffer); + var decompress: Decompress = undefined; + var decompress_br = decompress.readable(&limited_br, self.compression_method, &decompress_read_buffer); + const start_out = file_bw.count; + var hash_writer = file_bw.hashed(std.hash.Crc32.init()); + var hash_bw = hash_writer.writer(&.{}); + decompress_br.readAll(&hash_bw, .limited(self.uncompressed_size)) catch |err| switch (err) { + error.ReadFailed => return stream.err.?, + error.WriteFailed => return file_writer.err.?, + error.EndOfStream => return error.ZipDecompressTruncated, + }; + if (limited_file_reader.remaining.nonzero()) return error.ZipDecompressTruncated; + const written = file_bw.count - start_out; + if (written != self.uncompressed_size) return error.ZipUncompressSizeMismatch; + return hash_writer.hasher.final(); + } }; -} +}; // returns true if `filename` starts with `root` followed by a forward slash fn filenameInRoot(filename: []const u8, root: []const u8) bool { @@ -614,17 +640,13 @@ pub const ExtractOptions = struct { diagnostics: ?*Diagnostics = null, }; -/// Extract the zipped files inside `seekable_stream` to the given `dest` directory. -/// Note that `seekable_stream` must be an instance of `std.io.SeekableStream` and -/// its context must also have a `.reader()` method that returns an instance of -/// `std.io.GenericReader`. -pub fn extract(dest: std.fs.Dir, seekable_stream: anytype, options: ExtractOptions) !void { - const SeekableStream = @TypeOf(seekable_stream); - var iter = try Iterator(SeekableStream).init(seekable_stream); +/// Extract the zipped files to the given `dest` directory. +pub fn extract(dest: std.fs.Dir, fr: *File.Reader, options: ExtractOptions) !void { + var iter = try Iterator.init(fr); var filename_buf: [std.fs.max_path_bytes]u8 = undefined; while (try iter.next()) |entry| { - const crc32 = try entry.extract(seekable_stream, options, &filename_buf, dest); + const crc32 = try entry.extract(fr, options, &filename_buf, dest); if (crc32 != entry.crc32) return error.ZipCrcMismatch; if (options.diagnostics) |d| { @@ -633,173 +655,6 @@ pub fn extract(dest: std.fs.Dir, seekable_stream: anytype, options: ExtractOptio } } -fn testZip(options: ExtractOptions, comptime files: []const File, write_opt: testutil.WriteZipOptions) !void { - var store: [files.len]FileStore = undefined; - try testZipWithStore(options, files, write_opt, &store); -} -fn testZipWithStore( - options: ExtractOptions, - test_files: []const File, - write_opt: testutil.WriteZipOptions, - store: []FileStore, -) !void { - var zip_buf: [4096]u8 = undefined; - var fbs = try testutil.makeZipWithStore(&zip_buf, test_files, write_opt, store); - - var tmp = testing.tmpDir(.{ .no_follow = true }); - defer tmp.cleanup(); - try extract(tmp.dir, fbs.seekableStream(), options); - try testutil.expectFiles(test_files, tmp.dir, .{}); -} -fn testZipError(expected_error: anyerror, file: File, options: ExtractOptions) !void { - var zip_buf: [4096]u8 = undefined; - var store: [1]FileStore = undefined; - var fbs = try testutil.makeZipWithStore(&zip_buf, &[_]File{file}, .{}, &store); - var tmp = testing.tmpDir(.{ .no_follow = true }); - defer tmp.cleanup(); - try testing.expectError(expected_error, extract(tmp.dir, fbs.seekableStream(), options)); -} - -test "zip one file" { - try testZip(.{}, &[_]File{ - .{ .name = "onefile.txt", .content = "Just a single file\n", .compression = .store }, - }, .{}); -} -test "zip multiple files" { - try testZip(.{ .allow_backslashes = true }, &[_]File{ - .{ .name = "foo", .content = "a foo file\n", .compression = .store }, - .{ .name = "subdir/bar", .content = "bar is this right?\nanother newline\n", .compression = .store }, - .{ .name = "subdir\\whoa", .content = "you can do backslashes", .compression = .store }, - .{ .name = "subdir/another/baz", .content = "bazzy mc bazzerson", .compression = .store }, - }, .{}); -} -test "zip deflated" { - try testZip(.{}, &[_]File{ - .{ .name = "deflateme", .content = "This is a deflated file.\nIt should be smaller in the Zip file1\n", .compression = .deflate }, - // TODO: re-enable this if/when we add support for deflate64 - //.{ .name = "deflateme64", .content = "The 64k version of deflate!\n", .compression = .deflate64 }, - .{ .name = "raw", .content = "Not all files need to be deflated in the same Zip.\n", .compression = .store }, - }, .{}); -} -test "zip verify filenames" { - // no empty filenames - try testZipError(error.ZipBadFilename, .{ .name = "", .content = "", .compression = .store }, .{}); - // no absolute paths - try testZipError(error.ZipBadFilename, .{ .name = "/", .content = "", .compression = .store }, .{}); - try testZipError(error.ZipBadFilename, .{ .name = "/foo", .content = "", .compression = .store }, .{}); - try testZipError(error.ZipBadFilename, .{ .name = "/foo/bar", .content = "", .compression = .store }, .{}); - // no '..' components - try testZipError(error.ZipBadFilename, .{ .name = "..", .content = "", .compression = .store }, .{}); - try testZipError(error.ZipBadFilename, .{ .name = "foo/..", .content = "", .compression = .store }, .{}); - try testZipError(error.ZipBadFilename, .{ .name = "foo/bar/..", .content = "", .compression = .store }, .{}); - try testZipError(error.ZipBadFilename, .{ .name = "foo/bar/../", .content = "", .compression = .store }, .{}); - // no backslashes - try testZipError(error.ZipFilenameHasBackslash, .{ .name = "foo\\bar", .content = "", .compression = .store }, .{}); -} - -test "zip64" { - const test_files = [_]File{ - .{ .name = "fram", .content = "fram foo fro fraba", .compression = .store }, - .{ .name = "subdir/barro", .content = "aljdk;jal;jfd;lajkf", .compression = .store }, - }; - - try testZip(.{}, &test_files, .{ - .end = .{ - .zip64 = .{}, - .record_count_disk = std.math.maxInt(u16), // trigger zip64 - }, - }); - try testZip(.{}, &test_files, .{ - .end = .{ - .zip64 = .{}, - .record_count_total = std.math.maxInt(u16), // trigger zip64 - }, - }); - try testZip(.{}, &test_files, .{ - .end = .{ - .zip64 = .{}, - .record_count_disk = std.math.maxInt(u16), // trigger zip64 - .record_count_total = std.math.maxInt(u16), // trigger zip64 - }, - }); - try testZip(.{}, &test_files, .{ - .end = .{ - .zip64 = .{}, - .central_directory_size = std.math.maxInt(u32), // trigger zip64 - }, - }); - try testZip(.{}, &test_files, .{ - .end = .{ - .zip64 = .{}, - .central_directory_offset = std.math.maxInt(u32), // trigger zip64 - }, - }); - try testZip(.{}, &test_files, .{ - .end = .{ - .zip64 = .{}, - .central_directory_offset = std.math.maxInt(u32), // trigger zip64 - }, - .local_header = .{ - .zip64 = .{ // trigger local header zip64 - .data_size = 16, - }, - .compressed_size = std.math.maxInt(u32), - .uncompressed_size = std.math.maxInt(u32), - .extra_len = 20, - }, - }); -} - -test "bad zip files" { - var tmp = testing.tmpDir(.{ .no_follow = true }); - defer tmp.cleanup(); - var zip_buf: [4096]u8 = undefined; - - const file_a = [_]File{.{ .name = "a", .content = "", .compression = .store }}; - - { - var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .sig = [_]u8{ 1, 2, 3, 4 } } }); - try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{})); - } - { - var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .comment_len = 1 } }); - try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{})); - } - { - var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .comment = "a", .comment_len = 0 } }); - try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{})); - } - { - var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .disk_number = 1 } }); - try testing.expectError(error.ZipMultiDiskUnsupported, extract(tmp.dir, fbs.seekableStream(), .{})); - } - { - var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .central_directory_disk_number = 1 } }); - try testing.expectError(error.ZipMultiDiskUnsupported, extract(tmp.dir, fbs.seekableStream(), .{})); - } - { - var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .record_count_disk = 1 } }); - try testing.expectError(error.ZipDiskRecordCountTooLarge, extract(tmp.dir, fbs.seekableStream(), .{})); - } - { - var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .central_directory_size = 1 } }); - try testing.expectError(error.ZipCdOversized, extract(tmp.dir, fbs.seekableStream(), .{})); - } - { - var fbs = try testutil.makeZip(&zip_buf, &file_a, .{ .end = .{ .central_directory_size = 0 } }); - try testing.expectError(error.ZipCdUndersized, extract(tmp.dir, fbs.seekableStream(), .{})); - } - { - var fbs = try testutil.makeZip(&zip_buf, &file_a, .{ .end = .{ .central_directory_offset = 0 } }); - try testing.expectError(error.ZipBadCdOffset, extract(tmp.dir, fbs.seekableStream(), .{})); - } - { - var fbs = try testutil.makeZip(&zip_buf, &file_a, .{ - .end = .{ - .zip64 = .{ .locator_sig = [_]u8{ 1, 2, 3, 4 } }, - .central_directory_size = std.math.maxInt(u32), // trigger 64 - }, - }); - try testing.expectError(error.ZipBadLocatorSig, extract(tmp.dir, fbs.seekableStream(), .{})); - } +test { + _ = @import("zip/test.zig"); } From a4f05a4588100c5e7f311dd5319e97e394f109a4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 27 Jul 2025 10:25:46 -0700 Subject: [PATCH 02/47] delete flate implementation --- lib/std/compress/flate.zig | 208 ++-- lib/std/compress/flate/BlockWriter.zig | 259 ++--- lib/std/compress/flate/Compress.zig | 977 +++--------------- lib/std/compress/flate/Decompress.zig | 5 +- lib/std/compress/flate/HuffmanEncoder.zig | 475 +++++++++ .../compress/flate/testdata/block_writer.zig | 606 ----------- .../block_writer/huffman-null-max.dyn.expect | Bin 78 -> 0 bytes .../huffman-null-max.dyn.expect-noinput | Bin 78 -> 0 bytes .../block_writer/huffman-null-max.huff.expect | Bin 8204 -> 0 bytes .../block_writer/huffman-null-max.input | Bin 65535 -> 0 bytes .../block_writer/huffman-null-max.wb.expect | Bin 78 -> 0 bytes .../huffman-null-max.wb.expect-noinput | Bin 78 -> 0 bytes .../block_writer/huffman-pi.dyn.expect | Bin 1696 -> 0 bytes .../huffman-pi.dyn.expect-noinput | Bin 1696 -> 0 bytes .../block_writer/huffman-pi.huff.expect | Bin 1606 -> 0 bytes .../testdata/block_writer/huffman-pi.input | 1 - .../block_writer/huffman-pi.wb.expect | Bin 1696 -> 0 bytes .../block_writer/huffman-pi.wb.expect-noinput | Bin 1696 -> 0 bytes .../block_writer/huffman-rand-1k.dyn.expect | Bin 1005 -> 0 bytes .../huffman-rand-1k.dyn.expect-noinput | Bin 1054 -> 0 bytes .../block_writer/huffman-rand-1k.huff.expect | Bin 1005 -> 0 bytes .../block_writer/huffman-rand-1k.input | Bin 1000 -> 0 bytes .../block_writer/huffman-rand-1k.wb.expect | Bin 1005 -> 0 bytes .../huffman-rand-1k.wb.expect-noinput | Bin 1054 -> 0 bytes .../huffman-rand-limit.dyn.expect | Bin 229 -> 0 bytes .../huffman-rand-limit.dyn.expect-noinput | Bin 229 -> 0 bytes .../huffman-rand-limit.huff.expect | Bin 252 -> 0 bytes .../block_writer/huffman-rand-limit.input | 4 - .../block_writer/huffman-rand-limit.wb.expect | Bin 186 -> 0 bytes .../huffman-rand-limit.wb.expect-noinput | Bin 186 -> 0 bytes .../block_writer/huffman-rand-max.huff.expect | Bin 65540 -> 0 bytes .../block_writer/huffman-rand-max.input | Bin 65535 -> 0 bytes .../block_writer/huffman-shifts.dyn.expect | Bin 32 -> 0 bytes .../huffman-shifts.dyn.expect-noinput | Bin 32 -> 0 bytes .../block_writer/huffman-shifts.huff.expect | Bin 1812 -> 0 bytes .../block_writer/huffman-shifts.input | 2 - .../block_writer/huffman-shifts.wb.expect | Bin 32 -> 0 bytes .../huffman-shifts.wb.expect-noinput | Bin 32 -> 0 bytes .../huffman-text-shift.dyn.expect | Bin 231 -> 0 bytes .../huffman-text-shift.dyn.expect-noinput | Bin 231 -> 0 bytes .../huffman-text-shift.huff.expect | Bin 231 -> 0 bytes .../block_writer/huffman-text-shift.input | 14 - .../block_writer/huffman-text-shift.wb.expect | Bin 231 -> 0 bytes .../huffman-text-shift.wb.expect-noinput | Bin 231 -> 0 bytes .../block_writer/huffman-text.dyn.expect | Bin 217 -> 0 bytes .../huffman-text.dyn.expect-noinput | Bin 217 -> 0 bytes .../block_writer/huffman-text.huff.expect | Bin 219 -> 0 bytes .../testdata/block_writer/huffman-text.input | 14 - .../block_writer/huffman-text.wb.expect | Bin 217 -> 0 bytes .../huffman-text.wb.expect-noinput | Bin 217 -> 0 bytes .../block_writer/huffman-zero.dyn.expect | Bin 17 -> 0 bytes .../huffman-zero.dyn.expect-noinput | Bin 17 -> 0 bytes .../block_writer/huffman-zero.huff.expect | Bin 51 -> 0 bytes .../testdata/block_writer/huffman-zero.input | 1 - .../block_writer/huffman-zero.wb.expect | Bin 6 -> 0 bytes .../huffman-zero.wb.expect-noinput | Bin 6 -> 0 bytes .../null-long-match.dyn.expect-noinput | Bin 206 -> 0 bytes .../null-long-match.wb.expect-noinput | Bin 206 -> 0 bytes 58 files changed, 787 insertions(+), 1779 deletions(-) create mode 100644 lib/std/compress/flate/HuffmanEncoder.zig delete mode 100644 lib/std/compress/flate/testdata/block_writer.zig delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-null-max.input delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-pi.input delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.input delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-shifts.input delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-text-shift.input delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-text.input delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-zero.input delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect delete mode 100644 lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput delete mode 100644 lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput diff --git a/lib/std/compress/flate.zig b/lib/std/compress/flate.zig index 5a54643f45..73f98271a4 100644 --- a/lib/std/compress/flate.zig +++ b/lib/std/compress/flate.zig @@ -1,7 +1,7 @@ const builtin = @import("builtin"); const std = @import("../std.zig"); const testing = std.testing; -const Writer = std.io.Writer; +const Writer = std.Io.Writer; /// Container of the deflate bit stream body. Container adds header before /// deflate bit stream and footer after. It can bi gzip, zlib or raw (no header, @@ -77,7 +77,7 @@ pub const Container = enum { raw: void, gzip: struct { crc: std.hash.Crc32 = .init(), - count: usize = 0, + count: u32 = 0, }, zlib: std.hash.Adler32, @@ -98,7 +98,7 @@ pub const Container = enum { .raw => {}, .gzip => |*gzip| { gzip.update(buf); - gzip.count += buf.len; + gzip.count +%= buf.len; }, .zlib => |*zlib| { zlib.update(buf); @@ -148,35 +148,9 @@ pub const Compress = @import("flate/Compress.zig"); /// decompression and correctly produces the original full-size data or file. pub const Decompress = @import("flate/Decompress.zig"); -/// Huffman only compression. Without Lempel-Ziv match searching. Faster -/// compression, less memory requirements but bigger compressed sizes. -pub const huffman = struct { - // The odd order in which the codegen code sizes are written. - pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; - // The number of codegen codes. - pub const codegen_code_count = 19; - - // The largest distance code. - pub const distance_code_count = 30; - - // Maximum number of literals. - pub const max_num_lit = 286; - - // Max number of frequencies used for a Huffman Code - // Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286). - // The largest of these is max_num_lit. - pub const max_num_frequencies = max_num_lit; - - // Biggest block size for uncompressed block. - pub const max_store_block_size = 65535; - // The special code used to mark the end of a block. - pub const end_block_marker = 256; -}; - -test { - _ = Compress; - _ = Decompress; -} +/// Compression without Lempel-Ziv match searching. Faster compression, less +/// memory requirements but bigger compressed sizes. +pub const HuffmanEncoder = @import("flate/HuffmanEncoder.zig"); test "compress/decompress" { const print = std.debug.print; @@ -217,12 +191,11 @@ test "compress/decompress" { }, }; - for (cases, 0..) |case, case_no| { // for each case + for (cases, 0..) |case, case_no| { const data = case.data; - for (levels, 0..) |level, i| { // for each compression level - - inline for (Container.list) |container| { // for each wrapping + for (levels, 0..) |level, i| { + for (Container.list) |container| { var compressed_size: usize = if (case.gzip_sizes[i] > 0) case.gzip_sizes[i] - Container.gzip.size() + container.size() else @@ -230,21 +203,21 @@ test "compress/decompress" { // compress original stream to compressed stream { - var original: std.io.Reader = .fixed(data); var compressed: Writer = .fixed(&cmp_buf); - var compress: Compress = .init(&original, &.{}, .{ .container = .raw, .level = level }); - const n = try compress.reader.streamRemaining(&compressed); + var compress: Compress = .init(&compressed, &.{}, .{ .container = .raw, .level = level }); + try compress.writer.writeAll(data); + try compress.end(); + if (compressed_size == 0) { if (container == .gzip) print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos }); compressed_size = compressed.end; } - try testing.expectEqual(compressed_size, n); try testing.expectEqual(compressed_size, compressed.end); } // decompress compressed stream to decompressed stream { - var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]); + var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]); var decompressed: Writer = .fixed(&dcm_buf); var decompress: Decompress = .init(&compressed, container, &.{}); _ = try decompress.reader.streamRemaining(&decompressed); @@ -266,7 +239,7 @@ test "compress/decompress" { } // decompressor reader interface { - var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]); + var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]); var decompress: Decompress = .init(&compressed, container, &.{}); const n = try decompress.reader.readSliceShort(&dcm_buf); try testing.expectEqual(data.len, n); @@ -276,7 +249,7 @@ test "compress/decompress" { } // huffman only compression { - inline for (Container.list) |container| { // for each wrapping + for (Container.list) |container| { var compressed_size: usize = if (case.huffman_only_size > 0) case.huffman_only_size - Container.gzip.size() + container.size() else @@ -284,7 +257,7 @@ test "compress/decompress" { // compress original stream to compressed stream { - var original: std.io.Reader = .fixed(data); + var original: std.Io.Reader = .fixed(data); var compressed: Writer = .fixed(&cmp_buf); var cmp = try Compress.Huffman.init(container, &compressed); try cmp.compress(original.reader()); @@ -298,7 +271,7 @@ test "compress/decompress" { } // decompress compressed stream to decompressed stream { - var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]); + var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]); var decompress: Decompress = .init(&compressed, container, &.{}); var decompressed: Writer = .fixed(&dcm_buf); _ = try decompress.reader.streamRemaining(&decompressed); @@ -309,7 +282,7 @@ test "compress/decompress" { // store only { - inline for (Container.list) |container| { // for each wrapping + for (Container.list) |container| { var compressed_size: usize = if (case.store_size > 0) case.store_size - Container.gzip.size() + container.size() else @@ -317,7 +290,7 @@ test "compress/decompress" { // compress original stream to compressed stream { - var original: std.io.Reader = .fixed(data); + var original: std.Io.Reader = .fixed(data); var compressed: Writer = .fixed(&cmp_buf); var cmp = try Compress.SimpleCompressor(.store, container).init(&compressed); try cmp.compress(original.reader()); @@ -332,7 +305,7 @@ test "compress/decompress" { } // decompress compressed stream to decompressed stream { - var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]); + var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]); var decompress: Decompress = .init(&compressed, container, &.{}); var decompressed: Writer = .fixed(&dcm_buf); _ = try decompress.reader.streamRemaining(&decompressed); @@ -344,13 +317,13 @@ test "compress/decompress" { } fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void { - var in: std.io.Reader = .fixed(compressed); - var aw: std.io.Writer.Allocating = .init(testing.allocator); + var in: std.Io.Reader = .fixed(compressed); + var aw: std.Io.Writer.Allocating = .init(testing.allocator); defer aw.deinit(); var decompress: Decompress = .init(&in, container, &.{}); _ = try decompress.reader.streamRemaining(&aw.writer); - try testing.expectEqualSlices(u8, expected_plain, aw.items); + try testing.expectEqualSlices(u8, expected_plain, aw.getWritten()); } test "don't read past deflate stream's end" { @@ -483,17 +456,12 @@ test "public interface" { var buffer1: [64]u8 = undefined; var buffer2: [64]u8 = undefined; - // TODO These used to be functions, need to migrate the tests - const decompress = void; - const compress = void; - const store = void; - // decompress { var plain: Writer = .fixed(&buffer2); - - var in: std.io.Reader = .fixed(gzip_data); - try decompress(&in, &plain); + var in: std.Io.Reader = .fixed(gzip_data); + var d: Decompress = .init(&in, .raw, &.{}); + _ = try d.reader.streamRemaining(&plain); try testing.expectEqualSlices(u8, plain_data, plain.buffered()); } @@ -502,11 +470,13 @@ test "public interface" { var plain: Writer = .fixed(&buffer2); var compressed: Writer = .fixed(&buffer1); - var in: std.io.Reader = .fixed(plain_data); - try compress(&in, &compressed, .{}); + var cmp: Compress = .init(&compressed, &.{}, .{}); + try cmp.writer.writeAll(plain_data); + try cmp.end(); - var r: std.io.Reader = .fixed(&buffer1); - try decompress(&r, &plain); + var r: std.Io.Reader = .fixed(&buffer1); + var d: Decompress = .init(&r, .raw, &.{}); + _ = try d.reader.streamRemaining(&plain); try testing.expectEqualSlices(u8, plain_data, plain.buffered()); } @@ -515,12 +485,11 @@ test "public interface" { var plain: Writer = .fixed(&buffer2); var compressed: Writer = .fixed(&buffer1); - var in: std.io.Reader = .fixed(plain_data); - var cmp = try Compress(&compressed, .{}); - try cmp.compress(&in); - try cmp.finish(); + var cmp: Compress = .init(&compressed, &.{}, .{}); + try cmp.writer.writeAll(plain_data); + try cmp.end(); - var r: std.io.Reader = .fixed(&buffer1); + var r: std.Io.Reader = .fixed(&buffer1); var dcp = Decompress(&r); try dcp.decompress(&plain); try testing.expectEqualSlices(u8, plain_data, plain.buffered()); @@ -533,11 +502,12 @@ test "public interface" { var plain: Writer = .fixed(&buffer2); var compressed: Writer = .fixed(&buffer1); - var in: std.io.Reader = .fixed(plain_data); - try huffman.compress(&in, &compressed); + var in: std.Io.Reader = .fixed(plain_data); + try HuffmanEncoder.compress(&in, &compressed); - var r: std.io.Reader = .fixed(&buffer1); - try decompress(&r, &plain); + var r: std.Io.Reader = .fixed(&buffer1); + var d: Decompress = .init(&r, .raw, &.{}); + _ = try d.reader.streamRemaining(&plain); try testing.expectEqualSlices(u8, plain_data, plain.buffered()); } @@ -546,47 +516,50 @@ test "public interface" { var plain: Writer = .fixed(&buffer2); var compressed: Writer = .fixed(&buffer1); - var in: std.io.Reader = .fixed(plain_data); - var cmp = try huffman.Compressor(&compressed); + var in: std.Io.Reader = .fixed(plain_data); + var cmp = try HuffmanEncoder.Compressor(&compressed); try cmp.compress(&in); try cmp.finish(); - var r: std.io.Reader = .fixed(&buffer1); - try decompress(&r, &plain); + var r: std.Io.Reader = .fixed(&buffer1); + var d: Decompress = .init(&r, .raw, &.{}); + _ = try d.reader.streamRemaining(&plain); try testing.expectEqualSlices(u8, plain_data, plain.buffered()); } } - // store - { - // store compress/decompress - { - var plain: Writer = .fixed(&buffer2); - var compressed: Writer = .fixed(&buffer1); + // TODO + //{ + // // store compress/decompress + // { + // var plain: Writer = .fixed(&buffer2); + // var compressed: Writer = .fixed(&buffer1); - var in: std.io.Reader = .fixed(plain_data); - try store.compress(&in, &compressed); + // var in: std.Io.Reader = .fixed(plain_data); + // try store.compress(&in, &compressed); - var r: std.io.Reader = .fixed(&buffer1); - try decompress(&r, &plain); - try testing.expectEqualSlices(u8, plain_data, plain.buffered()); - } + // var r: std.Io.Reader = .fixed(&buffer1); + // var d: Decompress = .init(&r, .raw, &.{}); + // _ = try d.reader.streamRemaining(&plain); + // try testing.expectEqualSlices(u8, plain_data, plain.buffered()); + // } - // store compressor/decompressor - { - var plain: Writer = .fixed(&buffer2); - var compressed: Writer = .fixed(&buffer1); + // // store compressor/decompressor + // { + // var plain: Writer = .fixed(&buffer2); + // var compressed: Writer = .fixed(&buffer1); - var in: std.io.Reader = .fixed(plain_data); - var cmp = try store.compressor(&compressed); - try cmp.compress(&in); - try cmp.finish(); + // var in: std.Io.Reader = .fixed(plain_data); + // var cmp = try store.compressor(&compressed); + // try cmp.compress(&in); + // try cmp.finish(); - var r: std.io.Reader = .fixed(&buffer1); - try decompress(&r, &plain); - try testing.expectEqualSlices(u8, plain_data, plain.buffered()); - } - } + // var r: std.Io.Reader = .fixed(&buffer1); + // var d: Decompress = .init(&r, .raw, &.{}); + // _ = try d.reader.streamRemaining(&plain); + // try testing.expectEqualSlices(u8, plain_data, plain.buffered()); + // } + //} } pub const match = struct { @@ -615,26 +588,33 @@ test "zlib should not overshoot" { 0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12, }; - var stream: std.io.Reader = .fixed(&data); - const reader = stream.reader(); + var reader: std.Io.Reader = .fixed(&data); - var dcp = Decompress.init(reader); + var decompress: Decompress = .init(&reader, .zlib, &.{}); var out: [128]u8 = undefined; - // Decompress - var n = try dcp.reader().readAll(out[0..]); + { + const n = try decompress.reader.readSliceShort(out[0..]); - // Expected decompressed data - try std.testing.expectEqual(46, n); - try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]); + // Expected decompressed data + try std.testing.expectEqual(46, n); + try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]); - // Decompressor don't overshoot underlying reader. - // It is leaving it at the end of compressed data chunk. - try std.testing.expectEqual(data.len - 4, stream.getPos()); - try std.testing.expectEqual(0, dcp.unreadBytes()); + // Decompressor don't overshoot underlying reader. + // It is leaving it at the end of compressed data chunk. + try std.testing.expectEqual(data.len - 4, reader.seek); + // TODO what was this testing, exactly? + //try std.testing.expectEqual(0, decompress.unreadBytes()); + } // 4 bytes after compressed chunk are available in reader. - n = try reader.readAll(out[0..]); + const n = try reader.readSliceShort(out[0..]); try std.testing.expectEqual(n, 4); try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]); } + +test { + _ = HuffmanEncoder; + _ = Compress; + _ = Decompress; +} diff --git a/lib/std/compress/flate/BlockWriter.zig b/lib/std/compress/flate/BlockWriter.zig index d1eb3a068e..b3af65051a 100644 --- a/lib/std/compress/flate/BlockWriter.zig +++ b/lib/std/compress/flate/BlockWriter.zig @@ -8,32 +8,33 @@ const Writer = std.io.Writer; const BlockWriter = @This(); const flate = @import("../flate.zig"); const Compress = flate.Compress; -const huffman = flate.huffman; +const HuffmanEncoder = flate.HuffmanEncoder; const Token = @import("Token.zig"); -const codegen_order = huffman.codegen_order; +const codegen_order = HuffmanEncoder.codegen_order; const end_code_mark = 255; output: *Writer, -codegen_freq: [huffman.codegen_code_count]u16 = undefined, -literal_freq: [huffman.max_num_lit]u16 = undefined, -distance_freq: [huffman.distance_code_count]u16 = undefined, -codegen: [huffman.max_num_lit + huffman.distance_code_count + 1]u8 = undefined, -literal_encoding: Compress.LiteralEncoder = .{}, -distance_encoding: Compress.DistanceEncoder = .{}, -codegen_encoding: Compress.CodegenEncoder = .{}, -fixed_literal_encoding: Compress.LiteralEncoder, -fixed_distance_encoding: Compress.DistanceEncoder, -huff_distance: Compress.DistanceEncoder, +codegen_freq: [HuffmanEncoder.codegen_code_count]u16, +literal_freq: [HuffmanEncoder.max_num_lit]u16, +distance_freq: [HuffmanEncoder.distance_code_count]u16, +codegen: [HuffmanEncoder.max_num_lit + HuffmanEncoder.distance_code_count + 1]u8, +literal_encoding: HuffmanEncoder, +distance_encoding: HuffmanEncoder, +codegen_encoding: HuffmanEncoder, +fixed_literal_encoding: HuffmanEncoder, +fixed_distance_encoding: HuffmanEncoder, +huff_distance: HuffmanEncoder, -pub fn init(output: *Writer) BlockWriter { - return .{ - .output = output, - .fixed_literal_encoding = Compress.fixedLiteralEncoder(), - .fixed_distance_encoding = Compress.fixedDistanceEncoder(), - .huff_distance = Compress.huffmanDistanceEncoder(), - }; +fixed_literal_codes: [HuffmanEncoder.max_num_frequencies]HuffmanEncoder.Code, +fixed_distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code, +distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code, + +pub fn init(bw: *BlockWriter) void { + bw.fixed_literal_encoding = .fixedLiteralEncoder(&bw.fixed_literal_codes); + bw.fixed_distance_encoding = .fixedDistanceEncoder(&bw.fixed_distance_codes); + bw.huff_distance = .huffmanDistanceEncoder(&bw.distance_codes); } /// Flush intrenal bit buffer to the writer. @@ -46,27 +47,23 @@ pub fn flush(self: *BlockWriter) Writer.Error!void { try self.bit_writer.flush(); } -pub fn setWriter(self: *BlockWriter, new_writer: *Writer) void { - self.bit_writer.setWriter(new_writer); -} - fn writeCode(self: *BlockWriter, c: Compress.HuffCode) Writer.Error!void { try self.bit_writer.writeBits(c.code, c.len); } -// RFC 1951 3.2.7 specifies a special run-length encoding for specifying -// the literal and distance lengths arrays (which are concatenated into a single -// array). This method generates that run-length encoding. -// -// The result is written into the codegen array, and the frequencies -// of each code is written into the codegen_freq array. -// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional -// information. Code bad_code is an end marker -// -// num_literals: The number of literals in literal_encoding -// num_distances: The number of distances in distance_encoding -// lit_enc: The literal encoder to use -// dist_enc: The distance encoder to use +/// RFC 1951 3.2.7 specifies a special run-length encoding for specifying +/// the literal and distance lengths arrays (which are concatenated into a single +/// array). This method generates that run-length encoding. +/// +/// The result is written into the codegen array, and the frequencies +/// of each code is written into the codegen_freq array. +/// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional +/// information. Code bad_code is an end marker +/// +/// num_literals: The number of literals in literal_encoding +/// num_distances: The number of distances in distance_encoding +/// lit_enc: The literal encoder to use +/// dist_enc: The distance encoder to use fn generateCodegen( self: *BlockWriter, num_literals: u32, @@ -167,7 +164,7 @@ const DynamicSize = struct { num_codegens: u32, }; -// dynamicSize returns the size of dynamically encoded data in bits. +/// dynamicSize returns the size of dynamically encoded data in bits. fn dynamicSize( self: *BlockWriter, lit_enc: *Compress.LiteralEncoder, // literal encoder @@ -194,7 +191,7 @@ fn dynamicSize( }; } -// fixedSize returns the size of dynamically encoded data in bits. +/// fixedSize returns the size of dynamically encoded data in bits. fn fixedSize(self: *BlockWriter, extra_bits: u32) u32 { return 3 + self.fixed_literal_encoding.bitLength(&self.literal_freq) + @@ -207,25 +204,25 @@ const StoredSize = struct { storable: bool, }; -// storedSizeFits calculates the stored size, including header. -// The function returns the size in bits and whether the block -// fits inside a single block. +/// storedSizeFits calculates the stored size, including header. +/// The function returns the size in bits and whether the block +/// fits inside a single block. fn storedSizeFits(in: ?[]const u8) StoredSize { if (in == null) { return .{ .size = 0, .storable = false }; } - if (in.?.len <= huffman.max_store_block_size) { + if (in.?.len <= HuffmanEncoder.max_store_block_size) { return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true }; } return .{ .size = 0, .storable = false }; } -// Write the header of a dynamic Huffman block to the output stream. -// -// num_literals: The number of literals specified in codegen -// num_distances: The number of distances specified in codegen -// num_codegens: The number of codegens used in codegen -// eof: Is it the end-of-file? (end of stream) +/// Write the header of a dynamic Huffman block to the output stream. +/// +/// num_literals: The number of literals specified in codegen +/// num_distances: The number of distances specified in codegen +/// num_codegens: The number of codegens used in codegen +/// eof: Is it the end-of-file? (end of stream) fn dynamicHeader( self: *BlockWriter, num_literals: u32, @@ -291,11 +288,11 @@ fn fixedHeader(self: *BlockWriter, eof: bool) Writer.Error!void { try self.bit_writer.writeBits(value, 3); } -// Write a block of tokens with the smallest encoding. Will choose block type. -// The original input can be supplied, and if the huffman encoded data -// is larger than the original bytes, the data will be written as a -// stored block. -// If the input is null, the tokens will always be Huffman encoded. +/// Write a block of tokens with the smallest encoding. Will choose block type. +/// The original input can be supplied, and if the huffman encoded data +/// is larger than the original bytes, the data will be written as a +/// stored block. +/// If the input is null, the tokens will always be Huffman encoded. pub fn write(self: *BlockWriter, tokens: []const Token, eof: bool, input: ?[]const u8) Writer.Error!void { const lit_and_dist = self.indexTokens(tokens); const num_literals = lit_and_dist.num_literals; @@ -379,11 +376,11 @@ pub fn storedBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Erro try self.bit_writer.writeBytes(input); } -// writeBlockDynamic encodes a block using a dynamic Huffman table. -// This should be used if the symbols used have a disproportionate -// histogram distribution. -// If input is supplied and the compression savings are below 1/16th of the -// input size the block is stored. +/// writeBlockDynamic encodes a block using a dynamic Huffman table. +/// This should be used if the symbols used have a disproportionate +/// histogram distribution. +/// If input is supplied and the compression savings are below 1/16th of the +/// input size the block is stored. fn dynamicBlock( self: *BlockWriter, tokens: []const Token, @@ -429,10 +426,10 @@ const TotalIndexedTokens = struct { num_distances: u32, }; -// Indexes a slice of tokens followed by an end_block_marker, and updates -// literal_freq and distance_freq, and generates literal_encoding -// and distance_encoding. -// The number of literal and distance tokens is returned. +/// Indexes a slice of tokens followed by an end_block_marker, and updates +/// literal_freq and distance_freq, and generates literal_encoding +/// and distance_encoding. +/// The number of literal and distance tokens is returned. fn indexTokens(self: *BlockWriter, tokens: []const Token) TotalIndexedTokens { var num_literals: u32 = 0; var num_distances: u32 = 0; @@ -453,7 +450,7 @@ fn indexTokens(self: *BlockWriter, tokens: []const Token) TotalIndexedTokens { self.distance_freq[t.distanceCode()] += 1; } // add end_block_marker token at the end - self.literal_freq[huffman.end_block_marker] += 1; + self.literal_freq[HuffmanEncoder.end_block_marker] += 1; // get the number of literals num_literals = @as(u32, @intCast(self.literal_freq.len)); @@ -479,8 +476,8 @@ fn indexTokens(self: *BlockWriter, tokens: []const Token) TotalIndexedTokens { }; } -// Writes a slice of tokens to the output followed by and end_block_marker. -// codes for literal and distance encoding must be supplied. +/// Writes a slice of tokens to the output followed by and end_block_marker. +/// codes for literal and distance encoding must be supplied. fn writeTokens( self: *BlockWriter, tokens: []const Token, @@ -508,18 +505,18 @@ fn writeTokens( } } // add end_block_marker at the end - try self.writeCode(le_codes[huffman.end_block_marker]); + try self.writeCode(le_codes[HuffmanEncoder.end_block_marker]); } -// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes -// if the results only gains very little from compression. +/// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes +/// if the results only gains very little from compression. pub fn huffmanBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Error!void { // Add everything as literals histogram(input, &self.literal_freq); - self.literal_freq[huffman.end_block_marker] = 1; + self.literal_freq[HuffmanEncoder.end_block_marker] = 1; - const num_literals = huffman.end_block_marker + 1; + const num_literals = HuffmanEncoder.end_block_marker + 1; self.distance_freq[0] = 1; const num_distances = 1; @@ -560,10 +557,9 @@ pub fn huffmanBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Err const c = encoding[t]; try self.bit_writer.writeBits(c.code, c.len); } - try self.writeCode(encoding[huffman.end_block_marker]); + try self.writeCode(encoding[HuffmanEncoder.end_block_marker]); } -// histogram accumulates a histogram of b in h. fn histogram(b: []const u8, h: *[286]u16) void { // Clear histogram for (h, 0..) |_, i| { @@ -575,122 +571,3 @@ fn histogram(b: []const u8, h: *[286]u16) void { lh[t] += 1; } } - -// tests -const expect = std.testing.expect; -const fmt = std.fmt; -const testing = std.testing; -const ArrayList = std.ArrayList; - -const TestCase = @import("testdata/block_writer.zig").TestCase; -const testCases = @import("testdata/block_writer.zig").testCases; - -// tests if the writeBlock encoding has changed. -test "write" { - inline for (0..testCases.len) |i| { - try testBlock(testCases[i], .write_block); - } -} - -// tests if the writeBlockDynamic encoding has changed. -test "dynamicBlock" { - inline for (0..testCases.len) |i| { - try testBlock(testCases[i], .write_dyn_block); - } -} - -test "huffmanBlock" { - inline for (0..testCases.len) |i| { - try testBlock(testCases[i], .write_huffman_block); - } - try testBlock(.{ - .tokens = &[_]Token{}, - .input = "huffman-rand-max.input", - .want = "huffman-rand-max.{s}.expect", - }, .write_huffman_block); -} - -const TestFn = enum { - write_block, - write_dyn_block, // write dynamic block - write_huffman_block, - - fn to_s(self: TestFn) []const u8 { - return switch (self) { - .write_block => "wb", - .write_dyn_block => "dyn", - .write_huffman_block => "huff", - }; - } - - fn write( - comptime self: TestFn, - bw: anytype, - tok: []const Token, - input: ?[]const u8, - final: bool, - ) !void { - switch (self) { - .write_block => try bw.write(tok, final, input), - .write_dyn_block => try bw.dynamicBlock(tok, final, input), - .write_huffman_block => try bw.huffmanBlock(input.?, final), - } - try bw.flush(); - } -}; - -// testBlock tests a block against its references -// -// size -// 64K [file-name].input - input non compressed file -// 8.1K [file-name].golden - -// 78 [file-name].dyn.expect - output with writeBlockDynamic -// 78 [file-name].wb.expect - output with writeBlock -// 8.1K [file-name].huff.expect - output with writeBlockHuff -// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null -// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null -// -// wb - writeBlock -// dyn - writeBlockDynamic -// huff - writeBlockHuff -// -fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void { - if (tc.input.len != 0 and tc.want.len != 0) { - const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()}); - const input = @embedFile("testdata/block_writer/" ++ tc.input); - const want = @embedFile("testdata/block_writer/" ++ want_name); - try testWriteBlock(tfn, input, want, tc.tokens); - } - - if (tfn == .write_huffman_block) { - return; - } - - const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()}); - const want = @embedFile("testdata/block_writer/" ++ want_name_no_input); - try testWriteBlock(tfn, null, want, tc.tokens); -} - -// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output. -fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void { - var buf = ArrayList(u8).init(testing.allocator); - var bw: BlockWriter = .init(buf.writer()); - try tfn.write(&bw, tokens, input, false); - var got = buf.items; - try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result - try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set - // - // Test if the writer produces the same output after reset. - buf.deinit(); - buf = ArrayList(u8).init(testing.allocator); - defer buf.deinit(); - bw.setWriter(buf.writer()); - - try tfn.write(&bw, tokens, input, true); - try bw.flush(); - got = buf.items; - - try expect(got[0] & 1 == 1); // bfinal is set - buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices - try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result -} diff --git a/lib/std/compress/flate/Compress.zig b/lib/std/compress/flate/Compress.zig index 4d827fd590..f38f7b2703 100644 --- a/lib/std/compress/flate/Compress.zig +++ b/lib/std/compress/flate/Compress.zig @@ -39,6 +39,7 @@ //! //! //! Allocates statically ~400K (192K lookup, 128K tokens, 64K window). + const builtin = @import("builtin"); const std = @import("std"); const assert = std.debug.assert; @@ -47,7 +48,6 @@ const expect = testing.expect; const mem = std.mem; const math = std.math; const Writer = std.Io.Writer; -const Reader = std.Io.Reader; const Compress = @This(); const Token = @import("Token.zig"); @@ -55,22 +55,24 @@ const BlockWriter = @import("BlockWriter.zig"); const flate = @import("../flate.zig"); const Container = flate.Container; const Lookup = @import("Lookup.zig"); -const huffman = flate.huffman; +const HuffmanEncoder = flate.HuffmanEncoder; +const LiteralNode = HuffmanEncoder.LiteralNode; lookup: Lookup = .{}, tokens: Tokens = .{}, -/// Asserted to have a buffer capacity of at least `flate.max_window_len`. -input: *Reader, block_writer: BlockWriter, level: LevelArgs, hasher: Container.Hasher, -reader: Reader, +writer: Writer, +state: State, // Match and literal at the previous position. // Used for lazy match finding in processWindow. prev_match: ?Token = null, prev_literal: ?u8 = null, +pub const State = enum { header, middle, ended }; + /// Trades between speed and compression size. /// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43) /// levels 1-3 are using different algorithm to perform faster but with less @@ -118,188 +120,34 @@ pub const Options = struct { container: Container = .raw, }; -pub fn init(input: *Reader, buffer: []u8, options: Options) Compress { +pub fn init(output: *Writer, buffer: []u8, options: Options) Compress { return .{ - .input = input, - .block_writer = undefined, + .block_writer = .{ + .output = output, + .codegen_freq = undefined, + .literal_freq = undefined, + .distance_freq = undefined, + .codegen = undefined, + .literal_encoding = undefined, + .distance_encoding = undefined, + .codegen_encoding = undefined, + .fixed_literal_encoding = undefined, + .fixed_distance_encoding = undefined, + .huff_distance = undefined, + .fixed_literal_codes = undefined, + .fixed_distance_codes = undefined, + .distance_codes = undefined, + }, .level = .get(options.level), .hasher = .init(options.container), .state = .header, - .reader = .{ + .writer = .{ .buffer = buffer, - .stream = stream, + .vtable = &.{ .drain = drain }, }, }; } -const FlushOption = enum { none, flush, final }; - -/// Process data in window and create tokens. If token buffer is full -/// flush tokens to the token writer. -/// -/// Returns number of bytes consumed from `lh`. -fn tokenizeSlice(c: *Compress, bw: *Writer, limit: std.Io.Limit, lh: []const u8) !usize { - _ = bw; - _ = limit; - if (true) @panic("TODO"); - var step: u16 = 1; // 1 in the case of literal, match length otherwise - const pos: u16 = c.win.pos(); - const literal = lh[0]; // literal at current position - const min_len: u16 = if (c.prev_match) |m| m.length() else 0; - - // Try to find match at least min_len long. - if (c.findMatch(pos, lh, min_len)) |match| { - // Found better match than previous. - try c.addPrevLiteral(); - - // Is found match length good enough? - if (match.length() >= c.level.lazy) { - // Don't try to lazy find better match, use this. - step = try c.addMatch(match); - } else { - // Store this match. - c.prev_literal = literal; - c.prev_match = match; - } - } else { - // There is no better match at current pos then it was previous. - // Write previous match or literal. - if (c.prev_match) |m| { - // Write match from previous position. - step = try c.addMatch(m) - 1; // we already advanced 1 from previous position - } else { - // No match at previous position. - // Write previous literal if any, and remember this literal. - try c.addPrevLiteral(); - c.prev_literal = literal; - } - } - // Advance window and add hashes. - c.windowAdvance(step, lh, pos); -} - -fn windowAdvance(self: *Compress, step: u16, lh: []const u8, pos: u16) void { - // current position is already added in findMatch - self.lookup.bulkAdd(lh[1..], step - 1, pos + 1); - self.win.advance(step); -} - -// Add previous literal (if any) to the tokens list. -fn addPrevLiteral(self: *Compress) !void { - if (self.prev_literal) |l| try self.addToken(Token.initLiteral(l)); -} - -// Add match to the tokens list, reset prev pointers. -// Returns length of the added match. -fn addMatch(self: *Compress, m: Token) !u16 { - try self.addToken(m); - self.prev_literal = null; - self.prev_match = null; - return m.length(); -} - -fn addToken(self: *Compress, token: Token) !void { - self.tokens.add(token); - if (self.tokens.full()) try self.flushTokens(.none); -} - -// Finds largest match in the history window with the data at current pos. -fn findMatch(self: *Compress, pos: u16, lh: []const u8, min_len: u16) ?Token { - var len: u16 = min_len; - // Previous location with the same hash (same 4 bytes). - var prev_pos = self.lookup.add(lh, pos); - // Last found match. - var match: ?Token = null; - - // How much back-references to try, performance knob. - var chain: usize = self.level.chain; - if (len >= self.level.good) { - // If we've got a match that's good enough, only look in 1/4 the chain. - chain >>= 2; - } - - // Hot path loop! - while (prev_pos > 0 and chain > 0) : (chain -= 1) { - const distance = pos - prev_pos; - if (distance > flate.match.max_distance) - break; - - const new_len = self.win.match(prev_pos, pos, len); - if (new_len > len) { - match = Token.initMatch(@intCast(distance), new_len); - if (new_len >= self.level.nice) { - // The match is good enough that we don't try to find a better one. - return match; - } - len = new_len; - } - prev_pos = self.lookup.prev(prev_pos); - } - - return match; -} - -fn flushTokens(self: *Compress, flush_opt: FlushOption) !void { - // Pass tokens to the token writer - try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer()); - // Stored block ensures byte alignment. - // It has 3 bits (final, block_type) and then padding until byte boundary. - // After that everything is aligned to the boundary in the stored block. - // Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF. - // Last 4 bytes are byte aligned. - if (flush_opt == .flush) { - try self.block_writer.storedBlock("", false); - } - if (flush_opt != .none) { - // Safe to call only when byte aligned or it is OK to add - // padding bits (on last byte of the final block). - try self.block_writer.flush(); - } - // Reset internal tokens store. - self.tokens.reset(); - // Notify win that tokens are flushed. - self.win.flush(); -} - -// Slide win and if needed lookup tables. -fn slide(self: *Compress) void { - const n = self.win.slide(); - self.lookup.slide(n); -} - -/// Flushes internal buffers to the output writer. Outputs empty stored -/// block to sync bit stream to the byte boundary, so that the -/// decompressor can get all input data available so far. -/// -/// It is useful mainly in compressed network protocols, to ensure that -/// deflate bit stream can be used as byte stream. May degrade -/// compression so it should be used only when necessary. -/// -/// Completes the current deflate block and follows it with an empty -/// stored block that is three zero bits plus filler bits to the next -/// byte, followed by four bytes (00 00 ff ff). -/// -pub fn flush(c: *Compress) !void { - try c.tokenize(.flush); -} - -/// Completes deflate bit stream by writing any pending data as deflate -/// final deflate block. HAS to be called once all data are written to -/// the compressor as a signal that next block has to have final bit -/// set. -/// -pub fn finish(c: *Compress) !void { - _ = c; - @panic("TODO"); -} - -/// Use another writer while preserving history. Most probably flush -/// should be called on old writer before setting new. -pub fn setWriter(self: *Compress, new_writer: *Writer) void { - self.block_writer.setWriter(new_writer); - self.output = new_writer; -} - // Tokens store const Tokens = struct { list: [n_tokens]Token = undefined, @@ -323,528 +171,111 @@ const Tokens = struct { } }; -/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and -/// only performs Huffman entropy encoding. Results in faster compression, much -/// less memory requirements during compression but bigger compressed sizes. -pub const Huffman = SimpleCompressor(.huffman, .raw); - -/// Creates store blocks only. Data are not compressed only packed into deflate -/// store blocks. That adds 9 bytes of header for each block. Max stored block -/// size is 64K. Block is emitted when flush is called on on finish. -pub const store = struct { - pub fn Compressor(comptime container: Container, comptime WriterType: type) type { - return SimpleCompressor(.store, container, WriterType); - } - - pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) { - return try store.Compressor(container, @TypeOf(writer)).init(writer); - } -}; - -const SimpleCompressorKind = enum { - huffman, - store, -}; - -fn simpleCompressor( - comptime kind: SimpleCompressorKind, - comptime container: Container, - writer: anytype, -) !SimpleCompressor(kind, container, @TypeOf(writer)) { - return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer); -} - -fn SimpleCompressor( - comptime kind: SimpleCompressorKind, - comptime container: Container, - comptime WriterType: type, -) type { - const BlockWriterType = BlockWriter(WriterType); - return struct { - buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes - wp: usize = 0, - - output: WriterType, - block_writer: BlockWriterType, - hasher: container.Hasher() = .{}, - - const Self = @This(); - - pub fn init(output: WriterType) !Self { - const self = Self{ - .output = output, - .block_writer = BlockWriterType.init(output), - }; - try container.writeHeader(self.output); - return self; - } - - pub fn flush(self: *Self) !void { - try self.flushBuffer(false); - try self.block_writer.storedBlock("", false); - try self.block_writer.flush(); - } - - pub fn finish(self: *Self) !void { - try self.flushBuffer(true); - try self.block_writer.flush(); - try container.writeFooter(&self.hasher, self.output); - } - - fn flushBuffer(self: *Self, final: bool) !void { - const buf = self.buffer[0..self.wp]; - switch (kind) { - .huffman => try self.block_writer.huffmanBlock(buf, final), - .store => try self.block_writer.storedBlock(buf, final), - } - self.wp = 0; - } - }; -} - -const LiteralNode = struct { - literal: u16, - freq: u16, -}; - -// Describes the state of the constructed tree for a given depth. -const LevelInfo = struct { - // Our level. for better printing - level: u32, - - // The frequency of the last node at this level - last_freq: u32, - - // The frequency of the next character to add to this level - next_char_freq: u32, - - // The frequency of the next pair (from level below) to add to this level. - // Only valid if the "needed" value of the next lower level is 0. - next_pair_freq: u32, - - // The number of chains remaining to generate for this level before moving - // up to the next level - needed: u32, -}; - -// hcode is a huffman code with a bit code and bit length. -pub const HuffCode = struct { - code: u16 = 0, - len: u16 = 0, - - // set sets the code and length of an hcode. - fn set(self: *HuffCode, code: u16, length: u16) void { - self.len = length; - self.code = code; - } -}; - -pub fn HuffmanEncoder(comptime size: usize) type { - return struct { - codes: [size]HuffCode = undefined, - // Reusable buffer with the longest possible frequency table. - freq_cache: [huffman.max_num_frequencies + 1]LiteralNode = undefined, - bit_count: [17]u32 = undefined, - lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate - lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate - - const Self = @This(); - - // Update this Huffman Code object to be the minimum code for the specified frequency count. - // - // freq An array of frequencies, in which frequency[i] gives the frequency of literal i. - // max_bits The maximum number of bits to use for any literal. - pub fn generate(self: *Self, freq: []u16, max_bits: u32) void { - var list = self.freq_cache[0 .. freq.len + 1]; - // Number of non-zero literals - var count: u32 = 0; - // Set list to be the set of all non-zero literals and their frequencies - for (freq, 0..) |f, i| { - if (f != 0) { - list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f }; - count += 1; - } else { - list[count] = LiteralNode{ .literal = 0x00, .freq = 0 }; - self.codes[i].len = 0; - } - } - list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 }; - - list = list[0..count]; - if (count <= 2) { - // Handle the small cases here, because they are awkward for the general case code. With - // two or fewer literals, everything has bit length 1. - for (list, 0..) |node, i| { - // "list" is in order of increasing literal value. - self.codes[node.literal].set(@as(u16, @intCast(i)), 1); - } - return; - } - self.lfs = list; - mem.sort(LiteralNode, self.lfs, {}, byFreq); - - // Get the number of literals for each bit count - const bit_count = self.bitCounts(list, max_bits); - // And do the assignment - self.assignEncodingAndSize(bit_count, list); - } - - pub fn bitLength(self: *Self, freq: []u16) u32 { - var total: u32 = 0; - for (freq, 0..) |f, i| { - if (f != 0) { - total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len)); - } - } - return total; - } - - // Return the number of literals assigned to each bit size in the Huffman encoding - // - // This method is only called when list.len >= 3 - // The cases of 0, 1, and 2 literals are handled by special case code. - // - // list: An array of the literals with non-zero frequencies - // and their associated frequencies. The array is in order of increasing - // frequency, and has as its last element a special element with frequency - // `math.maxInt(i32)` - // - // max_bits: The maximum number of bits that should be used to encode any literal. - // Must be less than 16. - // - // Returns an integer array in which array[i] indicates the number of literals - // that should be encoded in i bits. - fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 { - var max_bits = max_bits_to_use; - const n = list.len; - const max_bits_limit = 16; - - assert(max_bits < max_bits_limit); - - // The tree can't have greater depth than n - 1, no matter what. This - // saves a little bit of work in some small cases - max_bits = @min(max_bits, n - 1); - - // Create information about each of the levels. - // A bogus "Level 0" whose sole purpose is so that - // level1.prev.needed == 0. This makes level1.next_pair_freq - // be a legitimate value that never gets chosen. - var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo); - // leaf_counts[i] counts the number of literals at the left - // of ancestors of the rightmost node at level i. - // leaf_counts[i][j] is the number of literals at the left - // of the level j ancestor. - var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32); - - { - var level = @as(u32, 1); - while (level <= max_bits) : (level += 1) { - // For every level, the first two items are the first two characters. - // We initialize the levels as if we had already figured this out. - levels[level] = LevelInfo{ - .level = level, - .last_freq = list[1].freq, - .next_char_freq = list[2].freq, - .next_pair_freq = list[0].freq + list[1].freq, - .needed = 0, - }; - leaf_counts[level][level] = 2; - if (level == 1) { - levels[level].next_pair_freq = math.maxInt(i32); - } - } - } - - // We need a total of 2*n - 2 items at top level and have already generated 2. - levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4; - - { - var level = max_bits; - while (true) { - var l = &levels[level]; - if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) { - // We've run out of both leaves and pairs. - // End all calculations for this level. - // To make sure we never come back to this level or any lower level, - // set next_pair_freq impossibly large. - l.needed = 0; - levels[level + 1].next_pair_freq = math.maxInt(i32); - level += 1; - continue; - } - - const prev_freq = l.last_freq; - if (l.next_char_freq < l.next_pair_freq) { - // The next item on this row is a leaf node. - const next = leaf_counts[level][level] + 1; - l.last_freq = l.next_char_freq; - // Lower leaf_counts are the same of the previous node. - leaf_counts[level][level] = next; - if (next >= list.len) { - l.next_char_freq = maxNode().freq; - } else { - l.next_char_freq = list[next].freq; - } - } else { - // The next item on this row is a pair from the previous row. - // next_pair_freq isn't valid until we generate two - // more values in the level below - l.last_freq = l.next_pair_freq; - // Take leaf counts from the lower level, except counts[level] remains the same. - @memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]); - levels[l.level - 1].needed = 2; - } - - l.needed -= 1; - if (l.needed == 0) { - // We've done everything we need to do for this level. - // Continue calculating one level up. Fill in next_pair_freq - // of that level with the sum of the two nodes we've just calculated on - // this level. - if (l.level == max_bits) { - // All done! - break; - } - levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq; - level += 1; - } else { - // If we stole from below, move down temporarily to replenish it. - while (levels[level - 1].needed > 0) { - level -= 1; - if (level == 0) { - break; - } - } - } - } - } - - // Somethings is wrong if at the end, the top level is null or hasn't used - // all of the leaves. - assert(leaf_counts[max_bits][max_bits] == n); - - var bit_count = self.bit_count[0 .. max_bits + 1]; - var bits: u32 = 1; - const counts = &leaf_counts[max_bits]; - { - var level = max_bits; - while (level > 0) : (level -= 1) { - // counts[level] gives the number of literals requiring at least "bits" - // bits to encode. - bit_count[bits] = counts[level] - counts[level - 1]; - bits += 1; - if (level == 0) { - break; - } - } - } - return bit_count; - } - - // Look at the leaves and assign them a bit count and an encoding as specified - // in RFC 1951 3.2.2 - fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void { - var code = @as(u16, 0); - var list = list_arg; - - for (bit_count, 0..) |bits, n| { - code <<= 1; - if (n == 0 or bits == 0) { - continue; - } - // The literals list[list.len-bits] .. list[list.len-bits] - // are encoded using "bits" bits, and get the values - // code, code + 1, .... The code values are - // assigned in literal order (not frequency order). - const chunk = list[list.len - @as(u32, @intCast(bits)) ..]; - - self.lns = chunk; - mem.sort(LiteralNode, self.lns, {}, byLiteral); - - for (chunk) |node| { - self.codes[node.literal] = HuffCode{ - .code = bitReverse(u16, code, @as(u5, @intCast(n))), - .len = @as(u16, @intCast(n)), - }; - code += 1; - } - list = list[0 .. list.len - @as(u32, @intCast(bits))]; - } - } - }; -} - -fn maxNode() LiteralNode { - return LiteralNode{ - .literal = math.maxInt(u16), - .freq = math.maxInt(u16), - }; -} - -pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) { - return .{}; -} - -pub const LiteralEncoder = HuffmanEncoder(huffman.max_num_frequencies); -pub const DistanceEncoder = HuffmanEncoder(huffman.distance_code_count); -pub const CodegenEncoder = HuffmanEncoder(19); - -// Generates a HuffmanCode corresponding to the fixed literal table -pub fn fixedLiteralEncoder() LiteralEncoder { - var h: LiteralEncoder = undefined; - var ch: u16 = 0; - - while (ch < huffman.max_num_frequencies) : (ch += 1) { - var bits: u16 = undefined; - var size: u16 = undefined; - switch (ch) { - 0...143 => { - // size 8, 000110000 .. 10111111 - bits = ch + 48; - size = 8; - }, - 144...255 => { - // size 9, 110010000 .. 111111111 - bits = ch + 400 - 144; - size = 9; - }, - 256...279 => { - // size 7, 0000000 .. 0010111 - bits = ch - 256; - size = 7; - }, - else => { - // size 8, 11000000 .. 11000111 - bits = ch + 192 - 280; - size = 8; - }, - } - h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size }; - } - return h; -} - -pub fn fixedDistanceEncoder() DistanceEncoder { - var h: DistanceEncoder = undefined; - for (h.codes, 0..) |_, ch| { - h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 }; - } - return h; -} - -pub fn huffmanDistanceEncoder() DistanceEncoder { - var distance_freq = [1]u16{0} ** huffman.distance_code_count; - distance_freq[0] = 1; - // huff_distance is a static distance encoder used for huffman only encoding. - // It can be reused since we will not be encoding distance values. - var h: DistanceEncoder = .{}; - h.generate(distance_freq[0..], 15); - return h; -} - -fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool { - _ = context; - return a.literal < b.literal; -} - -fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool { - _ = context; - if (a.freq == b.freq) { - return a.literal < b.literal; - } - return a.freq < b.freq; -} - -fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize { - const c: *Compress = @fieldParentPtr("reader", r); +fn drain(me: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize { + _ = data; + _ = splat; + const c: *Compress = @fieldParentPtr("writer", me); + const out = c.block_writer.output; switch (c.state) { - .header => |i| { + .header => { + c.state = .middle; const header = c.hasher.container().header(); - const n = try w.write(header[i..]); - if (header.len - i - n == 0) { - c.state = .middle; - } else { - c.state.header += n; - } - return n; + try out.writeAll(header); + return header.len; }, - .middle => { - c.input.fillMore() catch |err| switch (err) { - error.EndOfStream => { - c.state = .final; - return 0; - }, - else => |e| return e, - }; - const buffer_contents = c.input.buffered(); - const min_lookahead = flate.match.min_length + flate.match.max_length; - const history_plus_lookahead_len = flate.history_len + min_lookahead; - if (buffer_contents.len < history_plus_lookahead_len) return 0; - const lookahead = buffer_contents[flate.history_len..]; - const start = w.count; - const n = try c.tokenizeSlice(w, limit, lookahead) catch |err| switch (err) { - error.WriteFailed => return error.WriteFailed, - }; - c.hasher.update(lookahead[0..n]); - c.input.toss(n); - return w.count - start; - }, - .final => { - const buffer_contents = c.input.buffered(); - const start = w.count; - const n = c.tokenizeSlice(w, limit, buffer_contents) catch |err| switch (err) { - error.WriteFailed => return error.WriteFailed, - }; - if (buffer_contents.len - n == 0) { - c.hasher.update(buffer_contents); - c.input.tossAll(); - { - // In the case of flushing, last few lookahead buffers were - // smaller than min match len, so only last literal can be - // unwritten. - assert(c.prev_match == null); - try c.addPrevLiteral(); - c.prev_literal = null; + .middle => {}, + .ended => unreachable, + } - try c.flushTokens(.final); - } - switch (c.hasher) { - .gzip => |*gzip| { - // GZIP 8 bytes footer - // - 4 bytes, CRC32 (CRC-32) - // - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32 - comptime assert(c.footer_buffer.len == 8); - std.mem.writeInt(u32, c.footer_buffer[0..4], gzip.final(), .little); - std.mem.writeInt(u32, c.footer_buffer[4..8], gzip.bytes_read, .little); - c.state = .{ .footer = 0 }; - }, - .zlib => |*zlib| { - // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). - // 4 bytes of ADLER32 (Adler-32 checksum) - // Checksum value of the uncompressed data (excluding any - // dictionary data) computed according to Adler-32 - // algorithm. - comptime assert(c.footer_buffer.len == 8); - std.mem.writeInt(u32, c.footer_buffer[4..8], zlib.final, .big); - c.state = .{ .footer = 4 }; - }, - .raw => { - c.state = .ended; - }, - } - } - return w.count - start; + const buffered = me.buffered(); + const min_lookahead = flate.match.min_length + flate.match.max_length; + const history_plus_lookahead_len = flate.history_len + min_lookahead; + if (buffered.len < history_plus_lookahead_len) return 0; + const lookahead = buffered[flate.history_len..]; + + _ = lookahead; + // TODO tokenize + //c.hasher.update(lookahead[0..n]); + @panic("TODO"); +} + +pub fn end(c: *Compress) !void { + try endUnflushed(c); + try c.output.flush(); +} + +pub fn endUnflushed(c: *Compress) !void { + while (c.writer.end != 0) _ = try drain(&c.writer, &.{""}, 1); + c.state = .ended; + + const out = c.block_writer.output; + + // TODO flush tokens + + switch (c.hasher) { + .gzip => |*gzip| { + // GZIP 8 bytes footer + // - 4 bytes, CRC32 (CRC-32) + // - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32 + const footer = try out.writableArray(8); + std.mem.writeInt(u32, footer[0..4], gzip.crc.final(), .little); + std.mem.writeInt(u32, footer[4..8], @truncate(gzip.count), .little); }, - .ended => return error.EndOfStream, - .footer => |i| { - const remaining = c.footer_buffer[i..]; - const n = try w.write(limit.slice(remaining)); - c.state = if (n == remaining) .ended else .{ .footer = i - n }; - return n; + .zlib => |*zlib| { + // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). + // 4 bytes of ADLER32 (Adler-32 checksum) + // Checksum value of the uncompressed data (excluding any + // dictionary data) computed according to Adler-32 + // algorithm. + std.mem.writeInt(u32, try out.writableArray(4), zlib.final, .big); }, + .raw => {}, } } +pub const Simple = struct { + /// Note that store blocks are limited to 65535 bytes. + buffer: []u8, + wp: usize, + block_writer: BlockWriter, + hasher: Container.Hasher, + strategy: Strategy, + + pub const Strategy = enum { huffman, store }; + + pub fn init(out: *Writer, buffer: []u8, container: Container) !Simple { + const self: Simple = .{ + .buffer = buffer, + .wp = 0, + .block_writer = .init(out), + .hasher = .init(container), + }; + try container.writeHeader(self.out); + return self; + } + + pub fn flush(self: *Simple) !void { + try self.flushBuffer(false); + try self.block_writer.storedBlock("", false); + try self.block_writer.flush(); + } + + pub fn finish(self: *Simple) !void { + try self.flushBuffer(true); + try self.block_writer.flush(); + try self.hasher.container().writeFooter(&self.hasher, self.out); + } + + fn flushBuffer(self: *Simple, final: bool) !void { + const buf = self.buffer[0..self.wp]; + switch (self.strategy) { + .huffman => try self.block_writer.huffmanBlock(buf, final), + .store => try self.block_writer.storedBlock(buf, final), + } + self.wp = 0; + } +}; + test "generate a Huffman code from an array of frequencies" { var freqs: [19]u16 = [_]u16{ 8, // 0 @@ -868,7 +299,8 @@ test "generate a Huffman code from an array of frequencies" { 5, // 18 }; - var enc = huffmanEncoder(19); + var codes: [19]HuffmanEncoder.Code = undefined; + var enc: HuffmanEncoder = .{ .codes = &codes }; enc.generate(freqs[0..], 7); try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..])); @@ -906,120 +338,6 @@ test "generate a Huffman code from an array of frequencies" { try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code); } -test "generate a Huffman code for the fixed literal table specific to Deflate" { - const enc = fixedLiteralEncoder(); - for (enc.codes) |c| { - switch (c.len) { - 7 => { - const v = @bitReverse(@as(u7, @intCast(c.code))); - try testing.expect(v <= 0b0010111); - }, - 8 => { - const v = @bitReverse(@as(u8, @intCast(c.code))); - try testing.expect((v >= 0b000110000 and v <= 0b10111111) or - (v >= 0b11000000 and v <= 11000111)); - }, - 9 => { - const v = @bitReverse(@as(u9, @intCast(c.code))); - try testing.expect(v >= 0b110010000 and v <= 0b111111111); - }, - else => unreachable, - } - } -} - -test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" { - const enc = fixedDistanceEncoder(); - for (enc.codes) |c| { - const v = @bitReverse(@as(u5, @intCast(c.code))); - try testing.expect(v <= 29); - try testing.expect(c.len == 5); - } -} - -// Reverse bit-by-bit a N-bit code. -fn bitReverse(comptime T: type, value: T, n: usize) T { - const r = @bitReverse(value); - return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n)); -} - -test bitReverse { - const ReverseBitsTest = struct { - in: u16, - bit_count: u5, - out: u16, - }; - - const reverse_bits_tests = [_]ReverseBitsTest{ - .{ .in = 1, .bit_count = 1, .out = 1 }, - .{ .in = 1, .bit_count = 2, .out = 2 }, - .{ .in = 1, .bit_count = 3, .out = 4 }, - .{ .in = 1, .bit_count = 4, .out = 8 }, - .{ .in = 1, .bit_count = 5, .out = 16 }, - .{ .in = 17, .bit_count = 5, .out = 17 }, - .{ .in = 257, .bit_count = 9, .out = 257 }, - .{ .in = 29, .bit_count = 5, .out = 23 }, - }; - - for (reverse_bits_tests) |h| { - const v = bitReverse(u16, h.in, h.bit_count); - try std.testing.expectEqual(h.out, v); - } -} - -test "fixedLiteralEncoder codes" { - var al = std.ArrayList(u8).init(testing.allocator); - defer al.deinit(); - var bw = std.Io.bitWriter(.little, al.writer()); - - const f = fixedLiteralEncoder(); - for (f.codes) |c| { - try bw.writeBits(c.code, c.len); - } - try testing.expectEqualSlices(u8, &fixed_codes, al.items); -} - -pub const fixed_codes = [_]u8{ - 0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100, - 0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100, - 0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010, - 0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010, - 0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010, - 0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010, - 0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110, - 0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110, - 0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110, - 0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110, - 0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001, - 0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001, - 0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001, - 0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001, - 0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101, - 0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101, - 0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101, - 0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101, - 0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100, - 0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110, - 0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100, - 0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101, - 0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010, - 0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110, - 0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110, - 0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111, - 0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011, - 0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101, - 0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110, - 0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100, - 0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101, - 0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010, - 0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110, - 0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000, - 0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010, - 0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100, - 0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011, - 0b10100011, -}; - test "tokenization" { const L = Token.initLiteral; const M = Token.initMatch; @@ -1133,7 +451,7 @@ test "file tokenization" { const data = case.data; for (levels, 0..) |level, i| { // for each compression level - var original: Reader = .fixed(data); + var original: std.Io.Reader = .fixed(data); // buffer for decompressed data var al = std.ArrayList(u8).init(testing.allocator); @@ -1198,32 +516,33 @@ const TokenDecoder = struct { }; test "store simple compressor" { - const data = "Hello world!"; - const expected = [_]u8{ - 0x1, // block type 0, final bit set - 0xc, 0x0, // len = 12 - 0xf3, 0xff, // ~len - 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', // - //0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21, - }; + if (true) return error.SkipZigTest; + //const data = "Hello world!"; + //const expected = [_]u8{ + // 0x1, // block type 0, final bit set + // 0xc, 0x0, // len = 12 + // 0xf3, 0xff, // ~len + // 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', // + // //0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21, + //}; - var fbs: Reader = .fixed(data); - var al = std.ArrayList(u8).init(testing.allocator); - defer al.deinit(); + //var fbs: std.Io.Reader = .fixed(data); + //var al = std.ArrayList(u8).init(testing.allocator); + //defer al.deinit(); - var cmp = try store.compressor(.raw, al.writer()); - try cmp.compress(&fbs); - try cmp.finish(); - try testing.expectEqualSlices(u8, &expected, al.items); + //var cmp = try store.compressor(.raw, al.writer()); + //try cmp.compress(&fbs); + //try cmp.finish(); + //try testing.expectEqualSlices(u8, &expected, al.items); - fbs = .fixed(data); - try al.resize(0); + //fbs = .fixed(data); + //try al.resize(0); - // huffman only compresoor will also emit store block for this small sample - var hc = try huffman.compressor(.raw, al.writer()); - try hc.compress(&fbs); - try hc.finish(); - try testing.expectEqualSlices(u8, &expected, al.items); + //// huffman only compresoor will also emit store block for this small sample + //var hc = try huffman.compressor(.raw, al.writer()); + //try hc.compress(&fbs); + //try hc.finish(); + //try testing.expectEqualSlices(u8, &expected, al.items); } test "sliding window match" { diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 6cb5953763..ed9c0f3798 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -620,10 +620,9 @@ test "init/find" { } test "encode/decode literals" { - const LiteralEncoder = std.compress.flate.Compress.LiteralEncoder; - + var codes: [flate.HuffmanEncoder.max_num_frequencies]flate.HuffmanEncoder.Code = undefined; for (1..286) |j| { // for all different number of codes - var enc: LiteralEncoder = .{}; + var enc: flate.HuffmanEncoder = .{ .codes = &codes }; // create frequencies var freq = [_]u16{0} ** 286; freq[256] = 1; // ensure we have end of block code diff --git a/lib/std/compress/flate/HuffmanEncoder.zig b/lib/std/compress/flate/HuffmanEncoder.zig new file mode 100644 index 0000000000..bdcaf75801 --- /dev/null +++ b/lib/std/compress/flate/HuffmanEncoder.zig @@ -0,0 +1,475 @@ +const HuffmanEncoder = @This(); +const std = @import("std"); +const assert = std.debug.assert; +const testing = std.testing; + +codes: []Code, +// Reusable buffer with the longest possible frequency table. +freq_cache: [max_num_frequencies + 1]LiteralNode, +bit_count: [17]u32, +lns: []LiteralNode, // sorted by literal, stored to avoid repeated allocation in generate +lfs: []LiteralNode, // sorted by frequency, stored to avoid repeated allocation in generate + +pub const LiteralNode = struct { + literal: u16, + freq: u16, + + pub fn max() LiteralNode { + return .{ + .literal = std.math.maxInt(u16), + .freq = std.math.maxInt(u16), + }; + } +}; + +pub const Code = struct { + code: u16 = 0, + len: u16 = 0, +}; + +/// The odd order in which the codegen code sizes are written. +pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; +/// The number of codegen codes. +pub const codegen_code_count = 19; + +/// The largest distance code. +pub const distance_code_count = 30; + +/// Maximum number of literals. +pub const max_num_lit = 286; + +/// Max number of frequencies used for a Huffman Code +/// Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286). +/// The largest of these is max_num_lit. +pub const max_num_frequencies = max_num_lit; + +/// Biggest block size for uncompressed block. +pub const max_store_block_size = 65535; +/// The special code used to mark the end of a block. +pub const end_block_marker = 256; + +/// Update this Huffman Code object to be the minimum code for the specified frequency count. +/// +/// freq An array of frequencies, in which frequency[i] gives the frequency of literal i. +/// max_bits The maximum number of bits to use for any literal. +pub fn generate(self: *HuffmanEncoder, freq: []u16, max_bits: u32) void { + var list = self.freq_cache[0 .. freq.len + 1]; + // Number of non-zero literals + var count: u32 = 0; + // Set list to be the set of all non-zero literals and their frequencies + for (freq, 0..) |f, i| { + if (f != 0) { + list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f }; + count += 1; + } else { + list[count] = LiteralNode{ .literal = 0x00, .freq = 0 }; + self.codes[i].len = 0; + } + } + list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 }; + + list = list[0..count]; + if (count <= 2) { + // Handle the small cases here, because they are awkward for the general case code. With + // two or fewer literals, everything has bit length 1. + for (list, 0..) |node, i| { + // "list" is in order of increasing literal value. + self.codes[node.literal] = .{ + .code = @intCast(i), + .len = 1, + }; + } + return; + } + self.lfs = list; + std.mem.sort(LiteralNode, self.lfs, {}, byFreq); + + // Get the number of literals for each bit count + const bit_count = self.bitCounts(list, max_bits); + // And do the assignment + self.assignEncodingAndSize(bit_count, list); +} + +pub fn bitLength(self: *HuffmanEncoder, freq: []u16) u32 { + var total: u32 = 0; + for (freq, 0..) |f, i| { + if (f != 0) { + total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len)); + } + } + return total; +} + +/// Return the number of literals assigned to each bit size in the Huffman encoding +/// +/// This method is only called when list.len >= 3 +/// The cases of 0, 1, and 2 literals are handled by special case code. +/// +/// list: An array of the literals with non-zero frequencies +/// and their associated frequencies. The array is in order of increasing +/// frequency, and has as its last element a special element with frequency +/// `math.maxInt(i32)` +/// +/// max_bits: The maximum number of bits that should be used to encode any literal. +/// Must be less than 16. +/// +/// Returns an integer array in which array[i] indicates the number of literals +/// that should be encoded in i bits. +fn bitCounts(self: *HuffmanEncoder, list: []LiteralNode, max_bits_to_use: usize) []u32 { + var max_bits = max_bits_to_use; + const n = list.len; + const max_bits_limit = 16; + + assert(max_bits < max_bits_limit); + + // The tree can't have greater depth than n - 1, no matter what. This + // saves a little bit of work in some small cases + max_bits = @min(max_bits, n - 1); + + // Create information about each of the levels. + // A bogus "Level 0" whose sole purpose is so that + // level1.prev.needed == 0. This makes level1.next_pair_freq + // be a legitimate value that never gets chosen. + var levels: [max_bits_limit]LevelInfo = std.mem.zeroes([max_bits_limit]LevelInfo); + // leaf_counts[i] counts the number of literals at the left + // of ancestors of the rightmost node at level i. + // leaf_counts[i][j] is the number of literals at the left + // of the level j ancestor. + var leaf_counts: [max_bits_limit][max_bits_limit]u32 = @splat(0); + + { + var level = @as(u32, 1); + while (level <= max_bits) : (level += 1) { + // For every level, the first two items are the first two characters. + // We initialize the levels as if we had already figured this out. + levels[level] = LevelInfo{ + .level = level, + .last_freq = list[1].freq, + .next_char_freq = list[2].freq, + .next_pair_freq = list[0].freq + list[1].freq, + .needed = 0, + }; + leaf_counts[level][level] = 2; + if (level == 1) { + levels[level].next_pair_freq = std.math.maxInt(i32); + } + } + } + + // We need a total of 2*n - 2 items at top level and have already generated 2. + levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4; + + { + var level = max_bits; + while (true) { + var l = &levels[level]; + if (l.next_pair_freq == std.math.maxInt(i32) and l.next_char_freq == std.math.maxInt(i32)) { + // We've run out of both leaves and pairs. + // End all calculations for this level. + // To make sure we never come back to this level or any lower level, + // set next_pair_freq impossibly large. + l.needed = 0; + levels[level + 1].next_pair_freq = std.math.maxInt(i32); + level += 1; + continue; + } + + const prev_freq = l.last_freq; + if (l.next_char_freq < l.next_pair_freq) { + // The next item on this row is a leaf node. + const next = leaf_counts[level][level] + 1; + l.last_freq = l.next_char_freq; + // Lower leaf_counts are the same of the previous node. + leaf_counts[level][level] = next; + if (next >= list.len) { + l.next_char_freq = LiteralNode.max().freq; + } else { + l.next_char_freq = list[next].freq; + } + } else { + // The next item on this row is a pair from the previous row. + // next_pair_freq isn't valid until we generate two + // more values in the level below + l.last_freq = l.next_pair_freq; + // Take leaf counts from the lower level, except counts[level] remains the same. + @memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]); + levels[l.level - 1].needed = 2; + } + + l.needed -= 1; + if (l.needed == 0) { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in next_pair_freq + // of that level with the sum of the two nodes we've just calculated on + // this level. + if (l.level == max_bits) { + // All done! + break; + } + levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq; + level += 1; + } else { + // If we stole from below, move down temporarily to replenish it. + while (levels[level - 1].needed > 0) { + level -= 1; + if (level == 0) { + break; + } + } + } + } + } + + // Somethings is wrong if at the end, the top level is null or hasn't used + // all of the leaves. + assert(leaf_counts[max_bits][max_bits] == n); + + var bit_count = self.bit_count[0 .. max_bits + 1]; + var bits: u32 = 1; + const counts = &leaf_counts[max_bits]; + { + var level = max_bits; + while (level > 0) : (level -= 1) { + // counts[level] gives the number of literals requiring at least "bits" + // bits to encode. + bit_count[bits] = counts[level] - counts[level - 1]; + bits += 1; + if (level == 0) { + break; + } + } + } + return bit_count; +} + +/// Look at the leaves and assign them a bit count and an encoding as specified +/// in RFC 1951 3.2.2 +fn assignEncodingAndSize(self: *HuffmanEncoder, bit_count: []u32, list_arg: []LiteralNode) void { + var code = @as(u16, 0); + var list = list_arg; + + for (bit_count, 0..) |bits, n| { + code <<= 1; + if (n == 0 or bits == 0) { + continue; + } + // The literals list[list.len-bits] .. list[list.len-bits] + // are encoded using "bits" bits, and get the values + // code, code + 1, .... The code values are + // assigned in literal order (not frequency order). + const chunk = list[list.len - @as(u32, @intCast(bits)) ..]; + + self.lns = chunk; + std.mem.sort(LiteralNode, self.lns, {}, byLiteral); + + for (chunk) |node| { + self.codes[node.literal] = .{ + .code = bitReverse(u16, code, @as(u5, @intCast(n))), + .len = @as(u16, @intCast(n)), + }; + code += 1; + } + list = list[0 .. list.len - @as(u32, @intCast(bits))]; + } +} + +fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool { + _ = context; + if (a.freq == b.freq) { + return a.literal < b.literal; + } + return a.freq < b.freq; +} + +/// Describes the state of the constructed tree for a given depth. +const LevelInfo = struct { + /// Our level. for better printing + level: u32, + /// The frequency of the last node at this level + last_freq: u32, + /// The frequency of the next character to add to this level + next_char_freq: u32, + /// The frequency of the next pair (from level below) to add to this level. + /// Only valid if the "needed" value of the next lower level is 0. + next_pair_freq: u32, + /// The number of chains remaining to generate for this level before moving + /// up to the next level + needed: u32, +}; + +fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool { + _ = context; + return a.literal < b.literal; +} + +/// Reverse bit-by-bit a N-bit code. +fn bitReverse(comptime T: type, value: T, n: usize) T { + const r = @bitReverse(value); + return r >> @as(std.math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n)); +} + +test bitReverse { + const ReverseBitsTest = struct { + in: u16, + bit_count: u5, + out: u16, + }; + + const reverse_bits_tests = [_]ReverseBitsTest{ + .{ .in = 1, .bit_count = 1, .out = 1 }, + .{ .in = 1, .bit_count = 2, .out = 2 }, + .{ .in = 1, .bit_count = 3, .out = 4 }, + .{ .in = 1, .bit_count = 4, .out = 8 }, + .{ .in = 1, .bit_count = 5, .out = 16 }, + .{ .in = 17, .bit_count = 5, .out = 17 }, + .{ .in = 257, .bit_count = 9, .out = 257 }, + .{ .in = 29, .bit_count = 5, .out = 23 }, + }; + + for (reverse_bits_tests) |h| { + const v = bitReverse(u16, h.in, h.bit_count); + try std.testing.expectEqual(h.out, v); + } +} + +/// Generates a HuffmanCode corresponding to the fixed literal table +pub fn fixedLiteralEncoder(codes: *[max_num_frequencies]Code) HuffmanEncoder { + var h: HuffmanEncoder = undefined; + h.codes = codes; + var ch: u16 = 0; + + while (ch < max_num_frequencies) : (ch += 1) { + var bits: u16 = undefined; + var size: u16 = undefined; + switch (ch) { + 0...143 => { + // size 8, 000110000 .. 10111111 + bits = ch + 48; + size = 8; + }, + 144...255 => { + // size 9, 110010000 .. 111111111 + bits = ch + 400 - 144; + size = 9; + }, + 256...279 => { + // size 7, 0000000 .. 0010111 + bits = ch - 256; + size = 7; + }, + else => { + // size 8, 11000000 .. 11000111 + bits = ch + 192 - 280; + size = 8; + }, + } + h.codes[ch] = .{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size }; + } + return h; +} + +pub fn fixedDistanceEncoder(codes: *[distance_code_count]Code) HuffmanEncoder { + var h: HuffmanEncoder = undefined; + h.codes = codes; + for (h.codes, 0..) |_, ch| { + h.codes[ch] = .{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 }; + } + return h; +} + +pub fn huffmanDistanceEncoder(codes: *[distance_code_count]Code) HuffmanEncoder { + var distance_freq: [distance_code_count]u16 = @splat(0); + distance_freq[0] = 1; + // huff_distance is a static distance encoder used for huffman only encoding. + // It can be reused since we will not be encoding distance values. + var h: HuffmanEncoder = .{}; + h.codes = codes; + h.generate(distance_freq[0..], 15); + return h; +} + +test "generate a Huffman code for the fixed literal table specific to Deflate" { + const enc = fixedLiteralEncoder(); + for (enc.codes) |c| { + switch (c.len) { + 7 => { + const v = @bitReverse(@as(u7, @intCast(c.code))); + try testing.expect(v <= 0b0010111); + }, + 8 => { + const v = @bitReverse(@as(u8, @intCast(c.code))); + try testing.expect((v >= 0b000110000 and v <= 0b10111111) or + (v >= 0b11000000 and v <= 11000111)); + }, + 9 => { + const v = @bitReverse(@as(u9, @intCast(c.code))); + try testing.expect(v >= 0b110010000 and v <= 0b111111111); + }, + else => unreachable, + } + } +} + +test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" { + var codes: [distance_code_count]Code = undefined; + const enc = fixedDistanceEncoder(&codes); + for (enc.codes) |c| { + const v = @bitReverse(@as(u5, @intCast(c.code))); + try testing.expect(v <= 29); + try testing.expect(c.len == 5); + } +} + +test "fixedLiteralEncoder codes" { + var al = std.ArrayList(u8).init(testing.allocator); + defer al.deinit(); + var bw = std.Io.bitWriter(.little, al.writer()); + + var codes: [max_num_frequencies]Code = undefined; + const f = fixedLiteralEncoder(&codes); + for (f.codes) |c| { + try bw.writeBits(c.code, c.len); + } + try testing.expectEqualSlices(u8, &fixed_codes, al.items); +} + +pub const fixed_codes = [_]u8{ + 0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100, + 0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100, + 0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010, + 0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010, + 0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010, + 0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010, + 0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110, + 0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110, + 0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110, + 0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110, + 0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001, + 0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001, + 0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001, + 0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001, + 0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101, + 0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101, + 0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101, + 0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101, + 0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100, + 0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110, + 0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100, + 0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101, + 0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010, + 0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110, + 0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110, + 0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111, + 0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011, + 0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101, + 0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110, + 0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100, + 0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101, + 0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010, + 0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110, + 0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000, + 0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010, + 0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100, + 0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011, + 0b10100011, +}; diff --git a/lib/std/compress/flate/testdata/block_writer.zig b/lib/std/compress/flate/testdata/block_writer.zig deleted file mode 100644 index cb8f3028d1..0000000000 --- a/lib/std/compress/flate/testdata/block_writer.zig +++ /dev/null @@ -1,606 +0,0 @@ -const Token = @import("../Token.zig"); - -pub const TestCase = struct { - tokens: []const Token, - input: []const u8 = "", // File name of input data matching the tokens. - want: []const u8 = "", // File name of data with the expected output with input available. - want_no_input: []const u8 = "", // File name of the expected output when no input is available. -}; - -pub const testCases = blk: { - @setEvalBranchQuota(4096 * 2); - - const L = Token.initLiteral; - const M = Token.initMatch; - const ml = M(1, 258); // Maximum length token. Used to reduce the size of writeBlockTests - - break :blk &[_]TestCase{ - TestCase{ - .input = "huffman-null-max.input", - .want = "huffman-null-max.{s}.expect", - .want_no_input = "huffman-null-max.{s}.expect-noinput", - .tokens = &[_]Token{ - L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, L(0x0), L(0x0), - }, - }, - TestCase{ - .input = "huffman-pi.input", - .want = "huffman-pi.{s}.expect", - .want_no_input = "huffman-pi.{s}.expect-noinput", - .tokens = &[_]Token{ - L('3'), L('.'), L('1'), L('4'), L('1'), L('5'), L('9'), L('2'), - L('6'), L('5'), L('3'), L('5'), L('8'), L('9'), L('7'), L('9'), - L('3'), L('2'), L('3'), L('8'), L('4'), L('6'), L('2'), L('6'), - L('4'), L('3'), L('3'), L('8'), L('3'), L('2'), L('7'), L('9'), - L('5'), L('0'), L('2'), L('8'), L('8'), L('4'), L('1'), L('9'), - L('7'), L('1'), L('6'), L('9'), L('3'), L('9'), L('9'), L('3'), - L('7'), L('5'), L('1'), L('0'), L('5'), L('8'), L('2'), L('0'), - L('9'), L('7'), L('4'), L('9'), L('4'), L('4'), L('5'), L('9'), - L('2'), L('3'), L('0'), L('7'), L('8'), L('1'), L('6'), L('4'), - L('0'), L('6'), L('2'), L('8'), L('6'), L('2'), L('0'), L('8'), - L('9'), L('9'), L('8'), L('6'), L('2'), L('8'), L('0'), L('3'), - L('4'), L('8'), L('2'), L('5'), L('3'), L('4'), L('2'), L('1'), - L('1'), L('7'), L('0'), L('6'), L('7'), L('9'), L('8'), L('2'), - L('1'), L('4'), L('8'), L('0'), L('8'), L('6'), L('5'), L('1'), - L('3'), L('2'), L('8'), L('2'), L('3'), L('0'), L('6'), L('6'), - L('4'), L('7'), L('0'), L('9'), L('3'), L('8'), L('4'), L('4'), - L('6'), L('0'), L('9'), L('5'), L('5'), L('0'), L('5'), L('8'), - L('2'), L('2'), L('3'), L('1'), L('7'), L('2'), L('5'), L('3'), - L('5'), L('9'), L('4'), L('0'), L('8'), L('1'), L('2'), L('8'), - L('4'), L('8'), L('1'), L('1'), L('1'), L('7'), L('4'), M(127, 4), - L('4'), L('1'), L('0'), L('2'), L('7'), L('0'), L('1'), L('9'), - L('3'), L('8'), L('5'), L('2'), L('1'), L('1'), L('0'), L('5'), - L('5'), L('5'), L('9'), L('6'), L('4'), L('4'), L('6'), L('2'), - L('2'), L('9'), L('4'), L('8'), L('9'), L('5'), L('4'), L('9'), - L('3'), L('0'), L('3'), L('8'), L('1'), M(19, 4), L('2'), L('8'), - L('8'), L('1'), L('0'), L('9'), L('7'), L('5'), L('6'), L('6'), - L('5'), L('9'), L('3'), L('3'), L('4'), L('4'), L('6'), M(72, 4), - L('7'), L('5'), L('6'), L('4'), L('8'), L('2'), L('3'), L('3'), - L('7'), L('8'), L('6'), L('7'), L('8'), L('3'), L('1'), L('6'), - L('5'), L('2'), L('7'), L('1'), L('2'), L('0'), L('1'), L('9'), - L('0'), L('9'), L('1'), L('4'), M(27, 4), L('5'), L('6'), L('6'), - L('9'), L('2'), L('3'), L('4'), L('6'), M(179, 4), L('6'), L('1'), - L('0'), L('4'), L('5'), L('4'), L('3'), L('2'), L('6'), M(51, 4), - L('1'), L('3'), L('3'), L('9'), L('3'), L('6'), L('0'), L('7'), - L('2'), L('6'), L('0'), L('2'), L('4'), L('9'), L('1'), L('4'), - L('1'), L('2'), L('7'), L('3'), L('7'), L('2'), L('4'), L('5'), - L('8'), L('7'), L('0'), L('0'), L('6'), L('6'), L('0'), L('6'), - L('3'), L('1'), L('5'), L('5'), L('8'), L('8'), L('1'), L('7'), - L('4'), L('8'), L('8'), L('1'), L('5'), L('2'), L('0'), L('9'), - L('2'), L('0'), L('9'), L('6'), L('2'), L('8'), L('2'), L('9'), - L('2'), L('5'), L('4'), L('0'), L('9'), L('1'), L('7'), L('1'), - L('5'), L('3'), L('6'), L('4'), L('3'), L('6'), L('7'), L('8'), - L('9'), L('2'), L('5'), L('9'), L('0'), L('3'), L('6'), L('0'), - L('0'), L('1'), L('1'), L('3'), L('3'), L('0'), L('5'), L('3'), - L('0'), L('5'), L('4'), L('8'), L('8'), L('2'), L('0'), L('4'), - L('6'), L('6'), L('5'), L('2'), L('1'), L('3'), L('8'), L('4'), - L('1'), L('4'), L('6'), L('9'), L('5'), L('1'), L('9'), L('4'), - L('1'), L('5'), L('1'), L('1'), L('6'), L('0'), L('9'), L('4'), - L('3'), L('3'), L('0'), L('5'), L('7'), L('2'), L('7'), L('0'), - L('3'), L('6'), L('5'), L('7'), L('5'), L('9'), L('5'), L('9'), - L('1'), L('9'), L('5'), L('3'), L('0'), L('9'), L('2'), L('1'), - L('8'), L('6'), L('1'), L('1'), L('7'), M(234, 4), L('3'), L('2'), - M(10, 4), L('9'), L('3'), L('1'), L('0'), L('5'), L('1'), L('1'), - L('8'), L('5'), L('4'), L('8'), L('0'), L('7'), M(271, 4), L('3'), - L('7'), L('9'), L('9'), L('6'), L('2'), L('7'), L('4'), L('9'), - L('5'), L('6'), L('7'), L('3'), L('5'), L('1'), L('8'), L('8'), - L('5'), L('7'), L('5'), L('2'), L('7'), L('2'), L('4'), L('8'), - L('9'), L('1'), L('2'), L('2'), L('7'), L('9'), L('3'), L('8'), - L('1'), L('8'), L('3'), L('0'), L('1'), L('1'), L('9'), L('4'), - L('9'), L('1'), L('2'), L('9'), L('8'), L('3'), L('3'), L('6'), - L('7'), L('3'), L('3'), L('6'), L('2'), L('4'), L('4'), L('0'), - L('6'), L('5'), L('6'), L('6'), L('4'), L('3'), L('0'), L('8'), - L('6'), L('0'), L('2'), L('1'), L('3'), L('9'), L('4'), L('9'), - L('4'), L('6'), L('3'), L('9'), L('5'), L('2'), L('2'), L('4'), - L('7'), L('3'), L('7'), L('1'), L('9'), L('0'), L('7'), L('0'), - L('2'), L('1'), L('7'), L('9'), L('8'), M(154, 5), L('7'), L('0'), - L('2'), L('7'), L('7'), L('0'), L('5'), L('3'), L('9'), L('2'), - L('1'), L('7'), L('1'), L('7'), L('6'), L('2'), L('9'), L('3'), - L('1'), L('7'), L('6'), L('7'), L('5'), M(563, 5), L('7'), L('4'), - L('8'), L('1'), M(7, 4), L('6'), L('6'), L('9'), L('4'), L('0'), - M(488, 4), L('0'), L('0'), L('0'), L('5'), L('6'), L('8'), L('1'), - L('2'), L('7'), L('1'), L('4'), L('5'), L('2'), L('6'), L('3'), - L('5'), L('6'), L('0'), L('8'), L('2'), L('7'), L('7'), L('8'), - L('5'), L('7'), L('7'), L('1'), L('3'), L('4'), L('2'), L('7'), - L('5'), L('7'), L('7'), L('8'), L('9'), L('6'), M(298, 4), L('3'), - L('6'), L('3'), L('7'), L('1'), L('7'), L('8'), L('7'), L('2'), - L('1'), L('4'), L('6'), L('8'), L('4'), L('4'), L('0'), L('9'), - L('0'), L('1'), L('2'), L('2'), L('4'), L('9'), L('5'), L('3'), - L('4'), L('3'), L('0'), L('1'), L('4'), L('6'), L('5'), L('4'), - L('9'), L('5'), L('8'), L('5'), L('3'), L('7'), L('1'), L('0'), - L('5'), L('0'), L('7'), L('9'), M(203, 4), L('6'), M(340, 4), L('8'), - L('9'), L('2'), L('3'), L('5'), L('4'), M(458, 4), L('9'), L('5'), - L('6'), L('1'), L('1'), L('2'), L('1'), L('2'), L('9'), L('0'), - L('2'), L('1'), L('9'), L('6'), L('0'), L('8'), L('6'), L('4'), - L('0'), L('3'), L('4'), L('4'), L('1'), L('8'), L('1'), L('5'), - L('9'), L('8'), L('1'), L('3'), L('6'), L('2'), L('9'), L('7'), - L('7'), L('4'), M(117, 4), L('0'), L('9'), L('9'), L('6'), L('0'), - L('5'), L('1'), L('8'), L('7'), L('0'), L('7'), L('2'), L('1'), - L('1'), L('3'), L('4'), L('9'), M(1, 5), L('8'), L('3'), L('7'), - L('2'), L('9'), L('7'), L('8'), L('0'), L('4'), L('9'), L('9'), - M(731, 4), L('9'), L('7'), L('3'), L('1'), L('7'), L('3'), L('2'), - L('8'), M(395, 4), L('6'), L('3'), L('1'), L('8'), L('5'), M(770, 4), - M(745, 4), L('4'), L('5'), L('5'), L('3'), L('4'), L('6'), L('9'), - L('0'), L('8'), L('3'), L('0'), L('2'), L('6'), L('4'), L('2'), - L('5'), L('2'), L('2'), L('3'), L('0'), M(740, 4), M(616, 4), L('8'), - L('5'), L('0'), L('3'), L('5'), L('2'), L('6'), L('1'), L('9'), - L('3'), L('1'), L('1'), M(531, 4), L('1'), L('0'), L('1'), L('0'), - L('0'), L('0'), L('3'), L('1'), L('3'), L('7'), L('8'), L('3'), - L('8'), L('7'), L('5'), L('2'), L('8'), L('8'), L('6'), L('5'), - L('8'), L('7'), L('5'), L('3'), L('3'), L('2'), L('0'), L('8'), - L('3'), L('8'), L('1'), L('4'), L('2'), L('0'), L('6'), M(321, 4), - M(300, 4), L('1'), L('4'), L('7'), L('3'), L('0'), L('3'), L('5'), - L('9'), M(815, 5), L('9'), L('0'), L('4'), L('2'), L('8'), L('7'), - L('5'), L('5'), L('4'), L('6'), L('8'), L('7'), L('3'), L('1'), - L('1'), L('5'), L('9'), L('5'), M(854, 4), L('3'), L('8'), L('8'), - L('2'), L('3'), L('5'), L('3'), L('7'), L('8'), L('7'), L('5'), - M(896, 5), L('9'), M(315, 4), L('1'), M(329, 4), L('8'), L('0'), L('5'), - L('3'), M(395, 4), L('2'), L('2'), L('6'), L('8'), L('0'), L('6'), - L('6'), L('1'), L('3'), L('0'), L('0'), L('1'), L('9'), L('2'), - L('7'), L('8'), L('7'), L('6'), L('6'), L('1'), L('1'), L('1'), - L('9'), L('5'), L('9'), M(568, 4), L('6'), M(293, 5), L('8'), L('9'), - L('3'), L('8'), L('0'), L('9'), L('5'), L('2'), L('5'), L('7'), - L('2'), L('0'), L('1'), L('0'), L('6'), L('5'), L('4'), L('8'), - L('5'), L('8'), L('6'), L('3'), L('2'), L('7'), M(155, 4), L('9'), - L('3'), L('6'), L('1'), L('5'), L('3'), M(545, 4), M(349, 5), L('2'), - L('3'), L('0'), L('3'), L('0'), L('1'), L('9'), L('5'), L('2'), - L('0'), L('3'), L('5'), L('3'), L('0'), L('1'), L('8'), L('5'), - L('2'), M(370, 4), M(118, 4), L('3'), L('6'), L('2'), L('2'), L('5'), - L('9'), L('9'), L('4'), L('1'), L('3'), M(597, 4), L('4'), L('9'), - L('7'), L('2'), L('1'), L('7'), M(223, 4), L('3'), L('4'), L('7'), - L('9'), L('1'), L('3'), L('1'), L('5'), L('1'), L('5'), L('5'), - L('7'), L('4'), L('8'), L('5'), L('7'), L('2'), L('4'), L('2'), - L('4'), L('5'), L('4'), L('1'), L('5'), L('0'), L('6'), L('9'), - M(320, 4), L('8'), L('2'), L('9'), L('5'), L('3'), L('3'), L('1'), - L('1'), L('6'), L('8'), L('6'), L('1'), L('7'), L('2'), L('7'), - L('8'), M(824, 4), L('9'), L('0'), L('7'), L('5'), L('0'), L('9'), - M(270, 4), L('7'), L('5'), L('4'), L('6'), L('3'), L('7'), L('4'), - L('6'), L('4'), L('9'), L('3'), L('9'), L('3'), L('1'), L('9'), - L('2'), L('5'), L('5'), L('0'), L('6'), L('0'), L('4'), L('0'), - L('0'), L('9'), M(620, 4), L('1'), L('6'), L('7'), L('1'), L('1'), - L('3'), L('9'), L('0'), L('0'), L('9'), L('8'), M(822, 4), L('4'), - L('0'), L('1'), L('2'), L('8'), L('5'), L('8'), L('3'), L('6'), - L('1'), L('6'), L('0'), L('3'), L('5'), L('6'), L('3'), L('7'), - L('0'), L('7'), L('6'), L('6'), L('0'), L('1'), L('0'), L('4'), - M(371, 4), L('8'), L('1'), L('9'), L('4'), L('2'), L('9'), M(1055, 5), - M(240, 4), M(652, 4), L('7'), L('8'), L('3'), L('7'), L('4'), M(1193, 4), - L('8'), L('2'), L('5'), L('5'), L('3'), L('7'), M(522, 5), L('2'), - L('6'), L('8'), M(47, 4), L('4'), L('0'), L('4'), L('7'), M(466, 4), - L('4'), M(1206, 4), M(910, 4), L('8'), L('4'), M(937, 4), L('6'), M(800, 6), - L('3'), L('3'), L('1'), L('3'), L('6'), L('7'), L('7'), L('0'), - L('2'), L('8'), L('9'), L('8'), L('9'), L('1'), L('5'), L('2'), - M(99, 4), L('5'), L('2'), L('1'), L('6'), L('2'), L('0'), L('5'), - L('6'), L('9'), L('6'), M(1042, 4), L('0'), L('5'), L('8'), M(1144, 4), - L('5'), M(1177, 4), L('5'), L('1'), L('1'), M(522, 4), L('8'), L('2'), - L('4'), L('3'), L('0'), L('0'), L('3'), L('5'), L('5'), L('8'), - L('7'), L('6'), L('4'), L('0'), L('2'), L('4'), L('7'), L('4'), - L('9'), L('6'), L('4'), L('7'), L('3'), L('2'), L('6'), L('3'), - M(1087, 4), L('9'), L('9'), L('2'), M(1100, 4), L('4'), L('2'), L('6'), - L('9'), M(710, 6), L('7'), M(471, 4), L('4'), M(1342, 4), M(1054, 4), L('9'), - L('3'), L('4'), L('1'), L('7'), M(430, 4), L('1'), L('2'), M(43, 4), - L('4'), M(415, 4), L('1'), L('5'), L('0'), L('3'), L('0'), L('2'), - L('8'), L('6'), L('1'), L('8'), L('2'), L('9'), L('7'), L('4'), - L('5'), L('5'), L('5'), L('7'), L('0'), L('6'), L('7'), L('4'), - M(310, 4), L('5'), L('0'), L('5'), L('4'), L('9'), L('4'), L('5'), - L('8'), M(454, 4), L('9'), M(82, 4), L('5'), L('6'), M(493, 4), L('7'), - L('2'), L('1'), L('0'), L('7'), L('9'), M(346, 4), L('3'), L('0'), - M(267, 4), L('3'), L('2'), L('1'), L('1'), L('6'), L('5'), L('3'), - L('4'), L('4'), L('9'), L('8'), L('7'), L('2'), L('0'), L('2'), - L('7'), M(284, 4), L('0'), L('2'), L('3'), L('6'), L('4'), M(559, 4), - L('5'), L('4'), L('9'), L('9'), L('1'), L('1'), L('9'), L('8'), - M(1049, 4), L('4'), M(284, 4), L('5'), L('3'), L('5'), L('6'), L('6'), - L('3'), L('6'), L('9'), M(1105, 4), L('2'), L('6'), L('5'), M(741, 4), - L('7'), L('8'), L('6'), L('2'), L('5'), L('5'), L('1'), M(987, 4), - L('1'), L('7'), L('5'), L('7'), L('4'), L('6'), L('7'), L('2'), - L('8'), L('9'), L('0'), L('9'), L('7'), L('7'), L('7'), L('7'), - M(1108, 5), L('0'), L('0'), L('0'), M(1534, 4), L('7'), L('0'), M(1248, 4), - L('6'), M(1002, 4), L('4'), L('9'), L('1'), M(1055, 4), M(664, 4), L('2'), - L('1'), L('4'), L('7'), L('7'), L('2'), L('3'), L('5'), L('0'), - L('1'), L('4'), L('1'), L('4'), M(1604, 4), L('3'), L('5'), L('6'), - M(1200, 4), L('1'), L('6'), L('1'), L('3'), L('6'), L('1'), L('1'), - L('5'), L('7'), L('3'), L('5'), L('2'), L('5'), M(1285, 4), L('3'), - L('4'), M(92, 4), L('1'), L('8'), M(1148, 4), L('8'), L('4'), M(1512, 4), - L('3'), L('3'), L('2'), L('3'), L('9'), L('0'), L('7'), L('3'), - L('9'), L('4'), L('1'), L('4'), L('3'), L('3'), L('3'), L('4'), - L('5'), L('4'), L('7'), L('7'), L('6'), L('2'), L('4'), M(579, 4), - L('2'), L('5'), L('1'), L('8'), L('9'), L('8'), L('3'), L('5'), - L('6'), L('9'), L('4'), L('8'), L('5'), L('5'), L('6'), L('2'), - L('0'), L('9'), L('9'), L('2'), L('1'), L('9'), L('2'), L('2'), - L('2'), L('1'), L('8'), L('4'), L('2'), L('7'), M(575, 4), L('2'), - M(187, 4), L('6'), L('8'), L('8'), L('7'), L('6'), L('7'), L('1'), - L('7'), L('9'), L('0'), M(86, 4), L('0'), M(263, 5), L('6'), L('6'), - M(1000, 4), L('8'), L('8'), L('6'), L('2'), L('7'), L('2'), M(1757, 4), - L('1'), L('7'), L('8'), L('6'), L('0'), L('8'), L('5'), L('7'), - M(116, 4), L('3'), M(765, 5), L('7'), L('9'), L('7'), L('6'), L('6'), - L('8'), L('1'), M(702, 4), L('0'), L('0'), L('9'), L('5'), L('3'), - L('8'), L('8'), M(1593, 4), L('3'), M(1702, 4), L('0'), L('6'), L('8'), - L('0'), L('0'), L('6'), L('4'), L('2'), L('2'), L('5'), L('1'), - L('2'), L('5'), L('2'), M(1404, 4), L('7'), L('3'), L('9'), L('2'), - M(664, 4), M(1141, 4), L('4'), M(1716, 5), L('8'), L('6'), L('2'), L('6'), - L('9'), L('4'), L('5'), M(486, 4), L('4'), L('1'), L('9'), L('6'), - L('5'), L('2'), L('8'), L('5'), L('0'), M(154, 4), M(925, 4), L('1'), - L('8'), L('6'), L('3'), M(447, 4), L('4'), M(341, 5), L('2'), L('0'), - L('3'), L('9'), M(1420, 4), L('4'), L('5'), M(701, 4), L('2'), L('3'), - L('7'), M(1069, 4), L('6'), M(1297, 4), L('5'), L('6'), M(1593, 4), L('7'), - L('1'), L('9'), L('1'), L('7'), L('2'), L('8'), M(370, 4), L('7'), - L('6'), L('4'), L('6'), L('5'), L('7'), L('5'), L('7'), L('3'), - L('9'), M(258, 4), L('3'), L('8'), L('9'), M(1865, 4), L('8'), L('3'), - L('2'), L('6'), L('4'), L('5'), L('9'), L('9'), L('5'), L('8'), - M(1704, 4), L('0'), L('4'), L('7'), L('8'), M(479, 4), M(809, 4), L('9'), - M(46, 4), L('6'), L('4'), L('0'), L('7'), L('8'), L('9'), L('5'), - L('1'), M(143, 4), L('6'), L('8'), L('3'), M(304, 4), L('2'), L('5'), - L('9'), L('5'), L('7'), L('0'), M(1129, 4), L('8'), L('2'), L('2'), - M(713, 4), L('2'), M(1564, 4), L('4'), L('0'), L('7'), L('7'), L('2'), - L('6'), L('7'), L('1'), L('9'), L('4'), L('7'), L('8'), M(794, 4), - L('8'), L('2'), L('6'), L('0'), L('1'), L('4'), L('7'), L('6'), - L('9'), L('9'), L('0'), L('9'), M(1257, 4), L('0'), L('1'), L('3'), - L('6'), L('3'), L('9'), L('4'), L('4'), L('3'), M(640, 4), L('3'), - L('0'), M(262, 4), L('2'), L('0'), L('3'), L('4'), L('9'), L('6'), - L('2'), L('5'), L('2'), L('4'), L('5'), L('1'), L('7'), M(950, 4), - L('9'), L('6'), L('5'), L('1'), L('4'), L('3'), L('1'), L('4'), - L('2'), L('9'), L('8'), L('0'), L('9'), L('1'), L('9'), L('0'), - L('6'), L('5'), L('9'), L('2'), M(643, 4), L('7'), L('2'), L('2'), - L('1'), L('6'), L('9'), L('6'), L('4'), L('6'), M(1050, 4), M(123, 4), - L('5'), M(1295, 4), L('4'), M(1382, 5), L('8'), M(1370, 4), L('9'), L('7'), - M(1404, 4), L('5'), L('4'), M(1182, 4), M(575, 4), L('7'), M(1627, 4), L('8'), - L('4'), L('6'), L('8'), L('1'), L('3'), M(141, 4), L('6'), L('8'), - L('3'), L('8'), L('6'), L('8'), L('9'), L('4'), L('2'), L('7'), - L('7'), L('4'), L('1'), L('5'), L('5'), L('9'), L('9'), L('1'), - L('8'), L('5'), M(91, 4), L('2'), L('4'), L('5'), L('9'), L('5'), - L('3'), L('9'), L('5'), L('9'), L('4'), L('3'), L('1'), M(1464, 4), - L('7'), M(19, 4), L('6'), L('8'), L('0'), L('8'), L('4'), L('5'), - M(744, 4), L('7'), L('3'), M(2079, 4), L('9'), L('5'), L('8'), L('4'), - L('8'), L('6'), L('5'), L('3'), L('8'), M(1769, 4), L('6'), L('2'), - M(243, 4), L('6'), L('0'), L('9'), M(1207, 4), L('6'), L('0'), L('8'), - L('0'), L('5'), L('1'), L('2'), L('4'), L('3'), L('8'), L('8'), - L('4'), M(315, 4), M(12, 4), L('4'), L('1'), L('3'), M(784, 4), L('7'), - L('6'), L('2'), L('7'), L('8'), M(834, 4), L('7'), L('1'), L('5'), - M(1436, 4), L('3'), L('5'), L('9'), L('9'), L('7'), L('7'), L('0'), - L('0'), L('1'), L('2'), L('9'), M(1139, 4), L('8'), L('9'), L('4'), - L('4'), L('1'), M(632, 4), L('6'), L('8'), L('5'), L('5'), M(96, 4), - L('4'), L('0'), L('6'), L('3'), M(2279, 4), L('2'), L('0'), L('7'), - L('2'), L('2'), M(345, 4), M(516, 5), L('4'), L('8'), L('1'), L('5'), - L('8'), M(518, 4), M(511, 4), M(635, 4), M(665, 4), L('3'), L('9'), L('4'), - L('5'), L('2'), L('2'), L('6'), L('7'), M(1175, 6), L('8'), M(1419, 4), - L('2'), L('1'), M(747, 4), L('2'), M(904, 4), L('5'), L('4'), L('6'), - L('6'), L('6'), M(1308, 4), L('2'), L('3'), L('9'), L('8'), L('6'), - L('4'), L('5'), L('6'), M(1221, 4), L('1'), L('6'), L('3'), L('5'), - M(596, 5), M(2066, 4), L('7'), M(2222, 4), L('9'), L('8'), M(1119, 4), L('9'), - L('3'), L('6'), L('3'), L('4'), M(1884, 4), L('7'), L('4'), L('3'), - L('2'), L('4'), M(1148, 4), L('1'), L('5'), L('0'), L('7'), L('6'), - M(1212, 4), L('7'), L('9'), L('4'), L('5'), L('1'), L('0'), L('9'), - M(63, 4), L('0'), L('9'), L('4'), L('0'), M(1703, 4), L('8'), L('8'), - L('7'), L('9'), L('7'), L('1'), L('0'), L('8'), L('9'), L('3'), - M(2289, 4), L('6'), L('9'), L('1'), L('3'), L('6'), L('8'), L('6'), - L('7'), L('2'), M(604, 4), M(511, 4), L('5'), M(1344, 4), M(1129, 4), M(2050, 4), - L('1'), L('7'), L('9'), L('2'), L('8'), L('6'), L('8'), M(2253, 4), - L('8'), L('7'), L('4'), L('7'), M(1951, 5), L('8'), L('2'), L('4'), - M(2427, 4), L('8'), M(604, 4), L('7'), L('1'), L('4'), L('9'), L('0'), - L('9'), L('6'), L('7'), L('5'), L('9'), L('8'), M(1776, 4), L('3'), - L('6'), L('5'), M(309, 4), L('8'), L('1'), M(93, 4), M(1862, 4), M(2359, 4), - L('6'), L('8'), L('2'), L('9'), M(1407, 4), L('8'), L('7'), L('2'), - L('2'), L('6'), L('5'), L('8'), L('8'), L('0'), M(1554, 4), L('5'), - M(586, 4), L('4'), L('2'), L('7'), L('0'), L('4'), L('7'), L('7'), - L('5'), L('5'), M(2079, 4), L('3'), L('7'), L('9'), L('6'), L('4'), - L('1'), L('4'), L('5'), L('1'), L('5'), L('2'), M(1534, 4), L('2'), - L('3'), L('4'), L('3'), L('6'), L('4'), L('5'), L('4'), M(1503, 4), - L('4'), L('4'), L('4'), L('7'), L('9'), L('5'), M(61, 4), M(1316, 4), - M(2279, 5), L('4'), L('1'), M(1323, 4), L('3'), M(773, 4), L('5'), L('2'), - L('3'), L('1'), M(2114, 5), L('1'), L('6'), L('6'), L('1'), M(2227, 4), - L('5'), L('9'), L('6'), L('9'), L('5'), L('3'), L('6'), L('2'), - L('3'), L('1'), L('4'), M(1536, 4), L('2'), L('4'), L('8'), L('4'), - L('9'), L('3'), L('7'), L('1'), L('8'), L('7'), L('1'), L('1'), - L('0'), L('1'), L('4'), L('5'), L('7'), L('6'), L('5'), L('4'), - M(1890, 4), L('0'), L('2'), L('7'), L('9'), L('9'), L('3'), L('4'), - L('4'), L('0'), L('3'), L('7'), L('4'), L('2'), L('0'), L('0'), - L('7'), M(2368, 4), L('7'), L('8'), L('5'), L('3'), L('9'), L('0'), - L('6'), L('2'), L('1'), L('9'), M(666, 5), M(838, 4), L('8'), L('4'), - L('7'), M(979, 5), L('8'), L('3'), L('3'), L('2'), L('1'), L('4'), - L('4'), L('5'), L('7'), L('1'), M(645, 4), M(1911, 4), L('4'), L('3'), - L('5'), L('0'), M(2345, 4), M(1129, 4), L('5'), L('3'), L('1'), L('9'), - L('1'), L('0'), L('4'), L('8'), L('4'), L('8'), L('1'), L('0'), - L('0'), L('5'), L('3'), L('7'), L('0'), L('6'), M(2237, 4), M(1438, 5), - M(1922, 5), L('1'), M(1370, 4), L('7'), M(796, 4), L('5'), M(2029, 4), M(1037, 4), - L('6'), L('3'), M(2013, 5), L('4'), M(2418, 4), M(847, 5), M(1014, 5), L('8'), - M(1326, 5), M(2184, 5), L('9'), M(392, 4), L('9'), L('1'), M(2255, 4), L('8'), - L('1'), L('4'), L('6'), L('7'), L('5'), L('1'), M(1580, 4), L('1'), - L('2'), L('3'), L('9'), M(426, 6), L('9'), L('0'), L('7'), L('1'), - L('8'), L('6'), L('4'), L('9'), L('4'), L('2'), L('3'), L('1'), - L('9'), L('6'), L('1'), L('5'), L('6'), M(493, 4), M(1725, 4), L('9'), - L('5'), M(2343, 4), M(1130, 4), M(284, 4), L('6'), L('0'), L('3'), L('8'), - M(2598, 4), M(368, 4), M(901, 4), L('6'), L('2'), M(1115, 4), L('5'), M(2125, 4), - L('6'), L('3'), L('8'), L('9'), L('3'), L('7'), L('7'), L('8'), - L('7'), M(2246, 4), M(249, 4), L('9'), L('7'), L('9'), L('2'), L('0'), - L('7'), L('7'), L('3'), M(1496, 4), L('2'), L('1'), L('8'), L('2'), - L('5'), L('6'), M(2016, 4), L('6'), L('6'), M(1751, 4), L('4'), L('2'), - M(1663, 5), L('6'), M(1767, 4), L('4'), L('4'), M(37, 4), L('5'), L('4'), - L('9'), L('2'), L('0'), L('2'), L('6'), L('0'), L('5'), M(2740, 4), - M(997, 5), L('2'), L('0'), L('1'), L('4'), L('9'), M(1235, 4), L('8'), - L('5'), L('0'), L('7'), L('3'), M(1434, 4), L('6'), L('6'), L('6'), - L('0'), M(405, 4), L('2'), L('4'), L('3'), L('4'), L('0'), M(136, 4), - L('0'), M(1900, 4), L('8'), L('6'), L('3'), M(2391, 4), M(2021, 4), M(1068, 4), - M(373, 4), L('5'), L('7'), L('9'), L('6'), L('2'), L('6'), L('8'), - L('5'), L('6'), M(321, 4), L('5'), L('0'), L('8'), M(1316, 4), L('5'), - L('8'), L('7'), L('9'), L('6'), L('9'), L('9'), M(1810, 4), L('5'), - L('7'), L('4'), M(2585, 4), L('8'), L('4'), L('0'), M(2228, 4), L('1'), - L('4'), L('5'), L('9'), L('1'), M(1933, 4), L('7'), L('0'), M(565, 4), - L('0'), L('1'), M(3048, 4), L('1'), L('2'), M(3189, 4), L('0'), M(964, 4), - L('3'), L('9'), M(2859, 4), M(275, 4), L('7'), L('1'), L('5'), M(945, 4), - L('4'), L('2'), L('0'), M(3059, 5), L('9'), M(3011, 4), L('0'), L('7'), - M(834, 4), M(1942, 4), M(2736, 4), M(3171, 4), L('2'), L('1'), M(2401, 4), L('2'), - L('5'), L('1'), M(1404, 4), M(2373, 4), L('9'), L('2'), M(435, 4), L('8'), - L('2'), L('6'), M(2919, 4), L('2'), M(633, 4), L('3'), L('2'), L('1'), - L('5'), L('7'), L('9'), L('1'), L('9'), L('8'), L('4'), L('1'), - L('4'), M(2172, 5), L('9'), L('1'), L('6'), L('4'), M(1769, 5), L('9'), - M(2905, 5), M(2268, 4), L('7'), L('2'), L('2'), M(802, 4), L('5'), M(2213, 4), - M(322, 4), L('9'), L('1'), L('0'), M(189, 4), M(3164, 4), L('5'), L('2'), - L('8'), L('0'), L('1'), L('7'), M(562, 4), L('7'), L('1'), L('2'), - M(2325, 4), L('8'), L('3'), L('2'), M(884, 4), L('1'), M(1418, 4), L('0'), - L('9'), L('3'), L('5'), L('3'), L('9'), L('6'), L('5'), L('7'), - M(1612, 4), L('1'), L('0'), L('8'), L('3'), M(106, 4), L('5'), L('1'), - M(1915, 4), M(3419, 4), L('1'), L('4'), L('4'), L('4'), L('2'), L('1'), - L('0'), L('0'), M(515, 4), L('0'), L('3'), M(413, 4), L('1'), L('1'), - L('0'), L('3'), M(3202, 4), M(10, 4), M(39, 4), M(1539, 6), L('5'), L('1'), - L('6'), M(1498, 4), M(2180, 5), M(2347, 4), L('5'), M(3139, 5), L('8'), L('5'), - L('1'), L('7'), L('1'), L('4'), L('3'), L('7'), M(1542, 4), M(110, 4), - L('1'), L('5'), L('5'), L('6'), L('5'), L('0'), L('8'), L('8'), - M(954, 4), L('9'), L('8'), L('9'), L('8'), L('5'), L('9'), L('9'), - L('8'), L('2'), L('3'), L('8'), M(464, 4), M(2491, 4), L('3'), M(365, 4), - M(1087, 4), M(2500, 4), L('8'), M(3590, 5), L('3'), L('2'), M(264, 4), L('5'), - M(774, 4), L('3'), M(459, 4), L('9'), M(1052, 4), L('9'), L('8'), M(2174, 4), - L('4'), M(3257, 4), L('7'), M(1612, 4), L('0'), L('7'), M(230, 4), L('4'), - L('8'), L('1'), L('4'), L('1'), M(1338, 4), L('8'), L('5'), L('9'), - L('4'), L('6'), L('1'), M(3018, 4), L('8'), L('0'), - }, - }, - TestCase{ - .input = "huffman-rand-1k.input", - .want = "huffman-rand-1k.{s}.expect", - .want_no_input = "huffman-rand-1k.{s}.expect-noinput", - .tokens = &[_]Token{ - L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xd), L(0x85), L(0x94), L(0x25), L(0x80), L(0xaf), L(0xc2), L(0xfe), L(0x8d), - L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde), L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b), - L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73), L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4), - L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15), L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde), - L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9), L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14), - L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xd), L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c), - L(0x99), L(0xdf), L(0xfd), L(0x70), L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5), - L(0xd4), L(0x80), L(0x59), L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20), - L(0x1b), L(0x46), L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4), - L(0x7b), L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28), - L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f), L(0xa0), - L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4), L(0x75), L(0xda), - L(0xea), L(0x9b), L(0xa), L(0x64), L(0xb), L(0xe0), L(0x23), L(0x29), L(0xbd), L(0xf7), L(0xe7), L(0x83), L(0x3c), L(0xfb), - L(0xdf), L(0xb3), L(0xae), L(0x4f), L(0xa4), L(0x47), L(0x55), L(0x99), L(0xde), L(0x2f), L(0x96), L(0x6e), L(0x1c), L(0x43), - L(0x4c), L(0x87), L(0xe2), L(0x7c), L(0xd9), L(0x5f), L(0x4c), L(0x7c), L(0xe8), L(0x90), L(0x3), L(0xdb), L(0x30), L(0x95), - L(0xd6), L(0x22), L(0xc), L(0x47), L(0xb8), L(0x4d), L(0x6b), L(0xbd), L(0x24), L(0x11), L(0xab), L(0x2c), L(0xd7), L(0xbe), - L(0x6e), L(0x7a), L(0xd6), L(0x8), L(0xa3), L(0x98), L(0xd8), L(0xdd), L(0x15), L(0x6a), L(0xfa), L(0x93), L(0x30), L(0x1), - L(0x25), L(0x1d), L(0xa2), L(0x74), L(0x86), L(0x4b), L(0x6a), L(0x95), L(0xe8), L(0xe1), L(0x4e), L(0xe), L(0x76), L(0xb9), - L(0x49), L(0xa9), L(0x5f), L(0xa0), L(0xa6), L(0x63), L(0x3c), L(0x7e), L(0x7e), L(0x20), L(0x13), L(0x4f), L(0xbb), L(0x66), - L(0x92), L(0xb8), L(0x2e), L(0xa4), L(0xfa), L(0x48), L(0xcb), L(0xae), L(0xb9), L(0x3c), L(0xaf), L(0xd3), L(0x1f), L(0xe1), - L(0xd5), L(0x8d), L(0x42), L(0x6d), L(0xf0), L(0xfc), L(0x8c), L(0xc), L(0x0), L(0xde), L(0x40), L(0xab), L(0x8b), L(0x47), - L(0x97), L(0x4e), L(0xa8), L(0xcf), L(0x8e), L(0xdb), L(0xa6), L(0x8b), L(0x20), L(0x9), L(0x84), L(0x7a), L(0x66), L(0xe5), - L(0x98), L(0x29), L(0x2), L(0x95), L(0xe6), L(0x38), L(0x32), L(0x60), L(0x3), L(0xe3), L(0x9a), L(0x1e), L(0x54), L(0xe8), - L(0x63), L(0x80), L(0x48), L(0x9c), L(0xe7), L(0x63), L(0x33), L(0x6e), L(0xa0), L(0x65), L(0x83), L(0xfa), L(0xc6), L(0xba), - L(0x7a), L(0x43), L(0x71), L(0x5), L(0xf5), L(0x68), L(0x69), L(0x85), L(0x9c), L(0xba), L(0x45), L(0xcd), L(0x6b), L(0xb), - L(0x19), L(0xd1), L(0xbb), L(0x7f), L(0x70), L(0x85), L(0x92), L(0xd1), L(0xb4), L(0x64), L(0x82), L(0xb1), L(0xe4), L(0x62), - L(0xc5), L(0x3c), L(0x46), L(0x1f), L(0x92), L(0x31), L(0x1c), L(0x4e), L(0x41), L(0x77), L(0xf7), L(0xe7), L(0x87), L(0xa2), - L(0xf), L(0x6e), L(0xe8), L(0x92), L(0x3), L(0x6b), L(0xa), L(0xe7), L(0xa9), L(0x3b), L(0x11), L(0xda), L(0x66), L(0x8a), - L(0x29), L(0xda), L(0x79), L(0xe1), L(0x64), L(0x8d), L(0xe3), L(0x54), L(0xd4), L(0xf5), L(0xef), L(0x64), L(0x87), L(0x3b), - L(0xf4), L(0xc2), L(0xf4), L(0x71), L(0x13), L(0xa9), L(0xe9), L(0xe0), L(0xa2), L(0x6), L(0x14), L(0xab), L(0x5d), L(0xa7), - L(0x96), L(0x0), L(0xd6), L(0xc3), L(0xcc), L(0x57), L(0xed), L(0x39), L(0x6a), L(0x25), L(0xcd), L(0x76), L(0xea), L(0xba), - L(0x3a), L(0xf2), L(0xa1), L(0x95), L(0x5d), L(0xe5), L(0x71), L(0xcf), L(0x9c), L(0x62), L(0x9e), L(0x6a), L(0xfa), L(0xd5), - L(0x31), L(0xd1), L(0xa8), L(0x66), L(0x30), L(0x33), L(0xaa), L(0x51), L(0x17), L(0x13), L(0x82), L(0x99), L(0xc8), L(0x14), - L(0x60), L(0x9f), L(0x4d), L(0x32), L(0x6d), L(0xda), L(0x19), L(0x26), L(0x21), L(0xdc), L(0x7e), L(0x2e), L(0x25), L(0x67), - L(0x72), L(0xca), L(0xf), L(0x92), L(0xcd), L(0xf6), L(0xd6), L(0xcb), L(0x97), L(0x8a), L(0x33), L(0x58), L(0x73), L(0x70), - L(0x91), L(0x1d), L(0xbf), L(0x28), L(0x23), L(0xa3), L(0xc), L(0xf1), L(0x83), L(0xc3), L(0xc8), L(0x56), L(0x77), L(0x68), - L(0xe3), L(0x82), L(0xba), L(0xb9), L(0x57), L(0x56), L(0x57), L(0x9c), L(0xc3), L(0xd6), L(0x14), L(0x5), L(0x3c), L(0xb1), - L(0xaf), L(0x93), L(0xc8), L(0x8a), L(0x57), L(0x7f), L(0x53), L(0xfa), L(0x2f), L(0xaa), L(0x6e), L(0x66), L(0x83), L(0xfa), - L(0x33), L(0xd1), L(0x21), L(0xab), L(0x1b), L(0x71), L(0xb4), L(0x7c), L(0xda), L(0xfd), L(0xfb), L(0x7f), L(0x20), L(0xab), - L(0x5e), L(0xd5), L(0xca), L(0xfd), L(0xdd), L(0xe0), L(0xee), L(0xda), L(0xba), L(0xa8), L(0x27), L(0x99), L(0x97), L(0x69), - L(0xc1), L(0x3c), L(0x82), L(0x8c), L(0xa), L(0x5c), L(0x2d), L(0x5b), L(0x88), L(0x3e), L(0x34), L(0x35), L(0x86), L(0x37), - L(0x46), L(0x79), L(0xe1), L(0xaa), L(0x19), L(0xfb), L(0xaa), L(0xde), L(0x15), L(0x9), L(0xd), L(0x1a), L(0x57), L(0xff), - L(0xb5), L(0xf), L(0xf3), L(0x2b), L(0x5a), L(0x6a), L(0x4d), L(0x19), L(0x77), L(0x71), L(0x45), L(0xdf), L(0x4f), L(0xb3), - L(0xec), L(0xf1), L(0xeb), L(0x18), L(0x53), L(0x3e), L(0x3b), L(0x47), L(0x8), L(0x9a), L(0x73), L(0xa0), L(0x5c), L(0x8c), - L(0x5f), L(0xeb), L(0xf), L(0x3a), L(0xc2), L(0x43), L(0x67), L(0xb4), L(0x66), L(0x67), L(0x80), L(0x58), L(0xe), L(0xc1), - L(0xec), L(0x40), L(0xd4), L(0x22), L(0x94), L(0xca), L(0xf9), L(0xe8), L(0x92), L(0xe4), L(0x69), L(0x38), L(0xbe), L(0x67), - L(0x64), L(0xca), L(0x50), L(0xc7), L(0x6), L(0x67), L(0x42), L(0x6e), L(0xa3), L(0xf0), L(0xb7), L(0x6c), L(0xf2), L(0xe8), - L(0x5f), L(0xb1), L(0xaf), L(0xe7), L(0xdb), L(0xbb), L(0x77), L(0xb5), L(0xf8), L(0xcb), L(0x8), L(0xc4), L(0x75), L(0x7e), - L(0xc0), L(0xf9), L(0x1c), L(0x7f), L(0x3c), L(0x89), L(0x2f), L(0xd2), L(0x58), L(0x3a), L(0xe2), L(0xf8), L(0x91), L(0xb6), - L(0x7b), L(0x24), L(0x27), L(0xe9), L(0xae), L(0x84), L(0x8b), L(0xde), L(0x74), L(0xac), L(0xfd), L(0xd9), L(0xb7), L(0x69), - L(0x2a), L(0xec), L(0x32), L(0x6f), L(0xf0), L(0x92), L(0x84), L(0xf1), L(0x40), L(0xc), L(0x8a), L(0xbc), L(0x39), L(0x6e), - L(0x2e), L(0x73), L(0xd4), L(0x6e), L(0x8a), L(0x74), L(0x2a), L(0xdc), L(0x60), L(0x1f), L(0xa3), L(0x7), L(0xde), L(0x75), - L(0x8b), L(0x74), L(0xc8), L(0xfe), L(0x63), L(0x75), L(0xf6), L(0x3d), L(0x63), L(0xac), L(0x33), L(0x89), L(0xc3), L(0xf0), - L(0xf8), L(0x2d), L(0x6b), L(0xb4), L(0x9e), L(0x74), L(0x8b), L(0x5c), L(0x33), L(0xb4), L(0xca), L(0xa8), L(0xe4), L(0x99), - L(0xb6), L(0x90), L(0xa1), L(0xef), L(0xf), L(0xd3), L(0x61), L(0xb2), L(0xc6), L(0x1a), L(0x94), L(0x7c), L(0x44), L(0x55), - L(0xf4), L(0x45), L(0xff), L(0x9e), L(0xa5), L(0x5a), L(0xc6), L(0xa0), L(0xe8), L(0x2a), L(0xc1), L(0x8d), L(0x6f), L(0x34), - L(0x11), L(0xb9), L(0xbe), L(0x4e), L(0xd9), L(0x87), L(0x97), L(0x73), L(0xcf), L(0x3d), L(0x23), L(0xae), L(0xd5), L(0x1a), - L(0x5e), L(0xae), L(0x5d), L(0x6a), L(0x3), L(0xf9), L(0x22), L(0xd), L(0x10), L(0xd9), L(0x47), L(0x69), L(0x15), L(0x3f), - L(0xee), L(0x52), L(0xa3), L(0x8), L(0xd2), L(0x3c), L(0x51), L(0xf4), L(0xf8), L(0x9d), L(0xe4), L(0x98), L(0x89), L(0xc8), - L(0x67), L(0x39), L(0xd5), L(0x5e), L(0x35), L(0x78), L(0x27), L(0xe8), L(0x3c), L(0x80), L(0xae), L(0x79), L(0x71), L(0xd2), - L(0x93), L(0xf4), L(0xaa), L(0x51), L(0x12), L(0x1c), L(0x4b), L(0x1b), L(0xe5), L(0x6e), L(0x15), L(0x6f), L(0xe4), L(0xbb), - L(0x51), L(0x9b), L(0x45), L(0x9f), L(0xf9), L(0xc4), L(0x8c), L(0x2a), L(0xfb), L(0x1a), L(0xdf), L(0x55), L(0xd3), L(0x48), - L(0x93), L(0x27), L(0x1), L(0x26), L(0xc2), L(0x6b), L(0x55), L(0x6d), L(0xa2), L(0xfb), L(0x84), L(0x8b), L(0xc9), L(0x9e), - L(0x28), L(0xc2), L(0xef), L(0x1a), L(0x24), L(0xec), L(0x9b), L(0xae), L(0xbd), L(0x60), L(0xe9), L(0x15), L(0x35), L(0xee), - L(0x42), L(0xa4), L(0x33), L(0x5b), L(0xfa), L(0xf), L(0xb6), L(0xf7), L(0x1), L(0xa6), L(0x2), L(0x4c), L(0xca), L(0x90), - L(0x58), L(0x3a), L(0x96), L(0x41), L(0xe7), L(0xcb), L(0x9), L(0x8c), L(0xdb), L(0x85), L(0x4d), L(0xa8), L(0x89), L(0xf3), - L(0xb5), L(0x8e), L(0xfd), L(0x75), L(0x5b), L(0x4f), L(0xed), L(0xde), L(0x3f), L(0xeb), L(0x38), L(0xa3), L(0xbe), L(0xb0), - L(0x73), L(0xfc), L(0xb8), L(0x54), L(0xf7), L(0x4c), L(0x30), L(0x67), L(0x2e), L(0x38), L(0xa2), L(0x54), L(0x18), L(0xba), - L(0x8), L(0xbf), L(0xf2), L(0x39), L(0xd5), L(0xfe), L(0xa5), L(0x41), L(0xc6), L(0x66), L(0x66), L(0xba), L(0x81), L(0xef), - L(0x67), L(0xe4), L(0xe6), L(0x3c), L(0xc), L(0xca), L(0xa4), L(0xa), L(0x79), L(0xb3), L(0x57), L(0x8b), L(0x8a), L(0x75), - L(0x98), L(0x18), L(0x42), L(0x2f), L(0x29), L(0xa3), L(0x82), L(0xef), L(0x9f), L(0x86), L(0x6), L(0x23), L(0xe1), L(0x75), - L(0xfa), L(0x8), L(0xb1), L(0xde), L(0x17), L(0x4a), - }, - }, - TestCase{ - .input = "huffman-rand-limit.input", - .want = "huffman-rand-limit.{s}.expect", - .want_no_input = "huffman-rand-limit.{s}.expect-noinput", - .tokens = &[_]Token{ - L(0x61), M(1, 74), L(0xa), L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xa), L(0x85), L(0x94), L(0x25), L(0x80), - L(0xaf), L(0xc2), L(0xfe), L(0x8d), L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde), - L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b), L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73), - L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4), L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15), - L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde), L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9), - L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14), L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xa), - L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c), L(0x99), L(0xdf), L(0xfd), L(0x70), - L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5), L(0xd4), L(0x80), L(0x59), - L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20), L(0x1b), L(0x46), - L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4), L(0x7b), - L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28), - L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f), - L(0xa0), L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4), - L(0x75), L(0xda), L(0xea), L(0x9b), L(0xa), - }, - }, - TestCase{ - .input = "huffman-shifts.input", - .want = "huffman-shifts.{s}.expect", - .want_no_input = "huffman-shifts.{s}.expect-noinput", - .tokens = &[_]Token{ - L('1'), L('0'), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), - M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), - M(2, 258), M(2, 76), L(0xd), L(0xa), L('2'), L('3'), M(2, 258), M(2, 258), - M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 256), - }, - }, - TestCase{ - .input = "huffman-text-shift.input", - .want = "huffman-text-shift.{s}.expect", - .want_no_input = "huffman-text-shift.{s}.expect-noinput", - .tokens = &[_]Token{ - L('/'), L('/'), L('C'), L('o'), L('p'), L('y'), L('r'), L('i'), - L('g'), L('h'), L('t'), L('2'), L('0'), L('0'), L('9'), L('T'), - L('h'), L('G'), L('o'), L('A'), L('u'), L('t'), L('h'), L('o'), - L('r'), L('.'), L('A'), L('l'), L('l'), M(23, 5), L('r'), L('r'), - L('v'), L('d'), L('.'), L(0xd), L(0xa), L('/'), L('/'), L('U'), - L('o'), L('f'), L('t'), L('h'), L('i'), L('o'), L('u'), L('r'), - L('c'), L('c'), L('o'), L('d'), L('i'), L('g'), L('o'), L('v'), - L('r'), L('n'), L('d'), L('b'), L('y'), L('B'), L('S'), L('D'), - L('-'), L('t'), L('y'), L('l'), M(33, 4), L('l'), L('i'), L('c'), - L('n'), L('t'), L('h'), L('t'), L('c'), L('n'), L('b'), L('f'), - L('o'), L('u'), L('n'), L('d'), L('i'), L('n'), L('t'), L('h'), - L('L'), L('I'), L('C'), L('E'), L('N'), L('S'), L('E'), L('f'), - L('i'), L('l'), L('.'), L(0xd), L(0xa), L(0xd), L(0xa), L('p'), - L('c'), L('k'), L('g'), L('m'), L('i'), L('n'), M(11, 4), L('i'), - L('m'), L('p'), L('o'), L('r'), L('t'), L('"'), L('o'), L('"'), - M(13, 4), L('f'), L('u'), L('n'), L('c'), L('m'), L('i'), L('n'), - L('('), L(')'), L('{'), L(0xd), L(0xa), L(0x9), L('v'), L('r'), - L('b'), L('='), L('m'), L('k'), L('('), L('['), L(']'), L('b'), - L('y'), L('t'), L(','), L('6'), L('5'), L('5'), L('3'), L('5'), - L(')'), L(0xd), L(0xa), L(0x9), L('f'), L(','), L('_'), L(':'), - L('='), L('o'), L('.'), L('C'), L('r'), L('t'), L('('), L('"'), - L('h'), L('u'), L('f'), L('f'), L('m'), L('n'), L('-'), L('n'), - L('u'), L('l'), L('l'), L('-'), L('m'), L('x'), L('.'), L('i'), - L('n'), L('"'), M(34, 5), L('.'), L('W'), L('r'), L('i'), L('t'), - L('('), L('b'), L(')'), L(0xd), L(0xa), L('}'), L(0xd), L(0xa), - L('A'), L('B'), L('C'), L('D'), L('E'), L('F'), L('G'), L('H'), - L('I'), L('J'), L('K'), L('L'), L('M'), L('N'), L('O'), L('P'), - L('Q'), L('R'), L('S'), L('T'), L('U'), L('V'), L('X'), L('x'), - L('y'), L('z'), L('!'), L('"'), L('#'), L(0xc2), L(0xa4), L('%'), - L('&'), L('/'), L('?'), L('"'), - }, - }, - TestCase{ - .input = "huffman-text.input", - .want = "huffman-text.{s}.expect", - .want_no_input = "huffman-text.{s}.expect-noinput", - .tokens = &[_]Token{ - L('/'), L('/'), L(' '), L('z'), L('i'), L('g'), L(' '), L('v'), - L('0'), L('.'), L('1'), L('0'), L('.'), L('0'), L(0xa), L('/'), - L('/'), L(' '), L('c'), L('r'), L('e'), L('a'), L('t'), L('e'), - L(' '), L('a'), L(' '), L('f'), L('i'), L('l'), L('e'), M(5, 4), - L('l'), L('e'), L('d'), L(' '), L('w'), L('i'), L('t'), L('h'), - L(' '), L('0'), L('x'), L('0'), L('0'), L(0xa), L('c'), L('o'), - L('n'), L('s'), L('t'), L(' '), L('s'), L('t'), L('d'), L(' '), - L('='), L(' '), L('@'), L('i'), L('m'), L('p'), L('o'), L('r'), - L('t'), L('('), L('"'), L('s'), L('t'), L('d'), L('"'), L(')'), - L(';'), L(0xa), L(0xa), L('p'), L('u'), L('b'), L(' '), L('f'), - L('n'), L(' '), L('m'), L('a'), L('i'), L('n'), L('('), L(')'), - L(' '), L('!'), L('v'), L('o'), L('i'), L('d'), L(' '), L('{'), - L(0xa), L(' '), L(' '), L(' '), L(' '), L('v'), L('a'), L('r'), - L(' '), L('b'), L(' '), L('='), L(' '), L('['), L('1'), L(']'), - L('u'), L('8'), L('{'), L('0'), L('}'), L(' '), L('*'), L('*'), - L(' '), L('6'), L('5'), L('5'), L('3'), L('5'), L(';'), M(31, 5), - M(86, 6), L('f'), L(' '), L('='), L(' '), L('t'), L('r'), L('y'), - M(94, 4), L('.'), L('f'), L('s'), L('.'), L('c'), L('w'), L('d'), - L('('), L(')'), L('.'), M(144, 6), L('F'), L('i'), L('l'), L('e'), - L('('), M(43, 5), M(1, 4), L('"'), L('h'), L('u'), L('f'), L('f'), - L('m'), L('a'), L('n'), L('-'), L('n'), L('u'), L('l'), L('l'), - L('-'), L('m'), L('a'), L('x'), L('.'), L('i'), L('n'), L('"'), - L(','), M(31, 9), L('.'), L('{'), L(' '), L('.'), L('r'), L('e'), - L('a'), L('d'), M(79, 5), L('u'), L('e'), L(' '), L('}'), M(27, 6), - L(')'), M(108, 6), L('d'), L('e'), L('f'), L('e'), L('r'), L(' '), - L('f'), L('.'), L('c'), L('l'), L('o'), L('s'), L('e'), L('('), - M(183, 4), M(22, 4), L('_'), M(124, 7), L('f'), L('.'), L('w'), L('r'), - L('i'), L('t'), L('e'), L('A'), L('l'), L('l'), L('('), L('b'), - L('['), L('0'), L('.'), L('.'), L(']'), L(')'), L(';'), L(0xa), - L('}'), L(0xa), - }, - }, - TestCase{ - .input = "huffman-zero.input", - .want = "huffman-zero.{s}.expect", - .want_no_input = "huffman-zero.{s}.expect-noinput", - .tokens = &[_]Token{ L(0x30), ml, M(1, 49) }, - }, - TestCase{ - .input = "", - .want = "", - .want_no_input = "null-long-match.{s}.expect-noinput", - .tokens = &[_]Token{ - L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, M(1, 8), - }, - }, - }; -}; diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect deleted file mode 100644 index c08165143f2c570013c4916cbac5addfe9622a55..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 78 ZcmaEJppgLx8W#LrDZUcKq5v#l0|1+Y23i0B diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput deleted file mode 100644 index c08165143f2c570013c4916cbac5addfe9622a55..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 78 ZcmaEJppgLx8W#LrDZUcKq5v#l0|1+Y23i0B diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect deleted file mode 100644 index db422ca3983d12e71e31979d7b3dddd080dcbca7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8204 zcmeIuK@9*P3YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ t0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFwg)F00961 diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect deleted file mode 100644 index c08165143f2c570013c4916cbac5addfe9622a55..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 78 ZcmaEJppgLx8W#LrDZUcKq5v#l0|1+Y23i0B diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput deleted file mode 100644 index c08165143f2c570013c4916cbac5addfe9622a55..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 78 ZcmaEJppgLx8W#LrDZUcKq5v#l0|1+Y23i0B diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect deleted file mode 100644 index e4396ac6fe5e34609ccb7ea0bc359e6adb48c7f4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1696 zcmV;R24DFkmtE59U+}dp3M$E(%$UAJ`ff>rsvsiW8T+$6ZwCa`!Y=s-_luo9MajP$09#>I(F*#bYgkvGSvgH9cjqJxOtZL@E-R zxap{F9H>K0YPWsSkS2)R*aWKe{#|WqFIuv-wS}!bk75c(Z-9;7Wc4VnBBzs?752D& zc8p>URDdmkKtvR3uWp%l!&_EmTpc=NETFYQZ$(jWT@; zgN3|cc@&v*F@uVLa&KnX>Fd2bZUkkwfB)b_MW1tl319U*%S zvp^|A=dI~L9VRO0%SM^tpIF);2v& z2UTM|Eu;@^j|Ys3yuqcmNp8%xRb#N#JWo+RBgezuM69fAg{7zjhSjaxj9hCIS<|)) zTLN?jLt7gbXKG}iEUuqR-jG}(yN@N#B)wX z?|Hml6#3}s*c0K~nJep+6gLc-%e0Zx+0e0@vrzAOGcG64J5tD?3)Gal%l@md3K`X! zWHzzhS`E>KPF)C!q0$!IOpK<-WbmbF9QLE^nXFo~mu))PKI>??oiY z2eq0;6HL=Tt81EVym$AC{;?VPYEHwbEH44G@EQbW;L1XcSd)b||Ff@Ei(4Sj++jOm zBUh^KsO^kc_oqFUViJ1J^cG$3Tj{GxbaP=7I(EAlE=mRs3qthuA%e9rE-#PHFM(mQ zu6KhDd&6Mrg?qbky>)t9e~*^0hsbjfTxSkFOE@c#rEgM-#Z9ZTpaI9jc6f=dNhXc8 znW%G1wBBCANuz}>6H}+!y>*N6gKL$sTjqM=lH+`zajbQ|_!-Asw+~_~BPZz2`j$Kc zEhFt1TPE|&golz{9lnon*4~tBl|$aFu;^S(&T%XtkV=$yRZ5cBjJLTgxTv7rS!-y$2B``yh?Bd zU87(35T;+y=@n~to6Yow&?UtR3gMggy9M(CYsW0orRXZXb1;cR#nNz{C5S6uiE#A# z)e7C6h_D5sJRBg(Zy^5U!@dY0#$+}dp3M$E(%$UAJ`ff>rsvsiW8T+$6ZwCa`!Y=s-_luo9MajP$09#>I(F*#bYgkvGSvgH9cjqJxOtZL@E-R zxap{F9H>K0YPWsSkS2)R*aWKe{#|WqFIuv-wS}!bk75c(Z-9;7Wc4VnBBzs?752D& zc8p>URDdmkKtvR3uWp%l!&_EmTpc=NETFYQZ$(jWT@; zgN3|cc@&v*F@uVLa&KnX>Fd2bZUkkwfB)b_MW1tl319U*%S zvp^|A=dI~L9VRO0%SM^tpIF);2v& z2UTM|Eu;@^j|Ys3yuqcmNp8%xRb#N#JWo+RBgezuM69fAg{7zjhSjaxj9hCIS<|)) zTLN?jLt7gbXKG}iEUuqR-jG}(yN@N#B)wX z?|Hml6#3}s*c0K~nJep+6gLc-%e0Zx+0e0@vrzAOGcG64J5tD?3)Gal%l@md3K`X! zWHzzhS`E>KPF)C!q0$!IOpK<-WbmbF9QLE^nXFo~mu))PKI>??oiY z2eq0;6HL=Tt81EVym$AC{;?VPYEHwbEH44G@EQbW;L1XcSd)b||Ff@Ei(4Sj++jOm zBUh^KsO^kc_oqFUViJ1J^cG$3Tj{GxbaP=7I(EAlE=mRs3qthuA%e9rE-#PHFM(mQ zu6KhDd&6Mrg?qbky>)t9e~*^0hsbjfTxSkFOE@c#rEgM-#Z9ZTpaI9jc6f=dNhXc8 znW%G1wBBCANuz}>6H}+!y>*N6gKL$sTjqM=lH+`zajbQ|_!-Asw+~_~BPZz2`j$Kc zEhFt1TPE|&golz{9lnon*4~tBl|$aFu;^S(&T%XtkV=$yRZ5cBjJLTgxTv7rS!-y$2B``yh?Bd zU87(35T;+y=@n~to6Yow&?UtR3gMggy9M(CYsW0orRXZXb1;cR#nNz{C5S6uiE#A# z)e7C6h_D5sJRBg(Zy^5U!@dY0#$WHH?*pN(^^{)UzR+vFm$z5VCPWYJYu^c@?DSYt z-8UDQT!V^U$bB)7T#vx*u3cr>8Uiz?!&E~$_X|MfTLzZ%-*0cT3{sA zq<70y?3)+V47+RWm;KF_UZSrB{g zmdef8;|D3hF@bivQ*X8_PI1sPpD$f?o@apfMsLJqhqe|k1YMzo08jbe==SDWr>dyl zbP)shg1$9yeHZ}|Ge>&gwccapdqSeq-ZI7R^#yqYl*Kmh`QG!AW7&KY&*GJ(U$x77 zhtAF78c+Fd_HOjLIGnc+=hME>#~}3C0p=D;~UUo6}`h~uDe&Kt8|~ZgH{F>+ga2Ta~_F64W74myu+K;Bjn~5cx>A?@3xm= zD~x&%_crj<4^cHss+1nx$(uD-yKl03k90IoYxDsL4uhF_c$b&cCRR@G?c@dAF%EZj z!F%)6tR6euoJ8jypvoJ+WS`#U)OSL47esd%y~SbI85?f~-OJeks!kW?4w}A9wYnXQ z#VCGziPdN5y&!va_RSkqJ5O&AZJkzk&%0`gmtxsl+-dW8#e*OnqgyUTXk1m3t=~z ztn{vTymfdO4o{D!cQUlYRmWwqeGkufvuO$~hNp#hD}F4UuGsSZ#)Bo zS{7a0Tik+o`!pa(^qw3sFrES3(@Kl?cse319lmJa+t;~Qg6}kRGv`+WxC0;tQ z^43(V;Jpqnv!~9qw9cGMg>~t?=4n;kG^^}-N4rcpck4rWS%+ByUK=%87P!zqw&Du-Yd21qtBOAtHq0U#a6U^d)e2RvTDFj zOuTP&N14q$sn3scaC}Q(1^H&3hB$x^bpE zIi9DuEc>E#rU!QFQ=>JWCH8{!VS8Z~&bBFYyH+xcPUe7n!#hcM`>rM^Xy4KM_)Kx5 zki2TG__7!Q#1rr7UbPQ++=Go-Jk{0nK9E~2!@fff7*J<%o;rK&Bj+7=<{m~Dy^a^n z%aNA}FHD{H7Nia@&I#qym~R%L9*SWY%;?=1bjHV`dqII-U#lI1sQtJ(ksR|Kg?)#h zi)))#3{+G`K%c;Mv9HIJ(>cT}_o0S-rFJqEXlZt}51yBxZ@`_t^Wt5k%6{p)%T3#2f#2i?tZ=e>*D=b76# zQqB+Or*e^dSb>Kfz3m1?)`+o+?=5siLoiIXFnQytPJ83(cBeS`0N-vg4csZT8QjzO zu*WEGcXPz1E zbw-{bS(C8Srcs-HxkqX99{i*^^NQ#xeY-}))Nk>&?@b!rm^<`o)wwCefUwx8H2e;k EAEW(DiU0rr diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-pi.input b/lib/std/compress/flate/testdata/block_writer/huffman-pi.input deleted file mode 100644 index efaed43431..0000000000 --- a/lib/std/compress/flate/testdata/block_writer/huffman-pi.input +++ /dev/null @@ -1 +0,0 @@ -3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180 \ No newline at end of file diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect deleted file mode 100644 index e4396ac6fe5e34609ccb7ea0bc359e6adb48c7f4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1696 zcmV;R24DFkmtE59U+}dp3M$E(%$UAJ`ff>rsvsiW8T+$6ZwCa`!Y=s-_luo9MajP$09#>I(F*#bYgkvGSvgH9cjqJxOtZL@E-R zxap{F9H>K0YPWsSkS2)R*aWKe{#|WqFIuv-wS}!bk75c(Z-9;7Wc4VnBBzs?752D& zc8p>URDdmkKtvR3uWp%l!&_EmTpc=NETFYQZ$(jWT@; zgN3|cc@&v*F@uVLa&KnX>Fd2bZUkkwfB)b_MW1tl319U*%S zvp^|A=dI~L9VRO0%SM^tpIF);2v& z2UTM|Eu;@^j|Ys3yuqcmNp8%xRb#N#JWo+RBgezuM69fAg{7zjhSjaxj9hCIS<|)) zTLN?jLt7gbXKG}iEUuqR-jG}(yN@N#B)wX z?|Hml6#3}s*c0K~nJep+6gLc-%e0Zx+0e0@vrzAOGcG64J5tD?3)Gal%l@md3K`X! zWHzzhS`E>KPF)C!q0$!IOpK<-WbmbF9QLE^nXFo~mu))PKI>??oiY z2eq0;6HL=Tt81EVym$AC{;?VPYEHwbEH44G@EQbW;L1XcSd)b||Ff@Ei(4Sj++jOm zBUh^KsO^kc_oqFUViJ1J^cG$3Tj{GxbaP=7I(EAlE=mRs3qthuA%e9rE-#PHFM(mQ zu6KhDd&6Mrg?qbky>)t9e~*^0hsbjfTxSkFOE@c#rEgM-#Z9ZTpaI9jc6f=dNhXc8 znW%G1wBBCANuz}>6H}+!y>*N6gKL$sTjqM=lH+`zajbQ|_!-Asw+~_~BPZz2`j$Kc zEhFt1TPE|&golz{9lnon*4~tBl|$aFu;^S(&T%XtkV=$yRZ5cBjJLTgxTv7rS!-y$2B``yh?Bd zU87(35T;+y=@n~to6Yow&?UtR3gMggy9M(CYsW0orRXZXb1;cR#nNz{C5S6uiE#A# z)e7C6h_D5sJRBg(Zy^5U!@dY0#$+}dp3M$E(%$UAJ`ff>rsvsiW8T+$6ZwCa`!Y=s-_luo9MajP$09#>I(F*#bYgkvGSvgH9cjqJxOtZL@E-R zxap{F9H>K0YPWsSkS2)R*aWKe{#|WqFIuv-wS}!bk75c(Z-9;7Wc4VnBBzs?752D& zc8p>URDdmkKtvR3uWp%l!&_EmTpc=NETFYQZ$(jWT@; zgN3|cc@&v*F@uVLa&KnX>Fd2bZUkkwfB)b_MW1tl319U*%S zvp^|A=dI~L9VRO0%SM^tpIF);2v& z2UTM|Eu;@^j|Ys3yuqcmNp8%xRb#N#JWo+RBgezuM69fAg{7zjhSjaxj9hCIS<|)) zTLN?jLt7gbXKG}iEUuqR-jG}(yN@N#B)wX z?|Hml6#3}s*c0K~nJep+6gLc-%e0Zx+0e0@vrzAOGcG64J5tD?3)Gal%l@md3K`X! zWHzzhS`E>KPF)C!q0$!IOpK<-WbmbF9QLE^nXFo~mu))PKI>??oiY z2eq0;6HL=Tt81EVym$AC{;?VPYEHwbEH44G@EQbW;L1XcSd)b||Ff@Ei(4Sj++jOm zBUh^KsO^kc_oqFUViJ1J^cG$3Tj{GxbaP=7I(EAlE=mRs3qthuA%e9rE-#PHFM(mQ zu6KhDd&6Mrg?qbky>)t9e~*^0hsbjfTxSkFOE@c#rEgM-#Z9ZTpaI9jc6f=dNhXc8 znW%G1wBBCANuz}>6H}+!y>*N6gKL$sTjqM=lH+`zajbQ|_!-Asw+~_~BPZz2`j$Kc zEhFt1TPE|&golz{9lnon*4~tBl|$aFu;^S(&T%XtkV=$yRZ5cBjJLTgxTv7rS!-y$2B``yh?Bd zU87(35T;+y=@n~to6Yow&?UtR3gMggy9M(CYsW0orRXZXb1;cR#nNz{C5S6uiE#A# z)e7C6h_D5sJRBg(Zy^5U!@dY0#$lcQ}x5eHD>U|iC=ROD8-~ViL-puE1 zjRYA__oQ{&>YEB=3*aLuz4zyXJp13Xu1};#Rhix|mTnwF zOo!rp*PZhF=TqnOy;6>9pEFaaeUqI8B!YL)2W zP7ZdtNvU6;rei#QejpQ1yJnKOE~NTM%dWXRuhSpl)r~@J@cfJn0Ny~Wi$|AEsLzhu zri&m6gnDM>m?;94<~TB71LK+=ROn-XNSxENOU6sujQmH^hn%vbF>Y9-Bf>bg4ep_N_banGD$o@)BlG0~`IFf*!A z7ZZY+$P{3oO)_oT873jzel8_va>@^q&Gy#Imx?o3b8wLzzbGT44Do}*$X0h~ljl$J4Xnb zbD&&|U+WJ#!b4}YW@ms{4#Dg|)FPD1`RJ15X*j-TWXe#-24_NUqwu$E^5|c&ujkvl zceVJ-2*h=M!1)}1Jc%#TSUTePk+ypzC+V()i{5ms{n@u^D(o_E@REe_Kn#k!Ic_d< z)NYD&D%@ZnqX*t~i*(5TV|DgDW2`fY!|?bmYqXwpi(E6b%BbX-wveIk57S|?#u}7- zL{;=f|DL5<#-Qjb!HsV;5xKrj*@u^N&pjiq)f!%|U1|gQA`KAPM`;y5?oy)&(mYZ0 z_?_gKiO6R;)m}AtC+IwYu6c3Nlk}=l5*$k#%8*z(mO5DYDWih#pN0k_;dS~5vECO-S0Dj5 diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput deleted file mode 100644 index 0c24742fde2487e3a454ec3364f15e541693c37c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1054 zcmV+(1mXJxzzaAU2mk=!nayIXbMy_f)7H$mL&SF;F?3`%k8@)&&%@Oe(UOiioadDG zS>BI}35WJ&PF@*1*&LbA=aF5pFj3x*HIFRrKcto>d1~bp8)vlgPG~al`sLh_uD4>f zwcquqQs)bz`O{dU_?0E5ZyfOj3vL$R|;Io1R(-}eKi+pE+?-hv`IeFsDFRE4SU5j~y=5(3C6?qYw^br64(dswwJMG1iwh9bz5{6%{CK{d z?OTrws1RG0;tdgAc^^}S;a;h-Le*Jl$;@?4WVbi2?}j$(yZ8P0lo@^JyA?I@?GEt7oU6m&;AhmaN!WN2o4Ue&a8T%J8g~M#1p4zh)_hxG4z2`Ogny za;mxRW4Md@6TRsPIrIrmbY`0*@-5uMh;C)*<4Qh|=G6i5GP){GL)z@9EkaXFMahfN zv?c%P&)d;?j&h!ypwqm%P^YHL3jM3}%*^0B)TTYwcr0m+>#+B{By^cDULDE6Dg;&& zO=u{r#qY9CX2q~>M2)v~oJjxXwYfA4W6UEykUq9QGg?N01rigUU44BE!qnW&8XUe) zez=s$?Lpl~RS(YSo`<)!77bHS>Gu?tEqHX60yc4tb%nr56)BI?!K^R=U-@BSOT=w5 zsVIvXfM*tHgqR0EakjC|{oW&au|@y5MGR8cGC-Yn06%^E0PC$!Cb-Z0wr}jN>)ms9 zL;@;_wIK!J>p%w{0>eRLG6F9RY`9EcFXkV~=#m(_eoQp~r+?KjZg}QSSL~iFyscF_ z+e_{^90j}~rTuecm=4dkoD6jMc=)XI@ePwzb~aU<_(Cb{iZR=;j^CkY@49GGUfJU< zh|_pVqS;)85%YqWL`@t%i6ZSgMZJLK5AGbA<2Z~&Y6y(OZ<17W7CzqwPW|%tkU??k z4*_!bQ%1vsp<0>Q04?4|yQkkrm{&#|cY?4524U<_tm@Hqt*?a07|`vlpP7J3xS#y+ zPf2l=uqk^9aS%w&GBx^|J3nR zNecGe6yILAWA?u!e(Cl<@PcA2?CSjWxPaGt_JWfJ*C8~T`^Pp$vI5uS*J~uVqo@>8 Yxt^P)DKm4sCRYuzNKydW#Fu~kAGX;UBLDyZ diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect deleted file mode 100644 index 09dc798ee37df82176b8b7c9998c88a14207c1ad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1005 zcmVlcQ}x5eHD>U|iC=ROD8-~ViL-puE1 zjRYA__oQ{&>YEB=3*aLuz4zyXJp13Xu1};#Rhix|mTnwF zOo!rp*PZhF=TqnOy;6>9pEFaaeUqI8B!YL)2W zP7ZdtNvU6;rei#QejpQ1yJnKOE~NTM%dWXRuhSpl)r~@J@cfJn0Ny~Wi$|AEsLzhu zri&m6gnDM>m?;94<~TB71LK+=ROn-XNSxENOU6sujQmH^hn%vbF>Y9-Bf>bg4ep_N_banGD$o@)BlG0~`IFf*!A z7ZZY+$P{3oO)_oT873jzel8_va>@^q&Gy#Imx?o3b8wLzzbGT44Do}*$X0h~ljl$J4Xnb zbD&&|U+WJ#!b4}YW@ms{4#Dg|)FPD1`RJ15X*j-TWXe#-24_NUqwu$E^5|c&ujkvl zceVJ-2*h=M!1)}1Jc%#TSUTePk+ypzC+V()i{5ms{n@u^D(o_E@REe_Kn#k!Ic_d< z)NYD&D%@ZnqX*t~i*(5TV|DgDW2`fY!|?bmYqXwpi(E6b%BbX-wveIk57S|?#u}7- zL{;=f|DL5<#-Qjb!HsV;5xKrj*@u^N&pjiq)f!%|U1|gQA`KAPM`;y5?oy)&(mYZ0 z_?_gKiO6R;)m}AtC+IwYu6c3Nlk}=l5*$k#%8*z(mO5DYDWih#pN0k_;dS~5vECO-S0Dj5 diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input deleted file mode 100644 index ce038ebb5bd911cd054b86c044fd26e6003225e2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1000 zcmV&K%;#;51Q|(x zM;}NPu;`xhFDh+BMJ6ccYFsSUg>Bfi<~bp&jg-~DiA=I+_Co^FF# z)zpAln0JXoILWUtGMXS8Mm=Y4*K(dtAy3BO)O!Str33Z_n`_)ElXocnv|`#I=O3$U zQA0T|pppS>bw2bp{X;JIq;=Zrn+jwL;3Fx$_veE=``@#!Pozgxncgp!ZX82QhvIzM zUrc=HkOSK=mDVB*N4QOEy(AHOADFT(|I5J=Zkm4@Lua&RXMk7^!R$cPB9zMc=#u1VIKF3O%23A!XF_hH@V9L8=wGp~=i9q?wfM^j z#C3ka`5b>di7(PvI^y_|wtFNe>8^x}-gK<}*|%vb>@sigl7#U<42rxtZZ31wZi;j& z++ZK02i|pybjbc=b@n}DtTTzj@c1ojw4QW}Tr;%FsN|WpkfHAn(_ym48kBrQRrE#w zo~2sGpy(>Wjc+s&xxP->hnI8DJtMBw8eXnlY6JNq4G`H!X%#>2QlkjcJW=%co#dE_ z$Y(j#UNv|p=sbX~d2!N{^r}%397`MJZWV9jyHT4(pZUa$D*GDWRnth5CjlnHYgKKc z`-F?ho+!fa8YJwSuDxLC6*cZcq%&Lk54QIKrUFdLkXSmFLFdZ}jN64xsEPBnj{S98 zPwn16>o}vnuyg#lRQF6UXD&FRR2aGlzw$ZN{-r_2W@fs9?`P!ZJPgXD3VE|vi;8ua W7(y>8qk`|Bh6W?yb@~Xg-WN)Vp#LfW diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect deleted file mode 100644 index 09dc798ee37df82176b8b7c9998c88a14207c1ad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1005 zcmVlcQ}x5eHD>U|iC=ROD8-~ViL-puE1 zjRYA__oQ{&>YEB=3*aLuz4zyXJp13Xu1};#Rhix|mTnwF zOo!rp*PZhF=TqnOy;6>9pEFaaeUqI8B!YL)2W zP7ZdtNvU6;rei#QejpQ1yJnKOE~NTM%dWXRuhSpl)r~@J@cfJn0Ny~Wi$|AEsLzhu zri&m6gnDM>m?;94<~TB71LK+=ROn-XNSxENOU6sujQmH^hn%vbF>Y9-Bf>bg4ep_N_banGD$o@)BlG0~`IFf*!A z7ZZY+$P{3oO)_oT873jzel8_va>@^q&Gy#Imx?o3b8wLzzbGT44Do}*$X0h~ljl$J4Xnb zbD&&|U+WJ#!b4}YW@ms{4#Dg|)FPD1`RJ15X*j-TWXe#-24_NUqwu$E^5|c&ujkvl zceVJ-2*h=M!1)}1Jc%#TSUTePk+ypzC+V()i{5ms{n@u^D(o_E@REe_Kn#k!Ic_d< z)NYD&D%@ZnqX*t~i*(5TV|DgDW2`fY!|?bmYqXwpi(E6b%BbX-wveIk57S|?#u}7- zL{;=f|DL5<#-Qjb!HsV;5xKrj*@u^N&pjiq)f!%|U1|gQA`KAPM`;y5?oy)&(mYZ0 z_?_gKiO6R;)m}AtC+IwYu6c3Nlk}=l5*$k#%8*z(mO5DYDWih#pN0k_;dS~5vECO-S0Dj5 diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput deleted file mode 100644 index 0c24742fde2487e3a454ec3364f15e541693c37c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1054 zcmV+(1mXJxzzaAU2mk=!nayIXbMy_f)7H$mL&SF;F?3`%k8@)&&%@Oe(UOiioadDG zS>BI}35WJ&PF@*1*&LbA=aF5pFj3x*HIFRrKcto>d1~bp8)vlgPG~al`sLh_uD4>f zwcquqQs)bz`O{dU_?0E5ZyfOj3vL$R|;Io1R(-}eKi+pE+?-hv`IeFsDFRE4SU5j~y=5(3C6?qYw^br64(dswwJMG1iwh9bz5{6%{CK{d z?OTrws1RG0;tdgAc^^}S;a;h-Le*Jl$;@?4WVbi2?}j$(yZ8P0lo@^JyA?I@?GEt7oU6m&;AhmaN!WN2o4Ue&a8T%J8g~M#1p4zh)_hxG4z2`Ogny za;mxRW4Md@6TRsPIrIrmbY`0*@-5uMh;C)*<4Qh|=G6i5GP){GL)z@9EkaXFMahfN zv?c%P&)d;?j&h!ypwqm%P^YHL3jM3}%*^0B)TTYwcr0m+>#+B{By^cDULDE6Dg;&& zO=u{r#qY9CX2q~>M2)v~oJjxXwYfA4W6UEykUq9QGg?N01rigUU44BE!qnW&8XUe) zez=s$?Lpl~RS(YSo`<)!77bHS>Gu?tEqHX60yc4tb%nr56)BI?!K^R=U-@BSOT=w5 zsVIvXfM*tHgqR0EakjC|{oW&au|@y5MGR8cGC-Yn06%^E0PC$!Cb-Z0wr}jN>)ms9 zL;@;_wIK!J>p%w{0>eRLG6F9RY`9EcFXkV~=#m(_eoQp~r+?KjZg}QSSL~iFyscF_ z+e_{^90j}~rTuecm=4dkoD6jMc=)XI@ePwzb~aU<_(Cb{iZR=;j^CkY@49GGUfJU< zh|_pVqS;)85%YqWL`@t%i6ZSgMZJLK5AGbA<2Z~&Y6y(OZ<17W7CzqwPW|%tkU??k z4*_!bQ%1vsp<0>Q04?4|yQkkrm{&#|cY?4524U<_tm@Hqt*?a07|`vlpP7J3xS#y+ zPf2l=uqk^9aS%w&GBx^|J3nR zNecGe6yILAWA?u!e(Cl<@PcA2?CSjWxPaGt_JWfJ*C8~T`^Pp$vI5uS*J~uVqo@>8 Yxt^P)DKm4sCRYuzNKydW#Fu~kAGX;UBLDyZ diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect deleted file mode 100644 index 2d6527934e98300d744c7558a025250f67e0f1c9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 229 zcmVoXRI~IhW&XxGJNu5- o$p8QV diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.input b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.input deleted file mode 100644 index fb5b1be619..0000000000 --- a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.input +++ /dev/null @@ -1,4 +0,0 @@ -aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -ø‹–vH -…”%€¯Âþè ë†É·ÅÞê}‹ç>ÚßÿlsÞÌçmIGH°èžò1YÞ4´[åà 0ˆ[|]o#© -¼-#¾Ùíul™ßýpfæîÙ±žnƒYÕÔ€Y˜w‰C8ɯ02š F=gn×ržN!OÆàÔ{¥ö›kÜ*“w(ý´bÚ ç«kQC9/ ’lu>ô5ýC.÷¤uÚê› diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect deleted file mode 100644 index 881e59c9ab9bb356c5f1b8f2e188818bd42dbcf0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 186 zcmV;r07d^wq#oe<(LJrqgR6ClYQy?N|9W32ycTaex&7!pwpX+&C|&*fKV2Rd8oFPOxbQ)>6c^slqt_a&vbUd`qL0Dk3ZG5`Po diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput deleted file mode 100644 index 881e59c9ab9bb356c5f1b8f2e188818bd42dbcf0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 186 zcmV;r07d^wq#oe<(LJrqgR6ClYQy?N|9W32ycTaex&7!pwpX+&C|&*fKV2Rd8oFPOxbQ)>6c^slqt_a&vbUd`qL0Dk3ZG5`Po diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect b/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect deleted file mode 100644 index 47d53c89c077d0e62aeaf818154f60921960de5c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 65540 zcmV(pK=8i+|Nj60_=}cyNDYOQC4jHO{*CA$>lcQ}x5eHD>U|iC=ROD8-~ViL-puE1 zjRYA__oQ{&>YEB=3*aLuz4zyXJp13Xu1};#Rhix|mTnwF zOo!rp*PZhF=TqnOy;6>9pEFaaeUqI8B!YL)2W zP7ZdtNvU6;rei#QejpQ1yJnKOE~NTM%dWXRuhSpl)r~@J@cfJn0Ny~Wi$|AEsLzhu zri&m6gnDM>m?;94<~TB71LK+=ROn-XNSxENOU6sujQmH^hn%vbF>Y9-Bf>bg4ep_N_banGD$o@)BlG0~`IFf*!A z7ZZY+$P{3oO)_oT873jzel8_va>@^q&Gy#Imx?o3b8wLzzbGT44Do}*$X0h~ljl$J4Xnb zbD&&|U+WJ#!b4}YW@ms{4#Dg|)FPD1`RJ15X*j-TWXe#-24_NUqwu$E^5|c&ujkvl zceVJ-2*h=M!1)}1Jc%#TSUTePk+ypzC+V()i{5ms{n@u^D(o_E@REe_Kn#k!Ic_d< z)NYD&D%@ZnqX*t~i*(5TV|DgDW2`fY!|?bmYqXwpi(E6b%BbX-wveIk57S|?#u}7- zL{;=f|DL5<#-Qjb!HsV;5xKrj*@u^N&pjiq)f!%|U1|gQA`KAPM`;y5?oy)&(mYZ0 z_?_gKiO6R;)m}AtC+IwYu6c3Nlk}=l5*$k#%8*z(mO5DYDWih#pN0k_;dS~5vECO-{Sz67BvqUSnW{Z4qSwk8%vpHvn(29}%TV=C zHoW^0>~#+!vE}ML_o3Oq>J0sCA))J2%?N)^T+vPSA29CXybmFgXG{)+RryHV(XWGV z*!JOk+FDxX8)8v|T&yQh!z%Hf9GF$fj!tc+URbObr5q!ZD2Z*V=P6(r@qLP$jo3H}n3C zBVZ@K;Q%0MI>Yab7)`=^3;o8B9x++Y8BsAZ{pg*WYdMe`U8HPGjk((k3nQoMAQXPu z3$E{&7ziBb6n~;Dpp%I853jGu-D|VlliNfn0-&leAM!2dUBAg^yr310r zxw+}MT8v@Jdq@H%5db-3bw~gyz`)9(VSmo@KOADTm)UKO?D{5=*C3(C?oP*LUFOK$ z=^hys;TqN|qn)Tn%$rg!`J92y{NJ5uHr=){&GF9y zt4*>NtPcrBh6lcR<9Bcq2Ylz1;YLk+vc0^%Mt-yh@O!qVdwFp@xL=V5P`3MlGB1>Rqq+>Y7-oT&p;d^3 zZ~ogY{%hrg`-S^_^F?a=Qd-W-TSh}pl~6?6>I>_&$_18*4jnKQ6`j<+(?9aUHBOkqNpbGxUJ;rHWy&ylQIv zPQJl>*v?}_ixmi8Q))ysZi;J-&~)agct+P~s4Xeu%ft}mN*wyA)3+6lKZD*2`h>e7 z>k$U!NJB_ap*6|~X3`?S-3T|}yd1x5!kV75kT8>YnLKYVRk>FDqAgq0vqyl*MzZU1 z?(qL4))R@QXc#I(;*fj~*i)N*p4naH8kITw<}nAGY@4C#B!qP$yPf;Q z>$H6_XGCq$^T=R{O$Q3Juf+>+qR4Emp?Vj=r$8y9( z_C3Eb^-L3Ztw1AjcuXZ%kbmaSqqA0u*rn;g&jz4L6ErD^4TrB&+6x-<=nh35GIDsY6SW67&X0(&=>rcX1~=`j5`#!HU)T6@YGh4&@}1 z>?#WZ!K%|o$nMcyMDerArf}p&x~thmP6Yo>a8o*JqmcmiU%I+5%Zu<`G_1xG!-~0+J#uPW5vn>GcqZ*0Q8f{+KY`Rf6q)>Xt`nfOBxANZSabMysrUFl zFw<{su%&7TWf4n*X-4Px99P;)Keq|12z;N~nE|1!?Qr0o!0Maz#cLk^0M!EUEq$8T4NU2R||4psE#ku+CA)8hijH@}!AGBe{R%N}KcmW|%pdiK~a07;I&TdRVupK8_s-TsVG z=`3E9WZ$DlN|4O3>2Y6rgvF{jpYjl_OG1*xYhWHW%_q$iH*+`~V&}=$@v$p5w`Bpu z$%wIy$i{8mZ^4+j&Fw^JMf1@sWbi)g`ceV#F0hrGbgE=`+*N9Fz1Rx7RX07CUQiwv zVCqC?CBszzG9&|*3ij?ktQOXLgKm>FQOhOZEiUmygxTFJ*^yy2h7s6WkE%>_2Kn&< z>qJn4woxTm6{nSivU3hw9)B(~THM^atj1Oxirb<68?{7m$CV#&u&`r?deElS|Id)Q zD2Bd+Zh#S2pHEd)C9WqV;uE-CFLM0vVg1HB6K$+axcb+C&B!MeoY&3-b~%X%)X*I0 zIDzqueUF7Y)c)^NBTXFyKJ)W*DvR387=m6f7Tw&Lpdb=A=cp$=CkHzrUXn_fudW6`sVeW(N12g{kRTMvbZ8 zElwE`YtgJSK-0)2M0_=tivB2uE%tYPf*kt|F*>Vrt@ec0F##wSaYfnc{WLJ~w4IT~ zDj~n6k->N8s{t&nK{n))cOXwX8Se8 z&;st9)N$>lX1pZh>_10>>t zB7yOYbE!fM znWPz(ERaT7ridH~+BjAN_4kAwRCJD?f4zT#gVke8)H-Dn$mgBJz+$j7CpMWa1Q^@u z6m}x0qdn>r=@Np_>4?UP8O#kr6nvn^>ioBMHS*>CJ@ zsiy!F8_Ta1NNZ(*`Mx^2A~@&iPm3+8^j?Qu5Nk!8HdpN;{)d+6 z91^{}n+G*vc*u8X#*djSf8~PMTDf<|8)bR}Q78xWM z*(voo$SVO-yC5!G<6#@S%x2?`R0=z1^r2s(SVz8xZ{1VRS~7U}I1`rkpOui=wTpo_Hy z+lsCl1W;t>$p}r8K}SX_C0b$S!{)krMFDt^(MaoK%Azf{q;YgF?7voq7JOU^2vF;a zd_$j8Mka+IlZdrS&vN0Lm9$-*kPR)&k+7A8RGaF4nY{*Jp$F)M>+dD2(Q&_}&4eK5 zKSoS}HLSjtvZr$#fRVp0^bH=@T&2iSkSuSFd5wCGta}iO25r+_7Ip{egm-+~xh+JA z079ysye}8rj(BDN5pDB#b%&Vfu%uJl$&-C>aOhvf+pFgo=vXp{kMhsEjVWjaq^Hvw z!~vt}=2WJWZ_0rzp2*{q~`5#p1yw2-@g}|ng*z#hZuq|9U5g! z2etp1EG_?%HU3V;7z{%-y>p%;01shJ?pxGQQ4`shW;l-pt!Wbq&qrl`Mfj+VzAvft z_M|lC+*HWBd?#St1@>atD$>O0sp5<+*Lux;DEzY@AoWHj5mu7ssAQzfwtP~jSjGmg z`nzLV#K7f}yu0c}`W_UkEt^3%uGCpupB*doCo6ayPwQ<;&b}2M2ROhgjChHa4|XCYBI@dnJ(3Op_Nc zFf@@bdJ0HO5LC*jZ{FPRS2SBKKlp5dG-zRht}3SRDdVsx z)%nHQs7OBfjF@Gc?j2U}*({(Dc|3!?{8>7mk}c(nC;a-KjTu#FHDq9d?OgObo+!A8 z)~N~+jbAM4QZQ*lW*P!?e;Vy>sD*?mCWQexu2LJiqqbb1@NVCI2{ zY)C*!&bP73^Q8g#*(diZy4WEWMmr2}2)3hkB=B^EX!M0q#(XYPR%hndGzN$ckHc?Te2B*227W<~uI7Qe zV`YOQK>}2@XKFDIcCRb3uvkwjOP>mw>v^eH@FKPMg#J_ftK01eCbIKNJ?-R_?LLAd z+DysSOB13^XQfaJb@MJX9s5%J^55Z;QNl08qGztF1HQWJRcWffEC4}WGkQ8{$+s`Z$oKFbb_hXQX3Y@>lv#moE1+y_ zGVz#9GfPve7rI zEuCmlxD|r??q!eh=b2#O9w(0jWfb5+B=bjERR1dby#~!fu7rH<>kr#&g;mwk=BVyzxJVf}leE9w1ZK21BJ6#w%MCBrxUIc`TW5 zW0KWKK2`X(X)v^lT?`0{c?r-9K{8GXta!g)ew`igN8zZ6tqQS?;d5s?-n5m(!9fjp z0idOi{nI$(efOO*z%c5R*ABCX7?FF9Ad9G`Yj05Zw2xCDSLAT-ND@2i^mYw%z*=VO2 zn`O_od)N8cp~xN~zjgtr%#*U;G|53`VydMZv7nq zZzs`N>q*>}qR&DliI%Rr0@VrlhtFK`;?Bd_<~zvScQL;UueU<)d1u62?DToDx-K)O z4rSj`jVW@6JU#|)oS$|kmD8ZR4vpGB635Crvl^5aHfnrwWMP1f{l%daA?I3hT~!ZZ zDK_pYT>)Jh*TPA(o&g;Gk}}=+(9a+;wx}Mg6;(Z_e_)<7P&?fa0UIf85w=;O0XVeA z8qQwO9+9FsaA{vrlW$#D*B-Q(oZGb^*GV45$LN#B7M9NB4N34&-dVPC`8&qkw8=Aq z{nCiY#(wm|&MJxD;0*RPV3f8LU`l6EKD{Kz%BtU3F;X^SUO&YK-v6n_8b~|-_+dM1I-`8`)3$pBPyxE}V_+`v@Y3$^z zOs`mDT@QG%Gcds&8LlX7rAa0kUvs=_LTGj7DuwyiJl+|-)X!&12#K&N^kU1dSqD&- zD-82a9bPuF{=W-_+7cXR(~tQD!+7-+=%aEyHM+ZU2#c)p-qVuN(J zGiru!dT(#^O4izY)Ye^El4^3fkn0up`g_OZkp+IDM8DJWXkNz&?%Cr^vY0X)*vv%D zl?jNR(>?LVhnKr`Gj|FxQCMB3PgXKx&~cLNtNWi#Tfr>NaxRNNk0NdeG42l0K~=aw z-ee5OP2$;LEqwZMA66IjL&f0WiX(Zbd&CrWN)~ulmZu zsv>50ROtCugW9h;Z_jleB^1JdLWzt2bkEI**=brU|AUpmUIxNu%~peD+%cXIzV^j8 zR8e3A;ZItvRpCd%)r&BrqHY5StQnM4$ebxZVSdoqOJmq2=ee-ing{>~uoxQH>b%8Xrf zC=Z>0w;uuCSrEf|*-p>;rJCZ3;{2Q+4%4O;7GDFj!`D_N5Z=Vu#3E8Sti1OjFSEdv za~_n@>gt``-32pDWkJ!W!FwZ6uM?N|X2hZj2VV6t$9aHhD61~_Il_0wP|&-!$uy19 ze@CK9TI^U)9gkc*$8oKUdi%2mun|t5` z5rK8qSWk^*Ljh;C;ddhy)S;9;kP73Z54b{|sw-n6mD&EBTFmY*|B>tlvx!Or%JqxH zjFmCwf2ctEj>K0UyhKspjStW7&YJ8prmJ6RGLsb6mrwJ{qs~_2Ox%u-UiHe4D7c~; zBP0(ns*VUlHsIE-xk`Yd9J=`JYZUXBdV~p9=XV?;yf+fFYQ+@d2QO32=G*lHsl#*>OZBR zxRmf*iUreZ}Cp-mEKo&L5^0ijLyVj@WG-6C>-Np!JV^eiUK5z)!&&NR~``hAGhS>(Cfmr4Xghl<6$-uTZ{D&)3zaZ zme2;#vV=M!P&WeoY)hOiux)pQ)d!`wz~v&WpNig?N8|2BDB~>q>H4bV07cDoWd#_G z(J^^|{Tu6EApAD>eurgPZYm}{ebhQO3{dy_6|BQxbCbWJlx|KDcpojVLlvTy5|SY1 z7tah8fV~iBg#Hj1XT8KgB~L(@Qhd{IiqJq@W30UK`6AyRM4R$g|2rwQF?`50@9RQ;le@}5 zn|~bvcX4*_)K;2w=XCz+#)-x@#Bqx*B9YecQ^7P&!)0z5 zO|IW4vav+jK8GI97YIqx0~tfimf;@u(Kdw<4A?BrF9oS`b}& z{-M6Bd?C;w4RGChoRv7C$1?@xCvDCs?pc-a)sSpR(We@@vqaWgox3!%mcO1P0I zq8OuY?F#ym?W5oQrae&`()P*f?WG}^-8R-P!L3gttdWN&_XX#9wL zA=9&a8#A$zQejT@+wK=S?Lk_ymt5)l6iY2?=iKIN(B$(>c@^CImaAdgI=fY`r zji10dts2f8Z!d>sbV$u0!Zr zx6LoK#JpE+|90p2FFtS=NH`TxOhvt7^TPVzE!xo6K6=2Y0*gwaVrwiq*VW0i7T!tT(cKf#5Oy zG=~!1j#Rj^ZKA@vjc9^8CBKlt*xPqPQfxy6}s!9%0 z*gngWx*rjf-&hbw6`@|f$aoO^g$+e6p@Xe!oExG|Ba}`$-b(4wjXxZ^w!%N!iillu zB^^+Xw7fFp?yzu%71|ax`fm{;A|yn35}`6^uG;v*7dUaFpE=ZK$++o^Kr>X)emCf& zl);d*UU9+bHW0*muj3x~U~+o`MCq4bNiC+Dj{t=~n6SnVN~!Q*F|^svZrBaMH0_;# z6`2yACNKXqGYPBSkcKH01W^rUK7z-XgHcBRzeoy}bVs^D-JdcbBCz=zx-PaMd@_Ip zm{i*#sau!2B(h1b`^o?%vP=Vz>QC(*+P!eb3*KBVBiDk#lTMCxoDLO5R_tMf41xeA z!2s8~jurz!v2M1TOuDEG-dTVg0<1MVp;^H+G>23^*+YE%guwE4QjxKIS=~@hET4wn z9F_FRP*8X|jMKGWWU}hK0#%2RZnxQxC7z_h9>rAcR2SxE-gz8495r`AIxD^5RrflM zkzP}Fr#UEDEnN1O=J4cM+ruG1TJhX%dc+JGyD7Zt!bG&7!r>;eS>SB}xJJ6JKN^M$ zA#jcwVPhU+fY-zZZWdBwTH^g-6Xon1@1SX+n#j=F!f1SdtVkirubcn}tuVfEz>2$> z=quL9W!sh30ccwm58QA!Al+>+p}60hEh72KbB#I)Pkrm>ITnQEq$i<#R{^X!@`U!Q z=YpbfRKqE6d3Y7oEj1JI2fKksGSX`>`%+b0Q+V5rL$KG&7KAu=VGL2=DE*(wxVAO& zEnzM|uY0K@ODo!NrN`wgjm(I-yuTKRS%#W$zzStX6oWv{#<&7Bj=(kRBx_c@W4dI( zjB$pEV5R)9nn(hyXNAZ)zQP+a`%ZjztNA2e&4rX82Cgk~ZtJ8XF^%%b7CHWi?m6!1 zLb50hcW|ul&IBKe~Mab{+)j^^|-tA*txF~zV=)v-nSxL}+d?M?#0acYz!eQUHl`R0@VY2q_b)^T6#(ioeQa%J21M1>m!Gsr?O0mpg^^Wi_9EYF%`0)r+SaR<2l2wK z3k?l=$)YEwD$zzvsXu?0TPc~f3?3kSyBlQ#58VDQ#q)A}&z;G%`|Kr`eD8-#?%QO) zIYS9=f>{!i94{EJUm2HU!SRg12zS}n)eU#o5iOwEj0hE37Hgygx3ZVojWiClrfNP# z(jx0(TVzGj)HxM<3z>$iU*^rA*iieFNXX?}&>t5NeB!5rL>djddI)|W7zkCMn;wVQ zMabTK1$yHp=U%g}!d;z!u_4MWQ}O9WzQ($Xc6zfv@uMRlYEJk9k}67I@M*l2=7Jys z46Pbk>;f>Z@5}`XCS7%ymB_ny>fS#b;5@1!r`Yn>#I2_jZvVQ|=j#o)5@f*j3wS5& z0T0Llf5aGtWUXJAk^Z7>3)v_TuIyqZ@@u8y3@a4-)`_Q||WfqQgV{eWpCD`7k z>%EDmBK)CWjIl`fU)*}^@hA;7AYaE2zhD{Sj^1vWcNVl zl1M{)9E9%q1$EXdk11@V<= zTm;^{NqXPLf$`DXX)s}6kWAAB$KP1To=3OQW$lc$w35Qhy$)>HqKpS``C7VqvB*|p z@juPms!^1Y@C?G-m2qPUf*&*fH-%hgX@g{xv`YIdCIrFCZ*VF{7>aKn9{h$Z*^&Uj zxDvl<`Iw zMjUdj#NwO0@RtY>oh|LRt%AEKo?YFNoN`%a6u={@iuYB6gdr{gJ8!t&E2!&<*^W=i zb|Nq(l;@+hTu_#Ov)8t70TZ~ahrMgqbAyqm$t^Z+lJ%gVMSD9TfbZqPK63W?Q3Tsy zl@QQRbrdm?%f-GsVz6x{;MUL!bYBp7fGeW5sb%N70_DY)Sq$7V>kbutg^f%|EMF5r z&>!)qz#WWbFkq!=l=fNnwSnjM1GoQTM)RDBr)X&<)|>K-MGgq3F<%(%F2uZ zwA4??Zc%6ReRUf8rr6K7sS;mB?r#=?U0w>J>i^kQCK!kTwEnxOi0(lHJW`_4Y4rpcDW}lz?RsJD*+WX6Q&-e&`Or2e?=~cNXWJ> zp*H3|^**%5qF%6Dmbl%a;%WAa)f$>G4#&9mk53b%73}YR=~re3t6OSejAN0OAriUpsoYcVZ*X?w9iR&8Tt_Rqa z2=q6vY0EVnPJm{05hHv4zbcXfLuFWhB4dKL28Z_09xaE2xO3d9O|6d|D?0o)vXWY- zLoTpPsN^BbyQ{K7A+lFlt3(apL+W#66*UZ=WZ$p3}ytAS*GX{_j z{O%huQx0;3$iqMEse8UgwsHjTGc*s1ij6q(7s(l8#3G|EiC6v-w`AhPdvc`zdgxr7 zWlN-%o0N1JsQZY>aq(bEkT*8d3;MVXXITNLVlVEqHE6qtX%8;Q4tMca9luppTa;S z_J*H^_V?fPZyi~L#dS3m>q9POn+Cp~X1v^3I+iT8JjDPhh5U8qf?Z6S738GWu+y^G zX)w2|oThn%Ixlf-e@s4pe^hbVrji&Cl!4kKK@9s;fP{!xhXi4V&IU~4|y=9WgWEDAcR z|L)wfKB1Vh9lj+N{A%G52G9X_1PZy}ImsG?XiG87I_ktFwIUqzC2=OVV$9EV-{V-e zjB! zGlhm==4bNxLHddK2gsS~lih~FX)y6TV<%7pe*BZuAmEFS7^Bvb{}g6p3{Em?&6NJz zPiDFDEek6kKlAX@vUp;}w7HHzlE|)<0gvq|mCM|7equWXUsEmG85C-BCdI%V`Kzal z8srgvflKE(8>J~;9CGv(QTe7O6`P~LE_(6)+4)sV*CFj+FveSR?}VZw*~0@pB{!;v zD|~EU8nz&CA3WYSs~l8O-v+qUk%U7cU#FWl^2PF1m>|*LQjK7jlg-(eAp6rE?dfyZ z8ziEinWWY@}d{fV9I0ra(p0ikov&!vzyxDwm6Rg!5r;I4F zh!l#m&3q5YmILjur1k%fr#p%}pkIy8!G>C9;1u^hAwtoRyZRM-r05NCR^Xc-D<}u` z5G`DU{*=6?yQjB2WhcOoQ97nmW~wD6@iz)}dLm^Zsaocf09h$;q}_QPQSRPZ)-K#o z9oDLVw-Z?KdrPph2hV_q)Da|I$4(DOf4yg(dQ4haA?{#{_^e+bmf;^L&(AES+KFH{ z?Ibdux!3(!5mwwh$*JRp^&H_!)CylUU=H7R=7i0ZPN@qdEqQHFQueys<)Pp`=VyDuEuz-{(JkSi z7^4t1gm5Mf*&Y|HbypT*FqD*$hIUEbKy}$`beH|{kF5hzsJ_e(>#n6QA&KBKy`cLN%u`2OBmm8E23Ochyc0BTn~%KM ziQeHABLewOoqVD*lnZ_6ON)mV4{aS*k@HI{zvF`PkmT2#!o-#!ZShzR+Afs3&Gs_W z(~m`KhD-Ijn5hg@p&Prg6X=c1|bLwBRnys^?m;MB53a0I9K!h~J8KVshy% zvTNQ;K};9l`eMNF$*>;~Q>D`x-KGiZd*#L+hl1z;l*!wsB3ndp#k{-+47>3fy)f$m zU%1Fk+(oJgAnZBMC1T|A$(EH)Qfps3$Zm^9W=)`OoH%?=#ubd!X41IP(0XiM*BHOX zNZL<=k}~=-l&L68vN>R^6iD>wyf?rfkW|=xtqoVk%f~*M&F$WY95}*?bf2hHUd0pG z-g(-Hf9{9e3+R10+t8uwWR&64yO4yYMlllx>mNk(AjYo;3E$60LR_oU6fm#qLWX;V zriJ~dV@9pUV(C~97Hs&hJH%I|#C-jQ!vX{~g^RKo(Qa0ZeXVj zJ76Wgi8GR}sj0NE8jQBE-~gc55FrwB?87>WNP$YH9lPcc30-%k&jug0$oLK;vLZ>v zi<_yNJceMKDsN#TQ3d$ntPbKIzseQ?3N7+wlIT%HMFnf>#uett(gU3T8^`9yWL0zr ztcN4HrTFI)Fb=AJfzgs5<=&cF&^csg#W{Y@R7N-Zc*h!zeP(rgBB~FJ1lJEQ4Lw8sQfZ=Fu))6JEC)NQ8WN zJCLER?6o9Nz1WfC!iKg?Z|c-i5et(eXx?XxurzJfaDgb}u{(9#P?Qhf!E|L$yYn1` zS!z-aE+dmuRuD&+AcF>i>#231-&=c|M&Ab>0oR6~B`qw71;Hrk0afQ&)CZIMcQfZp zJRQ3x9DSfsa{$kNxj+Ie{$dDjW{`uxiQyy;!{A(RXuR3sE!#aZKgx8_Sa8Nf2WrR8 zo|m5-`_PbJpdU>1c8gADk-re9HTn0*R?@v@vX#(`6wLWL^*kocYt+Km_R_8QvEK1N zo~(>MHy#TNxTs~P$4{>+d(3LRmfNH0h$BeUWw|Y;Wlq+ugA0XJCMLHCHcg$^ipMK4 zjw!4Z_ibg{@1kTFTei{^=$rl5Z@6fZpEeXnSN(}--qqIR#w_^u18b8r3b}5u8jY3A z5OCkga_t)?xKf(qBY?FO>_%C;5@%*3vo7+EcLL;|CVsk}Qh5n2;1ZOB$qUWmX!bz^ z6?2+wkRHMxz)$<#NfaR9erW*0GO#C3t$POg6199O$V+tV0CY{BRyw(^8DnJ|3Od+q zy$V&@jF+mU&dTYRh>7Y+ck;&)vwEVs&CG7`b3}PGiq7}gxYA+_;2OOz#?+Wp#SWn! z$C?`m&rCD4FZ^z%&j7`~Ofd@93*tPOUo^pA+aITLA%v-0Mn*bsWnKfcq_dL%HXZ9& zYjR7(TD;VCQhp0+8(aSqlt{hEnT%^@ywh>*S}HLBb&$)#@N}^V6ECpAs#DQZe^B~B zO&?4v=_TJl=cuqLF(0L`{JhH_fNpqQ70?=u>Gc!#V>cHfma9Cy*@6fSNwnEmlC?QZ zptE|krB!I0-8=ZW3CQl`TdU2NFGxuKN6q!51_hva|42S9a| zADr<7H#&bv^X3BD4RBXA!pEx18$xI<<5w4bmr+{umUv67xyi@O??B&k>V;QC4-XoG zlcbHQW++WWmH7%xMoa2EW*ee;GrOm9SL6E+c8}Fr~vO6=ZI?iB+N`t zZ~*L0Pm>zgXb5xw3DHh1F8W_-F*IzGeUC;#VpQXm9YWzx#4>sbJ5Xi;L9YeSYZbre zCY^>XF|u0aX31G~8%2`Idov{1!aVW=?L+2w`!#fOnWOH!>^PXzyx?R#XJe}{Hpt}F zsGSa;s&$kp+o$lb^PDVogXN17_?Jp}eo$}~t(5ULk=rP6vzh%tB@rQ4fEIE8L<9L3 z>LSo&HUXL>5Z!BKl5Za|5;XR0i0b|h`WI*kc)$0B*)mC>gU|4QTbKq6e*L09IgQC7WD_9;zi;MINrpxt>a>_@bNkWhs_g?G(&;HI z72K-meFJp6aS146qS|2-`nh+QzFtqS7El4WO}xeKpCw)a@T~}J*khe}Gr{nEeKE)D zTyMv6_XS@(5G*i(O7kw0t;jQf0WC1D0OT7$;I>ykx5UyuavFxH=saQ$G4T|UP1>G0xQ z&o9O1>Jg>Hv%2ZnT&45`Ms7KMo-S_^#Xgxpjv?2-96D7z#-=jtabQnW^70N_209lS zwOg5;4>-s6gZI8^=eGxq(tq3f_}oT|IJ8gGXE<+22_s8W5A}6Ct`ZfplF)Z-etI_c z#A~!NI>00O?CBqrNqj2|avA+_lOb;xsVbOH=&*#4NA`tIv!%5Ig|$l$CBGgFY{9gC zN8)I}YLCLM>q6d~4Z%I%U10wX7y8UzlJfJpDR&l>*q2IB4y{EcjYdBKEF-%RMncfX z+uI$UKr18XtZ7a1XS$JAj3%1~%chNC5W!NfoflQmiV6NHd&7B(ZEgTFcAN+y;;@Lf zH<%n=z{DQR73|NM#>Y+%9n$!rCI%by|9D!J~2Yknxf z5h#t-aSok2k>xqkQIUXs+1LGD5`#}u-ZhH#_K3w`uQG@hRkLzWpJ-!w_)Tjk#drVO zyHsl6PccW-R!p)%U%&YlAwaa#y4W^_SPc3e$cTppq0;}T^nGXiJmb}aQona`UkMM^ zv`tfpv=}JKb57&!dX#;9iYRPMfm3-?-Rb2;noxe%sf2 z*$$@b0B14|4TGw`K8*WxIkb=e{{`^MLLP?qW%VF{>jl><0U38R*^ELqaka3pk#<^_>@_vflqO*4!UFMTy&HzDFk zNI3-zoBKT>CILYa7)Hli;3E|9)aRym#{tg&_sj zJw@CB>kuV(Ts6|?+Qz|#Tgpn#0`gK8D~AFM(s0FABdRleV4T?InDt8o5^DnMVO=8) zPn|b!>^~N=YLR@I8Xx{7otX_XF~Z6?xb)gU9B5a=dG+|0q_6?vU=St;%&vxDIJssU zC^6xXCNq;QUO z7HsU91Z>nxo+VE(irz>^*XX_ROdz5l&kyVXgnf8;`%8<UovBz`HVYDwLvy`gv{||Xqf|b(8E9_MF*^<8YAMCi|i`o-2(!j?tw0KA@XhmQ2 zx`I}q2l`VmS&6ulOPJ67 z0zM?ydL=0gY>~qhit{g(s-T4uC=-p3qxqW%b37&z&mwiKklLGA{qCOr_@5Bob#phD zhb``W=1(jZGCl*&~l9^CGyq@ zO>-`-u@TzQ6~@$-RSzWCxmW7Fm-}0uV6U*gJ@fFI_pzO8?U{Y3K(ZYE&3{Kz7Ug__ zGn9OaL54=UtN7csQG*VpNu)~hMqR0%+F0h{Tc+`?R!P?dSL2Zj%=@P`oeX{@%n;8Q zhdW|$TT($+^5SKJ#TYsB;I^FRZO~C+!Oe%Aj=pZtk&F5qz`gzoi7y2a#|?Gs19SVS zi1vZ^BApmLYYtvZg#6A3{4NK5Jodu?Mlo7z`I~j?lk$2cYn2y=8{-1O&ZJ`r&}2pF zD>|}8DH<{aI|DrO+$9~fupHanGr``>Q$^hw)l)`*S`I;gmC>or%#lEK=d|v?4Ue)p zASTYTSOR@hqb0qVf6+x;+(`_g!qE&2KT5c>V;1uofDN5NcK+zD3D^-@Gs2uScsqJ zybYn;2>TY#z#DmP+tV-BpT3Tb5-yOos1`Q}TP8<)rf@}s zeMb$(V7=($JcRp7@}2af+5aBx``B8y26&pe5!P43L_Sv!r6~aC#L^JRgdI_~2{gBo z=GN>NSNT84-Ih&J{vuEZ9q0Ln*I@`pJ1STTn>})1g6Ac%Gbh_>41`aG@%umS4YqHW zdgS^>8)603prB8xXuFmE@1WLiej#&R$P_xR9)g zSP%`5uM-6g@Hg&F%`esu1DEfCo={9!!+9#shF(n zHobMRowYMGybc<@d6mG%NST297v??=vZH<33)xe;{unEwnzcCkQ3ZYKKw%a(C6L6^ z+q*?!C2aKzmmHcdDgD1(DVi!X6TS~z=EW#WpL+HMZxH$cuc_pHTY1}5Gz$a6infgX z2Pv{%{nR5*7FCq}F=M8nWKgtRHPFsCP_h1^a-w1%+Tcj-c)qLgq2C6X83~Makw1IO zp~=Q#Y|t!(TxUP&ts!5O!2SpmcwsUqp3;`abvh&(>g#ghq^|qK&m_jsaQ9C%!B2A7!Ss@)-s|j{jIP!L zj`ni^h_Epkyi+K=_RexFZ2pqmZ6$PNQM^C1?_PA<5HY%A2utm&pw7#K7J9&#Fu9Ts zWzLaG4U}~-1ObxhOE5$^2re{&d+XKKedq{uZ;=sqZ~)52(A2^C|jp| zIBabt#u$C1%x^-ZqIzc;OxDABdpV^BLg&?cqE<$R3XrPupne9be;@DevI&cNe7y(= zT}cvXH{Y=@qRAq;S)!&q@@as>I&zWhK~iX&e2 zyp9ri?%E-rz*Bme+KzocSel)qo@VfJZ}F=7uIQ{>5Km36$VoKGg7zOrwk^^4Pt|Df z3qG^}b>A}$rC!TNu2K96!I+uPbda2r2t2&`3xwe^5n(#Ql@`1al_{I*f4e$e<&5k- zg=3f?u7h3c+F8x?0H6YQ*}_ujdQFN4P`uYekoBT(vGti!iYHiDn}34^Vkp!z`QU?L zSRZ>tY?r6a8)xH8S9#xz=v%Wzj+v>LPQdokQUxfYTM8i5`>>{e@os8JCrPZb3m0{DnW<^s-xqf2-v|>7C{Tjmzm^;A0C&~o8rNR*XEpwfnONI%-vFx5+v zk5O)Z#W)S9HQmdo@$>Bs*8tT>cK5vhkb}JkRW1$SqI{b5x)U5~-MooH@#Pd4Z^_9) z2~qFI@?c6`P*{6v+HiL$mz`VblL{_AY)c5^c@t-s$o)C_dCRHklXP zF&b*9Cs?s@4MXsrxP+GuDd3Cr3|v z_Iif!f>2x0*IH>wfpo_T{vXC?+=8VoM( zd(JB}GkrH7)Kq`4eg)A7>6cb)Pparo+$NVR!M2DB?-3P;VFyZl&>)Ar6Vy;Tb)Rlw zO2A&(N|Fvd5%??xjS=886~{X#+zVnjrsyAmbwH?uCRhQPJ#sgE{WP}{jsFgzcF zPIFO>wz9lTXHfWjO3A0Tn{!%b?d4qH>> zzSljbK5SQm3OUBY_=ejpW1sHv;O%M6PWo#qw?$sCPU*XVY&2aDPM+NyuI*iGv_S&0 z4z)|shL)i9m4cZ}dhzRI5!EG%Q4tq`F9_W;x=*Yc&qjQ&ho-Q8N>8uh-@(XV|E&n? z$N1Fz3u-6#KfDtjmRe4u=GWv7d^R?XeMKbVgD(SC_{6AV0N)!hKV;7xHMpm1X_-uA zCTRFg+rhs*4u4AO70vB z)E=lh7@=Ic#WQ}tjV>W}1X;_u+!(ono3T~fY`8;sDj9d>x*4>*JBW|G&9!M`WV9C? zhk;f5&u?O3C1*AooBPL;J*fd5ang$Wz8^L(d=c$J;UZQKS+YEu1;sr= zZ2#T-tSPQ>HyF_L_P^LVzA`Ww+Erk4F}+jw$7ar7j&Vz;M&fb6O?FY%$*@dk$qQPy z-XX~i>v@~#OC4>F@q!(Z+5@lCF2|W&y%kBQ0K${;d(m#2+fYCf+8AM$!!OP-o(oE6IDpF+!9v*_^S`rk zS!rH-IC%}(AIbg|PL+EG1?|=6)9ECwl?C!+MP{L91qran6N-shI%$DC`hh&c{qE?w zWf2l?RjPQn7%z}PUMByQ85a0>djcA&<}CY%8TN9B3v(4g^G{m%%#^HPFb-IG85~w_ z&jTWj8%2qZ5nZvWcXL$ma*epwU8d>o;HqVh`N?2==uKi$j8t^Ak4<|c$JH?qQxm5Q zNP$g&hm77@x(T{RB7x4mk)%JU9Yydh%DZ@C)r0}zx>LUqF*a76)C1C@jc$Ooq_z{8 zcbCeeX#xv6y}8A~{ShR60dEi^yxUyrh$?_bW{296pi|7UfjY!Zodf=;O-?j+>15Ix zvDG-^#+Ca#i;p3~A%jR8mP*K3>{(`jm zf2nq`da3m5M_~We+0?k`T)7KrWpwkf6?_rp|OA~Kd zmr%d9P?XB2osSv6OTr~o75pv>)8^&R;PiCzx+z3Nc+4DMF1Lq(+bQf1q&CP6MdIRQ zyi*98uc`hIaleHc&jhY~%-s8{E{OgV28 z+_ic*IE#qKTbFum{21e)v<#WnuQ*R3FDfC83pvy5dpGXH-|sBU|>ZMPnpqS@ZxUB;bU(}Y7Pw)tCF4)E|yJuT$)+1-?ec< zY;Ocwre_y@-~5GAY6k^n^JN62h`^Wed_AN!O<}S-vB#m%!FX`dY)T? z+bnSb?G3`}VLZ+waEJk^QR>_5`CClBh5Dd z)*y_NAEuq@*6BuWKArHMMj`ESAvfC2I=DC24I1yBIwAihl$gI7B@##8#y^=FF}=UP z2yJuBZDXb*CKm{usUh4*p)mDzeqpefd3?iPzb$2$)-~U9-bR;8HbMl7RN4}H4WOI> z|F)Ta!ug}=i?+qleKtn6u=Qn?wD}doK(6MiyXiH0TiamS5j~zN4g+LVjz3cYaC_`| zlfi#Nyr0+6#KOx=*titigvi1yu^Y}Vn09}dWz1n^>~eFt5y8YcQwvOZ;4r++@Q-Uy z01d^Mx+mrB!4~0MH3y6I=Wb^4wQ_7N2o?V)&*3*Yc|_4rT7Ob1vybR6-f;f^shdBY z=o_u7w5f~`OW^-7xVnXoF&&?+cPOUXxk%*flV)wSJPp>#d(Dn40{iGNX~L6Nt;*-h zNaAPTd^rkOg1H%o*&hUMT zb4sn5wxMC+L>84HOQW3Yd2|M^PF`@~Lzkh}Yjg}P1AX6|=~~(Ofl&yY5SOd5BSR&f zG%dj)$AM7}jB>0FBb>1q3OPF;SJR(55D#*B*Nk-UyGR1BdVc)4dQZG*%j8|JY`^~B zct9pH??6}XrCLH-GI2#C9fFGfQKSOj^T>~2p-J1Zf-vc>6ZDwuaQ;1<$UI#6)tn-gYLZ0)A zm8v8WEK!5TJ_~9lcOnxuz&L(*IG@aK+#;7nJ>8fXr&i&XC8?h;P}MG3YW=}-O6oDV zYb@ctz6(E10#eO+3~Gwaj6*GzMv9KTyA0Psh=$^I5K8Mg8uCG`q(b~H$^gPmI%k63ZzXV} z3{1Eg6|E#|+Pe&b)s5x~D}mZoefFXs*M-u!d|Rr_pz`uMLJN4N;{S&%aybs9XXKO| zEB^_4?eYQ^yV{x{ycHOhIvc-B3>V&|JMWyld9k09oVUBq^R|IV z@Abfm^s3=z)qC$;MZUe)Hphmq|t4q}P6{PZ(B!PhsB>k-bomCayWZPFIPG zs&fC?K%!Ir6;bx5GLI&a2;@RbuJK!IS57Bb?+mO+ep_OCvA(39@MZZm@EPpAI6{)n zRH@RYl^G(+irYHjeIL8|QgwrFpvw&x1}(oahEiVYm}6H3L&I<4{Qd;|yATVjT2*~A zvgnp_nqh=DsBUN{b2Knsd6%PY;74C-E)AJgYEl&A+gpBLDK1~%3CZt%YG!w;iG4Rg zP;fJ&0~vaMHT-r9j1P3FsaE&zKjJ1uFI-kmFRN@kH%$_{y6p+<*)HdbWcCI2O7}D|g~ii=~Ln z{b@WlrRhdB-k$HWJu(d@l|DdBP@)3_`}8dGTj|+YW0bVq16N)~Xm*RQTuM=I&kg*y zd~gp)D%zDsuJ&65agv9IP~W?zcrl}`fXm;bHr_umTsIf`^&$W3fsYEby5G@1k>fP5 z%q)&zV`-Q>sFMtG-*juB;t`3%`-I$G1hc-6>gs!eZvP!1VVjUlBU4GrHVM5tdn0F_ zBqO5w75-9dRtWOcyCV`*dUM|v9Eul^mECjPa_9c6X#Ta&1ga#|0Ps% zJ-zgmw_Ii#LUGYis&5Tx)00hZpDGX{-v#SNeQ|1P_w?WFPjZ^tkNw(BDsUjl@A25( z5O{M&KJT14q3f2LxYb2`UFtmdR29+CYIu2zx^>*|G>eeAi9~jUmGa&xLi!5WTUFu) zRWl2V;io;?GKepx$~EgS)IG3A17baDeAF8PS=l0)p5hpg>w`APFGfmw;{!f}XVm$4 z7>g5OWqbbD@j5-Y27id5#UME5@#aKv$E_F#k-Wdy@fA8fpIEP)!m0n8(`#F8Fb7#% zc?d@Cq7+7h^-$SfM4$)JAivvP@(6gXtN+PLPZrI_`;=U(9R&Sx5>jDd!>p9 ze?!BGvCfR_2aA|`P}YXmx!@>09NnASQXuK}kRJ9vi+*PGiER<2pE!%4Om)yE$W;+Q zCa9$O2fp0}T^6969soz-!=b!D>W7y0&I$2vE(xeBrB`U=ts~Z$sS(mI#XX5dorX`A z(CaqpRPGi4$F!i&$rAA^l2(0&kWV3+Y*R)gB}$ieMvOBs!X#RKOfWsKz!CJ$bYOIr zy*gE-eCF0O>#6EP%P}H~dmVi2`zJfYMOb73JLyW=gUfSPJWyR^q=p!8{vI=U{+$jt z<#q&6A8JH>eZ8C>HkVi(4tSYacq3HW zIwTNgdvn`~GiR$?Hfjhg=~$p>jKjwk&#NbdioM_Ecj+)%4c;jEl+GcuAt;B)JCWXQ z(n}U&P$$8EYYmU*09rb@6wxMbu6BD!A<}Rrh9tk(qLFEcN}Cl5p_4{OOlQWm+cG8l z=py;*h@?&pn5NxtEK*f)l3^YSWJxp)6iQOk7jHT@)7(0zdcrk=8oBt6Z(|*WC7HOL zh&=xpQa;I{|3Zmwfx;a1f(abW?f6eBR($UbUDL9Yi1xqXW;a+-J1hb+k^2~XLZ8k| zTZWCIHh+~v(b=#Z30hef2&f=~gPmSc(V70YC;EBeF>kW;N;5yYR(#YPzf{Gh^+FDg zJG9Ene5We=H8_WbR-R*kZX{;ke?hapxd^Ve+3*h@IBeVl?!qA-gg&vOlJwwY!kv}3 zUax=*9_nEKqz#WvGD9{XX>BCVF6>DhJhHdAM%$8u_@OI(5f^&O>Q9rO{E^@&9Jnzk zBCf5~r%*H01LLuKV3pwu?Gwbr)GGc6z$OAtW=|S4bRLCPQCJV_~RUEH(?(7Pnz+w0FP^eQL;b-B!U)m zRGGBZ1~^TGY-3!*9$y4?Mtok=?z!-|h3oB#Dj(yxU}?YC?i$-bSX(Yh{C8C}QG z%N>n3Nn@116pGX*M%d|3vVAtTD!FieL)(F?B8Le$ds}w%Wm`&UVb z)F$3bji6M#iVgmn)lHn6nQ$dyZ*Z+`REf^7Chi?89-WunGZ;<^T zW6206?G!r0SC$QH+d-~TS1mY=o)p-%0sv49OmoE|=FxSs8}BIi0eJq2ma-9wy4IT= zGEMNq&g`{i>U8PAE;D4XBBZ=4W^`}281o8nCo{}l1*IzMDLoZD8>im5tMjK8C<2~L z;E1;!^92f;7If)x-C!QFSSlbK)+S6+@}P|si-A$*9gIF0Cg`FX|ZR)Tlv^vH6E1JuBY^E z1Xr44s53pj0PMfKZK)Sw$YruWb_G!Rv<&CsC=Q&y( zk+jxgOTbD~V*A4Ep!HgkJZQ88G%!16QV6&BLFXyGf(^ z<+vPW%TRF{Dg}{FadJdNNI^ zYkOe)Wa=E+xTZBLq0-oR)TkFA&0yfq7D3wp0Rs)F3JT z6eOWlh!BP=;^as;5{J_X=Mf~$f1UZ*9WpC#e!?j@DJd`hO?9dzHmKDq=;{#-6cccI zTd=yh#K?+dnT%1d{eX?DS-hyxN$Fu}L9QGHH4L900$&}&7Q>0bR!%#!Tqvr@Mkj#c zL8y0Z9W5OKWU?qrLNP*u3V%b{m=f{BzoehZLXT>JqDz?-VUVrrh_43Nk0I%6>fptj z1BeASWD(j5b6Q;`JPfCPR8{g*sE5Pa(f)ZsJB0`YMf)C_ho5K(f8pe51A3)AfsTRy zzMG#TS83`i-PhEt8E4uL$K%`RPWI3V2zXLJRzs|COD6iNsrM$J+8@Tqn!t|NcRGlt z0yNkNpIjEotsHSR+}ug{q_efV?xnseYF7vcRLJ6Q1kP@-+smom=z=!M3EKdCY%E)(IJ&PBy3|4ZV6Y3O;!v-c zElvd+_~05#Gkg*Y%pD)`_oP}L59a7n8ZE2I5!F#?%x)Ox9IjRI=%ibt0Drm)2SR4U zcyycS(;Lf3Z*oxHCL05rW&F^`SpZ0m@`j(J-D zy3P=8(7s*#RIaB97lJ&qJDaSkMWR(r5pt%C0aAAF^m=WQfSF#fBLc=vZfX<;B0k~h zPfeMKA_*!fpA>-~?!DPhx+IEqHBLFa&SgmGtxNlF-Mx-oL92=-b16V&`elaQE^*0I zuAW0-)M$1v07#|gf?6cTKlgo;=eY{jVls}>gKQ01#90E&*&HxZP$Ga)|j_M7@ z`ro?YPJ_NIYyE9IXYbl#5}jiLo(?_Xy>~-Tf}ydhjU~!_CFOkp{OeYcY70KG7sJk_ z{rOr_D84IAH}Xvva^(pF0LT3Ih+}z3DPrh68L?Isnxf$@9XjoWBF}O;qU|Lloj0$2 zMS#%Ci}Z(p;rDcSS(ZCWp-TUs*@w5iaFmLIGPDRQ+SXvqgN4JsNHfmD`~k$_-<5dn zpr@$<>|6M{58xbo##*XYf)~~_&J-iKYM?aY;FVt%3QWVLfgVF&V1j>DRsmvf0$kIj zr%)6Es)I6WMVf(YE=T3{k~nMJvk_bqfXo(g!j48QGgG5YSIb#aZ>=%0$TwX;+!*x> zQJq$iV`~rs<=?}gZ>d@zFk!j4nKM^)03JJ@HI41HhY9{kmk)ay*WYqtB z288RHK%hboPCB6_4Jt_Sb6lf_qyRP%A-SF6nK2E>YDZ0^S;0sSo`v+IC@~!{znMS4 znhhhO6b~K{$Iq1keZp2Z@!m1^z}}&j;x>x7Uv|@=4Sr+mKkQ?PUvGmu%NkWMiRC>8*QLC2rV&{e`BNqEXU-y87C$oCRMkmX4hVB2R#q57)x`n|E~q_d)xo^T3rZuw=F^M2$z zl^&k)f&xs+Ww+}t@fCO@Xeumpq^@?0wi?`nbr1S<2N`QVhP3UbNvN+OPC1_r&F zIgT2~^^^sGcro?>_NCc`KbJq&BJhhk?+o`yFVgB8ywTc=6;&?KJ@ef%yzXsxv>;U0 zd^f+hm;3>Nkqxt0kSOPY9MQC-1;TdjjxPN=D$$nFr$EfcbT%pwm)^P=>bn+?J&!fx z`Zn|hXD1-eiV4;E7W+ge|FpLjNjSi~I63yAo-^Byd)@AhKyhrD%-8}3YU|xM?`nY5 z@nOEt8EJBmDUuxzsMQ;Ib|~)0YHUglkUewF8n%x>(PJmfDp-gVHYj;4vVtQ6%`95= zYvB0!J{aS52KSoQ zf$b)5+F@lr!T%oyno$X{#xUZE=I{i<IE?HV+iBkp;iHU&#I6X?n3Nh0!%A5GMk6 z*;LckK^7fh8!#EwicaBns#!hk<$V%6UJlJbedIVCBsCdyS$iOJ(g>*bU5-8Wj~p%Z z>hLi1942y$8xs-7*Wmn+tlteA7s#0gi1GjjJNfvalfz?RS%aI*Tj4M~NqHj=aWy*L z;7u&Rc2w?2Vn!AoMAVm9@$YYdIG!8|XTK2s<|g!j3_O=;2b9^LqfCKYg&lstrfc`V8j~-+?suGn z376nNG*fj3hV*XxX@IthkeFJ1=%lVs8CgBpIT5-4bFDDUP*iHUM;iP!(r~aYP7rh5 zkMK@U-=YwP@V!!h8;Nhz$8EZ8=~4zUREXP;JM{2OS85{9Z~7mZCjVysEmhhwA$~KULw; z4(tYjGP?0@1CX&87(@jmdh6RD1ghb&FttI3`BTX)hWS&$W}r&JO(1C!&h6`OQ-_f` zqHiikGA11H!xG+g#ph8rW0Ec3waMmaKy<_Y04m0WEW%Ctqxki2a@KT~=^im|8b`h8^IE9{rruSqq({1!C>H?s zeb4WXBX!rD+e8V_1oxWFnko0XwHb;Q9le*_h?luZEAqqmt{EgJE>T#_8B?tExp|72 zsvGsdlUwlrxHrOp)5D$7lG$mf0bMk+Tn8O9*bMbCpD;pUejoQoe2S{i+iYDy$j5i``Byb}HhRNO+V#2jMeUTy53+Gvs(3eGD)@K)| zJ*BhHq+W~wYFJL5Fu>Or!pTUG^ORh#VR_;*r3z}G5$p+314AsN+QqNE{*w(3pq&=$ zG}Vynl@l{8fi(1-g5$^03V2nRofd&*M1@G?iSo@A7blbeL1JX6L}1?doUmebJ1uE_ zh-zXLdsH2S@knf$A?MFchI8jmuwOT1fJt_WZm~KyEY2C)pJpV!k&WT0hZ3q;q?CQ` zkm3uZK}@s{CaI#iTA5aS=V*ENF=4^xukf=0BP6x1X{|X62}Kj%C5hcjS1Sqp!nfzT zi#p;?lRl$mI)-=}w96Jlh@} zZ2J!(^0U2=Ud^?4HVLqgXZ0h>MhTaOt6Og;INw*%f9r_yCRbl`bUBRh_veuIa`uAq za6fzgl2Kuf9_R-e*x=axz=gAjg$0r_R*T+xG6C@2lI-1CUWVr1Bzz^BR2SVVPTi8&xzf$s?79zMf~mRV5p9gjBilM6 zktacmfZ0BE#XP52opj8ey`C2W0C@Qgz&V22F`9EaM|t&wUJa<(30>}r#ZUvl=a~jP z$?&O5(W=7!&^RvgzX)a!?6q`o#vt*^7k)IZrmTg44?(Ll{fv%y7w?~v)dC%%+!%we zxkOt^uIWoF0lFKB!@`Xw(0L z$->~S{JKdLX+kXORNa<4F+=5&?#y-Tk6P}}d4>V8p8Mr>h&V81l&DORy^*}YqbU6S ztgObQi2w${u+LMN*}F;CXdd0`Q0`P{AqG1vr|_|y?=e@bD*A0vd}DoNmSPS1yXZRN z9RzgBx0pVpT|n=9_4Ksq*0Sjw5MkfZ(i8dR?b-rxMCQj$xJ)<`*^#MPLc%Mj%Edo_ zkhjUVw*U@L5&^@ji_Z|Y_8D|lp{r~Og3k;1_Q|wMUl|bAV=Ms6&cFU&09w*k=~^KqaRKTL7=bQ< zi>JqGYdHW+Q__e$w?X|{TT3Q~o!fl1sAWLq2~nl@nWAJM=RGY7s2^x{Yv8kePg(b< zAM`dwrfC+$pFX-ldjxu^ePC$NTu);Mj+BNX5%2Mp+(i5b(?^rKXW&W z8r-UUr03Mi@xbsgW8Y3iF!}MmKCXWeRB(Rx`3}$AB7=xPWX{XN6=FJAU$tbt-@Jak zZ;goILeG_IN$d7P_Es&v{5|Nu|L4nFVf2$JIg->=+6l0K4le+nQqf`QlIeF^Z8()` zv4Z!tiWD)T<{!3PgDi0wpt~BKd~Q%Pi>X#h`R^fWMDIjqt)zF64#pX<#{E86{rbbw z55o=BPOz9%(+~QF$v5zh^fwOs+w^`tnpf1n3H#xCWox zn%FW`QzAR<*gUj;RMk>_pyR!W{ZnOndU8acLfNF)_lGB4#}?7la=F4vi6=SfEqbC4 z)IMIZurtI)blrFcx)t@yg5AGJ+ReDheu&iQN-+Hy>9zjKC@&r=uAj z%fqYu{&Lex_Fy@jX?AwlG<2uY(lS1D2-26+2_kONiJ!5Zw&E<*7TAM<#Se6y3Y3{|1pyE144wV22ETJH5r-S_)$lpJ9r0i35Q;BjYZPZu%+e7wwg3w%6oacwn zwJ5AY;l0f)O|=UQ=zilLUo!e;z2vdkAqt297dT4(j+uIwap*ZX@hl5j0_!@yW&Mci zh;J08zP@~Y$1+cjlQ)4wmobB^CK^4$$z{MO#ERST26y|3HZ<<+q|e4?~~xp6Yc7;d>3KW4MAlvmR(soD4* z*f2RD{<4Dil6VH)e32u0l`Lc$;#F&s{aAiSHq%Pq!kMHYR{-x>om`5OPnky5;| zJWrM19bQ@0O$K<-b5sdP*C)5sWdT1bD@u)caQj?o=84Ma^zi(i&1@;pKfDF)-)NX} z#Shza(1wRtOXyza7CN6%o-PSNU|2_|GNy zP7i`j+&N*#et}|a5=_3VU|B{3;FA!Fy4gFMPYx5*+W#qhxi?t?ChyT&3dwdy$1jUq z0moUbC?1WrYU3{?V%9l?^$U}AEGRNftC$wZ1$&T?AMVyi5WB&Bd9Ta9SJr9<+y=?} zWlkS`GY@=qa!TkLc$yIboC{~Y>JTm?_Ls;8 z`Vb8c;dB^E+!ZYR7RE*i=oTr{1nSPLLQNCA)Gmt`05~EF!dCYMi6~3;UUE%1oNWP| zpSYMFOut*(RcjqdY*zj#sT!*n+b{FSH#A3-B``D|wZj`m1rvW(T^=jQywr+H2CfgZ z1&q8eZGjwG*-5Z&y;a^M8|U>lzpFzcTeVvrqI(MQ{n4Rf03Afd0-@awNn8@OYiA;i zOq*!DF575ZyxM%t9hDGQI0kd7=a|s>G3E+2gXVKSOsAULa?ASw8!u^-%Y%rr6Mg-c z)h&%8*ByK6zNES`v_z6XN1;XLP2P1|4Sx*}Hw|8P&5%t6b|s5vHR~svJd>3GDzkIrM;XNNlZ?s%NWSMl^nmR8&cVP*rijtjPnitIu|n zWy21$_sk~Q=?pwkuV&vFN_LBh;WJOtk3WwSwvBpfW~)0?r;09ZZ5BEHXFV((^YCSb zy^u9UDVgF&ST!y@7ez6N2=hoVhowh7DYWZ)3ve~Do=(hL)9l$Pv^sCo0cfWthl16u zwn@K%(Tx0WbhQ`NaAl`mBofLl@NBR9=!>iJu9}HmuAO2S1%x2VFq^o5d1wxwzSFhn z-o_7<*ouo0MAhvRYYB-SygzT?bYHoaE!`wgOz$An5z8=EjbS-I*S2^ha~%q)2E%NcV9%y-jNvb?M`K{Z&X2e0X9QhiRk*i+7B{!IW;Q0 zu&3HK!4|d;$6|E@O$ytC6>i(FK3ZrocQ6})aeOD4%M(V(dC?@?y2gcd&PY8KoO#Yj zr?q1BVwwNBUM^HWd++v`6IbpL9i%Qb*(_LGL9PM*hjq_`_m}VCqRu^$-_??uQnM31 z8M}15{|u}j)>+R$bUZeoIOYsPc~=$?2t5ZxxrU^w;U{#bc?_AXQTXl?GR8WXMP8#2 zgnt*FTwa(V#=a7}vSZxGdoE_9sFYgs=TrpPsPTUbVfBXfrz47k_=f~{DbH?QQJXX3 z9$d@@;*P27@SVxTe&lllO9&AS8eRj+gD~N7DtI|!$wQCK>z+g5qct23L*Do`hEc0} z#{M#dkH@iy9W{_YK6Vel0Qp9~zx|hk0M-}>PXx~@P@ad{exVF1e``hNuu-t$HvZUGbhOqX326gMnnFd!B&Ie<{IiDkD41mW% zRqQdrXm;+MTO>V`JDy0X5TaVqFzZ0!%bHufqS=jl6B3H0Hd!0FvXWf(1M#TqIlSNp z@wTML1(GrzeoWljQ-%77PvNhIzf{qIu9FiW^N`yqikrGaR=vuQ`1Xuuil7j08 z&tw#>-q#-ydDW&3WZRuMKZzEG`2`RF>u;~LJI0s%(sS^MeTV0mV@A?ePC7VDbTKQ) zL`b&LUVwxaz2^+Y;$d#&f8<1Z6w6Q>UA~dVc~120u|0cz1o_$!+1?8;T1GPgl>X0l&mByr3%+PBYSQXi?9?e&)jY63t!E+2wqPzg zuOQy!>~)>Om1T@^OIiI5X}6zv*vQ7(e(4xOPwtM_Y~xmQLl8j zymOmX%0h1*|25&-7f}Jn{Q{ujP)~3J(+Y^g%B=}o9sa=|*T$y^@pyThw*puqxfZ(6 z$3Ydd2lRIN`=px+3+#p#z0NNf-15CI%Zt=5_$!70j<3wx5uj&YDWDwx0_8FOcsu0g zYwJwRP2o2EYv-@-m5iXY3qC)3-zhEN9|%(CBN?O%ZN%k%C?{Q-%2G}qqU809h`jq* zE+uwMhVSN^mt0WE5$OFJ!A&ANdMs@rY08^aMgf~;nEo6O6!wM4(iN>VNPJ2u2!0bA){>P0RNB| z$SUZ4gCE52sE&6O`;hB0ZN z6X7$h)Z1p`ag-OrV@GlTx9s`;;8-X>oc?sJElFH@fGG7BFt2= z>(*=r37bt*04jPhOI<_40Y|O+Ya$8k!f*w+q5_HarWpLguizKB6sGAHXXACqTM4&N-BpL?QP7hJklNBY~obzfX-b`Qw*Wp z<|6n@-N+19so`4WG0~B9G1830#TO8l4!_+=Im>yAgX&^)jNq@!#zeM*q81=xUuW0c z39;h>+xv9XcgB0!Q-WI-yc|1u#~cXx1+R=i2)l^VO1Ni;CxXuQhsSSgi&zYZ?RwtW zM7p<{h&39g@2|Ce!r&`8oBYCzi`SGcbf*b?#^<=SZ37TWA`fTTiT_@IzluOo@%Hx`t*2J1cns|n9NRyl|K}QskPiA*vSnk z(8yxPfK&NkZ7DMQm9a*j>JRbPp*LnDG+G7d$A{IYO*p>Ml5Jzp&JtWl^`s{zwU&OG z>hMKM7q}LQYa{<9(7a%Q2?G2TK|xivXSi;JtLJWl@(JUDCz^?r(G54MUW@wN`w)A6 z|LC-rknfSu-II!4G2tnEPA0NRH+?MmcJNHdUH(#t?Od`V^9trU0-7@1!!)1ACGNtz z2cs3ylgC(c(N|AL?Es48#B0^pl58%4W|f7N>N=!Ev|1}-31LKxq>Im4J{Os+FL-_# zu&XySwX+o@u)0V{nJ+alaYtrG49eV`^3sv^h?1`31Q%Whe;fm@#f$eh@ZP$?W=cBJ zGYxnCLr0&_7prGY;I)4thZ(TAve+f1Dh(C`r%1z#+9fo>|K<-Elxdhi-75bI$x)FB zqI3}=SV1yt%;i`-YRvBo%MlSt69Kt!NBz?(k6ZZ6M`m4lDgy9FNTvl1)QL>H;CLvp z$Yos6=ax)IKwuUS`(B6GA%R}5A@S&<(^4FjjXNC}+^T0eQIepPb7Lk{Du;PZJb%@x z#)hT^-or$2U-R5Sv zOwC;S4Mx#~mDEjO9Ywyf*V_)m78PVW5)5<+tzb_rLGL1IYwUFKG5_7)~Z=DtM+oDAC=P|Ilm7)C4KV z*V){;u8VQCWPWcGbri56eh2QKjsfoBuD+Wn2Ed4Ow+Lz>cEribP$b+XZkwc>%zQM9 z;|Ei|TOelQ1@Y2e)64fL%}jPchl2jbt#CwI_8r2vVhl>oZ(=b!>l6Eo1zv6_9Gx&O z4+i2yZA?Vr)JJo#Yy+@#POvMObQX^esxG@~;$|m~dB?M&fhg_aE3*Wx;S!yq9Cjb% zN-#R`NxYSm97BvU)65zVnT>6B>DyLdY?3KEi&8jH+mRYoI!E zPylyi2N1p&^nA+k>>Gk8JS7qp4c!WJy%(zOhZ1YjLf#4Y_h->IWVM1oklBp!A|?NU zE#EY=>x3m(l}LFA2PofYNp#dB5Cmo3;V8ftpUCfLY>|dqKazaTj;#H8N5LZW;EHCJ zz4L0_wACcRjV_{0V!g`O;1Gd^;+(tqfz1V|dmIs9gk9I^DpW}O^lPcwUYQhOq;n9) zSE4%~f+0+#xk6~MFc;)ih=Y24XPElcOVN9n$d?|_5AxIzI?tfpa$47-eAwgv(dQ%B ziTAq~Tco-;yTYMvgh1#^5?gSQ-6@p_h}Q7v5GQ>C!FUsCM+7AT_zKS4V9ZJVnm>YZ z;R2QSUCHaqqCp(s1hZ`^gqNS$aqMa^-AlHS~%zd~So?U30Wm~r#Y7}kP2i>RmPmQF0^To^$| zP(QPuTRcc2Tpm?{BtTA7%oD>eaYB!Dkn?Ua^Ov$%+adYiP-_!I-?T%$h)j?0G~8H< zU|Vt$YrJN2b^y?ZwEj=0ZG@H4C#tz*VGo9n1jhrbHMdGluSr@}Ji#_(%Y}^yt7j&y zFIkT(z@jARk**ESA<`fmmk*2 zNMJa~$vAcLgwUWzK`S3?8I)$tG>f>Ea@PO)5Ha4)J}j=Hp<8$ib|H+4Ib@kWO()9p z^%fzI5eF!0am8-jZn3s3L%FY3w4s={Cdk{vhO=dIqCVA7!Z3oLvN!c#=`)q{vav<1 zAo@8#CstcX&*JQj$z z=eFxxnQ>PE9w|G7>B4sS%L?9s+9%~+2uh;9nibH-&6!TDSvN%40=BI=}>rLnXezH30d2ogL&qBR4tphSviC zrbC@IikDfRPrBEMc@|<6!e8wT9C^rH6X=urQFcBj>OFEC;p&S^s-oUvKVZ#wTr6x4 zbSpKm?Wn70Ar#_MAY4~G9;7Kk(~7{adjq5Tl2xqH;t_OZwJ=7@$NGLM9M`Gw%(!i? z*qng13C>H-Pv8uf(Ue22QGz7kp7ORy0+CJu!!MS~LxU2Wl|UwQA0l?>{gNT-S{uE> zuWY?aXluIR%m$Va2G@s<{OCc{S)WwEOF)eK=JOI|;4XPhSnSCs$o9yR$VM9k^<$R8 z`Rnaa0igzeW9_B%!RTp!FiA?Dr>0GD^u(KrifnA|FpgGYarg3uCCF6&Ci&s_#ZK3K zJptkuiF!Up#g#9Z`SckI&Wq=+p3X6E6gHj~@83aMPd8p?s!HL9K$4eGXMuC@8+lbY zD4!JeeEoqk7%RN;0h3-bCsh|Z;KXm%c4+Vk*FU2=8$Jrws| zdcvCVa9(|hei#Kvt07*ugVJF9St@7b5qnXdxBL{%j%V1oCyPNd#y`n|P~p6`EArb; zdiFn^=4#3wds1V88J|Sg%{bq^Oq%U!BVA4|{{X|;t!G>7?c_VT0R_KxU~wqtQHe4- zfC)dhb!(5)jNOuyUB$=mohSSS9sRK%z;LEo2f>WJoEV>HI>bQVss#!ye2|9UXV>nw zo%+0I1UGUW0>-6SC_umMXzi1Ra7*7S=^L_h5z|fXmAkc@b1vPA!lui{jB-H!RTGTs zO^6<9c#9dsD2eMwhfNwS0H~Lkeog4N&II%hCaFXY352A-KbZ%&ES%)ZY6~KyX-fq0 zgMBF{p0_-8Q;ln(@co&Fus7{1M+i?JtFH}vA~c*2w#ZhQv3X3<=pAZRnSY~&HO-E# zs3Fhhw(?vxY(zTxcbg-nwUGAeOWGU=PVO&-Swtb8tRg+vH{g@jJwB&5eaN(EibOBF z2(6N8F52@dzd>NjS6aMkSv*nv{z#CTJ!|~B?!Q~7OFkPu!ma% z1T%%TqL6+dGPvrDctJ&lH&;Dua8zpSFA;e;SxflOnkB5N4S%U*g6yMt_0$7wk;%&x zP~rdGkQJ(Lpb%p!oX_93AgKxL1hoDxQl?bEaEEeuIJZj3{1c4Gf!*u&B{SZ+h)k4J zeNjWrvZX8dBd1k$DVBgIHOUcd?v`w>u(o=Oz35vG=3OeQFXN9&7)!i?WHkYMfehX3 z{3BN!7>eWNeGdj3IXnmv3x;&+W8}yzB(B~sA9rJ4a(YI=EU`JU zY@fLrcV+OG*>}5TrZ*!pK+sx&naK`=z!ewnZh1jJzJ2?LYUB5DyF^pFi|>!>t@T>a zv`ANGY`h!^pd%It!N)0wstFt@$UdzmY^Ifmn^-1pA^Y5>w%uW17Fb;trl);EQfNs4 z&?H2%+zNCueB$*qVfW4bZxDT$(eCgmYk#i`uQhlAa@BH(2*b~cc#HNDF^xQ(gJ4uV z*BSEK9iG7uWZnv)>gHbExN~~A^gMp(7SZlTe&Is57dTNXF&{jTRz=+&r|IBZjH!&f z`43p+BC@QXRYz{HjV{5^(4qo&V^JEZ5S!8ZFX5A{Op;-lBOKK9B^U~;7H5oX#Y;FQ<$&~Dz1$5lf|)n*oos&iM15k| zy>Zg=r!IPi_|a*%s;%*KS=*E2Vmo2A1@ys{Scs`iO;htu60=E3-+OKL+VWCRz&tS8w6Ma$5f zYX`f{%$qm3~i|Rhq5C<|2bk3vPfM)TUre`7EvlwCC-Qe%>#?4FA9utaWO> zeZq4qpEGwsl|9{H$xlzr5R`*&e`WOP3&?j50FU-sE#TmS110Z19I1P0tS3IU8pQZp zD#I?IPgbr*(}U7Kz!ZHi#FTq}SY-Rr0^@u+s`H#CVs(&qMjR;k0tf*p)7(xr2XJp(g3UiCgrl#Zf83%DuDx!Mus#9|JGS{}8@Y<%${y^y^VDoME;Vyrrom7*lak2Nt z3o0v=Iv1mqB4XD}(>rl9%WZd&fRo-*-8tmCdg4){I zqnhz4*{KEtd(Rm;8IM+jSK^f#{P=N*L+7p|Cb;lBZ`f}+s#~o?eM=Mp7$J3#cPRNG znqBXFh#0ysIZw%bywltwbdMIf%`I3$l$}k8=@y^z94VT_d@8t*84*lsF+4M}sXp1JT{?{`0)CK=kA&(%|Wc?5sB-#5SP zP`b-@1n7Tpb(*4+W53;&RO2gekjw)Yke*kh6)p0A8z4SmCL?)Bu{ zGtq+e#KUpNm;~E= zlT=NXbPP&u)6xtXhNpkn*a(!CH3i{Zxq@*2NIWOMAg=I)q(dv#W7e1euQgHtELMc! zQ$D)WZ$q_eP>QMVkxkr4Lhz-!yWM4H=yzjlCc9DEewcZfTttO2SmyM;*bb6x)%^U| z+5cY^-KdwNU9j!&WindEO}OkAWxx%c=J3J&`Q zx8_NwXJP(w;bBh}DVQ+9pT+6rD$;xGB9#`K)j|eA{-|0(xXb$wHfhytHXVnRWKWA; zku>Jhf)}0H2R8J%iULU=#Xh&|j3dY$#vaRQP~7yQQN6+B4qt$1g)RdYoR4o75u6BL3~=HbSV)HFQ_$J5m7-r!uU8Nzvk zKN)4fMopEO-dr!o$Hole-C%NmyzT~uD+>;E79t(mr2SQSD0oOj-Lv;@p-9eRD&8UzDVG_?UqacN^zmYGrML)m!$>G`%JqW;ay%@ zjl;dkBGD1Bzb@xJmd(7#jo^1n{C2IB1fOLCvR^dymIcc_0gZ!W%}@sCEHPszAM0q{ zr=$qoat^_;tqQJjDk`ESCk(~;724$-3z!7)G@vBZ8FNw89jFHW08MC=?gcYOY5(=h zEiYR?NP;pPc`dm{)dChjy8)7~{J)HVA3WLmgv4tBlI5WB>wYacNVx1eoel(Devxf} z$rbr)2^%t3H?9Ff#YSmYqet$C1vp7y5aw*dKq`9ASWuj9U3;~&O^T=f1>IkWx?8ti zF!G{b$7tT@=4E#@ay`t(J@!k$j~l)8?|C1Q5TRU|cZWJv!NEmx7?V3B{9^Yp^1wp^0=;TA}Fo+v@^*3(a3Zu4RxF?qyS<)4BSYAG|)A zD0#;R7tb++=Dm5gX(~1O^r4;uS&b(DD?ZCS^19YXp?c2}K;oJqGlkA-FnPP0SDy%T zT5HmY>EL&i**qR-pBnAyWsCBy)q^NY6m&l`t7+cxUSgb!QH*NiN}VHtvKVA6{S@7L z-+lh4i%VFe4(3Hco3DDgevsqPaIx13xzuqrick=;1zOHdPMvD2rWuFIKW51dGFHw& zKBXKWvQN=*Ne6<+C#=V+#z5l?P*AnvxfOtfBR+--vIdmi;BQ)dbHW{by*up$pr{Qz z7CnyvVkBuBs0SdtKOpS0FA65~iEvkNk9se1!$ty`A&iXNxuY6e(n|Wm%2%=!&v`92 z$qdz8Amdbb>CclCG6r)I=&8R>XR{A#vi#T;*+R5Qnzf^xdsFJ0a%92kv$9O`B4oo` z2^fEd7jGm=_uxeF|#&+9X(nrzhbzYg$)=dou z{#N&vDvnTkpP)a7Z29HKj}odOOarogW3iQb2=yI~t6B0=!jAuTA)46O! z>6VvaP3S>=sg;&2HqqGlND^@vjTvWk%&Oe!k+KkswfBEmnSzgO*9LWN8SgG+823nE zm1wZ-QGz!#jIyi8JJ;v9U80?3p7{nkw99U-GI0Q+)L%FTtQ3y1)sFO)RjevAr@=`; z%p_emCSp;fO#)d~J&h#VtN6*Ker#j&Bk=qK#stBmW&nS&WXImSKtY|MdF*;%nLBkl zI*LWh7E=g}Ickdzh?{|6VuCey+)tIJy9oT>^1^aaNr_W4bbev$sYxOhS-l@EM{%P0 zB(C@MLt=3XM73~}n5k!;pV|we(xbg)vRMy)l>DEnKt3GSidiDfyA8f_`ag{A8zm5B zYqh7hUAnYtw}u^P<17uXg5AM+vPIv!^T1L3$JqL_8&i0 zf-GzPM4#p|%P)Mv*Tw>T-Jl9v>MZRWYz+sC%hx%^x%Sw01vkaK%7*LW#$g>g*6mW; z$1XrtdF|5LT2O(Rmny8>>sq~)sJZ;ElV`Y`_yaj0)P=x9{oD3-d5o^T1+-S<+iOY1 zXwjj>XGn1|7uO$C`K1R40(wYa9SABnIFhJi=b=Iv~-MVfvVZtoBg|_#D8TDe6KzqP9av8D)Av>j#(Zr?HM~HCLHj8@+l_V~|te*Nx+u z+&CRRE2=-A(Ckx{Gk#6-pG!9Ps}~e#n({G1XtE2Pp@cG+gr=VQnxi7-DG$=)zVCf} z_3c407=s?cvQ_X~=X0v}Z%Xb)u5SX(IQL+ECYsmhSq%`y#s#lhC6Z{Sv`hFp5rLFB zZ9LyE%tCP7*sGyk&_7#&Mk>~M834J_BO+|O&Kh%Eu#mVL1WS*?IcgmrHRr$wZm?;U zL=AdaB;=_kmJMi-ABB8{0PYgzJ*eTVJ?D6UWq|bqXX_4MpuI)5kI z!^jR;*a4ha2)9HB@ZDQ3G%+cFXEtJjS|ILNX>{pokUtZ1r;*(Az(~bQU1h|KANr&8 z7=YHV{JqvTZsw337J@~_+{sud z7$fp_8Rc(=0myV$xOQRLld=rlnvG>4`?hS~&(4yv*2G)@L=(!2`3~jhTHt{ihwNuf zscGb6zNd#_k2(S=&f@B>@rsHm4^0$%&E0pi3#f5IQu_pg3iMm!&AOs^jNzr{!_lo( z1H|P=ryJxiJCX8Qk47}zq2k;C*1m7-CZ)h#&y~{b2X5!HujM!IJ&;NVb}-c@4Ff{k z%&WHcC(iM*fRA7R@wFf-iiNV&mg%Y{pcX%U+yBV3<*o6w24BW;nE15(Q?jhVi1>4M z0*L|2sNa-SiGe(@C1I{fq=YlV63529pd$MzL#>&{)JL*SZZsd{RZrANVw{az!$~>| z0IfVxa3T-eS!9T}K5rP|ypcN)5uNAoW<|~m61i$HGNEaL8Ky!dT;V8O-e7{I+-h~t z+J`o+Kv*jye>_JLbGo1U6xRZ@=NZy2W2uM4wVikLP*O zeBCb_V*mS4_ydYpJ$}grp^u@#%mj4MxQsTuDJzHu3?U5@^#9Bc>lRd*^Glfp<}0>e zk-k5_&B05$znb}J1ka%yAuhAYX<#a1nxeh;OgDd*lwBw4WT0R#?z<-+HA3QV4v`cU zaCR@C0ol0}yq|+yc`RUqlPPsVF;K_B-)ff#9M2s~=4p;~YwR=dD(7aB6zCvQ5r}5j z6z*|j$T~hrf0JU=1fHsHY^Fh9nE^U1e(;=rbKE=n>wRk$eUP3s+Py(2ZJLAb-lP+^Q z#mW!^u6d^M=;-HSqV|6Kr`lG#F@Qi1y>x*Uw}gBmX&#-LN10MRZh99&n0+! zc{LGVNysBq9@{0orei~*@}-#~M@y`n{d>96cd85?r2NnBnu~^=pJ*oHCd<6Kw@bk^ zm?4l5Wv8)1{)fQBo-*XgLmuYeZK^JFu|+zwZPOdg=A4c~Cin9vg!3#5xOlD~CvoU) zUkL%ljtBv#MiX?5Q1-3Bg;qY&MsMZoE34$JpZkl&mi~B6`%&_QQ$d4GUND@9t9?b%G^340{+tph4qaX{7`M z{XZF9wT5P`S{o4%g?Ww^kDsBr#HZ2&-mdVot*FO{M#!tt4SWXah`)QL%X?A1d8nFg zyLJj-^3#3>@jYJ{DA;=w=`>Q>TBNU5Ax3UMbq=v{DXc|=4&&alLG}RCab@^$(V`%$ zQ%g9x?=?)zQT4jSvn|mGQd8v!BrFBs42w7ZH6)$)->4;Sw?y6Yux%Y2P(1m`)fUSH zyDmX-wVbOVS(yM8YIxaApMYrj@V3fa@no$3AS3u`zaPox^tjWm@Dp%k2O&NceOY0= z7t#N9CrNx@d1FH_I21UtjxRm`^bUS%VKX4HxDBY!iY{x6;($Efv7G2l1cXd zlqbubgc`*)1%mj)G51qt))xoQ81-(M5ZI|_E>HvMhs>2&JsK==!fUbqdflP;E+WYv z#K(g8Rnuh2=cwJ7578i;)AADX93dc3|Eu&53p3ZCcZe3iRMlNV=R5ebJD##hdCk?C z^o6c+x7nZ^16;xc4ty7aga?@NGWH~I6psCYk70vk#7izxEPiVN{xVs7F+SLw898IU z^Z?nUqn^+Es3_p=<Im)xBpW^0b?#!Ur6z6jg9%Vc0rv)eB; z9><2Lt{$CjTFEw~_w3jcFu9F3l@P|-3t4|`BB*5d2vn>blvd<%{(N(M#wC0RW@b*!-ULzc3=C!>zW#^}3fCw|Grr;E*4(Jc zsh5(QdHfq2^I?6Tx#wIi^T)92&xZAl!ynAAj!Y@M*qOUD^e7t>?}P*6-QQh6EfpH- zbp4l4c&6{ka@k&^l9SsjNJc*^?Pz#Pm)t-qS_`y<_n-Z;?aoNJ(#||$LR-P#F-leq zBo^b@uBa=o0CwQ)$Q3BW%9L>REx(Y!XHPlZ^CFs3|{_PD%(JnXi=7z8~K#E~=w<^OqtBxn_m z(ENM?2w2SO_W^C1KhR5!py%$iNNpWbln|<%WDQ7%#iVN_EZ@@kn1~8-j}WG4HNxBg zuj8)}x0_7uCSNw$Cb~$;voR;j$%GFHlpzn!DuXGO-NAfr^-jK+Ef#P&n=amu?r$Zj zZ!UZE|4{GVerd=$7zg;IB`5t9p!USxQw-o09T4?{mB~;_yQuW1C-!$>b7oprpIf5> z4;UOKs%<=i8g|iUUJEbL&k=_z>u$KUg+uOThS7$^MGZ^=B!~GOh0DN$i?0c=lRk_d z{9c0($eookrSZT7(W+0xS8x;v(uxd;+kYY)u>e_HzDy``a~Aq30e?uKqu#VvAQqfsCk5?0wD^F|ZB`K|7NlF;arSed*0^#6!0J=4 zXd&Yy)%oDF8;(ffDGt}M7r&svh8|GvIu=L#O3$kKsA2v=`AOZGxZHzeXFggDf!Tq% z#i9rH=VQB9=1f!cd*i%lre;>`ZPh~5Z7yV!j}CmW3lc#$kuh1XxM+p##S%HuiHC&O zp!jOgjM$FRb9n1Kgu%J>dsKf}YT!V{#}T1E#g@N7BnMVLNzn1ZmxS4zr(%;CKJyh! zeg>4g%7(+Af%XP13`wS~z;HeWmgN46Yvb+}4`r9o%~tYvTSU;d3H3t0W1Hk;RAqB4 z(O7ikGkemQ#C*Vr#z8?-&btrAm5lu%99te(aM5V=(5I&en-B@_)SOQ05cRTmB0_ec zJ=qz95cWfXG39_C_)ih#jQSs_cs^hyx)@;X$qr){GnWc8LN34s?J7&rV}9oipI!uQ za*X9AtN(TtRvH)j(&j5)_(RK%K*_=v*2}or?YD)&7mnzIcRb)Ji)L5~ z)Mg}v$Ot{Cy*rf&bW5Fx8~;sC6W#V^S}1tn)N~WD*13)rsqhi z;n%PP;cF2r^7#j8O+j2g8%BB}SR|vPTFeT4^|QGO(&n zpxT~rYloS15HRR5t;Wf3yFYHDm@Z8lRCV=yLx{+r?#~BgI~5YFkH^m%BG&@tb9g}h zUKG%H!>VHL>*njF>yTA_F7=H8)p~iA$v!8id}xS;FCT*A5_vW@i}0k2Ye@NVC8dMo z|3(Mw@LNPLwIACyjBEPgpRpONH`j3mPc@S;+tld2yP7du_sXrPIpGMU!XKO;2n8~y zrF%VneRLR;i=MaZaK=n+{eC#BtOaTpfO6&$$?q}Bc29ck>xyB|-K(!TM?kWT2yMtT z*(&xG{6U)7bQ&L!=9Hq$O6ImreMR2>JmQ6nSs$=%S6M&mcs{uwgXLrWmGv2_>EBp@ zjnt3A!L&Mf?7#x46(?c7a`Ez|k(9hi0tX#5j*(sZo>^}V;clsP&x&Z}@2~LMNYB=_Qjd-u<{8$!?Dj(V zsU=rADR1k61d_ztr6wqp7h**!b&p;*Z?(Xk2J9(2f)Ob|^y&ZrfuX&ra^!AlwzbP{ z%)TwxN~ zskkuek^w-A<)e+8eUHFY$QT=RC6$^}J|tZiVk@ll=XaGU(}8##2d><#Kt@{qdx)bBVEQIRQbGnC%JcfMc7B1kmZn2TQGhG@Y>DzmpP_ zGYIC7BAsJ&CIQ0loYL~`IS|SlR9~#T*q@2!{ypdAN)4fT$iNMwvepjLi`mEaD|~k7N%2mXbt6 zS!!Bsj5$~J1VePeLk|M`_`Pda^9n7(h%JJoh3*4%Kua1K9YxFmLKeo9zpr@YhiMx( zwcxrKRoO|ZYX~pPsL_V0Z~$~3iH88|Y~-(`IGCSvI5S0|=w)q#C>g>!ANFR6o-LBL z0|Rs9q1R*nfxw|G#fgg@V_e;Rx>=z!R>jJy%wegZ#tIT4=-M{;TRKqFV)(SD-5|^k zZ1u?wI9_nl1J84pj@p{Cm=SK=H2c1eGRczHwTijQU#dVvrB${gq<3ZCq3<9`%U=c8 z0V&NldB*YFgXz_X{(6MqvHfe)KE)G$#QkT92a~j$HE{j+<=z2udPJ>ZnW!3po zNge|oyOo;HcCg1dG%mz(FpTSqOZWA(>)$+-tNp9P{>9a>a&#`fU;BYY)k$!+CC`8b zWsILrQ0r+xgt<#^=NVLZWyP(mRsJJSz49FSeQ))efI-mI7Nq>~WG*gJL55>!GNh(_ z;J+7+kRBS%Yq^Re^v`$=mmFoBOVy(^jEn-#U4|>{no@)6(MgVf3g}h~)=KHmxA@qp zg1dtFb`PC!dt8$14#%v|7s zlBRh?tU1%X%KJ_Yt5fO(ja_dgVRKl&^B^lLt2$0g##UXP7x&x`m>CnH7c%6Z%1^;j z8{Eb!{1Yrgsg92xdOr#uaKcAkf=RO;n`r~njNAu{LjCi-G7~aR-~t6zY|Jq0@pswTA z52J8^ARl~cVX9@9%pJf*%5UcDdQ#V7;t_E9u{=YyPk%MyR z3@(FjcL3>F5Ya%InVX7mo7`iZKa~Xr;~-=SmSA~K55mC{Ofh0`z0#RAdg&1nWY@{v zeME$ycqPo>{O`Ls>@Awj8aesyx@?oRq_0Ns%=#xM6Yb=uFi?2((oSL^)IFAwn~fQi z7AnYrC*V00S&X~ria&*sj-S)qPs&(iSY=;Ach^ia20IK6#U#9*)cEhDPRp^`pxov6 zG9U$cMr9-KatXENFW@~)_@mWKb)R$-uPyvj&}<&|!6%f8iE1OwKEEhY9%iT+ii|KS zI4~d4Y;AH&MBG`?N)b(n!V8!#dte35^$-B@!*P{y8ba?%#N{?dHQ{)X$5Ziqn}B;A zV?)Wvu)0%3$MW?MfB~dz2ElrJm7eTqR5)=ji6(E`q%$#$P5K7DJXHZ8)am5nJ9hkW z**GRNPSMCv1}``CZ2Y{dqzdG*KRvr)rwMrM8ujUyCrBB=(!9qFqJ+R=e}NJ|gARv*zT9JjK%R zq2Du<&vf6HGeb69g)WcW;Qke%tzFmmpQU=JnXvI z;vkNhTyY17ry5(8DO#*2@ID#IgpsyzX(NFQePzG(pFeMeX=#OQ0%n67F~-Y)RbE^w z0w$n0gf)#&-374ey@o=>FNVc0*i#f@b|n)78>nhLZ>r(YiC>M}$9iTNzRFQLYzhV_ zUBE`>bsvOg)ykwrc-AWDvK$C60k7Gzuk&WmUpWiG-BVqp_uC%nI>LO*PJI?c6yapK zMj2~$*)%|5jVAC?uuzT#J>zUrsCre`>o^H5^@qpmC@04DIDQYXn(Ampz{?m@ep{9f z;v;4d*^5-9e{-5a9`mPH9Sqz%&mr662O{3QG)5WUii;jivUS_&;3yAb;4_H;(*$Kp zvAYv!J=PWf2@uI6oWNs1Km(|jPn)yXAl4;~c|qVgRS-TcchvaR%5P-{ctlV_S~of8?C7ujKPFb$^mT4|itE3`;?=;bMS9LI8j znjSp%V8^U}SzFF~qCXHk6HPvq_#u~a^c!PhmuastF`uL#-``T91KZPFqo$_?+hi!f znW^sc=B!t4v{Ljma9Z+nONm2&+_?PvD4QOKvL1;ktSAfUbT%x}a(gtpY`A89LddlG zy#Pdmh*-+!5H`KEE>BX0exaPo42jnD`!T+b1T;?P=W^8gJx_O8HQPt}PFm@dvarWu zsLIwiA+V!)gX#&A4~Qx7dH$77?yOh5H7$XuA{A{T81&D~iGz<(cIl<>zLu6nzvCmK z+$))X_mu@uo}Mb5Wg^w5n^31#-*>0;e8~B)@K&CHQB}L+bFym** zaoqokwo$d6ZL@;E4f-14-$Yxf6d5U}OIZ_=_y*!%ZE#C$3TCAymD#EPt+laWB&79^ zS=+HeWY6d87E^>3pqZ?Iy$(V3QWA6+7X^=F#xj=~A>hD_?{JptqEk zoosvA8lr~N7Y=t)Zc4g{9Q?0A*p%_i$kFeCKyG%RHAX}0432-$T%>xYxn1G^N%Fl{ zdH68-!+o1e{b$gw_s9$K8rIky!Zo^^eNQayLl5O<0L~Mp(O6iY#3cs@_VA7n)uMCS z6yZU9WN07AWq#>R)V7emqP^X#@F*T~Ob?_aQIE}aTZKiiWQH<$8bheZ7ciCRV0`pW zo6Z8lNMCX3dDx8@?-_T*S%}Adb^?qZRnzDXuC}p-3&k^l-<+@qYuRPbp$#JLVHhW6 zby~}=2c|inz-6K&+r2A-mbLY7*?~2AjC8WO1s}E&E)c5+-tx$cn{no&{1Ie3uX(29 zidiYtb11P~K8Z-GSymaC-RkOs9CB0zwtfp#%a7oXEM(~Tsl=+b(Kx^>lymkp>CUGWY*?x!{z-%`EUk+Nl(BA4b`KbPx7Ogq zqk;S%>&D~qb~Legj_wN&aWw2L>s46_YN^4{r5|ndecxZnc?r#E-4C$&YxIT7_4pM~ zVg||~SR6Vc{0)BQWj47pqoeH-t` ztES|uKCSDGF8m0*e@V4T^3~o5 zq1RMm6yAIm5qt1j&a>FBd|3F699| z>xW6Vt55>4+z$>n752?)8WClo#fplv>{pBLds%2;Hhw0i&@pY zBIvSH+_exP3^TfOPRf6`suWRC>|C~HCbV7aM(d8T>5{n}L4}iN0t-NHqVqiz66&8q zSbrl24kvKJ7m7|%jJGnHoChcIyDZ?tt^VTGF==CEoEGvzFcO4bBn(nR9Iyu1Cy&N8fQG`DN7yZybJxja$VHFre-Bt^9?`)jQv5s!?&CU)~@ zbKxGW9v2Qcma4Q|4*IXD;=ht@e1V3_s;N;(_D_Gf@%qQdo4J8NEHq<@0cN{_MSs5c zE>_m1HLp-S%~z`21ctU?#*%m-G%D;ZLfbI??`ZhgWy3QjDW#t9c~r`|`Jo6%`w0NyJM`0h<}0?SptSKDz!6Vg zWT~H-mB{AGa;we0Xz>WyA91f%CJQ}NMBY3T$d7)!$t64fd#NKGS3;`_NUaog@RK@D zrVa?r$*>=^fSE!)I?4wotuKMKnOmTRhD8nkFUJ=Wy1>Of35XOlQ^Z9UF@~RkLAU28 zK(I;~$}Nnq&Vl&fu0hfY*%+FdMRCY}Wfss=;qy_kolRXH<_Y=Jr&^v`^Sj&-|@Wh5M02SJGJMm_ei6Uf)4#rOI`>Yeol3SIo)rTHxw&Xw3e!aZ4Osi4o<8 z1Dm(wy+I@W;Rku+nC=FUIDFF_n#qG`QgMwR77k(3N_1&dabubT?il~}WkhtqOYLxQ zyWI-|0N|*^uIbTrp+u<lS|7pXT#}#(?^k>>Fu z7e)||4K^$vyk`6)ds9LL+QnjE^1!-G1S|B{WX|;E?u($F)L~EMvOP|ZRUA6?kDcgM zi=K_k{0()@S!J);atYDz5zS=C#icFYgfId5dDGOcCzcFP`af^SrhpQy1hTko-5oCMF1B#c@yRZ@RoS{ zDuo|iM%QRz9+e1Z_=RG?sH?-yuY<~?lz%h+X6%+3p_4#bl`M_xSdD8XDU}8Wn$kdw9-Dn>ne&*|+8ooWl=6uqYyp~g4b0(SpZ!%c z<($xudmkCEcxKMCrdgQ8ZC94^qNXr@tFcb)mgn{N9qfolUV3YJASxT4!9ESh3bMvV zN+@D-MI`S4mc^Bub`Un|`jY%+Q!io!lJ3iiTLMw$)b__=kE;;sNOlbdzmHKa9fL&u zqR*s;n`u5R*eLsvi8nSXlW?Ct;Bi?fEtn_iO`&uN^)UXsA{?syv@xEMS!K!(P#FOc zg*Wj^OskHd*~ghdwg*XuA{zT)kp>_{@|0A^iqeN*O({Lnw3O?7uU!58JlCK`S*9H= z_OO6Jt4%7bLWABVZqa#)kz229+uZ|%3w)hwaG3HTJw9_k5Y+#NEQ$QHpO`b2#IvCH zz21dkeqtb&)Tbn?$x>Gm1P1fA5JPs;?X(i7a+$G_Lw%D>okM`$nH4E34I2ze&?TgfDa~rq2UVM&|*7Kc-*M-luh7HXAT`XekTp0-F7JY&q+y5j}zkwrP!8y`2Ll z5o0E<*%i#v%H-M1!gH6z_?vcsJcP6C!nIpgHnAg?(zQF4)6$u4`NqO{0!LbX$q$B3 z%J8`Ersame-w*SO)qe{ZEm~&5LDwCoS}$xcE2vP4-LlP;I0slZY_8wS66E}neAQ-L z)_O$&7Pu=1V_xJNsQ8Racpy((yD?U?7xLBTpaDo_hYVMe27IBmjqpRSvUm~;Q0vICv*0^3nw~q z_DS&_WSxupE&n3wW=9Su=<=`5XG^MhKEQrP?~ z!u{nCp8#o}3yeG{^av%_0rr?VDGETA`D75C?>~l96&ADBio>#yD}&n!E@BK8B>Cwz z)@~vD26N$J`6RLLcw>BzW9(73G%Ry<`KN0CK}t%MSC41bRa%ISrL?vn{pZ0Vtg)Pz zSj6kUKd)RU_1G-2<`BU)~n=~nO4$X`*@|xUsnoV?r?tb~C!#*M2mY6gF{dV50717KF zm>H)wMIgs;yS~)LORzj8%23Yz8rfa^KWPiIA+fTRrb>HUoIX0LNWc`8arsUrhdiY~ zW31JIn`9EBXsR%R_BiZbm#yqGT` z?}Qh^sZjq}ch*D5FW8shlk!HC(%LCL6yoW?Mw>~;M5>hFqVlDhk&A2|1R=AN4`#wA zPa2;Q0ZfX8W^Y83|9Qu0qRWE|zSa$%i>}c*wXO5>OFyrVHSiyCS^NL*YxOXT7PyXN zNa8ugU}=8I2dS6)q5lWO|MZWyz#%w#!><3%D$rHhL^?Ki*>%oe&fKLYqGbW?s(zWf zIg*GPTUIaw61zczY*b?ZjjKv3zc~8_%GB>S)vW!};r^oG7<+T#07`_IuWC@(E3^Q3 z+K-j;Q|!_dff(2a$g2P<0owYz;ADOqdsZReC@oJE#gq^;AX9vNpM&`vxx8Pi_aT*? zXC1(f(3rAT*i-Fr&6D4Qk}JM2S%M?y)ry?W+-Bf%mH5v0tlM6&9l960{t8Fg6# z)4ubSU4~ZLTzuM6;wPs%I3mf(`eRmIe)XmOP^i;`<26XyUrAfsenM$s?N`?x@VdWP zwG@`-U4mUNw<`D)^RGj>b+7qvYSo-DCdQf{LcF%gxJy46(M{n)fsYUz+xA{`!l_&l zy@JHcH4pvK>+KW%Xj4z2k+B2@jX)_}9mAa5F~(rL0`d+6WhK%mfo{ab;c)?rKXkxi zc!r-}snmX=up51=q?|zIH8|h@wB@t1)zbHQxx}5@l#om=U&;ThoO^sxdc<&u`<)Jv z$_(%ni(i&{CQqYgk=qjiOjmAzt&QRBx8A#@(Qvu*ZLL7xqaSE{mP3v5*AaK(3AMrn z0PL)RMyARfrs9u1(#P{nZp7=?bgqL||6?c73mqJyiX21K6k!!Uc8f|Otoojpl=VA; zz7~Qxt-0>LtdyJU=B&MRRe}_4FTJC(AaZ!Nz23I$k&%EKpGI;*_LnBOKUts_R`J?- zjQ+o*gw+8wxd43(=+&;dPtCMStcolY&?Z@MLe!Y$x@ILr*$y`Er?GHDL1kPlTDV^Q zMuY{NEqs&Q1T22ayIL<$J_C=^j335RuB(1CQySrtMlP56w~#@?vrNcG1DH(+gIry! zwtH^nsrmC!01!_C;9iP(DL<;zxHu0=>TllbUqLbEz`vqRY0CtiJDEaX(59}xa4hx> zPCtB4wk)1uSV5ko$-xKFgfh1hdw~szKK3$aMO;Lh273Y=oU@(qxJCB{o+{#Y)awf@ zanWuKOZGamGVs$b&3z3^(V#@%f$3dPh+9Y9Qx9WiuF90mcx}Y)mw_YCI*B=Y3fD^8 zPFWE{6{H%!@)FugtIMb?J3Py)CK)eaV5+_! z+TLJ4Mobku#_!Cc<1TkOF;;GXuW-)Gb?_Glps0?MfVu|5?hH>%k}AEJ!_PwILd)*9%=zc96>SyLYg)wk`D$kpzvHE?t>7DKD2JY%fAu zYDSua;8J<4Vh&p;)$1&&{k1OG8?4L{M!OZpVzEDn2%e|C2164Gw$a&JgmvS=e;jEe zZSTl><}Czb5d9@_xnnDLb3O7*R6eOsb)1#TU2SnPunqn;N7>e7ti$@z3a`31^mP(mjZVk^Nt}G&=aR-yMx`}Us)*o z{~U*|DEe*mY*pOU41BP+aJU{Ea1RmjO4l7rV_jlD%3 z--qiLepV0^4r`b_@zv3fMf_`oSy6$W6#Z?nGy2F-JRejjQhKGhmIPi97H3d=6yU4s z<&-stzwXD7u9Fr3!`CHt+Hc2m994118SY~Tq)v7{@&ev-f?}S6f6Kcwl4b}e7u|A| zT=-k1NR>q_e=DUesljy!XL|k`T0`j)INA)D3^au=04Ska_F3Z0Z552QEB;|xWF|+X zA7w@4Q=xP3MdSBZM|oLKa}ddy%hSShD!a%3C}6hAkZUQ?>2yOsicZ4936*@uSukwS z@7f!pQ8f4tqAGIBP=CwzZUv442jjUA z<}0puPT>JY`;IwLt`@n9pM44>x_pY$E=zC*Ie?OD*Huvwjt&g;+Z|asf!-O}2(b@F zV8uW+#sP+yEv_VO3yDCbX7zr&PJ|x<7Bj_U;PiJ&2-g2Z!DUhBCa(CrbC5Oj`&7qo zqFxsv@=XAjN~D$rwPC*HwXkn=%3T^WkCQwe@1xmnMG|Mt*A)ZrUcS zTsXtjBQecwz|V9)5ba$mz~0!+wtzhaDUJ9@n#V$o7ZmXqGwV4uL^gEM)HQe`e4d|BXR0$+?ZrY8EshWg5?*0YefGdy-0)D+~Q> z380Ssu}f}o!MfHDOtL~-_6}NDW~`Jw+x8g{9KnzDRVQ1ofyc}P1vs^8lXGW>9GFA> z2Zd`PIs=pDmGo1qo%7ZCm}CAYL4MC~1kb0Q;nle7W91@~)gw3S@N*&k1{4#tR&JTL zK!=2{b?N2-(i&hh_}dKR@?B3%)cX*cnTnN7=05I<-tFBVs*$_4PDMN|3PU`0|FI%+ zYKxm46xNdO1(*8$j|DE-8H(S!NSP{^h6|>lq=ZncXWV4GYZw%V(x#3D39d z@sKWb4kuO7vvaL>#2@y^inWw_WIU?J#*n?FPlJlv*o}QS4dIX5Bc)Tymg++T60W|H z%5nxl&RdkF-wCvu6|%J|ZKK*Y|Nr{Cff+_~zTje7DqV-L#}`Z_Xe@c0HDhZs&A;Ly z(IQ72;|ueSgCVa7$P>(yl#|;_rP-fJC!@Zm@O)x(6T1P--^sqyqW|OfL-7mzez95S z+;n@Er3MqkS|^Z|0B@<8U4XN>ViT?|zJ8 zwBLK$F;UmZ+kThe6lIN9ST6eB^4H2Re9k$0hAa%w^3t8ct+Ab7I`SxZa0v+5tDcme z7-Ah92#K|D1dIu8xtyEz{9LRk$#>QZ$py^yo-~d7e+aAfVWZ5fw}@vGOQX%+7P)!N zM@%`OtwF+K+oDhjK1_Q&9NhuJ?gZm%s;BWR334Nw=b@2C3^*c1=%43p+m-}i3LgAG zk2VHqp>})-wuz9qX&Sp`nZr`WQM|Q)?8ks{Rnv+zc|Eu@+)Q)Q{Z3=NPNP@|KdtwD zb-UZVsNQ-x=#-n?+fmtJz}H4q+K1hQVqG4AE@zT^2`}pI;JQR_zEuWl6PTREx{K%J zr`i0u0dWd3(D;qy_q%{CG%v}kfgLL`<@h=$v;F&17v0!&CWXHBK{%gjZ?XJ%@v_jB zAA{z|#uLw9q-#A)`KF(ZPU$DAno1c4rzk&ncW3tks!=|Spu1tPFHP>D5cta-R(hY% z|AlY)Lo=@41s9J*SUsgDI8MtPi%hvWM0ChHPHu1jkgC9i$Z`>vh4*I#F6wzeb00ge z!c9N62xTEUGQ63QZv1}_Xo6>0OF`~%lZF-sIV~lNIc%{ZSV1m4H(AL6C z?zlUjt`(M@R5i$C?J0_2cn(Gc(wNBgo?e6Y&;Cu(OIvRMPI#S-*xR%U=zlL9H^4oD`<b^EVoZ85KVnNt?p?ObMsub*8*s=%&m+RfRF(zYN=w$ENps8&ani?X*muffyzB|) zaKssLLzH|lb`+Gt_Vo#U|xe0bfkmMlZq}p(O1Y`5E2nOx-wIPfupt+uuyTr`A`v@!| zc6p4etjS-P<`?iKnWX2~e!wLKy!mjpP!q1zCB^LqGIOXOf51Vrr=m99k1ul-b*6f0__qQr_^ zZMR~H>Jn2z>+4>Jo_GjX4RD@QQUT&LsA_))`Iqn70K-Z7E=#8ghC}GHt<)+gFtCC@`=sNOr1RO+G#_0c&h*wrcNnz*n4?W^(sho{iMBJX6)ReN z_QKW3h#xNqF5z$WO%j!doVvV4LT@gE5E^*ljb^AONH`1wjRcLA(p9kSud`7qJtk-9 z7Z=;eq5#$49XivnsZhP_2t-@r?X`}R?+Xt5-r=_(`88H)2xz!`<%M)=*lrsl2hqak z`4LV+r7z6}qNTb!TrQ{1EYqt0tZ>WUMm?bHD=w(MhX{flDcl=b9I$&5O3Qf#ww6!v zBOwhyuH_ic-TT#;#rblZW$2wN2^X0hSC%cDqGM$Wn#!s+4VB(b88Bx6MkShE3i+Ed z)PB&yc$MT{h0VgzFb%EB;oFM28;tqMe|szGK*y~56?DKc@T z$%Li;;tK9ZnGj1T30RV^?W{s`r={Fa36pNXoB7qML%b&`cNbIWAal%sm3O-#*e$=d8g(tu0pPMLCrGj4n0*IZ?g;%hLmX@!3a!!HCz1v{$wkb zBf31^e|j~%v7L)}y*2iQ$Xz}#`0PKDp8DP{=Y)hYZI&$YOWgb%I=eS3#w^ z&3zLjYSuSL42annN*k8-H#SdE!uGdi9Dlm&_K?IYx%!HQMqdEh%vZ=l-v?#V6WW4g zLt}ybOF=M-Whaxs#s#H?t#Ax*y3YW5&e!!*_2vMooNGf17AYoZP_qp(PvN;st!BfH{;>IgRTk8 zgWS+RMaj&+^;|jSRMvQriQ?~Ejtj^+q|cch)1DRP=~HL66QGX}Yx(KjA&c}OS6v>F zQ|rDWCji(zR4RWG>XvywHo-snLHclISb~uNPWnj*1FW8mc|U8~RuWCw%1Zs*cwr!X zlN*cI0v##UB=rWhWzxuikOX=VrR}Jn0xybW7xxUgRT!??-)@Q(W=9e{pYO(0%<`+k zy*MYFz@>)M9slI*-v$ibLxoKWw%&E+0}!U!ggPti(A^$68#!vK%%*WDGF9ka{>&iQ zp|`t|p!Iw0(cb!2Q*%HBi1v}tD$L>v&?W%`b)ry*8A1TfBhOpWeJ6+CJ##(A&y9F? z)jJR`IBM_UNt@!|9U}Ok{2C%)0x)#8P4cU6M813eTu8Tf-fwV*G^Pmlpj2KZ{H>#u z{&=fqT7d~=+d$W(E|N|`{b!}Cw}gVJj$7|Wdpo9?Gn~BLBuO+aO`CE5?CNkU5HH2j zQZkss@(IY)pK5f`eIO&As&XfYyd-f$uz~msNxMx8Cd}^@3~~P$QXlaqeBq-vONo{? zYZzya#e+w8q~bI^mk1SFk>lwq-~AnC^X^hoS(7?S-=oSuUhycrO&?6+hCZ7#Dxt0=dzT0dC)n`fIF?RQiZ&3BfDvC^00E z?maGf){?#AnH9-eYlgK~o#ZoB-hZ<{$3aeT%=P!tg`Z

-55T5k7()$J-=RUylfA^ZiEtzZH6ID??)hZ@OKB0?ai72(d` zm27a8o!VnV`#vgMJe7}RSc_m+^<{pctK&5^l6D?I5Y1e%T@f9UOL`l|8uFgEkxYTF zN2IG*eKdLg!82rgNv3^rbgGoIqK zK^t;zbHxcKRhKyfMIA@JK_-abk0mXnpIiV7)5ORo(XKZx*$Hy3)HUv3(f%q3|J6Kx zpZ{G_ql31_-1;L#%!4d$G?j}b4ZuP3@1g8xzg*k-kMm7|ZoEyc3ja%c^**ug(^Ab* zTDy5pR5V?JNr4@7C31wU?^!ZEv|4!nv)af(v?1D>NOE1fjtkv$;JKi~1iw|fdYQ@A z#aRW0`B9i9VR=Do{2uk2=R~#^VS4;++cP4g1eua-kn^*K8su+AnLZNh&Ll4vxG7CYruL;zmX zd~xYk=vx7Q&u+Om(1|j2pj64(UMM7ow1m0dYlzs4>KM9Z=i& zk-oj!zghj5>-_;!hu1P~&XM-vBaD{oDhwd!co(SWD&Y6WzHdBLkxmITzeBv+VI6ax zJziAh5S)?bpvT&9Lx?Y8885u)xh87?f@|LFeqjlAq1o86jsCUUwuxsRW6$B?*V%$v zk@#kO!aS1^(8Vq)zK@r(&!x)MM+`7}SNDM9V}C^Z8t-@yNr@+8TYL}-A5FkOR_0@*0TrG`sX zkVKd2PmM}4hGugS86YWyrHEc}25jXxpqN03AXm7=Ki)x3gH=M}uK#5PS{^}4tFx+x z+miEG2~zbIoJo6dKbz1U2(7-B&#)M&iLG13uJ;geaM)~i)f^Ekk`Ev~3&2gEY3eS~ zu6~Cbo^~tQB0AlN7`HwsI}F@5E6}MEY<>wZX4F30 zKTWKoO3&9mktDLVSMNyy>`L>W^-E#$(;9<>`y2(I&UfX(Buq(Z1FA1lsU-7hGo=9f z&cLK8N?yb=AXvW$XCZGFb^21Ob=CR|TcN;>Dc5++KhIc}J`?8%!QIIo8r6paGW_?? zg?2Os8CUjwwPsvb36~iL#swHMM->Wraht8}##kM^ut$6OXTfzf6>AzYcYK3D#O*UC zfAdc65dq!feMg=FoNNIMiw%tp(o!agn1r>fOO}4H>k-iGyx|8VB8FO2cjA`i*?-QvW4O1Oq~}I9=|L0xnHQZnGf;Z8+u8v9_?eC zjDsUm5}tRHACVBNK+PM*IsR(F7v$nS%~sZ_Msk9-$L|&H&2Fj$cK3Gio31a)W$Y$wp#)-)3el8Pic*t>;;}(do z2h!m_sQkLS=@K(6Kna>e5Zg!J3~=H+rEXcG2ncW_XRn39@TiA9#JU~$5HeABzuBY1 zGukFa)^-HHU-iGRc95Z4iJ- zdrE)ad`WHocjbUDg?xh{Qd}V2?!vo=c&!~O@7=rx8jxcW+H9zBz`F~RfF4DgFWUUk z9~f-G>#BI>&6by(iE~(Bg6HSn9_^&k0d^^kRDCx!Q^I!oyv;w9KxW6u++th?WAtNI zz3di8S~K{!d?{svpg5?37~?^)`-+s-tY4?SwFJh|SM4^q)B^QCUDkN$Q{HJE zLC;Tb7yQZ|IeJyWqCI^c0DT`bT8q}{_)L97a@Ai>>7os;0RtOhC8;hIPz}+WU42l~3uvmcrRpkrg|?-49eg_*0s*csm8c)k zyJ90Mtp0`ZVEloct;>y{NNFPIof}B>HQ&=s(ndG);O;Ddb;VMu{F?-&67WXsY*~zI zC2g_=r+MMM0Spt^Hjex(t;ibVEqUsuux`F#E$|HA4qJ0|ExPoXW$9UxA=^YK1 zzwlT-di%-%bgG+rDL&zDHt~1~ZJrR#;!A#4>)Q{q$P`D`+}PVk$@?TQCmhLE$vlmt%D;i12Indt8qV6v8Yh*kXnrKgizpcEZKEj9ehPB92!>g*5?%SK|dZ+D+s>@HJQE*P#y!P z{6rCI-W{@*FKUB*Iuj>X<&8$3WwgzS0)hWS31JlRoae%VN3_vR3{>`cK%Pcdq=6?qe$ zVO^l?o7j#;3z6!owByEYt+(IR;hf*Cu2Ql9aM|dTQ#~tZ#B=4I_1an+m4UbhO2S^>cb^GG8pG zRr6BR+(0P1m4}q{--b5y?qFB;Ff^Jd&Nx|5 z)F+PLWiOG?hv1=*ebn3|&uIVg!93~&q}EJoMZjCpN^~mPf#J#HOOKO6sb1$ZmOpW5 ziGaG^Y(Neg5^E$*J4>s+b%n^f3Sq5qm5_0sC&?!dxQBMtrWkomp%fW7nnL_PPfWwLj5knN(4&y!$ijiKxj-ED4{FHtbCIO=3MRUOo{!z_2+9xE~iH+ub;B| z2vm(ooOqvT0M6lmQ_Y`aREKKkqX(|7`m0b&v7ojP>SZ{LXt@?LoN4(+M*|xb zfFsnt4l<8zMo9AmV%dT^FC+RjtqyCBV5_%w1*!#{ww8J~Ne@C?-~?hlwA>QX3LQLP znhEF&S@OJS6`xMd`!QQdf0aIg#xWo|#Pl2Kls_1e2sGsRj#eT}8ID#>TwvEdk`nnp zY2VJMWty7TE4s|qN9R&JGLxECnc`-!w$7kdb0e$y1N$; zSpOzfv|LP9t@d7ST1MXIc=D46O^dLe*xM%Powwo3DY2noj`F;BWZQ*t78VDF2fP%j z*73@@v?MTN;CSK&07<}lriX%WgfTz!i=;LB4Of&j%%(DUu1i7lt-(MQOo%2-p>2T; zn+h5hSFY6rzVsRr#Ul;6DlbjEqM7DG9R!24KK(ztLVlK~ZocFPj$Y696uIxb8JYUk@$r`eT zoiEr?)qu zdjhHnJ+!?@(xI%v_|pzx7UlvW_72NH59oi9OZ&M3ZY}O%V>Uq?WXrQRp8WAgK%E#W zvNoar^N}*~17bV*SkW=_;BOIfN^?w5)ComMBNrGas-3lxdC>jp?@J2A%=?!9Iss8&up%ulBH^#gsHS zMRnROql40mguqr8!3G&~Fcg98ntjq{D^#asPcuM&482lDRwjO6ap#XFjj$m#dguE& zyBifobP;ePZ&Vl5#0^?vYuxrK^OG7y6yjV4C}sep06tdlJ3(&z;+5EW4`?h|RXw4J z7TKyuQ1t~FyA%^+hNPa`X{H)V-@Ok=QpLDHs?RO{zh{Bj+V8AVu0R``26lU6rjQ>! z3E|b4$TL?&Iie8(pDqYu49FGUt$R1X%8FON=FIizBNqcgTQgg6qGT8G_h*Xxy$Nu` zn#lMcA2|e(^p~%?6ESJ6{6e$TYJ+w-CW%d$4u4C`Ek|T~^xc7zMU#i7cC>(Zmy<%K zef~}++B;V4pSxJt2tZ?#0trr6NXZoUj#4C{r+UxP`K4_;q_a3C!qOQWA&D==*m%jf4#O)&<86w+1 z0e+(gSFPv`#b+2A7meT9w4B5zPNS*!p?R=%-s4D`_nr-wWQMQi%UTG3tnkSrNkMnQ z$`~cLUebT=r_Iu7#%Os_S7ZZ&XD5Hev!L<9JNMPIas!u44CgRoKoQ#YS*Z+#rbiKv zKct0^A98s$%vfUr{|{+NBX-K|9(BlD93jrbmxTb0%>c@{bFI+3yHTT^FGrwWoGHhf z_Fdz~<6<9<&jD+}Qi6b$S8wt68mT`eS|*DSh|@><4>E1v>)AMNU?E44S6tuXhcg+d z{XBS%;&9v{g%8FY-QzG=nSn;YAiMX{`8&A@)+>pGzBjllLb>qZS|P_Y9u3Tyy;|gX zj=o>}ddnV#IY=*BmZ%nC<7oFK-Oo}!xITn2V{uDxPmjRm~=UL%c<v7hlU1fiKGF)t6Po24`MfS*|EaWQEc^w_kg1|Kq=^BthTgMa z{&5jtTSTl{bSshwGyTQAi%Z|AQW;G)2-)~K&6!moWdH@a`C7>J z68UP5UrB-9^@iuY73#0z06rs8wPP4A`nj!^>g1E}IX$j@sk+k`dC4tjG*`Af8n31t zLrQaDhPnl<)bdnbLmbblHrY2{QVvY@A3sl2^yz$zW7@$AO~#~E)XYyt@Vl>8XbKeb z+HycZgvoF%NhW6)$g;U2Jx5rvZ}K{{P8RI{`^w6g{OgUbNrisZtBH($8TmA4;=N64_JWP*#tGKjW~J#%v(lUp#3Y<(1`#ir;J3NPUPv(Q5+s7!)A~ zeLOM-)pc}kFlrMV^okkM8zuR({4ZCgPUEFZFhHQ5TBUL?&J{we;)Jzg*Fu>_01hG5 zF#LwRsO5Mx^3Vn0Y`6(^Wkz7v2U5BENjsrI8p1{DUG^8ypYe(c-f$(BYarn&QO|G$ zj%SG8;q*Q)vfmSSh+PW_VXPFz6>e^Hq)~|CZ)&ooGndMvH z!Hv&UerFu-nx)~L9DX}GKayMlAQ3%$=!6QLoLgpR*cC?MN&dGbRKSjX8g$%x)hGA%=BT)a@w10IvN0J}$8M!YwOBL=muFB%6mgqyPY zRLP7PSeM`S&?87Qs2V3C)r4{Ca9y4DIhKXCr>arMrSvqK?`9@gJ}? z%Yw?*BDlMk3@uM#j*TADw2EB7RpSNEF_T%gl>IU}y~t z4~kX%Y4zQ1N9JYqm73Cm1iJ3IjLz`3Z#}wG`(bK;b?GHEz&=B*@yoe5(?tM(FF!r4C)~w`l6PeTTcdCR-t(e z3UAM$9L48^Dqi1Z`Gv+TC#%9=KZlK?(A7}sd`#Y)mg09YAo*_8lrOPcKedta-3!!3 z*mj|E;0+{`E~l>GH%fs+93YAu=u3j33{A{bf~Vb^#>SRjCya1nxe#7>!rFW$?+IxW zLiT0D`)cb#ZORG;KhKpzoi1K|8Li)A!5)6w=B5a_r6F&LXEj1NZ{w{{K>Ts+(Fpi= zeQ0pB2HIqcE!bFEeQs=}$w%`gzW^N4W59+Sb`yRWpatc3OIKC9TE(xCdjpybu?EhQ zm>(onCXecczi1FEqIMu7tD)NS&!FHhHY(5klM!t-XL; z5#-Oov~de%_;XN>izBR$O9vI?XUlPS_C<#~V|Rwora>Zc{7R`w*va9Eo4_ZN_;}m; zQ&~z`X5uB;5@^$EX0o8fhImm*KI;reBQ#_N=SBf5XjNj*Ftgqtv|Usv zkpemClG9Ov2NS1duzkBBra_3ED-*2I)d`fLrnYpwiEljNONUb<8gg%Z%oEgTRIJX5 z`Gy}c0)=)i(NDsvpPTsxKIPW`t&&p~qKla?dD<(uulDP5A zY=GeU!$PVc{3$~<_s&9NQik=#vOt)@w&u}Hqk(#o`@eV#SDsB6F;$YX{(j%A7{GXJ zH1ZPBT;~hN2OO`BG*YTXr%qVJT>F5{s}`Qn9$*w~W5n8gE(1ngcu)~tuV6>{bQ)Z{ zEXBJQ;HUVxrxNPo`&*2)&eCjf*DGGFIn;wGTi+ZAs$aeVu@!szOh}vJ%YOHA6-T{Z z!F3Xhg7RppPnvE$guKi0L$#-ZNhTtC?3Y zFChWl;h3(v!Uc|HH$N{u8fy*6Zb^#?lP>k2{dUSe}0u@Rtw zw>>(n=tn~?5fqP3O|$D3t19hWRmh0a*a8_ja!Ayj3_EpH{ zN2lRm5W*x!+>fHJ?OQ_f&jl{yC&^l!4y4&cv#tB&odW6)lW!q8Mf!CMG@fCVzU0z- zDldPC_|$@`S*NW)o*vOb}|KijDiT#c9>NC0B@s?Ca%=AT28 z)~+X}fPD;+vD7)MotM-=Iybu8T~ptfF-f5T2p23kAm{WA3aBShQ8};uSjx}UTlJU!{41%NC$isNk*43Q6dV#&|| zuIY5DqK{j%nC;kuja>+XSvJV7yykEkD$KWrkreRNT&7sHy6H);e9orjIhFuKkSmAa(L=!e)pq^KK2sUXZk%HFf7-c` zRY7M>Ny%(SXDRV`O!i?!`*xYuuZ5uRzP}%z_+1;`-J2%a$U^~0m7DW6Dn+jnM# zWJ}yE69KQe-^wk_x(gNnVcey=G{V`#slloD2fxdHZup{h7_)A0?4EIeij-eb*xY0O zZ}s`(5}(LXF!h5*j*#h*iHMm4oqmZn8_dQ8Y>B*<%ASS)5w-~yK9~TnT&@+{u$h5D zMh1fi;u}bE@$twV-wc@>;0@qdmyr1z#hn(idtlgkF%t|!Fxm>%BKl$m11A;ZX0?{->+$sr{<4&Y z>F`Le8kDo^Vd*!8GWgH-?TnasL@^Y}xLveUl#K=ldWnQjy00fQgGpy8;7Oys-da|3 zQJ*?SUprZ}{ny}eWD1-9XmnrsJY#?__l)x&$@5?EpB5~M53#_)mS%|n_fiq18lzt! z8}k;fDw{x!&R;hV{~*^LC0wILH^GvxBx-P_9rHK1p z-}*}*sw{>*#{Os6@}L(MacWQ)D*h30M8QxuI+IaQ%IZ8Gtksh3N2tMWZ_hIhM9*U9 z(osZjZOQ)mH`pgNXztEcH2C*whT=Bb*K4o^p7a6F>Pw^Ya3B_!;JG(MmG99-7o=2; z5eNYdlz~(re649zXrtpW<=Q9NB}Ig%z9!Qw2>L2S5qeqmV`!;CVB=>Gy&_ZZx+YQh zNi7;3<(79Frwa8sjU4=CEe2s8{JT)jU+S46L}6pblqe@G3Max;OXL?r4Ai(Oc%Lt! zZ#Jx#rH}3SYW0Ti1U=IE-LyWMyJzOmk*C|PJ2!acRwS#d}Jm0#QUOmGNm=Ui>!$v9C_M2c$5#jSW~P{S2^S4{by7f1U0XmnLSM z^487nw#ns=5nzc9Iv`j;I;5E<%TF11Ew3BEZQI2&ec>i9aeiU~y24$4)ReIU1^~)1 z0a3MAF1$(1^y~@nt{mn4G?+xBZKNyi`f*FZQ@JMH4&Wb8 z3obl}oI7mcUk+Nln#dvYID>RvC`HJ|gvh^45GoK!%%b-=mYu;Re!Z}$C@)DBiPJgT z+?ng@i*sY#qkWj!k524ZGM?z}%BR>wQ@S<%5UT#^;Uc_W33?Os8U4_|U6cGF>yBhL z;sYPVf#keNK7Oyzfgdn+$t`HQq?&HpO}8OF;yaj14lX8JOcH2c)mHBLy;ne-d!!8~(@lrH64txgRS?~ag(qYal}^1e8i>~^V+ zrwvoW(4g*S&4yOOcPnB4wZ+$mwQ$FLMXrqK2d(#jwatyf9MYZ*)i+7{SM0tJ8S+Y(tv6e3Mv|4n{`4r#4;4TAYWkVwbVWp3NcqiK)V1N9 zLC4X4%l87WtvN@z7+O!XVtdyr+pn@V!kQBq=mBiMD}UOEIkXT-b4cn)D@&t(I$@l= z_N5U)dF>yqiP01*t9k@LCipG%sLwf=Rmgz=v*tY#RQ=qgY`~fDiIC=ZTkL&OmLtZ} zVL-?~HDI)dN7T^?Mug0U!7?Szu9@&$m+*`L=8A7^A5fGAgS1-2(Sz?*+#dQ>(CX29 zDH)R4Zd3I1^@zyz^u5B3s)HaWPyN4+eFm;4V-~RegINw$g_3Xo+?$6$AyrOj)Y1rd zEPye)xO?11*fTCGC~|_tkEyz(#f?lOyOXG_#GFd8`QieBa$tfinsxo%4LJ-R-gAR$ zZK?B;G9i2_-ncB&aZ(T15?9A_5JqT{X>6xV!#pPoaQsGG&av&H9+L^B zJU{4u`D6-fH%(#-G-m1w=;I#?PgX>9ak_?BV=p!}z`6DBo^**anUGD$66-VlRu@;o z0dBZEEZ^YE>8|c=VGcVdjyc#H>0SJjUB0>d;?mdOiA2;&xZJA^o<*i4v;V%2U{3?A=Wk1A z6(b&LJ8R(L(Mpv&(sdyamlZn5#g2Wtkf$!Gn*G^M6_5IFIZmsGlku!2$X2i0y}LCBgLm zK9NK(*aX7?PdvREsWTicL<t+?{C110p{_41y!q# z8-AWC6#9HS)Ah^DxqQ?TH)_kvX9hkq^AV( zKGHKe=Ud5}iqcDf0WlRKFS*L;KFbeZ@J&RkO(=H%>38HFLr@b zO*ro%79crg1ow-PQOjPz%e~?O1A(Q%l|CVsLiBH;Oy$sx>N*qUMB08~_uRtSNM?Qu z|5VJBC9Y-nWYdG4X>o8rs5SZweb}1WalC}_Ny+u4^Q__usq8N{ganb7)odeEEWYrr z4w$a4js$)cKX=gEQ<`TzYT7}k2-WPw?tFjzU5qFV?uO_6Jl%eV^{pqhtVx}Pi=&ji zY`*%lT?1IRuB<&VE4-XsF|a6FTr!-XKtqDxSvYRf=l9#CD@=dvRsY_96PQzWY?*8n z6y-!2BE{P$+9fd#f48$x2_B7Pw9d-(v%=C*09lBbmt5{Fs}Mr>sE~#~{O;@u30(*W zOrnB#k40C?SEM?la3aYt-MJ-C$PCZ0Tnd%LW6dMIDFy3kDqLcotOveH(V1}3_{uoQ zu?0n6@}YKgCJrV5f`_5-eUt3k=I(J9dK(8{CsAovLlEpQF(N{~y&RI>gtP*DNXm~5 zyt&F5>-*nG)mv+!;xj=6_vQ0PPal5BlUVuNP!j@x)jV*xlICM2Dh#SypF02FA7A?I z9lyEtZ}&Bpu5fR8vy6Gn8w-yBa#&LOf(PHTK9Ti)@*S+%6PGtst@qvP*3n5=)3K2y zd6BIb@paminY+&&M#i$TEn{$^)RLdxLUIg5?IN4F5IzeJQ2z^|W0PUZxN1qYe4?!e zV(eeY<=?~^UwxpsUu+!DDf@iL9K`fHoF+Kh>qK340jPV|EBQM3_U`l?=FeB($!U~sS`GjJ diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input b/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input deleted file mode 100644 index 8418633d2ac9791cc75a316d53c8490a328daa39..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 65535 zcmV(pK=8l#i&K%;#;51Q|(x zM;}NPu;`xhFDh+BMJ6ccYFsSUg>Bfi<~bp&jg-~DiA=I+_Co^FF# z)zpAln0JXoILWUtGMXS8Mm=Y4*K(dtAy3BO)O!Str33Z_n`_)ElXocnv|`#I=O3$U zQA0T|pppS>bw2bp{X;JIq;=Zrn+jwL;3Fx$_veE=``@#!Pozgxncgp!ZX82QhvIzM zUrc=HkOSK=mDVB*N4QOEy(AHOADFT(|I5J=Zkm4@Lua&RXMk7^!R$cPB9zMc=#u1VIKF3O%23A!XF_hH@V9L8=wGp~=i9q?wfM^j z#C3ka`5b>di7(PvI^y_|wtFNe>8^x}-gK<}*|%vb>@sigl7#U<42rxtZZ31wZi;j& z++ZK02i|pybjbc=b@n}DtTTzj@c1ojw4QW}Tr;%FsN|WpkfHAn(_ym48kBrQRrE#w zo~2sGpy(>Wjc+s&xxP->hnI8DJtMBw8eXnlY6JNq4G`H!X%#>2QlkjcJW=%co#dE_ z$Y(j#UNv|p=sbX~d2!N{^r}%397`MJZWV9jyHT4(pZUa$D*GDWRnth5CjlnHYgKKc z`-F?ho+!fa8YJwSuDxLC6*cZcq%&Lk54QIKrUFdLkXSmFLFdZ}jN64xsEPBnj{S98 zPwn16>o}vnuyg#lRQF6UXD&FRR2aGlzw$ZN{-r_2W@fs9?`P!ZJPgXD3VE|vi;8ua z7(y>8qk`|Bh6W?yb@~Xg-WN*!6B&mjRh$Nysy!{D*UA{oS$OZ7>3OQlQ1wqXy!#RC zbq^x3&iqo0dF+K`mQiw4!w_K`F6Bc*qmTnSPWGyxBR zi4@9y1JoKtPhF6juC_k+(rV80ZqR5e!Dl-K=1yM5ugioX(a?kngK z?!wLWA-Ajr;y!RTW(O9?rfUay&i>v^4FZG0F^u=u+H*D1Z|~wzCATX#^Ztw@U?;xe z03c~P!|#k3O~QK%{l<_UF&O`X-UrAfdSdow#%Ug3+GD+h!;h?*QS1f z_380|P(Qa(x^zsEri%`A@C35EDYOgjjtx=OPeci^9ctBOmr!dFY|F?c)=*9888!Og z-~m7|^YQ-K`fnOuxXP;~P3x_1(DN-y=8mxdL|gKi@VF_Z4x}n(rY?f23qPzftgFf( zmgF)Fqg-WkVd{#bo@^?js3!BeLW$Lpotr<+eaqquQO2FB#v%%(iea$4YHIvWzQKIh z&SON26$oEbYD6?{iffI~bmpjdM%QSlEh*#6#1Q049Qvr!w-t^*gWe1Jgu5Z@5eDT* zLr74eHOdHP(jvg!2shxo9KUSBnx3mxmNt5EnC#HM}Wyjvg>i~@c$&% z6N#p17%D^JkbDo=Q=5IB*;Z*pJ{Zd~|*p0gc-j3L} zaO5+qlH#x*V_mF^|Iy&`pTM9u{1)+V{rDe^yV>M99!{lfo1y9?gmogjo%_V=w0$vW zL~YUY$Y6<02MV;W#S3tv$ZV~ldLZ5^KRYqDR-pM3&YX*N8~B5}w^u&ZurR&6hjp5A z-d^u_JHZ0)g8P6P-MZdHPXTJ-;#a zOcQvmKqGN@OeI&4f9B7lvsQ}OrRl-X2B1mhI2v;puh4#ztSGmt2@9S0SMSRbP8Ag^ zP9yCktLB8?ofnn~hB8a3Lranp^ae=M>2(8laVI+ZkIv=6iq-lRfNp#asNZ9gBB?aWRkF0~ zKU&_yin)_La%x)YMe&Yaadp)dKM2wug-=i9Cm91F9Znv`wc%g zuf-zs0Bixtw_POP-{FMsd#Xbl@|@}&nV`4grS$hV?uJZZ-3ni^8(rs^v9%SNu0kiI z-Cye4n9E0wa*1+~-HXq16CM7Du_cz4jo#XN_Si1~Nshl;tAehdYSsYV{)|!SEMAmk z-=jxLkj${@abJ3b#i}@;@(`>`LXyR6U>-KjC(RT$b2uDg=gHRbu`4#WWdX#=h_Q{x z#%O^NH z!&Lt=BmIZLCbV`qzNX$R`z?*UkiXIf)3=&>ZMEf$@xe zkA*ta{_j&GO&tW}p6|#bOO#KeroBsu{CF>JV>I9j)P`P(*wQ9aMxeGS(9iUsUSs?1 zFjCL@T*A2BI!nQhZkKIJ4N^SMxb8-kFNDv7)ci`nD^92No3*Hcv*V>#nb-wsZY4)% z7_IIJp<~PxEFXo=B&&wzq(!&M*Z#%7zn>EKQ(zwzp2V+a2KSwXsp(lpjj7))P8ktv z(X28+)5s-6d^MJe{wRhm_IG`P9QzG1I;(T7_Jr0k0Vo%7McL~8G%)eBosq>VA-|=O z!FT7Y0W7USHsq3c>{F2TpEPQkQNmagxXhqyDF=RXH!av~9%ar@d>BBLSq^;$9FP6m zhDZ;vg z`&y|*Y>w4RBPP`fkr-J9@E`G2j7|{RYud0)u2h9*gJ~zmVNDt3<5q8G`!&VT0`8pD zaqXpMyd>l7KSzS=W@U(&qp$FzoTWV@8_t8>Y7_p}jba0N!*%%2Ht~beY*1(9n%5kN zD=sS3G%7O45(en(|4idgc#MXPqpYFi)VJ~fB{ZLz@#WX^%Bi!-X!&lue`2H6SsXlZ zqgJ1BQ}%S(yJc$w zJ?a$c5`xg_h{lQ;%nd>me4xkb{I_;B^5y+K&*bbUnmDZy@*uWoacSnfuy3%}ieFZm zo@>~MaO{^Vd!VgvkIBr{l5mb$Z$?Aj7UMwFcL%ZzYwzshc7wCTrpd+^kEfgy%@T2 zfKQ3fjR`_m15$-$^B~9dps-s}5(TRxwpBbQVyrjJB`q;Rj<}~gwF~pL zqpK+#IU+$er6!NfM;B_49V1BS(mAK$a8?zbV~myG6Vg-9Y&8vj&fqU10bt#jmyc(m zY&)d0Gp;dVT;FaH)o`xH*1_0Ygni6p~L)Nt2yvEo&MOKqUowZn5u@8t$- z3=HrGj%VI%(McrCOOgl&yQuKqBa*`1P|3)N2)Fyy#*sa7aZb0By}`cLHR(2I&!5k- zwY-c_dRK%{_LB$Jt3uo8dN20~I(xRh9U`a%LI(#H>FpEx-$ACY;ge0Ei?s#Yimn<2 zP-N%H2u+hgM@B0pT4CkG=DK=C0eFwmNb6(DqAj+hada>2zgC78d|V0$Q0t0(L!VPd zCWRoAh_y=3a^ahmv|XK$4K2)(u$6^Wo9cdR2k3yXr;y9u%uBn?X0O)LC1f9V_%FD|j4F@Q)8%l_I%29-Ahe5&;U8BwHkidY>ly z1)4X!TJFnwW^WzM`ohOUz0pVI%iM1V2XjG(Sk}olHnmhHmJon@C6LiflNT>AG?6dl z;x(D{Q;(n9`^<%`QaojP3P?*3RLZDt-rVn3G+Qk{_-ukSXkmh`DyHx$ zx3S9er2+$gnB$0TeFxD2xv#we2P%leH;i@ET)27uxNtiIZ6IwleHTPu+Gei+WfT3d z?Y7*TtxP?gEjC2a=@u0YpN2S>N!&!!v?#CUt_^;{wp_o_ufE#L)*f3AyT_yFn_r>iQ^)9x#jiX+5T+qmt)UfuDU{Wi#GMijfHRIp7yzSWhZTp9-4md8t?MBDME~{!{#`+wBM@vhzwk?c|j0K7u3KOv%+t z6QWILrBDlX^DZX;pd z9EhqT!Lc7>JojX!jw%})PGG0&49_&lw=c)Y_wXKe2titA%@GEaS%GaUplob1@t90B zb;u@z)f10&b&M41qZmRtTQrFfIonPA+E4edU->WjR(Ko9tooG?G z6@vTjWsmXanPA`^CyxVV6yQN5^G8`!|0?^v1(4;8eH?~+b^~2My~jN=S(!p-8d{@tKQ_UN;B7^Tx@eg&C?I-1RLyriH&rmVIpcG{!?>V4;a=cjoi8MN_d;MD8j(R_)d9q+{+1Fuv;mYMK;= z$om-Qt)m=53rlOC;d&lIF1%9GLzti$)UdIXU+2c&Im@*vL%tXzV35cH4 zJ@Lkem%DW{cM364SY4)1Rx)GIagyw-`=3o)!7R;kE{i~qB5nvV?hesGRk%RjWDLkn z;@My=eEM-8Ru}a{#o*wIBYCKM#1wYq)bJJ8ymf8xMU$WC;jC+)@~?Pjdx{)Wfoq}I zzy_XAmc#Si3rY9nrx$IehReMwQ4C_d1l9@?p^_@&Zh7&iANSKWXCKsWuxx+idBIU0 zgKms~kz1hQh&fS7yR%J9o(@-y{4{t0u7O#z=?nEKKTD`t{6m44MlzC>q2Z<~INBV@ z|HHg;W!RJj?;oN63z-cOMKfpco22+3#}}&?$|RItyh5Idug~{4!zt)#S@O0F?G6rV z8TJ`ipug;y%@zbUamB#h$PF3(@&(i|q@boM)D$RbAjUJ!=ko?JZ8g=e`pUwpB4&3~ z==oNI+OIlq&vhOp6vBW)iHrYq&&`P0X<94)gO$Nv2Et~|R)b~SF`f{<_Qf|;QD6k& zPg<>2;YY&cB?#+5PWxV!f_>u;YnqW8k`-uhhxfuf9bd*s>$cqx8_2oFi?NK@3rqr1 z3TC5akpgFyK~;qkTgm(lgXdI`HR<~*lCBxTITkxU=PLLHc7jvB<{4Uze2MR!s-zji z&YO({f0%B~$1?M2`=Y#=L==m4OY~-YGKAolKze|G68WC~&LHl%h%vItj9qjn51oLw z9|7K35W{-ePS5(Kn&OJ${G1;S)20;`Ujwwm*H$GE-o)9&B2qZ4y!Rn5v%r;e9+c7Q z>Yd!(1v5-#LD8qddm~V<6PNd9#G(oZUiC4@d4Opst1kCB!gt0{(7U$DG>y`KN1{qv z>{w47k6b**ajlGc`?Ch*?$(G*zt04ozY&dnennyfHm1tpx{5QAq3j}W0;aFz%FmTl z8xr#uRHDS?Pom9>+9g0LjDc~*fvZO*k5rm z}(e1X-D>}M0Q;H3u94`S&-`My$#v7d?_e(1FK>4~HWx*%@SmK6Z zBGb`_xkU~i^e05-SC~d?T^-Uiyspy=`W%8?`z?nW$9F#?^^3%el`-aj zs6hIT#8)1?L{Z?456|z;n(Q*Bt6ym{lN8pMPxH&8&Q{}0+>Vc4^~#SZxS|>(Bo8pE zjtE0G;MT6WN`Rsqy7=vD6!VvQgb7yXcN`+THxjdI#T4QPFH_Ct+w}vf#D#~0*Lazk z2|3!EXQoF!Nc4QWDeE4hLUjorlP2b62UjD49e)U~$C9QJ5G{Bf(K2}IKc%3!m$zp$ z9)hNR^=0$h2(Xh)q7Bi%Oq9keFwvhZ_Z#~RDsK88Bs|%hNkeAW6ROmB3~Z@hTXt?r z6X$oirl$Ra)?^WE)i5rW;J-YkVnq|7O%iCG{*_dDQmKvmqfL__vBW56Nq{1d42Gt?Zc>#&)oO zhLVt9s2FZg`-NG<=ofHDyYdmTQ)t7F*@5mKI|HibIe6vzC_e%THwl%JPzx>flR6>Q zLjdp<1zWm#N~(3PaP43Y2M^xVGEl^mA%8ys^M8EVp!~sqR-;7t9k=_ZD%XciG%a-t zY+xLw4cb0k)!Q;4o|2k#8}MU;=z7hvb0&H5&(lo+NhGkhYo%u;!_O5>lL!Leg@>c6 z#Ha2k9OGcYowI6+0wj#p-%z4StN$V6VKx$5i}euGwjp`l{prMa^_&1sILdF?oRf z8|z*m{5JP~hhe8@EK>rwoiwQFDI zmlRAl89on?wN6rmwT6XpKjmd*vO*U?X>_{;4wKs*>_TiMgs5~-tS=msyUIYDe;omL zadz+2R+@C@bpGnbiN-d>af>b_>3m+U46ujrL!`KPrUb|7vjZ?w!8A_8Wo{TvuHPuM z2*mK+gW=+1baLv={=5k*%Z%M|9tg-PT4ncGdQygp~eVGxREHL7^80O z3i^`dTDEdX(cW_x+`_&QW)Vhy=p7YMR4J!4+QQ&go+zRU}b{D^!Z)3bXU zGqI9VVNUhi?iS%?Ti5(TuV$5?b^=aJ)*1gyvT|Al% zQazS*I!y_-$i30DmSLKAG3Md~+`)kC6MyQG3ea>C6W*g&#qsrx*#V!;WUXx0L?7wt zDAgsM%~mLI4~>uC4^-?6L=pCU)8iGutNpg&GbMqY7W;DI3j8_eM+Gl1w@1AT(ka5T z6H=H8+=o8mJRwztV)cBJ!{l`96a{h#32x{(;6jhnIn);>1Usa9L@6Om?Kiddq>5zciEDq7`2O|5z$FdOX4Hu~qx;Bne+Y zb#_chPUd+>u6VZdo{8d72Ngy_`i&e%(+2YLm&o{d!=VC{Xtk?cai8fxD<~u~z+VX}@VI_YldTyN^vEbfq&FtGf}QJuz`&)dN)A%kKFgB2 z9}$z^SP(}QpnXZxJFQBt&=;p)zQ$+W5j3IB}z&In-v!xao{QGgQ%jH|V33!H~0F zalz;|5X5?~;~w{5a(e6c$gEvA}}0EIu8u*MHcsqkPiwAs#X*bTun?VW!WnG&8R zFaI<%39H_ahA9;UQ4MB3g2$MHQAYp2ND7v8N4i1XpE4jKu=yLhF18_jGJph_RNEn` zTbH^dvPrM|$^a#@OaqYWPwgGry>P}0-drvt*Mh;5PL6e)4i!aK>|ulqf&eDL0N1*X z76U=CZnm3Dx~L1@S%4e@tTj8KS-~_khg3eK77v^T(c^o<%HFrTeE4|@W_d1S|UQ>3b zIVf2zT=tjd@Z?$B!y!Og@!V~C#0(m{DZJ{!M6{s7;U=H7E)tc;{9P0K!+;BG_-EA9g~&O+!W%OCPJDK&`6OP=g_Ix$t}SwI>!czvjq=DAIsS<5IqvB~vM3IB zaIEjn1Rty@dk;)S$nWyiL83$6?PFZHD0{-_!Sa*j4pm?91CCiC*x*08C@e1sL`0`1 z2MRj4V4NK7P6EMkYLp{=YqUK1=9B+v;xkazabN7x7@L!FW!v{eg&PDj$U!X|xg)ru zcR6y<%j9$4pD>Cf+bvZvjFTVGr1L;kJuYj;!vIY|f^0csTmZ9IYO~tnG{DkqO<67( zXY=`+r1?3%w7!IiaAq`#X^NE!mU9F*$A!ml*f4F)K!VRQ`5KnzfNw+KbbXURkGcYp zCvcOAm**(MC+EKYy27DVenl9w2?N0c?}tn7+ho5vLkVw! zSrU^RFBq?18JAgnTD%h=FOnkQ2Uff$mLwn9~TgO;-`Z|8V$R82!0kYUPWWe|!PIYo+20D-`?IiKmvoa8D>@7LIjeZ;mG=*xsh=y@{tH z{Gnfru}Jq{+;>=P)YccF)knw{p{CPS$W1-s?TodwlETZq4s6(>j0bP|TDp3%$W~(UKh4{! zQIwJJ48q)%abpRBA2a?pg~+z$2=P_f><0Aua(sZ@AtosOyQ@j!(&UA}}SC z=cBb;P?mqQ*S2o~6S%C0y=&NWgOR7nEjDhF^`M|ddpjY3@8!Zia`yRA1lwSh5YSI` z6fu#@#lAaYux%#b*3b)dUl4eJE26fkW#_sA<;9j+4BRs74i$ZcjZ8=^UlT#lAMvNa z9gJl#V5Mo4_F49|f#>!ExBp^B^PGvNXlW(ZoAQiB4hW|)Ul{E!#+5ENqL;C>)KAE6 zQD^ggbsGAn*w44A5?@8`Zx(`GUJ9b>|JhY08MtiD$z&|HoB;=EKqv@_agdp=|K{tm z_}UHM2X_>`%!GpWqN_NeG^f(A0m-)Ytq&qq9}|C(q0M&xVglEOY_76`V+~*#Bz$+< zRMpa4TqRvhNY}zOntfLdu`up-xhE~ame`Fe0Tne9rXHoxN|vR6MJ|X)$hIz_Hs(I{ zKD5T7Ua(u1xZR=RY4(fN8k#T;$GG;7PZOjS?C*Z*S7rvQTWVp9W098Rh3++A;~l-^ zXNf*sconTY0+qa}2EJ99D@}dp+@eAVnXhA}hbc6i)WGo9?R0sG>l)&&2iTPe^f#|* z%QYNMfM#_OBYXb8Dv|<2WmtbAV}iE^hxX7OEr*1-bKI&;t&bfmI{Y`Xl3J%jF0f3f zT_fjH4L6+LMhTO9?lrX4?!@FoKG+t?Mxpi0~cEx8UnUd zYr=K9$R>&8%L1#oAVWp*U$H<7)B%m#DJS7*a*-I%0o&3Vb6@iD<2-%DrnTHG9)|L; z8WJ62=Rywwc($3fL!lE|fZ4V8)d3kl@AAZEF3pY!F$)H7Lx3Q>v!XIH29OT??i(>v z4swLZ!$0h)d%i}tas=-)G!Kf3jX3fb$r)qBBBL&eSN;;WWa7koa-{!y=vY zlyn)W`-sMfQh+%NXoz5DEL8TAGqNra&mM%!04;|Uc+Y4L zhjTcHuwprCP&a)w;B`PcRs0L~K4mJ{I5E0J6`egEv!z~EPc?kS-})~v2W{}<_>HwJ zk|QT+#2K}P8jTjj5WKmIQmP9MBV4~80cl0rA{_H2aVEE7%+Gb-<5;$gaaClv zfO=GqWZ7%s$z(I2559;`N?KPXw zq2&{Yc>OHcN-%tRdgEhXN6PI1!^HJVuGX*kfp~?~)g_lUZ2))*{oe3|G5T@Kqu438 z95|ko==DgXY$viW5s!25!*z<;!t0ya!E}M>)W)-_fCv0)tfeIKSz6W3{}fBg@$0} zXY%<$`ib}l$eHSs-G;$wF!4NNCr|@^{FBol;ERtKqt=rD6lP-#PBLoEl>XaKX1Ve$ z3o9T$^YGKMcw)u0xsE`R$gY$DkL@Xy%iME*Vmk$2Q!UyV6l!xO#lRi;tEY?_ zOXoQor72z!s#d~9DD zwjgjHJl;2}98^)?2DsFbghL`2uc`B%+_0 zq}Dj(D_w&Q6kzF{-pWh-MN8BmFN2jn(T)=f(7U&dS z8b38tJ6O5Y3STu~4&Qg?gw2#rsS6}6d2LWq_PX5Vq2N8|XM4geqSpV>E#aUTqYyQO za3&7f9v7^2R~BM0l$4Q%c1hkqb=hllm;LdNtpif1z9bpLUVFfy_mnHlf)&ZNPa7(* zwYsq_>82czvT^Ghqg_A=Ack40waljmQPAAi};4aOo=UT8t+Xo(a_KFyYu-yi zOc&q!V!-gpupbaprPCSRrU~kM<;EU|g6II0$=jwPTSRfiyu1etyYU*mFzW$dxX4Z1 zMXCrO>^aXRV&wA4mX%IYYhOFaZi_}{O`vX^IDAgV6^zwp(zwykdTd_T7{A6y+E0R# zGWs%s34Ohm?$3B_O?cRqRIKql_pQuz`#S_@xdD@77 z?uXn9=zTfc(4p&Ol;P96kc6g2F%t&sA4KyY#;*qn-_J)vT&vR*Ft6%DhI@skh5e^v zMyfjJB}g0HD|qArf-z!#avcfl8+xyXFxIU3aC=1|PM^_zokoB1y%Io2i>T zhG3g2Z($-)1^D5t4&oob$`%0%E%IfO=ut#P1#9ZY73Rs(1DyXG$L7amRdffeha$VIeyPnMmPI-#~O`&W_5cast=3=*AHI4rfSc**A$R7 zY{hJ47w>vRR@q+-%guJ(-v?beNyZ3zH*g-e-)kG;P*!fhgm#J9XSpln>v*bY)Mw^Bja(YElj^ zBa>5B5J#9Gg9d`@sdb>=TYH;E-v=H6*M^@ZEi8xy!6@kgRp(jM2b24EGv`Y@9lItR zeV|cu0MCB8KmsiOVhC<#kb}U9;Uo^j;9PHLyxHL`+dVQr%5>0JaK=OjYRAr=m!BN_ z(2!uDA58Rii%w^ezYwQ2`S-|H(!FM~mC%e7%=tR?JSNR+)WX;H(yjNg-tj-4tc*T4 z9t#Y(sAZ?ePp>L_%xb-s+oS1-BS_R`xhnoBh{sxM-4}HWWu!{fTJa)z;+3Eco^VYm+kyxo)r;jg`z0aNo&t z?HeYzQkvrO2MPy5|T6d>S!X#m19uqRHfdj|RvwR|baOLXf1bWNUCI=QYHV`Uo(I@oNz3RT*S zm#U=B%ITMgiRwvr^2ZXhdZN0`%x>{>M0qrd&iB~3(qatY8oe*Z)R8%FR;O?Q_)j@Q2IbkA51Ih zCEq~jsIVz9AEmGSyvrYeZg^c4&>D^D^%M4EHy0w7t318gf(Q*swAomawK+_nvwF0p zRcM^uJNUQ>$nNA@tIe0=<#|fuL)643p_Cbghnx|)p_IU+5Puto<^tLca91_L$EwR4LTE1IR~LPkQCjqtcuTCg$;Zs^K;Lufg;zum4;q4$?l&K{ z4Zioc4G*LPAH|FkomdmOwk_mQy!bA+Na{T+wW-oR+KZ~-;JCkQ0;C#&wl=uF zRW=9HG&}vjEODLQ1AmmbW=ugW!pIxop&*SN30~gYFBqwc)yIGEJD;AA~#W2-PW$mG?ioerL= zb(AUFr|_`zoGf*N<%<&dmr8hkP;eEkl<_u^+bD3enf*Z}5g}KA7IFVX1Nj%~BG6i!P;7ibB1zxRdNGD)E28P$36$)5OOW_;gg;}j8s`)f=s zi3bFoBjH&{iz8*-P#+dBm?FxT4US9&y;#K0fa9t=d#|8tvzw(jInSPaWC%9KE*W>W z1%w`qSVgy9I3@ivxWC!RWDc~=!C+&+vd-m<9}f{h~iPjmaTo6Cnk^Z{|@+hDIUkw3uph`_bvD?E@du=_xD~+^Xn( z19ZD_2`FQt+F=v=xp$bpUQe$UPyx72yv6RHC0+sWtq5$`W1V?3!SH>3F~{p%Z^v@? z1zpfr=-YG(tyU#m%iM|1*`+NqyTMG{m(BMjaIDu0ox~?jy@>VwC&0sb!bk7A!551d z)wX&7G)Q%tDFZHgYQEZ_-C#G80Enimk&Q-0r3UDaH&~ou^B)d6IvSeq?A=1p_iy~{ zGttP{Q-52LKWqz(`+RUEVFor@gT}(^l0`vZkOc2A)}N1X{a}h+KF0Iu@Zwz0FU98S z5v9bly6MsY15*4zN(06QpdN%gNYqT;t zz$5wW=^vCyd@Brc8U1jRA#WF{Dwt2`u!N9D_JvNfrL_ZvwM!2rza9*1!L)uy;%LBX zkHW6&Lf)JW!9CwyVE+ym`pjOE^7FYVcNUb`mr76$twkk`Mn3^8BfAhrLeR(C+Z~=j zDgc@ z&o{t;;CN{)1LRsKSUvn-)4mERX%| z4xKuYK*Zo}*gHKc5HH!82h{a&9GKdycvvN?#waYojdlQpHfT@tmKaDzie6yRCJI!(CvNZU!cl^)@@ zK?D^u)RA(Pv!Nkt!mMc>Sp4q6~Uz8@1RzF zVx`(ygrFlr7VUU#k9ekMdO_Md-C2~+ql)X`UX4ypo3gv#xY%9@;WNm7+t+&84yNk> zXEF{AgQ~wijQeysw2%M)1@Ov39)}ErK4G!$_29c*$eTO&RCVvQ`2EgYhFUWrNDZqv ztlRJOPs8qi0lWS@mGvv#taHeNu)7dOecB!cGm3mmuigk^5|y40PK*ny6psK}3$j1@ zu4KOH0`U%E38-(C12|0yqn5QSS<=;TBy@}B1(AC9=c-gqGmH^0eKlY=A>v3#IRy-x z`#m8h0YMQMM#o#=BNXt~L3B5ufnkb~viN^G8yNo0A?uC5$5}={cMVTf;IS0ExkC?{ zG-shlMO|aZg4(DnQ})t_)W@0_v7gDMvTi3jslYv2!E7@PSacCY@jSL7B%U0AqCbwMce`F z5G8k9HPYwW#=(YL%1X`x@=_KnhXM@JaK%<5sxy3GoY>}=^-BX1YXa+GT_X)ooi}gn zKNhlTk$jmNAO0krnGG^A!pbhumR#=5GDuAu7+SZxn>(EIX58Q z&$_Q$sDRL~B3jJ+ySr&N+*O>aBCHVh9>ldSV+v{_)qVs< zKux1vK~yX_{2P+)3CJ*Q7C|aE^2qZ0wl? zY}8AhB~LJl-bhE+=)LhwAfh1859|PheRz2LON-0y80rxoL!lwb8?^4!rD;was;$73 z*PisgoJtgj7Cxkdr_wSPO=@zonE9DS`X3;*SF*#%-tJD_+>c>7Vf`|QKx>HyPL(a9 z>!2BMOJ3c;ULERcBH_@g=8l_^(1L(PxOy!pymThLn~9PU>hH66ogy8#pvsjOk-Q?Y z$8*kMv@a#Il&bLm4|!ICmD0y6>{R#JlD_sI?6~5K+7mOxji z=z)30gb^8IPBTjbVq=%iSg%%pjmi{RMlOs4O|5E)=f>MHCIL`*__fVi0SOa8_Fvru z3^fkI^!IHs+e3csHl7cFnKA~Y1`Ts|wU>9Uj%jWGsTN6$Y#>>(C}=gcD3C@TK4^td zG<%n#<^DmXEpoJ1B6OE6f`I`uHJSGy)B6X~g+M8JY?w4pkPx8*MvZT7u&(wk+u$DXkcFr}j@RZvyuD;p9QpQVhHz3{ zCvm*U1Y}Q}gY%1PiJ!mjtG7QDy<#70i1h$FK*Ya@^YMJ91P4!}_%d6aI36j0wsq3H z(nS~}1R%ur1^o#!Ok641_u7Vqyx9p1q_5z0z%`6Uu6y;iC&mrzb2nin^41AWb1tp1 z5!%re#?+Qo4RX41Oid5YHHgJ7REK zQbAYp;$?!x7&-Icww&f|&{1K*&4-1Fi6CP#awa6&p`1IAbR0$XB?DoQK+%)|bDM-9ec zz3AgSg!@YJo%EyG{~qo8*jl#+c$&Ep)>p$sK35N=DFElh(h$gm9Z|LkG`EuG*6bHo z`9H|rmQ7LqB2Wh%=lO@%VF*V%Dp(4eJ#t`z=OwW-C);WaginR>`#6e70Dke9SY=uGtN9Qs}BGJbH^+b>+Hb_xI_p4dXUQd0vkgSPV5Dky8 z?4C+pj zwKFul4jR3AmB7YGnSlEj<~|OxqkY*6*;Bgy7%QThwK)1w1%2v3VHP$eki^s5yG3Co zZ1oG59GWgE{l8o(nkq9Bz7JgH#VAXkdiDix5c&bHspNfIdD~Pp3j@Q7wv7D;DY9Pu z)FV$8Rh0cPW2T^FP_$e%(9SndvHqfRqGBJ~;7ILwzN_(}-v*f(35<1-KYPoe$;M)A z&@6;pXFut!Azzfi0|rNkI^3vxr@Z=4(F*0tjP|HYm~~njPHh=S%d7qH@8%C*T6pes z1B};kJrUg|v*6$M7sgC|3r^TRlYev>BAsMJ*e4vb)l2M8nuS8bV9=pZM?;sYC~hd( zak_taVKOM5(w4?`IwTtE>vG|wuKUE#B*xHi_fIszPjcA7^pd6C>+G3~uGRyN_HzJ; zurV3DQz*Rl&T=el{*v5nC3Iy`yg#$=UUb_KF}h<2OYN(m&dY-qdcc@4xsnfM&XGzD zlyxu!0g~uTH@bD*U56N7ta{6pivb3BiA{P_KwI2v=^Xi(G3Xqm9R7{7JEC2eYHxZe zgSu=V>tn~W|8a}v96XUPxPcE!HHZB;N>`GF^PO(IdZm(2 zgYP;d!0S;5*cw8!*sU@2{!!$ENi$tb`8ThcLWSq95nx^Cp(?s;Mih1l4iCbn4ldR- zIH6^MUO$-ys(pu$@(7io;0ngWl*LX)JyNnuB)%Ri={n92N*U zw*hdAesYq01@Pvr**All`r~o($q_Ej&u&~lwUHpNjS26g97m`Kt^AuPTc>aeg>+4AMft635$Asy$A?hNfKx` z-?1;E$s)N~qNY6ZX@J8zbK~!KOPTfu|D5PuK+V`dL!*JVFREp}{6JWWBVP5qjuLt9 z+9997Q+k=&j(tB^nw_JbX7F-v@v8c+=&W22Pff1KNi@lV_8&*KEz$T-)oAbwKC}RJ z-!l!RUduOXkerhUJiPe}gyAv~VLHN<7Q7LaDVyqlyE zm#57eXX8v)dEboaTeC%unW>mg!1mHo1t_9h3Lw<`ux2X^o;rL=Gj;0Au>i7@q)YkA zN$B-`;~Q)-Bp96J+DME*J*H3S_YhTST+!VaZEYmy%+KnPDz&)?0zOgk2VT~qR)W=k zOWp}Plq`I6K$Tr8L5KyaqvN3n*uhg4K?};4ndFD#@h3C!D6IM3wXe2U+)S8Jb#%fv zg{F-N2+{Bd(%bj-atgN>ctgtDE zarHaYVQFvPj7}X|$nCQHCXA?DRwW8B>pM}MGY<$Q4$v;0L?)SPSCBInRwy`FZ>CN) zB>7WRI*(h+w$*}-;BH-Ppqaxz&!)tcccp{Gw!Ly~Es~ZCb8$3!7(hy&)EyRU;Y4yx zrAEFtx`+AIF%WcUpsL3n-cN%&3OGhOZx3=HJ*t*R5Kr1;Ex`tH4O!b)W`mij@BPRF zW71|YIC#nSC?UI*VmJGCruKGZcfnHStK8fvH~ zSg~;pL-3xuL@};C!{3p{&4&NO^7prQnlA`A557%H?a8+zxYg-1){4U?M^Ah9dWP_V zSRjMW;tVB-nP@5ZA2W`+15WghpJn7AwNFjA0p(o{zC-uU+O8@yfJ(CDElUBMVVaim zomOj~F)KK|v1x6+PTVWc0Tfm*k@k>{hJkJK$_Gb>@t9A$+7Xz}zlEJtLHha44AR+{wd4ou2CI(>|3@+|_&MPxB zeK#M}RDZC31R_9Pz)#y{@`UMzg=%D7h-JS&*P@KSzjLw4a>8O`7}Mgh<9x@JMbTi&C-*FB~_ zY*&K{ImW{HhTAP;pYHMC?P<+U`fDn;MP9H@>AQezG+hr)p4}X-?OkiMK?1T4wM)^4 zmZ0^Of|*Qu@#|y})g_8i5f_0k2;DQfPplfxMtrY_rm%iWPp{$M!N_0#tqAMK_|*Ih zYA5$Uyb~UlT27bF9TNi#HeEc-y1MLWX~NnxTkArnM`FSX!uRr z!M{On#-D8?*}@)?O){%qd-@*#o;E5n#Fy}T<5Dv(Jh z^>U*HoW$p<>J!I}$rw33jfJ5cz;AhRJQr{jD7(;b?*ZW}=%$d?+igEe?i>r$9;iAP zpyI@wP|Byv=%J0is`-m-|K0qo zDXwuh7|`_gzt}pyGB6q1RbX>5y;JwcX3k%ZaZ9I0;&H%Dc2U;JuuNyk3tG3{A;}Hv zd7J1<9c_;Bf*q0MsI9;$P&0J^=Rjx!`+|D33oIV@`}MQBx0lyt{E&tZ!(pJ!gA_a3 z1FzFA$C+Kd6-lT7!jtiP(QcdDP(Tve7-5#fFU~NY3rc4=fXf%bLfIVizq4^!XW}#*U39!c#iiue|X@NZYfjq+f?&!H?5fW}y zs(82MFflYvi zjNV$h3A#rjfzG{=q(7(~Mer@kyLe*NgaP5YQ@;^0HddX~1Ja_6Zh*Cb$#1OBK@PBeDuWYQY3)i~qE zmHRx4k)0o$I>1Y(w^spu^>0#dI@SHDx2_kS)|Bt9Uqi{%K?Qk_*lK)zu@W=@5HzL4 z-Whk7uw+;RbdV7TOYD>$v}?x=Fv_@03f&naxPzWCA1HgP>F%tp*8p83%~p$b*UY~s z*;f5G-zjNzrwm4%($zy_Ur52cN<28GjrnWMV5+9;IXSD#=3RAAQNZZ_g0%U9Kmn3( zVFl{NNeSWgr3?$&oq(=NGV1Hz>ZNk0QKd!9^l$^_!a~<6HwzqeA7`p^5MF_$@at}7 z!z^fXu{;eLF{8Q{39)$y3JVHwBoZ3< zi-^ZtmwIjd7~`O{44KxiI8PxjDj|&vIn(TWH}3qOh|RWYHf|W!WEQKWlblpN%Uob| z7Ro;PG&_tphd;MqU_}s5nbBeJ;%>;{V{b`n4hlwQ)jhZvo?C+3EO7zt z4Z`YSJkBFf7x3TTH&=?h>fq2IO3$m!pcIIRH|5?(56)qL|m}?qfgJAdHhA zrk&~5=|*loo$#JUA?lRxHs1g8t2rzF!gnQVX&Bae8XSAEoGS2HQ#dHMwd%ALIjFb+7fyVpqv5!wwZpy z`J?HJw#CtXHb%Cv^<|Z``4z)JuI8(|={0&=+hExdJ)SBK17uW=KT`p4d+d3W!GA)$ zpV!gE!plt9xD?xj$ighK8_q76c7K><%wc8ha&x&6!NfUJ3ru+6Fucw1k84l>4aJzc zC*|$I7U5hq2aELQZf5bda%?RK75^vC;Ws&XMA1-Me^M&5kLWMnaQ^?Pn?IfC8?CCe zsf-Xy;QuhVx`mE09iOdtD5l!ENaXC3W^J@Q4c5tf&5kSr`{*!f!jo66%IC^R;%DD{ zJ3~Z&y!dkC5Rf%3lt;@-_N6x|VkYH_ZD(O9y(aEVf&OuVg`SX!3l5hl0a)R0-N!B2 zrGMG0SR;|71pd7pe!R`5-x+DmSG0})@axPl!lM94c07ISP@zl%!wtsH@O_GNO0Ai; zp<&@f7L_4Oqnzt`bOx_ZUU1<LptOVt_3R%W*hxE6u--jo0Ra{$mr-dQB42QW>ARriaq% zA)vT88wgl{AEARkLv&YN0~-z9n*guh135o!W^LkAh+$Y>2Q9Iz^N&YDp7V;8sw5FC zQG>=l3u-2JA`>>iIDU9IpUiLEB9}!y-Iy4sR^gTM^)$EaAPr z3qMT)Qq6e`YKqO}Gnffd3xRg6`NMn3$2UubOCLVNR!FMy4+zX1bs6?Lf$Qf<)c768 z_Z?qqWxExOLoJm?ijKXz4A()3hT?S)O6xfq@DMmv)2tJ>a^LHs@H#1oz%1m zTMd;>I9(1_O}LfFv|?+1^W%S)Nk!eH*M6%{7*>E!Vc!suy-<)Qt~tL>SBZEx$2_QeNtqV^;-3!*AmJ{sjEH5DTkXRedtD=$3Mt zVT3oRZfGZSG%#Lym!oaqM_+0#4VhJHQWWFcTYg_DE??gX$?tw@W_PNIeK$c+a5JL= z8HU^HuB+GH+JPHQ=F5350DCSwEoqhBohx%nW|36N4tZXoMu@!_VDOs-lIUy|7UeV) z+WhhnQ}BfDL-H=kR9Op_zX`lb8(X(jW;qlB#({Bqwu1sV7P?9+cj8ftrHIV^X*@Qi z=|(l)p6{|fG7Tn`K0r)Rq5}l`^epmQ>DgFgl(gIfS6)VFc8jlEN>Ojm4g9x!a1Tf- z+LcDG_FDvTl81&+-@B%GF{7=3%ip6m-aj#1Hy8T#A^+=vj|#N9-_bvj<210$ERJAf zX_z~xlMHg-bZemE5sAb5gxp;Ov%Zh&>U)81{~aJleVA7}c3jC%BL`O#=Tv{qXhIzSB~)@fz4Vp0 zTxJ?VanVt#Zw+bFlTB`)Di9*y1?xtAacXM!^xy1Ha+=zY{n||`a3IO=@z~uEcymTR z@0>ZI>z13i)kS<=>OA*U717XYczKJub=>bXi;%gAM0SLg^4=*z`U==vRpJI!GYgC1 zr#;#-h%ctfHR~|cJ+MauVm)en)Efd>*&>;q;uw(YgEq)7MoN0)13rUi)cJTAixXjG zd;Zt)Iz6}se~6*QAUNjn=0tJFtr!Q9yuaA-6*@hiSg)MIssEePYg=tF2U%Nr2uALr z6h?#fP}yEYpa;<)zuR5%2zafl|H(>E7R|={lw7f#rBn1jTUR&Hq|o=t?h|lMhlKLb zB~pQrdGZ`HJUI5Mq}J5L&LX2ErD1-SoqY4W{JcUhtca1&&LFY3$-^CcrHTlDL&J%& z&W!8_io)3C z?iK*Yw4l$)67egNR(*z$Pa&FYQ${2uN|$yrp5%kV<^F(Qh49enKjCp*JMSY!b^=}Ovz%X3ydP+erCh8S=D9y56UoenqUb_Thh zYBE$j7)N@KhZ!C!rEFGjY6MeX(paG{u!uwuQ%nLx27sy|S+EVzg8S~|_F>z{GmvoI z=ZHi`D4?1Vq(%FT@@h2&VkK8fHLEYD9f~y__C4mslMRc$ryvBUIWtBoJnM zbK8kCXRBK_Y6vXpSfFT(!^alSt0#ntz2D_`=`dOi-YEH$&LOiQD2K>9k=}07OBQ2L zC&7Pf4UgvlS~|EC(I#%Lc6&%6(r_k*B){0Ak!grZn-vP7lSW5OXU4VLG9~-yBKhiw zq)rW(rrmEWQdMx0VIB%(Ni+=b7nZ#p;A+&ZUv!Zm^#x%iH6V;zMhnYf*ZJpUO| zKFOi~LWypH!W{L22^`Js_)jWUeD4ii)3TF@_P^m~H&{_SECMo-`xtvdpUzBMhK-^& zf0aYg*{~c5T3Htes33!bonBGVnf|vY`g!3oZ?g1CGe5dkeAFDjRK=$CLJp2Qw93nT zrz-n3IERH+o@0P+Bxc`#L9@QO2(Gu;@DCn1Y}^Cx!XY1oKCz>c^x$N|ot3vR|t*4UbJSLpC63Z6wYv>`5FvvbVTK+meI$p(}k67kbL-Pm`bgk>DsCxG^XquC3Ll zP&3p6@_=%6anv5pIgk36q2lxHx~8zMZAfx-L-}UB}VO9gR0h zW0b!Xiqt1Y*y&HQeKxi#xp00%+kvYhhY2`)TXyqhTS{nQ*5W$Rr)pJ8ko_NH$p|Lx z6gtCKmJMs$L9S6(EjW#y6xg)_08k7}bHyU&(RH#L?al&vJr~9)|(wNP4L6c z?6qa;bm_n@Gi0zLq`WI;bZ@s9^9pb$Gt6BDr7G(wJrz6~r{1`$^QRUl0-j9Zh_@Z{ z1qzxLbm?*3U>>qqDj*!zCQMTDpp7V`H2I4MIQB3Z0M6MW>Hd*qTf*ytg8B&vKArt? z2qEY)0Hr_lseEg^{OFo=T$=e_bo?V+7Ju!}5)&ZMk2St#Ev<+|xx;ZmK$~azM2SA4 z))|847gcXbxynKp*LP6t<{y_DD{xWcd+=d1LK4p2dlhWO4WcBMdhlcRRbffihb^PO z-CRl17y%0`_~oqh9-H6*C&FQ=s3mjP2*GUtbuSfZv1h?s`Pg7J9+cItr}S+ESDIs} zGd;cl?7zHisiyC`8lOZgAQaa_1|M<#t;_=J+dz$Lza2)a$(3JGBupgmDaD1*>p8f;_D!=IYFNu&DZxEy87 z?4?p6p&fxAz%>M=J@E)j^T86%E2hEdv!Mjn5WAN-5CqLJq)dkVmXPjxGEJ##dtm%z z>KxkS^{mUqsIv_YY3I|DcNEM$<1=|91UToMmUe_M5W!YMc@DKGv_b*d#csMRXy>JbeT6L5N4u)4X# z$cki{j8U)sfQ_qJyr|Jh>0xO>t{eq544)qYUme31!->IGPCK()D5}UtCxGHXsCR4~ zEgb@6vM5VJF+ze0e?!@r67j>oq@T$`k7|LUOPLj6kge*7uLjtUA?a)C;KiE*hy^xe z5!woKT3sbP45xlnRq|7)hr`;@{&_(=g$M*i`yQHypJ)kx;pAxpdZj#pj)DKao1Y_B zY3eK8*VL>TXW9?Pm)AI8Xe2V%crQEL)^Fx~~(u)It1UunVT*P_LLRP6ZqI z;2KLad=d-H9Ut-cq*@*i=IBxyEvv~9)lq58ZW!kru2u2qq+6o^f4T|>LT1BwberhY z8_P&DKe%O$ZcYv%Mg`UGdj@de)iI$Z?O z*C+U(xFhuSiu-j4>)AWjMqwW2IVq}*_Y<1L;|F5c3c%Y7PJ>o#>xjgTd0PLv&Jb?U zzFquOuBQnXf;_W3o2;ruqE$^1a;A&{Qg-k3dTo+`nO?9X0>({lY7_<{KH=z3O__)y z2`VX{6oDV^z1dH?B#L!4PC2~JWk~3)OZ#u#y^dW$tBNIaDL`fVWrp1@amiDzoJ7&F-@4&W zgT5?l{cSsE@7iJ#onr!?4n5($cSBEtp|PrsCCYpy<$VDB>sFC!3qG+I!_KAs`C3va zzAH^P@=X_Vu~rqDqTwzbI_-rb&vH4U?Ik3gH?Ms~fY8c| z^oM}q_jGt!mODzJO8=kPhqt|Ol!}5fvN0mR|om3Zx-r>O$$ zTll&U;2e9#TB=rp7uGb+6eGB5pfuv(m0uPLOv9yt9z$PXf`3(30b*|gT+^keP!s~H zgEDGGnt^LBN9FaBIBVRq5nL01%ocINjz%prQ=?5+%UM!ytue94H(fy781)NLpP4Ot z(G$t*Nksbm0*4ss555-a(C2HBY0SPq5jG?W3Co8#Ch{+8Ci7O2p&IvO)c<@2gzK6> zph6E$I-w;EDoF8jT%(4h05%aJxt-#fF%8ISM@^(z!AK6Ch4iB+F&!|!nLogq4I`oy z4;~Q5&y@jv!d5r&-ZA#T-l3M_Hj21kcGI8@eq-xD>|=>vZ-YC_8dWgl4k(Kwue=6Y zR+~H44kmzU1Bt9r{tB(gNTLTHX2yuuSO{EgxN2uzS1i*}4#s4Jsp<1a>y|8+xA)jlkL?*&%kDZ$qevest79K=A&K^x}m~iN2zTWVCI0N?v*Sz_$Ou zk5Fsrt$SA`ZrFhRg{H~CNEpXD3J|NRgU5EO9$^A++74SC^FVICS}8CZ$yBGaCLR4; zo0l^NxHVCyhPn(m<$fWK{2%jS2<_Y~$K<#fCng~#Rkx>R+NdMX(KGD(K}%4y=9cu` zQ!u8kld$EGF-rPXJ6UGF&RLmY+h<0h`l|E#y|L+}v!a!ra0+p5`DK>#e&joq9-i@n z0!+$fx9cwP6?h|PDlBxQu6B#I8r*|*5BhWm8EZa3Dx-)`$Q-Iw6_*XIKaF(IrgERGuw`P-R_M*acr5)*a8M>>)ki+YJk-7VZP59 zX>yP$k{u7I)f;$rDDKB zDbJYRFtp~E{=PvW2;pWG<5f6<; zQZPNqblP(V$Rvg{&wcO?DjR#Be)%7#<6U3jGvXN39nhIKGwY$G-I zina=jmnYqwvIGPt@X@Xt!N1cqvrqu>Qf&y#;@KXu7QnwfSWDinbCoYSlRUeD?Iv&9 zVP!tS{~rdLQ3yvLIp1}m5+RLtO!9JnG~|he9e%*3YxlnzlP|vGfw+;8KAA(+Df0z)Dp`c}*%*sq zsajtgu5{^Z(=|5j5mu6h>Fv`I+B9=-;oG>I2%)OM0)GKC%mF4gy}U*4cbtO>m*7A& zQ*{Q0^ltlUfVPT|m|A`4q^?gHSv}Y}5xM_!tuV|`RBE|L8vHfVaIh{;5Odv+@J>(V zC?78m^`4{_;?7mBRZsZ@T!!h5f0O1Pej7Cpq=V*;`EBj)){@{D3X*t&{)nYVIwL5x zXx#0>d=7Mo4EZW;IbZ$0=e)g8ZO6+89SDQ`UP=s>qCHc*s=Bj>>iauCRpHSN>;{1{ zy76uUkg*sTL)Rj%s^PIPwLymYQ^_rc`BTAWpi03_AZZfL?dxw-hmkp=Zz@MJ zCLHm@65e&i=TSCek}co0$>wN4bi@7tD#nB?!#GnHbab#a<`R*!aN076*sJgrD?=>PUdLd^uf$8N zhU^*itRXM&P&5xD1*LtwH(-*=97&0bLz3H@37Tc3I9H>y*%zMo9oB1u8O?%gwN_%U z#!;G36Ul0pgx_-6@6BT-@amoIZg;QE(sOAA{4^8{1LZR7QM}}&N4l3N7XbEs&+m>S zb=RERL^s4d5W2;8}-1G zTk!w5H^P9^!=2HR*=eW&T{N>?2OTrm4D~UeFhXH|ANNOpCvJDDe?0L7NT1eyjQE9F`ysJ(^5zmAMVMNK$>FCCm-u!UnH+>GjqXK2HuMQ#m6qv zuxcDui4s`{NN72VChYf&HzH~za3chU$=(-Y!nHenks;Iz=UJ4{mqlOJXBVeErL)hZ zUW@>0SWccWz}FYT$w-j%lw7Z2dEzpq3TmJc>5~^9Glzr}y;tQlf zOtcRssiL`BnO1z~XnFTBVZr9F@UsCUB(<(-tvL$`MHAm8iQP+AD+&CPd-`j9wcb?ck46Np6jJN+a4Wk`wt=V zv%QdB&9!$n39yf6^&`qg373YeTW=>g-&fIp>xl9uS6_21%zI4<(P2xbuMwRCXCAo0oLvgsTUVc*fx6Zz%s+5&Jy=EqIAOgIzSk*Qfi!YilB#XoA+ z01ksUDI;{Ai7Vk`f^oqJm&#Um=>;sFMr@g$*|d&`F~wFHLNC+6$rV%R$HCcV4|^MI z&k5g0k*G;^;7^?OiVvC~Ko3Nzmg{zeqRr(Dr~e}JHFx;M`=}bATK#k1pd8Gp@#4is zLM@3K3F%!@8r`Of<~8$~w2WqKN+M*|hl>#8;cQVBrodW@(=-;22tSr1SX6N1Fj;wZ z!ev=e94eL~Gg(d(9sc?rmda~R!}5!dp@xaAI5GJRVLP>RDEWAA@Cdn$kEOlMBas-i z98v%F8FW>lt858^&kOkW$+Syf84%WEEC9>Szy4nUTGCeOS|KEH0qPAHfi8iIr^jk* zIRH#k(uh2_LH$}=OD2b%+kCaCWkBT#QKk2pqGTZFJuM2TA82-K;In;CS@)Ph$s;n_vUt^xVR`Xw`@CB+516-HJW6VIy^mh%aQ^lX*$xG0nkT1Ap8$j<8`*R&#~WT;~=Y{Mb({kn}DM~b2p0`+^T$} z=hVva!0<6+-%dp^`SHF!u743!aDMms4$s^ogNQ(6&db6TVmeq~wPd~DynelJjfmkw z&y{LP>-IwSRxQ5#J?Ovx=gV7R^phz$lGIe%39x?-F94oW(P8P5>33RfIF)L#g7>wG z6fvUaAGTbBEO8m2yBeK*ZcsCesa8t)?;&bL??h&;q<4`H#u>22{XSUz`oq!>!wuC= zu$WZS5Bi45H}H=1HxB#T^nN{>SJb}@1Ywx&#$!&J_XQU6nN@MAI3iM&A7>ah}7sxF#Q?nwf@Q|FC@96T{ADhN?4MoqZuB{!>j!M za??xpU^$#=c6Qh_bf?kMGCp(&(wEZ-B5u-&pRt{`;w;n_*n@$^4~pE_>m9&f(Szh- zhc#}R^)<$uxzd7q1aJm&klmK>2fhuZ`Ims{3WNUtmxJ+I;c<-2A{*+)kSOCbhT(f- zK4o{nXkmb}>1@`nL(tYjNCOfk*GJ4RRD~Fy{3<08fex)+$TkPldZ?Vs?45mSY^Q~3 z?!}Fu;yOkSl>iAWp&)FhgZ&%G-#>Pw>{E49iE=1y)K~o5L-u)s&|FrW=ZDX=D6B%^ zz0E95wF?aBe&ZisGWuq{pH$={fO#_Zxp7! zzI=VhGEa_^H<%@bs;3WNs2#fv=6*sIrG{8cB^@^{G&az5+UbvfUs#Y?$M=HihR~aH zj^8r!-3?#-*50sH3zNLPuiM$>)v-E!qO^dyaWcplZn+pgX0x!ASJN)3+4vsVFgYOp zvV!-Lcm~~kkt2EJ?}gZ@`wB}cBYWe0SATv!ou$UI_v5Ge6eDi?X|ZFE^ijFI-uyiJ zP7kd=GrzcGAsP>$l&u&-!XO$k98QQJyqmkrEyiv|7Jbd%8Ur)=8vn7=wA!LXPo{;&Uq~W(TBaaUc_>CHu_P=3?-}Ttpd>%5z!%6`E-%^&n5Uy4}wkH zIbp|sfnsbDOunpOSw;iklMsr!**lw04inVc|0#UAH(3HE@6lQc$#zG_FN<6O$62i? z9*wqY<1Zv));War3zKy$C^Ai}m=?$ddytSH?$$>TyTN^Vugkty)@laa2Fd$nPA)Dq z{3nv7njh%V+Qc6I)X&(~$>8TpE|Bmu4}5iUO6VGRnh^n<3unFR5H2J3m&gYC5DgCD zbQnt96)gM~#zqO~7AeyN>dve}O%uG-E{hicI3fzdR`&&oC`YjJz*x zfgD=dNw99cRo)~U=k+$ft3x7NwObyddkXRW(V=1h9Yn^e4PJK5kWB@4C5vY@>nEE$lVQkrH!XO0Ut;{a z6Io^_NGhShoZ@%@i)LUF79fthkkCgL5jlGW4-!qgOqFr>FLNq>~4H5JiJ$4!& zz6K!tsvWt2cBrcf?Ep79^nh|mY^{{4XRBUDG=7X!R7rwRRdK+q$pf;h&vuk$!w$3e z%qH3C3_MYPRv`=?Aa-_I&aefXs0HJg4M0INxy;7 zjQnqOwHMWJWv5*v63Q;{Y_I$1i>vaknu%SmonjaTgdoZ=o49~^Xbzvg)3xZ{#t)R( zii;6M)$J2&35g!OKX2f4U%8eo-6T*%P>}rVL3n7ws<6S9T9IJF8)D@oQ&OJ z6I%;++gmdl!XGBbKkBttU*y^F|=KUn7qbG$|cZ)i-mj!_;ScKnoJtdt>8o z@G0CPmI#1c0gm$1r=6&GUqs8^kr?sqPGzreR6%$FHbYy9==#3e4>ET-H7dNYr`k5b z7Pb$^Vs!&e3fqGfZriUuT4*tMFdKn!d?%U96Gq5+(Iniu#)WmxNIeytdCo|uwPN*R zng6+7E>u5z@AjAzSMCuVq%JktELdDYt^xjsb<_tr5R~8TmJqJX&hNPNI44JG^`0f)j#yXfqUZW6%e;1xy zUYH@qz7o5#W8BAkE@q>slv?xWR0P8ee*w!2dM+vd)&+<*hJq}lU^u=aljb?eEQ23Hi$2V=rHpCe`rfX6~r>@mS; zcJ7^9Bt4Tmo=B+>qFT`~>pc|+j?sRSO9vGg6jv*WE8F5 z*B=pi)us((+nqQ+i57#!?w_p+WB~ZJQVO>)xW8@p6y%gquXMM(bDLGl zLT?`bHR0M9Q31#O0-)hgPjCa%3W&qXtqEHl{=pyD#-|AJczK++0$3us7P`>KK^3zH z^mh6Cq?-y0?1mP-&Mz3;^1Uz1i_|XoD~14$uguyJpl4nwpd9}KrBi| z;Wqti=dbRSjG(j&K0kWjDJ|e12vX-G8Kets#N~b{CtaG#QcfPC*BE_H2)1Lk)a(r!qM68&9B!S7v(f1|KQKL4<|Upo z$ks{H2sN*LZj0&0qq)O4Hi#{S!4K}#P;qOHbk8zFJZC6XC5roF<3tXX1sH2$X>J7x zScaDT6y}>^qQg&h5}xcD?Hogy4W%h(SraheT73LWt%v7=K)@%j| zn@v*yDta+XT|>eFN3Hs6A_?rma0R%c0poutN|gQ}fg(M?GSWHK5PA7P6SrbL0Y%`l zesPVloUsLb^0N{hM@md{Vu2eyFx96*PXq4e`LIHy$f*E3NPEObJlm6>O1K;1T|tD* zTr#zB_UIXezjp&7cjk#U9kT_Z-6Kg#Du)^EZOpAa(jDGx;#BT{&RtGZ458fSBKS+) z$P8Ag;acP|(UEj9(u~B#7Z8^YzuieW%Xy1~>SA+@;IGTZM7D#X79e6@XV=^bvEu^U z`*hTI#(UXQf?F2696Nc(90>UZuZ%zlyNJ?CxMzqbg3k7b$8T$kSPY2mdfwPXy0@B$ zH5#YyueE)`;43+s{KAZj*OV@FrwM$<=c~;z$KfT5vmOf+P&y^()oSV0w#-no-Mq&X|G(!8vBXQ&Nq)`h83bzl`-y@XiT$Cv$yW? zt=KmzEq;&I`)}5qK65ROLK=HlUP5ZuG}t>y6D$cin)mRqf&#i%!<7Yzs5QcY? zpG=R$@p1+A2H>LTc`s%sBhT?t5Yw@!)v&=?4Q)*&)72Nvv{9gk#ok*RNySktDb=%N z=I#C3A_0nRyEW|P+GyDp+U#RK>plUi3C}&`;f!mXv*lag zG;Q%2TLM1x188dCyO#7o?bmGP$%?;*#J)YE-6nS$VgiHnCXpxJSDJSjYpA@{Y&_`R zu$pcpz1BaPuHG6k(~+tRgNw^`KwmyW*!HZmRs3-iE7JmZl<%^E@;rfZXEI^@kcMNm zby2*dsIa|>GEAN5D1ieh#f`1jXDEYiga<#<16)gA)3U3TKNN@9=Vi1Q{oG)k@KvxQL5J2~jj$ zv5$qKcVi-_>ArGP<}2rANfE(Ay%{9+ZNYMs3P>1SnYaI+jttlD)VEg*6!aua&0P8o zM$v?o)J_ppwPqx9{xtzwg2W$plR=X!dIuP9|t7c&0um(cPB+&}+)n1S!YY+1$CV zi*dDNes2?X6tE$F2kxJa0q)_hzMCipz=(6V2x=jA#L3D~B-|x#o1~n~d^C&W2UEUV zAZFqP@zP$?%l9bFOm;wrg8s&>a70@69m2O_3`)*#Vlg}G6Z?z>UT!EHoiHvB2I53* zOhn<-M{}=i1F&>Xuq&8!7LN|9F1u>tW+#q$$Frh=DDB}ZvjnZ-5}l$Pb|2(QFgox_ zyp@z3Ly>wf8|H}t!03HA@K+wN}A^|z# z}4?91&oIUDxRMCw&6JcoS(y1SJCa3eMbM%t`&4KZ0@L0+sh& z$?MCaK^))&vu!Daog_0HS_L6$9AEzqzAqP#(EIyP;_7wgN2EQ&ZD2v7CTayWiyZ^W zGnB9Btt*L0ooG8n&2fd2-rSwPLSTCBkl7uWar4d?)`B~WsHf+aPAunK7(qu+KeL}( zJV+v39#w%PKu%Q56T>fYLXUKi^KLQom$F#fA^G1>YZF7?v_rj!Opow1+*pcWTXGU> zyk>KD0MLfC{!gcEgq6`Js<~ug4~CBf#{;W1w@OW~Nm^Ar!8T;eg^dWSXC|#LS&u8g zq9o{%t_}M;b%D+@=g4uo%In^6ez4I~Ir~$7_Tf}sUHMUA#}T_%gkPAKAJ)l8U^vLh zICb)b(4a>_D<5kalxEE|i@23?*8lntG2YHTEUu!VTX+n1A&iPSWSKrqC(85n79o!j z2PkTB#ctegv9>Hjxvy5Vp_sQO$lJq)vt@FkKGjgdFoK`5H}zlXGnMnQu|=#P`Z+)+ zR$EBV=;*dB368-*2+y2lJwoE0`_>O=@jq0CSWX5nK08WL#kDtAFMeP)PQXVvb#T=) zB|J=GQHsYa?i*8X^qSl4qb0LNT9JxM#ICMP{=oyULW2eDdd1e?ikz;{^@>vY3%X`;s& z8d_MxUZqtwb%6naq{PMP*V`n@V?{zbzyym!CBDEl0Qr5L9ppbFH#zi%*8>2iL!C8> zmsy`ry4Q($7Ge~_U+oPXdB|N8=#%WfRNqTXUZV9j@2ENl;SD>bm~ zsHFhY`sco zYr5gg29^*8*N2Y$=t0z3pH#q0K#cq5^Acs?E_qE@?8zs{_Q;aRMjHh6W0u1C>+Mhh zp$30r?WOd==xKj2NlKolrcH75#G8tWY;5i@j#grE_wt4%$W;F(`Qi7)PS<@s0pb^l zdOk+Ql`ok2^cf1yi|4MM&M|KkHl7ym-$7bWH(qC|O5ukRTnzo#BbhnX|fJz382o8O=Lp&TYA%o^{VRA$1zzwqa`g`Zax^s$ zA9+p^f^YBkya&`JW6!%_w!kY1L zUVVvv7zIeHAzrtG(qQ~qDre*odr_XZ{1nZOXV|zWi$OESKgohn;k>ph^4m^&_CKBG zYRVpaQe%M`pG4QqIN!ZYn(b*LT~03l0K?g>XIty-y1?UROZOW!N$8?ti|(@pM`yS1BhF5Qa4rpv~RazOr76O8Lkh#qQq ziy6ZxiR(v)O&To#sF#?2P3X7I1oRFjsYDJ5grvVenFqHloaD-C3nHXxO9b(QeJLlN zw>))IjccLs{h5ZaH|;A&2u~lYuMK-5G@K8%$X1!Lc}&sh9copXf1`yp&5o_8AQTD)pmJW>4qNRXO6Yy7(IzgwqEJ{wPj6eGZ19&f&CfQW*yhg$;#GljLH zkbWRCxay2}K}Cf(S3PWSRBG%m5qUXTOZd;4C9J9qf2m}G?4x=0)B|jh$;%W_;s4!` z6{>Hb5MwHw&)>EnsR`@^wEiwqrc}UihjMs0w@S$T6O71#-Rt%xGv2v~Oq5i8QA5qL zr7QR&r&V<+mVhTU$q{VsmTa!Dwt9=b=vxlvT`H?DZM1Tiwir*9M@2zd*3Yy0_h8OC;Qa9Q=i$&^d-&GNCbtloR4o2%m6 z8J~(M$LpJ5-T8$0?@MLEtBFejR3m!TLQf{aSI253uHG*ncVl02dPc!4u{p79pSc=$ zW$>5Tce`b#HzP7Y&{~0+$qs|S6&LPqc|kwEefx)M)Z2^=WMKCLEfrj>`ASSD^E``o6s-Cg4 zp1}}g-U^}W=3d>nb9%V+JbvgF(e6fm;X=0;I8iGxA3Ts&Mcp2!>EK(8sf@e%4_M?P zvaFv~M{cls z;jlD!gobM&jYQVhKm11o(j@R***_59uaTqWfb?9w+zm22A!T6+0YC|Su1Q)`rC)oo<%g~%_2fNM8 z>j3*CGPKi`eovWInytg;B7;l|Zh#!rreI9@EUpE#=k1Sv-Y>rl|G*Zkb!xtS!gDL1 zGj~CiJ>6i*PfyGcl!I@7W%TI_$afC_kM>$E;NXG-CGS2Qse5UxCqA|s#Q0k(!!Dpt zR<1_VgVI016n!tmlzVgk0S#8E=l zc9nb^zqNm(E0~;Ri4_&YO!~w2(|6=kr}IS80JpMu2c%@ zc-@nxrsMY+2XRpLcDl3#a z7o(ISV%JR5J8?71ZFiA?lipI@Ipn%};!&bHva!OhVPH(>#G?o@yI6_##qyh@n(-;w zsRjdk&lx!xk5+?M;*}cw_;H9s=dL3rxbQn~*l#(iTdhNVOB4YZA$5>nKTUVeaZX$h!aa2y8uZ&_1eZz6?_2k?$(S!ru z7s487u7B_cx=w^{A8(n^$YemAVnNX?`BV$L`#iHsYT~J1%3<-u!*R!$1lxR*R85w2 z3`%X&(hM1fr+?Vk2$YsJ1>sz|f^h#xJSV>(uJDAULo3!})|ddVHBta9R)pbGKDyIy zL$zv9imC6BP25OA@TI!D-DPL!cVlZNyHVPHn0c67M1?U}=JdYU4w7uu{QTG1|6di| zsF$N%uxa=2Y!H;h%t%^&}2bE`_SuA+iLym=j3YCNXw?pg`Mz1eB=Am^2 z95aJzCH#Zs4sBy@-%-_fV8KN+-NT!;__k*;IN%{Y!c3as#pG?_$iC? zJ(mYwdjG*?cO^ZssI<2cE-M>Zc=YuKYtEtPA5(M$A?&!|CVO+a_xKYE4*LeT=1He# zVg7RAVNVt*m@vVg#p&fL(tGS8l@^=TLIy$ps9Hg|%li*DY1M2t9fy@H3l&44KG zK~e(IweatLmhn$gZS}`ggBmQpNZ`EfmP`OjaiPRByJkd}r3KCVOuHQ6U0zy^!@bEO z(Gjq}F6TX#&AiBs;CD;>cCC~IpJfBGUo`cW1uBAlqzK(| z4#BXk3a)V~DxxJP48{2s+T|Pzm;~`Opd{28b5YbCs0RK3O=y(v1v5rz|Mkl)FIzuI zf-)R=ExAV30v12J0g|u$zl?w%JlXn$#A^YP<)HEFel0mjxa>Ne4g_6(k!^s<75Qrk z8!}fnt^q>DMrl{0NA8COI7wg-=4``2DtgaYP@HaEd$qGoil_bs-Cu~hTen^?@}ghI zXx`}NWp^}kJjHZV&0j#SWsnu_WmBQkx%!zOygr&JdB+DA z&oP7My?M52DmD4^p`HU-jVAvqKFd7vy4FXbde0I-;+i2dh0ba)dApidp9phWYto77 z;CGbSJRWGD8tv(2i}J12gD6WBbU!nzY2NW(Vw{UnjB4Xbog;y=7-TH{6y18?eg3D5 zOIV{0=0!o9uX?$DkmJyBvDXQ?)NwV6P!O^OTFy>RoocJ58HdV0X2}gQR?a{^r5qr# zPtkEn2ZG2atjDUxK;sNhP_^Q@6@Y{zK86ai29)05Z(4kF!X13QJM9FZs0}<8J&yrm zBxxI{2OzvZAnda*3MTZ4a941TdM|RrMgo~3jEvm5qZ(V%O8UaeSF#k(c`Y`{4Aop9 z<5YL)&yy4~26GYUslQHVvkz*r{MZ%QLbOSmwWFMSQ|g;?WWnmQvP|+KWW!qs>MkQZ zJ5;@|STmTr32!&0sNSWLtpk0EChpUSG-mx1i&w-62#1t|Q!XTtvF@*{`Ggh2obhVT zf~*YCzgN`C*q7*!ASwa5@WBl6fv&+7ncI|xcSQ0fv$1^}abvMJZGuD*RM4ZJ4~Au5 zZwjxK>sXpuFHnL(WMmKuD$4b8f{OTOa5{#2L8>4AniI-igSvWHr|GE%NkH5VB03 zf;Tjbva82C*XOuhqMc=)`35?)%WkbQaR8#!UpNM=6ppdgj`WpPtSU37!AU^OBwaTq zVo{_`0$El)jU?Ku_{pVyY-95y@caYD1i_jpj!g5hbiBmIleqrmWNg@_my&o+{aiaMouJ`pr zVsQ#YwQ!S|sb`*_+6$u6qrGLaSr2}c{GX~oJ{;DHSt8B54Zd>vKaA}gB@kt6wWqgT zy0mJyh8<|+9C^lHQS;8mv}fliV|+fH;qGR74_O~&tr5@Ej*o(07+_V3j?uVJ(WrxD zo&=cjwwc@$cn{7A(w3D#u77|IKaNTzPEGQ~5- zvImJ`nKQ=;<+_ZlH^sclhU?P=T43Dy-b=TD_I1x%{q^XSkgB134hng}_7o+xB*OjIO-}v{vKWYe~gu(V@g= zNO3V2*B?{)r3VNCdPrX#2r4%?lBi?np+YFKw6+baML`~rKUN?b+`vC&h?=%MG?`d6 zll`9zI{Af3a#vkJQ3#u$R>#5i+63-H^9p%jhax3{}>mt(b`6_8( z*R6WsTl;kDdR+Zs`j~^P{V84%^)O-EE}*A2G;u11BKXoWD5m9OGNJwcaOwrCwnIl5WqLsC2bb-qv5sstSD9QJy?Rk&kW=8-jpLczI2}GK zsz0C5>{FF9eogY9OE&kb7Zhlk@-aeavJ0J|gff_frk?tmqax-h57Ohl?|ppr?LjaY zgC4=MRq$KqbE@}mO72FkZvxFY_h5Y{n%C!94G_h~1+Q8ql4z#1OZYkwfs{FIJl`+O zLU7#JtD#-cKU;xDD%N=!0J+g4B5b?P8gpE*khmKJOOL`iY8@Xn=fDSUuxXV<4SHB4 z4QP-bg?xno?h@ubsNt+V=Xii+fb|1s>keO_y+yW<(3u{E9qd6g@c$;m$PQT8 z0i0L}w?qf<-CHg+F)4s&He!NWAnsUcbm?l4KNEAOk=*mZNX1KCWyFjh`lIw1fYz`4 zz#-cUIMDBG30K>TZazjmCh=1>HE@X@l3f$y7Xe^c>W!PT|A7?};bdF$TCxb#K!5TF z%e|n4{pxx}V9}|?IGI}7Xf%M(cZEEUN^4;>JaZL>3`}`qmwH&!1IF`=pX4ld^M;{n z8xajWw}O4>Y2;(R zr-xyWIsz%q;_9yPii#-@O%!|0-FLGKsBuD4`vigt^jqW2x}tcD;ic!p(XCYj#N|h) z8{{uLk@8xPMl{`_;@kk%zHjU%rNCX!mD217Zs)VF8d867C(L4|H!lDt?{%5U&e8m__X{}vaG>~_;Ypwi2=%} z-;`8|fjqD!VXjG}gfqet$Hu*&BKs*rt(nHuN3uN1 zSSurcJVz39x}W+K*8;TX8PYCeshG<6bh=2*XyJoi^tSjB0y*CqXZ&x>jQtF_hRcgP zkI}7H7EjHHXs$*pYs@Br)Me-k{aBdX`K6!9piX@)Oo5px(AsuHpHfSY=Xue5-7gzr z|NBt*1BzEYe#r%)kD@)Bx=Vp===pa%Nh-TLm?r~$t zIzCB%lW_@TMjfi8y^E^iduZmHc2?U4`KsMs_ObKzrpPuR(BoR#+ zdrp`%sOpCqlDinxJ3WmSvZlTjOEE2aJD9g3NQYX}U4efP8?Xz*NZm@irpL~}R3adE zx#dL~(LGYEU0z&bpx=~o9ovV3t-Z?#+_Vj=TP)^jmwZ<+6aq9kjb>1w*LV~<>lnJ9 zD7G#%ih;m@HizT$LB5hMZG8{ZA*VAw#mCgOcsjpcLO8-u0+@|E&5v^M&xQ?~kW(rs ztO7a^h<%sjd|SXq9Zfzh8qA+Ut^kR`5@N;PBSiMfldp4Ut4KfSTw;w_03%c+E;4uO zm_2H`Ad2btP*h*ORQoSA7O^#dj?pxu2$l3uco|X{jm#=SE`}dfPPwbkC3t&zH4$G) z$Rku9+a(;Lm^oQ^^!_wy%&^DGOvc&;BOap-Md2?529 z2mz-?6LgGF_N~B$RzA{3Z{`A1_xdI0) z>`xMkk60nz&K(U+8I9`D*d88X@W>U08h$n}=AAmU z$(YjXF!n~r*y8_`tS>ntdeqSN!-?Vz3s>^*?o%3df+af)dl*8XLE~O&r33^0KN(%M zhGwl=8xatNd5#v3pP{+Lr_ux7uJE(1sKq9ChNOE|gj zHB8G<^}58fEzt;4Q{@OGECt{Ui#Pr?B%Sx)s3mTkiIrk+MRYwPbs4f4JN%sDfC(E6L z8pSmQg80KR_fuuo7YEN6^=_FE*r{hOPy^|Q%#~O@8Z2?bYq9=%-J$p{BFP`b$Ab7( z(`3o#sNI+k(IA}D@)GhKAs|rytMm^GGuNPZh!((9)m=j8JNUCZp0Y`K&DELog|2e9 z*`OQ)T*3qnd>4X*2bl6Q_9SlWYIb*%_0NJFY zp3nQJDB$ho(gVe)F=byW>|&xL>zuon+?^q2YmV#2O$9=}2-~~MWMEIT+b=X8$A+k` z9-VDk$u^|-?AQ}9xs5iJ5XRaIS$}LIsAUQB_l`& zc}%Vh;zl6Fj|2^V%ZDU+aZVC+-SGv@JS4wH$SiMFHf8?9gv@<*U=Pyn2obQnsq*WI z>HsiE6S}M$kyE=DgCQ`Z&(hxtS67)*CaZKh!HaIb{)i0<*Cs;+gahN<-(5g06&mVv{g+R8 zrtise*&Xd7US8j zs4GP0;%BF`_TlAo1eJc#p`#COJ0ESE(5%r*&Gg>^rv}4ws?nD=gI})S;aQHnE9!Zf zDuWr!MMp61G}MYSzDg2|YOSMi5hWHZPNM}QS5t4E?;VGd!iZ0$XjS5UQMH4M>N@q-!KB-_rS*hzfC!5TQ09jkWee%B&ZAv?Na3!AA%+_^UvqPQk?1A>$;~ z`QWn~j!5At4%e|4zo5Z}9#HN&7DxO_&#L*TVg5n+N!^*a+=FCiK3WZd*@3ymq6hZp zW4l-8OjGoG)NN)k4&5E@YFB4t%f+5@Vv`v@^A${f29&$X zhQptM_698sNv5s9a6Sf>cjQt@TTOL?&(P;G0r>6*;5DD+poKETx^|E&&LUy1%*%^Zn z_CtX&<$xdfPZ8ye`X8uxK42xf7+~$m4r3NGmkKjNF2DutDofF0e&-FJUIcA&jO8V( z|8^Bt8W;M~=}RGHU?nys;bkWMVl&so1%6%9CRDE=AmDo>=3>5&$tMI-vf^Z;X@iSg z7Qfbn-8*r!ioXD}v;LJQkRsEHKW?L#E=?O$b@hBhh{&Ms&j(~X6%wqE$Iluf*8=5pctHMM6wr9X zs$%Zz=If>FkX3yy^^F15dU=(}J}0MqXo!U`AA;i&c{VnS@T7}tNcnLkrGw-DMhEQh zTSPClAKNyJYx?1zu^Fs4*Kq|;HIp#g)abptnlW4V%B`q5;RvO|ADkZu1v00ldp&)9 zbQqJ1p1138#!PMfemJYF1!@<7a^?}q?=i}DPkQa^ieb;)tFJjnK(dVpZOAm)D)trp zL7Lcf8Xu76l%mW^=C)0JMc)2A;)RS^AFyp#SwHG{KDi%*2>KDe6UZmD$7ifHBUukhMP&(^h4kB%MY8P>e)_ColnC098q zZ|i{slEmAkCMc8_Vnr%-k6t%#wZNSQ>?u2f5h+0AUIt!<^Hlsq*2EE(w+yA%H?l%z z4c=o{g(ZR|9*HPE_gENM9JyRI9NVX5rQ+WFXtwMY#6JWkPB715ZqHxU0CKMPYuL`= zH~#OE2_UN){_zmNm{WUT%yF)HA;Gu6Ux}iyk~%*QZF9$O^_1dD(gBVo)GMd7U|ry_ zJP8*}Q&dHexfLs;MW*Sr`UW#>y3{auxI)N=jRlAF>Hq+Np}nbc1P{e3iM3oj-+cxJMOtq3by?MT-ZyXu?MH-=MJz7>|Bn3&YVze#EBdkzITgk- z=;>I-Q%RELa(U4G@uaVFiLq-r0YQ|Q?Fs9EW1EWv(CNnqORa!3ov`1(lM<9O2 zonv$+0mAQ`((>&&5Xu`=U#z^?pNZ!FJ?G_04WW6+zzw6a)(+C-${Mx12{nkm0ISI7 zlONxyS{i|v5m_TCo(B^7`f@3s?Px)RS?HIBeqt(ty(!9rXs=~@u!$yE&1!`?9Eat4!NdoB zYcO}WFlH|s%q(~J6>{1_g#b3s{tib%=z#9UoPiuY;hi2V;w5#DWDft9l0-vUYFcfK zIal=rLv+DI4+8u6y=zzV3N6BjErO(l?gMl{OBxv+Ma%(07RHpnuXyB#X&X1S;JO%9 z*-5Hv2rtX1(T1sT0CXOShXCtrPsz5}gRkkCfcV*w9?;uIbUj^3zDwT!n z$MM!A#11oP@p>oFv2sKC4pFgEm%apZ8rl9Xi&!`4dPLMtgHdB;)%j9M9s?b_ zm733Xu*WzwF2r##jO&X__w}^v-#nD7{j0uErQxl3>78B};>#jUJW{v%Jl@*MenZ}pmhLD19|r2O$@E-q3*hGS?lq^5k}zZZ^> z9vaPSxr!t7&v*@&9A%tK)uS|wi~`PGhAZrvQiJNzNsfOC=vE8XO6kwH_}Hm}yMp+3 z51o5;SY+x{26y|c`Ho(E2pJ(kYjJP&$E=c*g>mMx3JZw(>i9c;$av(YNp7yblv;X> zjz15k|DT*tkLpH~FtJL`i1JL~F&r0^Frq5WY`I^TCkraN*g&y80cT^}r*^iS?(mPn z%RfpTIKzPgb?_1wKep39R-d48J(@$bTCzdh@=t?qL&Xn|;l=8%IB}P7#rqFVT=9xm zxU|;{KD1l=PnZo_+o=i;Q#3E;khxb4|7;uv9St}Q5ipmbffK76JN}f+T;PF{rg=oH zIn%t#`%Vq3Q|bhbU2i2}b6CIgAS)}YI!;T*R$ZPK_uLPd855xwGUT7iPr*?e+{P;W z6D&iij*lLCKMEgk!be_$NwXfCX#>-Y+y{$7{qwyt6EaTV0tHp%0hJ)e4Z&upIT9@h z^(NYZji@~XQ)xRT2&eMMMy`w{wbkG0BpoGvPgfcth*jC5SDfJg+DIy(uH)7Zqi}nc zXWBNNvr&W7Y0uQBW3NwD!@Hkn3Gs=aWfvt7tL!OLH$%^^23vjpzd#z1gL3E$E`x7( z0O?o|(LkD+n~HFo++&H2Q^7Rma0i>Ez-&cKmVKI3_es z(a2B+FE{jT{Jg8A3godrJ-cD233%)p_34-8RBslpx57Y^DaM$iPo=^(vqXwuE4u}n zr$E-+|EWNh!efL4X`Kaq0_uc%A0nBv24bR|a~)WR19A!du^n-df|}@=Fcbc%z@M3t zA^TmeHS%0cJS|{_EFgun^ft{~Cp|CeBIlcGi?>{*#p^oWS4%Wg@t>~P_RtC9c2$>ZpC-5k7F3+_g>zAH1B65_M4}oT}A*_yYss~BIVSx=H!Vy#nSMh-!qit z_m1*`HS6nt%U>vSSaVur_)Kt+o$1doa_)#=4<+HU3Q*kcyzI}^ux?K>LVQ_pR`?9^ zhlq}5_*CRX;Q$?6ABPBq6mxtW)QJ78`ZsA4#|x%@u36l=4iF7Yf&juARp?0KAdZ<_ zaR-N|8e5eqTC6AVJ{ie`k+yJYBY_NkWxw^GKW~I-X@zV8W`i3s#>;?JUR){yCZIQj zHH}c+1+eM8hC;+IhQ%+~Qxsx$B@+W1sA@cKs^QRyUyauyX<7`r>dR5o!I0-HFhsWwDC&uS#s4%NSFBTb2&uBW4iU zi&Uh4bDBXO^QTuG4BR`Ax1Z7LHyAxM7iK$5yEZlIv{xatU@y)m zpyrlEnKu9Xt5#f4YeV9bXPRl96B(Bm*(fxcvJln;wU<9*HTeC=2LxHZ0L{do;UjxMqDq$h7*s07Qd` zSjy)RHodeiPf~?`p`6MLiPrV|F}{ujG*0K|a@6`gPj^{0+ei9NTIrOsu*YJk%GNg_ zu%mf{>Isq$h$--S{*_MdtXI4>ErF>b6>TFJ^v}$RgO5>m>802`1SapSY_4xPRO-V6>$N2+XIMvH~+3F9Usq>1;N)7q3`Z8 z&|{yv`}8wKwHK)Vb{j7CsdhAlj~E2<%)B7kZmTpK_?VzTrY~6~MdE@m<7dfn-2aNU zQMH|Ivx2`3`WoTiL|ds887ZerSrd}@2I60Ba7%0oW~C;T*{T1nwXt9%r1g(k+p$4p z&*$qFQ-l?unXG`l4)IG4n8AeFs7ZPq+39n@{n0Q{zC>dtehL6&N8>X^7D}473Vpd? zH@Qm+d{)6F(HwBjq-5>`yJQJPU-f|N(j($y?+g8DkVor?o4U*Hq(viB6~A0gLI6jW zhoMs^XH1+p>)NB+nq;}2B{~F{2_5?YDL#W3WFqPgs|Va#RJkC{`wC0vrQ_0?{24IRND&SK$hqLDVB4Z9i=5V%rtV zkKm6iWa#**#HzN@IKV5EbM`dp&ZiY@SgIucNrWyet&0bgv2IUx4;Xv5*5JgWf&3rq z#^dsKG_iP&?h6lbH0&+wRapvZslm{tA8qt~-(Sgj3C(ET53u=b^o7gy_!UrM2Ff8= z96BQW4SwcjHn}sSqwNx{&?4P?qmaE@2InKzna3;F8#0@{>#)9;9mua zzuXqqdSm52o5%_oH{$<=3#gzn9hklpzqS~Jnn|IMwUDabtfpf z;jivAx1Fb%NlDBTO*Hej^`}=9zmNTjvO2qZiJ*xN2w#;Z(37F}*KX+n*D~~CzR&8y9h{9EFa9cLD^jF3piB@S2i5(@WGWCHL zv}U4mjl?&Ze%qHYONzaR+o)NdI1%&}Nt1m6P6_A#fYUubdPGKk=@8?{L&PeprsS(W zt?P|0{2<-KDTc01%EM2Yq4f5r4-PjK_RVS<5oMvpii)%BSBvj^S!iLYZZ8@lBnB}1f9I0msc>40S=GEE=(1DX zwGbf;GrDt5%73@26j4#^T()K=v|Z~)>yEMMlDQp0g_CCj3qWq7^F0(2>YqYbel~re0#@Q5*1Buy-T z8B79jU5y&%ye~%TXoz9>{9ZU{_c^wWIiE4HYhwDB9j5l>!Zsh^pZ z$mYs&tIfV>@d();aj#Y;3q4ar-aHe?kAA$#B|H9msUsa%LaPc$trT_elR8eO4hYT3 zuphL5nL<4}$_FN`FM+k0TcCx8MGgNi#}^X1z{NcYh!iwa#6=b{hM$2!x92B7uu2)q zEsU?uf%xC9LDCA@7@C?zamapU7SL1S^HH&#OJ}-OB`K^5#@&io44b= zK_mU)2YKU|?go%JeA677$%AN8ag84q4q?(tbZJy^W10i*82|QVM0CJQ?Qn3r-3tQ% z;Hbo|>CtteM5zyf9p#%Kmo#9y4=o7IFq*d^ABj@`X~UYw6?XXaXWA{W?wn*7Mi7q; zHY^{!X8a_3Q$hsV#bRLcz`9HXEA-c7&h+K(i=dv=VNd09DM z`nO%PlyFE_XSq=%lU9Re>U}+-95wh+QJfiq9&X3$^wC*G02erU6Xpf*mU#Org&$o; z*JxoLl?Z3}g<`;{tHaK(gUX|ne>47O?3Nj!lR#RPERE|}jeyX%K|=GIZ3aaiY3Kbz zzU6Ptz55y$ct(Bne`}Arf7KmVQ#V)$7Auo`Yz*k8yoPGOmE3hLMpp7NkxJJ2Ks$|o zg2&X7Y+nkYV#rh}l?Dcy(m;$Jn|*1S^O)A!9Mh_l@`)pC0h)IW%;8|4{Z%vNoY0SZ z9~rNBX3n#wS(wCaSC;akrZ9f1u}zrLF#fwD9IE}aF`kiGWy%jw837T6H}OhL ztB#=A$C*L42T6t^8v9|91|UT8lvKxx(uZJ8DLvA(lbq#*PupOrX4Nzuz*0T zO)9KHgWe@>(RqrITd!=}-2;RRe4T1=nDQY#K65`1)c=PpiTtylm@}5dv!M6A-i2X) zVjz~(rzES%QdbfL2J^NMLw3{cv=XOsnX!>WeUnU`LxA3y6)7v^U9f%3z=d*{k*cO8 z*U;xUSY#zf@JUgG?8ZqChAgcaCv*@v6x2{9IbbdX4*5-Sxpw{y8w_fW{?Eng*%a=C zFLW%X&jV0K=K+F0reDzBr*&X98!&ljDGTcYn*DfeIqRwsJ%R|fX^mRFodYHjV|4M_PT!4~9<4@VM@# z<%YlC5A%xEe+w8bT4upP*Bz!>FKjR?s8EXCvdxq@2Us?2uHVZN4<=#lrx9$h$z?M zXJe(|Jh~>1MlYdr2*zN(MoNET2RD;M)Q@GLopQG;t0b)mEs?qc5-WV|iEz__9Jyc9 z%w--RfuwP*#^H)3pr#o9m`w4t)j&u=_AseZNVRnVEtr5>$MWjM{;%>9{`O`A>7%}f z3}}jg*CEAWMdgmu^DP-yzhGnAIZ9!Ch(#8c1YLJoI0X~n`3Tu3bN0LoCpvQYe|>S! z-!kPWeGRNw7%qz=|45ui%i2is-a(b6dqHHV(mExCrl%V3`f4V;;xdMzXgysi6{+2w zZK@5z?`{1Qb^f+`&nA@G?!(ksk-W*dr2OWXm-2ia4N-XMES+5QgIbYN*!(QQ{pAp! z0BN5Kj65jx2qo76_Lw;-3P6_mWDuS2KZa8k7PHoh!?KVogWCx%Vhk1}`RO&*ZXx>y zbKzq7B(d*!V|`}HfEOT}Fr)vK}N=lVik7w3ZT8NINw6-Aq=fNVZv7DD!#OuF5 zuUshg*etQ+x6U$6DR4)4509mY-Dgc~I>`-9i|JfBYMe|IhTm+x{9K)VKXZ&rr|UDF z?)cDvb{aCd`Cu72;q@+}%)4D?FISu2SkH7S%`-A|EiLgfn|vyM@8RMaUfJBp6dhJ9 z3-h!7F7?p)Q}Gp>G%0ir&5Sbgn%s7pO>~3qe)*)sJ|W(gm^1N_$+KK02yMz!a5n`A#N>Jf%Qmtkr>= zWD=ujsxX4~IP6}Rt?V=8e*A8`EaVT*h29l5jOiiwY%r+9^I1;_1Lfn@Pt+s+8cO@}-)Qi)tm`~UB2^)QPTxQ=8<;yJ}& zX@1EEsh9hq{|ChX^pCf|Avk)&uK&&|&{f(*IyQINbWdZG~ewn*Dl872x zRxkq+yFr6&RAT>)t4b=rIQs_5)bBUdto_pA{-WU+dvoFdN`#oNYEalKv;cV8kCpOM z?9vp07}y8Ms{kqi+WNcTWPTfaRw3UgEl(81ln^r@Q+#}%gZUh}ykD#LA(fqH9l(yz zn6g&bQ|)lgli!1qE50z~gxvZ^LP@v1dggs2!6FI~q|B^Dvg)E7IR4`qby)(_zVnq` zhF02KeA-gtC#O0%BFV}6V^&>$^`-q#sMCYvHAvcDNn6~0LTO>`SJxi!y1!Vp6qe>) zf?Y4SD)OFtOVP2od=N$zFcpYFu^Y7i?5OEfYW&+?2h>8X{UQXsLHc!cK{H3(!85?Iz7I(=t~@-X z`?Efm(O;EQ%cTXu6HoW)6t&d2OSN(hXekkB?h@1lO^UXYwQ*rOlR&v4GxQSLKe+?v zEwF{;(@oApmm{5cnML{`$R(ta034s8Y(n0rUII$6_ofq}TSY^Vt-P?3wqYRKREEy$ zf*b|80DTPT)vmcu&9q9aiYye+CRuPo)R^VEW+g<~4mR(nv2a5{Wn3&;xL*B6gaw=} ze3RS+EPl$nS}#yO1CP>-AI4Lzt9~<68sU;gE|>YYkU_$;OvpzAm`wkBM#(QXY( z_ByjN@Y62MeGN;|phVw+>0MBWTSwhf4`XJo%9PA_ZN%=Efg{g4i8*=-*Gk(?SrJ1O zq#D2S652|u%cv|nJj<&l882X9s=goO(c_1x@`)FfUIsa8itjHy7rzrawp#cFCGloi zzxc+eqmupB}1%8IYXAUbXiJ5d(0DZ>bpBUQcb8 z&$B9)kSa_)PP`7YGVdm3<3#lGz36Z(`;Qo--VfM^AIz|4YPrKZZfF0I!9cg#-e5mQ zOcguE@64j(E_XRGR&Ic=aL&th@D~T5sE(6>x(37U3{OmwD!tAkRT9pb&0iB;%@J3S z9DA%yP>1&F6Wf%vAsVRH3tdunkjnJCcd)g#E%gzR1dW(3U70s2FP9^1FG5;sMw)}* zQhBUm4qGSH>ny4LwJz8jtjrQdyA{S_u|J3io~OMALlX(M(b-#sb>qQ*9BCuufP_OM zvzg9ku^GN_SRvF-OcFqagRKmIYKdnqA6+#HTTB=5vDwKJ2z`!VYYQ&mdsaI41ocv~ z#YC;Jqug6&Su_G|@5p-QEd*i^{Uvd^V=H%aJ@QOcKB-T2oR!L5ZE-TN?^~Z(*mk&=S={KO}h0(Twrjv*J&6Q^dogWcd?St$Ge9EYX3 zF%^(UJ4Hr+0H?^GlBIqlEEy+><8xEQFF>d$9Lrc>P%O?>$jKa%gV$k=y+s}0hwB%9 zRuB{pYnVOp)zObd{A+|+QGuQm{cW)``p8f`A5lNJEO*Clq^Z^v^SRdL7}?qdg}PIf)=0^W0iVxEG3%eym@W(X%2-Ex&&_*nuX6^ykj{$W~VCP$yv=;N&@fbil=NPjM#o7G!Qaf%QJbbjyX}T7P*R_eF`MHe2UX9OK=7`fRbz1RZ$U+4h;0$9a%Yn-Wk~lu@6RI#XvR2 z0fv|@t|V;>i9n@h^?tlggdYMHGsR@!^mj@K*8fDoWl`rQuK2ujkTvuBRL5_kUKb(q zO#qilq?QG>VZPp3;$9gU&fySR=A$mABYa&{2_ zmA)Ew6JCJ#zx{F|vSJ#Dnf4L7Bk+X%dXX7=XGzFMl2Nd^^NH8GF|7~p{FUa46agI~ zyT9*ON~orwUAyp`=TAKkK2%Qs>aorBVINd5$4N@q3b|?UwiF%9H{0iL;&__S@V>LV zN?;tvlz+o!hJ7N4a5*bxh!|3ZNrtdhxGLNNGyi(eVL2&~wZJjwZl!wutw~a3DIAPx zLF1a@Du=F20HlsAWbW^OX3!h|jX^QVxsA?h7BnGc8qc5sLlO^rl1i2<3;k>fppN~q zOKx$&y4DX&vO-(-4q8}dtdu?5_8AZy!H@J+CtI(9$IJr-IJIh%b7zMfm_z*sg=--? z1C!>J^i!*y^VRv7WBw;We$Q_N&!?W@)wt_pE;2_8elW{+YIFLT~AEZ`w*I$ij_^~KJJR%?cE=$k-N4|MLaDELp*l>u_AJ6i<=!3 z){^f9m-_vW1uoecir>0OnJSlt3#Or@gix&H6oy;9Ag=bhMHUwfzPhkQCVRE68VG93 z8QIRXhRuUm%UK6>KeW^GzVVgi6l43af)z{*nzauF-vT%d3(Wt^XQpKd&$sLGkS=r% zCson2bFFs7ANI(KwUl~fJgUdWkiDc&gNobOjeR%`;g8!RrBlk5>O%t(uD+4Vat1-p zTa=~W3ACCOvb8F0quMtA|N6Uu8Afxy;9^=TU5Bv87fd8*EP0$YV{0v8liN$B*`G-#qrRr_d}4DGy8+DK$-dK~|Ks;V@eBNZv03NbbbFSi z1{1_uCygDGg&l1R)+Luka9a(}hZwzNnh!Xz2&?sBqs*+gh-VW^qs`tHxp~b;OgW&f zLBe9&qEHDwOnW>W-2uYx1mkL|r|~QaawD7Pp^-)mI3h*ppXY7cmIPo59{fO$HU?;+ zc6JWEyMQe;FUhNc9V;>A_&O)E{rgiF-Pm*{g}(JcIG<^6vHW=Pve1U&o zGp^nR7mq|(J*6i&PRksNOu0EkbjUhRZg2pQs=$TFauJt>_h$tz>UltOA3LwYO+U5> zWg$8;yqS@1{C^K()Mfk*1tLe*Qp1Wk#H7+3!=Y1 zGgfs2(l^i!gMw)z9R!(EbuXr(nO6f%>D%}!2;$G4AW`aHvD%Nz^quNgvbT%F`nNsm zSbozeAb&CkqMgWiwf5zNioK3(JPg>;S+A8vU%h1Cb?sAN6UK!csFK<%Xp713b)YYz z79K!4rbHCtob-|c(|0nmNc{bE`>$qgF|T`>Qx@9oTz6PbbR{Uxqcz*Hy{B4wigBf`j3mId5OOV*)DbIcBL(#LApcgkbD>t=UiPnZa~33f%0qhR()af3FTb1U2# zWfOI$>gd5DxfgL;Hed$nRO?g;m z7TRPCRz9SNxUHvh^tV?&U*_?PosdOy?w6q*!Qnc1(ejSJfb6NdAw}rP5|CfMup(Ny zl7Uugvjyfdn+|$fDaSKe@n_a?(`|b`jaV2@AF9ka@v!B7H!sHQ$YNNiIFj)-m@z$U z8sfJeo9P%quO9AX+k9sNeCjsdWkoj3Ht+2uPWR@n2QWf-YuT;X#6LT9p*^R zd;uPIhB<4I^gNJH&aKp;+Yo*8diYgKrC!>t2VRcnDVwaGq0A0pc{MYJUg$m+#sD!%6urOT2|NVbG1dHkRTE(nf-;UIrP4 zL+G=u)G8=2u!2APq~nvM^V!ohA6+2M^wvdp7_|VHqfKwpb&k)8wll32D_VT^!qv!# zA1??l;cxX#5|xLXy1Yd~Z!Uun8hGK2W~e4eI1B@g1dWx_Rj}=^vr#HNCTHju7u(39 z0M+0fI@7SJP`&I3L|fwRwT_eT3l96<;kO|9HCAZ|Xt;dkg>-7zZW|&8(Zc8X5l%v- zFUE~vhT2!b6eC0s_AoIxaXPuqcZ$HgpfsWbg^ zb(L-qZ`&i~Z<=7DV=t-+jn+W0k7*#=pOv}K1A5s=7k5WNc>$X?3GlhLmQV2`Aq_yT zpo>IJ4MNamr7A1bT|k{qAC< zv-OWJMkNH*Y78I>{iZd&xrB=S5o;9(ENX6wX(BCcX{k^2c?wS$T|CB+R}-CA++Y_L zhI87k=PFoF;FzI^1|m@LFJptU`0ArQA;mlWxG9`PHgJyeBDl7gOjUbIgF1ce}2NJD=gjwi^C} zt7}q=#{x&jkGtY|r{*uNLauf}%`)o_JyjfUvkVf3lxZHp2uyu7Tm1n3WGj{NG2~BYT*-@G@ftXA7ZNx*m4JBp)0f9Q6 zWFXTC-tFnm>7DTJi*8^;B->W&vJMwMxzX)V$TCic497gkVt3qif?R%AL8ZFQeG?>V z);C8Ch}jrQ8KhTkzP$<#uwm6t6NnvV!OI zc?6>|4!E0$1FMuv36(`gn8S!UZ!thLQ(~CUeE|d39!{bf!v`?3TvTN*EX>~t6YPFn ztnRj1?Y=s{wUL|%HgO(b;Igy5mDrPAMipRDxcP6NYBNiV>x)mPlHo%8fX23&+?yb= z*qM*kBc|nzbFJnHrB5y>&d02sX#;cvUqa$-*)V3&XHXQX4m;mB%Jl< z0N6cLDt{8{mU%xm!9V#y`fz1ff{_4D`bh`_te%W{KWo}n5>464O8wk;VIX{y8;jQh z9Vyi$^#-+N(#U|21bPsq?WmsuFN$Or_YApJ7_QpiZi*FVM-n`r@5WTj@~gtVI47LI zrH0cT|K#o81`ORpg-r^!-gV^z5T@CLIxFnZ-5xm`IcloRrg11TRp?&+%plmIx4V*{ z^?U8n-uhNkb3g=$_L0yk%;E~rCIJI=qELq!LIBMp&s))bCx_oXb3Mk-jd*s|I}k58 zYVY7lo8sRcBKV;E8X{l?+R9+?gt)rCwc&lbw zfeB^XK-Z)$l1@SWXQivRgo3G#Tkl4DJEoa4oV?v6Ni;1@n{of_>ToL%FU8VQGMK~i z3CPr+YIMf zgGYCy;xs*%2o+k9C$y3jzBn#<%c8_Dqm-wxL=M>ns29$#t!`6f3rWwK~8YY_4m<*pJP=5xygp4 zKz;aw*m8#}v2%R?z~em%Vkcv6=3#hths?J7i|hXG^BuQ!n#p(VW%scXd5PU($HbQo zv;|IToUC}ix}R$ahH!Vx!oo#JRFi!>r$_J=Csm0mYvHyh27@86Uz|qYjdp+$+Q!mo z(=YOcS{&Plg6;;@CIPog4HZgWnZbmdp7>8fWYXE?KDzHeQJ{p5nAY8**-Q z#R(@>mpKDP9Y?-FCWzmUB`u_%TmTEx#Kn)A=;Wqa$UQQ3*B?zxuC-Yzg4<=naS40Sp|mq zQJ5uRc|mLZ9`&2&M79=Tdi-tMGa{n|nUZXf^Ar^V;}!UDtQ|kb6iT5jDE7Gn?K}sS zdotaWONz5;hH6{G_ZwnKiCWkPUQwVL5bkFJXZ4^v7L%iEz9dn*NUR31} zoRQ|B$J%g1h%aIpFTClwCTjtLYu@aBVF`Aj+1Rm-{T&*9)Qd=LsBO~61_=3+NKuTYS~ zKpuE7tRL5J(=~_tXFweHR-~pMqf)euo>L zb}QK;LD6^;;N~(Iz;y)VH_ZJ=7~I44BR#=(5Vw_ehDvT)IQrkO{}9z z&(}VYB(kTA#5DvX0*L=!M3E6S~~uj!d*RtP*Lci7&sdf|6XyuQ-N_yr)rSHy{P)j=b~FYV zSN46iW?WYZml+1e1sF0%6$*KAo2~7}SRK5uM|=5a!F4qiYZ@|le1ky5?K38S^G@y& z0o~($N1g$kYyk|54UG-bQYML*gteu)k|fbGV?gh2iE*ofi=vzbp2+U#Pa35Aod_dP%Py?PHvbgCkNB zo_CWUkr1mu%^Su!{%XM&Kf!=Xy@1Bph{U2INi^$$Ya| zhb=RT0Bd7AEri;sczY|#ZeT?Fk73$^s*FM@n5aTg+N!#*j!op;wjSLC2P$+Y&aqEA z`sg1y@nWug8f|&3G`AZZLuE5Oj1lSPfr~e=YZB@d=qCoYIhG;t`0@6~*zwA+`u0?T ze1PbTs{Kqd$$^LCqHG!c*Em4aBNo04Dj4T$2OtMA$k*)}=s8_ zGx)cBDP@D8IH-Xb<3X|eij>x^(Kw9ingx-y1jf-z4!8qvD zfN*$kc0MDALwxxF+d^;Ng0j@8Vs2|X~Vk0W7 z{)O>i{DGXU%Z;B%X(H&I8%Xpu-_uRfMmO`|?ks?H#Zs#Ln*^m2@J8%xS&V8WZL$TY zdEvbQ3=`Nkj{GdG$Qt7sJ4 zCzq)B0W6y&4YfeKTh$&=cek!!3=KAOZ8kM`g9$~4dOZV@_Kraokt!QdQ6^U|uhvHZ zW!TJb`EX(5SZm9F>dm@>OZEI)xw!&b!bd0`x8rrsrXiDE=W2||*esGhy9r+(dIIc@ ze&Ge3+vEqqCYy9dOTz#}hq#LlbSTA19gFl8D$>^=^MlBpM^U-%qrtfBIg&`f@K`>2 z`^o@xs+)Q#KH+XQ@puVso)FFAOMX}D+YhqH6i3$F-Y;#P6j5T$QEZ_NYK*k9Xr1x1 zuxLy6$R9GQwI5LJi7Q~!BA-c6ciJX>zdQ9}Do7$*Fb?KH;WY7)&P|LBAP>b~f*t{x zxTAP(!C8~I(*omhA5EQkc8&Jgw@CnrqSC^c*X31eagP``Wm;%l!+QKwo}c8KT19;8 z7^;#h8cE1G(B8?S?k@6cWIPp`XitQ|d5P;mAhoTV6%_`zPVH0K?bsw90VhD1QxqQi z?Pfrs`uW;*xIn>tHGge!A@dn~{$bYnHY~cT@lJRE@>ien2Z})Ns>@$J!?}qLRSu^^ zCy@OL)TewWCUXb;!t>;|VAMFLI2p~ARArA61nShLa{Bwf8+O8iGysZcNmZjvT_~>X zq=$u;hvp_6?~d#PMrzhl_my-#hI2VEHoW`05^ypdR(MNvR1)fLowJ@;;S#jc26xb* z5;4NoH$sF2c)dyHYswW>BXl_|*>zkUf0C{}VQTO9Igw9+h-~f;eetnq4^oxy%tAV6 zLeln&Tsl5vxub7XFlH;s?j^JZ;2H}Y8dmbw=M?inKORym2)_k2nZ6BB9s{T30=e^% zJ~emc{?FB#+`6QWoHeWjxFzU5*sMaOS^swa-zm0W3X2_)gkeAKgypOa=l^oa;Yl5@ z>jqNItW&n$LO|ipS&O*)l@n=U>uJPgFT+zLcM2N9bPN((Y5I+*jGEfl6!&puX>X-Q zU*g*slzB2SEG_qS-du@YH*+^pd<`B2+Ou16Gc@v&tU7+ln z*p5XDk?N|nUux^o~}}9Y29ZWg5|;XboW|pu8Y+mB7LKU{Z#}ghUS&Lso6|b9!nrUo59p`ojlO zAMcyMzpzG!BJ+Ratf~|$)?r$@4^@AR3s4~mpP)O2a9rFYg)BB)ua@~?EK}HiqplJ; zwRC)Vdf@-%G&9^%=oP?BOV++kYwaK0Kq$MFhm`c+hBox>U|04qG@2;RI9X5BCyw7` zFOkrP;GvLx)Z8P_X#erSJn96b)=X+ez+2HubSm0`;mPAmkCQ^FUgtBGKXGV@fV$pn zKn@xbYa~uPORK(hg~+-JVXbhLka3p)1C$e3J&|TkiT%Fy=W9nUr$;HTpR)Q0RE_?*@8MRBlpI@DZNToJ;)+x3prr2vxx4>jn64IN#eU((*m8nT9+FW6Dm zMfYa3>eQaR0cdNYObpu+wcTVeqv_p9+Wf9jGMJlc%B7ePmXR+mjsRT3pNFG+0;&l; zw7p2up{&CA(+*%3<^m!14$D9f=zoz*`?&&cE$(4sHbER@%dA3pi(qmqD?1@ z1GczK7@0j06xonAA};rvc6%BaUgLt3GFd7!Oechbm~aGA5g5W(jCo8024FBmvAjM$ zhr5Ab|M&@WL_)NDC!hd^i>q|GV^NlIk(Y4Z1)uU(0V5D?-L#den^TK|(0G>7QRPG` z88r`_pwsap(s+i<)KrZRMj%E-Q>yiDSrTeSSs^yTK8INwRNP9h_OPMFlr%U+b=obX zgVKwHz*ZN*1{rfO6oKrTebQztRHtN5GeCa~y;4S2CVpUX=Z_|hupu>i=leOk8x=-$ z5pW}KR2S654O(Jr-1aK-lNv=7;#>wOW&orBK34BLL2mrwmDqU?Xe?P(J)wyf*{VoT z^#vKb6cb~Hq@LSprW#7$y$?uI#kfGK&n^DHXMx$;@2pa;KpUF|c6(!{kRLq>;nkSP zGgm}8q7ea~E(l@_$Q9nLdpE$!idVnp%=PFa7Xw0DGh1<@WEb)GXNvp132?)j$oL;0 zIRugPm#?}LF=?&*LbKFrgLXJ3iA|Uee@o0QM`U~S-GP%ulZU2uw19V)lR~C_{!S*^ zJ67zUyI9x=Kx2~v2~Jl?$rSgFS+$S9Xs0{(9;Dc!uEL_KKXBZk6jo;a{oWv(iqpA0yd9Zcf<4BtKo(+~{hOg$!S_prv@W~@dL3hH+7$vt} z(tqx!&C+ScXn9drWCMd|Cx67Vpz*>x_tmp<1D8w;=P+YH5!&@xsSJgtM-h)dq=kg#eAs0Lr*?tQKOwNN1$GuDaV`kUE{{% zVjqsr0c*ihf`FA*Z}ImUsXrxJCW{b=(?|LbGHu`M**I=sAxDr`T;Jk{Ga0D;Ja~@c zaNHq<55^qb<1kp6fkwa}yZ6%hJGlwgD~W`@H@GW8x$xjxA;&Zx4a}LnTI6|-zF+%# z%N~U}NH1EJs1{-4X!j-E&r&|PK7=u2aZ7MdlGrKZzH6%9H2BX!aga(9uv+s+5?MRi;Ni(h9~Cn&lh$ye?q>skCM+`~}O9siHTei2<^P-m_r-aS>o! zM66nLE0PE^{l&eDOW&wc8BI0_+4wolnN=WV00rJGfZJV*!m(5T2+U2@iRuj2m1LF| z3QWsVLCN4alf~M=OTBu5cm(9tMU3T=VC@io2t4vxVg9T$k%(i}PP{%8Yg{{i%cOxY z9f)ElbR4=tuO}R6a*iOhuT6Y2lmQ8Hxc2STy-QWLrmJHrq=sl!+F_FUTFCVh`D%_| zNrB$=hUdK%>aXGeJ|j`JV;C;_xviG!E_!9M7sY**9NO4ovkQKTlNj>3oc1+QAA<#-vr$%uhz}yRTMg3Ka9&azH?Y z$#5-6CTAGPvbiEXM_93M@;bCm7VQ7~%F3Ah>y55Sg?`nmiHwIsfCHj#Oyt*>eqEmi zN)CkpZ2~lyZntXHsY^FUt@J*mXw>4lt3wyg?6i`(aTv0}r2hxCsoZ{XkOd zvxc-EO0!QA*;C+9R*!@~?e8jQBYwrDyMk4sj@kY3AFwscg38w9 z4&uF=5?&<7j&DjJuyeN18bAa_BIH0u#?X`>oo1CHepe(&6yA)>%!@-{XblVxidFn+ z_1$hq=4JJjn$m&?-vylPeXmD07(m!_DhyNTB3gW@h1qN6Q>LDWfqL!XpPX=37p?M4nZ_l9| z#pi@7Uf*T;g~lu=tHNJDhmE4p)llhtOx~Q9;&(A1`EJydFR@!cwUP4O3)Dr}cA;|M z4J4B;r>@{PN`XThAc`F5OM;;cP0Un+r`?;z#+F_ujBsMP5MFq~+I%MO3275T_GQES zYU@I6$_fTQ&y_=+E?#{Zt>0t89)8>ArU<#EA#aIiH9|OVCY{Bi8j2>5q>XmGR! z+GL9@*jQS9ZfvE=NAo4W036a|z=j-l6Mh(=1?6{3S5>=O#jlZj1DXu62F{b1A1mvq zcVR|GfE&1#Gh*0uWlBH?9ugJuy>xbBO!BXnl)vUh1+t=Dd4S9|`LSdu+P7c`^V2vZ z?9qcO)gx2fZD|}$|CJM+UURs2rje#EG`RV$gtP}povA7|d93CULfj6my?|U1E8cz$cUVc-#6@SxQ-E z;w9MkLODG-L+o?B?2-H=LoRXf1!#!J)xVcnq=IR&eybZV%%P z-erc;xPr!!w4plVNf}&mC6Qw`nnt8~P4rIz-Gfu}A>!Ww`~H=5*jKAQBF}tFzQ3K? zl>YVfGE3_dsicZN7T6-0!VKARSQ+0z&>o}i@RHSEp;$&i?8G<8ULRcM+yJ4DJMPvvKOvf0urmA<|;k1t4PgvR)uG5HWl$m zJbnDlR*^KK`cj*P*;?-r;Ag>g-65!@*HdcKOA5BjLfs4geT1DM2un=dX{EWLLb1Qq# zM|#|0nwXVHyJ=hSoiePlTGetqsU;SE&O)!EifRh#VujpjRbtODv)&)HT~sKM0y*iD z(@}v36Q^aceY+v1L5Q6z6RgqI36!CxwsgITZ#>~khf^aOa&LUh6Vzx_tj>!0h96X9 zAijz6^H^#Qo2vh zm0a%4#n^_T0WJCPYn=xbe(vfZ+PW zLaHGADMK~)&O&2ShV{j=K$yX{=Fv=}fqIhrzjzB*o=q4rRg$v)e&4JZz<6vl@)FTp z=L^RN9IuTuQmRF#PFTcT`+&`>7M{=^U=(a)#M*o=14dnVP!V0PU`P3M8eF?9#k&{a zr}(+266)gnTa2~N(rj?oD_*TR)PpHo-y8_4U%mmc6?^(jNSorze)n<}N4;LbbrOt% z@@T5$7T2|MT+UW-zmFHOha45p=g;LSystfoi|aWp>#+2Q=`@($GhYF#nO86`Apzat zn6A3ybw`UF-0O)J8&}vVWm}1PJ^$WHjU+9-He`482Rh;F3PJB)Vsjy}5uky$JvyxD zM?)?V6pv0#v+EYCD(zfV$cWPAYRzy2XW_Dh-xXaw?JnzZR%H0WOoVXuRmkQ?r{P}^ z!X!uBkD{*aTSD^B1uo+!$y%Kbq}fEXt^4Gi0_qQwZy`BF`gIF5o?(@~5KB2!q+o=*!fLg5JA8;VrCQDW4H((ypR^!6J?WqYfGjzR<82fSkq>%e$2#{1 zk6W{t?bw5jT?m6&Hps5L=5QJ+%(sT*iYvyp62E0H;#6s%k6POJB3iTn!z|etCZDCI zVNY<)p~wUiQUMno+Z$7cT@2`0;dVVl6EpGbAvcV>lTOWZ6I z0k65=$}P;g3l;!j+@-rT!r8;A!KwELzsr7Z_@Z_gvu<$go^gPRlwVQU+++T4_4(rx zpU6=#^@B!^km-?$h?xVOeu*|4%*F(4iM*D|o`wGrwh0zKm;kR_t`*y`nSnt@27?FU z8%T2T@yH$D44E6?4d7UpkogO3B-)spQ;sKIV;&od50&tm7&QABTT z$^Q8_*e5k;?#@*-`1fjt;x^jXYp?~L^a0Q6OQZ5|AQqS4xi>_W@6kmUq*RR&2muY0 zfm9%Tt!Y$fqvJ5;+9%m1MTDomCethk`YJ>bdRg^jXsJP9<7W@OB2(|WCQX{-$VPnRWC?_lmC&E-q)VL{lpD&?rHmsMW zkL~zs^@i^RJ<|Ezv_6`cw@*Js&^FbZE85s6sE>MLp);7r zI&;Jsui6;o)|oj_V2KVoAXq>;q?snmPZ@VDuN%N^+r=||;U+F|eqsW;!d-sUl(7Q_0Lm}{QMFet zyh+UT>wo%)n1&@9+xwv9zr{ejFKnEs_uMKJ&y&1|=K;~7d?H_W}vAX?R zieN7pC7v%nHHVISB-W*}ca*?9Kf2+qcN^t=+G7Nfy1QXXi_{sWdKHSlDfczOC5^~4 z@|1$52~-KEQ-OQXd|EDqx6%9D)glDktyn?d0k(D_m6DfP4^+xV_R=b;w?X!mSM>?KSSo~HRrS>wBJfu= z+ksl|ZPbDu7!CLps^8ZHC6)VYOt1AbOo>+sBLoa3wU_d|G@pQ#eFst#tdyxV<4nS@ z;$@ki;rnVkyhw=?K+tg*0egBu=X<2|^uH?r>9cKT&kwj%QLW#|_V=jsU>^w}kevvp zx|-&^N`{2wLjV}fH(F-CJXJOJqDQBi8!?Xz@GmNOj4a^-*v@8|XMlwHxAqcUMJ`Au z?}$}Vxqx`yrxC#diUu9{;@}VeI&vqQPTz|AG@)KSAZ9!p)sRHbd|{h=YipgBSIlx2 zdGRnd|C{ zb7S12eVEygPV87Rp6Kq%r`SYOx;6a}s{ZNWBD`M-dK2^+{m{Q%ll&p;j$}6C10TeJ z}VIV!(j zb*k5MklSrr5|3ZpSFtzC4;ub;Bk*1h0)58zm!6hE*J!4EZ3Dj)cfFMsGB~HUMgs*Q z{Rm^p;v5Jx`_$DnPDs(2^c(5HJadMWF6CRTP7M?9j*y9?4VPf@zBrfccBzi14O7C< zpzdbPhE~FND`Ee&#n*?maL0T_u8imht@nVn&xhg;5p>$XyV%SBC)ZCK^Z$@28!QhT z(w+^~H%awc(sW$I*Vv z_X4l2IY+q|T2HiMd)F%4ud+76niCo50c^i3f7*ySv=B*iNa{!{OQU`|VVu17r4d1S z?H{d)(G)AIdIUfw_$~CP&pDV?$bkT}<~gv^G)G9}Kgnebef@QeWFif?TnP?QFPv|7Z`gYQ+`9{N?#>d|{C8Isv< zQ}p!ph{*Nyy~2&EgCHkQ{lAWV2CgS#7O?$;Sq@c&l5hXqn}OD zA$%&{xGd9gQV-Y?SI2YYJilztT{w1#`I)Q*x(iT!ZlXkW+x*&wHKch;1g-bb#`ELz z)o==TW$z|sZ)w@2U*2Dxc~0*Kk-eV~Mre|0Y^O}aJSPiq{6<{PvF)NBlL@6fKj?n> zWD09HO=1f)X6g#);~xu8Rz!4hx`tR|FE%y6x%Ka!bcr&VkWI)E>ofgU7gxdoZn!%v z-{8yXuI_DN4m&4~IoKQNUHp?>zPbD2(%0XKMAS;S+^Y@by=!tC-3xB_M75_-z1rre zDygmU7?gp}(P5&&tZ9g3+P`CWN84_OU-PCC71pAjMW!UP|Gtl4PXnyyZ%bztBOYlx zYvTD$>s#_P9R5k?k6C~;Fm|hDP&2an=~1HV(H`WXU*c!8%Xx#m1pDG){0r~HU?T2h7!Swz(o28=F%=>&xytE2%MV}jL>9%d#Mm!BGH5Q8{aP-N2sxC@{MW=Uc7an(IPW1A zAUS0O_luEH%U;3Dz2X4_fu+HfJ|UJu^lza|<@PNi1d*53Y$H=FzVNONn69pl z1b!7ichK8YnrA&~+CirX)$GLXe1H61j3^E6hUfh}-F}AkttYjtNu7m@qm;dDzWTFW z16a4NtUWO+yqsJyuqawwGMu15LxSH~IBwJD_uHi_On>ZE|K5KSm{WFanQRmk}j84kiGChoSI&lkD2&?r|4-8wXz}QE6905bQ58B0|2s9FpFIv;urc%8w1axyl*q z``<{_TWg`>GeHFRlAqo}atuW6BAd7nJ_`>}{|ljGlVQrZYDu+xqOAsE>|e;` z-^3YTeW18sY#h%i`+Ue8#Qyq}r}*U7Ld&!=2PniLDYPzR4n3&f1Xjw@rOAT3Le4pm zZX|L9J62vz%I`T4nTU0ogc>PbWU*>iK*aq;-X`&%{=5sECQ4bZn>J_tgS2#LrKqqD oCa6$IPPKuZ3Eqha$LhuFL|t|PsC(Bd`8xRa?(`hy&sX2cX>p8L5C8xG diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect deleted file mode 100644 index 7812c1c62da3cbaeb6399e9aa8ab65ae7efa9b08..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32 ocmaEJ(2|$IfP>+{UeCQBetd7^G}D{T$iTpm^J~2nL&Iw}0NYm#xc~qF diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput deleted file mode 100644 index 7812c1c62da3cbaeb6399e9aa8ab65ae7efa9b08..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32 ocmaEJ(2|$IfP>+{UeCQBetd7^G}D{T$iTpm^J~2nL&Iw}0NYm#xc~qF diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect deleted file mode 100644 index f5133778e1c783a6da12f76fec3f04014c77694e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1812 zcmZQM;K<8hAkcE4esWdg(f!+{1xLYX2#kinFbsi-|F1=5uiZLIjE2EzI>3_+tN@OD BEKdLc diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.input b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.input deleted file mode 100644 index 7c7a50d158..0000000000 --- a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.input +++ /dev/null @@ -1,2 +0,0 @@ -101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010 -232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323 \ No newline at end of file diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect deleted file mode 100644 index 7812c1c62da3cbaeb6399e9aa8ab65ae7efa9b08..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32 ocmaEJ(2|$IfP>+{UeCQBetd7^G}D{T$iTpm^J~2nL&Iw}0NYm#xc~qF diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput deleted file mode 100644 index 7812c1c62da3cbaeb6399e9aa8ab65ae7efa9b08..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32 ocmaEJ(2|$IfP>+{UeCQBetd7^G}D{T$iTpm^J~2nL&Iw}0NYm#xc~qF diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect deleted file mode 100644 index 71ce3aeb75a86e8375d9ac4350b7d83b9229a3ed..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 231 zcmVb2j)h-%-Q8H+K zIkmg!?Y-=9be1Hi$&iwP9DQ6&foC2grh=5#ja@KiZ1-F{b`bob2j)h-%-Q8H+K zIkmg!?Y-=9be1Hi$&iwP9DQ6&foC2grh=5#ja@KiZ1-F{b`bow{lnwaYQ1@WJ+zb2j)h-%-Q8H+K zIkmg!?Y-=9be1Hi$&iwP9DQ6&foC2grh=5#ja@KiZ1-F{b`bob2j)h-%-Q8H+K zIkmg!?Y-=9be1Hi$&iwP9DQ6&foC2grh=5#ja@KiZ1-F{b`boihFqUl1P&?kcmudcm!G0Ch?Cd*49 z?n4b5&}tWCj(=n43}yiUqmHOi;c~hTFSA;3INm*W!Q3rrN(zX%eD4;{rJtc TG)0&A&3jp26wx&=#Ug$H8a`?5 diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput deleted file mode 100644 index fbffc3f36b78cc48e8c3a33f97fa86f1d0a52272..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 217 zcmV;~04D!Dj?rqgFc5|B^AzXzDlY$yS<04z2<=T@Vp&QwlWri!B}Uy=eD|W3&c$5J zH+<(51-{)UMnxw@N)!c}$T~4Jtn<7s&jyoAH>ihFqUl1P&?kcmudcm!G0Ch?Cd*49 z?n4b5&}tWCj(=n43}yiUqmHOi;c~hTFSA;3INm*W!Q3rrN(zX%eD4;{rJtc TG)0&A&3jp26wx&=#Ug$H8a`?5 diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect b/lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect deleted file mode 100644 index 46fa51fdad7c4186a2cbe48866d511bdf6fb75ea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 219 zcmV<103`nez}<@CFc^U0^Y1F&?NKY5Mi!TaQrJD2n-CddlZ=5hl_d3N#CxBo@A(d2 z+_c(jrKRtgvNP3T@F6;Uh|yW@pihFqUl1P&?kcmudcm!G0Ch?Cd*49 z?n4b5&}tWCj(=n43}yiUqmHOi;c~hTFSA;3INm*W!Q3rrN(zX%eD4;{rJtc TG)0&A&3jp26wx&=#Ug$H8a`?5 diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput deleted file mode 100644 index fbffc3f36b78cc48e8c3a33f97fa86f1d0a52272..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 217 zcmV;~04D!Dj?rqgFc5|B^AzXzDlY$yS<04z2<=T@Vp&QwlWri!B}Uy=eD|W3&c$5J zH+<(51-{)UMnxw@N)!c}$T~4Jtn<7s&jyoAH>ihFqUl1P&?kcmudcm!G0Ch?Cd*49 z?n4b5&}tWCj(=n43}yiUqmHOi;c~hTFSA;3INm*W!Q3rrN(zX%eD4;{rJtc TG)0&A&3jp26wx&=#Ug$H8a`?5 diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect deleted file mode 100644 index 830348a79ad9ab38d0edc449e8335c056f7d185f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17 XcmaEJU?T$%G#D)X^D^m0zK$>eMUV%O diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput deleted file mode 100644 index 830348a79ad9ab38d0edc449e8335c056f7d185f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17 XcmaEJU?T$%G#D)X^D^m0zK$>eMUV%O diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect b/lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect deleted file mode 100644 index 5abdbaff9a69ad9c71178ba3641fa548818c9030..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 51 VcmZQM(8vG+6IB0~f*Aw}2LPDS1Frx8 diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-zero.input b/lib/std/compress/flate/testdata/block_writer/huffman-zero.input deleted file mode 100644 index 349be0e6ec..0000000000 --- a/lib/std/compress/flate/testdata/block_writer/huffman-zero.input +++ /dev/null @@ -1 +0,0 @@ -00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 \ No newline at end of file diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect deleted file mode 100644 index dbe401c54c4b6f45f3169376185a476dcf00dde9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6 NcmXq#U{zse0006o0CxZY diff --git a/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput deleted file mode 100644 index dbe401c54c4b6f45f3169376185a476dcf00dde9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6 NcmXq#U{zse0006o0CxZY diff --git a/lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput deleted file mode 100644 index 8b92d9fc20f1ee1fea5e4cc84d18aeea26a6fdaa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 206 ccmaEJz>txFf#HzC@8#d3xFvwhAq<`X0E^!Sx&QzG diff --git a/lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput deleted file mode 100644 index 8b92d9fc20f1ee1fea5e4cc84d18aeea26a6fdaa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 206 ccmaEJz>txFf#HzC@8#d3xFvwhAq<`X0E^!Sx&QzG From 73c98ca0e6aa52b942b92135ecf0305362030733 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 27 Jul 2025 14:10:55 -0700 Subject: [PATCH 03/47] simplify std.hash.Adler32 --- lib/std/hash.zig | 5 +- lib/std/hash/Adler32.zig | 117 ++++++++++++++++++++++++++++++++++ lib/std/hash/adler.zig | 134 --------------------------------------- lib/std/hash/verify.zig | 2 +- 4 files changed, 120 insertions(+), 138 deletions(-) create mode 100644 lib/std/hash/Adler32.zig delete mode 100644 lib/std/hash/adler.zig diff --git a/lib/std/hash.zig b/lib/std/hash.zig index 5f1697a236..781971bd13 100644 --- a/lib/std/hash.zig +++ b/lib/std/hash.zig @@ -1,5 +1,4 @@ -const adler = @import("hash/adler.zig"); -pub const Adler32 = adler.Adler32; +pub const Adler32 = @import("hash/Adler32.zig"); const auto_hash = @import("hash/auto_hash.zig"); pub const autoHash = auto_hash.autoHash; @@ -116,7 +115,7 @@ test int { } test { - _ = adler; + _ = Adler32; _ = auto_hash; _ = crc; _ = fnv; diff --git a/lib/std/hash/Adler32.zig b/lib/std/hash/Adler32.zig new file mode 100644 index 0000000000..6932dc59da --- /dev/null +++ b/lib/std/hash/Adler32.zig @@ -0,0 +1,117 @@ +//! https://tools.ietf.org/html/rfc1950#section-9 +//! https://github.com/madler/zlib/blob/master/adler32.c + +const Adler32 = @This(); +const std = @import("std"); +const testing = std.testing; + +adler: u32 = 1, + +pub fn permute(state: u32, input: []const u8) u32 { + const base = 65521; + const nmax = 5552; + + var s1 = state & 0xffff; + var s2 = (state >> 16) & 0xffff; + + if (input.len == 1) { + s1 +%= input[0]; + if (s1 >= base) { + s1 -= base; + } + s2 +%= s1; + if (s2 >= base) { + s2 -= base; + } + } else if (input.len < 16) { + for (input) |b| { + s1 +%= b; + s2 +%= s1; + } + if (s1 >= base) { + s1 -= base; + } + + s2 %= base; + } else { + const n = nmax / 16; // note: 16 | nmax + + var i: usize = 0; + + while (i + nmax <= input.len) { + var rounds: usize = 0; + while (rounds < n) : (rounds += 1) { + comptime var j: usize = 0; + inline while (j < 16) : (j += 1) { + s1 +%= input[i + j]; + s2 +%= s1; + } + i += 16; + } + + s1 %= base; + s2 %= base; + } + + if (i < input.len) { + while (i + 16 <= input.len) : (i += 16) { + comptime var j: usize = 0; + inline while (j < 16) : (j += 1) { + s1 +%= input[i + j]; + s2 +%= s1; + } + } + while (i < input.len) : (i += 1) { + s1 +%= input[i]; + s2 +%= s1; + } + + s1 %= base; + s2 %= base; + } + } + + return s1 | (s2 << 16); +} + +pub fn update(a: *Adler32, input: []const u8) void { + a.adler = permute(a.adler, input); +} + +pub fn hash(input: []const u8) u32 { + return permute(1, input); +} + +test "sanity" { + try testing.expectEqual(@as(u32, 0x620062), hash("a")); + try testing.expectEqual(@as(u32, 0xbc002ed), hash("example")); +} + +test "long" { + const long1 = [_]u8{1} ** 1024; + try testing.expectEqual(@as(u32, 0x06780401), hash(long1[0..])); + + const long2 = [_]u8{1} ** 1025; + try testing.expectEqual(@as(u32, 0x0a7a0402), hash(long2[0..])); +} + +test "very long" { + const long = [_]u8{1} ** 5553; + try testing.expectEqual(@as(u32, 0x707f15b2), hash(long[0..])); +} + +test "very long with variation" { + const long = comptime blk: { + @setEvalBranchQuota(7000); + var result: [6000]u8 = undefined; + + var i: usize = 0; + while (i < result.len) : (i += 1) { + result[i] = @as(u8, @truncate(i)); + } + + break :blk result; + }; + + try testing.expectEqual(@as(u32, 0x5af38d6e), hash(long[0..])); +} diff --git a/lib/std/hash/adler.zig b/lib/std/hash/adler.zig deleted file mode 100644 index 52f7b2691a..0000000000 --- a/lib/std/hash/adler.zig +++ /dev/null @@ -1,134 +0,0 @@ -// Adler32 checksum. -// -// https://tools.ietf.org/html/rfc1950#section-9 -// https://github.com/madler/zlib/blob/master/adler32.c - -const std = @import("std"); -const testing = std.testing; - -pub const Adler32 = struct { - const base = 65521; - const nmax = 5552; - - adler: u32, - - pub fn init() Adler32 { - return Adler32{ .adler = 1 }; - } - - // This fast variant is taken from zlib. It reduces the required modulos and unrolls longer - // buffer inputs and should be much quicker. - pub fn update(self: *Adler32, input: []const u8) void { - var s1 = self.adler & 0xffff; - var s2 = (self.adler >> 16) & 0xffff; - - if (input.len == 1) { - s1 +%= input[0]; - if (s1 >= base) { - s1 -= base; - } - s2 +%= s1; - if (s2 >= base) { - s2 -= base; - } - } else if (input.len < 16) { - for (input) |b| { - s1 +%= b; - s2 +%= s1; - } - if (s1 >= base) { - s1 -= base; - } - - s2 %= base; - } else { - const n = nmax / 16; // note: 16 | nmax - - var i: usize = 0; - - while (i + nmax <= input.len) { - var rounds: usize = 0; - while (rounds < n) : (rounds += 1) { - comptime var j: usize = 0; - inline while (j < 16) : (j += 1) { - s1 +%= input[i + j]; - s2 +%= s1; - } - i += 16; - } - - s1 %= base; - s2 %= base; - } - - if (i < input.len) { - while (i + 16 <= input.len) : (i += 16) { - comptime var j: usize = 0; - inline while (j < 16) : (j += 1) { - s1 +%= input[i + j]; - s2 +%= s1; - } - } - while (i < input.len) : (i += 1) { - s1 +%= input[i]; - s2 +%= s1; - } - - s1 %= base; - s2 %= base; - } - } - - self.adler = s1 | (s2 << 16); - } - - pub fn final(self: *Adler32) u32 { - return self.adler; - } - - pub fn hash(input: []const u8) u32 { - var c = Adler32.init(); - c.update(input); - return c.final(); - } -}; - -test "adler32 sanity" { - try testing.expectEqual(@as(u32, 0x620062), Adler32.hash("a")); - try testing.expectEqual(@as(u32, 0xbc002ed), Adler32.hash("example")); -} - -test "adler32 long" { - const long1 = [_]u8{1} ** 1024; - try testing.expectEqual(@as(u32, 0x06780401), Adler32.hash(long1[0..])); - - const long2 = [_]u8{1} ** 1025; - try testing.expectEqual(@as(u32, 0x0a7a0402), Adler32.hash(long2[0..])); -} - -test "adler32 very long" { - const long = [_]u8{1} ** 5553; - try testing.expectEqual(@as(u32, 0x707f15b2), Adler32.hash(long[0..])); -} - -test "adler32 very long with variation" { - const long = comptime blk: { - @setEvalBranchQuota(7000); - var result: [6000]u8 = undefined; - - var i: usize = 0; - while (i < result.len) : (i += 1) { - result[i] = @as(u8, @truncate(i)); - } - - break :blk result; - }; - - try testing.expectEqual(@as(u32, 0x5af38d6e), std.hash.Adler32.hash(long[0..])); -} - -const verify = @import("verify.zig"); - -test "adler32 iterative" { - try verify.iterativeApi(Adler32); -} diff --git a/lib/std/hash/verify.zig b/lib/std/hash/verify.zig index 61f501a881..a96a36c050 100644 --- a/lib/std/hash/verify.zig +++ b/lib/std/hash/verify.zig @@ -45,7 +45,7 @@ pub fn smhasher(comptime hash_fn: anytype) u32 { pub fn iterativeApi(comptime Hash: anytype) !void { // Sum(1..32) = 528 - var buf: [528]u8 = [_]u8{0} ** 528; + var buf: [528]u8 = @splat(0); var len: usize = 0; const seed = 0; From 824c157e0c25a9337ffd036f7ad5cb811b1f18cd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 27 Jul 2025 16:05:32 -0700 Subject: [PATCH 04/47] std.compress.flate: finish reorganizing --- lib/std/compress/flate.zig | 505 +--------------------- lib/std/compress/flate/BlockWriter.zig | 21 +- lib/std/compress/flate/Compress.zig | 291 +------------ lib/std/compress/flate/Decompress.zig | 217 ++++++++-- lib/std/compress/flate/HuffmanEncoder.zig | 5 +- lib/std/compress/flate/Lookup.zig | 15 +- lib/std/compress/flate/Token.zig | 20 +- 7 files changed, 271 insertions(+), 803 deletions(-) diff --git a/lib/std/compress/flate.zig b/lib/std/compress/flate.zig index 73f98271a4..032ea8a779 100644 --- a/lib/std/compress/flate.zig +++ b/lib/std/compress/flate.zig @@ -1,7 +1,23 @@ -const builtin = @import("builtin"); const std = @import("../std.zig"); -const testing = std.testing; -const Writer = std.Io.Writer; + +/// When decompressing, the output buffer is used as the history window, so +/// less than this may result in failure to decompress streams that were +/// compressed with a larger window. +pub const max_window_len = history_len * 2; + +pub const history_len = 32768; + +/// Deflate is a lossless data compression file format that uses a combination +/// of LZ77 and Huffman coding. +pub const Compress = @import("flate/Compress.zig"); + +/// Inflate is the decoding process that takes a Deflate bitstream for +/// decompression and correctly produces the original full-size data or file. +pub const Decompress = @import("flate/Decompress.zig"); + +/// Compression without Lempel-Ziv match searching. Faster compression, less +/// memory requirements but bigger compressed sizes. +pub const HuffmanEncoder = @import("flate/HuffmanEncoder.zig"); /// Container of the deflate bit stream body. Container adds header before /// deflate bit stream and footer after. It can bi gzip, zlib or raw (no header, @@ -13,7 +29,6 @@ const Writer = std.Io.Writer; /// Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes /// crc32 checksum and 4 bytes of uncompressed data length. /// -/// /// rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4 /// rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5 pub const Container = enum { @@ -84,7 +99,7 @@ pub const Container = enum { pub fn init(containter: Container) Hasher { return switch (containter) { .gzip => .{ .gzip = .{} }, - .zlib => .{ .zlib = .init() }, + .zlib => .{ .zlib = .{} }, .raw => .raw, }; } @@ -107,7 +122,7 @@ pub const Container = enum { } } - pub fn writeFooter(hasher: *Hasher, writer: *Writer) Writer.Error!void { + pub fn writeFooter(hasher: *Hasher, writer: *std.Io.Writer) std.Io.Writer.Error!void { var bits: [4]u8 = undefined; switch (hasher.*) { .gzip => |*gzip| { @@ -135,484 +150,6 @@ pub const Container = enum { }; }; -/// When decompressing, the output buffer is used as the history window, so -/// less than this may result in failure to decompress streams that were -/// compressed with a larger window. -pub const max_window_len = 1 << 16; - -/// Deflate is a lossless data compression file format that uses a combination -/// of LZ77 and Huffman coding. -pub const Compress = @import("flate/Compress.zig"); - -/// Inflate is the decoding process that takes a Deflate bitstream for -/// decompression and correctly produces the original full-size data or file. -pub const Decompress = @import("flate/Decompress.zig"); - -/// Compression without Lempel-Ziv match searching. Faster compression, less -/// memory requirements but bigger compressed sizes. -pub const HuffmanEncoder = @import("flate/HuffmanEncoder.zig"); - -test "compress/decompress" { - const print = std.debug.print; - var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer - var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer - - const levels = [_]Compress.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 }; - const cases = [_]struct { - data: []const u8, // uncompressed content - // compressed data sizes per level 4-9 - gzip_sizes: [levels.len]usize = [_]usize{0} ** levels.len, - huffman_only_size: usize = 0, - store_size: usize = 0, - }{ - .{ - .data = @embedFile("flate/testdata/rfc1951.txt"), - .gzip_sizes = [_]usize{ 11513, 11217, 11139, 11126, 11122, 11119 }, - .huffman_only_size = 20287, - .store_size = 36967, - }, - .{ - .data = @embedFile("flate/testdata/fuzz/roundtrip1.input"), - .gzip_sizes = [_]usize{ 373, 370, 370, 370, 370, 370 }, - .huffman_only_size = 393, - .store_size = 393, - }, - .{ - .data = @embedFile("flate/testdata/fuzz/roundtrip2.input"), - .gzip_sizes = [_]usize{ 373, 373, 373, 373, 373, 373 }, - .huffman_only_size = 394, - .store_size = 394, - }, - .{ - .data = @embedFile("flate/testdata/fuzz/deflate-stream.expect"), - .gzip_sizes = [_]usize{ 351, 347, 347, 347, 347, 347 }, - .huffman_only_size = 498, - .store_size = 747, - }, - }; - - for (cases, 0..) |case, case_no| { - const data = case.data; - - for (levels, 0..) |level, i| { - for (Container.list) |container| { - var compressed_size: usize = if (case.gzip_sizes[i] > 0) - case.gzip_sizes[i] - Container.gzip.size() + container.size() - else - 0; - - // compress original stream to compressed stream - { - var compressed: Writer = .fixed(&cmp_buf); - var compress: Compress = .init(&compressed, &.{}, .{ .container = .raw, .level = level }); - try compress.writer.writeAll(data); - try compress.end(); - - if (compressed_size == 0) { - if (container == .gzip) - print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos }); - compressed_size = compressed.end; - } - try testing.expectEqual(compressed_size, compressed.end); - } - // decompress compressed stream to decompressed stream - { - var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]); - var decompressed: Writer = .fixed(&dcm_buf); - var decompress: Decompress = .init(&compressed, container, &.{}); - _ = try decompress.reader.streamRemaining(&decompressed); - try testing.expectEqualSlices(u8, data, decompressed.buffered()); - } - - // compressor writer interface - { - var compressed: Writer = .fixed(&cmp_buf); - var cmp = try Compress.init(&compressed, &.{}, .{ - .level = level, - .container = container, - }); - var cmp_wrt = cmp.writer(); - try cmp_wrt.writeAll(data); - try cmp.finish(); - - try testing.expectEqual(compressed_size, compressed.pos); - } - // decompressor reader interface - { - var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]); - var decompress: Decompress = .init(&compressed, container, &.{}); - const n = try decompress.reader.readSliceShort(&dcm_buf); - try testing.expectEqual(data.len, n); - try testing.expectEqualSlices(u8, data, dcm_buf[0..n]); - } - } - } - // huffman only compression - { - for (Container.list) |container| { - var compressed_size: usize = if (case.huffman_only_size > 0) - case.huffman_only_size - Container.gzip.size() + container.size() - else - 0; - - // compress original stream to compressed stream - { - var original: std.Io.Reader = .fixed(data); - var compressed: Writer = .fixed(&cmp_buf); - var cmp = try Compress.Huffman.init(container, &compressed); - try cmp.compress(original.reader()); - try cmp.finish(); - if (compressed_size == 0) { - if (container == .gzip) - print("case {d} huffman only compressed size: {d}\n", .{ case_no, compressed.pos }); - compressed_size = compressed.pos; - } - try testing.expectEqual(compressed_size, compressed.pos); - } - // decompress compressed stream to decompressed stream - { - var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]); - var decompress: Decompress = .init(&compressed, container, &.{}); - var decompressed: Writer = .fixed(&dcm_buf); - _ = try decompress.reader.streamRemaining(&decompressed); - try testing.expectEqualSlices(u8, data, decompressed.buffered()); - } - } - } - - // store only - { - for (Container.list) |container| { - var compressed_size: usize = if (case.store_size > 0) - case.store_size - Container.gzip.size() + container.size() - else - 0; - - // compress original stream to compressed stream - { - var original: std.Io.Reader = .fixed(data); - var compressed: Writer = .fixed(&cmp_buf); - var cmp = try Compress.SimpleCompressor(.store, container).init(&compressed); - try cmp.compress(original.reader()); - try cmp.finish(); - if (compressed_size == 0) { - if (container == .gzip) - print("case {d} store only compressed size: {d}\n", .{ case_no, compressed.pos }); - compressed_size = compressed.pos; - } - - try testing.expectEqual(compressed_size, compressed.pos); - } - // decompress compressed stream to decompressed stream - { - var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]); - var decompress: Decompress = .init(&compressed, container, &.{}); - var decompressed: Writer = .fixed(&dcm_buf); - _ = try decompress.reader.streamRemaining(&decompressed); - try testing.expectEqualSlices(u8, data, decompressed.buffered()); - } - } - } - } -} - -fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void { - var in: std.Io.Reader = .fixed(compressed); - var aw: std.Io.Writer.Allocating = .init(testing.allocator); - defer aw.deinit(); - - var decompress: Decompress = .init(&in, container, &.{}); - _ = try decompress.reader.streamRemaining(&aw.writer); - try testing.expectEqualSlices(u8, expected_plain, aw.getWritten()); -} - -test "don't read past deflate stream's end" { - try testDecompress(.zlib, &[_]u8{ - 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff, - 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01, - 0x83, 0x95, 0x0b, 0xf5, - }, &[_]u8{ - 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, - 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, - }); -} - -test "zlib header" { - // Truncated header - try testing.expectError( - error.EndOfStream, - testDecompress(.zlib, &[_]u8{0x78}, ""), - ); - // Wrong CM - try testing.expectError( - error.BadZlibHeader, - testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""), - ); - // Wrong CINFO - try testing.expectError( - error.BadZlibHeader, - testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""), - ); - // Wrong checksum - try testing.expectError( - error.WrongZlibChecksum, - testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""), - ); - // Truncated checksum - try testing.expectError( - error.EndOfStream, - testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""), - ); -} - -test "gzip header" { - // Truncated header - try testing.expectError( - error.EndOfStream, - testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined), - ); - // Wrong CM - try testing.expectError( - error.BadGzipHeader, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, - }, undefined), - ); - - // Wrong checksum - try testing.expectError( - error.WrongGzipChecksum, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, - }, undefined), - ); - // Truncated checksum - try testing.expectError( - error.EndOfStream, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, - }, undefined), - ); - // Wrong initial size - try testing.expectError( - error.WrongGzipSize, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, - }, undefined), - ); - // Truncated initial size field - try testing.expectError( - error.EndOfStream, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, - }, undefined), - ); - - try testDecompress(.gzip, &[_]u8{ - // GZIP header - 0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00, - // header.FHCRC (should cover entire header) - 0x99, 0xd6, - // GZIP data - 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - }, ""); -} - -test "public interface" { - const plain_data_buf = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a }; - - // deflate final stored block, header + plain (stored) data - const deflate_block = [_]u8{ - 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen - } ++ plain_data_buf; - - const plain_data: []const u8 = &plain_data_buf; - const gzip_data: []const u8 = &deflate_block; - - //// gzip header/footer + deflate block - //const gzip_data = - // [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes) - // deflate_block ++ - // [_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes) - - //// zlib header/footer + deflate block - //const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)} - // deflate_block ++ - // [_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum - - // TODO - //const gzip = @import("gzip.zig"); - //const zlib = @import("zlib.zig"); - - var buffer1: [64]u8 = undefined; - var buffer2: [64]u8 = undefined; - - // decompress - { - var plain: Writer = .fixed(&buffer2); - var in: std.Io.Reader = .fixed(gzip_data); - var d: Decompress = .init(&in, .raw, &.{}); - _ = try d.reader.streamRemaining(&plain); - try testing.expectEqualSlices(u8, plain_data, plain.buffered()); - } - - // compress/decompress - { - var plain: Writer = .fixed(&buffer2); - var compressed: Writer = .fixed(&buffer1); - - var cmp: Compress = .init(&compressed, &.{}, .{}); - try cmp.writer.writeAll(plain_data); - try cmp.end(); - - var r: std.Io.Reader = .fixed(&buffer1); - var d: Decompress = .init(&r, .raw, &.{}); - _ = try d.reader.streamRemaining(&plain); - try testing.expectEqualSlices(u8, plain_data, plain.buffered()); - } - - // compressor/decompressor - { - var plain: Writer = .fixed(&buffer2); - var compressed: Writer = .fixed(&buffer1); - - var cmp: Compress = .init(&compressed, &.{}, .{}); - try cmp.writer.writeAll(plain_data); - try cmp.end(); - - var r: std.Io.Reader = .fixed(&buffer1); - var dcp = Decompress(&r); - try dcp.decompress(&plain); - try testing.expectEqualSlices(u8, plain_data, plain.buffered()); - } - - // huffman - { - // huffman compress/decompress - { - var plain: Writer = .fixed(&buffer2); - var compressed: Writer = .fixed(&buffer1); - - var in: std.Io.Reader = .fixed(plain_data); - try HuffmanEncoder.compress(&in, &compressed); - - var r: std.Io.Reader = .fixed(&buffer1); - var d: Decompress = .init(&r, .raw, &.{}); - _ = try d.reader.streamRemaining(&plain); - try testing.expectEqualSlices(u8, plain_data, plain.buffered()); - } - - // huffman compressor/decompressor - { - var plain: Writer = .fixed(&buffer2); - var compressed: Writer = .fixed(&buffer1); - - var in: std.Io.Reader = .fixed(plain_data); - var cmp = try HuffmanEncoder.Compressor(&compressed); - try cmp.compress(&in); - try cmp.finish(); - - var r: std.Io.Reader = .fixed(&buffer1); - var d: Decompress = .init(&r, .raw, &.{}); - _ = try d.reader.streamRemaining(&plain); - try testing.expectEqualSlices(u8, plain_data, plain.buffered()); - } - } - - // TODO - //{ - // // store compress/decompress - // { - // var plain: Writer = .fixed(&buffer2); - // var compressed: Writer = .fixed(&buffer1); - - // var in: std.Io.Reader = .fixed(plain_data); - // try store.compress(&in, &compressed); - - // var r: std.Io.Reader = .fixed(&buffer1); - // var d: Decompress = .init(&r, .raw, &.{}); - // _ = try d.reader.streamRemaining(&plain); - // try testing.expectEqualSlices(u8, plain_data, plain.buffered()); - // } - - // // store compressor/decompressor - // { - // var plain: Writer = .fixed(&buffer2); - // var compressed: Writer = .fixed(&buffer1); - - // var in: std.Io.Reader = .fixed(plain_data); - // var cmp = try store.compressor(&compressed); - // try cmp.compress(&in); - // try cmp.finish(); - - // var r: std.Io.Reader = .fixed(&buffer1); - // var d: Decompress = .init(&r, .raw, &.{}); - // _ = try d.reader.streamRemaining(&plain); - // try testing.expectEqualSlices(u8, plain_data, plain.buffered()); - // } - //} -} - -pub const match = struct { - pub const base_length = 3; // smallest match length per the RFC section 3.2.5 - pub const min_length = 4; // min length used in this algorithm - pub const max_length = 258; - - pub const min_distance = 1; - pub const max_distance = 32768; -}; - -pub const history_len = match.max_distance; - -pub const lookup = struct { - pub const bits = 15; - pub const len = 1 << bits; - pub const shift = 32 - bits; -}; - -test "zlib should not overshoot" { - // Compressed zlib data with extra 4 bytes at the end. - const data = [_]u8{ - 0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9, - 0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08, - 0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34, - 0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12, - }; - - var reader: std.Io.Reader = .fixed(&data); - - var decompress: Decompress = .init(&reader, .zlib, &.{}); - var out: [128]u8 = undefined; - - { - const n = try decompress.reader.readSliceShort(out[0..]); - - // Expected decompressed data - try std.testing.expectEqual(46, n); - try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]); - - // Decompressor don't overshoot underlying reader. - // It is leaving it at the end of compressed data chunk. - try std.testing.expectEqual(data.len - 4, reader.seek); - // TODO what was this testing, exactly? - //try std.testing.expectEqual(0, decompress.unreadBytes()); - } - - // 4 bytes after compressed chunk are available in reader. - const n = try reader.readSliceShort(out[0..]); - try std.testing.expectEqual(n, 4); - try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]); -} - test { _ = HuffmanEncoder; _ = Compress; diff --git a/lib/std/compress/flate/BlockWriter.zig b/lib/std/compress/flate/BlockWriter.zig index b3af65051a..d0e9dc1203 100644 --- a/lib/std/compress/flate/BlockWriter.zig +++ b/lib/std/compress/flate/BlockWriter.zig @@ -31,7 +31,26 @@ fixed_literal_codes: [HuffmanEncoder.max_num_frequencies]HuffmanEncoder.Code, fixed_distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code, distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code, -pub fn init(bw: *BlockWriter) void { +pub fn init(output: *Writer) BlockWriter { + return .{ + .output = output, + .codegen_freq = undefined, + .literal_freq = undefined, + .distance_freq = undefined, + .codegen = undefined, + .literal_encoding = undefined, + .distance_encoding = undefined, + .codegen_encoding = undefined, + .fixed_literal_encoding = undefined, + .fixed_distance_encoding = undefined, + .huff_distance = undefined, + .fixed_literal_codes = undefined, + .fixed_distance_codes = undefined, + .distance_codes = undefined, + }; +} + +pub fn initBuffers(bw: *BlockWriter) void { bw.fixed_literal_encoding = .fixedLiteralEncoder(&bw.fixed_literal_codes); bw.fixed_distance_encoding = .fixedDistanceEncoder(&bw.fixed_distance_codes); bw.huff_distance = .huffmanDistanceEncoder(&bw.distance_codes); diff --git a/lib/std/compress/flate/Compress.zig b/lib/std/compress/flate/Compress.zig index f38f7b2703..2249ece4c0 100644 --- a/lib/std/compress/flate/Compress.zig +++ b/lib/std/compress/flate/Compress.zig @@ -122,22 +122,7 @@ pub const Options = struct { pub fn init(output: *Writer, buffer: []u8, options: Options) Compress { return .{ - .block_writer = .{ - .output = output, - .codegen_freq = undefined, - .literal_freq = undefined, - .distance_freq = undefined, - .codegen = undefined, - .literal_encoding = undefined, - .distance_encoding = undefined, - .codegen_encoding = undefined, - .fixed_literal_encoding = undefined, - .fixed_distance_encoding = undefined, - .huff_distance = undefined, - .fixed_literal_codes = undefined, - .fixed_distance_codes = undefined, - .distance_codes = undefined, - }, + .block_writer = .init(output), .level = .get(options.level), .hasher = .init(options.container), .state = .header, @@ -188,20 +173,21 @@ fn drain(me: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize } const buffered = me.buffered(); - const min_lookahead = flate.match.min_length + flate.match.max_length; + const min_lookahead = Token.min_length + Token.max_length; const history_plus_lookahead_len = flate.history_len + min_lookahead; if (buffered.len < history_plus_lookahead_len) return 0; const lookahead = buffered[flate.history_len..]; - _ = lookahead; // TODO tokenize + _ = lookahead; //c.hasher.update(lookahead[0..n]); @panic("TODO"); } pub fn end(c: *Compress) !void { try endUnflushed(c); - try c.output.flush(); + const out = c.block_writer.output; + try out.flush(); } pub fn endUnflushed(c: *Compress) !void { @@ -227,7 +213,7 @@ pub fn endUnflushed(c: *Compress) !void { // Checksum value of the uncompressed data (excluding any // dictionary data) computed according to Adler-32 // algorithm. - std.mem.writeInt(u32, try out.writableArray(4), zlib.final, .big); + std.mem.writeInt(u32, try out.writableArray(4), zlib.adler, .big); }, .raw => {}, } @@ -243,15 +229,16 @@ pub const Simple = struct { pub const Strategy = enum { huffman, store }; - pub fn init(out: *Writer, buffer: []u8, container: Container) !Simple { - const self: Simple = .{ + pub fn init(output: *Writer, buffer: []u8, container: Container, strategy: Strategy) !Simple { + const header = container.header(); + try output.writeAll(header); + return .{ .buffer = buffer, .wp = 0, - .block_writer = .init(out), + .block_writer = .init(output), .hasher = .init(container), + .strategy = strategy, }; - try container.writeHeader(self.out); - return self; } pub fn flush(self: *Simple) !void { @@ -263,7 +250,7 @@ pub const Simple = struct { pub fn finish(self: *Simple) !void { try self.flushBuffer(true); try self.block_writer.flush(); - try self.hasher.container().writeFooter(&self.hasher, self.out); + try self.hasher.container().writeFooter(&self.hasher, self.block_writer.output); } fn flushBuffer(self: *Simple, final: bool) !void { @@ -300,7 +287,13 @@ test "generate a Huffman code from an array of frequencies" { }; var codes: [19]HuffmanEncoder.Code = undefined; - var enc: HuffmanEncoder = .{ .codes = &codes }; + var enc: HuffmanEncoder = .{ + .codes = &codes, + .freq_cache = undefined, + .bit_count = undefined, + .lns = undefined, + .lfs = undefined, + }; enc.generate(freqs[0..], 7); try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..])); @@ -337,247 +330,3 @@ test "generate a Huffman code from an array of frequencies" { try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code); try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code); } - -test "tokenization" { - const L = Token.initLiteral; - const M = Token.initMatch; - - const cases = [_]struct { - data: []const u8, - tokens: []const Token, - }{ - .{ - .data = "Blah blah blah blah blah!", - .tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') }, - }, - .{ - .data = "ABCDEABCD ABCDEABCD", - .tokens = &[_]Token{ - L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '), - L('A'), M(10, 8), - }, - }, - }; - - for (cases) |c| { - inline for (Container.list) |container| { // for each wrapping - - var cw = std.Io.countingWriter(std.Io.null_writer); - const cww = cw.writer(); - var df = try Compress(container, @TypeOf(cww), TestTokenWriter).init(cww, .{}); - - _ = try df.write(c.data); - try df.flush(); - - // df.token_writer.show(); - try expect(df.block_writer.pos == c.tokens.len); // number of tokens written - try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match - - try testing.expectEqual(container.headerSize(), cw.bytes_written); - try df.finish(); - try testing.expectEqual(container.size(), cw.bytes_written); - } - } -} - -// Tests that tokens written are equal to expected token list. -const TestTokenWriter = struct { - const Self = @This(); - - pos: usize = 0, - actual: [128]Token = undefined, - - pub fn init(_: anytype) Self { - return .{}; - } - pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void { - for (tokens) |t| { - self.actual[self.pos] = t; - self.pos += 1; - } - } - - pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {} - - pub fn get(self: *Self) []Token { - return self.actual[0..self.pos]; - } - - pub fn show(self: *Self) void { - std.debug.print("\n", .{}); - for (self.get()) |t| { - t.show(); - } - } - - pub fn flush(_: *Self) !void {} -}; - -test "file tokenization" { - const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 }; - const cases = [_]struct { - data: []const u8, // uncompressed content - // expected number of tokens producet in deflate tokenization - tokens_count: [levels.len]usize = .{0} ** levels.len, - }{ - .{ - .data = @embedFile("testdata/rfc1951.txt"), - .tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 }, - }, - - .{ - .data = @embedFile("testdata/block_writer/huffman-null-max.input"), - .tokens_count = .{ 257, 257, 257, 257, 257, 257 }, - }, - .{ - .data = @embedFile("testdata/block_writer/huffman-pi.input"), - .tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 }, - }, - .{ - .data = @embedFile("testdata/block_writer/huffman-text.input"), - .tokens_count = .{ 235, 234, 234, 234, 234, 234 }, - }, - .{ - .data = @embedFile("testdata/fuzz/roundtrip1.input"), - .tokens_count = .{ 333, 331, 331, 331, 331, 331 }, - }, - .{ - .data = @embedFile("testdata/fuzz/roundtrip2.input"), - .tokens_count = .{ 334, 334, 334, 334, 334, 334 }, - }, - }; - - for (cases) |case| { // for each case - const data = case.data; - - for (levels, 0..) |level, i| { // for each compression level - var original: std.Io.Reader = .fixed(data); - - // buffer for decompressed data - var al = std.ArrayList(u8).init(testing.allocator); - defer al.deinit(); - const writer = al.writer(); - - // create compressor - const WriterType = @TypeOf(writer); - const TokenWriter = TokenDecoder(@TypeOf(writer)); - var cmp = try Compress(.raw, WriterType, TokenWriter).init(writer, .{ .level = level }); - - // Stream uncompressed `original` data to the compressor. It will - // produce tokens list and pass that list to the TokenDecoder. This - // TokenDecoder uses CircularBuffer from inflate to convert list of - // tokens back to the uncompressed stream. - try cmp.compress(original.reader()); - try cmp.flush(); - const expected_count = case.tokens_count[i]; - const actual = cmp.block_writer.tokens_count; - if (expected_count == 0) { - std.debug.print("actual token count {d}\n", .{actual}); - } else { - try testing.expectEqual(expected_count, actual); - } - - try testing.expectEqual(data.len, al.items.len); - try testing.expectEqualSlices(u8, data, al.items); - } - } -} - -const TokenDecoder = struct { - output: *Writer, - tokens_count: usize, - - pub fn init(output: *Writer) TokenDecoder { - return .{ - .output = output, - .tokens_count = 0, - }; - } - - pub fn write(self: *TokenDecoder, tokens: []const Token, _: bool, _: ?[]const u8) !void { - self.tokens_count += tokens.len; - for (tokens) |t| { - switch (t.kind) { - .literal => self.hist.write(t.literal()), - .match => try self.hist.writeMatch(t.length(), t.distance()), - } - if (self.hist.free() < 285) try self.flushWin(); - } - try self.flushWin(); - } - - fn flushWin(self: *TokenDecoder) !void { - while (true) { - const buf = self.hist.read(); - if (buf.len == 0) break; - try self.output.writeAll(buf); - } - } -}; - -test "store simple compressor" { - if (true) return error.SkipZigTest; - //const data = "Hello world!"; - //const expected = [_]u8{ - // 0x1, // block type 0, final bit set - // 0xc, 0x0, // len = 12 - // 0xf3, 0xff, // ~len - // 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', // - // //0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21, - //}; - - //var fbs: std.Io.Reader = .fixed(data); - //var al = std.ArrayList(u8).init(testing.allocator); - //defer al.deinit(); - - //var cmp = try store.compressor(.raw, al.writer()); - //try cmp.compress(&fbs); - //try cmp.finish(); - //try testing.expectEqualSlices(u8, &expected, al.items); - - //fbs = .fixed(data); - //try al.resize(0); - - //// huffman only compresoor will also emit store block for this small sample - //var hc = try huffman.compressor(.raw, al.writer()); - //try hc.compress(&fbs); - //try hc.finish(); - //try testing.expectEqualSlices(u8, &expected, al.items); -} - -test "sliding window match" { - const data = "Blah blah blah blah blah!"; - var win: Writer = .{}; - try expect(win.write(data) == data.len); - try expect(win.wp == data.len); - try expect(win.rp == 0); - - // length between l symbols - try expect(win.match(1, 6, 0) == 18); - try expect(win.match(1, 11, 0) == 13); - try expect(win.match(1, 16, 0) == 8); - try expect(win.match(1, 21, 0) == 0); - - // position 15 = "blah blah!" - // position 20 = "blah!" - try expect(win.match(15, 20, 0) == 4); - try expect(win.match(15, 20, 3) == 4); - try expect(win.match(15, 20, 4) == 0); -} - -test "sliding window slide" { - var win: Writer = .{}; - win.wp = Writer.buffer_len - 11; - win.rp = Writer.buffer_len - 111; - win.buffer[win.rp] = 0xab; - try expect(win.lookahead().len == 100); - try expect(win.tokensBuffer().?.len == win.rp); - - const n = win.slide(); - try expect(n == 32757); - try expect(win.buffer[win.rp] == 0xab); - try expect(win.rp == Writer.hist_len - 111); - try expect(win.wp == Writer.hist_len - 11); - try expect(win.lookahead().len == 100); - try expect(win.tokensBuffer() == null); -} diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index ed9c0f3798..5f603baf9a 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -4,8 +4,8 @@ const Container = flate.Container; const Token = @import("Token.zig"); const testing = std.testing; const Decompress = @This(); -const Writer = std.io.Writer; -const Reader = std.io.Reader; +const Writer = std.Io.Writer; +const Reader = std.Io.Reader; input: *Reader, reader: Reader, @@ -129,7 +129,7 @@ fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol { return sym; } -pub fn stream(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!usize { +pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize { const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); return readInner(d, w, limit) catch |err| switch (err) { error.EndOfStream => return error.EndOfStream, @@ -143,7 +143,8 @@ pub fn stream(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!us }; } -fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.StreamError)!usize { +fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.StreamError)!usize { + var remaining = @intFromEnum(limit); const in = d.input; sw: switch (d.state) { .protocol_header => switch (d.hasher.container()) { @@ -182,15 +183,9 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S continue :sw .block_header; }, .zlib => { - const Header = extern struct { - cmf: packed struct(u8) { - cm: u4, - cinfo: u4, - }, - flg: u8, - }; - const header = try in.takeStruct(Header); - if (header.cmf.cm != 8 or header.cmf.cinfo > 7) return error.BadZlibHeader; + const header = try in.takeArray(2); + const cmf: packed struct(u8) { cm: u4, cinfo: u4 } = @bitCast(header[0]); + if (cmf.cm != 8 or cmf.cinfo > 7) return error.BadZlibHeader; continue :sw .block_header; }, .raw => continue :sw .block_header, @@ -219,7 +214,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S // lengths for code lengths var cl_lens = [_]u4{0} ** 19; for (0..hclen) |i| { - cl_lens[flate.huffman.codegen_order[i]] = try d.takeBits(u3); + cl_lens[flate.HuffmanEncoder.codegen_order[i]] = try d.takeBits(u3); } var cl_dec: CodegenDecoder = .{}; try cl_dec.generate(&cl_lens); @@ -259,52 +254,56 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S return n; }, .fixed_block => { - const start = w.count; - while (@intFromEnum(limit) > w.count - start) { + while (remaining > 0) { const code = try d.readFixedCode(); switch (code) { - 0...255 => try w.writeBytePreserve(flate.history_len, @intCast(code)), + 0...255 => { + try w.writeBytePreserve(flate.history_len, @intCast(code)); + remaining -= 1; + }, 256 => { d.state = if (d.final_block) .protocol_footer else .block_header; - return w.count - start; + return @intFromEnum(limit) - remaining; }, 257...285 => { // Handles fixed block non literal (length) code. // Length code is followed by 5 bits of distance code. const length = try d.decodeLength(@intCast(code - 257)); const distance = try d.decodeDistance(try d.takeBitsReverseBuffered(u5)); - try writeMatch(w, length, distance); + remaining = try writeMatch(w, length, distance, remaining); }, else => return error.InvalidCode, } } d.state = .fixed_block; - return w.count - start; + return @intFromEnum(limit) - remaining; }, .dynamic_block => { - // In larger archives most blocks are usually dynamic, so decompression - // performance depends on this logic. - const start = w.count; - while (@intFromEnum(limit) > w.count - start) { + // In larger archives most blocks are usually dynamic, so + // decompression performance depends on this logic. + while (remaining > 0) { const sym = try d.decodeSymbol(&d.lit_dec); switch (sym.kind) { - .literal => try w.writeBytePreserve(flate.history_len, sym.symbol), + .literal => { + try w.writeBytePreserve(flate.history_len, sym.symbol); + remaining -= 1; + }, .match => { // Decode match backreference const length = try d.decodeLength(sym.symbol); const dsm = try d.decodeSymbol(&d.dst_dec); const distance = try d.decodeDistance(dsm.symbol); - try writeMatch(w, length, distance); + remaining = try writeMatch(w, length, distance, remaining); }, .end_of_block => { d.state = if (d.final_block) .protocol_footer else .block_header; - return w.count - start; + return @intFromEnum(limit) - remaining; }, } } d.state = .dynamic_block; - return w.count - start; + return @intFromEnum(limit) - remaining; }, .protocol_footer => { d.alignBitsToByte(); @@ -314,7 +313,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S if (try in.takeInt(u32, .little) != gzip.count) return error.WrongGzipSize; }, .zlib => |*zlib| { - const chksum: u32 = @byteSwap(zlib.final()); + const chksum: u32 = @byteSwap(zlib.adler); if (try in.takeInt(u32, .big) != chksum) return error.WrongZlibChecksum; }, .raw => {}, @@ -328,10 +327,11 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S /// Write match (back-reference to the same data slice) starting at `distance` /// back from current write position, and `length` of bytes. -fn writeMatch(bw: *Writer, length: u16, distance: u16) !void { - _ = bw; +fn writeMatch(w: *Writer, length: u16, distance: u16, remaining: usize) !usize { + _ = w; _ = length; _ = distance; + _ = remaining; @panic("TODO"); } @@ -622,7 +622,13 @@ test "init/find" { test "encode/decode literals" { var codes: [flate.HuffmanEncoder.max_num_frequencies]flate.HuffmanEncoder.Code = undefined; for (1..286) |j| { // for all different number of codes - var enc: flate.HuffmanEncoder = .{ .codes = &codes }; + var enc: flate.HuffmanEncoder = .{ + .codes = &codes, + .freq_cache = undefined, + .bit_count = undefined, + .lns = undefined, + .lfs = undefined, + }; // create frequencies var freq = [_]u16{0} ** 286; freq[256] = 1; // ensure we have end of block code @@ -857,7 +863,7 @@ test "fuzzing tests" { const r = &decompress.reader; if (c.err) |expected_err| { try testing.expectError(error.ReadFailed, r.streamRemaining(&aw.writer)); - try testing.expectError(expected_err, decompress.read_err.?); + try testing.expectEqual(expected_err, decompress.read_err orelse return error.TestFailed); } else { _ = try r.streamRemaining(&aw.writer); try testing.expectEqualStrings(c.out, aw.getWritten()); @@ -891,3 +897,148 @@ test "reading into empty buffer" { var buf: [0]u8 = undefined; try testing.expectEqual(0, try r.readVec(&.{&buf})); } + +test "don't read past deflate stream's end" { + try testDecompress(.zlib, &[_]u8{ + 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff, + 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01, + 0x83, 0x95, 0x0b, 0xf5, + }, &[_]u8{ + 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, + 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, + }); +} + +test "zlib header" { + // Truncated header + try testing.expectError( + error.EndOfStream, + testDecompress(.zlib, &[_]u8{0x78}, ""), + ); + // Wrong CM + try testing.expectError( + error.BadZlibHeader, + testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""), + ); + // Wrong CINFO + try testing.expectError( + error.BadZlibHeader, + testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""), + ); + // Wrong checksum + try testing.expectError( + error.WrongZlibChecksum, + testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""), + ); + // Truncated checksum + try testing.expectError( + error.EndOfStream, + testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""), + ); +} + +test "gzip header" { + // Truncated header + try testing.expectError( + error.EndOfStream, + testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined), + ); + // Wrong CM + try testing.expectError( + error.BadGzipHeader, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, + }, undefined), + ); + + // Wrong checksum + try testing.expectError( + error.WrongGzipChecksum, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, + }, undefined), + ); + // Truncated checksum + try testing.expectError( + error.EndOfStream, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, + }, undefined), + ); + // Wrong initial size + try testing.expectError( + error.WrongGzipSize, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, + }, undefined), + ); + // Truncated initial size field + try testing.expectError( + error.EndOfStream, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, + }, undefined), + ); + + try testDecompress(.gzip, &[_]u8{ + // GZIP header + 0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00, + // header.FHCRC (should cover entire header) + 0x99, 0xd6, + // GZIP data + 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, ""); +} + +fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void { + var in: std.Io.Reader = .fixed(compressed); + var aw: std.Io.Writer.Allocating = .init(testing.allocator); + defer aw.deinit(); + + var decompress: Decompress = .init(&in, container, &.{}); + _ = try decompress.reader.streamRemaining(&aw.writer); + try testing.expectEqualSlices(u8, expected_plain, aw.getWritten()); +} + +test "zlib should not overshoot" { + // Compressed zlib data with extra 4 bytes at the end. + const data = [_]u8{ + 0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9, + 0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08, + 0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34, + 0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12, + }; + + var reader: std.Io.Reader = .fixed(&data); + + var decompress: Decompress = .init(&reader, .zlib, &.{}); + var out: [128]u8 = undefined; + + { + const n = try decompress.reader.readSliceShort(out[0..]); + + // Expected decompressed data + try std.testing.expectEqual(46, n); + try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]); + + // Decompressor don't overshoot underlying reader. + // It is leaving it at the end of compressed data chunk. + try std.testing.expectEqual(data.len - 4, reader.seek); + // TODO what was this testing, exactly? + //try std.testing.expectEqual(0, decompress.unreadBytes()); + } + + // 4 bytes after compressed chunk are available in reader. + const n = try reader.readSliceShort(out[0..]); + try std.testing.expectEqual(n, 4); + try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]); +} diff --git a/lib/std/compress/flate/HuffmanEncoder.zig b/lib/std/compress/flate/HuffmanEncoder.zig index bdcaf75801..2057038057 100644 --- a/lib/std/compress/flate/HuffmanEncoder.zig +++ b/lib/std/compress/flate/HuffmanEncoder.zig @@ -135,7 +135,7 @@ fn bitCounts(self: *HuffmanEncoder, list: []LiteralNode, max_bits_to_use: usize) // of ancestors of the rightmost node at level i. // leaf_counts[i][j] is the number of literals at the left // of the level j ancestor. - var leaf_counts: [max_bits_limit][max_bits_limit]u32 = @splat(0); + var leaf_counts: [max_bits_limit][max_bits_limit]u32 = @splat(@splat(0)); { var level = @as(u32, 1); @@ -389,7 +389,8 @@ pub fn huffmanDistanceEncoder(codes: *[distance_code_count]Code) HuffmanEncoder } test "generate a Huffman code for the fixed literal table specific to Deflate" { - const enc = fixedLiteralEncoder(); + var codes: [max_num_frequencies]Code = undefined; + const enc: HuffmanEncoder = .fixedLiteralEncoder(&codes); for (enc.codes) |c| { switch (c.len) { 7 => { diff --git a/lib/std/compress/flate/Lookup.zig b/lib/std/compress/flate/Lookup.zig index 722e175c8a..d1d93de50a 100644 --- a/lib/std/compress/flate/Lookup.zig +++ b/lib/std/compress/flate/Lookup.zig @@ -6,14 +6,19 @@ const std = @import("std"); const testing = std.testing; const expect = testing.expect; const flate = @import("../flate.zig"); +const Token = @import("Token.zig"); const Lookup = @This(); const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761 const chain_len = 2 * flate.history_len; +pub const bits = 15; +pub const len = 1 << bits; +pub const shift = 32 - bits; + // Maps hash => first position -head: [flate.lookup.len]u16 = [_]u16{0} ** flate.lookup.len, +head: [len]u16 = [_]u16{0} ** len, // Maps position => previous positions for the same hash value chain: [chain_len]u16 = [_]u16{0} ** (chain_len), @@ -52,8 +57,8 @@ pub fn slide(self: *Lookup, n: u16) void { // Add `len` 4 bytes hashes from `data` into lookup. // Position of the first byte is `pos`. -pub fn bulkAdd(self: *Lookup, data: []const u8, len: u16, pos: u16) void { - if (len == 0 or data.len < flate.match.min_length) { +pub fn bulkAdd(self: *Lookup, data: []const u8, length: u16, pos: u16) void { + if (length == 0 or data.len < Token.min_length) { return; } var hb = @@ -64,7 +69,7 @@ pub fn bulkAdd(self: *Lookup, data: []const u8, len: u16, pos: u16) void { _ = self.set(hashu(hb), pos); var i = pos; - for (4..@min(len + 3, data.len)) |j| { + for (4..@min(length + 3, data.len)) |j| { hb = (hb << 8) | @as(u32, data[j]); i += 1; _ = self.set(hashu(hb), i); @@ -80,7 +85,7 @@ fn hash(b: *const [4]u8) u32 { } fn hashu(v: u32) u32 { - return @intCast((v *% prime4) >> flate.lookup.shift); + return @intCast((v *% prime4) >> shift); } test add { diff --git a/lib/std/compress/flate/Token.zig b/lib/std/compress/flate/Token.zig index 293a786cef..1383047693 100644 --- a/lib/std/compress/flate/Token.zig +++ b/lib/std/compress/flate/Token.zig @@ -6,7 +6,6 @@ const std = @import("std"); const assert = std.debug.assert; const print = std.debug.print; const expect = std.testing.expect; -const match = std.compress.flate.match; const Token = @This(); @@ -21,16 +20,23 @@ dist: u15 = 0, len_lit: u8 = 0, kind: Kind = .literal, +pub const base_length = 3; // smallest match length per the RFC section 3.2.5 +pub const min_length = 4; // min length used in this algorithm +pub const max_length = 258; + +pub const min_distance = 1; +pub const max_distance = std.compress.flate.history_len; + pub fn literal(t: Token) u8 { return t.len_lit; } pub fn distance(t: Token) u16 { - return @as(u16, t.dist) + match.min_distance; + return @as(u16, t.dist) + min_distance; } pub fn length(t: Token) u16 { - return @as(u16, t.len_lit) + match.base_length; + return @as(u16, t.len_lit) + base_length; } pub fn initLiteral(lit: u8) Token { @@ -40,12 +46,12 @@ pub fn initLiteral(lit: u8) Token { // distance range 1 - 32768, stored in dist as 0 - 32767 (u15) // length range 3 - 258, stored in len_lit as 0 - 255 (u8) pub fn initMatch(dist: u16, len: u16) Token { - assert(len >= match.min_length and len <= match.max_length); - assert(dist >= match.min_distance and dist <= match.max_distance); + assert(len >= min_length and len <= max_length); + assert(dist >= min_distance and dist <= max_distance); return .{ .kind = .match, - .dist = @intCast(dist - match.min_distance), - .len_lit = @intCast(len - match.base_length), + .dist = @intCast(dist - min_distance), + .len_lit = @intCast(len - base_length), }; } From 1b43551190fc9aa83ae175e1db3a9db60dba5bb9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 27 Jul 2025 16:16:56 -0700 Subject: [PATCH 05/47] std.Io: remove BitWriter --- lib/std/Io.zig | 4 - lib/std/Io/bit_writer.zig | 179 ---------------------- lib/std/Io/test.zig | 45 ------ lib/std/compress/flate/HuffmanEncoder.zig | 13 -- 4 files changed, 241 deletions(-) delete mode 100644 lib/std/Io/bit_writer.zig diff --git a/lib/std/Io.zig b/lib/std/Io.zig index b90276cfab..bb50bb2f81 100644 --- a/lib/std/Io.zig +++ b/lib/std/Io.zig @@ -470,9 +470,6 @@ pub const countingReader = @import("Io/counting_reader.zig").countingReader; pub const BitReader = @import("Io/bit_reader.zig").BitReader; pub const bitReader = @import("Io/bit_reader.zig").bitReader; -pub const BitWriter = @import("Io/bit_writer.zig").BitWriter; -pub const bitWriter = @import("Io/bit_writer.zig").bitWriter; - pub const tty = @import("Io/tty.zig"); /// Deprecated in favor of `Writer.Discarding`. @@ -951,7 +948,6 @@ test { _ = Reader.Limited; _ = Writer; _ = BitReader; - _ = BitWriter; _ = BufferedReader; _ = BufferedWriter; _ = CountingWriter; diff --git a/lib/std/Io/bit_writer.zig b/lib/std/Io/bit_writer.zig deleted file mode 100644 index eef0ece81b..0000000000 --- a/lib/std/Io/bit_writer.zig +++ /dev/null @@ -1,179 +0,0 @@ -const std = @import("../std.zig"); - -//General note on endianess: -//Big endian is packed starting in the most significant part of the byte and subsequent -// bytes contain less significant bits. Thus we write out bits from the high end -// of our input first. -//Little endian is packed starting in the least significant part of the byte and -// subsequent bytes contain more significant bits. Thus we write out bits from -// the low end of our input first. -//Regardless of endianess, within any given byte the bits are always in most -// to least significant order. -//Also regardless of endianess, the buffer always aligns bits to the low end -// of the byte. - -/// Creates a bit writer which allows for writing bits to an underlying standard writer -pub fn BitWriter(comptime endian: std.builtin.Endian, comptime Writer: type) type { - return struct { - writer: Writer, - bits: u8 = 0, - count: u4 = 0, - - const low_bit_mask = [9]u8{ - 0b00000000, - 0b00000001, - 0b00000011, - 0b00000111, - 0b00001111, - 0b00011111, - 0b00111111, - 0b01111111, - 0b11111111, - }; - - /// Write the specified number of bits to the writer from the least significant bits of - /// the specified value. Bits will only be written to the writer when there - /// are enough to fill a byte. - pub fn writeBits(self: *@This(), value: anytype, num: u16) !void { - const T = @TypeOf(value); - const UT = std.meta.Int(.unsigned, @bitSizeOf(T)); - const U = if (@bitSizeOf(T) < 8) u8 else UT; // 0) { - //if we can't fill the buffer, add what we have - const bits_free = 8 - self.count; - if (num < bits_free) { - self.addBits(@truncate(in), @intCast(num)); - return; - } - - //finish filling the buffer and flush it - if (num == bits_free) { - self.addBits(@truncate(in), @intCast(num)); - return self.flushBits(); - } - - switch (endian) { - .big => { - const bits = in >> @intCast(in_count - bits_free); - self.addBits(@truncate(bits), bits_free); - }, - .little => { - self.addBits(@truncate(in), bits_free); - in >>= @intCast(bits_free); - }, - } - in_count -= bits_free; - try self.flushBits(); - } - - //write full bytes while we can - const full_bytes_left = in_count / 8; - for (0..full_bytes_left) |_| { - switch (endian) { - .big => { - const bits = in >> @intCast(in_count - 8); - try self.writer.writeByte(@truncate(bits)); - }, - .little => { - try self.writer.writeByte(@truncate(in)); - if (U == u8) in = 0 else in >>= 8; - }, - } - in_count -= 8; - } - - //save the remaining bits in the buffer - self.addBits(@truncate(in), @intCast(in_count)); - } - - //convenience funciton for adding bits to the buffer - //in the appropriate position based on endianess - fn addBits(self: *@This(), bits: u8, num: u4) void { - if (num == 8) self.bits = bits else switch (endian) { - .big => { - self.bits <<= @intCast(num); - self.bits |= bits & low_bit_mask[num]; - }, - .little => { - const pos = bits << @intCast(self.count); - self.bits |= pos; - }, - } - self.count += num; - } - - /// Flush any remaining bits to the writer, filling - /// unused bits with 0s. - pub fn flushBits(self: *@This()) !void { - if (self.count == 0) return; - if (endian == .big) self.bits <<= @intCast(8 - self.count); - try self.writer.writeByte(self.bits); - self.bits = 0; - self.count = 0; - } - }; -} - -pub fn bitWriter(comptime endian: std.builtin.Endian, writer: anytype) BitWriter(endian, @TypeOf(writer)) { - return .{ .writer = writer }; -} - -/////////////////////////////// - -test "api coverage" { - var mem_be = [_]u8{0} ** 2; - var mem_le = [_]u8{0} ** 2; - - var mem_out_be = std.io.fixedBufferStream(&mem_be); - var bit_stream_be = bitWriter(.big, mem_out_be.writer()); - - const testing = std.testing; - - try bit_stream_be.writeBits(@as(u2, 1), 1); - try bit_stream_be.writeBits(@as(u5, 2), 2); - try bit_stream_be.writeBits(@as(u128, 3), 3); - try bit_stream_be.writeBits(@as(u8, 4), 4); - try bit_stream_be.writeBits(@as(u9, 5), 5); - try bit_stream_be.writeBits(@as(u1, 1), 1); - - try testing.expect(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001011); - - mem_out_be.pos = 0; - - try bit_stream_be.writeBits(@as(u15, 0b110011010000101), 15); - try bit_stream_be.flushBits(); - try testing.expect(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001010); - - mem_out_be.pos = 0; - try bit_stream_be.writeBits(@as(u32, 0b110011010000101), 16); - try testing.expect(mem_be[0] == 0b01100110 and mem_be[1] == 0b10000101); - - try bit_stream_be.writeBits(@as(u0, 0), 0); - - var mem_out_le = std.io.fixedBufferStream(&mem_le); - var bit_stream_le = bitWriter(.little, mem_out_le.writer()); - - try bit_stream_le.writeBits(@as(u2, 1), 1); - try bit_stream_le.writeBits(@as(u5, 2), 2); - try bit_stream_le.writeBits(@as(u128, 3), 3); - try bit_stream_le.writeBits(@as(u8, 4), 4); - try bit_stream_le.writeBits(@as(u9, 5), 5); - try bit_stream_le.writeBits(@as(u1, 1), 1); - - try testing.expect(mem_le[0] == 0b00011101 and mem_le[1] == 0b10010101); - - mem_out_le.pos = 0; - try bit_stream_le.writeBits(@as(u15, 0b110011010000101), 15); - try bit_stream_le.flushBits(); - try testing.expect(mem_le[0] == 0b10000101 and mem_le[1] == 0b01100110); - - mem_out_le.pos = 0; - try bit_stream_le.writeBits(@as(u32, 0b1100110100001011), 16); - try testing.expect(mem_le[0] == 0b00001011 and mem_le[1] == 0b11001101); - - try bit_stream_le.writeBits(@as(u0, 0), 0); -} diff --git a/lib/std/Io/test.zig b/lib/std/Io/test.zig index c08879316e..e0fcea7674 100644 --- a/lib/std/Io/test.zig +++ b/lib/std/Io/test.zig @@ -57,51 +57,6 @@ test "write a file, read it, then delete it" { try tmp.dir.deleteFile(tmp_file_name); } -test "BitStreams with File Stream" { - var tmp = tmpDir(.{}); - defer tmp.cleanup(); - - const tmp_file_name = "temp_test_file.txt"; - { - var file = try tmp.dir.createFile(tmp_file_name, .{}); - defer file.close(); - - var bit_stream = io.bitWriter(native_endian, file.deprecatedWriter()); - - try bit_stream.writeBits(@as(u2, 1), 1); - try bit_stream.writeBits(@as(u5, 2), 2); - try bit_stream.writeBits(@as(u128, 3), 3); - try bit_stream.writeBits(@as(u8, 4), 4); - try bit_stream.writeBits(@as(u9, 5), 5); - try bit_stream.writeBits(@as(u1, 1), 1); - try bit_stream.flushBits(); - } - { - var file = try tmp.dir.openFile(tmp_file_name, .{}); - defer file.close(); - - var bit_stream = io.bitReader(native_endian, file.deprecatedReader()); - - var out_bits: u16 = undefined; - - try expect(1 == try bit_stream.readBits(u2, 1, &out_bits)); - try expect(out_bits == 1); - try expect(2 == try bit_stream.readBits(u5, 2, &out_bits)); - try expect(out_bits == 2); - try expect(3 == try bit_stream.readBits(u128, 3, &out_bits)); - try expect(out_bits == 3); - try expect(4 == try bit_stream.readBits(u8, 4, &out_bits)); - try expect(out_bits == 4); - try expect(5 == try bit_stream.readBits(u9, 5, &out_bits)); - try expect(out_bits == 5); - try expect(1 == try bit_stream.readBits(u1, 1, &out_bits)); - try expect(out_bits == 1); - - try expectError(error.EndOfStream, bit_stream.readBitsNoEof(u1, 1)); - } - try tmp.dir.deleteFile(tmp_file_name); -} - test "File seek ops" { var tmp = tmpDir(.{}); defer tmp.cleanup(); diff --git a/lib/std/compress/flate/HuffmanEncoder.zig b/lib/std/compress/flate/HuffmanEncoder.zig index 2057038057..28a405b886 100644 --- a/lib/std/compress/flate/HuffmanEncoder.zig +++ b/lib/std/compress/flate/HuffmanEncoder.zig @@ -421,19 +421,6 @@ test "generate a Huffman code for the 30 possible relative distances (LZ77 dista } } -test "fixedLiteralEncoder codes" { - var al = std.ArrayList(u8).init(testing.allocator); - defer al.deinit(); - var bw = std.Io.bitWriter(.little, al.writer()); - - var codes: [max_num_frequencies]Code = undefined; - const f = fixedLiteralEncoder(&codes); - for (f.codes) |c| { - try bw.writeBits(c.code, c.len); - } - try testing.expectEqualSlices(u8, &fixed_codes, al.items); -} - pub const fixed_codes = [_]u8{ 0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100, 0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100, From fa410cc234f4db16f9bdff3f9b0d0d66caa54c6b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 27 Jul 2025 16:48:17 -0700 Subject: [PATCH 06/47] std.Io: delete BitReader --- lib/std/Io.zig | 5 - lib/std/Io/Reader.zig | 4 + lib/std/Io/bit_reader.zig | 238 -------------------------------------- 3 files changed, 4 insertions(+), 243 deletions(-) delete mode 100644 lib/std/Io/bit_reader.zig diff --git a/lib/std/Io.zig b/lib/std/Io.zig index bb50bb2f81..971a720f25 100644 --- a/lib/std/Io.zig +++ b/lib/std/Io.zig @@ -467,9 +467,6 @@ pub const CountingReader = @import("Io/counting_reader.zig").CountingReader; /// Deprecated with no replacement; inefficient pattern pub const countingReader = @import("Io/counting_reader.zig").countingReader; -pub const BitReader = @import("Io/bit_reader.zig").BitReader; -pub const bitReader = @import("Io/bit_reader.zig").bitReader; - pub const tty = @import("Io/tty.zig"); /// Deprecated in favor of `Writer.Discarding`. @@ -945,9 +942,7 @@ pub fn PollFiles(comptime StreamEnum: type) type { test { _ = Reader; - _ = Reader.Limited; _ = Writer; - _ = BitReader; _ = BufferedReader; _ = BufferedWriter; _ = CountingWriter; diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index 7ba58bfbf4..8b389d83c1 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -1874,3 +1874,7 @@ pub fn writableVectorWsa( } return .{ i, n }; } + +test { + _ = Limited; +} diff --git a/lib/std/Io/bit_reader.zig b/lib/std/Io/bit_reader.zig deleted file mode 100644 index 7823e47d43..0000000000 --- a/lib/std/Io/bit_reader.zig +++ /dev/null @@ -1,238 +0,0 @@ -const std = @import("../std.zig"); - -//General note on endianess: -//Big endian is packed starting in the most significant part of the byte and subsequent -// bytes contain less significant bits. Thus we always take bits from the high -// end and place them below existing bits in our output. -//Little endian is packed starting in the least significant part of the byte and -// subsequent bytes contain more significant bits. Thus we always take bits from -// the low end and place them above existing bits in our output. -//Regardless of endianess, within any given byte the bits are always in most -// to least significant order. -//Also regardless of endianess, the buffer always aligns bits to the low end -// of the byte. - -/// Creates a bit reader which allows for reading bits from an underlying standard reader -pub fn BitReader(comptime endian: std.builtin.Endian, comptime Reader: type) type { - return struct { - reader: Reader, - bits: u8 = 0, - count: u4 = 0, - - const low_bit_mask = [9]u8{ - 0b00000000, - 0b00000001, - 0b00000011, - 0b00000111, - 0b00001111, - 0b00011111, - 0b00111111, - 0b01111111, - 0b11111111, - }; - - fn Bits(comptime T: type) type { - return struct { - T, - u16, - }; - } - - fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) { - const UT = std.meta.Int(.unsigned, @bitSizeOf(T)); - return .{ - @bitCast(@as(UT, @intCast(out))), - num, - }; - } - - /// Reads `bits` bits from the reader and returns a specified type - /// containing them in the least significant end, returning an error if the - /// specified number of bits could not be read. - pub fn readBitsNoEof(self: *@This(), comptime T: type, num: u16) !T { - const b, const c = try self.readBitsTuple(T, num); - if (c < num) return error.EndOfStream; - return b; - } - - /// Reads `bits` bits from the reader and returns a specified type - /// containing them in the least significant end. The number of bits successfully - /// read is placed in `out_bits`, as reaching the end of the stream is not an error. - pub fn readBits(self: *@This(), comptime T: type, num: u16, out_bits: *u16) !T { - const b, const c = try self.readBitsTuple(T, num); - out_bits.* = c; - return b; - } - - /// Reads `bits` bits from the reader and returns a tuple of the specified type - /// containing them in the least significant end, and the number of bits successfully - /// read. Reaching the end of the stream is not an error. - pub fn readBitsTuple(self: *@This(), comptime T: type, num: u16) !Bits(T) { - const UT = std.meta.Int(.unsigned, @bitSizeOf(T)); - const U = if (@bitSizeOf(T) < 8) u8 else UT; //it is a pain to work with return initBits(T, out, out_count), - else => |e| return e, - }; - - switch (endian) { - .big => { - if (U == u8) out = 0 else out <<= 8; //shifting u8 by 8 is illegal in Zig - out |= byte; - }, - .little => { - const pos = @as(U, byte) << @intCast(out_count); - out |= pos; - }, - } - out_count += 8; - } - - const bits_left = num - out_count; - const keep = 8 - bits_left; - - if (bits_left == 0) return initBits(T, out, out_count); - - const final_byte = self.reader.readByte() catch |err| switch (err) { - error.EndOfStream => return initBits(T, out, out_count), - else => |e| return e, - }; - - switch (endian) { - .big => { - out <<= @intCast(bits_left); - out |= final_byte >> @intCast(keep); - self.bits = final_byte & low_bit_mask[keep]; - }, - .little => { - const pos = @as(U, final_byte & low_bit_mask[bits_left]) << @intCast(out_count); - out |= pos; - self.bits = final_byte >> @intCast(bits_left); - }, - } - - self.count = @intCast(keep); - return initBits(T, out, num); - } - - //convenience function for removing bits from - //the appropriate part of the buffer based on - //endianess. - fn removeBits(self: *@This(), num: u4) u8 { - if (num == 8) { - self.count = 0; - return self.bits; - } - - const keep = self.count - num; - const bits = switch (endian) { - .big => self.bits >> @intCast(keep), - .little => self.bits & low_bit_mask[num], - }; - switch (endian) { - .big => self.bits &= low_bit_mask[keep], - .little => self.bits >>= @intCast(num), - } - - self.count = keep; - return bits; - } - - pub fn alignToByte(self: *@This()) void { - self.bits = 0; - self.count = 0; - } - }; -} - -pub fn bitReader(comptime endian: std.builtin.Endian, reader: anytype) BitReader(endian, @TypeOf(reader)) { - return .{ .reader = reader }; -} - -/////////////////////////////// - -test "api coverage" { - const mem_be = [_]u8{ 0b11001101, 0b00001011 }; - const mem_le = [_]u8{ 0b00011101, 0b10010101 }; - - var mem_in_be = std.io.fixedBufferStream(&mem_be); - var bit_stream_be = bitReader(.big, mem_in_be.reader()); - - var out_bits: u16 = undefined; - - const expect = std.testing.expect; - const expectError = std.testing.expectError; - - try expect(1 == try bit_stream_be.readBits(u2, 1, &out_bits)); - try expect(out_bits == 1); - try expect(2 == try bit_stream_be.readBits(u5, 2, &out_bits)); - try expect(out_bits == 2); - try expect(3 == try bit_stream_be.readBits(u128, 3, &out_bits)); - try expect(out_bits == 3); - try expect(4 == try bit_stream_be.readBits(u8, 4, &out_bits)); - try expect(out_bits == 4); - try expect(5 == try bit_stream_be.readBits(u9, 5, &out_bits)); - try expect(out_bits == 5); - try expect(1 == try bit_stream_be.readBits(u1, 1, &out_bits)); - try expect(out_bits == 1); - - mem_in_be.pos = 0; - bit_stream_be.count = 0; - try expect(0b110011010000101 == try bit_stream_be.readBits(u15, 15, &out_bits)); - try expect(out_bits == 15); - - mem_in_be.pos = 0; - bit_stream_be.count = 0; - try expect(0b1100110100001011 == try bit_stream_be.readBits(u16, 16, &out_bits)); - try expect(out_bits == 16); - - _ = try bit_stream_be.readBits(u0, 0, &out_bits); - - try expect(0 == try bit_stream_be.readBits(u1, 1, &out_bits)); - try expect(out_bits == 0); - try expectError(error.EndOfStream, bit_stream_be.readBitsNoEof(u1, 1)); - - var mem_in_le = std.io.fixedBufferStream(&mem_le); - var bit_stream_le = bitReader(.little, mem_in_le.reader()); - - try expect(1 == try bit_stream_le.readBits(u2, 1, &out_bits)); - try expect(out_bits == 1); - try expect(2 == try bit_stream_le.readBits(u5, 2, &out_bits)); - try expect(out_bits == 2); - try expect(3 == try bit_stream_le.readBits(u128, 3, &out_bits)); - try expect(out_bits == 3); - try expect(4 == try bit_stream_le.readBits(u8, 4, &out_bits)); - try expect(out_bits == 4); - try expect(5 == try bit_stream_le.readBits(u9, 5, &out_bits)); - try expect(out_bits == 5); - try expect(1 == try bit_stream_le.readBits(u1, 1, &out_bits)); - try expect(out_bits == 1); - - mem_in_le.pos = 0; - bit_stream_le.count = 0; - try expect(0b001010100011101 == try bit_stream_le.readBits(u15, 15, &out_bits)); - try expect(out_bits == 15); - - mem_in_le.pos = 0; - bit_stream_le.count = 0; - try expect(0b1001010100011101 == try bit_stream_le.readBits(u16, 16, &out_bits)); - try expect(out_bits == 16); - - _ = try bit_stream_le.readBits(u0, 0, &out_bits); - - try expect(0 == try bit_stream_le.readBits(u1, 1, &out_bits)); - try expect(out_bits == 0); - try expectError(error.EndOfStream, bit_stream_le.readBitsNoEof(u1, 1)); -} From 88ca75020909f922a5d3edfd1ca0be3ce148040f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 27 Jul 2025 16:55:23 -0700 Subject: [PATCH 07/47] std.compress.flate.Decompress: add rebase impl --- lib/std/compress/flate/Decompress.zig | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 5f603baf9a..b55041ed78 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -1,11 +1,13 @@ const std = @import("../../std.zig"); +const assert = std.debug.assert; const flate = std.compress.flate; -const Container = flate.Container; -const Token = @import("Token.zig"); const testing = std.testing; -const Decompress = @This(); const Writer = std.Io.Writer; const Reader = std.Io.Reader; +const Container = flate.Container; + +const Decompress = @This(); +const Token = @import("Token.zig"); input: *Reader, reader: Reader, @@ -54,7 +56,10 @@ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { .reader = .{ // TODO populate discard so that when an amount is discarded that // includes an entire frame, skip decoding that frame. - .vtable = &.{ .stream = stream }, + .vtable = &.{ + .stream = stream, + .rebase = rebase, + }, .buffer = buffer, .seek = 0, .end = 0, @@ -69,6 +74,17 @@ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { }; } +fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void { + assert(capacity <= r.buffer.len - flate.history_len); + assert(r.end + capacity > r.buffer.len); + const buffered = r.buffer[0..r.end]; + const discard = buffered.len - flate.history_len; + const keep = buffered[discard..]; + @memmove(r.buffer[0..keep.len], keep); + r.end = keep.len; + r.seek -= discard; +} + fn decodeLength(self: *Decompress, code: u8) !u16 { if (code > 28) return error.InvalidCode; const ml = Token.matchLength(code); From 6509fa1cf3694ae16333739fa0f3fae1b63f1eaf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 11:42:43 -0700 Subject: [PATCH 08/47] std.compress.flate.Decompress: passing basic test case --- lib/std/compress/flate/Decompress.zig | 126 ++++++++++++++++---------- 1 file changed, 78 insertions(+), 48 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index b55041ed78..29756eb742 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -10,7 +10,11 @@ const Decompress = @This(); const Token = @import("Token.zig"); input: *Reader, +next_bits: usize, +remaining_bits: std.math.Log2Int(usize), + reader: Reader, + /// Hashes, produces checksum, of uncompressed data for gzip/zlib footer. hasher: Container.Hasher, @@ -65,6 +69,8 @@ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { .end = 0, }, .input = input, + .next_bits = 0, + .remaining_bits = 0, .hasher = .init(container), .lit_dec = .{}, .dst_dec = .{}, @@ -228,15 +234,15 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S return error.InvalidDynamicBlockHeader; // lengths for code lengths - var cl_lens = [_]u4{0} ** 19; - for (0..hclen) |i| { - cl_lens[flate.HuffmanEncoder.codegen_order[i]] = try d.takeBits(u3); + var cl_lens: [19]u4 = @splat(0); + for (flate.HuffmanEncoder.codegen_order[0..hclen]) |i| { + cl_lens[i] = try d.takeBits(u3); } var cl_dec: CodegenDecoder = .{}; try cl_dec.generate(&cl_lens); // decoded code lengths - var dec_lens = [_]u4{0} ** (286 + 30); + var dec_lens: [286 + 30]u4 = @splat(0); var pos: usize = 0; while (pos < hlit + hdist) { const sym = try cl_dec.find(try d.peekBitsReverse(u7)); @@ -352,8 +358,30 @@ fn writeMatch(w: *Writer, length: u16, distance: u16, remaining: usize) !usize { } fn takeBits(d: *Decompress, comptime T: type) !T { - _ = d; - @panic("TODO"); + const U = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(T) } }); + const remaining_bits = d.remaining_bits; + const next_bits = d.next_bits; + if (remaining_bits >= @bitSizeOf(T)) { + const u: U = @truncate(next_bits); + d.next_bits = next_bits >> @bitSizeOf(T); + d.remaining_bits = remaining_bits - @bitSizeOf(T); + return switch (@typeInfo(T)) { + .int => u, + .@"enum" => @enumFromInt(u), + else => @bitCast(u), + }; + } + const in = d.input; + const next_int = try in.takeInt(usize, .little); + const needed_bits = @bitSizeOf(T) - remaining_bits; + const u: U = @intCast((next_bits << needed_bits) | (next_int & ((@as(usize, 1) << needed_bits) - 1))); + d.next_bits = next_int >> needed_bits; + d.remaining_bits = @intCast(@bitSizeOf(usize) - @as(usize, needed_bits)); + return switch (@typeInfo(T)) { + .int => u, + .@"enum" => @enumFromInt(u), + else => @bitCast(u), + }; } fn takeBitsReverseBuffered(d: *Decompress, comptime T: type) !T { @@ -378,8 +406,20 @@ fn peekBitsReverseBuffered(d: *Decompress, comptime T: type) !T { } fn alignBitsToByte(d: *Decompress) void { - _ = d; - @panic("TODO"); + const remaining_bits = d.remaining_bits; + const next_bits = d.next_bits; + if (remaining_bits == 0) return; + const discard_bits = remaining_bits % 8; + const n_bytes = remaining_bits / 8; + var put_back_bits = next_bits >> discard_bits; + const in = d.input; + in.seek -= n_bytes; + for (in.buffer[in.seek..][0..n_bytes]) |*b| { + b.* = @truncate(put_back_bits); + put_back_bits >>= 8; + } + d.remaining_bits = 0; + d.next_bits = 0; } fn shiftBits(d: *Decompress, n: u6) !void { @@ -691,47 +731,37 @@ test "encode/decode literals" { } } -test "decompress" { - const cases = [_]struct { - in: []const u8, - out: []const u8, - }{ - // non compressed block (type 0) - .{ - .in = &[_]u8{ - 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen - 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data - }, - .out = "Hello world\n", - }, - // fixed code block (type 1) - .{ - .in = &[_]u8{ - 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 - 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, - }, - .out = "Hello world\n", - }, - // dynamic block (type 2) - .{ - .in = &[_]u8{ - 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 - 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, - 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01, - }, - .out = "ABCDEABCD ABCDEABCD", - }, - }; - for (cases) |c| { - var fb: Reader = .fixed(c.in); - var aw: Writer.Allocating = .init(testing.allocator); - defer aw.deinit(); +test "basic" { + // non compressed block (type 0) + try testBasicCase(&[_]u8{ + 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen + 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data + }, "Hello world\n"); - var decompress: Decompress = .init(&fb, .raw, &.{}); - const r = &decompress.reader; - _ = try r.streamRemaining(&aw.writer); - try testing.expectEqualStrings(c.out, aw.getWritten()); - } + // fixed code block (type 1) + try testBasicCase(&[_]u8{ + 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 + 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, + }, "Hello world\n"); + + // dynamic block (type 2) + try testBasicCase(&[_]u8{ + 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 + 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, + 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01, + }, "ABCDEABCD ABCDEABCD"); +} + +fn testBasicCase(in: []const u8, out: []const u8) !void { + var reader: Reader = .fixed(in); + var aw: Writer.Allocating = .init(testing.allocator); + try aw.ensureUnusedCapacity(flate.history_len + 1); + defer aw.deinit(); + + var decompress: Decompress = .init(&reader, .raw, &.{}); + const r = &decompress.reader; + _ = try r.streamRemaining(&aw.writer); + try testing.expectEqualStrings(out, aw.getWritten()); } test "gzip decompress" { From 9c8cb777d461406c4185bdb38b9462abde5d5e52 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 13:13:59 -0700 Subject: [PATCH 09/47] std.compress.flate.Decompress: implement more bit reading --- lib/std/compress/flate/Decompress.zig | 95 ++++++++++++++++++--------- 1 file changed, 65 insertions(+), 30 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 29756eb742..1f7d4346cd 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -146,8 +146,8 @@ fn dynamicCodeLength(self: *Decompress, code: u16, lens: []u4, pos: usize) !usiz // used. Shift bit reader for that much bits, those bits are used. And // return symbol. fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol { - const sym = try decoder.find(try self.peekBitsReverseBuffered(u15)); - try self.shiftBits(sym.code_bits); + const sym = try decoder.find(@bitReverse(try self.peekBits(u15))); + try self.tossBits(sym.code_bits); return sym; } @@ -245,8 +245,8 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S var dec_lens: [286 + 30]u4 = @splat(0); var pos: usize = 0; while (pos < hlit + hdist) { - const sym = try cl_dec.find(try d.peekBitsReverse(u7)); - try d.shiftBits(sym.code_bits); + const sym = try cl_dec.find(@bitReverse(try d.peekBits(u7))); + try d.tossBits(sym.code_bits); pos += try d.dynamicCodeLength(sym.symbol, &dec_lens, pos); } if (pos > hlit + hdist) { @@ -291,7 +291,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S // Handles fixed block non literal (length) code. // Length code is followed by 5 bits of distance code. const length = try d.decodeLength(@intCast(code - 257)); - const distance = try d.decodeDistance(try d.takeBitsReverseBuffered(u5)); + const distance = try d.decodeDistance(@bitReverse(try d.takeBits(u5))); remaining = try writeMatch(w, length, distance, remaining); }, else => return error.InvalidCode, @@ -384,9 +384,42 @@ fn takeBits(d: *Decompress, comptime T: type) !T { }; } -fn takeBitsReverseBuffered(d: *Decompress, comptime T: type) !T { - _ = d; - @panic("TODO"); +fn peekBits(d: *Decompress, comptime T: type) !T { + const U = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(T) } }); + const remaining_bits = d.remaining_bits; + const next_bits = d.next_bits; + if (remaining_bits >= @bitSizeOf(T)) { + const u: U = @truncate(next_bits); + return switch (@typeInfo(T)) { + .int => u, + .@"enum" => @enumFromInt(u), + else => @bitCast(u), + }; + } + const in = d.input; + const next_int = try in.peekInt(usize, .little); + const needed_bits = @bitSizeOf(T) - remaining_bits; + const u: U = @intCast((next_bits << needed_bits) | (next_int & ((@as(usize, 1) << needed_bits) - 1))); + return switch (@typeInfo(T)) { + .int => u, + .@"enum" => @enumFromInt(u), + else => @bitCast(u), + }; +} + +fn tossBits(d: *Decompress, n: u6) !void { + const remaining_bits = d.remaining_bits; + const next_bits = d.next_bits; + if (remaining_bits >= n) { + d.next_bits = next_bits >> n; + d.remaining_bits = remaining_bits - n; + } else { + const in = d.input; + const next_int = try in.takeInt(usize, .little); + const needed_bits = n - remaining_bits; + d.next_bits = next_int >> needed_bits; + d.remaining_bits = @intCast(@bitSizeOf(usize) - @as(usize, needed_bits)); + } } fn takeNBitsBuffered(d: *Decompress, n: u4) !u16 { @@ -395,16 +428,6 @@ fn takeNBitsBuffered(d: *Decompress, n: u4) !u16 { @panic("TODO"); } -fn peekBitsReverse(d: *Decompress, comptime T: type) !T { - _ = d; - @panic("TODO"); -} - -fn peekBitsReverseBuffered(d: *Decompress, comptime T: type) !T { - _ = d; - @panic("TODO"); -} - fn alignBitsToByte(d: *Decompress) void { const remaining_bits = d.remaining_bits; const next_bits = d.next_bits; @@ -422,15 +445,26 @@ fn alignBitsToByte(d: *Decompress) void { d.next_bits = 0; } -fn shiftBits(d: *Decompress, n: u6) !void { - _ = d; - _ = n; - @panic("TODO"); -} - +/// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code. +/// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12 +/// Lit Value Bits Codes +/// --------- ---- ----- +/// 0 - 143 8 00110000 through +/// 10111111 +/// 144 - 255 9 110010000 through +/// 111111111 +/// 256 - 279 7 0000000 through +/// 0010111 +/// 280 - 287 8 11000000 through +/// 11000111 fn readFixedCode(d: *Decompress) !u16 { - _ = d; - @panic("TODO"); + const code7 = @bitReverse(try d.takeBits(u7)); + return switch (code7) { + 0...0b0010_111 => @as(u16, code7) + 256, + 0b0010_111 + 1...0b1011_111 => (@as(u16, code7) << 1) + @as(u16, try d.takeBits(u1)) - 0b0011_0000, + 0b1011_111 + 1...0b1100_011 => (@as(u16, code7 - 0b1100000) << 1) + try d.takeBits(u1) + 280, + else => (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, @bitReverse(try d.takeBits(u2))) + 144, + }; } pub const Symbol = packed struct { @@ -731,20 +765,21 @@ test "encode/decode literals" { } } -test "basic" { - // non compressed block (type 0) +test "non compressed block (type 0)" { try testBasicCase(&[_]u8{ 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data }, "Hello world\n"); +} - // fixed code block (type 1) +test "fixed code block (type 1)" { try testBasicCase(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, }, "Hello world\n"); +} - // dynamic block (type 2) +test "dynamic block (type 2)" { try testBasicCase(&[_]u8{ 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, From 73e5594c78a66fb943638d99f8540bec0b5ed839 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 14:36:26 -0700 Subject: [PATCH 10/47] std.compress.flate.Decompress: fix bit read at eof --- lib/std/compress/flate/Decompress.zig | 55 +++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 1f7d4346cd..ef9e551a15 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -372,7 +372,10 @@ fn takeBits(d: *Decompress, comptime T: type) !T { }; } const in = d.input; - const next_int = try in.takeInt(usize, .little); + const next_int = in.takeInt(usize, .little) catch |err| switch (err) { + error.ReadFailed => return error.ReadFailed, + error.EndOfStream => return takeBitsEnding(d, T), + }; const needed_bits = @bitSizeOf(T) - remaining_bits; const u: U = @intCast((next_bits << needed_bits) | (next_int & ((@as(usize, 1) << needed_bits) - 1))); d.next_bits = next_int >> needed_bits; @@ -384,6 +387,35 @@ fn takeBits(d: *Decompress, comptime T: type) !T { }; } +fn takeBitsEnding(d: *Decompress, comptime T: type) !T { + const remaining_bits = d.remaining_bits; + const next_bits = d.next_bits; + const in = d.input; + const U = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(T) } }); + var u: U = 0; + var remaining_needed_bits = @bitSizeOf(U) - remaining_bits; + while (@bitSizeOf(U) >= 8 and remaining_needed_bits >= 8) { + const byte = try in.takeByte(); + u = (u << 8) | byte; + remaining_needed_bits -= 8; + } + if (remaining_needed_bits == 0) { + d.next_bits = 0; + d.remaining_bits = 0; + } else { + const byte = try in.takeByte(); + u = @intCast((@as(usize, u) << remaining_needed_bits) | (byte & ((@as(usize, 1) << remaining_needed_bits) - 1))); + d.next_bits = @as(usize, byte) >> remaining_needed_bits; + d.remaining_bits = @intCast(8 - remaining_needed_bits); + } + u = @intCast((@as(usize, u) << remaining_bits) | next_bits); + return switch (@typeInfo(T)) { + .int => u, + .@"enum" => @enumFromInt(u), + else => @bitCast(u), + }; +} + fn peekBits(d: *Decompress, comptime T: type) !T { const U = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(T) } }); const remaining_bits = d.remaining_bits; @@ -397,7 +429,10 @@ fn peekBits(d: *Decompress, comptime T: type) !T { }; } const in = d.input; - const next_int = try in.peekInt(usize, .little); + const next_int = in.peekInt(usize, .little) catch |err| switch (err) { + error.ReadFailed => return error.ReadFailed, + error.EndOfStream => return peekBitsEnding(d, T), + }; const needed_bits = @bitSizeOf(T) - remaining_bits; const u: U = @intCast((next_bits << needed_bits) | (next_int & ((@as(usize, 1) << needed_bits) - 1))); return switch (@typeInfo(T)) { @@ -407,6 +442,11 @@ fn peekBits(d: *Decompress, comptime T: type) !T { }; } +fn peekBitsEnding(d: *Decompress, comptime T: type) !T { + _ = d; + @panic("TODO"); +} + fn tossBits(d: *Decompress, n: u6) !void { const remaining_bits = d.remaining_bits; const next_bits = d.next_bits; @@ -415,13 +455,22 @@ fn tossBits(d: *Decompress, n: u6) !void { d.remaining_bits = remaining_bits - n; } else { const in = d.input; - const next_int = try in.takeInt(usize, .little); + const next_int = in.takeInt(usize, .little) catch |err| switch (err) { + error.ReadFailed => return error.ReadFailed, + error.EndOfStream => return tossBitsEnding(d, n), + }; const needed_bits = n - remaining_bits; d.next_bits = next_int >> needed_bits; d.remaining_bits = @intCast(@bitSizeOf(usize) - @as(usize, needed_bits)); } } +fn tossBitsEnding(d: *Decompress, n: u6) !void { + _ = d; + _ = n; + @panic("TODO"); +} + fn takeNBitsBuffered(d: *Decompress, n: u4) !u16 { _ = d; _ = n; From 7bf91d705c241609eb9d459d7556b2fc2afee7a1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 15:18:43 -0700 Subject: [PATCH 11/47] fix bit read not at eof --- lib/std/compress/flate/Decompress.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index ef9e551a15..ca2d45df21 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -245,7 +245,8 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S var dec_lens: [286 + 30]u4 = @splat(0); var pos: usize = 0; while (pos < hlit + hdist) { - const sym = try cl_dec.find(@bitReverse(try d.peekBits(u7))); + const peeked = @bitReverse(try d.peekBits(u7)); + const sym = try cl_dec.find(peeked); try d.tossBits(sym.code_bits); pos += try d.dynamicCodeLength(sym.symbol, &dec_lens, pos); } @@ -377,7 +378,7 @@ fn takeBits(d: *Decompress, comptime T: type) !T { error.EndOfStream => return takeBitsEnding(d, T), }; const needed_bits = @bitSizeOf(T) - remaining_bits; - const u: U = @intCast((next_bits << needed_bits) | (next_int & ((@as(usize, 1) << needed_bits) - 1))); + const u: U = @intCast(((next_int & ((@as(usize, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); d.next_bits = next_int >> needed_bits; d.remaining_bits = @intCast(@bitSizeOf(usize) - @as(usize, needed_bits)); return switch (@typeInfo(T)) { @@ -434,7 +435,7 @@ fn peekBits(d: *Decompress, comptime T: type) !T { error.EndOfStream => return peekBitsEnding(d, T), }; const needed_bits = @bitSizeOf(T) - remaining_bits; - const u: U = @intCast((next_bits << needed_bits) | (next_int & ((@as(usize, 1) << needed_bits) - 1))); + const u: U = @intCast(((next_int & ((@as(usize, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); return switch (@typeInfo(T)) { .int => u, .@"enum" => @enumFromInt(u), @@ -983,11 +984,10 @@ test "fuzzing tests" { .{ .input = "puff27", .err = error.InvalidDynamicBlockHeader }, }; - inline for (cases, 0..) |c, case_no| { + inline for (cases) |c| { var in: Reader = .fixed(@embedFile("testdata/fuzz/" ++ c.input ++ ".input")); var aw: Writer.Allocating = .init(testing.allocator); defer aw.deinit(); - errdefer std.debug.print("test case failed {}\n", .{case_no}); var decompress: Decompress = .init(&in, .raw, &.{}); const r = &decompress.reader; From e73ca2444e1d7a42266af5496f24ec122fd18f2c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 16:46:16 -0700 Subject: [PATCH 12/47] std.compress.flate.Decompress: implement peekBitsEnding and writeMatch --- lib/std/compress/flate.zig | 4 +- lib/std/compress/flate/Decompress.zig | 94 +++++++++++++++++---------- 2 files changed, 61 insertions(+), 37 deletions(-) diff --git a/lib/std/compress/flate.zig b/lib/std/compress/flate.zig index 032ea8a779..f36afb52cf 100644 --- a/lib/std/compress/flate.zig +++ b/lib/std/compress/flate.zig @@ -11,8 +11,8 @@ pub const history_len = 32768; /// of LZ77 and Huffman coding. pub const Compress = @import("flate/Compress.zig"); -/// Inflate is the decoding process that takes a Deflate bitstream for -/// decompression and correctly produces the original full-size data or file. +/// Inflate is the decoding process that consumes a Deflate bitstream and +/// produces the original full-size data. pub const Decompress = @import("flate/Decompress.zig"); /// Compression without Lempel-Ziv match searching. Faster compression, less diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index ca2d45df21..26a8a5a1ed 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -97,7 +97,7 @@ fn decodeLength(self: *Decompress, code: u8) !u16 { return if (ml.extra_bits == 0) // 0 - 5 extra bits ml.base else - ml.base + try self.takeNBitsBuffered(ml.extra_bits); + ml.base + try self.takeBitsRuntime(ml.extra_bits); } fn decodeDistance(self: *Decompress, code: u8) !u16 { @@ -106,7 +106,7 @@ fn decodeDistance(self: *Decompress, code: u8) !u16 { return if (md.extra_bits == 0) // 0 - 13 extra bits md.base else - md.base + try self.takeNBitsBuffered(md.extra_bits); + md.base + try self.takeBitsRuntime(md.extra_bits); } // Decode code length symbol to code length. Writes decoded length into @@ -293,7 +293,8 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S // Length code is followed by 5 bits of distance code. const length = try d.decodeLength(@intCast(code - 257)); const distance = try d.decodeDistance(@bitReverse(try d.takeBits(u5))); - remaining = try writeMatch(w, length, distance, remaining); + try writeMatch(w, length, distance); + remaining -= length; }, else => return error.InvalidCode, } @@ -317,7 +318,8 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S const length = try d.decodeLength(sym.symbol); const dsm = try d.decodeSymbol(&d.dst_dec); const distance = try d.decodeDistance(dsm.symbol); - remaining = try writeMatch(w, length, distance, remaining); + try writeMatch(w, length, distance); + remaining -= length; }, .end_of_block => { d.state = if (d.final_block) .protocol_footer else .block_header; @@ -350,12 +352,19 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S /// Write match (back-reference to the same data slice) starting at `distance` /// back from current write position, and `length` of bytes. -fn writeMatch(w: *Writer, length: u16, distance: u16, remaining: usize) !usize { - _ = w; - _ = length; - _ = distance; - _ = remaining; - @panic("TODO"); +fn writeMatch(w: *Writer, length: u16, distance: u16) !void { + if (w.end < length) return error.InvalidMatch; + if (length < Token.base_length) return error.InvalidMatch; + if (length > Token.max_length) return error.InvalidMatch; + if (distance < Token.min_distance) return error.InvalidMatch; + if (distance > Token.max_distance) return error.InvalidMatch; + + // This is not a @memmove; it intentionally repeats patterns caused by + // iterating one byte at a time. + const dest = try w.writableSlicePreserve(flate.history_len, length); + const end = dest.ptr - w.buffer.ptr; + const src = w.buffer[end - distance ..][0..length]; + for (dest, src) |*d, s| d.* = s; } fn takeBits(d: *Decompress, comptime T: type) !T { @@ -417,35 +426,48 @@ fn takeBitsEnding(d: *Decompress, comptime T: type) !T { }; } -fn peekBits(d: *Decompress, comptime T: type) !T { - const U = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(T) } }); +fn peekBits(d: *Decompress, comptime U: type) !U { const remaining_bits = d.remaining_bits; const next_bits = d.next_bits; - if (remaining_bits >= @bitSizeOf(T)) { - const u: U = @truncate(next_bits); - return switch (@typeInfo(T)) { - .int => u, - .@"enum" => @enumFromInt(u), - else => @bitCast(u), - }; - } + if (remaining_bits >= @bitSizeOf(U)) return @truncate(next_bits); const in = d.input; const next_int = in.peekInt(usize, .little) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, - error.EndOfStream => return peekBitsEnding(d, T), - }; - const needed_bits = @bitSizeOf(T) - remaining_bits; - const u: U = @intCast(((next_int & ((@as(usize, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); - return switch (@typeInfo(T)) { - .int => u, - .@"enum" => @enumFromInt(u), - else => @bitCast(u), + error.EndOfStream => return peekBitsEnding(d, U), }; + const needed_bits = @bitSizeOf(U) - remaining_bits; + return @intCast(((next_int & ((@as(usize, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); } -fn peekBitsEnding(d: *Decompress, comptime T: type) !T { - _ = d; - @panic("TODO"); +fn peekBitsEnding(d: *Decompress, comptime U: type) !U { + const remaining_bits = d.remaining_bits; + const next_bits = d.next_bits; + const in = d.input; + var u: U = 0; + var remaining_needed_bits = @bitSizeOf(U) - remaining_bits; + var peek_len: usize = 0; + while (@bitSizeOf(U) >= 8 and remaining_needed_bits >= 8) { + peek_len += 1; + const byte = try specialPeek(in, next_bits, peek_len); + u = (u << 8) | byte; + remaining_needed_bits -= 8; + } + if (remaining_needed_bits != 0) { + peek_len += 1; + const byte = try specialPeek(in, next_bits, peek_len); + u = @intCast((@as(usize, u) << remaining_needed_bits) | (byte & ((@as(usize, 1) << remaining_needed_bits) - 1))); + } + return @intCast((@as(usize, u) << remaining_bits) | next_bits); +} + +/// If there is any unconsumed data, handles EndOfStream by pretending there +/// are zeroes afterwards. +fn specialPeek(in: *Reader, next_bits: usize, n: usize) Reader.Error!u8 { + const peeked = in.peek(n) catch |err| switch (err) { + error.ReadFailed => return error.ReadFailed, + error.EndOfStream => if (next_bits == 0 and n == 0) return error.EndOfStream else return 0, + }; + return peeked[n - 1]; } fn tossBits(d: *Decompress, n: u6) !void { @@ -472,10 +494,12 @@ fn tossBitsEnding(d: *Decompress, n: u6) !void { @panic("TODO"); } -fn takeNBitsBuffered(d: *Decompress, n: u4) !u16 { - _ = d; - _ = n; - @panic("TODO"); +fn takeBitsRuntime(d: *Decompress, n: u4) !u16 { + const x = try peekBits(d, u16); + const mask: u16 = (@as(u16, 1) << n) - 1; + const u: u16 = @as(u16, @truncate(x)) & mask; + try tossBits(d, n); + return u; } fn alignBitsToByte(d: *Decompress) void { From 5f571f53d63e8742374c3c3cdda1ca0bb432d8fc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 16:53:01 -0700 Subject: [PATCH 13/47] refactor gzip test cases zig newbies love using for loops in unit tests --- lib/std/compress/flate/Decompress.zig | 104 ++++++++++++-------------- 1 file changed, 47 insertions(+), 57 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 26a8a5a1ed..a1cf438f8e 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -864,7 +864,7 @@ test "dynamic block (type 2)" { fn testBasicCase(in: []const u8, out: []const u8) !void { var reader: Reader = .fixed(in); var aw: Writer.Allocating = .init(testing.allocator); - try aw.ensureUnusedCapacity(flate.history_len + 1); + try aw.ensureUnusedCapacity(flate.history_len); defer aw.deinit(); var decompress: Decompress = .init(&reader, .raw, &.{}); @@ -873,63 +873,53 @@ fn testBasicCase(in: []const u8, out: []const u8) !void { try testing.expectEqualStrings(out, aw.getWritten()); } -test "gzip decompress" { - const cases = [_]struct { - in: []const u8, - out: []const u8, - }{ - // non compressed block (type 0) - .{ - .in = &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes) - 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen - 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data - 0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum - 0x0c, 0x00, 0x00, 0x00, // gzip footer: size - }, - .out = "Hello world\n", - }, - // fixed code block (type 1) - .{ - .in = &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes) - 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 - 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, - 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len) - }, - .out = "Hello world\n", - }, - // dynamic block (type 2) - .{ - .in = &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes) - 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 - 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, - 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01, - 0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len) - }, - .out = "ABCDEABCD ABCDEABCD", - }, - // gzip header with name - .{ - .in = &[_]u8{ - 0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, - 0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, - 0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, - }, - .out = "Hello world\n", - }, - }; - for (cases) |c| { - var fb: Reader = .fixed(c.in); - var aw: Writer.Allocating = .init(testing.allocator); - defer aw.deinit(); +test "gzip non compressed block (type 0)" { + try testGzipDecompress(&[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes) + 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen + 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data + 0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum + 0x0c, 0x00, 0x00, 0x00, // gzip footer: size + }, "Hello world\n"); +} - var decompress: Decompress = .init(&fb, .gzip, &.{}); - const r = &decompress.reader; - _ = try r.streamRemaining(&aw.writer); - try testing.expectEqualStrings(c.out, aw.getWritten()); - } +test "gzip fixed code block (type 1)" { + try testGzipDecompress(&[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes) + 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 + 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, + 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len) + }, "Hello world\n"); +} + +test "gzip dynamic block (type 2)" { + try testGzipDecompress(&[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes) + 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 + 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, + 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01, + 0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len) + }, "ABCDEABCD ABCDEABCD"); +} + +test "gzip header with name" { + try testGzipDecompress(&[_]u8{ + 0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, + }, "Hello world\n"); +} + +fn testGzipDecompress(in: []const u8, out: []const u8) !void { + var reader: Reader = .fixed(in); + var aw: Writer.Allocating = .init(testing.allocator); + try aw.ensureUnusedCapacity(flate.history_len); + defer aw.deinit(); + + var decompress: Decompress = .init(&reader, .gzip, &.{}); + const r = &decompress.reader; + _ = try r.streamRemaining(&aw.writer); + try testing.expectEqualStrings(out, aw.getWritten()); } test "zlib decompress" { From ac4fbb427ba71e20fe8042b86179409598241d6a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 18:00:20 -0700 Subject: [PATCH 14/47] std.compress.flate.Decompress: don't compute checksums These have no business being in-bound; simply provide the expected values to user code for maximum flexibility. --- lib/std/compress/flate.zig | 23 ++++++++++++++++++++++ lib/std/compress/flate/Decompress.zig | 28 +++++++++++++-------------- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/lib/std/compress/flate.zig b/lib/std/compress/flate.zig index f36afb52cf..4391a06787 100644 --- a/lib/std/compress/flate.zig +++ b/lib/std/compress/flate.zig @@ -148,6 +148,29 @@ pub const Container = enum { } } }; + + pub const Metadata = union(Container) { + raw: void, + gzip: struct { + crc: u32 = 0, + count: u32 = 0, + }, + zlib: struct { + adler: u32 = 0, + }, + + pub fn init(containter: Container) Metadata { + return switch (containter) { + .gzip => .{ .gzip = .{} }, + .zlib => .{ .zlib = .{} }, + .raw => .raw, + }; + } + + pub fn container(m: Metadata) Container { + return m; + } + }; }; test { diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index a1cf438f8e..5fd4b282da 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -15,8 +15,7 @@ remaining_bits: std.math.Log2Int(usize), reader: Reader, -/// Hashes, produces checksum, of uncompressed data for gzip/zlib footer. -hasher: Container.Hasher, +container_metadata: Container.Metadata, lit_dec: LiteralDecoder, dst_dec: DistanceDecoder, @@ -71,7 +70,7 @@ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { .input = input, .next_bits = 0, .remaining_bits = 0, - .hasher = .init(container), + .container_metadata = .init(container), .lit_dec = .{}, .dst_dec = .{}, .final_block = false, @@ -169,7 +168,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S var remaining = @intFromEnum(limit); const in = d.input; sw: switch (d.state) { - .protocol_header => switch (d.hasher.container()) { + .protocol_header => switch (d.container_metadata.container()) { .gzip => { const Header = extern struct { magic: u16 align(1), @@ -258,7 +257,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S try d.lit_dec.generate(dec_lens[0..hlit]); // distance code lengths to distance decoder - try d.dst_dec.generate(dec_lens[hlit .. hlit + hdist]); + try d.dst_dec.generate(dec_lens[hlit..][0..hdist]); continue :sw .dynamic_block; }, @@ -332,14 +331,17 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S }, .protocol_footer => { d.alignBitsToByte(); - switch (d.hasher) { + switch (d.container_metadata) { .gzip => |*gzip| { - if (try in.takeInt(u32, .little) != gzip.crc.final()) return error.WrongGzipChecksum; - if (try in.takeInt(u32, .little) != gzip.count) return error.WrongGzipSize; + gzip.* = .{ + .crc = try in.takeInt(u32, .little), + .count = try in.takeInt(u32, .little), + }; }, .zlib => |*zlib| { - const chksum: u32 = @byteSwap(zlib.adler); - if (try in.takeInt(u32, .big) != chksum) return error.WrongZlibChecksum; + zlib.* = .{ + .adler = try in.takeInt(u32, .little), + }; }, .raw => {}, } @@ -868,8 +870,7 @@ fn testBasicCase(in: []const u8, out: []const u8) !void { defer aw.deinit(); var decompress: Decompress = .init(&reader, .raw, &.{}); - const r = &decompress.reader; - _ = try r.streamRemaining(&aw.writer); + _ = try decompress.reader.streamRemaining(&aw.writer); try testing.expectEqualStrings(out, aw.getWritten()); } @@ -917,8 +918,7 @@ fn testGzipDecompress(in: []const u8, out: []const u8) !void { defer aw.deinit(); var decompress: Decompress = .init(&reader, .gzip, &.{}); - const r = &decompress.reader; - _ = try r.streamRemaining(&aw.writer); + _ = try decompress.reader.streamRemaining(&aw.writer); try testing.expectEqualStrings(out, aw.getWritten()); } From c684b21b4f8f13b32535d799d2c76cdb2dccc8f0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 18:05:29 -0700 Subject: [PATCH 15/47] simplify test cases --- lib/std/compress/flate/Decompress.zig | 72 +++++++-------------------- 1 file changed, 17 insertions(+), 55 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 5fd4b282da..26aa668c76 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -842,40 +842,29 @@ test "encode/decode literals" { } test "non compressed block (type 0)" { - try testBasicCase(&[_]u8{ + try testDecompress(.raw, &[_]u8{ 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data }, "Hello world\n"); } test "fixed code block (type 1)" { - try testBasicCase(&[_]u8{ + try testDecompress(.raw, &[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, }, "Hello world\n"); } test "dynamic block (type 2)" { - try testBasicCase(&[_]u8{ + try testDecompress(.raw, &[_]u8{ 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01, }, "ABCDEABCD ABCDEABCD"); } -fn testBasicCase(in: []const u8, out: []const u8) !void { - var reader: Reader = .fixed(in); - var aw: Writer.Allocating = .init(testing.allocator); - try aw.ensureUnusedCapacity(flate.history_len); - defer aw.deinit(); - - var decompress: Decompress = .init(&reader, .raw, &.{}); - _ = try decompress.reader.streamRemaining(&aw.writer); - try testing.expectEqualStrings(out, aw.getWritten()); -} - test "gzip non compressed block (type 0)" { - try testGzipDecompress(&[_]u8{ + try testDecompress(.gzip, &[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes) 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data @@ -885,7 +874,7 @@ test "gzip non compressed block (type 0)" { } test "gzip fixed code block (type 1)" { - try testGzipDecompress(&[_]u8{ + try testDecompress(.gzip, &[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes) 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, @@ -894,7 +883,7 @@ test "gzip fixed code block (type 1)" { } test "gzip dynamic block (type 2)" { - try testGzipDecompress(&[_]u8{ + try testDecompress(.gzip, &[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes) 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, @@ -904,50 +893,20 @@ test "gzip dynamic block (type 2)" { } test "gzip header with name" { - try testGzipDecompress(&[_]u8{ + try testDecompress(.gzip, &[_]u8{ 0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, 0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, }, "Hello world\n"); } -fn testGzipDecompress(in: []const u8, out: []const u8) !void { - var reader: Reader = .fixed(in); - var aw: Writer.Allocating = .init(testing.allocator); - try aw.ensureUnusedCapacity(flate.history_len); - defer aw.deinit(); - - var decompress: Decompress = .init(&reader, .gzip, &.{}); - _ = try decompress.reader.streamRemaining(&aw.writer); - try testing.expectEqualStrings(out, aw.getWritten()); -} - -test "zlib decompress" { - const cases = [_]struct { - in: []const u8, - out: []const u8, - }{ - // non compressed block (type 0) - .{ - .in = &[_]u8{ - 0x78, 0b10_0_11100, // zlib header (2 bytes) - 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen - 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data - 0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum - }, - .out = "Hello world\n", - }, - }; - for (cases) |c| { - var fb: Reader = .fixed(c.in); - var aw: Writer.Allocating = .init(testing.allocator); - defer aw.deinit(); - - var decompress: Decompress = .init(&fb, .zlib, &.{}); - const r = &decompress.reader; - _ = try r.streamRemaining(&aw.writer); - try testing.expectEqualStrings(c.out, aw.getWritten()); - } +test "zlib decompress non compressed block (type 0)" { + try testDecompress(.zlib, &[_]u8{ + 0x78, 0b10_0_11100, // zlib header (2 bytes) + 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen + 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data + 0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum + }, "Hello world\n"); } test "fuzzing tests" { @@ -1001,6 +960,7 @@ test "fuzzing tests" { inline for (cases) |c| { var in: Reader = .fixed(@embedFile("testdata/fuzz/" ++ c.input ++ ".input")); var aw: Writer.Allocating = .init(testing.allocator); + try aw.ensureUnusedCapacity(flate.history_len); defer aw.deinit(); var decompress: Decompress = .init(&in, .raw, &.{}); @@ -1021,6 +981,7 @@ test "bug 18966" { var in: Reader = .fixed(input); var aw: Writer.Allocating = .init(testing.allocator); + try aw.ensureUnusedCapacity(flate.history_len); defer aw.deinit(); var decompress: Decompress = .init(&in, .gzip, &.{}); @@ -1146,6 +1107,7 @@ test "gzip header" { fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void { var in: std.Io.Reader = .fixed(compressed); var aw: std.Io.Writer.Allocating = .init(testing.allocator); + try aw.ensureUnusedCapacity(flate.history_len); defer aw.deinit(); var decompress: Decompress = .init(&in, container, &.{}); From 2d8d0dd9b0c4bd8d51be33da540b45fef2b20ec2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 18:23:34 -0700 Subject: [PATCH 16/47] std.compress.flate.Decompress: unfuck the test suite --- lib/std/compress/flate/Decompress.zig | 226 ++++++++++++++++---------- 1 file changed, 143 insertions(+), 83 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 26aa668c76..166d5c22f8 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -909,85 +909,153 @@ test "zlib decompress non compressed block (type 0)" { }, "Hello world\n"); } -test "fuzzing tests" { - const cases = [_]struct { - input: []const u8, - out: []const u8 = "", - err: ?anyerror = null, - }{ - .{ .input = "deflate-stream", .out = @embedFile("testdata/fuzz/deflate-stream.expect") }, // 0 - .{ .input = "empty-distance-alphabet01" }, - .{ .input = "empty-distance-alphabet02" }, - .{ .input = "end-of-stream", .err = error.EndOfStream }, - .{ .input = "invalid-distance", .err = error.InvalidMatch }, - .{ .input = "invalid-tree01", .err = error.IncompleteHuffmanTree }, // 5 - .{ .input = "invalid-tree02", .err = error.IncompleteHuffmanTree }, - .{ .input = "invalid-tree03", .err = error.IncompleteHuffmanTree }, - .{ .input = "lengths-overflow", .err = error.InvalidDynamicBlockHeader }, - .{ .input = "out-of-codes", .err = error.InvalidCode }, - .{ .input = "puff01", .err = error.WrongStoredBlockNlen }, // 10 - .{ .input = "puff02", .err = error.EndOfStream }, - .{ .input = "puff03", .out = &[_]u8{0xa} }, - .{ .input = "puff04", .err = error.InvalidCode }, - .{ .input = "puff05", .err = error.EndOfStream }, - .{ .input = "puff06", .err = error.EndOfStream }, - .{ .input = "puff08", .err = error.InvalidCode }, - .{ .input = "puff09", .out = "P" }, - .{ .input = "puff10", .err = error.InvalidCode }, - .{ .input = "puff11", .err = error.InvalidMatch }, - .{ .input = "puff12", .err = error.InvalidDynamicBlockHeader }, // 20 - .{ .input = "puff13", .err = error.IncompleteHuffmanTree }, - .{ .input = "puff14", .err = error.EndOfStream }, - .{ .input = "puff15", .err = error.IncompleteHuffmanTree }, - .{ .input = "puff16", .err = error.InvalidDynamicBlockHeader }, - .{ .input = "puff17", .err = error.MissingEndOfBlockCode }, // 25 - .{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader }, - .{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader }, - .{ .input = "fuzz3", .err = error.InvalidMatch }, - .{ .input = "fuzz4", .err = error.OversubscribedHuffmanTree }, - .{ .input = "puff18", .err = error.OversubscribedHuffmanTree }, // 30 - .{ .input = "puff19", .err = error.OversubscribedHuffmanTree }, - .{ .input = "puff20", .err = error.OversubscribedHuffmanTree }, - .{ .input = "puff21", .err = error.OversubscribedHuffmanTree }, - .{ .input = "puff22", .err = error.OversubscribedHuffmanTree }, - .{ .input = "puff23", .err = error.OversubscribedHuffmanTree }, // 35 - .{ .input = "puff24", .err = error.IncompleteHuffmanTree }, - .{ .input = "puff25", .err = error.OversubscribedHuffmanTree }, - .{ .input = "puff26", .err = error.InvalidDynamicBlockHeader }, - .{ .input = "puff27", .err = error.InvalidDynamicBlockHeader }, - }; - - inline for (cases) |c| { - var in: Reader = .fixed(@embedFile("testdata/fuzz/" ++ c.input ++ ".input")); - var aw: Writer.Allocating = .init(testing.allocator); - try aw.ensureUnusedCapacity(flate.history_len); - defer aw.deinit(); - - var decompress: Decompress = .init(&in, .raw, &.{}); - const r = &decompress.reader; - if (c.err) |expected_err| { - try testing.expectError(error.ReadFailed, r.streamRemaining(&aw.writer)); - try testing.expectEqual(expected_err, decompress.read_err orelse return error.TestFailed); - } else { - _ = try r.streamRemaining(&aw.writer); - try testing.expectEqualStrings(c.out, aw.getWritten()); - } - } +test "failing end-of-stream" { + try testFailure(@embedFile("testdata/fuzz/end-of-stream.input"), error.EndOfStream); +} +test "failing invalid-distance" { + try testFailure(@embedFile("testdata/fuzz/invalid-distance.input"), error.InvalidMatch); +} +test "failing invalid-tree01" { + try testFailure(@embedFile("testdata/fuzz/invalid-tree01.input"), error.IncompleteHuffmanTree); +} +test "failing invalid-tree02" { + try testFailure(@embedFile("testdata/fuzz/invalid-tree02.input"), error.IncompleteHuffmanTree); +} +test "failing invalid-tree03" { + try testFailure(@embedFile("testdata/fuzz/invalid-tree03.input"), error.IncompleteHuffmanTree); +} +test "failing lengths-overflow" { + try testFailure(@embedFile("testdata/fuzz/lengths-overflow.input"), error.InvalidDynamicBlockHeader); +} +test "failing out-of-codes" { + try testFailure(@embedFile("testdata/fuzz/out-of-codes.input"), error.InvalidCode); +} +test "failing puff01" { + try testFailure(@embedFile("testdata/fuzz/puff01.input"), error.WrongStoredBlockNlen); +} +test "failing puff02" { + try testFailure(@embedFile("testdata/fuzz/puff02.input"), error.EndOfStream); +} +test "failing puff04" { + try testFailure(@embedFile("testdata/fuzz/puff04.input"), error.InvalidCode); +} +test "failing puff05" { + try testFailure(@embedFile("testdata/fuzz/puff05.input"), error.EndOfStream); +} +test "failing puff06" { + try testFailure(@embedFile("testdata/fuzz/puff06.input"), error.EndOfStream); +} +test "failing puff08" { + try testFailure(@embedFile("testdata/fuzz/puff08.input"), error.InvalidCode); +} +test "failing puff10" { + try testFailure(@embedFile("testdata/fuzz/puff10.input"), error.InvalidCode); +} +test "failing puff11" { + try testFailure(@embedFile("testdata/fuzz/puff11.input"), error.InvalidMatch); +} +test "failing puff12" { + try testFailure(@embedFile("testdata/fuzz/puff12.input"), error.InvalidDynamicBlockHeader); +} +test "failing puff13" { + try testFailure(@embedFile("testdata/fuzz/puff13.input"), error.IncompleteHuffmanTree); +} +test "failing puff14" { + try testFailure(@embedFile("testdata/fuzz/puff14.input"), error.EndOfStream); +} +test "failing puff15" { + try testFailure(@embedFile("testdata/fuzz/puff15.input"), error.IncompleteHuffmanTree); +} +test "failing puff16" { + try testFailure(@embedFile("testdata/fuzz/puff16.input"), error.InvalidDynamicBlockHeader); +} +test "failing puff17" { + try testFailure(@embedFile("testdata/fuzz/puff17.input"), error.MissingEndOfBlockCode); +} +test "failing fuzz1" { + try testFailure(@embedFile("testdata/fuzz/fuzz1.input"), error.InvalidDynamicBlockHeader); +} +test "failing fuzz2" { + try testFailure(@embedFile("testdata/fuzz/fuzz2.input"), error.InvalidDynamicBlockHeader); +} +test "failing fuzz3" { + try testFailure(@embedFile("testdata/fuzz/fuzz3.input"), error.InvalidMatch); +} +test "failing fuzz4" { + try testFailure(@embedFile("testdata/fuzz/fuzz4.input"), error.OversubscribedHuffmanTree); +} +test "failing puff18" { + try testFailure(@embedFile("testdata/fuzz/puff18.input"), error.OversubscribedHuffmanTree); +} +test "failing puff19" { + try testFailure(@embedFile("testdata/fuzz/puff19.input"), error.OversubscribedHuffmanTree); +} +test "failing puff20" { + try testFailure(@embedFile("testdata/fuzz/puff20.input"), error.OversubscribedHuffmanTree); +} +test "failing puff21" { + try testFailure(@embedFile("testdata/fuzz/puff21.input"), error.OversubscribedHuffmanTree); +} +test "failing puff22" { + try testFailure(@embedFile("testdata/fuzz/puff22.input"), error.OversubscribedHuffmanTree); +} +test "failing puff23" { + try testFailure(@embedFile("testdata/fuzz/puff23.input"), error.OversubscribedHuffmanTree); +} +test "failing puff24" { + try testFailure(@embedFile("testdata/fuzz/puff24.input"), error.IncompleteHuffmanTree); +} +test "failing puff25" { + try testFailure(@embedFile("testdata/fuzz/puff25.input"), error.OversubscribedHuffmanTree); +} +test "failing puff26" { + try testFailure(@embedFile("testdata/fuzz/puff26.input"), error.InvalidDynamicBlockHeader); +} +test "failing puff27" { + try testFailure(@embedFile("testdata/fuzz/puff27.input"), error.InvalidDynamicBlockHeader); } -test "bug 18966" { - const input = @embedFile("testdata/fuzz/bug_18966.input"); - const expect = @embedFile("testdata/fuzz/bug_18966.expect"); - - var in: Reader = .fixed(input); +fn testFailure(in: []const u8, expected_err: anyerror) !void { + var reader: Reader = .fixed(in); var aw: Writer.Allocating = .init(testing.allocator); try aw.ensureUnusedCapacity(flate.history_len); defer aw.deinit(); - var decompress: Decompress = .init(&in, .gzip, &.{}); - const r = &decompress.reader; - _ = try r.streamRemaining(&aw.writer); - try testing.expectEqualStrings(expect, aw.getWritten()); + var decompress: Decompress = .init(&reader, .raw, &.{}); + try testing.expectError(error.ReadFailed, decompress.reader.streamRemaining(&aw.writer)); + try testing.expectEqual(expected_err, decompress.read_err orelse return error.TestFailed); +} + +test "deflate-stream" { + try testDecompress( + .raw, + @embedFile("testdata/fuzz/deflate-stream.input"), + @embedFile("testdata/fuzz/deflate-stream.expect"), + ); +} + +test "empty-distance-alphabet01" { + try testDecompress(.raw, @embedFile("testdata/fuzz/empty-distance-alphabet01.input"), ""); +} + +test "empty-distance-alphabet02" { + try testDecompress(.raw, @embedFile("testdata/fuzz/empty-distance-alphabet02.input"), ""); +} + +test "puff03" { + try testDecompress(.raw, @embedFile("testdata/fuzz/puff03.input"), &.{0xa}); +} + +test "puff09" { + try testDecompress(.raw, @embedFile("testdata/fuzz/puff09.input"), "P"); +} + +test "bug 18966" { + try testDecompress( + .gzip, + @embedFile("testdata/fuzz/bug_18966.input"), + @embedFile("testdata/fuzz/bug_18966.expect"), + ); } test "reading into empty buffer" { @@ -1130,21 +1198,13 @@ test "zlib should not overshoot" { var out: [128]u8 = undefined; { - const n = try decompress.reader.readSliceShort(out[0..]); - - // Expected decompressed data + const n = try decompress.reader.readSliceShort(&out); try std.testing.expectEqual(46, n); try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]); - - // Decompressor don't overshoot underlying reader. - // It is leaving it at the end of compressed data chunk. - try std.testing.expectEqual(data.len - 4, reader.seek); - // TODO what was this testing, exactly? - //try std.testing.expectEqual(0, decompress.unreadBytes()); } // 4 bytes after compressed chunk are available in reader. - const n = try reader.readSliceShort(out[0..]); + const n = try reader.readSliceShort(&out); try std.testing.expectEqual(n, 4); try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]); } From f644f40702fa6c6d9ec1a1d396ddd896048c966e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 18:35:51 -0700 Subject: [PATCH 17/47] implement tossBitsEnding --- lib/std/compress/flate/Decompress.zig | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 166d5c22f8..1fde977f3e 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -491,9 +491,21 @@ fn tossBits(d: *Decompress, n: u6) !void { } fn tossBitsEnding(d: *Decompress, n: u6) !void { - _ = d; - _ = n; - @panic("TODO"); + const remaining_bits = d.remaining_bits; + const in = d.input; + var remaining_needed_bits = n - remaining_bits; + while (remaining_needed_bits >= 8) { + try in.discardAll(1); + remaining_needed_bits -= 8; + } + if (remaining_needed_bits == 0) { + d.next_bits = 0; + d.remaining_bits = 0; + } else { + const byte = try in.takeByte(); + d.next_bits = @as(usize, byte) >> remaining_needed_bits; + d.remaining_bits = @intCast(8 - remaining_needed_bits); + } } fn takeBitsRuntime(d: *Decompress, n: u4) !u16 { From 8ab91a6fe9d37ffc51645718a077b781be5d8873 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 18:59:03 -0700 Subject: [PATCH 18/47] error.EndOfStream disambiguation --- lib/std/compress/flate/Decompress.zig | 225 ++++++++++++-------------- 1 file changed, 103 insertions(+), 122 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 1fde977f3e..f3cd45fe7e 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -44,14 +44,13 @@ const State = union(enum) { pub const Error = Container.Error || error{ InvalidCode, InvalidMatch, - InvalidBlockType, WrongStoredBlockNlen, InvalidDynamicBlockHeader, - EndOfStream, ReadFailed, OversubscribedHuffmanTree, IncompleteHuffmanTree, MissingEndOfBlockCode, + EndOfStream, }; pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { @@ -153,7 +152,14 @@ fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol { pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize { const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); return readInner(d, w, limit) catch |err| switch (err) { - error.EndOfStream => return error.EndOfStream, + error.EndOfStream => { + if (d.state == .end) { + return error.EndOfStream; + } else { + d.read_err = error.EndOfStream; + return error.ReadFailed; + } + }, error.WriteFailed => return error.WriteFailed, else => |e| { // In the event of an error, state is unmodified so that it can be @@ -922,120 +928,109 @@ test "zlib decompress non compressed block (type 0)" { } test "failing end-of-stream" { - try testFailure(@embedFile("testdata/fuzz/end-of-stream.input"), error.EndOfStream); + try testFailure(.raw, @embedFile("testdata/fuzz/end-of-stream.input"), error.EndOfStream); } test "failing invalid-distance" { - try testFailure(@embedFile("testdata/fuzz/invalid-distance.input"), error.InvalidMatch); + try testFailure(.raw, @embedFile("testdata/fuzz/invalid-distance.input"), error.InvalidMatch); } test "failing invalid-tree01" { - try testFailure(@embedFile("testdata/fuzz/invalid-tree01.input"), error.IncompleteHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree01.input"), error.IncompleteHuffmanTree); } test "failing invalid-tree02" { - try testFailure(@embedFile("testdata/fuzz/invalid-tree02.input"), error.IncompleteHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree02.input"), error.IncompleteHuffmanTree); } test "failing invalid-tree03" { - try testFailure(@embedFile("testdata/fuzz/invalid-tree03.input"), error.IncompleteHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree03.input"), error.IncompleteHuffmanTree); } test "failing lengths-overflow" { - try testFailure(@embedFile("testdata/fuzz/lengths-overflow.input"), error.InvalidDynamicBlockHeader); + try testFailure(.raw, @embedFile("testdata/fuzz/lengths-overflow.input"), error.InvalidDynamicBlockHeader); } test "failing out-of-codes" { - try testFailure(@embedFile("testdata/fuzz/out-of-codes.input"), error.InvalidCode); + try testFailure(.raw, @embedFile("testdata/fuzz/out-of-codes.input"), error.InvalidCode); } test "failing puff01" { - try testFailure(@embedFile("testdata/fuzz/puff01.input"), error.WrongStoredBlockNlen); + try testFailure(.raw, @embedFile("testdata/fuzz/puff01.input"), error.WrongStoredBlockNlen); } test "failing puff02" { - try testFailure(@embedFile("testdata/fuzz/puff02.input"), error.EndOfStream); + try testFailure(.raw, @embedFile("testdata/fuzz/puff02.input"), error.EndOfStream); } test "failing puff04" { - try testFailure(@embedFile("testdata/fuzz/puff04.input"), error.InvalidCode); + try testFailure(.raw, @embedFile("testdata/fuzz/puff04.input"), error.InvalidCode); } test "failing puff05" { - try testFailure(@embedFile("testdata/fuzz/puff05.input"), error.EndOfStream); + try testFailure(.raw, @embedFile("testdata/fuzz/puff05.input"), error.EndOfStream); } test "failing puff06" { - try testFailure(@embedFile("testdata/fuzz/puff06.input"), error.EndOfStream); + try testFailure(.raw, @embedFile("testdata/fuzz/puff06.input"), error.EndOfStream); } test "failing puff08" { - try testFailure(@embedFile("testdata/fuzz/puff08.input"), error.InvalidCode); + try testFailure(.raw, @embedFile("testdata/fuzz/puff08.input"), error.InvalidCode); } test "failing puff10" { - try testFailure(@embedFile("testdata/fuzz/puff10.input"), error.InvalidCode); + try testFailure(.raw, @embedFile("testdata/fuzz/puff10.input"), error.InvalidCode); } test "failing puff11" { - try testFailure(@embedFile("testdata/fuzz/puff11.input"), error.InvalidMatch); + try testFailure(.raw, @embedFile("testdata/fuzz/puff11.input"), error.InvalidMatch); } test "failing puff12" { - try testFailure(@embedFile("testdata/fuzz/puff12.input"), error.InvalidDynamicBlockHeader); + try testFailure(.raw, @embedFile("testdata/fuzz/puff12.input"), error.InvalidDynamicBlockHeader); } test "failing puff13" { - try testFailure(@embedFile("testdata/fuzz/puff13.input"), error.IncompleteHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/puff13.input"), error.IncompleteHuffmanTree); } test "failing puff14" { - try testFailure(@embedFile("testdata/fuzz/puff14.input"), error.EndOfStream); + try testFailure(.raw, @embedFile("testdata/fuzz/puff14.input"), error.EndOfStream); } test "failing puff15" { - try testFailure(@embedFile("testdata/fuzz/puff15.input"), error.IncompleteHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/puff15.input"), error.IncompleteHuffmanTree); } test "failing puff16" { - try testFailure(@embedFile("testdata/fuzz/puff16.input"), error.InvalidDynamicBlockHeader); + try testFailure(.raw, @embedFile("testdata/fuzz/puff16.input"), error.InvalidDynamicBlockHeader); } test "failing puff17" { - try testFailure(@embedFile("testdata/fuzz/puff17.input"), error.MissingEndOfBlockCode); + try testFailure(.raw, @embedFile("testdata/fuzz/puff17.input"), error.MissingEndOfBlockCode); } test "failing fuzz1" { - try testFailure(@embedFile("testdata/fuzz/fuzz1.input"), error.InvalidDynamicBlockHeader); + try testFailure(.raw, @embedFile("testdata/fuzz/fuzz1.input"), error.InvalidDynamicBlockHeader); } test "failing fuzz2" { - try testFailure(@embedFile("testdata/fuzz/fuzz2.input"), error.InvalidDynamicBlockHeader); + try testFailure(.raw, @embedFile("testdata/fuzz/fuzz2.input"), error.InvalidDynamicBlockHeader); } test "failing fuzz3" { - try testFailure(@embedFile("testdata/fuzz/fuzz3.input"), error.InvalidMatch); + try testFailure(.raw, @embedFile("testdata/fuzz/fuzz3.input"), error.InvalidMatch); } test "failing fuzz4" { - try testFailure(@embedFile("testdata/fuzz/fuzz4.input"), error.OversubscribedHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/fuzz4.input"), error.OversubscribedHuffmanTree); } test "failing puff18" { - try testFailure(@embedFile("testdata/fuzz/puff18.input"), error.OversubscribedHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/puff18.input"), error.OversubscribedHuffmanTree); } test "failing puff19" { - try testFailure(@embedFile("testdata/fuzz/puff19.input"), error.OversubscribedHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/puff19.input"), error.OversubscribedHuffmanTree); } test "failing puff20" { - try testFailure(@embedFile("testdata/fuzz/puff20.input"), error.OversubscribedHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/puff20.input"), error.OversubscribedHuffmanTree); } test "failing puff21" { - try testFailure(@embedFile("testdata/fuzz/puff21.input"), error.OversubscribedHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/puff21.input"), error.OversubscribedHuffmanTree); } test "failing puff22" { - try testFailure(@embedFile("testdata/fuzz/puff22.input"), error.OversubscribedHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/puff22.input"), error.OversubscribedHuffmanTree); } test "failing puff23" { - try testFailure(@embedFile("testdata/fuzz/puff23.input"), error.OversubscribedHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/puff23.input"), error.OversubscribedHuffmanTree); } test "failing puff24" { - try testFailure(@embedFile("testdata/fuzz/puff24.input"), error.IncompleteHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/puff24.input"), error.IncompleteHuffmanTree); } test "failing puff25" { - try testFailure(@embedFile("testdata/fuzz/puff25.input"), error.OversubscribedHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/puff25.input"), error.OversubscribedHuffmanTree); } test "failing puff26" { - try testFailure(@embedFile("testdata/fuzz/puff26.input"), error.InvalidDynamicBlockHeader); + try testFailure(.raw, @embedFile("testdata/fuzz/puff26.input"), error.InvalidDynamicBlockHeader); } test "failing puff27" { - try testFailure(@embedFile("testdata/fuzz/puff27.input"), error.InvalidDynamicBlockHeader); -} - -fn testFailure(in: []const u8, expected_err: anyerror) !void { - var reader: Reader = .fixed(in); - var aw: Writer.Allocating = .init(testing.allocator); - try aw.ensureUnusedCapacity(flate.history_len); - defer aw.deinit(); - - var decompress: Decompress = .init(&reader, .raw, &.{}); - try testing.expectError(error.ReadFailed, decompress.reader.streamRemaining(&aw.writer)); - try testing.expectEqual(expected_err, decompress.read_err orelse return error.TestFailed); + try testFailure(.raw, @embedFile("testdata/fuzz/puff27.input"), error.InvalidDynamicBlockHeader); } test "deflate-stream" { @@ -1097,82 +1092,57 @@ test "don't read past deflate stream's end" { test "zlib header" { // Truncated header - try testing.expectError( - error.EndOfStream, - testDecompress(.zlib, &[_]u8{0x78}, ""), - ); + try testFailure(.zlib, &[_]u8{0x78}, error.EndOfStream); + // Wrong CM - try testing.expectError( - error.BadZlibHeader, - testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""), - ); + try testFailure(.zlib, &[_]u8{ 0x79, 0x94 }, error.BadZlibHeader); + // Wrong CINFO - try testing.expectError( - error.BadZlibHeader, - testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""), - ); + try testFailure(.zlib, &[_]u8{ 0x88, 0x98 }, error.BadZlibHeader); + // Wrong checksum - try testing.expectError( - error.WrongZlibChecksum, - testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""), - ); + try testFailure(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, error.WrongZlibChecksum); + // Truncated checksum - try testing.expectError( - error.EndOfStream, - testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""), - ); + try testFailure(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, error.EndOfStream); } test "gzip header" { // Truncated header - try testing.expectError( - error.EndOfStream, - testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined), - ); + try testFailure(.gzip, &[_]u8{ 0x1f, 0x8B }, error.EndOfStream); + // Wrong CM - try testing.expectError( - error.BadGzipHeader, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, - }, undefined), - ); + try testFailure(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, + }, error.BadGzipHeader); // Wrong checksum - try testing.expectError( - error.WrongGzipChecksum, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, - }, undefined), - ); + try testFailure(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, + }, error.WrongGzipChecksum); + // Truncated checksum - try testing.expectError( - error.EndOfStream, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, - }, undefined), - ); + try testFailure(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, + }, error.EndOfStream); + // Wrong initial size - try testing.expectError( - error.WrongGzipSize, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, - }, undefined), - ); + try testFailure(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, + }, error.WrongGzipSize); + // Truncated initial size field - try testing.expectError( - error.EndOfStream, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, - }, undefined), - ); + try testFailure(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, + }, error.EndOfStream); try testDecompress(.gzip, &[_]u8{ // GZIP header @@ -1184,17 +1154,6 @@ test "gzip header" { }, ""); } -fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void { - var in: std.Io.Reader = .fixed(compressed); - var aw: std.Io.Writer.Allocating = .init(testing.allocator); - try aw.ensureUnusedCapacity(flate.history_len); - defer aw.deinit(); - - var decompress: Decompress = .init(&in, container, &.{}); - _ = try decompress.reader.streamRemaining(&aw.writer); - try testing.expectEqualSlices(u8, expected_plain, aw.getWritten()); -} - test "zlib should not overshoot" { // Compressed zlib data with extra 4 bytes at the end. const data = [_]u8{ @@ -1220,3 +1179,25 @@ test "zlib should not overshoot" { try std.testing.expectEqual(n, 4); try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]); } + +fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !void { + var reader: Reader = .fixed(in); + var aw: Writer.Allocating = .init(testing.allocator); + try aw.ensureUnusedCapacity(flate.history_len); + defer aw.deinit(); + + var decompress: Decompress = .init(&reader, container, &.{}); + try testing.expectError(error.ReadFailed, decompress.reader.streamRemaining(&aw.writer)); + try testing.expectEqual(expected_err, decompress.read_err orelse return error.TestFailed); +} + +fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void { + var in: std.Io.Reader = .fixed(compressed); + var aw: std.Io.Writer.Allocating = .init(testing.allocator); + try aw.ensureUnusedCapacity(flate.history_len); + defer aw.deinit(); + + var decompress: Decompress = .init(&in, container, &.{}); + _ = try decompress.reader.streamRemaining(&aw.writer); + try testing.expectEqualSlices(u8, expected_plain, aw.getWritten()); +} From c00fb86db6a9d1f1eb0684304eeb79b1f12bb805 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 20:28:07 -0700 Subject: [PATCH 19/47] fix peekBitsEnding --- lib/std/compress/flate/Decompress.zig | 43 +++++++++++++-------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index f3cd45fe7e..fb7f2bbf9f 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -139,11 +139,8 @@ fn dynamicCodeLength(self: *Decompress, code: u16, lens: []u4, pos: usize) !usiz } } -// Peek 15 bits from bits reader (maximum code len is 15 bits). Use -// decoder to find symbol for that code. We then know how many bits is -// used. Shift bit reader for that much bits, those bits are used. And -// return symbol. fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol { + // Maximum code len is 15 bits. const sym = try decoder.find(@bitReverse(try self.peekBits(u15))); try self.tossBits(sym.code_bits); return sym; @@ -451,31 +448,30 @@ fn peekBitsEnding(d: *Decompress, comptime U: type) !U { const remaining_bits = d.remaining_bits; const next_bits = d.next_bits; const in = d.input; - var u: U = 0; + var u: usize = 0; var remaining_needed_bits = @bitSizeOf(U) - remaining_bits; - var peek_len: usize = 0; - while (@bitSizeOf(U) >= 8 and remaining_needed_bits >= 8) { - peek_len += 1; - const byte = try specialPeek(in, next_bits, peek_len); - u = (u << 8) | byte; + var i: usize = 0; + while (remaining_needed_bits >= 8) { + const byte = try specialPeek(in, next_bits, i); + u |= @as(usize, byte) << @intCast(i * 8); remaining_needed_bits -= 8; + i += 1; } if (remaining_needed_bits != 0) { - peek_len += 1; - const byte = try specialPeek(in, next_bits, peek_len); - u = @intCast((@as(usize, u) << remaining_needed_bits) | (byte & ((@as(usize, 1) << remaining_needed_bits) - 1))); + const byte = try specialPeek(in, next_bits, i); + u |= @as(usize, byte) << @intCast((i * 8) + remaining_needed_bits); } - return @intCast((@as(usize, u) << remaining_bits) | next_bits); + return @truncate((u << remaining_bits) | next_bits); } /// If there is any unconsumed data, handles EndOfStream by pretending there /// are zeroes afterwards. -fn specialPeek(in: *Reader, next_bits: usize, n: usize) Reader.Error!u8 { - const peeked = in.peek(n) catch |err| switch (err) { +fn specialPeek(in: *Reader, next_bits: usize, i: usize) Reader.Error!u8 { + const peeked = in.peek(i + 1) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, - error.EndOfStream => if (next_bits == 0 and n == 0) return error.EndOfStream else return 0, + error.EndOfStream => if (next_bits == 0 and i == 0) return error.EndOfStream else return 0, }; - return peeked[n - 1]; + return peeked[i]; } fn tossBits(d: *Decompress, n: u6) !void { @@ -507,11 +503,14 @@ fn tossBitsEnding(d: *Decompress, n: u6) !void { if (remaining_needed_bits == 0) { d.next_bits = 0; d.remaining_bits = 0; - } else { - const byte = try in.takeByte(); - d.next_bits = @as(usize, byte) >> remaining_needed_bits; - d.remaining_bits = @intCast(8 - remaining_needed_bits); + return; } + const byte = in.takeByte() catch |err| switch (err) { + error.ReadFailed => return error.ReadFailed, + error.EndOfStream => if (remaining_bits == 0) return error.EndOfStream else 0, + }; + d.next_bits = @as(usize, byte) >> remaining_needed_bits; + d.remaining_bits = @intCast(8 - remaining_needed_bits); } fn takeBitsRuntime(d: *Decompress, n: u4) !u16 { From 63f496c4f9a44e145362d3b851d46e9326ecf1c2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 20:31:00 -0700 Subject: [PATCH 20/47] make takeBits deal with integers only --- lib/std/compress/flate/Decompress.zig | 36 ++++++++------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index fb7f2bbf9f..118c8f5d62 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -216,7 +216,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S }, .block_header => { d.final_block = (try d.takeBits(u1)) != 0; - const block_type = try d.takeBits(BlockType); + const block_type: BlockType = @enumFromInt(try d.takeBits(u2)); switch (block_type) { .stored => { d.alignBitsToByte(); // skip padding until byte boundary @@ -372,41 +372,31 @@ fn writeMatch(w: *Writer, length: u16, distance: u16) !void { for (dest, src) |*d, s| d.* = s; } -fn takeBits(d: *Decompress, comptime T: type) !T { - const U = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(T) } }); +fn takeBits(d: *Decompress, comptime U: type) !U { const remaining_bits = d.remaining_bits; const next_bits = d.next_bits; - if (remaining_bits >= @bitSizeOf(T)) { + if (remaining_bits >= @bitSizeOf(U)) { const u: U = @truncate(next_bits); - d.next_bits = next_bits >> @bitSizeOf(T); - d.remaining_bits = remaining_bits - @bitSizeOf(T); - return switch (@typeInfo(T)) { - .int => u, - .@"enum" => @enumFromInt(u), - else => @bitCast(u), - }; + d.next_bits = next_bits >> @bitSizeOf(U); + d.remaining_bits = remaining_bits - @bitSizeOf(U); + return u; } const in = d.input; const next_int = in.takeInt(usize, .little) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, - error.EndOfStream => return takeBitsEnding(d, T), + error.EndOfStream => return takeBitsEnding(d, U), }; - const needed_bits = @bitSizeOf(T) - remaining_bits; + const needed_bits = @bitSizeOf(U) - remaining_bits; const u: U = @intCast(((next_int & ((@as(usize, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); d.next_bits = next_int >> needed_bits; d.remaining_bits = @intCast(@bitSizeOf(usize) - @as(usize, needed_bits)); - return switch (@typeInfo(T)) { - .int => u, - .@"enum" => @enumFromInt(u), - else => @bitCast(u), - }; + return u; } -fn takeBitsEnding(d: *Decompress, comptime T: type) !T { +fn takeBitsEnding(d: *Decompress, comptime U: type) !U { const remaining_bits = d.remaining_bits; const next_bits = d.next_bits; const in = d.input; - const U = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(T) } }); var u: U = 0; var remaining_needed_bits = @bitSizeOf(U) - remaining_bits; while (@bitSizeOf(U) >= 8 and remaining_needed_bits >= 8) { @@ -424,11 +414,7 @@ fn takeBitsEnding(d: *Decompress, comptime T: type) !T { d.remaining_bits = @intCast(8 - remaining_needed_bits); } u = @intCast((@as(usize, u) << remaining_bits) | next_bits); - return switch (@typeInfo(T)) { - .int => u, - .@"enum" => @enumFromInt(u), - else => @bitCast(u), - }; + return u; } fn peekBits(d: *Decompress, comptime U: type) !U { From 5bc63794cc9f38801e674d306c2c9ef4ccf92f13 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 20:44:50 -0700 Subject: [PATCH 21/47] fix takeBitsEnding --- lib/std/compress/flate/Decompress.zig | 28 +++++++++++---------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 118c8f5d62..012c1a109d 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -397,23 +397,17 @@ fn takeBitsEnding(d: *Decompress, comptime U: type) !U { const remaining_bits = d.remaining_bits; const next_bits = d.next_bits; const in = d.input; - var u: U = 0; - var remaining_needed_bits = @bitSizeOf(U) - remaining_bits; - while (@bitSizeOf(U) >= 8 and remaining_needed_bits >= 8) { - const byte = try in.takeByte(); - u = (u << 8) | byte; - remaining_needed_bits -= 8; - } - if (remaining_needed_bits == 0) { - d.next_bits = 0; - d.remaining_bits = 0; - } else { - const byte = try in.takeByte(); - u = @intCast((@as(usize, u) << remaining_needed_bits) | (byte & ((@as(usize, 1) << remaining_needed_bits) - 1))); - d.next_bits = @as(usize, byte) >> remaining_needed_bits; - d.remaining_bits = @intCast(8 - remaining_needed_bits); - } - u = @intCast((@as(usize, u) << remaining_bits) | next_bits); + const n = in.bufferedLen(); + assert(n < @sizeOf(usize)); + const needed_bits = @bitSizeOf(U) - remaining_bits; + if (n * 8 < needed_bits) return error.EndOfStream; + const next_int = in.takeVarInt(usize, .little, n) catch |err| switch (err) { + error.ReadFailed => return error.ReadFailed, + error.EndOfStream => unreachable, + }; + const u: U = @intCast(((next_int & ((@as(usize, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); + d.next_bits = next_int >> needed_bits; + d.remaining_bits = @intCast(n * 8 - @as(usize, needed_bits)); return u; } From 2569f4ff8514f47730c1abd42691f8fc1cdd22d5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 21:05:33 -0700 Subject: [PATCH 22/47] simplify tossBitsEnding --- lib/std/compress/flate/Decompress.zig | 48 +++++++++++++++++---------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 012c1a109d..6a37987501 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -219,7 +219,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S const block_type: BlockType = @enumFromInt(try d.takeBits(u2)); switch (block_type) { .stored => { - d.alignBitsToByte(); // skip padding until byte boundary + d.alignBitsDiscarding(); // everything after this is byte aligned in stored block const len = try in.takeInt(u16, .little); const nlen = try in.takeInt(u16, .little); @@ -333,20 +333,23 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S return @intFromEnum(limit) - remaining; }, .protocol_footer => { - d.alignBitsToByte(); switch (d.container_metadata) { .gzip => |*gzip| { + d.alignBitsDiscarding(); gzip.* = .{ .crc = try in.takeInt(u32, .little), .count = try in.takeInt(u32, .little), }; }, .zlib => |*zlib| { + d.alignBitsDiscarding(); zlib.* = .{ .adler = try in.takeInt(u32, .little), }; }, - .raw => {}, + .raw => { + d.alignBitsPreserving(); + }, } d.state = .end; return 0; @@ -475,22 +478,16 @@ fn tossBits(d: *Decompress, n: u6) !void { fn tossBitsEnding(d: *Decompress, n: u6) !void { const remaining_bits = d.remaining_bits; const in = d.input; - var remaining_needed_bits = n - remaining_bits; - while (remaining_needed_bits >= 8) { - try in.discardAll(1); - remaining_needed_bits -= 8; - } - if (remaining_needed_bits == 0) { - d.next_bits = 0; - d.remaining_bits = 0; - return; - } - const byte = in.takeByte() catch |err| switch (err) { + const buffered_n = in.bufferedLen(); + if (buffered_n == 0) return error.EndOfStream; + assert(buffered_n < @sizeOf(usize)); + const needed_bits = n - remaining_bits; + const next_int = in.takeVarInt(usize, .little, buffered_n) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, - error.EndOfStream => if (remaining_bits == 0) return error.EndOfStream else 0, + error.EndOfStream => unreachable, }; - d.next_bits = @as(usize, byte) >> remaining_needed_bits; - d.remaining_bits = @intCast(8 - remaining_needed_bits); + d.next_bits = next_int >> needed_bits; + d.remaining_bits = @intCast(@as(usize, n) * 8 -| @as(usize, needed_bits)); } fn takeBitsRuntime(d: *Decompress, n: u4) !u16 { @@ -501,7 +498,7 @@ fn takeBitsRuntime(d: *Decompress, n: u4) !u16 { return u; } -fn alignBitsToByte(d: *Decompress) void { +fn alignBitsDiscarding(d: *Decompress) void { const remaining_bits = d.remaining_bits; const next_bits = d.next_bits; if (remaining_bits == 0) return; @@ -518,6 +515,21 @@ fn alignBitsToByte(d: *Decompress) void { d.next_bits = 0; } +fn alignBitsPreserving(d: *Decompress) void { + const remaining_bits: usize = d.remaining_bits; + if (remaining_bits == 0) return; + const n_bytes = (remaining_bits + 7) / 8; + const in = d.input; + in.seek -= n_bytes; + var put_back_bits = d.next_bits; + for (in.buffer[in.seek..][0..n_bytes]) |*b| { + b.* = @truncate(put_back_bits); + put_back_bits >>= 8; + } + d.remaining_bits = 0; + d.next_bits = 0; +} + /// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code. /// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12 /// Lit Value Bits Codes From 05ce1f99a60df0bcede00af1c98a1e5fdce35051 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 22:12:46 -0700 Subject: [PATCH 23/47] compiler: update to new flate API --- lib/std/Io/Reader.zig | 9 +++ lib/std/compress/flate/Decompress.zig | 10 ++-- lib/std/http/Client.zig | 19 +++--- lib/std/zip.zig | 84 ++++++++++++++------------- src/Package/Fetch.zig | 25 ++++---- src/Package/Fetch/git.zig | 67 ++++++++++----------- src/link/Elf/Object.zig | 15 +++-- 7 files changed, 122 insertions(+), 107 deletions(-) diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index 8b389d83c1..8dddc49ad2 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -1709,6 +1709,15 @@ fn failingDiscard(r: *Reader, limit: Limit) Error!usize { return error.ReadFailed; } +pub fn adaptToOldInterface(r: *Reader) std.Io.AnyReader { + return .{ .context = r, .readFn = derpRead }; +} + +fn derpRead(context: *const anyopaque, buffer: []u8) anyerror!usize { + const r: *Reader = @constCast(@alignCast(@ptrCast(context))); + return r.readSliceShort(buffer); +} + test "readAlloc when the backing reader provides one byte at a time" { const str = "This is a test"; var tiny_buffer: [1]u8 = undefined; diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 6a37987501..69764245a7 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -23,7 +23,7 @@ dst_dec: DistanceDecoder, final_block: bool, state: State, -read_err: ?Error, +err: ?Error, const BlockType = enum(u2) { stored = 0, @@ -74,7 +74,7 @@ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { .dst_dec = .{}, .final_block = false, .state = .protocol_header, - .read_err = null, + .err = null, }; } @@ -153,7 +153,7 @@ pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!us if (d.state == .end) { return error.EndOfStream; } else { - d.read_err = error.EndOfStream; + d.err = error.EndOfStream; return error.ReadFailed; } }, @@ -161,7 +161,7 @@ pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!us else => |e| { // In the event of an error, state is unmodified so that it can be // better used to diagnose the failure. - d.read_err = e; + d.err = e; return error.ReadFailed; }, }; @@ -1179,7 +1179,7 @@ fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !vo var decompress: Decompress = .init(&reader, container, &.{}); try testing.expectError(error.ReadFailed, decompress.reader.streamRemaining(&aw.writer)); - try testing.expectEqual(expected_err, decompress.read_err orelse return error.TestFailed); + try testing.expectEqual(expected_err, decompress.err orelse return error.TestFailed); } fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void { diff --git a/lib/std/http/Client.zig b/lib/std/http/Client.zig index 83c1c8b50b..20f6018e45 100644 --- a/lib/std/http/Client.zig +++ b/lib/std/http/Client.zig @@ -405,8 +405,8 @@ pub const RequestTransfer = union(enum) { /// The decompressor for response messages. pub const Compression = union(enum) { - deflate: std.compress.flate.Decompress, - gzip: std.compress.flate.Decompress, + //deflate: std.compress.flate.Decompress, + //gzip: std.compress.flate.Decompress, // https://github.com/ziglang/zig/issues/18937 //zstd: ZstdDecompressor, none: void, @@ -1074,12 +1074,10 @@ pub const Request = struct { switch (req.response.transfer_compression) { .identity => req.response.compression = .none, .compress, .@"x-compress" => return error.CompressionUnsupported, - .deflate => req.response.compression = .{ - .deflate = std.compress.zlib.decompressor(req.transferReader()), - }, - .gzip, .@"x-gzip" => req.response.compression = .{ - .gzip = std.compress.gzip.decompressor(req.transferReader()), - }, + // I'm about to upstream my http.Client rewrite + .deflate => return error.CompressionUnsupported, + // I'm about to upstream my http.Client rewrite + .gzip, .@"x-gzip" => return error.CompressionUnsupported, // https://github.com/ziglang/zig/issues/18937 //.zstd => req.response.compression = .{ // .zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()), @@ -1105,8 +1103,9 @@ pub const Request = struct { /// Reads data from the response body. Must be called after `wait`. pub fn read(req: *Request, buffer: []u8) ReadError!usize { const out_index = switch (req.response.compression) { - .deflate => |*deflate| deflate.read(buffer) catch return error.DecompressionFailure, - .gzip => |*gzip| gzip.read(buffer) catch return error.DecompressionFailure, + // I'm about to upstream my http client rewrite + //.deflate => |*deflate| deflate.readSlice(buffer) catch return error.DecompressionFailure, + //.gzip => |*gzip| gzip.read(buffer) catch return error.DecompressionFailure, // https://github.com/ziglang/zig/issues/18937 //.zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure, else => try req.transferRead(buffer), diff --git a/lib/std/zip.zig b/lib/std/zip.zig index b13c1d5010..b0da6ac266 100644 --- a/lib/std/zip.zig +++ b/lib/std/zip.zig @@ -7,8 +7,9 @@ const builtin = @import("builtin"); const std = @import("std"); const File = std.fs.File; const is_le = builtin.target.cpu.arch.endian() == .little; -const Writer = std.io.Writer; -const Reader = std.io.Reader; +const Writer = std.Io.Writer; +const Reader = std.Io.Reader; +const flate = std.compress.flate; pub const CompressionMethod = enum(u16) { store = 0, @@ -117,6 +118,7 @@ pub const EndRecord = extern struct { pub const FindFileError = File.GetEndPosError || File.SeekError || File.ReadError || error{ ZipNoEndRecord, EndOfStream, + ReadFailed, }; pub fn findFile(fr: *File.Reader) FindFileError!EndRecord { @@ -137,8 +139,7 @@ pub const EndRecord = extern struct { try fr.seekTo(end_pos - @as(u64, new_loaded_len)); const read_buf: []u8 = buf[buf.len - new_loaded_len ..][0..read_len]; - var br = fr.interface().unbuffered(); - br.readSlice(read_buf) catch |err| switch (err) { + fr.interface.readSliceAll(read_buf) catch |err| switch (err) { error.ReadFailed => return fr.err.?, error.EndOfStream => return error.EndOfStream, }; @@ -164,7 +165,7 @@ pub const EndRecord = extern struct { pub const Decompress = struct { interface: Reader, state: union { - inflate: std.compress.flate.Decompress, + inflate: flate.Decompress, store: *Reader, }, @@ -201,7 +202,7 @@ pub const Decompress = struct { fn streamDeflate(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!usize { const d: *Decompress = @fieldParentPtr("interface", r); - return std.compress.flate.Decompress.read(&d.inflate, w, limit); + return flate.Decompress.read(&d.inflate, w, limit); } }; @@ -305,7 +306,7 @@ pub const Iterator = struct { if (locator_end_offset > stream_len) return error.ZipTruncated; try input.seekTo(stream_len - locator_end_offset); - const locator = input.interface.takeStructEndian(EndLocator64, .little) catch |err| switch (err) { + const locator = input.interface.takeStruct(EndLocator64, .little) catch |err| switch (err) { error.ReadFailed => return input.err.?, error.EndOfStream => return error.EndOfStream, }; @@ -318,7 +319,7 @@ pub const Iterator = struct { try input.seekTo(locator.record_file_offset); - const record64 = input.interface.takeStructEndian(EndRecord64, .little) catch |err| switch (err) { + const record64 = input.interface.takeStruct(EndRecord64, .little) catch |err| switch (err) { error.ReadFailed => return input.err.?, error.EndOfStream => return error.EndOfStream, }; @@ -374,7 +375,7 @@ pub const Iterator = struct { const header_zip_offset = self.cd_zip_offset + self.cd_record_offset; const input = self.input; try input.seekTo(header_zip_offset); - const header = input.interface.takeStructEndian(CentralDirectoryFileHeader, .little) catch |err| switch (err) { + const header = input.interface.takeStruct(CentralDirectoryFileHeader, .little) catch |err| switch (err) { error.ReadFailed => return input.err.?, error.EndOfStream => return error.EndOfStream, }; @@ -405,7 +406,7 @@ pub const Iterator = struct { const extra = extra_buf[0..header.extra_len]; try input.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len); - input.interface.readSlice(extra) catch |err| switch (err) { + input.interface.readSliceAll(extra) catch |err| switch (err) { error.ReadFailed => return input.err.?, error.EndOfStream => return error.EndOfStream, }; @@ -460,7 +461,7 @@ pub const Iterator = struct { options: ExtractOptions, filename_buf: []u8, dest: std.fs.Dir, - ) !u32 { + ) !void { if (filename_buf.len < self.filename_len) return error.ZipInsufficientBuffer; switch (self.compression_method) { @@ -470,13 +471,13 @@ pub const Iterator = struct { const filename = filename_buf[0..self.filename_len]; { try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader)); - try stream.interface.readSlice(filename); + try stream.interface.readSliceAll(filename); } const local_data_header_offset: u64 = local_data_header_offset: { const local_header = blk: { try stream.seekTo(self.file_offset); - break :blk try stream.interface.takeStructEndian(LocalFileHeader, .little); + break :blk try stream.interface.takeStruct(LocalFileHeader, .little); }; if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig)) return error.ZipBadFileOffset; @@ -502,7 +503,7 @@ pub const Iterator = struct { { try stream.seekTo(self.file_offset + @sizeOf(LocalFileHeader) + local_header.filename_len); - try stream.interface.readSlice(extra); + try stream.interface.readSliceAll(extra); } var extra_offset: usize = 0; @@ -550,7 +551,7 @@ pub const Iterator = struct { if (self.uncompressed_size != 0) return error.ZipBadDirectorySize; try dest.makePath(filename[0 .. filename.len - 1]); - return std.hash.Crc32.hash(&.{}); + return; } const out_file = blk: { @@ -564,31 +565,36 @@ pub const Iterator = struct { break :blk try dest.createFile(filename, .{ .exclusive = true }); }; defer out_file.close(); - var file_writer = out_file.writer(); - var file_bw = file_writer.writer(&.{}); + var out_file_buffer: [1024]u8 = undefined; + var file_writer = out_file.writer(&out_file_buffer); const local_data_file_offset: u64 = @as(u64, self.file_offset) + @as(u64, @sizeOf(LocalFileHeader)) + local_data_header_offset; try stream.seekTo(local_data_file_offset); - var limited_file_reader = stream.interface.limited(.limited(self.compressed_size)); - var file_read_buffer: [1000]u8 = undefined; - var decompress_read_buffer: [1000]u8 = undefined; - var limited_br = limited_file_reader.reader().buffered(&file_read_buffer); - var decompress: Decompress = undefined; - var decompress_br = decompress.readable(&limited_br, self.compression_method, &decompress_read_buffer); - const start_out = file_bw.count; - var hash_writer = file_bw.hashed(std.hash.Crc32.init()); - var hash_bw = hash_writer.writer(&.{}); - decompress_br.readAll(&hash_bw, .limited(self.uncompressed_size)) catch |err| switch (err) { - error.ReadFailed => return stream.err.?, - error.WriteFailed => return file_writer.err.?, - error.EndOfStream => return error.ZipDecompressTruncated, - }; - if (limited_file_reader.remaining.nonzero()) return error.ZipDecompressTruncated; - const written = file_bw.count - start_out; - if (written != self.uncompressed_size) return error.ZipUncompressSizeMismatch; - return hash_writer.hasher.final(); + + // TODO limit based on self.compressed_size + + switch (self.compression_method) { + .store => { + stream.interface.streamExact(&file_writer.interface, self.uncompressed_size) catch |err| switch (err) { + error.ReadFailed => return stream.err.?, + error.WriteFailed => return file_writer.err.?, + error.EndOfStream => return error.ZipDecompressTruncated, + }; + }, + .deflate => { + var flate_buffer: [flate.max_window_len]u8 = undefined; + var decompress: flate.Decompress = .init(&stream.interface, .raw, &flate_buffer); + decompress.reader.streamExact(&file_writer.interface, self.uncompressed_size) catch |err| switch (err) { + error.ReadFailed => return stream.err.?, + error.WriteFailed => return file_writer.err orelse decompress.err.?, + error.EndOfStream => return error.ZipDecompressTruncated, + }; + }, + else => return error.UnsupportedCompressionMethod, + } + try file_writer.end(); } }; }; @@ -636,19 +642,19 @@ pub const ExtractOptions = struct { /// Allow filenames within the zip to use backslashes. Back slashes are normalized /// to forward slashes before forwarding them to platform APIs. allow_backslashes: bool = false, - diagnostics: ?*Diagnostics = null, + verify_checksums: bool = false, }; /// Extract the zipped files to the given `dest` directory. pub fn extract(dest: std.fs.Dir, fr: *File.Reader, options: ExtractOptions) !void { + if (options.verify_checksums) @panic("TODO unimplemented"); + var iter = try Iterator.init(fr); var filename_buf: [std.fs.max_path_bytes]u8 = undefined; while (try iter.next()) |entry| { - const crc32 = try entry.extract(fr, options, &filename_buf, dest); - if (crc32 != entry.crc32) - return error.ZipCrcMismatch; + try entry.extract(fr, options, &filename_buf, dest); if (options.diagnostics) |d| { try d.nextFilename(filename_buf[0..entry.filename_len]); } diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index f47086bf58..47dfdce7ac 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1203,12 +1203,11 @@ fn unpackResource( return unpackTarball(f, tmp_directory.handle, &adapter.new_interface); }, .@"tar.gz" => { - const reader = resource.reader(); - var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); - var dcp = std.compress.gzip.decompressor(br.reader()); - var adapter_buffer: [1024]u8 = undefined; - var adapter = dcp.reader().adaptToNewApi(&adapter_buffer); - return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface); + var adapter_buffer: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined; + var adapter = resource.reader().adaptToNewApi(&adapter_buffer); + var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; + var decompress: std.compress.flate.Decompress = .init(&adapter.new_interface, .gzip, &flate_buffer); + return try unpackTarball(f, tmp_directory.handle, &decompress.reader); }, .@"tar.xz" => { const gpa = f.arena.child_allocator; @@ -1352,7 +1351,10 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { )); defer zip_file.close(); - std.zip.extract(out_dir, zip_file.seekableStream(), .{ + var zip_file_buffer: [1024]u8 = undefined; + var zip_file_reader = zip_file.reader(&zip_file_buffer); + + std.zip.extract(out_dir, &zip_file_reader, .{ .allow_backslashes = true, .diagnostics = &diagnostics, }) catch |err| return f.fail(f.location_tok, try eb.printString( @@ -1384,23 +1386,26 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!U defer pack_dir.close(); var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); defer pack_file.close(); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); + var pack_file_buffer: [4096]u8 = undefined; + var fifo = std.fifo.LinearFifo(u8, .{ .Slice = {} }).init(&pack_file_buffer); try fifo.pump(resource.fetch_stream.reader(), pack_file.deprecatedWriter()); + var pack_file_reader = pack_file.reader(&pack_file_buffer); + var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); defer index_file.close(); { const index_prog_node = f.prog_node.start("Index pack", 0); defer index_prog_node.end(); var index_buffered_writer = std.io.bufferedWriter(index_file.deprecatedWriter()); - try git.indexPack(gpa, object_format, pack_file, index_buffered_writer.writer()); + try git.indexPack(gpa, object_format, &pack_file_reader, index_buffered_writer.writer()); try index_buffered_writer.flush(); } { const checkout_prog_node = f.prog_node.start("Checkout", 0); defer checkout_prog_node.end(); - var repository = try git.Repository.init(gpa, object_format, pack_file, index_file); + var repository = try git.Repository.init(gpa, object_format, &pack_file_reader, index_file); defer repository.deinit(); var diagnostics: git.Diagnostics = .{ .allocator = arena }; try repository.checkout(out_dir, resource.want_oid, &diagnostics); diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig index 34d373f553..a1e2e6419a 100644 --- a/src/Package/Fetch/git.zig +++ b/src/Package/Fetch/git.zig @@ -73,7 +73,7 @@ pub const Oid = union(Format) { }; } - pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid { + pub fn readBytes(oid_format: Format, reader: anytype) !Oid { return switch (oid_format) { inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())), }; @@ -166,7 +166,7 @@ pub const Diagnostics = struct { pub const Repository = struct { odb: Odb, - pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository { + pub fn init(allocator: Allocator, format: Oid.Format, pack_file: *std.fs.File.Reader, index_file: std.fs.File) !Repository { return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) }; } @@ -335,14 +335,14 @@ pub const Repository = struct { /// [pack-format](https://git-scm.com/docs/pack-format). const Odb = struct { format: Oid.Format, - pack_file: std.fs.File, + pack_file: *std.fs.File.Reader, index_header: IndexHeader, index_file: std.fs.File, cache: ObjectCache = .{}, allocator: Allocator, /// Initializes the database from open pack and index files. - fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb { + fn init(allocator: Allocator, format: Oid.Format, pack_file: *std.fs.File.Reader, index_file: std.fs.File) !Odb { try pack_file.seekTo(0); try index_file.seekTo(0); const index_header = try IndexHeader.read(index_file.deprecatedReader()); @@ -362,14 +362,14 @@ const Odb = struct { /// Reads the object at the current position in the database. fn readObject(odb: *Odb) !Object { - var base_offset = try odb.pack_file.getPos(); + var base_offset = odb.pack_file.logicalPos(); var base_header: EntryHeader = undefined; var delta_offsets: std.ArrayListUnmanaged(u64) = .empty; defer delta_offsets.deinit(odb.allocator); const base_object = while (true) { if (odb.cache.get(base_offset)) |base_object| break base_object; - base_header = try EntryHeader.read(odb.format, odb.pack_file.deprecatedReader()); + base_header = try EntryHeader.read(odb.format, odb.pack_file.interface.adaptToOldInterface()); switch (base_header) { .ofs_delta => |ofs_delta| { try delta_offsets.append(odb.allocator, base_offset); @@ -379,10 +379,10 @@ const Odb = struct { .ref_delta => |ref_delta| { try delta_offsets.append(odb.allocator, base_offset); try odb.seekOid(ref_delta.base_object); - base_offset = try odb.pack_file.getPos(); + base_offset = odb.pack_file.logicalPos(); }, else => { - const base_data = try readObjectRaw(odb.allocator, odb.pack_file.deprecatedReader(), base_header.uncompressedLength()); + const base_data = try readObjectRaw(odb.allocator, &odb.pack_file.interface, base_header.uncompressedLength()); errdefer odb.allocator.free(base_data); const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; try odb.cache.put(odb.allocator, base_offset, base_object); @@ -1227,7 +1227,7 @@ const IndexEntry = struct { /// Writes out a version 2 index for the given packfile, as documented in /// [pack-format](https://git-scm.com/docs/pack-format). -pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void { +pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: *std.fs.File.Reader, index_writer: anytype) !void { try pack.seekTo(0); var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; @@ -1324,12 +1324,11 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in fn indexPackFirstPass( allocator: Allocator, format: Oid.Format, - pack: std.fs.File, + pack: *std.fs.File.Reader, index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), pending_deltas: *std.ArrayListUnmanaged(IndexEntry), ) !Oid { - var pack_buffered_reader = std.io.bufferedReader(pack.deprecatedReader()); - var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader()); + var pack_counting_reader = std.io.countingReader(pack.interface.adaptToOldInterface()); var pack_hashed_reader = hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format)); const pack_reader = pack_hashed_reader.reader(); @@ -1340,15 +1339,19 @@ fn indexPackFirstPass( const entry_offset = pack_counting_reader.bytes_read; var entry_crc32_reader = hashedReader(pack_reader, std.hash.Crc32.init()); const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader()); + var adapter_buffer: [1024]u8 = undefined; + var adapter = entry_crc32_reader.reader().adaptToNewApi(&adapter_buffer); + var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; + var entry_decompress_stream: std.compress.flate.Decompress = .init(&adapter.new_interface, .zlib, &flate_buffer); + const old = entry_decompress_stream.reader.adaptToOldInterface(); + var entry_counting_reader = std.io.countingReader(old); switch (entry_header) { .commit, .tree, .blob, .tag => |object| { - var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); - var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); var entry_hashed_writer = hashedWriter(std.io.null_writer, Oid.Hasher.init(format)); const entry_writer = entry_hashed_writer.writer(); // The object header is not included in the pack data but is // part of the object's ID - try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length }); + try entry_writer.print("{s} {d}\x00", .{ @tagName(entry_header), object.uncompressed_length }); var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); try fifo.pump(entry_counting_reader.reader(), entry_writer); if (entry_counting_reader.bytes_read != object.uncompressed_length) { @@ -1361,8 +1364,6 @@ fn indexPackFirstPass( }); }, inline .ofs_delta, .ref_delta => |delta| { - var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); - var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); if (entry_counting_reader.bytes_read != delta.uncompressed_length) { @@ -1377,7 +1378,7 @@ fn indexPackFirstPass( } const pack_checksum = pack_hashed_reader.hasher.finalResult(); - const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader()); + const recorded_checksum = try Oid.readBytes(format, pack.interface.adaptToOldInterface()); if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) { return error.CorruptedPack; } @@ -1394,7 +1395,7 @@ fn indexPackFirstPass( fn indexPackHashDelta( allocator: Allocator, format: Oid.Format, - pack: std.fs.File, + pack: *std.fs.File.Reader, delta: IndexEntry, index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), cache: *ObjectCache, @@ -1408,7 +1409,7 @@ fn indexPackHashDelta( if (cache.get(base_offset)) |base_object| break base_object; try pack.seekTo(base_offset); - base_header = try EntryHeader.read(format, pack.deprecatedReader()); + base_header = try EntryHeader.read(format, pack.interface.adaptToOldInterface()); switch (base_header) { .ofs_delta => |ofs_delta| { try delta_offsets.append(allocator, base_offset); @@ -1419,7 +1420,7 @@ fn indexPackHashDelta( base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; }, else => { - const base_data = try readObjectRaw(allocator, pack.deprecatedReader(), base_header.uncompressedLength()); + const base_data = try readObjectRaw(allocator, &pack.interface, base_header.uncompressedLength()); errdefer allocator.free(base_data); const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; try cache.put(allocator, base_offset, base_object); @@ -1444,7 +1445,7 @@ fn indexPackHashDelta( fn resolveDeltaChain( allocator: Allocator, format: Oid.Format, - pack: std.fs.File, + pack: *std.fs.File.Reader, base_object: Object, delta_offsets: []const u64, cache: *ObjectCache, @@ -1456,8 +1457,8 @@ fn resolveDeltaChain( const delta_offset = delta_offsets[i]; try pack.seekTo(delta_offset); - const delta_header = try EntryHeader.read(format, pack.deprecatedReader()); - const delta_data = try readObjectRaw(allocator, pack.deprecatedReader(), delta_header.uncompressedLength()); + const delta_header = try EntryHeader.read(format, pack.interface.adaptToOldInterface()); + const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength()); defer allocator.free(delta_data); var delta_stream = std.io.fixedBufferStream(delta_data); const delta_reader = delta_stream.reader(); @@ -1481,18 +1482,14 @@ fn resolveDeltaChain( /// Reads the complete contents of an object from `reader`. This function may /// read more bytes than required from `reader`, so the reader position after /// returning is not reliable. -fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 { +fn readObjectRaw(allocator: Allocator, reader: *std.Io.Reader, size: u64) ![]u8 { const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; - var buffered_reader = std.io.bufferedReader(reader); - var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader()); - const data = try allocator.alloc(u8, alloc_size); - errdefer allocator.free(data); - try decompress_stream.reader().readNoEof(data); - _ = decompress_stream.reader().readByte() catch |e| switch (e) { - error.EndOfStream => return data, - else => |other| return other, - }; - return error.InvalidFormat; + var aw: std.Io.Writer.Allocating = .init(allocator); + try aw.ensureTotalCapacity(alloc_size); + defer aw.deinit(); + var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{}); + try decompress.reader.streamExact(&aw.writer, alloc_size); + return aw.toOwnedSlice(); } /// Expands delta data from `delta_reader` to `writer`. `base_object` must diff --git a/src/link/Elf/Object.zig b/src/link/Elf/Object.zig index 3681f07d66..a330a184be 100644 --- a/src/link/Elf/Object.zig +++ b/src/link/Elf/Object.zig @@ -1198,15 +1198,14 @@ pub fn codeDecompressAlloc(self: *Object, elf_file: *Elf, atom_index: Atom.Index const chdr = @as(*align(1) const elf.Elf64_Chdr, @ptrCast(data.ptr)).*; switch (chdr.ch_type) { .ZLIB => { - var stream = std.io.fixedBufferStream(data[@sizeOf(elf.Elf64_Chdr)..]); - var zlib_stream = std.compress.zlib.decompressor(stream.reader()); + var stream: std.Io.Reader = .fixed(data[@sizeOf(elf.Elf64_Chdr)..]); + var zlib_stream: std.compress.flate.Decompress = .init(&stream, .zlib, &.{}); const size = std.math.cast(usize, chdr.ch_size) orelse return error.Overflow; - const decomp = try gpa.alloc(u8, size); - const nread = zlib_stream.reader().readAll(decomp) catch return error.InputOutput; - if (nread != decomp.len) { - return error.InputOutput; - } - return decomp; + var aw: std.Io.Writer.Allocating = .init(gpa); + try aw.ensureUnusedCapacity(size); + defer aw.deinit(); + _ = try zlib_stream.reader.streamRemaining(&aw.writer); + return aw.toOwnedSlice(); }, else => @panic("TODO unhandled compression scheme"), } From 4741a16d9aab40bea8f1cf604337b3de067accb5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 22:16:52 -0700 Subject: [PATCH 24/47] putting stuff back does not require mutation --- lib/std/compress/flate/Decompress.zig | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 69764245a7..104be1c886 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -500,17 +500,10 @@ fn takeBitsRuntime(d: *Decompress, n: u4) !u16 { fn alignBitsDiscarding(d: *Decompress) void { const remaining_bits = d.remaining_bits; - const next_bits = d.next_bits; if (remaining_bits == 0) return; - const discard_bits = remaining_bits % 8; const n_bytes = remaining_bits / 8; - var put_back_bits = next_bits >> discard_bits; const in = d.input; in.seek -= n_bytes; - for (in.buffer[in.seek..][0..n_bytes]) |*b| { - b.* = @truncate(put_back_bits); - put_back_bits >>= 8; - } d.remaining_bits = 0; d.next_bits = 0; } @@ -521,11 +514,6 @@ fn alignBitsPreserving(d: *Decompress) void { const n_bytes = (remaining_bits + 7) / 8; const in = d.input; in.seek -= n_bytes; - var put_back_bits = d.next_bits; - for (in.buffer[in.seek..][0..n_bytes]) |*b| { - b.* = @truncate(put_back_bits); - put_back_bits >>= 8; - } d.remaining_bits = 0; d.next_bits = 0; } From f3a38e30faae2bfd9066dd187e9b4e9f914c46d6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Jul 2025 22:19:03 -0700 Subject: [PATCH 25/47] std.Io: delete SeekableStream Alternative is to use File.Reader and File.Writer directly. --- lib/std/Io.zig | 3 --- lib/std/Io/fixed_buffer_stream.zig | 17 +-------------- lib/std/Io/seekable_stream.zig | 35 ------------------------------ lib/std/fs/File.zig | 16 -------------- 4 files changed, 1 insertion(+), 70 deletions(-) delete mode 100644 lib/std/Io/seekable_stream.zig diff --git a/lib/std/Io.zig b/lib/std/Io.zig index 971a720f25..c55c28f177 100644 --- a/lib/std/Io.zig +++ b/lib/std/Io.zig @@ -438,8 +438,6 @@ pub fn GenericWriter( pub const AnyReader = @import("Io/DeprecatedReader.zig"); /// Deprecated in favor of `Writer`. pub const AnyWriter = @import("Io/DeprecatedWriter.zig"); -/// Deprecated in favor of `File.Reader` and `File.Writer`. -pub const SeekableStream = @import("Io/seekable_stream.zig").SeekableStream; /// Deprecated in favor of `Writer`. pub const BufferedWriter = @import("Io/buffered_writer.zig").BufferedWriter; /// Deprecated in favor of `Writer`. @@ -948,7 +946,6 @@ test { _ = CountingWriter; _ = CountingReader; _ = FixedBufferStream; - _ = SeekableStream; _ = tty; _ = @import("Io/test.zig"); } diff --git a/lib/std/Io/fixed_buffer_stream.zig b/lib/std/Io/fixed_buffer_stream.zig index 67d6f3d286..c284b9baf4 100644 --- a/lib/std/Io/fixed_buffer_stream.zig +++ b/lib/std/Io/fixed_buffer_stream.zig @@ -4,8 +4,7 @@ const testing = std.testing; const mem = std.mem; const assert = std.debug.assert; -/// This turns a byte buffer into an `io.GenericWriter`, `io.GenericReader`, or `io.SeekableStream`. -/// If the supplied byte buffer is const, then `io.GenericWriter` is not available. +/// Deprecated in favor of `std.Io.Reader.fixed` and `std.Io.Writer.fixed`. pub fn FixedBufferStream(comptime Buffer: type) type { return struct { /// `Buffer` is either a `[]u8` or `[]const u8`. @@ -20,16 +19,6 @@ pub fn FixedBufferStream(comptime Buffer: type) type { pub const Reader = io.GenericReader(*Self, ReadError, read); pub const Writer = io.GenericWriter(*Self, WriteError, write); - pub const SeekableStream = io.SeekableStream( - *Self, - SeekError, - GetSeekPosError, - seekTo, - seekBy, - getPos, - getEndPos, - ); - const Self = @This(); pub fn reader(self: *Self) Reader { @@ -40,10 +29,6 @@ pub fn FixedBufferStream(comptime Buffer: type) type { return .{ .context = self }; } - pub fn seekableStream(self: *Self) SeekableStream { - return .{ .context = self }; - } - pub fn read(self: *Self, dest: []u8) ReadError!usize { const size = @min(dest.len, self.buffer.len - self.pos); const end = self.pos + size; diff --git a/lib/std/Io/seekable_stream.zig b/lib/std/Io/seekable_stream.zig deleted file mode 100644 index 1aa653dbe5..0000000000 --- a/lib/std/Io/seekable_stream.zig +++ /dev/null @@ -1,35 +0,0 @@ -const std = @import("../std.zig"); - -pub fn SeekableStream( - comptime Context: type, - comptime SeekErrorType: type, - comptime GetSeekPosErrorType: type, - comptime seekToFn: fn (context: Context, pos: u64) SeekErrorType!void, - comptime seekByFn: fn (context: Context, pos: i64) SeekErrorType!void, - comptime getPosFn: fn (context: Context) GetSeekPosErrorType!u64, - comptime getEndPosFn: fn (context: Context) GetSeekPosErrorType!u64, -) type { - return struct { - context: Context, - - const Self = @This(); - pub const SeekError = SeekErrorType; - pub const GetSeekPosError = GetSeekPosErrorType; - - pub fn seekTo(self: Self, pos: u64) SeekError!void { - return seekToFn(self.context, pos); - } - - pub fn seekBy(self: Self, amt: i64) SeekError!void { - return seekByFn(self.context, amt); - } - - pub fn getEndPos(self: Self) GetSeekPosError!u64 { - return getEndPosFn(self.context); - } - - pub fn getPos(self: Self) GetSeekPosError!u64 { - return getPosFn(self.context); - } - }; -} diff --git a/lib/std/fs/File.zig b/lib/std/fs/File.zig index eca2d6667f..354d559d79 100644 --- a/lib/std/fs/File.zig +++ b/lib/std/fs/File.zig @@ -1105,22 +1105,6 @@ pub fn deprecatedWriter(file: File) DeprecatedWriter { return .{ .context = file }; } -/// Deprecated in favor of `Reader` and `Writer`. -pub const SeekableStream = io.SeekableStream( - File, - SeekError, - GetSeekPosError, - seekTo, - seekBy, - getPos, - getEndPos, -); - -/// Deprecated in favor of `Reader` and `Writer`. -pub fn seekableStream(file: File) SeekableStream { - return .{ .context = file }; -} - /// Memoizes key information about a file handle such as: /// * The size from calling stat, or the error that occurred therein. /// * The current seek position. From 5f790464b0d5da3c4c1a7252643e7cdd4c4b605e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 29 Jul 2025 11:48:54 -0700 Subject: [PATCH 26/47] std.compress.flate.Decompress: hashing is out of scope This API provides the data; applications can verify their own checksums. --- lib/std/compress/flate/Decompress.zig | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 104be1c886..972cc8f294 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -1096,26 +1096,12 @@ test "gzip header" { 0x00, 0x03, }, error.BadGzipHeader); - // Wrong checksum - try testFailure(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, - }, error.WrongGzipChecksum); - // Truncated checksum try testFailure(.gzip, &[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, }, error.EndOfStream); - // Wrong initial size - try testFailure(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, - }, error.WrongGzipSize); - // Truncated initial size field try testFailure(.gzip, &[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, From 42b10f08ccfa8d4912761371a470cf43bec0ae4e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 29 Jul 2025 16:02:10 -0700 Subject: [PATCH 27/47] std.compress.flate.Decompress: delete bad unit tests if I remove the last input byte from "don't read past deflate stream's end" (on master branch), the test fails with error.EndOfStream. what, then, is it supposed to be testing? --- lib/std/compress/flate/Decompress.zig | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 972cc8f294..57903ba751 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -916,7 +916,7 @@ test "failing invalid-tree01" { try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree01.input"), error.IncompleteHuffmanTree); } test "failing invalid-tree02" { - try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree02.input"), error.IncompleteHuffmanTree); + try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree02.input"), error.EndOfStream); } test "failing invalid-tree03" { try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree03.input"), error.IncompleteHuffmanTree); @@ -949,7 +949,7 @@ test "failing puff10" { try testFailure(.raw, @embedFile("testdata/fuzz/puff10.input"), error.InvalidCode); } test "failing puff11" { - try testFailure(.raw, @embedFile("testdata/fuzz/puff11.input"), error.InvalidMatch); + try testFailure(.raw, @embedFile("testdata/fuzz/puff11.input"), error.EndOfStream); } test "failing puff12" { try testFailure(.raw, @embedFile("testdata/fuzz/puff12.input"), error.InvalidDynamicBlockHeader); @@ -1021,7 +1021,7 @@ test "deflate-stream" { } test "empty-distance-alphabet01" { - try testDecompress(.raw, @embedFile("testdata/fuzz/empty-distance-alphabet01.input"), ""); + try testFailure(.raw, @embedFile("testdata/fuzz/empty-distance-alphabet01.input"), error.EndOfStream); } test "empty-distance-alphabet02" { @@ -1057,18 +1057,6 @@ test "reading into empty buffer" { try testing.expectEqual(0, try r.readVec(&.{&buf})); } -test "don't read past deflate stream's end" { - try testDecompress(.zlib, &[_]u8{ - 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff, - 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01, - 0x83, 0x95, 0x0b, 0xf5, - }, &[_]u8{ - 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, - 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, - }); -} - test "zlib header" { // Truncated header try testFailure(.zlib, &[_]u8{0x78}, error.EndOfStream); @@ -1079,9 +1067,6 @@ test "zlib header" { // Wrong CINFO try testFailure(.zlib, &[_]u8{ 0x88, 0x98 }, error.BadZlibHeader); - // Wrong checksum - try testFailure(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, error.WrongZlibChecksum); - // Truncated checksum try testFailure(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, error.EndOfStream); } From 6bcced31a04afeeead065af961f2571a95e4ad21 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 29 Jul 2025 16:18:52 -0700 Subject: [PATCH 28/47] fix 32-bit compilation --- lib/std/compress/flate/Decompress.zig | 4 ++-- lib/std/zip.zig | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 57903ba751..ffbaf156f6 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -457,7 +457,7 @@ fn specialPeek(in: *Reader, next_bits: usize, i: usize) Reader.Error!u8 { return peeked[i]; } -fn tossBits(d: *Decompress, n: u6) !void { +fn tossBits(d: *Decompress, n: u4) !void { const remaining_bits = d.remaining_bits; const next_bits = d.next_bits; if (remaining_bits >= n) { @@ -475,7 +475,7 @@ fn tossBits(d: *Decompress, n: u6) !void { } } -fn tossBitsEnding(d: *Decompress, n: u6) !void { +fn tossBitsEnding(d: *Decompress, n: u4) !void { const remaining_bits = d.remaining_bits; const in = d.input; const buffered_n = in.bufferedLen(); diff --git a/lib/std/zip.zig b/lib/std/zip.zig index b0da6ac266..ce2a5c43ae 100644 --- a/lib/std/zip.zig +++ b/lib/std/zip.zig @@ -577,7 +577,7 @@ pub const Iterator = struct { switch (self.compression_method) { .store => { - stream.interface.streamExact(&file_writer.interface, self.uncompressed_size) catch |err| switch (err) { + stream.interface.streamExact64(&file_writer.interface, self.uncompressed_size) catch |err| switch (err) { error.ReadFailed => return stream.err.?, error.WriteFailed => return file_writer.err.?, error.EndOfStream => return error.ZipDecompressTruncated, @@ -586,7 +586,7 @@ pub const Iterator = struct { .deflate => { var flate_buffer: [flate.max_window_len]u8 = undefined; var decompress: flate.Decompress = .init(&stream.interface, .raw, &flate_buffer); - decompress.reader.streamExact(&file_writer.interface, self.uncompressed_size) catch |err| switch (err) { + decompress.reader.streamExact64(&file_writer.interface, self.uncompressed_size) catch |err| switch (err) { error.ReadFailed => return stream.err.?, error.WriteFailed => return file_writer.err orelse decompress.err.?, error.EndOfStream => return error.ZipDecompressTruncated, From 3fff84a4a4fb8f9926436313d4e44773b8afc4eb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 29 Jul 2025 16:29:56 -0700 Subject: [PATCH 29/47] compiler: fix unit test compile errors sorry, zip file creation has regressed because std lib no longer has a deflate compression implementation --- lib/std/zip.zig | 4 - lib/std/zip/test.zig | 298 -------------------------------------- src/Package/Fetch.zig | 66 --------- src/Package/Fetch/git.zig | 14 +- 4 files changed, 10 insertions(+), 372 deletions(-) delete mode 100644 lib/std/zip/test.zig diff --git a/lib/std/zip.zig b/lib/std/zip.zig index ce2a5c43ae..e1e8e04a36 100644 --- a/lib/std/zip.zig +++ b/lib/std/zip.zig @@ -660,7 +660,3 @@ pub fn extract(dest: std.fs.Dir, fr: *File.Reader, options: ExtractOptions) !voi } } } - -test { - _ = @import("zip/test.zig"); -} diff --git a/lib/std/zip/test.zig b/lib/std/zip/test.zig deleted file mode 100644 index 27f12cf2a6..0000000000 --- a/lib/std/zip/test.zig +++ /dev/null @@ -1,298 +0,0 @@ -const std = @import("std"); -const testing = std.testing; -const zip = @import("../zip.zig"); -const maxInt = std.math.maxInt; - -pub const File = struct { - name: []const u8, - content: []const u8, - compression: zip.CompressionMethod, -}; - -pub fn expectFiles( - test_files: []const File, - dir: std.fs.Dir, - opt: struct { - strip_prefix: ?[]const u8 = null, - }, -) !void { - for (test_files) |test_file| { - var normalized_sub_path_buf: [std.fs.max_path_bytes]u8 = undefined; - - const name = blk: { - if (opt.strip_prefix) |strip_prefix| { - try testing.expect(test_file.name.len >= strip_prefix.len); - try testing.expectEqualStrings(strip_prefix, test_file.name[0..strip_prefix.len]); - break :blk test_file.name[strip_prefix.len..]; - } - break :blk test_file.name; - }; - const normalized_sub_path = normalized_sub_path_buf[0..name.len]; - @memcpy(normalized_sub_path, name); - std.mem.replaceScalar(u8, normalized_sub_path, '\\', '/'); - var file = try dir.openFile(normalized_sub_path, .{}); - defer file.close(); - var content_buf: [4096]u8 = undefined; - const n = try file.deprecatedReader().readAll(&content_buf); - try testing.expectEqualStrings(test_file.content, content_buf[0..n]); - } -} - -// Used to store any data from writing a file to the zip archive that's needed -// when writing the corresponding central directory record. -pub const FileStore = struct { - compression: zip.CompressionMethod, - file_offset: u64, - crc32: u32, - compressed_size: u32, - uncompressed_size: usize, -}; - -pub fn makeZip( - buf: []u8, - comptime files: []const File, - options: WriteZipOptions, -) !std.io.FixedBufferStream([]u8) { - var store: [files.len]FileStore = undefined; - return try makeZipWithStore(buf, files, options, &store); -} - -pub fn makeZipWithStore( - buf: []u8, - files: []const File, - options: WriteZipOptions, - store: []FileStore, -) !std.io.FixedBufferStream([]u8) { - var fbs = std.io.fixedBufferStream(buf); - try writeZip(fbs.writer(), files, store, options); - return std.io.fixedBufferStream(buf[0..fbs.pos]); -} - -pub const WriteZipOptions = struct { - end: ?EndRecordOptions = null, - local_header: ?LocalHeaderOptions = null, -}; -pub const LocalHeaderOptions = struct { - zip64: ?LocalHeaderZip64Options = null, - compressed_size: ?u32 = null, - uncompressed_size: ?u32 = null, - extra_len: ?u16 = null, -}; -pub const LocalHeaderZip64Options = struct { - data_size: ?u16 = null, -}; -pub const EndRecordOptions = struct { - zip64: ?Zip64Options = null, - sig: ?[4]u8 = null, - disk_number: ?u16 = null, - central_directory_disk_number: ?u16 = null, - record_count_disk: ?u16 = null, - record_count_total: ?u16 = null, - central_directory_size: ?u32 = null, - central_directory_offset: ?u32 = null, - comment_len: ?u16 = null, - comment: ?[]const u8 = null, -}; -pub const Zip64Options = struct { - locator_sig: ?[4]u8 = null, - locator_zip64_disk_count: ?u32 = null, - locator_record_file_offset: ?u64 = null, - locator_total_disk_count: ?u32 = null, - //record_size: ?u64 = null, - central_directory_size: ?u64 = null, -}; - -pub fn writeZip( - writer: anytype, - files: []const File, - store: []FileStore, - options: WriteZipOptions, -) !void { - if (store.len < files.len) return error.FileStoreTooSmall; - var zipper = initZipper(writer); - for (files, 0..) |file, i| { - store[i] = try zipper.writeFile(.{ - .name = file.name, - .content = file.content, - .compression = file.compression, - .write_options = options, - }); - } - for (files, 0..) |file, i| { - try zipper.writeCentralRecord(store[i], .{ - .name = file.name, - }); - } - try zipper.writeEndRecord(if (options.end) |e| e else .{}); -} - -pub fn initZipper(writer: anytype) Zipper(@TypeOf(writer)) { - return .{ .counting_writer = std.io.countingWriter(writer) }; -} - -/// Provides methods to format and write the contents of a zip archive -/// to the underlying Writer. -pub fn Zipper(comptime Writer: type) type { - return struct { - counting_writer: std.io.CountingWriter(Writer), - central_count: u64 = 0, - first_central_offset: ?u64 = null, - last_central_limit: ?u64 = null, - - const Self = @This(); - - pub fn writeFile( - self: *Self, - opt: struct { - name: []const u8, - content: []const u8, - compression: zip.CompressionMethod, - write_options: WriteZipOptions, - }, - ) !FileStore { - const writer = self.counting_writer.writer(); - - const file_offset: u64 = @intCast(self.counting_writer.bytes_written); - const crc32 = std.hash.Crc32.hash(opt.content); - - const header_options = opt.write_options.local_header; - { - var compressed_size: u32 = 0; - var uncompressed_size: u32 = 0; - var extra_len: u16 = 0; - if (header_options) |hdr_options| { - compressed_size = if (hdr_options.compressed_size) |size| size else 0; - uncompressed_size = if (hdr_options.uncompressed_size) |size| size else @intCast(opt.content.len); - extra_len = if (hdr_options.extra_len) |len| len else 0; - } - const hdr: zip.LocalFileHeader = .{ - .signature = zip.local_file_header_sig, - .version_needed_to_extract = 10, - .flags = .{ .encrypted = false, ._ = 0 }, - .compression_method = opt.compression, - .last_modification_time = 0, - .last_modification_date = 0, - .crc32 = crc32, - .compressed_size = compressed_size, - .uncompressed_size = uncompressed_size, - .filename_len = @intCast(opt.name.len), - .extra_len = extra_len, - }; - try writer.writeStructEndian(hdr, .little); - } - try writer.writeAll(opt.name); - - if (header_options) |hdr| { - if (hdr.zip64) |options| { - try writer.writeInt(u16, 0x0001, .little); - const data_size = if (options.data_size) |size| size else 8; - try writer.writeInt(u16, data_size, .little); - try writer.writeInt(u64, 0, .little); - try writer.writeInt(u64, @intCast(opt.content.len), .little); - } - } - - var compressed_size: u32 = undefined; - switch (opt.compression) { - .store => { - try writer.writeAll(opt.content); - compressed_size = @intCast(opt.content.len); - }, - .deflate => { - const offset = self.counting_writer.bytes_written; - var fbs = std.io.fixedBufferStream(opt.content); - try std.compress.flate.deflate.compress(.raw, fbs.reader(), writer, .{}); - std.debug.assert(fbs.pos == opt.content.len); - compressed_size = @intCast(self.counting_writer.bytes_written - offset); - }, - else => unreachable, - } - return .{ - .compression = opt.compression, - .file_offset = file_offset, - .crc32 = crc32, - .compressed_size = compressed_size, - .uncompressed_size = opt.content.len, - }; - } - - pub fn writeCentralRecord( - self: *Self, - store: FileStore, - opt: struct { - name: []const u8, - version_needed_to_extract: u16 = 10, - }, - ) !void { - if (self.first_central_offset == null) { - self.first_central_offset = self.counting_writer.bytes_written; - } - self.central_count += 1; - - const hdr: zip.CentralDirectoryFileHeader = .{ - .signature = zip.central_file_header_sig, - .version_made_by = 0, - .version_needed_to_extract = opt.version_needed_to_extract, - .flags = .{ .encrypted = false, ._ = 0 }, - .compression_method = store.compression, - .last_modification_time = 0, - .last_modification_date = 0, - .crc32 = store.crc32, - .compressed_size = store.compressed_size, - .uncompressed_size = @intCast(store.uncompressed_size), - .filename_len = @intCast(opt.name.len), - .extra_len = 0, - .comment_len = 0, - .disk_number = 0, - .internal_file_attributes = 0, - .external_file_attributes = 0, - .local_file_header_offset = @intCast(store.file_offset), - }; - try self.counting_writer.writer().writeStructEndian(hdr, .little); - try self.counting_writer.writer().writeAll(opt.name); - self.last_central_limit = self.counting_writer.bytes_written; - } - - pub fn writeEndRecord(self: *Self, opt: EndRecordOptions) !void { - const cd_offset = self.first_central_offset orelse 0; - const cd_end = self.last_central_limit orelse 0; - - if (opt.zip64) |zip64| { - const end64_off = cd_end; - const fixed: zip.EndRecord64 = .{ - .signature = zip.end_record64_sig, - .end_record_size = @sizeOf(zip.EndRecord64) - 12, - .version_made_by = 0, - .version_needed_to_extract = 45, - .disk_number = 0, - .central_directory_disk_number = 0, - .record_count_disk = @intCast(self.central_count), - .record_count_total = @intCast(self.central_count), - .central_directory_size = @intCast(cd_end - cd_offset), - .central_directory_offset = @intCast(cd_offset), - }; - try self.counting_writer.writer().writeStructEndian(fixed, .little); - const locator: zip.EndLocator64 = .{ - .signature = if (zip64.locator_sig) |s| s else zip.end_locator64_sig, - .zip64_disk_count = if (zip64.locator_zip64_disk_count) |c| c else 0, - .record_file_offset = if (zip64.locator_record_file_offset) |o| o else @intCast(end64_off), - .total_disk_count = if (zip64.locator_total_disk_count) |c| c else 1, - }; - try self.counting_writer.writer().writeStructEndian(locator, .little); - } - const hdr: zip.EndRecord = .{ - .signature = if (opt.sig) |s| s else zip.end_record_sig, - .disk_number = if (opt.disk_number) |n| n else 0, - .central_directory_disk_number = if (opt.central_directory_disk_number) |n| n else 0, - .record_count_disk = if (opt.record_count_disk) |c| c else @intCast(self.central_count), - .record_count_total = if (opt.record_count_total) |c| c else @intCast(self.central_count), - .central_directory_size = if (opt.central_directory_size) |s| s else @intCast(cd_end - cd_offset), - .central_directory_offset = if (opt.central_directory_offset) |o| o else @intCast(cd_offset), - .comment_len = if (opt.comment_len) |l| l else (if (opt.comment) |c| @as(u16, @intCast(c.len)) else 0), - }; - try self.counting_writer.writer().writeStructEndian(hdr, .little); - if (opt.comment) |c| - try self.counting_writer.writer().writeAll(c); - } - }; -} diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 47dfdce7ac..2fffd33f86 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -2076,72 +2076,6 @@ const UnpackResult = struct { } }; -test "zip" { - const gpa = std.testing.allocator; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const test_files = [_]std.zip.testutil.File{ - .{ .name = "foo", .content = "this is just foo\n", .compression = .store }, - .{ .name = "bar", .content = "another file\n", .compression = .deflate }, - }; - { - var zip_file = try tmp.dir.createFile("test.zip", .{}); - defer zip_file.close(); - var bw = std.io.bufferedWriter(zip_file.deprecatedWriter()); - var store: [test_files.len]std.zip.testutil.FileStore = undefined; - try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{}); - try bw.flush(); - } - - const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path}); - defer gpa.free(zip_path); - - var fb: TestFetchBuilder = undefined; - var fetch = try fb.build(gpa, tmp.dir, zip_path); - defer fb.deinit(); - - try fetch.run(); - - var out = try fb.packageDir(); - defer out.close(); - - try std.zip.testutil.expectFiles(&test_files, out, .{}); -} - -test "zip with one root folder" { - const gpa = std.testing.allocator; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const test_files = [_]std.zip.testutil.File{ - .{ .name = "the_root_folder/foo.zig", .content = "// this is foo.zig\n", .compression = .store }, - .{ .name = "the_root_folder/README.md", .content = "# The foo.zig README\n", .compression = .store }, - }; - { - var zip_file = try tmp.dir.createFile("test.zip", .{}); - defer zip_file.close(); - var bw = std.io.bufferedWriter(zip_file.deprecatedWriter()); - var store: [test_files.len]std.zip.testutil.FileStore = undefined; - try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{}); - try bw.flush(); - } - - const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path}); - defer gpa.free(zip_path); - - var fb: TestFetchBuilder = undefined; - var fetch = try fb.build(gpa, tmp.dir, zip_path); - defer fb.deinit(); - - try fetch.run(); - - var out = try fb.packageDir(); - defer out.close(); - - try std.zip.testutil.expectFiles(&test_files, out, .{ .strip_prefix = "the_root_folder/" }); -} - test "tarball with duplicate paths" { // This tarball has duplicate path 'dir1/file1' to simulate case sensitve // file system on any file sytstem. diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig index a1e2e6419a..f2d59f95e8 100644 --- a/src/Package/Fetch/git.zig +++ b/src/Package/Fetch/git.zig @@ -1564,9 +1564,12 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void defer pack_file.close(); try pack_file.writeAll(testrepo_pack); + var pack_file_buffer: [4096]u8 = undefined; + var pack_file_reader = pack_file.reader(&pack_file_buffer); + var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true }); defer index_file.close(); - try indexPack(testing.allocator, format, pack_file, index_file.deprecatedWriter()); + try indexPack(testing.allocator, format, &pack_file_reader, index_file.deprecatedWriter()); // Arbitrary size limit on files read while checking the repository contents // (all files in the test repo are known to be smaller than this) @@ -1580,7 +1583,7 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx"); try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); - var repository = try Repository.init(testing.allocator, format, pack_file, index_file); + var repository = try Repository.init(testing.allocator, format, &pack_file_reader, index_file); defer repository.deinit(); var worktree = testing.tmpDir(.{ .iterate = true }); @@ -1673,6 +1676,9 @@ pub fn main() !void { var pack_file = try std.fs.cwd().openFile(args[2], .{}); defer pack_file.close(); + var pack_file_buffer: [4096]u8 = undefined; + var pack_file_reader = pack_file.reader(&pack_file_buffer); + const commit = try Oid.parse(format, args[3]); var worktree = try std.fs.cwd().makeOpenPath(args[4], .{}); defer worktree.close(); @@ -1684,11 +1690,11 @@ pub fn main() !void { var index_file = try git_dir.createFile("idx", .{ .read = true }); defer index_file.close(); var index_buffered_writer = std.io.bufferedWriter(index_file.deprecatedWriter()); - try indexPack(allocator, format, pack_file, index_buffered_writer.writer()); + try indexPack(allocator, format, &pack_file_reader, index_buffered_writer.writer()); try index_buffered_writer.flush(); std.debug.print("Starting checkout...\n", .{}); - var repository = try Repository.init(allocator, format, pack_file, index_file); + var repository = try Repository.init(allocator, format, &pack_file_reader, index_file); defer repository.deinit(); var diagnostics: Diagnostics = .{ .allocator = allocator }; defer diagnostics.deinit(); From 81af4f3efc2e40db757426b9c870157d10ebec5a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 29 Jul 2025 17:02:44 -0700 Subject: [PATCH 30/47] link: update some dwarf code to non deprecated API --- src/link/Dwarf.zig | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/link/Dwarf.zig b/src/link/Dwarf.zig index 4262c329fa..07858ae50c 100644 --- a/src/link/Dwarf.zig +++ b/src/link/Dwarf.zig @@ -142,8 +142,8 @@ const DebugInfo = struct { &abbrev_code_buf, debug_info.section.off(dwarf) + unit_ptr.off + unit_ptr.header_len + entry_ptr.off, ) != abbrev_code_buf.len) return error.InputOutput; - var abbrev_code_fbs = std.io.fixedBufferStream(&abbrev_code_buf); - return @enumFromInt(std.leb.readUleb128(@typeInfo(AbbrevCode).@"enum".tag_type, abbrev_code_fbs.reader()) catch unreachable); + var abbrev_code_reader: std.Io.Reader = .fixed(&abbrev_code_buf); + return @enumFromInt(abbrev_code_reader.takeLeb128(@typeInfo(AbbrevCode).@"enum".tag_type) catch unreachable); } const trailer_bytes = 1 + 1; @@ -6021,15 +6021,17 @@ fn sectionOffsetBytes(dwarf: *Dwarf) u32 { } fn uleb128Bytes(value: anytype) u32 { - var cw = std.io.countingWriter(std.io.null_writer); - try uleb128(cw.writer(), value); - return @intCast(cw.bytes_written); + var trash_buffer: [64]u8 = undefined; + var d: std.Io.Writer.Discarding = .init(&trash_buffer); + d.writer.writeUleb128(value) catch unreachable; + return @intCast(d.count + d.writer.end); } fn sleb128Bytes(value: anytype) u32 { - var cw = std.io.countingWriter(std.io.null_writer); - try sleb128(cw.writer(), value); - return @intCast(cw.bytes_written); + var trash_buffer: [64]u8 = undefined; + var d: std.Io.Writer.Discarding = .init(&trash_buffer); + d.writer.writeSleb128(value) catch unreachable; + return @intCast(d.count + d.writer.end); } /// overrides `-fno-incremental` for testing incremental debug info until `-fincremental` is functional From afe9f3a9ec10e1d92fa02ef91168301daac47c67 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 29 Jul 2025 23:28:41 -0700 Subject: [PATCH 31/47] std.compress.flate.Decompress: implement readVec and discard --- lib/std/compress/flate/Decompress.zig | 36 +++++++++++++++++++++------ 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index ffbaf156f6..cd3a07fac6 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -56,11 +56,11 @@ pub const Error = Container.Error || error{ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { return .{ .reader = .{ - // TODO populate discard so that when an amount is discarded that - // includes an entire frame, skip decoding that frame. .vtable = &.{ .stream = stream, .rebase = rebase, + .discard = discard, + .readVec = Reader.indirectReadVec, }, .buffer = buffer, .seek = 0, @@ -81,12 +81,32 @@ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void { assert(capacity <= r.buffer.len - flate.history_len); assert(r.end + capacity > r.buffer.len); - const buffered = r.buffer[0..r.end]; - const discard = buffered.len - flate.history_len; - const keep = buffered[discard..]; + const discard_n = r.end - flate.history_len; + const keep = r.buffer[discard_n..r.end]; @memmove(r.buffer[0..keep.len], keep); r.end = keep.len; - r.seek -= discard; + r.seek -= discard_n; +} + +/// This could be improved so that when an amount is discarded that includes an +/// entire frame, skip decoding that frame. +fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { + r.rebase(flate.history_len) catch unreachable; + var writer: Writer = .{ + .vtable = &.{ + .drain = std.Io.Writer.Discarding.drain, + .sendFile = std.Io.Writer.Discarding.sendFile, + }, + .buffer = r.buffer, + .end = r.end, + }; + const n = r.stream(&writer, limit) catch |err| switch (err) { + error.WriteFailed => unreachable, + error.ReadFailed => return error.ReadFailed, + error.EndOfStream => return error.EndOfStream, + }; + assert(n <= @intFromEnum(limit)); + return n; } fn decodeLength(self: *Decompress, code: u8) !u16 { @@ -268,8 +288,8 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S }, .stored_block => |remaining_len| { const out = try w.writableSliceGreedyPreserve(flate.history_len, 1); - const limited_out = limit.min(.limited(remaining_len)).slice(out); - const n = try d.input.readVec(&.{limited_out}); + var limited_out: [1][]u8 = .{limit.min(.limited(remaining_len)).slice(out)}; + const n = try d.input.readVec(&limited_out); if (remaining_len - n == 0) { d.state = if (d.final_block) .protocol_footer else .block_header; } else { From 84e4343b0c26a11cc667f8df3f9863b65bf1fa36 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 30 Jul 2025 00:25:17 -0700 Subject: [PATCH 32/47] fix test failures by adding readVec --- lib/std/compress/flate/Decompress.zig | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index cd3a07fac6..8a3a3f3f71 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -60,7 +60,7 @@ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { .stream = stream, .rebase = rebase, .discard = discard, - .readVec = Reader.indirectReadVec, + .readVec = readVec, }, .buffer = buffer, .seek = 0, @@ -109,6 +109,22 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { return n; } +fn readVec(r: *Reader, data: []const []u8) Reader.Error!usize { + _ = data; + assert(r.seek == r.end); + r.rebase(flate.history_len) catch unreachable; + var writer: Writer = .{ + .buffer = r.buffer, + .end = r.end, + .vtable = &.{ .drain = Writer.fixedDrain }, + }; + r.end += r.vtable.stream(r, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) { + error.WriteFailed => unreachable, + else => |e| return e, + }; + return 0; +} + fn decodeLength(self: *Decompress, code: u8) !u16 { if (code > 28) return error.InvalidCode; const ml = Token.matchLength(code); @@ -1073,8 +1089,8 @@ test "reading into empty buffer" { var in: Reader = .fixed(input); var decomp: Decompress = .init(&in, .raw, &.{}); const r = &decomp.reader; - var buf: [0]u8 = undefined; - try testing.expectEqual(0, try r.readVec(&.{&buf})); + var bufs: [1][]u8 = .{&.{}}; + try testing.expectEqual(0, try r.readVec(&bufs)); } test "zlib header" { @@ -1135,7 +1151,8 @@ test "zlib should not overshoot" { var reader: std.Io.Reader = .fixed(&data); - var decompress: Decompress = .init(&reader, .zlib, &.{}); + var decompress_buffer: [flate.max_window_len]u8 = undefined; + var decompress: Decompress = .init(&reader, .zlib, &decompress_buffer); var out: [128]u8 = undefined; { From 4c04835a08097d18a30ba74ef9a3aa59f3b90fd3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 30 Jul 2025 00:28:11 -0700 Subject: [PATCH 33/47] std.compress.zstd.Decompress: implement discard and readVec --- lib/std/Io/Reader.zig | 17 ------------- lib/std/compress/zstd/Decompress.zig | 37 ++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index 8dddc49ad2..1cd6a3ebe3 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -438,23 +438,6 @@ pub fn defaultReadVec(r: *Reader, data: []const []u8) Error!usize { return 0; } -/// Always writes to `Reader.buffer` and returns 0. -pub fn indirectReadVec(r: *Reader, data: []const []u8) Error!usize { - _ = data; - assert(r.seek == r.end); - var writer: Writer = .{ - .buffer = r.buffer, - .end = r.end, - .vtable = &.{ .drain = Writer.fixedDrain }, - }; - const limit: Limit = .limited(writer.buffer.len - writer.end); - r.end += r.vtable.stream(r, &writer, limit) catch |err| switch (err) { - error.WriteFailed => unreachable, - else => |e| return e, - }; - return 0; -} - pub fn buffered(r: *Reader) []u8 { return r.buffer[r.seek..r.end]; } diff --git a/lib/std/compress/zstd/Decompress.zig b/lib/std/compress/zstd/Decompress.zig index db85474e00..fefdf11931 100644 --- a/lib/std/compress/zstd/Decompress.zig +++ b/lib/std/compress/zstd/Decompress.zig @@ -89,7 +89,7 @@ pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress { .stream = stream, .rebase = rebase, .discard = discard, - .readVec = Reader.indirectReadVec, + .readVec = readVec, }, .buffer = buffer, .seek = 0, @@ -109,10 +109,20 @@ fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void { r.seek -= discard_n; } -fn discard(r: *Reader, limit: Limit) Reader.Error!usize { - r.rebase(zstd.block_size_max) catch unreachable; - var d: Writer.Discarding = .init(r.buffer); - const n = r.stream(&d.writer, limit) catch |err| switch (err) { +/// This could be improved so that when an amount is discarded that includes an +/// entire frame, skip decoding that frame. +fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { + const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); + r.rebase(d.window_len) catch unreachable; + var writer: Writer = .{ + .vtable = &.{ + .drain = std.Io.Writer.Discarding.drain, + .sendFile = std.Io.Writer.Discarding.sendFile, + }, + .buffer = r.buffer, + .end = r.end, + }; + const n = r.stream(&writer, limit) catch |err| switch (err) { error.WriteFailed => unreachable, error.ReadFailed => return error.ReadFailed, error.EndOfStream => return error.EndOfStream, @@ -121,6 +131,23 @@ fn discard(r: *Reader, limit: Limit) Reader.Error!usize { return n; } +fn readVec(r: *Reader, data: []const []u8) Reader.Error!usize { + _ = data; + const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); + assert(r.seek == r.end); + r.rebase(d.window_len) catch unreachable; + var writer: Writer = .{ + .buffer = r.buffer, + .end = r.end, + .vtable = &.{ .drain = Writer.fixedDrain }, + }; + r.end += r.vtable.stream(r, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) { + error.WriteFailed => unreachable, + else => |e| return e, + }; + return 0; +} + fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize { const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); const in = d.input; From c49c90a42af225f6f125c1bedd0b3cb423de2b4d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 30 Jul 2025 19:02:00 -0700 Subject: [PATCH 34/47] fetch: update API usage --- lib/std/Io/Reader.zig | 49 +++--- lib/std/Io/Writer.zig | 48 ++++++ lib/std/net.zig | 3 +- src/Package/Fetch.zig | 10 +- src/Package/Fetch/git.zig | 320 +++++++++++++++++--------------------- 5 files changed, 225 insertions(+), 205 deletions(-) diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index 1cd6a3ebe3..079651082c 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -74,6 +74,10 @@ pub const VTable = struct { /// /// `data` may not contain an alias to `Reader.buffer`. /// + /// `data` is mutable because the implementation may to temporarily modify + /// the fields in order to handle partial reads. Implementations must + /// restore the original value before returning. + /// /// Implementations may ignore `data`, writing directly to `Reader.buffer`, /// modifying `seek` and `end` accordingly, and returning 0 from this /// function. Implementations are encouraged to take advantage of this if @@ -81,7 +85,7 @@ pub const VTable = struct { /// /// The default implementation calls `stream` with either `data[0]` or /// `Reader.buffer`, whichever is bigger. - readVec: *const fn (r: *Reader, data: []const []u8) Error!usize = defaultReadVec, + readVec: *const fn (r: *Reader, data: [][]u8) Error!usize = defaultReadVec, /// Ensures `capacity` more data can be buffered without rebasing. /// @@ -446,8 +450,8 @@ pub fn bufferedLen(r: *const Reader) usize { return r.end - r.seek; } -pub fn hashed(r: *Reader, hasher: anytype) Hashed(@TypeOf(hasher)) { - return .{ .in = r, .hasher = hasher }; +pub fn hashed(r: *Reader, hasher: anytype, buffer: []u8) Hashed(@TypeOf(hasher)) { + return .init(r, hasher, buffer); } pub fn readVecAll(r: *Reader, data: [][]u8) Error!void { @@ -1764,15 +1768,16 @@ pub fn Hashed(comptime Hasher: type) type { return struct { in: *Reader, hasher: Hasher, - interface: Reader, + reader: Reader, pub fn init(in: *Reader, hasher: Hasher, buffer: []u8) @This() { return .{ .in = in, .hasher = hasher, - .interface = .{ + .reader = .{ .vtable = &.{ - .read = @This().read, + .stream = @This().stream, + .readVec = @This().readVec, .discard = @This().discard, }, .buffer = buffer, @@ -1782,33 +1787,39 @@ pub fn Hashed(comptime Hasher: type) type { }; } - fn read(r: *Reader, w: *Writer, limit: Limit) StreamError!usize { - const this: *@This() = @alignCast(@fieldParentPtr("interface", r)); - const data = w.writableVector(limit); + fn stream(r: *Reader, w: *Writer, limit: Limit) StreamError!usize { + const this: *@This() = @alignCast(@fieldParentPtr("reader", r)); + const data = limit.slice(try w.writableSliceGreedy(1)); + var vec: [1][]u8 = .{data}; + const n = try this.in.readVec(&vec); + this.hasher.update(data[0..n]); + w.advance(n); + return n; + } + + fn readVec(r: *Reader, data: [][]u8) Error!usize { + const this: *@This() = @alignCast(@fieldParentPtr("reader", r)); const n = try this.in.readVec(data); - const result = w.advanceVector(n); var remaining: usize = n; for (data) |slice| { if (remaining < slice.len) { this.hasher.update(slice[0..remaining]); - return result; + return n; } else { remaining -= slice.len; this.hasher.update(slice); } } assert(remaining == 0); - return result; + return n; } fn discard(r: *Reader, limit: Limit) Error!usize { - const this: *@This() = @alignCast(@fieldParentPtr("interface", r)); - var w = this.hasher.writer(&.{}); - const n = this.in.stream(&w, limit) catch |err| switch (err) { - error.WriteFailed => unreachable, - else => |e| return e, - }; - return n; + const this: *@This() = @alignCast(@fieldParentPtr("reader", r)); + const peeked = limit.slice(try this.in.peekGreedy(1)); + this.hasher.update(peeked); + this.in.toss(peeked.len); + return peeked.len; } }; } diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index a177bda8ff..d8c2ec5541 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -2296,6 +2296,8 @@ pub fn fixedDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usiz /// generic. A better solution will involve creating a writer for each hash /// function, where the splat buffer can be tailored to the hash implementation /// details. +/// +/// Contrast with `Hashing` which terminates the stream pipeline. pub fn Hashed(comptime Hasher: type) type { return struct { out: *Writer, @@ -2368,6 +2370,52 @@ pub fn Hashed(comptime Hasher: type) type { }; } +/// Provides a `Writer` implementation based on calling `Hasher.update`, +/// discarding all data. +/// +/// This implementation makes suboptimal buffering decisions due to being +/// generic. A better solution will involve creating a writer for each hash +/// function, where the splat buffer can be tailored to the hash implementation +/// details. +/// +/// The total number of bytes written is stored in `hasher`. +/// +/// Contrast with `Hashed` which also passes the data to an underlying stream. +pub fn Hashing(comptime Hasher: type) type { + return struct { + hasher: Hasher, + writer: Writer, + + pub fn init(buffer: []u8) @This() { + return .initHasher(.init(.{}), buffer); + } + + pub fn initHasher(hasher: Hasher, buffer: []u8) @This() { + return .{ + .hasher = hasher, + .writer = .{ + .buffer = buffer, + .vtable = &.{ .drain = @This().drain }, + }, + }; + } + + fn drain(w: *Writer, data: []const []const u8, splat: usize) Error!usize { + const this: *@This() = @alignCast(@fieldParentPtr("writer", w)); + const hasher = &this.hasher; + hasher.update(w.buffered()); + w.end = 0; + var n: usize = 0; + for (data[0 .. data.len - 1]) |slice| { + hasher.update(slice); + n += slice.len; + } + for (0..splat) |_| hasher.update(data[data.len - 1]); + return n + splat; + } + }; +} + /// Maintains `Writer` state such that it writes to the unused capacity of an /// array list, filling it up completely before making a call through the /// vtable, causing a resize. Consequently, the same, optimized, non-generic diff --git a/lib/std/net.zig b/lib/std/net.zig index d7387662c0..2ffec84f9e 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -1932,7 +1932,8 @@ pub const Stream = struct { fn stream(io_r: *Io.Reader, io_w: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize { const dest = limit.slice(try io_w.writableSliceGreedy(1)); - const n = try readVec(io_r, &.{dest}); + var bufs: [1][]u8 = .{dest}; + const n = try readVec(io_r, &bufs); io_w.advance(n); return n; } diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 2fffd33f86..836435a86b 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1394,18 +1394,20 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!U var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); defer index_file.close(); + var index_file_buffer: [2000]u8 = undefined; + var index_file_writer = index_file.writer(&index_file_buffer); { const index_prog_node = f.prog_node.start("Index pack", 0); defer index_prog_node.end(); - var index_buffered_writer = std.io.bufferedWriter(index_file.deprecatedWriter()); - try git.indexPack(gpa, object_format, &pack_file_reader, index_buffered_writer.writer()); - try index_buffered_writer.flush(); + try git.indexPack(gpa, object_format, &pack_file_reader, &index_file_writer); } { + var index_file_reader = index_file.reader(&index_file_buffer); const checkout_prog_node = f.prog_node.start("Checkout", 0); defer checkout_prog_node.end(); - var repository = try git.Repository.init(gpa, object_format, &pack_file_reader, index_file); + var repository: git.Repository = undefined; + try repository.init(gpa, object_format, &pack_file_reader, &index_file_reader); defer repository.deinit(); var diagnostics: git.Diagnostics = .{ .allocator = arena }; try repository.checkout(out_dir, resource.want_oid, &diagnostics); diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig index f2d59f95e8..00e542b2d2 100644 --- a/src/Package/Fetch/git.zig +++ b/src/Package/Fetch/git.zig @@ -66,6 +66,33 @@ pub const Oid = union(Format) { } }; + const Hashing = union(Format) { + sha1: std.Io.Writer.Hashing(Sha1), + sha256: std.Io.Writer.Hashing(Sha256), + + fn init(oid_format: Format, buffer: []u8) Hashing { + return switch (oid_format) { + .sha1 => .{ .sha1 = .init(buffer) }, + .sha256 => .{ .sha256 = .init(buffer) }, + }; + } + + fn writer(h: *@This()) *std.Io.Writer { + return switch (h.*) { + inline else => |*inner| &inner.writer, + }; + } + + fn final(h: *@This()) Oid { + switch (h.*) { + inline else => |*inner, tag| { + inner.writer.flush() catch unreachable; // hashers cannot fail + return @unionInit(Oid, @tagName(tag), inner.hasher.finalResult()); + }, + } + } + }; + pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { assert(bytes.len == oid_format.byteLength()); return switch (oid_format) { @@ -73,9 +100,9 @@ pub const Oid = union(Format) { }; } - pub fn readBytes(oid_format: Format, reader: anytype) !Oid { + pub fn readBytes(oid_format: Format, reader: *std.Io.Reader) !Oid { return switch (oid_format) { - inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())), + inline else => |tag| @unionInit(Oid, @tagName(tag), (try reader.takeArray(tag.byteLength())).*), }; } @@ -166,8 +193,15 @@ pub const Diagnostics = struct { pub const Repository = struct { odb: Odb, - pub fn init(allocator: Allocator, format: Oid.Format, pack_file: *std.fs.File.Reader, index_file: std.fs.File) !Repository { - return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) }; + pub fn init( + repo: *Repository, + allocator: Allocator, + format: Oid.Format, + pack_file: *std.fs.File.Reader, + index_file: *std.fs.File.Reader, + ) !void { + repo.* = .{ .odb = undefined }; + try repo.odb.init(allocator, format, pack_file, index_file); } pub fn deinit(repository: *Repository) void { @@ -337,22 +371,28 @@ const Odb = struct { format: Oid.Format, pack_file: *std.fs.File.Reader, index_header: IndexHeader, - index_file: std.fs.File, + index_file: *std.fs.File.Reader, cache: ObjectCache = .{}, allocator: Allocator, /// Initializes the database from open pack and index files. - fn init(allocator: Allocator, format: Oid.Format, pack_file: *std.fs.File.Reader, index_file: std.fs.File) !Odb { + fn init( + odb: *Odb, + allocator: Allocator, + format: Oid.Format, + pack_file: *std.fs.File.Reader, + index_file: *std.fs.File.Reader, + ) !void { try pack_file.seekTo(0); try index_file.seekTo(0); - const index_header = try IndexHeader.read(index_file.deprecatedReader()); - return .{ + odb.* = .{ .format = format, .pack_file = pack_file, - .index_header = index_header, + .index_header = undefined, .index_file = index_file, .allocator = allocator, }; + try odb.index_header.read(&index_file.interface); } fn deinit(odb: *Odb) void { @@ -369,7 +409,7 @@ const Odb = struct { const base_object = while (true) { if (odb.cache.get(base_offset)) |base_object| break base_object; - base_header = try EntryHeader.read(odb.format, odb.pack_file.interface.adaptToOldInterface()); + base_header = try EntryHeader.read(odb.format, &odb.pack_file.interface); switch (base_header) { .ofs_delta => |ofs_delta| { try delta_offsets.append(odb.allocator, base_offset); @@ -412,7 +452,7 @@ const Odb = struct { const found_index = while (start_index < end_index) { const mid_index = start_index + (end_index - start_index) / 2; try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); - const mid_oid = try Oid.readBytes(odb.format, odb.index_file.deprecatedReader()); + const mid_oid = try Oid.readBytes(odb.format, &odb.index_file.interface); switch (mem.order(u8, mid_oid.slice(), oid.slice())) { .lt => start_index = mid_index + 1, .gt => end_index = mid_index, @@ -423,12 +463,12 @@ const Odb = struct { const n_objects = odb.index_header.fan_out_table[255]; const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); try odb.index_file.seekTo(offset_values_start + found_index * 4); - const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.deprecatedReader().readInt(u32, .big)); + const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.interface.takeInt(u32, .big)); const pack_offset = pack_offset: { if (l1_offset.big) { const l2_offset_values_start = offset_values_start + n_objects * 4; try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); - break :pack_offset try odb.index_file.deprecatedReader().readInt(u64, .big); + break :pack_offset try odb.index_file.interface.takeInt(u64, .big); } else { break :pack_offset l1_offset.value; } @@ -1080,18 +1120,18 @@ const PackHeader = struct { const signature = "PACK"; const supported_version = 2; - fn read(reader: anytype) !PackHeader { - const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) { + fn read(reader: *std.Io.Reader) !PackHeader { + const actual_signature = reader.take(4) catch |e| switch (e) { error.EndOfStream => return error.InvalidHeader, else => |other| return other, }; - if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader; - const version = reader.readInt(u32, .big) catch |e| switch (e) { + if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader; + const version = reader.takeInt(u32, .big) catch |e| switch (e) { error.EndOfStream => return error.InvalidHeader, else => |other| return other, }; if (version != supported_version) return error.UnsupportedVersion; - const total_objects = reader.readInt(u32, .big) catch |e| switch (e) { + const total_objects = reader.takeInt(u32, .big) catch |e| switch (e) { error.EndOfStream => return error.InvalidHeader, else => |other| return other, }; @@ -1143,13 +1183,13 @@ const EntryHeader = union(Type) { }; } - fn read(format: Oid.Format, reader: anytype) !EntryHeader { + fn read(format: Oid.Format, reader: *std.Io.Reader) !EntryHeader { const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; - const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) { + const initial: InitialByte = @bitCast(reader.takeByte() catch |e| switch (e) { error.EndOfStream => return error.InvalidFormat, else => |other| return other, }); - const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0; + const rest_len = if (initial.has_next) try reader.takeLeb128(u64) else 0; var uncompressed_length: u64 = initial.len; uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat; @@ -1172,25 +1212,12 @@ const EntryHeader = union(Type) { } }; -fn readSizeVarInt(r: anytype) !u64 { +fn readOffsetVarInt(r: *std.Io.Reader) !u64 { const Byte = packed struct { value: u7, has_next: bool }; - var b: Byte = @bitCast(try r.readByte()); - var value: u64 = b.value; - var shift: u6 = 0; - while (b.has_next) { - b = @bitCast(try r.readByte()); - shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat; - value |= @as(u64, b.value) << shift; - } - return value; -} - -fn readOffsetVarInt(r: anytype) !u64 { - const Byte = packed struct { value: u7, has_next: bool }; - var b: Byte = @bitCast(try r.readByte()); + var b: Byte = @bitCast(try r.takeByte()); var value: u64 = b.value; while (b.has_next) { - b = @bitCast(try r.readByte()); + b = @bitCast(try r.takeByte()); value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; value |= b.value; } @@ -1204,19 +1231,12 @@ const IndexHeader = struct { const supported_version = 2; const size = 4 + 4 + @sizeOf([256]u32); - fn read(reader: anytype) !IndexHeader { - var header_bytes = try reader.readBytesNoEof(size); - if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader; - const version = mem.readInt(u32, header_bytes[4..8], .big); + fn read(index_header: *IndexHeader, reader: *std.Io.Reader) !void { + const sig = try reader.take(4); + if (!mem.eql(u8, sig, signature)) return error.InvalidHeader; + const version = try reader.takeInt(u32, .big); if (version != supported_version) return error.UnsupportedVersion; - - var fan_out_table: [256]u32 = undefined; - var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]); - const fan_out_table_reader = fan_out_table_stream.reader(); - for (&fan_out_table) |*entry| { - entry.* = fan_out_table_reader.readInt(u32, .big) catch unreachable; - } - return .{ .fan_out_table = fan_out_table }; + try reader.readSliceEndian(u32, &index_header.fan_out_table, .big); } }; @@ -1227,7 +1247,12 @@ const IndexEntry = struct { /// Writes out a version 2 index for the given packfile, as documented in /// [pack-format](https://git-scm.com/docs/pack-format). -pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: *std.fs.File.Reader, index_writer: anytype) !void { +pub fn indexPack( + allocator: Allocator, + format: Oid.Format, + pack: *std.fs.File.Reader, + index_writer: *std.fs.File.Writer, +) !void { try pack.seekTo(0); var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; @@ -1280,8 +1305,8 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: *std.fs.File.Re } @memset(fan_out_table[fan_out_index..], count); - var index_hashed_writer = hashedWriter(index_writer, Oid.Hasher.init(format)); - const writer = index_hashed_writer.writer(); + var index_hashed_writer = std.Io.Writer.hashed(&index_writer.interface, Oid.Hasher.init(format), &.{}); + const writer = &index_hashed_writer.writer; try writer.writeAll(IndexHeader.signature); try writer.writeInt(u32, IndexHeader.supported_version, .big); for (fan_out_table) |fan_out_entry| { @@ -1314,7 +1339,8 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: *std.fs.File.Re try writer.writeAll(pack_checksum.slice()); const index_checksum = index_hashed_writer.hasher.finalResult(); - try index_writer.writeAll(index_checksum.slice()); + try index_writer.interface.writeAll(index_checksum.slice()); + try index_writer.end(); } /// Performs the first pass over the packfile data for index construction. @@ -1328,65 +1354,51 @@ fn indexPackFirstPass( index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), pending_deltas: *std.ArrayListUnmanaged(IndexEntry), ) !Oid { - var pack_counting_reader = std.io.countingReader(pack.interface.adaptToOldInterface()); - var pack_hashed_reader = hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format)); - const pack_reader = pack_hashed_reader.reader(); + var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; + var entry_buffer: [1024]u8 = undefined; // Input buffer to flate. + var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system. + var hasher_buffer: [64]u8 = undefined; + var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer); - const pack_header = try PackHeader.read(pack_reader); + const pack_header = try PackHeader.read(&pack_hashed.reader); - var current_entry: u32 = 0; - while (current_entry < pack_header.total_objects) : (current_entry += 1) { - const entry_offset = pack_counting_reader.bytes_read; - var entry_crc32_reader = hashedReader(pack_reader, std.hash.Crc32.init()); - const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader()); - var adapter_buffer: [1024]u8 = undefined; - var adapter = entry_crc32_reader.reader().adaptToNewApi(&adapter_buffer); - var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; - var entry_decompress_stream: std.compress.flate.Decompress = .init(&adapter.new_interface, .zlib, &flate_buffer); - const old = entry_decompress_stream.reader.adaptToOldInterface(); - var entry_counting_reader = std.io.countingReader(old); + for (0..pack_header.total_objects) |_| { + const entry_offset = pack.logicalPos(); + var entry_crc32_stream = pack_hashed.reader.hashed(std.hash.Crc32.init(), &entry_buffer); + const entry_header = try EntryHeader.read(format, &entry_crc32_stream.reader); + var entry_decompress: std.compress.flate.Decompress = .init(&entry_crc32_stream.reader, .zlib, &flate_buffer); switch (entry_header) { .commit, .tree, .blob, .tag => |object| { - var entry_hashed_writer = hashedWriter(std.io.null_writer, Oid.Hasher.init(format)); - const entry_writer = entry_hashed_writer.writer(); + var oid_hasher: Oid.Hashing = .init(format, &hasher_buffer); + const oid_hasher_w = oid_hasher.writer(); // The object header is not included in the pack data but is // part of the object's ID - try entry_writer.print("{s} {d}\x00", .{ @tagName(entry_header), object.uncompressed_length }); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(entry_counting_reader.reader(), entry_writer); - if (entry_counting_reader.bytes_read != object.uncompressed_length) { - return error.InvalidObject; - } - const oid = entry_hashed_writer.hasher.finalResult(); + try oid_hasher_w.print("{t} {d}\x00", .{ entry_header, object.uncompressed_length }); + const n = try entry_decompress.reader.streamRemaining(oid_hasher_w); + if (n != object.uncompressed_length) return error.InvalidObject; + const oid = oid_hasher.final(); try index_entries.put(allocator, oid, .{ .offset = entry_offset, - .crc32 = entry_crc32_reader.hasher.final(), + .crc32 = entry_crc32_stream.hasher.final(), }); }, inline .ofs_delta, .ref_delta => |delta| { - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); - if (entry_counting_reader.bytes_read != delta.uncompressed_length) { - return error.InvalidObject; - } + const n = try entry_decompress.reader.discardRemaining(); + if (n != delta.uncompressed_length) return error.InvalidObject; try pending_deltas.append(allocator, .{ .offset = entry_offset, - .crc32 = entry_crc32_reader.hasher.final(), + .crc32 = entry_crc32_stream.hasher.final(), }); }, } } - const pack_checksum = pack_hashed_reader.hasher.finalResult(); - const recorded_checksum = try Oid.readBytes(format, pack.interface.adaptToOldInterface()); + const pack_checksum = pack_hashed.hasher.finalResult(); + const recorded_checksum = try Oid.readBytes(format, &pack.interface); if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) { return error.CorruptedPack; } - _ = pack_reader.readByte() catch |e| switch (e) { - error.EndOfStream => return pack_checksum, - else => |other| return other, - }; - return error.InvalidFormat; + return pack_checksum; } /// Attempts to determine the final object ID of the given deltified object. @@ -1409,7 +1421,7 @@ fn indexPackHashDelta( if (cache.get(base_offset)) |base_object| break base_object; try pack.seekTo(base_offset); - base_header = try EntryHeader.read(format, pack.interface.adaptToOldInterface()); + base_header = try EntryHeader.read(format, &pack.interface); switch (base_header) { .ofs_delta => |ofs_delta| { try delta_offsets.append(allocator, base_offset); @@ -1431,11 +1443,13 @@ fn indexPackHashDelta( const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); - var entry_hasher: Oid.Hasher = .init(format); - var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher); - try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len }); - entry_hasher.update(base_data); - return entry_hasher.finalResult(); + var entry_hasher_buffer: [64]u8 = undefined; + var entry_hasher: Oid.Hashing = .init(format, &entry_hasher_buffer); + const entry_hasher_w = entry_hasher.writer(); + // Writes to hashers cannot fail. + entry_hasher_w.print("{t} {d}\x00", .{ base_object.type, base_data.len }) catch unreachable; + entry_hasher_w.writeAll(base_data) catch unreachable; + return entry_hasher.final(); } /// Resolves a chain of deltas, returning the final base object data. `pack` is @@ -1457,21 +1471,19 @@ fn resolveDeltaChain( const delta_offset = delta_offsets[i]; try pack.seekTo(delta_offset); - const delta_header = try EntryHeader.read(format, pack.interface.adaptToOldInterface()); + const delta_header = try EntryHeader.read(format, &pack.interface); const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength()); defer allocator.free(delta_data); - var delta_stream = std.io.fixedBufferStream(delta_data); - const delta_reader = delta_stream.reader(); - _ = try readSizeVarInt(delta_reader); // base object size - const expanded_size = try readSizeVarInt(delta_reader); + var delta_reader: std.Io.Reader = .fixed(delta_data); + _ = try delta_reader.takeLeb128(u64); // base object size + const expanded_size = try delta_reader.takeLeb128(u64); const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; const expanded_data = try allocator.alloc(u8, expanded_alloc_size); errdefer allocator.free(expanded_data); - var expanded_delta_stream = std.io.fixedBufferStream(expanded_data); - var base_stream = std.io.fixedBufferStream(base_data); - try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer()); - if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject; + var expanded_delta_stream: std.Io.Writer = .fixed(expanded_data); + try expandDelta(base_data, &delta_reader, &expanded_delta_stream); + if (expanded_delta_stream.end != expanded_size) return error.InvalidObject; try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); base_data = expanded_data; @@ -1497,9 +1509,10 @@ fn readObjectRaw(allocator: Allocator, reader: *std.Io.Reader, size: u64) ![]u8 /// /// The format of the delta data is documented in /// [pack-format](https://git-scm.com/docs/pack-format). -fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void { +fn expandDelta(base_object: []const u8, delta_reader: *std.Io.Reader, writer: *std.Io.Writer) !void { + var base_offset: u32 = 0; while (true) { - const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) { + const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch |e| switch (e) { error.EndOfStream => return, else => |other| return other, }); @@ -1514,27 +1527,23 @@ fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !vo size3: bool, } = @bitCast(inst.value); const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ - .offset1 = if (available.offset1) try delta_reader.readByte() else 0, - .offset2 = if (available.offset2) try delta_reader.readByte() else 0, - .offset3 = if (available.offset3) try delta_reader.readByte() else 0, - .offset4 = if (available.offset4) try delta_reader.readByte() else 0, + .offset1 = if (available.offset1) try delta_reader.takeByte() else 0, + .offset2 = if (available.offset2) try delta_reader.takeByte() else 0, + .offset3 = if (available.offset3) try delta_reader.takeByte() else 0, + .offset4 = if (available.offset4) try delta_reader.takeByte() else 0, }; - const offset: u32 = @bitCast(offset_parts); + base_offset = @bitCast(offset_parts); const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ - .size1 = if (available.size1) try delta_reader.readByte() else 0, - .size2 = if (available.size2) try delta_reader.readByte() else 0, - .size3 = if (available.size3) try delta_reader.readByte() else 0, + .size1 = if (available.size1) try delta_reader.takeByte() else 0, + .size2 = if (available.size2) try delta_reader.takeByte() else 0, + .size3 = if (available.size3) try delta_reader.takeByte() else 0, }; var size: u24 = @bitCast(size_parts); if (size == 0) size = 0x10000; - try base_object.seekTo(offset); - var copy_reader = std.io.limitedReader(base_object.reader(), size); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(copy_reader.reader(), writer); + try writer.writeAll(base_object[base_offset..][0..size]); + base_offset += size; } else if (inst.value != 0) { - var data_reader = std.io.limitedReader(delta_reader, inst.value); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(data_reader.reader(), writer); + try delta_reader.streamExact(writer, inst.value); } else { return error.InvalidDeltaInstruction; } @@ -1564,12 +1573,14 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void defer pack_file.close(); try pack_file.writeAll(testrepo_pack); - var pack_file_buffer: [4096]u8 = undefined; + var pack_file_buffer: [2000]u8 = undefined; var pack_file_reader = pack_file.reader(&pack_file_buffer); var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true }); defer index_file.close(); - try indexPack(testing.allocator, format, &pack_file_reader, index_file.deprecatedWriter()); + var index_file_buffer: [2000]u8 = undefined; + var index_file_writer = index_file.writer(&index_file_buffer); + try indexPack(testing.allocator, format, &pack_file_reader, &index_file_writer); // Arbitrary size limit on files read while checking the repository contents // (all files in the test repo are known to be smaller than this) @@ -1583,7 +1594,9 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx"); try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); - var repository = try Repository.init(testing.allocator, format, &pack_file_reader, index_file); + var index_file_reader = index_file.reader(&index_file_buffer); + var repository: Repository = undefined; + try repository.init(testing.allocator, format, &pack_file_reader, &index_file_reader); defer repository.deinit(); var worktree = testing.tmpDir(.{ .iterate = true }); @@ -1704,58 +1717,3 @@ pub fn main() !void { std.debug.print("Diagnostic: {}\n", .{err}); } } - -/// Deprecated -fn hashedReader(reader: anytype, hasher: anytype) HashedReader(@TypeOf(reader), @TypeOf(hasher)) { - return .{ .child_reader = reader, .hasher = hasher }; -} - -/// Deprecated -fn HashedReader(ReaderType: type, HasherType: type) type { - return struct { - child_reader: ReaderType, - hasher: HasherType, - - pub const Error = ReaderType.Error; - pub const Reader = std.io.GenericReader(*@This(), Error, read); - - pub fn read(self: *@This(), buf: []u8) Error!usize { - const amt = try self.child_reader.read(buf); - self.hasher.update(buf[0..amt]); - return amt; - } - - pub fn reader(self: *@This()) Reader { - return .{ .context = self }; - } - }; -} - -/// Deprecated -pub fn HashedWriter(WriterType: type, HasherType: type) type { - return struct { - child_writer: WriterType, - hasher: HasherType, - - pub const Error = WriterType.Error; - pub const Writer = std.io.GenericWriter(*@This(), Error, write); - - pub fn write(self: *@This(), buf: []const u8) Error!usize { - const amt = try self.child_writer.write(buf); - self.hasher.update(buf[0..amt]); - return amt; - } - - pub fn writer(self: *@This()) Writer { - return .{ .context = self }; - } - }; -} - -/// Deprecated -pub fn hashedWriter( - writer: anytype, - hasher: anytype, -) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) { - return .{ .child_writer = writer, .hasher = hasher }; -} From ae67c26cab7b430c254072ba21ee42288d023322 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 30 Jul 2025 21:07:28 -0700 Subject: [PATCH 35/47] disable failing incremental test cases due to dwarf linker logic tracked by #24634 --- src/link/Dwarf.zig | 2 +- test/incremental/change_panic_handler | 1 - test/incremental/change_panic_handler_explicit | 1 - test/incremental/type_becomes_comptime_only | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/link/Dwarf.zig b/src/link/Dwarf.zig index 07858ae50c..f11ebc94d4 100644 --- a/src/link/Dwarf.zig +++ b/src/link/Dwarf.zig @@ -2077,7 +2077,7 @@ pub const WipNav = struct { .generic_decl_const, .generic_decl_func, => true, - else => unreachable, + else => |t| std.debug.panic("bad decl abbrev code: {t}", .{t}), }; if (parent_type.getCaptures(zcu).len == 0) { if (was_generic_decl) try dwarf.freeCommonEntry(wip_nav.unit, decl_gop.value_ptr.*); diff --git a/test/incremental/change_panic_handler b/test/incremental/change_panic_handler index 699134100e..3e400b854e 100644 --- a/test/incremental/change_panic_handler +++ b/test/incremental/change_panic_handler @@ -1,4 +1,3 @@ -#target=x86_64-linux-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe #update=initial version diff --git a/test/incremental/change_panic_handler_explicit b/test/incremental/change_panic_handler_explicit index 2d068d593e..9f7eb6c12f 100644 --- a/test/incremental/change_panic_handler_explicit +++ b/test/incremental/change_panic_handler_explicit @@ -1,4 +1,3 @@ -#target=x86_64-linux-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe #update=initial version diff --git a/test/incremental/type_becomes_comptime_only b/test/incremental/type_becomes_comptime_only index 3bcae1cd21..5c5d332975 100644 --- a/test/incremental/type_becomes_comptime_only +++ b/test/incremental/type_becomes_comptime_only @@ -1,4 +1,3 @@ -#target=x86_64-linux-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe #target=wasm32-wasi-selfhosted From 111305678cc606fa71133fcb8bf517e59b35b772 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 31 Jul 2025 14:58:59 -0700 Subject: [PATCH 36/47] std: match readVec fn prototype exactly this is not necessary according to zig language, but works around a flaw in the C backend --- lib/std/Io/Reader.zig | 4 ++-- lib/std/compress/flate/Decompress.zig | 2 +- lib/std/compress/zstd/Decompress.zig | 2 +- lib/std/fs/File.zig | 2 +- lib/std/net.zig | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index 079651082c..5b4de6445d 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -421,7 +421,7 @@ pub fn readVec(r: *Reader, data: [][]u8) Error!usize { } /// Writes to `Reader.buffer` or `data`, whichever has larger capacity. -pub fn defaultReadVec(r: *Reader, data: []const []u8) Error!usize { +pub fn defaultReadVec(r: *Reader, data: [][]u8) Error!usize { assert(r.seek == r.end); r.seek = 0; r.end = 0; @@ -1665,7 +1665,7 @@ fn endingStream(r: *Reader, w: *Writer, limit: Limit) StreamError!usize { return error.EndOfStream; } -fn endingReadVec(r: *Reader, data: []const []u8) Error!usize { +fn endingReadVec(r: *Reader, data: [][]u8) Error!usize { _ = r; _ = data; return error.EndOfStream; diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 8a3a3f3f71..8faf8ac699 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -109,7 +109,7 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { return n; } -fn readVec(r: *Reader, data: []const []u8) Reader.Error!usize { +fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize { _ = data; assert(r.seek == r.end); r.rebase(flate.history_len) catch unreachable; diff --git a/lib/std/compress/zstd/Decompress.zig b/lib/std/compress/zstd/Decompress.zig index fefdf11931..d2f692ae61 100644 --- a/lib/std/compress/zstd/Decompress.zig +++ b/lib/std/compress/zstd/Decompress.zig @@ -131,7 +131,7 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { return n; } -fn readVec(r: *Reader, data: []const []u8) Reader.Error!usize { +fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize { _ = data; const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); assert(r.seek == r.end); diff --git a/lib/std/fs/File.zig b/lib/std/fs/File.zig index 354d559d79..9776854f18 100644 --- a/lib/std/fs/File.zig +++ b/lib/std/fs/File.zig @@ -1305,7 +1305,7 @@ pub const Reader = struct { } } - fn readVec(io_reader: *std.Io.Reader, data: []const []u8) std.Io.Reader.Error!usize { + fn readVec(io_reader: *std.Io.Reader, data: [][]u8) std.Io.Reader.Error!usize { const r: *Reader = @alignCast(@fieldParentPtr("interface", io_reader)); switch (r.mode) { .positional, .positional_reading => { diff --git a/lib/std/net.zig b/lib/std/net.zig index 2ffec84f9e..0146eab17c 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -1938,7 +1938,7 @@ pub const Stream = struct { return n; } - fn readVec(io_r: *std.Io.Reader, data: []const []u8) Io.Reader.Error!usize { + fn readVec(io_r: *std.Io.Reader, data: [][]u8) Io.Reader.Error!usize { const r: *Reader = @alignCast(@fieldParentPtr("interface_state", io_r)); var iovecs: [max_buffers_len]windows.ws2_32.WSABUF = undefined; const bufs_n, const data_size = try io_r.writableVectorWsa(&iovecs, data); From 165cd87c122b9c0a78377bb9f87b2d5d2b3ae661 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 31 Jul 2025 16:00:58 -0700 Subject: [PATCH 37/47] std.Io.Reader: don't set end to zero because it may be used as a ring buffer --- lib/std/Io/Reader.zig | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index 5b4de6445d..05ab489286 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -266,8 +266,7 @@ pub fn streamRemaining(r: *Reader, w: *Writer) StreamRemainingError!usize { /// number of bytes discarded. pub fn discardRemaining(r: *Reader) ShortError!usize { var offset: usize = r.end - r.seek; - r.seek = 0; - r.end = 0; + r.seek = r.end; while (true) { offset += r.vtable.discard(r, .unlimited) catch |err| switch (err) { error.EndOfStream => return offset, @@ -526,8 +525,7 @@ pub fn toss(r: *Reader, n: usize) void { /// Equivalent to `toss(r.bufferedLen())`. pub fn tossBuffered(r: *Reader) void { - r.seek = 0; - r.end = 0; + r.seek = r.end; } /// Equivalent to `peek` followed by `toss`. @@ -614,8 +612,7 @@ pub fn discardShort(r: *Reader, n: usize) ShortError!usize { return n; } var remaining = n - (r.end - r.seek); - r.end = 0; - r.seek = 0; + r.seek = r.end; while (true) { const discard_len = r.vtable.discard(r, .limited(remaining)) catch |err| switch (err) { error.EndOfStream => return n - remaining, From c9ff068391b18b0c6fe90a46c39e6e2956512660 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 31 Jul 2025 16:02:06 -0700 Subject: [PATCH 38/47] std.compress: fix discard impl and flate error detection --- lib/std/compress/flate/Decompress.zig | 6 +++++- lib/std/compress/zstd/Decompress.zig | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 8faf8ac699..8b16b3b353 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -100,6 +100,10 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { .buffer = r.buffer, .end = r.end, }; + defer { + r.end = writer.end; + r.seek = r.end; + } const n = r.stream(&writer, limit) catch |err| switch (err) { error.WriteFailed => unreachable, error.ReadFailed => return error.ReadFailed, @@ -397,7 +401,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.S /// Write match (back-reference to the same data slice) starting at `distance` /// back from current write position, and `length` of bytes. fn writeMatch(w: *Writer, length: u16, distance: u16) !void { - if (w.end < length) return error.InvalidMatch; + if (w.end < distance) return error.InvalidMatch; if (length < Token.base_length) return error.InvalidMatch; if (length > Token.max_length) return error.InvalidMatch; if (distance < Token.min_distance) return error.InvalidMatch; diff --git a/lib/std/compress/zstd/Decompress.zig b/lib/std/compress/zstd/Decompress.zig index d2f692ae61..bcb8e3b0da 100644 --- a/lib/std/compress/zstd/Decompress.zig +++ b/lib/std/compress/zstd/Decompress.zig @@ -122,6 +122,10 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { .buffer = r.buffer, .end = r.end, }; + defer { + r.end = writer.end; + r.seek = r.end; + } const n = r.stream(&writer, limit) catch |err| switch (err) { error.WriteFailed => unreachable, error.ReadFailed => return error.ReadFailed, From 95273337c507917e28a6be7dbd89a5c13fea9401 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 31 Jul 2025 16:04:53 -0700 Subject: [PATCH 39/47] fetch: remove checksum logic and fix new I/O API bugs Thanks Ian Johnson for finding these --- src/Package/Fetch/git.zig | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig index 00e542b2d2..c140335efd 100644 --- a/src/Package/Fetch/git.zig +++ b/src/Package/Fetch/git.zig @@ -1355,21 +1355,18 @@ fn indexPackFirstPass( pending_deltas: *std.ArrayListUnmanaged(IndexEntry), ) !Oid { var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; - var entry_buffer: [1024]u8 = undefined; // Input buffer to flate. var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system. - var hasher_buffer: [64]u8 = undefined; var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer); const pack_header = try PackHeader.read(&pack_hashed.reader); for (0..pack_header.total_objects) |_| { - const entry_offset = pack.logicalPos(); - var entry_crc32_stream = pack_hashed.reader.hashed(std.hash.Crc32.init(), &entry_buffer); - const entry_header = try EntryHeader.read(format, &entry_crc32_stream.reader); - var entry_decompress: std.compress.flate.Decompress = .init(&entry_crc32_stream.reader, .zlib, &flate_buffer); + const entry_offset = pack.logicalPos() - pack_hashed.reader.bufferedLen(); + const entry_header = try EntryHeader.read(format, &pack_hashed.reader); switch (entry_header) { .commit, .tree, .blob, .tag => |object| { - var oid_hasher: Oid.Hashing = .init(format, &hasher_buffer); + var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &.{}); + var oid_hasher: Oid.Hashing = .init(format, &flate_buffer); const oid_hasher_w = oid_hasher.writer(); // The object header is not included in the pack data but is // part of the object's ID @@ -1377,28 +1374,27 @@ fn indexPackFirstPass( const n = try entry_decompress.reader.streamRemaining(oid_hasher_w); if (n != object.uncompressed_length) return error.InvalidObject; const oid = oid_hasher.final(); + if (!skip_checksums) @compileError("TODO"); try index_entries.put(allocator, oid, .{ .offset = entry_offset, - .crc32 = entry_crc32_stream.hasher.final(), + .crc32 = 0, }); }, inline .ofs_delta, .ref_delta => |delta| { + var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &flate_buffer); const n = try entry_decompress.reader.discardRemaining(); if (n != delta.uncompressed_length) return error.InvalidObject; + if (!skip_checksums) @compileError("TODO"); try pending_deltas.append(allocator, .{ .offset = entry_offset, - .crc32 = entry_crc32_stream.hasher.final(), + .crc32 = 0, }); }, } } - const pack_checksum = pack_hashed.hasher.finalResult(); - const recorded_checksum = try Oid.readBytes(format, &pack.interface); - if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) { - return error.CorruptedPack; - } - return pack_checksum; + if (!skip_checksums) @compileError("TODO"); + return pack_hashed.hasher.finalResult(); } /// Attempts to determine the final object ID of the given deltified object. @@ -1497,7 +1493,7 @@ fn resolveDeltaChain( fn readObjectRaw(allocator: Allocator, reader: *std.Io.Reader, size: u64) ![]u8 { const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; var aw: std.Io.Writer.Allocating = .init(allocator); - try aw.ensureTotalCapacity(alloc_size); + try aw.ensureTotalCapacity(alloc_size + std.compress.flate.max_window_len); defer aw.deinit(); var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{}); try decompress.reader.streamExact(&aw.writer, alloc_size); @@ -1666,11 +1662,19 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void try testing.expectEqualStrings(expected_file_contents, actual_file_contents); } +/// Checksum calculation is useful for troubleshooting and debugging, but it's +/// redundant since the package manager already does content hashing at the +/// end. Let's save time by not doing that work, but, I left a cookie crumb +/// trail here if you want to restore the functionality for tinkering purposes. +const skip_checksums = true; + test "SHA-1 packfile indexing and checkout" { + if (skip_checksums) return error.SkipZigTest; try runRepositoryTest(.sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb"); } test "SHA-256 packfile indexing and checkout" { + if (skip_checksums) return error.SkipZigTest; try runRepositoryTest(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a"); } From 2024abda6ae7c02c7fc1667d221e98da23ebcd2a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 31 Jul 2025 16:21:28 -0700 Subject: [PATCH 40/47] std.debug.Dwarf: work around API deficiency need to supply a big enough buffer when working with decompression --- lib/std/debug/Dwarf.zig | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/std/debug/Dwarf.zig b/lib/std/debug/Dwarf.zig index 5fa5dd002a..6237d06cda 100644 --- a/lib/std/debug/Dwarf.zig +++ b/lib/std/debug/Dwarf.zig @@ -2019,10 +2019,14 @@ pub fn compactUnwindToDwarfRegNumber(unwind_reg_number: u3) !u8 { /// This function is to make it handy to comment out the return and make it /// into a crash when working on this file. pub fn bad() error{InvalidDebugInfo} { - if (debug_debug_mode) @panic("bad dwarf"); + invalidDebugInfoDetected(); return error.InvalidDebugInfo; } +fn invalidDebugInfoDetected() void { + if (debug_debug_mode) @panic("bad dwarf"); +} + fn missing() error{MissingDebugInfo} { if (debug_debug_mode) @panic("missing dwarf"); return error.MissingDebugInfo; @@ -2239,13 +2243,19 @@ pub const ElfModule = struct { const chdr = section_reader.takeStruct(elf.Chdr, endian) catch continue; if (chdr.ch_type != .ZLIB) continue; - var zlib_stream: std.compress.flate.Decompress = .init(§ion_reader, .zlib, &.{}); - const decompressed_section = zlib_stream.reader.allocRemaining(gpa, .unlimited) catch continue; - errdefer gpa.free(decompressed_section); - assert(chdr.ch_size == decompressed_section.len); - + var decompress: std.compress.flate.Decompress = .init(§ion_reader, .zlib, &.{}); + var decompressed_section: std.ArrayListUnmanaged(u8) = .empty; + defer decompressed_section.deinit(gpa); + decompress.reader.appendRemainingUnlimited(gpa, null, &decompressed_section, std.compress.flate.history_len) catch { + invalidDebugInfoDetected(); + continue; + }; + if (chdr.ch_size != decompressed_section.items.len) { + invalidDebugInfoDetected(); + continue; + } break :blk .{ - .data = decompressed_section, + .data = try decompressed_section.toOwnedSlice(gpa), .virtual_address = shdr.sh_addr, .owned = true, }; From 6eac56caf715bdc2dbd63b628ca48c0e32d5a70c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 31 Jul 2025 17:31:54 -0700 Subject: [PATCH 41/47] std.compress.flate.Decompress: allow users to swap out Writer --- lib/std/compress/flate/Decompress.zig | 33 +++++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 8b16b3b353..05a2354f09 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -58,7 +58,7 @@ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { .reader = .{ .vtable = &.{ .stream = stream, - .rebase = rebase, + .rebase = rebaseFallible, .discard = discard, .readVec = readVec, }, @@ -78,12 +78,19 @@ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { }; } -fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void { +fn rebaseFallible(r: *Reader, capacity: usize) Reader.RebaseError!void { + const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); + rebase(d, capacity); +} + +fn rebase(d: *Decompress, capacity: usize) void { + const r = &d.reader; assert(capacity <= r.buffer.len - flate.history_len); assert(r.end + capacity > r.buffer.len); const discard_n = r.end - flate.history_len; const keep = r.buffer[discard_n..r.end]; @memmove(r.buffer[0..keep.len], keep); + assert(keep.len != 0); r.end = keep.len; r.seek -= discard_n; } @@ -101,6 +108,7 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { .end = r.end, }; defer { + assert(writer.end != 0); r.end = writer.end; r.seek = r.end; } @@ -115,14 +123,20 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize { _ = data; - assert(r.seek == r.end); - r.rebase(flate.history_len) catch unreachable; + const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); + return streamIndirect(d); +} + +fn streamIndirect(d: *Decompress) Reader.Error!usize { + const r = &d.reader; + if (r.end + flate.history_len > r.buffer.len) rebase(d, flate.history_len); var writer: Writer = .{ .buffer = r.buffer, .end = r.end, .vtable = &.{ .drain = Writer.fixedDrain }, }; - r.end += r.vtable.stream(r, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) { + defer r.end = writer.end; + _ = streamFallible(d, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) { error.WriteFailed => unreachable, else => |e| return e, }; @@ -188,7 +202,12 @@ fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol { pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize { const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); - return readInner(d, w, limit) catch |err| switch (err) { + if (w.end >= r.end) return streamFallible(d, w, limit); + return streamIndirect(d); +} + +fn streamFallible(d: *Decompress, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize { + return streamInner(d, w, limit) catch |err| switch (err) { error.EndOfStream => { if (d.state == .end) { return error.EndOfStream; @@ -207,7 +226,7 @@ pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!us }; } -fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.StreamError)!usize { +fn streamInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.StreamError)!usize { var remaining = @intFromEnum(limit); const in = d.input; sw: switch (d.state) { From a7808892f71d0209a107e8d5e15405ce30f67a1e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 31 Jul 2025 18:05:22 -0700 Subject: [PATCH 42/47] std.compress.flate.Decompress: be in indirect or direct mode depending on whether buffered --- lib/std/compress/flate/Decompress.zig | 47 +++++++++++++++++---------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 05a2354f09..2060f52c02 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -53,15 +53,24 @@ pub const Error = Container.Error || error{ EndOfStream, }; +const direct_vtable: Reader.VTable = .{ + .stream = streamDirect, + .rebase = rebaseFallible, + .discard = discard, + .readVec = readVec, +}; + +const indirect_vtable: Reader.VTable = .{ + .stream = streamIndirect, + .rebase = rebaseFallible, + .discard = discard, + .readVec = readVec, +}; + pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { return .{ .reader = .{ - .vtable = &.{ - .stream = stream, - .rebase = rebaseFallible, - .discard = discard, - .readVec = readVec, - }, + .vtable = if (buffer.len == 0) &direct_vtable else &indirect_vtable, .buffer = buffer, .seek = 0, .end = 0, @@ -79,12 +88,10 @@ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress { } fn rebaseFallible(r: *Reader, capacity: usize) Reader.RebaseError!void { - const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); - rebase(d, capacity); + rebase(r, capacity); } -fn rebase(d: *Decompress, capacity: usize) void { - const r = &d.reader; +fn rebase(r: *Reader, capacity: usize) void { assert(capacity <= r.buffer.len - flate.history_len); assert(r.end + capacity > r.buffer.len); const discard_n = r.end - flate.history_len; @@ -98,7 +105,7 @@ fn rebase(d: *Decompress, capacity: usize) void { /// This could be improved so that when an amount is discarded that includes an /// entire frame, skip decoding that frame. fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { - r.rebase(flate.history_len) catch unreachable; + if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len); var writer: Writer = .{ .vtable = &.{ .drain = std.Io.Writer.Discarding.drain, @@ -124,12 +131,12 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize { _ = data; const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); - return streamIndirect(d); + return streamIndirectInner(d); } -fn streamIndirect(d: *Decompress) Reader.Error!usize { +fn streamIndirectInner(d: *Decompress) Reader.Error!usize { const r = &d.reader; - if (r.end + flate.history_len > r.buffer.len) rebase(d, flate.history_len); + if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len); var writer: Writer = .{ .buffer = r.buffer, .end = r.end, @@ -200,10 +207,16 @@ fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol { return sym; } -pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize { +fn streamDirect(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize { const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); - if (w.end >= r.end) return streamFallible(d, w, limit); - return streamIndirect(d); + return streamFallible(d, w, limit); +} + +fn streamIndirect(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize { + const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); + _ = limit; + _ = w; + return streamIndirectInner(d); } fn streamFallible(d: *Decompress, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize { From 6caa100f0d32ecb1bff9405748743f06e60921a2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 31 Jul 2025 18:16:13 -0700 Subject: [PATCH 43/47] std.Io.Writer: fix wrong return value from fixedDrain --- lib/std/Io/Writer.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index d8c2ec5541..1b0e4ad6f3 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -2266,7 +2266,7 @@ pub fn fixedDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usiz const pattern = data[data.len - 1]; const dest = w.buffer[w.end..]; switch (pattern.len) { - 0 => return w.end, + 0 => return 0, 1 => { assert(splat >= dest.len); @memset(dest, pattern[0]); From 64814dc986a6ea4ab06ad523c48b2d3302903620 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 31 Jul 2025 19:24:23 -0700 Subject: [PATCH 44/47] std.compress.flate.Decompress: respect stream limit --- lib/std/Io/Writer.zig | 7 +++ lib/std/compress/flate/Decompress.zig | 89 +++++++++++++++++++-------- 2 files changed, 72 insertions(+), 24 deletions(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 1b0e4ad6f3..2c787bafe7 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -2286,6 +2286,13 @@ pub fn fixedDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usiz } } +pub fn unreachableDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usize { + _ = w; + _ = data; + _ = splat; + unreachable; +} + /// Provides a `Writer` implementation based on calling `Hasher.update`, sending /// all data also to an underlying `Writer`. /// diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index 2060f52c02..a51fd05610 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -37,6 +37,8 @@ const State = union(enum) { stored_block: u16, fixed_block, dynamic_block, + dynamic_block_literal: u8, + dynamic_block_match: u16, protocol_footer, end, }; @@ -63,7 +65,7 @@ const direct_vtable: Reader.VTable = .{ const indirect_vtable: Reader.VTable = .{ .stream = streamIndirect, .rebase = rebaseFallible, - .discard = discard, + .discard = discardIndirect, .readVec = readVec, }; @@ -128,6 +130,26 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { return n; } +fn discardIndirect(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { + const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); + if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len); + var writer: Writer = .{ + .buffer = r.buffer, + .end = r.end, + .vtable = &.{ .drain = Writer.unreachableDrain }, + }; + { + defer r.end = writer.end; + _ = streamFallible(d, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) { + error.WriteFailed => unreachable, + else => |e| return e, + }; + } + const n = limit.minInt(r.end - r.seek); + r.seek += n; + return n; +} + fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize { _ = data; const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); @@ -140,7 +162,7 @@ fn streamIndirectInner(d: *Decompress) Reader.Error!usize { var writer: Writer = .{ .buffer = r.buffer, .end = r.end, - .vtable = &.{ .drain = Writer.fixedDrain }, + .vtable = &.{ .drain = Writer.unreachableDrain }, }; defer r.end = writer.end; _ = streamFallible(d, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) { @@ -379,30 +401,49 @@ fn streamInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader .dynamic_block => { // In larger archives most blocks are usually dynamic, so // decompression performance depends on this logic. - while (remaining > 0) { - const sym = try d.decodeSymbol(&d.lit_dec); - - switch (sym.kind) { - .literal => { - try w.writeBytePreserve(flate.history_len, sym.symbol); + var sym = try d.decodeSymbol(&d.lit_dec); + sym: switch (sym.kind) { + .literal => { + if (remaining != 0) { + @branchHint(.likely); remaining -= 1; - }, - .match => { - // Decode match backreference - const length = try d.decodeLength(sym.symbol); - const dsm = try d.decodeSymbol(&d.dst_dec); - const distance = try d.decodeDistance(dsm.symbol); - try writeMatch(w, length, distance); - remaining -= length; - }, - .end_of_block => { - d.state = if (d.final_block) .protocol_footer else .block_header; + try w.writeBytePreserve(flate.history_len, sym.symbol); + sym = try d.decodeSymbol(&d.lit_dec); + continue :sym sym.kind; + } else { + d.state = .{ .dynamic_block_literal = sym.symbol }; return @intFromEnum(limit) - remaining; - }, - } + } + }, + .match => { + // Decode match backreference + const length = try d.decodeLength(sym.symbol); + continue :sw .{ .dynamic_block_match = length }; + }, + .end_of_block => { + d.state = if (d.final_block) .protocol_footer else .block_header; + continue :sw d.state; + }, + } + }, + .dynamic_block_literal => |symbol| { + assert(remaining != 0); + remaining -= 1; + try w.writeBytePreserve(flate.history_len, symbol); + continue :sw .dynamic_block; + }, + .dynamic_block_match => |length| { + if (remaining >= length) { + @branchHint(.likely); + remaining -= length; + const dsm = try d.decodeSymbol(&d.dst_dec); + const distance = try d.decodeDistance(dsm.symbol); + try writeMatch(w, length, distance); + continue :sw .dynamic_block; + } else { + d.state = .{ .dynamic_block_match = length }; + return @intFromEnum(limit) - remaining; } - d.state = .dynamic_block; - return @intFromEnum(limit) - remaining; }, .protocol_footer => { switch (d.container_metadata) { @@ -424,7 +465,7 @@ fn streamInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader }, } d.state = .end; - return 0; + return @intFromEnum(limit) - remaining; }, .end => return error.EndOfStream, } From d91744401fd83fe2d603b5020fc141285aa017a1 Mon Sep 17 00:00:00 2001 From: Ian Johnson Date: Thu, 31 Jul 2025 21:56:08 -0400 Subject: [PATCH 45/47] fetch: More Git fixes --- lib/std/Io/Writer.zig | 2 +- src/Package/Fetch/git.zig | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 2c787bafe7..ef487958e5 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -2418,7 +2418,7 @@ pub fn Hashing(comptime Hasher: type) type { n += slice.len; } for (0..splat) |_| hasher.update(data[data.len - 1]); - return n + splat; + return n + splat * data[data.len - 1].len; } }; } diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig index c140335efd..8f5bff2522 100644 --- a/src/Package/Fetch/git.zig +++ b/src/Package/Fetch/git.zig @@ -1500,13 +1500,11 @@ fn readObjectRaw(allocator: Allocator, reader: *std.Io.Reader, size: u64) ![]u8 return aw.toOwnedSlice(); } -/// Expands delta data from `delta_reader` to `writer`. `base_object` must -/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`). +/// Expands delta data from `delta_reader` to `writer`. /// /// The format of the delta data is documented in /// [pack-format](https://git-scm.com/docs/pack-format). fn expandDelta(base_object: []const u8, delta_reader: *std.Io.Reader, writer: *std.Io.Writer) !void { - var base_offset: u32 = 0; while (true) { const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch |e| switch (e) { error.EndOfStream => return, @@ -1528,7 +1526,7 @@ fn expandDelta(base_object: []const u8, delta_reader: *std.Io.Reader, writer: *s .offset3 = if (available.offset3) try delta_reader.takeByte() else 0, .offset4 = if (available.offset4) try delta_reader.takeByte() else 0, }; - base_offset = @bitCast(offset_parts); + const base_offset: u32 = @bitCast(offset_parts); const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ .size1 = if (available.size1) try delta_reader.takeByte() else 0, .size2 = if (available.size2) try delta_reader.takeByte() else 0, @@ -1537,7 +1535,6 @@ fn expandDelta(base_object: []const u8, delta_reader: *std.Io.Reader, writer: *s var size: u24 = @bitCast(size_parts); if (size == 0) size = 0x10000; try writer.writeAll(base_object[base_offset..][0..size]); - base_offset += size; } else if (inst.value != 0) { try delta_reader.streamExact(writer, inst.value); } else { @@ -1582,13 +1579,15 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void // (all files in the test repo are known to be smaller than this) const max_file_size = 8192; - const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size); - defer testing.allocator.free(index_file_data); - // testrepo.idx is generated by Git. The index created by this file should - // match it exactly. Running `git verify-pack -v testrepo.pack` can verify - // this. - const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx"); - try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); + if (!skip_checksums) { + const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size); + defer testing.allocator.free(index_file_data); + // testrepo.idx is generated by Git. The index created by this file should + // match it exactly. Running `git verify-pack -v testrepo.pack` can verify + // this. + const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx"); + try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); + } var index_file_reader = index_file.reader(&index_file_buffer); var repository: Repository = undefined; @@ -1669,12 +1668,10 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void const skip_checksums = true; test "SHA-1 packfile indexing and checkout" { - if (skip_checksums) return error.SkipZigTest; try runRepositoryTest(.sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb"); } test "SHA-256 packfile indexing and checkout" { - if (skip_checksums) return error.SkipZigTest; try runRepositoryTest(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a"); } From eb17d4562a6515baa7128eb4cd3ffdab3e6c4835 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 31 Jul 2025 22:36:52 -0700 Subject: [PATCH 46/47] std.Io.Writer.Hashed: fix bad assert --- lib/std/Io/Writer.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index ef487958e5..a84077f8f3 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -2350,7 +2350,7 @@ pub fn Hashed(comptime Hasher: type) type { this.hasher.update(slice); } const pattern = data[data.len - 1]; - assert(remaining == splat * pattern.len); + assert(remaining <= splat * pattern.len); switch (pattern.len) { 0 => { assert(remaining == 0); From a6f7927764fde807cb077fb4385be0729035e1db Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 1 Aug 2025 09:04:27 -0700 Subject: [PATCH 47/47] std.compress.flate.Decompress: use 64 buffered bits will have to find out why usize doesn't work for 32 bit targets some other time --- lib/std/compress/flate/Decompress.zig | 39 ++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index a51fd05610..90f89ea6c9 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -10,8 +10,8 @@ const Decompress = @This(); const Token = @import("Token.zig"); input: *Reader, -next_bits: usize, -remaining_bits: std.math.Log2Int(usize), +next_bits: Bits, +remaining_bits: std.math.Log2Int(Bits), reader: Reader, @@ -25,6 +25,9 @@ state: State, err: ?Error, +/// TODO: change this to usize +const Bits = u64; + const BlockType = enum(u2) { stored = 0, fixed = 1, @@ -498,14 +501,14 @@ fn takeBits(d: *Decompress, comptime U: type) !U { return u; } const in = d.input; - const next_int = in.takeInt(usize, .little) catch |err| switch (err) { + const next_int = in.takeInt(Bits, .little) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, error.EndOfStream => return takeBitsEnding(d, U), }; const needed_bits = @bitSizeOf(U) - remaining_bits; - const u: U = @intCast(((next_int & ((@as(usize, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); + const u: U = @intCast(((next_int & ((@as(Bits, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); d.next_bits = next_int >> needed_bits; - d.remaining_bits = @intCast(@bitSizeOf(usize) - @as(usize, needed_bits)); + d.remaining_bits = @intCast(@bitSizeOf(Bits) - @as(usize, needed_bits)); return u; } @@ -514,14 +517,14 @@ fn takeBitsEnding(d: *Decompress, comptime U: type) !U { const next_bits = d.next_bits; const in = d.input; const n = in.bufferedLen(); - assert(n < @sizeOf(usize)); + assert(n < @sizeOf(Bits)); const needed_bits = @bitSizeOf(U) - remaining_bits; if (n * 8 < needed_bits) return error.EndOfStream; - const next_int = in.takeVarInt(usize, .little, n) catch |err| switch (err) { + const next_int = in.takeVarInt(Bits, .little, n) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, error.EndOfStream => unreachable, }; - const u: U = @intCast(((next_int & ((@as(usize, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); + const u: U = @intCast(((next_int & ((@as(Bits, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); d.next_bits = next_int >> needed_bits; d.remaining_bits = @intCast(n * 8 - @as(usize, needed_bits)); return u; @@ -532,37 +535,37 @@ fn peekBits(d: *Decompress, comptime U: type) !U { const next_bits = d.next_bits; if (remaining_bits >= @bitSizeOf(U)) return @truncate(next_bits); const in = d.input; - const next_int = in.peekInt(usize, .little) catch |err| switch (err) { + const next_int = in.peekInt(Bits, .little) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, error.EndOfStream => return peekBitsEnding(d, U), }; const needed_bits = @bitSizeOf(U) - remaining_bits; - return @intCast(((next_int & ((@as(usize, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); + return @intCast(((next_int & ((@as(Bits, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); } fn peekBitsEnding(d: *Decompress, comptime U: type) !U { const remaining_bits = d.remaining_bits; const next_bits = d.next_bits; const in = d.input; - var u: usize = 0; + var u: Bits = 0; var remaining_needed_bits = @bitSizeOf(U) - remaining_bits; var i: usize = 0; while (remaining_needed_bits >= 8) { const byte = try specialPeek(in, next_bits, i); - u |= @as(usize, byte) << @intCast(i * 8); + u |= @as(Bits, byte) << @intCast(i * 8); remaining_needed_bits -= 8; i += 1; } if (remaining_needed_bits != 0) { const byte = try specialPeek(in, next_bits, i); - u |= @as(usize, byte) << @intCast((i * 8) + remaining_needed_bits); + u |= @as(Bits, byte) << @intCast((i * 8) + remaining_needed_bits); } return @truncate((u << remaining_bits) | next_bits); } /// If there is any unconsumed data, handles EndOfStream by pretending there /// are zeroes afterwards. -fn specialPeek(in: *Reader, next_bits: usize, i: usize) Reader.Error!u8 { +fn specialPeek(in: *Reader, next_bits: Bits, i: usize) Reader.Error!u8 { const peeked = in.peek(i + 1) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, error.EndOfStream => if (next_bits == 0 and i == 0) return error.EndOfStream else return 0, @@ -578,13 +581,13 @@ fn tossBits(d: *Decompress, n: u4) !void { d.remaining_bits = remaining_bits - n; } else { const in = d.input; - const next_int = in.takeInt(usize, .little) catch |err| switch (err) { + const next_int = in.takeInt(Bits, .little) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, error.EndOfStream => return tossBitsEnding(d, n), }; const needed_bits = n - remaining_bits; d.next_bits = next_int >> needed_bits; - d.remaining_bits = @intCast(@bitSizeOf(usize) - @as(usize, needed_bits)); + d.remaining_bits = @intCast(@bitSizeOf(Bits) - @as(usize, needed_bits)); } } @@ -593,9 +596,9 @@ fn tossBitsEnding(d: *Decompress, n: u4) !void { const in = d.input; const buffered_n = in.bufferedLen(); if (buffered_n == 0) return error.EndOfStream; - assert(buffered_n < @sizeOf(usize)); + assert(buffered_n < @sizeOf(Bits)); const needed_bits = n - remaining_bits; - const next_int = in.takeVarInt(usize, .little, buffered_n) catch |err| switch (err) { + const next_int = in.takeVarInt(Bits, .little, buffered_n) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, error.EndOfStream => unreachable, };