mirror of
https://github.com/ziglang/zig.git
synced 2025-12-06 22:33:08 +00:00
30b41dc510
* std.Io.Reader: appendRemaining no longer supports alignment and has different rules about how exceeding limit. Fixed bug where it would return success instead of error.StreamTooLong like it was supposed to. * std.Io.Reader: simplify appendRemaining and appendRemainingUnlimited to be implemented based on std.Io.Writer.Allocating * std.Io.Writer: introduce unreachableRebase * std.Io.Writer: remove minimum_unused_capacity from Allocating. maybe that flexibility could have been handy, but let's see if anyone actually needs it. The field is redundant with the superlinear growth of ArrayList capacity. * std.Io.Writer: growingRebase also ensures total capacity on the preserve parameter, making it no longer necessary to do ensureTotalCapacity at the usage site of decompression streams. * std.compress.flate.Decompress: fix rebase not taking into account seek * std.compress.zstd.Decompress: split into "direct" and "indirect" usage patterns depending on whether a buffer is provided to init, matching how flate works. Remove some overzealous asserts that prevented buffer expansion from within rebase implementation. * std.zig: fix readSourceFileToAlloc returning an overaligned slice which was difficult to free correctly. fixes #24608
166 lines
6.5 KiB
Zig
166 lines
6.5 KiB
Zig
const std = @import("../std.zig");
|
|
const assert = std.debug.assert;
|
|
|
|
pub const Decompress = @import("zstd/Decompress.zig");
|
|
|
|
/// Recommended amount by the standard. Lower than this may result in inability
|
|
/// to decompress common streams.
|
|
pub const default_window_len = 8 * 1024 * 1024;
|
|
pub const block_size_max = 1 << 17;
|
|
|
|
pub const literals_length_default_distribution = [36]i16{
|
|
4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
|
|
2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
|
|
-1, -1, -1, -1,
|
|
};
|
|
|
|
pub const match_lengths_default_distribution = [53]i16{
|
|
1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
|
|
-1, -1, -1, -1, -1,
|
|
};
|
|
|
|
pub const offset_codes_default_distribution = [29]i16{
|
|
1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
|
|
};
|
|
|
|
pub const start_repeated_offset_1 = 1;
|
|
pub const start_repeated_offset_2 = 4;
|
|
pub const start_repeated_offset_3 = 8;
|
|
|
|
pub const literals_length_code_table = [36]struct { u32, u5 }{
|
|
.{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
|
|
.{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
|
|
.{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
|
|
.{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
|
|
.{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
|
|
.{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
|
|
.{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
|
|
.{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
|
|
.{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
|
|
};
|
|
|
|
pub const match_length_code_table = [53]struct { u32, u5 }{
|
|
.{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
|
|
.{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
|
|
.{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
|
|
.{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
|
|
.{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
|
|
.{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
|
|
.{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
|
|
.{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
|
|
.{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
|
|
};
|
|
|
|
pub const table_accuracy_log_max = struct {
|
|
pub const literal = 9;
|
|
pub const match = 9;
|
|
pub const offset = 8;
|
|
};
|
|
|
|
pub const table_symbol_count_max = struct {
|
|
pub const literal = 36;
|
|
pub const match = 53;
|
|
pub const offset = 32;
|
|
};
|
|
|
|
pub const default_accuracy_log = struct {
|
|
pub const literal = 6;
|
|
pub const match = 6;
|
|
pub const offset = 5;
|
|
};
|
|
pub const table_size_max = struct {
|
|
pub const literal = 1 << table_accuracy_log_max.literal;
|
|
pub const match = 1 << table_accuracy_log_max.match;
|
|
pub const offset = 1 << table_accuracy_log_max.offset;
|
|
};
|
|
|
|
fn testDecompress(gpa: std.mem.Allocator, compressed: []const u8) ![]u8 {
|
|
var out: std.Io.Writer.Allocating = .init(gpa);
|
|
defer out.deinit();
|
|
|
|
var in: std.Io.Reader = .fixed(compressed);
|
|
var zstd_stream: Decompress = .init(&in, &.{}, .{});
|
|
_ = try zstd_stream.reader.streamRemaining(&out.writer);
|
|
|
|
return out.toOwnedSlice();
|
|
}
|
|
|
|
fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void {
|
|
const gpa = std.testing.allocator;
|
|
const result = try testDecompress(gpa, compressed);
|
|
defer gpa.free(result);
|
|
try std.testing.expectEqualSlices(u8, uncompressed, result);
|
|
}
|
|
|
|
fn testExpectDecompressError(err: anyerror, compressed: []const u8) !void {
|
|
const gpa = std.testing.allocator;
|
|
|
|
var out: std.Io.Writer.Allocating = .init(gpa);
|
|
defer out.deinit();
|
|
|
|
var in: std.Io.Reader = .fixed(compressed);
|
|
var zstd_stream: Decompress = .init(&in, &.{}, .{});
|
|
try std.testing.expectError(
|
|
error.ReadFailed,
|
|
zstd_stream.reader.streamRemaining(&out.writer),
|
|
);
|
|
try std.testing.expectError(err, zstd_stream.err orelse {});
|
|
}
|
|
|
|
test Decompress {
|
|
const uncompressed = @embedFile("testdata/rfc8478.txt");
|
|
const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
|
|
const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
|
|
|
|
try testExpectDecompress(uncompressed, compressed3);
|
|
try testExpectDecompress(uncompressed, compressed19);
|
|
}
|
|
|
|
test "partial magic number" {
|
|
const input_raw =
|
|
"\x28\xb5\x2f"; // 3 bytes of the 4-byte zstandard frame magic number
|
|
try testExpectDecompressError(error.BadMagic, input_raw);
|
|
}
|
|
|
|
test "zero sized raw block" {
|
|
const input_raw =
|
|
"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
|
|
"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
|
|
"\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
|
|
try testExpectDecompress("", input_raw);
|
|
}
|
|
|
|
test "zero sized rle block" {
|
|
const input_rle =
|
|
"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
|
|
"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
|
|
"\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
|
|
"\xaa"; // block_content
|
|
try testExpectDecompress("", input_rle);
|
|
}
|
|
|
|
test "declared raw literals size too large" {
|
|
const input_raw =
|
|
"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
|
|
"\x00\x00" ++ // frame header: everything unset, window descriptor zero
|
|
"\x95\x00\x00" ++ // block header with: last_block set, block_type compressed, block_size 18
|
|
"\xbc\xf3\xae" ++ // literals section header with: type raw, size_format 3, regenerated_size 716603
|
|
"\xa5\x9f\xe3"; // some bytes of literal content - the content is shorter than regenerated_size
|
|
|
|
// Note that the regenerated_size in the above input is larger than block maximum size, so the
|
|
// block can't be valid as it is a raw literals block.
|
|
try testExpectDecompressError(error.MalformedLiteralsSection, input_raw);
|
|
}
|
|
|
|
test "skippable frame" {
|
|
const input_raw =
|
|
"\x50\x2a\x4d\x18" ++ // min magic number for a skippable frame
|
|
"\x02\x00\x00\x00" ++ // number of bytes to skip
|
|
"\xFF\xFF"; // the bytes that are skipped
|
|
|
|
try testExpectDecompress("", input_raw);
|
|
}
|