fix flate regression

Until now literal and distance code lengths where treated as two
different arrays. But according to rfc they can overlap:

  The code length repeat codes can cross from HLIT + 257 to the
  HDIST + 1 code lengths.  In other words, all code lengths form
  a single sequence of HLIT + HDIST + 258 values.

Now code lengths are decoded in single array which is then split
to literal and distance part.
This commit is contained in:
Igor Anić 2024-02-17 18:49:49 +01:00 committed by Andrew Kelley
parent 79e99c401c
commit 3e8cb153ea
3 changed files with 42 additions and 20 deletions

View File

@ -157,30 +157,23 @@ pub fn Inflate(comptime container: Container, comptime ReaderType: type) type {
var cl_dec: hfd.CodegenDecoder = .{};
try cl_dec.generate(&cl_lens);
// literal code lengths
var lit_lens = [_]u4{0} ** (286);
// decoded code lengths
var dec_lens = [_]u4{0} ** (286 + 30);
var pos: usize = 0;
while (pos < hlit) {
while (pos < hlit + hdist) {
const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
try self.bits.shift(sym.code_bits);
pos += try self.dynamicCodeLength(sym.symbol, &lit_lens, pos);
pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos);
}
if (pos > hlit)
if (pos > hlit + hdist) {
return error.InvalidDynamicBlockHeader;
// distance code lenths
var dst_lens = [_]u4{0} ** (30);
pos = 0;
while (pos < hdist) {
const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
try self.bits.shift(sym.code_bits);
pos += try self.dynamicCodeLength(sym.symbol, &dst_lens, pos);
}
if (pos > hdist)
return error.InvalidDynamicBlockHeader;
try self.lit_dec.generate(&lit_lens);
try self.dst_dec.generate(&dst_lens);
// literal code lengts to literal decoder
try self.lit_dec.generate(dec_lens[0..hlit]);
// distance code lengths to distance decoder
try self.dst_dec.generate(dec_lens[hlit .. hlit + hdist]);
}
// Decode code length symbol to code length. Writes decoded length into
@ -496,7 +489,7 @@ test "flate.Inflate fuzzing tests" {
.{ .input = "puff14", .err = error.EndOfStream },
.{ .input = "puff15", .err = error.IncompleteHuffmanTree },
.{ .input = "puff16", .err = error.InvalidDynamicBlockHeader },
.{ .input = "puff17", .err = error.InvalidDynamicBlockHeader }, // 25
.{ .input = "puff17", .err = error.MissingEndOfBlockCode }, // 25
.{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader },
.{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader },
.{ .input = "fuzz3", .err = error.InvalidMatch },
@ -506,8 +499,8 @@ test "flate.Inflate fuzzing tests" {
.{ .input = "puff20", .err = error.OversubscribedHuffmanTree },
.{ .input = "puff21", .err = error.OversubscribedHuffmanTree },
.{ .input = "puff22", .err = error.OversubscribedHuffmanTree },
.{ .input = "puff23", .err = error.InvalidDynamicBlockHeader }, // 35
.{ .input = "puff24", .err = error.InvalidDynamicBlockHeader },
.{ .input = "puff23", .err = error.OversubscribedHuffmanTree }, // 35
.{ .input = "puff24", .err = error.IncompleteHuffmanTree },
.{ .input = "puff25", .err = error.OversubscribedHuffmanTree },
.{ .input = "puff26", .err = error.InvalidDynamicBlockHeader },
.{ .input = "puff27", .err = error.InvalidDynamicBlockHeader },
@ -527,3 +520,15 @@ test "flate.Inflate fuzzing tests" {
}
}
}
test "flate bug 18966" {
const input = @embedFile("testdata/fuzz/bug_18966.input");
const expect = @embedFile("testdata/fuzz/bug_18966.expect");
var in = std.io.fixedBufferStream(input);
var out = std.ArrayList(u8).init(testing.allocator);
defer out.deinit();
try decompress(.gzip, in.reader(), out.writer());
try testing.expectEqualStrings(expect, out.items);
}

View File

@ -0,0 +1,17 @@
conditions,correction,exchange,id,participant_timestamp,price,sequence_number,sip_timestamp,size,tape,trf_id,trf_timestamp
,,1,,0,9.92,1929969624145677,1063224024145000000,200,1,,0
,,1,,0,9.86,1929967211195252,1063221611195000000,100,1,,0
,,1,,0,9.86,1929967094672373,1063221494672000000,200,1,,0
,,1,,0,9.85,1929967050174259,1063221450174000000,100,1,,0
,,1,,0,9.85,1929967013672246,1063221413672000000,200,1,,0
,,1,,0,9.85,1929966984671861,1063221384671000000,200,1,,0
,,1,,0,9.9,1929966976177115,1063221376177000000,200,1,,0
,,1,,0,9.85,1929966961186410,1063221361186000000,1000,1,,0
,,1,,0,9.85,1929957130426170,1063211530426000000,100,1,,0
,,1,,0,9.85,1929956938455804,1063211338455000000,100,1,,0
,,1,,0,9.85,1929956786452149,1063211186452000000,300,1,,0
,,1,,0,9.85,1929956658456522,1063211058456000000,500,1,,0
,,1,,0,9.85,1929956650952287,1063211050952000000,300,1,,0
,,1,,0,9.85,1929956528452578,1063210928452000000,1500,1,,0
,,1,,0,9.85,1929950990948195,1063205390948000000,100,1,,0
,,1,,0,9.85,1929947489432428,1063201889432000000,100,1,,0

Binary file not shown.