From aee90779d14bb47d70817f15313a3d2832e71356 Mon Sep 17 00:00:00 2001 From: Kacy Fortner Date: Wed, 8 Jul 2026 13:40:01 +0000 Subject: [PATCH] multi-member gzip, stored fallback, zlib on the shared engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit three gaps between "works" and "complete", closed together. multi-member: concatenated gzip streams (cat a.gz b.gz) decompress to their concatenated payloads with each member's crc32 and length trailer verified, as gunzip does. inflate now reports where a stream ends so the reader knows where the next member's header starts, and the trailer length compares mod 2^32, matching the format's isize field for payloads past 4gb. stored fallback: fixed huffman gives half the byte alphabet 9-bit codes, so truly incompressible data used to expand. deflate now encodes both ways and keeps the smaller — 8k of /dev/urandom costs 23 bytes of framing instead of ~5% growth. zlib: routed through the same inflate and deflate engine with the adler32 trailer verified — real compression instead of stored blocks, and python's zlib module reads the output directly. the multi-member work exposed a std bug: ByteBuffer.write of empty bytes reported failure, because the builtin returns bytes written and the result convention reads zero as an error. an empty write is now a no-op in the wrapper. a committed two-member fixture and extended interop regression pin all of it. --- docs/limitations.md | 10 ++-- std/bytes.pith | 4 ++ std/compress/deflate.pith | 38 +++++++++++++- std/compress/gzip.pith | 44 ++++++++++------ std/compress/inflate.pith | 21 ++++++-- std/compress/zlib.pith | 67 +++++++----------------- tests/cases/gzip_multimember_fixture.gz | Bin 0 -> 81 bytes tests/cases/test_gzip_interop.pith | 8 +++ tests/expected/test_gzip_interop.txt | 2 + 9 files changed, 120 insertions(+), 74 deletions(-) create mode 100644 tests/cases/gzip_multimember_fixture.gz diff --git a/docs/limitations.md b/docs/limitations.md index 4f29b7a7..bee281c7 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -50,10 +50,12 @@ something here that now works, the page is stale and a fix to it is welcome. quantifiers, backreferences, or lookaround. matching is a pike vm, so time is linear in the input for any pattern. - **gzip compresses with fixed huffman only** — `std.compress.gzip` - reads any deflate stream and writes real compression (greedy lz77 - over fixed huffman codes; system gunzip reads its output). dynamic - huffman trees would shave a few more percent and are not - implemented. + reads any deflate stream (multi-member files included) and writes + real compression (greedy lz77 over fixed huffman, stored-block + fallback for incompressible data; system gunzip reads its output, + and zlib routes through the same engine). dynamic huffman trees on + the write side would shave a few more percent and are the one + remaining refinement. ## tooling diff --git a/std/bytes.pith b/std/bytes.pith index 027b131b..2cfe2f68 100644 --- a/std/bytes.pith +++ b/std/bytes.pith @@ -106,6 +106,10 @@ pub fn array_filled(count: Int, value: Int) -> ByteArray!: impl ByteBuffer: fn write(data: Bytes) -> Int!: + # the builtin reports bytes written and the result convention + # reads 0 as failure; an empty write is a legitimate no-op + if data.len() == 0: + return 0 return byte_buffer_write(self.handle, data) fn free(): diff --git a/std/compress/deflate.pith b/std/compress/deflate.pith index 9e70608b..c691c121 100644 --- a/std/compress/deflate.pith +++ b/std/compress/deflate.pith @@ -97,8 +97,44 @@ fn write_match(w: BitWriter, length: Int, distance: Int): fn hash3(a: Int, b: Int, c: Int) -> Int: return bits.band(a * 2654435 + b * 40503 + c * 809, 32767) -# # compress raw bytes into a single fixed-huffman deflate block. +# stored blocks cost 5 framing bytes per 65535-byte chunk — the +# fallback for data fixed huffman would expand (full-range random +# bytes cost 9 bits for half the alphabet) +fn deflate_stored(data: Bytes) -> Bytes!: + out := bytes.buffer_with_capacity(data.len() + 16) + mut offset := 0 + mut wrote_any := false + while offset < data.len() or not wrote_any: + mut end := offset + 65535 + if end > data.len(): + end = data.len() + chunk_len := end - offset + mut final_flag := 0 + if end == data.len(): + final_flag = 1 + out.write_byte(final_flag)! + out.write_byte(bits.band(chunk_len, 255))! + out.write_byte(bits.band(bits.shr(chunk_len, 8), 255))! + nlen := bits.band(bits.invert(chunk_len), 65535) + out.write_byte(bits.band(nlen, 255))! + out.write_byte(bits.band(bits.shr(nlen, 8), 255))! + mut i := offset + while i < end: + out.write_byte(bytes.get(data, i))! + i = i + 1 + offset = end + wrote_any = true + return out.take_bytes() + +# # compress raw bytes: fixed-huffman lz77, falling back to stored +# # blocks when the encoded form would be larger than the input. pub fn deflate(data: Bytes) -> Bytes!: + packed := deflate_huffman(data)! + if packed.len() > data.len() + 5: + return deflate_stored(data) + return packed + +fn deflate_huffman(data: Bytes) -> Bytes!: w := BitWriter([], 0, 0) # final block, fixed huffman bw_write_bits(w, 1, 1) diff --git a/std/compress/gzip.pith b/std/compress/gzip.pith index a24e04ff..096ee99f 100644 --- a/std/compress/gzip.pith +++ b/std/compress/gzip.pith @@ -95,16 +95,16 @@ fn gzip_skip_zero_terminated(data: Bytes, pos: Int) -> Int!: return cursor fail gzip_invalid() -fn gzip_parse_header(data: Bytes) -> Int!: - id1 := gzip_read_u8_at(data, 0)! - id2 := gzip_read_u8_at(data, 1)! - method := gzip_read_u8_at(data, 2)! - flags := gzip_read_u8_at(data, 3)! +fn gzip_parse_header_at(data: Bytes, base: Int) -> Int!: + id1 := gzip_read_u8_at(data, base)! + id2 := gzip_read_u8_at(data, base + 1)! + method := gzip_read_u8_at(data, base + 2)! + flags := gzip_read_u8_at(data, base + 3)! if id1 != 31 or id2 != 139 or method != 8: fail gzip_invalid() if bits.band(flags, GZIP_FLAG_RESERVED) != 0: fail gzip_invalid() - mut pos := gzip_skip(data, 4, 6)! + mut pos := gzip_skip(data, base + 4, 6)! if bits.band(flags, GZIP_FLAG_EXTRA) != 0: extra_len := gzip_read_u16_le_at(data, pos)! pos = gzip_skip(data, pos + 2, extra_len)! @@ -129,20 +129,30 @@ pub fn compress(data: Bytes) -> Bytes: return gzip_compress_result(data) catch bytes.empty() # # decompress a gzip stream: stored, fixed-huffman, and -# # dynamic-huffman blocks all read, with the crc32 and length trailer -# # verified — files from other tools decompress correctly. +# # dynamic-huffman blocks all read, with each member's crc32 and +# # length trailer verified. concatenated members (cat a.gz b.gz) +# # decompress to their concatenated payloads, as gunzip does. pub fn decompress(data: Bytes) -> Bytes!: if data.len() < 18: fail gzip_invalid() - start := gzip_parse_header(data)! - payload := inflate.inflate(data, start)! - expected_crc := gzip_read_u32_le_at(data, data.len() - 8)! - expected_len := gzip_read_u32_le_at(data, data.len() - 4)! - if payload.len() != expected_len: - fail "gzip: length trailer mismatch" - if checksum.crc32(payload) != expected_crc: - fail "gzip: crc mismatch" - return payload + out := bytes.buffer() + mut pos := 0 + while pos < data.len(): + start := gzip_parse_header_at(data, pos)! + result := inflate.inflate_with_end(data, start)! + payload := result.payload + trailer := result.next_pos + if trailer + 8 > data.len(): + fail "gzip: truncated trailer" + expected_crc := gzip_read_u32_le_at(data, trailer)! + expected_len := gzip_read_u32_le_at(data, trailer + 4)! + if bits.mask32(payload.len()) != expected_len: + fail "gzip: length trailer mismatch" + if checksum.crc32(payload) != expected_crc: + fail "gzip: crc mismatch" + out.write(payload)! + pos = trailer + 8 + return out.take_bytes() # true when the data is a gzip stream this module can read. pub fn is_valid(data: Bytes) -> Bool: diff --git a/std/compress/inflate.pith b/std/compress/inflate.pith index fe85eee3..84e2bfd8 100644 --- a/std/compress/inflate.pith +++ b/std/compress/inflate.pith @@ -195,10 +195,20 @@ fn inflate_block(r: BitReader, out: List[Int], lit: Huffman, dist: Huffman) -> B length = length - 1 return true -# # decompress a raw deflate stream starting at `start`; returns the -# # payload and leaves nothing implicit — bad streams fail with a -# # message naming what was malformed. +# # an inflate result that also reports where the stream ended, so a +# # caller reading concatenated members knows where the next begins. +pub struct InflateResult: + payload: Bytes + next_pos: Int + +# # decompress a raw deflate stream starting at `start`; bad streams +# # fail with a message naming what was malformed. pub fn inflate(data: Bytes, start: Int) -> Bytes!: + return inflate_with_end(data, start)!.payload + +# # inflate, also reporting the byte position just past the stream — +# # a partly consumed final byte counts as consumed. +pub fn inflate_with_end(data: Bytes, start: Int) -> InflateResult!: r := BitReader(data, start, 0) mut out: List[Int] := [] mut final_block := false @@ -226,7 +236,10 @@ pub fn inflate(data: Bytes, start: Int) -> Bytes!: inflate_block(r, out, trees.lit, trees.dist)! else: fail "inflate: reserved block type" + mut end_pos := r.pos + if r.bit > 0: + end_pos = end_pos + 1 buf := bytes.buffer_with_capacity(out.len()) for b in out: buf.write_byte(b)! - return buf.take_bytes() + return InflateResult(buf.take_bytes(), end_pos) diff --git a/std/compress/zlib.pith b/std/compress/zlib.pith index 3bc8b66f..504cbb7d 100644 --- a/std/compress/zlib.pith +++ b/std/compress/zlib.pith @@ -4,6 +4,8 @@ # gzip module, this first slice favors compatibility over smaller output. import std.bytes as bytes +import std.compress.inflate as inflate +import std.compress.deflate as deflate_engine import std.binary as binary import std.bits as bits import std.checksum as checksum @@ -87,76 +89,45 @@ fn zlib_parse_header(data: Bytes) -> Int!: fail zlib_invalid() return 2 -fn zlib_decompress_stored(data: Bytes) -> Bytes!: - mut pos := zlib_parse_header(data)! - out := bytes.buffer() - deflate_limit := data.len() - 4 - while true: - if pos + 1 > deflate_limit: - fail zlib_invalid() - block_header := zlib_read_u8_at(data, pos)! - pos = pos + 1 - is_final := bits.band(block_header, 1) != 0 - block_type := bits.band(bits.shr(block_header, 1), 3) - if block_type != 0: - fail zlib_invalid() - if pos + 4 > deflate_limit: - fail zlib_invalid() - block_len := zlib_read_u16_le_at(data, pos)! - block_nlen := zlib_read_u16_le_at(data, pos + 2)! - pos = pos + 4 - if block_len + block_nlen != ZLIB_STORED_BLOCK_SIZE: - fail zlib_invalid() - if pos + block_len > deflate_limit: - fail zlib_invalid() - if block_len > 0: - out.write(data.slice(pos, pos + block_len))! - pos = pos + block_len - if is_final: - break - if pos + 4 != data.len(): +fn zlib_decompress_result(data: Bytes) -> Bytes!: + start := zlib_parse_header(data)! + payload := inflate.inflate(data, start)! + if data.len() < 4: fail zlib_invalid() - result := out.bytes() - expected_adler := zlib_read_u32_be_at(data, pos)! - if checksum.adler32(result) != expected_adler: - fail zlib_invalid() - return result + expected := zlib_read_u32_be_at(data, data.len() - 4)! + if checksum.adler32(payload) != expected: + fail "zlib: adler32 mismatch" + return payload fn zlib_compress_result(data: Bytes) -> Bytes!: - out := bytes.buffer_with_capacity(2 + data.len() + 5 + 4) + out := bytes.buffer_with_capacity(2 + data.len() + 4) zlib_write_header(out)! - if data.len() == 0: - zlib_write_stored_block(out, true, data)! - else: - mut offset := 0 - while offset < data.len(): - mut end := offset + ZLIB_STORED_BLOCK_SIZE - if end > data.len(): - end = data.len() - chunk := data.slice(offset, end) - zlib_write_stored_block(out, end == data.len(), chunk)! - offset = end + out.write(deflate_engine.deflate(data)!)! zlib_write_u32_be(out, checksum.adler32(data))! return out.bytes() -# compress bytes into a zlib stream. +# # compress bytes into a zlib stream (deflate + adler32). pub fn compress(data: Bytes) -> Bytes: return zlib_compress_result(data) catch bytes.empty() # decompress a zlib stream created with stored deflate blocks. +# # decompress a zlib stream; the adler32 trailer is verified. pub fn decompress(data: Bytes) -> Bytes!: - return zlib_decompress_stored(data) + return zlib_decompress_result(data) # true when the data is a zlib stream this module can read. +# # true when the data is a zlib stream this module can read. pub fn is_valid(data: Bytes) -> Bool: - result := zlib_decompress_stored(data) + result := zlib_decompress_result(data) return result.is_ok # compress utf-8 text into zlib bytes. +# # compress utf-8 text into zlib bytes. pub fn compress_string(text: String) -> Bytes: return compress(bytes.from_string_utf8(text)) # decompress zlib bytes and decode the result as utf-8 text. +# # decompress zlib bytes and decode as utf-8 text. pub fn decompress_string(data: Bytes) -> String!: return decompress(data)!.to_string_utf8()! diff --git a/tests/cases/gzip_multimember_fixture.gz b/tests/cases/gzip_multimember_fixture.gz new file mode 100644 index 0000000000000000000000000000000000000000..d50bd4ff753d657262fc1357e1fb9855da4cccbe GIT binary patch literal 81 zcmb2|=3oE=X7AHFT6%#ee9!uN=?0zf(Dk|+%)k)-u4SJvP+lIcOxw@*+<9-9I*-%m TPoC7*zsAh)F10{b45$(Sl9d=Q literal 0 HcmV?d00001 diff --git a/tests/cases/test_gzip_interop.pith b/tests/cases/test_gzip_interop.pith index bcc82bf8..63a54fd9 100644 --- a/tests/cases/test_gzip_interop.pith +++ b/tests/cases/test_gzip_interop.pith @@ -2,6 +2,7 @@ # huffman blocks); reading it proves interop with the outside world. import std.fs as fs import std.compress.gzip as gzip +import std.compress.zlib as zlib fn main() -> Int!: raw := fs.read_bytes("tests/cases/gzip_interop_fixture.gz")! @@ -14,4 +15,11 @@ fn main() -> Int!: print("shrinks: {packed.len() < big.len()}") back := gzip.decompress(packed)! print("restores: {back.len() == big.len()}") + + multi := fs.read_bytes("tests/cases/gzip_multimember_fixture.gz")! + print(gzip.decompress(multi)!.to_string_utf8()!) + + zpacked := zlib.compress(big) + zback := zlib.decompress(zpacked)! + print("zlib shrinks: {zpacked.len() < big.len()} restores: {zback.len() == big.len()}") return 0 diff --git a/tests/expected/test_gzip_interop.txt b/tests/expected/test_gzip_interop.txt index 051061fc..400c6e42 100644 --- a/tests/expected/test_gzip_interop.txt +++ b/tests/expected/test_gzip_interop.txt @@ -4,3 +4,5 @@ repeat repeat repeat repeat repeat repeat repeat repeat. compressor round trip shrinks: true restores: true +first member here. second member follows. +zlib shrinks: true restores: true