multi-member gzip, stored fallback, zlib on the shared engine

docs/limitations.md

@@ -50,10 +50,12 @@ something here that now works, the page is stale and a fix to it is welcome.
   quantifiers, backreferences, or lookaround. matching is a pike vm, so
   time is linear in the input for any pattern.
 - **gzip compresses with fixed huffman only** — `std.compress.gzip`
-  reads any deflate stream and writes real compression (greedy lz77
-  over fixed huffman codes; system gunzip reads its output). dynamic
-  huffman trees would shave a few more percent and are not
-  implemented.
+  reads any deflate stream (multi-member files included) and writes
+  real compression (greedy lz77 over fixed huffman, stored-block
+  fallback for incompressible data; system gunzip reads its output,
+  and zlib routes through the same engine). dynamic huffman trees on
+  the write side would shave a few more percent and are the one
+  remaining refinement.
 
 ## tooling
 

std/bytes.pith

@@ -106,6 +106,10 @@ pub fn array_filled(count: Int, value: Int) -> ByteArray!:
 
 impl ByteBuffer:
     fn write(data: Bytes) -> Int!:
+        # the builtin reports bytes written and the result convention
+        # reads 0 as failure; an empty write is a legitimate no-op
+        if data.len() == 0:
+            return 0
         return byte_buffer_write(self.handle, data)
 
     fn free():

std/compress/deflate.pith

@@ -97,8 +97,44 @@ fn write_match(w: BitWriter, length: Int, distance: Int):
 fn hash3(a: Int, b: Int, c: Int) -> Int:
     return bits.band(a * 2654435 + b * 40503 + c * 809, 32767)
 
-# # compress raw bytes into a single fixed-huffman deflate block.
+# stored blocks cost 5 framing bytes per 65535-byte chunk — the
+# fallback for data fixed huffman would expand (full-range random
+# bytes cost 9 bits for half the alphabet)
+fn deflate_stored(data: Bytes) -> Bytes!:
+    out := bytes.buffer_with_capacity(data.len() + 16)
+    mut offset := 0
+    mut wrote_any := false
+    while offset < data.len() or not wrote_any:
+        mut end := offset + 65535
+        if end > data.len():
+            end = data.len()
+        chunk_len := end - offset
+        mut final_flag := 0
+        if end == data.len():
+            final_flag = 1
+        out.write_byte(final_flag)!
+        out.write_byte(bits.band(chunk_len, 255))!
+        out.write_byte(bits.band(bits.shr(chunk_len, 8), 255))!
+        nlen := bits.band(bits.invert(chunk_len), 65535)
+        out.write_byte(bits.band(nlen, 255))!
+        out.write_byte(bits.band(bits.shr(nlen, 8), 255))!
+        mut i := offset
+        while i < end:
+            out.write_byte(bytes.get(data, i))!
+            i = i + 1
+        offset = end
+        wrote_any = true
+    return out.take_bytes()
+
+# # compress raw bytes: fixed-huffman lz77, falling back to stored
+# # blocks when the encoded form would be larger than the input.
 pub fn deflate(data: Bytes) -> Bytes!:
+    packed := deflate_huffman(data)!
+    if packed.len() > data.len() + 5:
+        return deflate_stored(data)
+    return packed
+
+fn deflate_huffman(data: Bytes) -> Bytes!:
     w := BitWriter([], 0, 0)
     # final block, fixed huffman
     bw_write_bits(w, 1, 1)

std/compress/gzip.pith

@@ -95,16 +95,16 @@ fn gzip_skip_zero_terminated(data: Bytes, pos: Int) -> Int!:
             return cursor
     fail gzip_invalid()
 
-fn gzip_parse_header(data: Bytes) -> Int!:
-    id1 := gzip_read_u8_at(data, 0)!
-    id2 := gzip_read_u8_at(data, 1)!
-    method := gzip_read_u8_at(data, 2)!
-    flags := gzip_read_u8_at(data, 3)!
+fn gzip_parse_header_at(data: Bytes, base: Int) -> Int!:
+    id1 := gzip_read_u8_at(data, base)!
+    id2 := gzip_read_u8_at(data, base + 1)!
+    method := gzip_read_u8_at(data, base + 2)!
+    flags := gzip_read_u8_at(data, base + 3)!
     if id1 != 31 or id2 != 139 or method != 8:
         fail gzip_invalid()
     if bits.band(flags, GZIP_FLAG_RESERVED) != 0:
         fail gzip_invalid()
-    mut pos := gzip_skip(data, 4, 6)!
+    mut pos := gzip_skip(data, base + 4, 6)!
     if bits.band(flags, GZIP_FLAG_EXTRA) != 0:
         extra_len := gzip_read_u16_le_at(data, pos)!
         pos = gzip_skip(data, pos + 2, extra_len)!
@@ -129,20 +129,30 @@ pub fn compress(data: Bytes) -> Bytes:
     return gzip_compress_result(data) catch bytes.empty()
 
 # # decompress a gzip stream: stored, fixed-huffman, and
-# # dynamic-huffman blocks all read, with the crc32 and length trailer
-# # verified — files from other tools decompress correctly.
+# # dynamic-huffman blocks all read, with each member's crc32 and
+# # length trailer verified. concatenated members (cat a.gz b.gz)
+# # decompress to their concatenated payloads, as gunzip does.
 pub fn decompress(data: Bytes) -> Bytes!:
     if data.len() < 18:
         fail gzip_invalid()
-    start := gzip_parse_header(data)!
-    payload := inflate.inflate(data, start)!
-    expected_crc := gzip_read_u32_le_at(data, data.len() - 8)!
-    expected_len := gzip_read_u32_le_at(data, data.len() - 4)!
-    if payload.len() != expected_len:
-        fail "gzip: length trailer mismatch"
-    if checksum.crc32(payload) != expected_crc:
-        fail "gzip: crc mismatch"
-    return payload
+    out := bytes.buffer()
+    mut pos := 0
+    while pos < data.len():
+        start := gzip_parse_header_at(data, pos)!
+        result := inflate.inflate_with_end(data, start)!
+        payload := result.payload
+        trailer := result.next_pos
+        if trailer + 8 > data.len():
+            fail "gzip: truncated trailer"
+        expected_crc := gzip_read_u32_le_at(data, trailer)!
+        expected_len := gzip_read_u32_le_at(data, trailer + 4)!
+        if bits.mask32(payload.len()) != expected_len:
+            fail "gzip: length trailer mismatch"
+        if checksum.crc32(payload) != expected_crc:
+            fail "gzip: crc mismatch"
+        out.write(payload)!
+        pos = trailer + 8
+    return out.take_bytes()
 
 # true when the data is a gzip stream this module can read.
 pub fn is_valid(data: Bytes) -> Bool:

std/compress/inflate.pith

@@ -195,10 +195,20 @@ fn inflate_block(r: BitReader, out: List[Int], lit: Huffman, dist: Huffman) -> B
                 length = length - 1
     return true
 
-# # decompress a raw deflate stream starting at `start`; returns the
-# # payload and leaves nothing implicit — bad streams fail with a
-# # message naming what was malformed.
+# # an inflate result that also reports where the stream ended, so a
+# # caller reading concatenated members knows where the next begins.
+pub struct InflateResult:
+    payload: Bytes
+    next_pos: Int
+
+# # decompress a raw deflate stream starting at `start`; bad streams
+# # fail with a message naming what was malformed.
 pub fn inflate(data: Bytes, start: Int) -> Bytes!:
+    return inflate_with_end(data, start)!.payload
+
+# # inflate, also reporting the byte position just past the stream —
+# # a partly consumed final byte counts as consumed.
+pub fn inflate_with_end(data: Bytes, start: Int) -> InflateResult!:
     r := BitReader(data, start, 0)
     mut out: List[Int] := []
     mut final_block := false
@@ -226,7 +236,10 @@ pub fn inflate(data: Bytes, start: Int) -> Bytes!:
             inflate_block(r, out, trees.lit, trees.dist)!
         else:
             fail "inflate: reserved block type"
+    mut end_pos := r.pos
+    if r.bit > 0:
+        end_pos = end_pos + 1
     buf := bytes.buffer_with_capacity(out.len())
     for b in out:
         buf.write_byte(b)!
-    return buf.take_bytes()
+    return InflateResult(buf.take_bytes(), end_pos)

std/compress/zlib.pith

@@ -4,6 +4,8 @@
 # gzip module, this first slice favors compatibility over smaller output.
 
 import std.bytes as bytes
+import std.compress.inflate as inflate
+import std.compress.deflate as deflate_engine
 import std.binary as binary
 import std.bits as bits
 import std.checksum as checksum
@@ -87,76 +89,45 @@ fn zlib_parse_header(data: Bytes) -> Int!:
         fail zlib_invalid()
     return 2
 
-fn zlib_decompress_stored(data: Bytes) -> Bytes!:
-    mut pos := zlib_parse_header(data)!
-    out := bytes.buffer()
-    deflate_limit := data.len() - 4
-    while true:
-        if pos + 1 > deflate_limit:
-            fail zlib_invalid()
-        block_header := zlib_read_u8_at(data, pos)!
-        pos = pos + 1
-        is_final := bits.band(block_header, 1) != 0
-        block_type := bits.band(bits.shr(block_header, 1), 3)
-        if block_type != 0:
-            fail zlib_invalid()
-        if pos + 4 > deflate_limit:
-            fail zlib_invalid()
-        block_len := zlib_read_u16_le_at(data, pos)!
-        block_nlen := zlib_read_u16_le_at(data, pos + 2)!
-        pos = pos + 4
-        if block_len + block_nlen != ZLIB_STORED_BLOCK_SIZE:
-            fail zlib_invalid()
-        if pos + block_len > deflate_limit:
-            fail zlib_invalid()
-        if block_len > 0:
-            out.write(data.slice(pos, pos + block_len))!
-        pos = pos + block_len
-        if is_final:
-            break
-    if pos + 4 != data.len():
+fn zlib_decompress_result(data: Bytes) -> Bytes!:
+    start := zlib_parse_header(data)!
+    payload := inflate.inflate(data, start)!
+    if data.len() < 4:
         fail zlib_invalid()
-    result := out.bytes()
-    expected_adler := zlib_read_u32_be_at(data, pos)!
-    if checksum.adler32(result) != expected_adler:
-        fail zlib_invalid()
-    return result
+    expected := zlib_read_u32_be_at(data, data.len() - 4)!
+    if checksum.adler32(payload) != expected:
+        fail "zlib: adler32 mismatch"
+    return payload
 
 fn zlib_compress_result(data: Bytes) -> Bytes!:
-    out := bytes.buffer_with_capacity(2 + data.len() + 5 + 4)
+    out := bytes.buffer_with_capacity(2 + data.len() + 4)
     zlib_write_header(out)!
-    if data.len() == 0:
-        zlib_write_stored_block(out, true, data)!
-    else:
-        mut offset := 0
-        while offset < data.len():
-            mut end := offset + ZLIB_STORED_BLOCK_SIZE
-            if end > data.len():
-                end = data.len()
-            chunk := data.slice(offset, end)
-            zlib_write_stored_block(out, end == data.len(), chunk)!
-            offset = end
+    out.write(deflate_engine.deflate(data)!)!
     zlib_write_u32_be(out, checksum.adler32(data))!
     return out.bytes()
 
-# compress bytes into a zlib stream.
+# # compress bytes into a zlib stream (deflate + adler32).
 pub fn compress(data: Bytes) -> Bytes:
     return zlib_compress_result(data) catch bytes.empty()
 
 # decompress a zlib stream created with stored deflate blocks.
+# # decompress a zlib stream; the adler32 trailer is verified.
 pub fn decompress(data: Bytes) -> Bytes!:
-    return zlib_decompress_stored(data)
+    return zlib_decompress_result(data)
 
 # true when the data is a zlib stream this module can read.
+# # true when the data is a zlib stream this module can read.
 pub fn is_valid(data: Bytes) -> Bool:
-    result := zlib_decompress_stored(data)
+    result := zlib_decompress_result(data)
     return result.is_ok
 
 # compress utf-8 text into zlib bytes.
+# # compress utf-8 text into zlib bytes.
 pub fn compress_string(text: String) -> Bytes:
     return compress(bytes.from_string_utf8(text))
 
 # decompress zlib bytes and decode the result as utf-8 text.
+# # decompress zlib bytes and decode as utf-8 text.
 pub fn decompress_string(data: Bytes) -> String!:
     return decompress(data)!.to_string_utf8()!
 

tests/cases/test_gzip_interop.pith

@@ -2,6 +2,7 @@
 # huffman blocks); reading it proves interop with the outside world.
 import std.fs as fs
 import std.compress.gzip as gzip
+import std.compress.zlib as zlib
 
 fn main() -> Int!:
     raw := fs.read_bytes("tests/cases/gzip_interop_fixture.gz")!
@@ -14,4 +15,11 @@ fn main() -> Int!:
     print("shrinks: {packed.len() < big.len()}")
     back := gzip.decompress(packed)!
     print("restores: {back.len() == big.len()}")
+
+    multi := fs.read_bytes("tests/cases/gzip_multimember_fixture.gz")!
+    print(gzip.decompress(multi)!.to_string_utf8()!)
+
+    zpacked := zlib.compress(big)
+    zback := zlib.decompress(zpacked)!
+    print("zlib shrinks: {zpacked.len() < big.len()} restores: {zback.len() == big.len()}")
     return 0

tests/expected/test_gzip_interop.txt

@@ -4,3 +4,5 @@ repeat repeat repeat repeat repeat repeat repeat repeat.
 compressor round trip
 shrinks: true
 restores: true
+first member here. second member follows.
+zlib shrinks: true restores: true