add a deflate compressor; gzip now interops both directions

.github/workflows/ci.yml

@@ -47,6 +47,9 @@ jobs:
       - name: parq golden check
         run: make parq-check
 
+      - name: gzip interop check
+        run: make gzip-interop-check
+
       - name: verify bootstrap fixed point
         run: make bootstrap-verify
 

Makefile

@@ -670,6 +670,19 @@ parq-check:
 	diff -u tools/parq/expected/report.txt "$$tmpdir/report1.txt" && \
 	echo "parq output matches golden files"
 
+# --- gzip interop check ---
+# both directions against the system tool: pith reads gzip's output
+# (covered in logscan-check too) and gzip reads pith's
+
+gzip-interop-check:
+	@echo "--- gzip interop check ---"
+	@tmpdir=$$(mktemp -d /tmp/pith-gzip-XXXXXX); \
+	trap 'rm -rf "$$tmpdir"' EXIT; \
+	printf 'import std.fs as fs\nimport std.compress.gzip as gzip\n\nfn main() -> Int!:\n    raw := fs.read_bytes("README.md")!\n    fs.write_bytes("'"$$tmpdir"'/out.gz", gzip.compress(raw))!\n    return 0\n' > "$$tmpdir/pack.pith"; \
+	./target/release/pith run "$$tmpdir/pack.pith" > /dev/null 2>&1 && \
+	gunzip -c "$$tmpdir/out.gz" | cmp - README.md && \
+	echo "system gunzip reads pith output byte-identical"
+
 # --- cli regressions ---
 
 cli-regressions: build cli-regressions-only

README.md

@@ -160,7 +160,7 @@ that path.
 - std.os.path, std.os.process, std.fs, std.glob — files, paths, and file discovery
 - std.cli, std.diagnostic, std.testing, std.text.scanner — small tooling layers
 - std.log, std.metrics, std.fmt, std.math, std.rand, std.time, std.datetime, std.uuid — common app helpers
-- std.compress (gzip reads any deflate stream — dynamic huffman included — via an inflate engine written in pith; compression still writes stored blocks), std.archive (tar/zip), std.net.sse, std.data.table — archives, server-sent events, and tabular data
+- std.compress (full gzip in pith: the inflate engine reads any deflate stream, and compression is lz77 over fixed huffman — interops with system gzip in both directions), std.archive (tar/zip), std.net.sse, std.data.table — archives, server-sent events, and tabular data
 - std.regex — a pike-vm regular expression engine written in pith itself: linear-time matching with no pathological backtracking, capture groups, classes, alternation, anchors (no {n,m}, lazy quantifiers, backreferences, or lookaround)
 
 for child processes, prefer `std.os.process.command(...)` and the structured

docs/limitations.md

@@ -49,11 +49,11 @@ something here that now works, the page is stale and a fix to it is welcome.
   groups, and `^ $` anchors. it does not support `{n,m}` counts, lazy
   quantifiers, backreferences, or lookaround. matching is a pike vm, so
   time is linear in the input for any pattern.
-- **gzip compression writes stored blocks** — `std.compress.gzip` now
-  reads any deflate stream (stored, fixed, and dynamic huffman — files
-  from other tools decompress correctly, crc-verified), but its own
-  compress() still emits stored blocks, so pith-written .gz files are
-  larger than their input. a real compressor is the remaining half.
+- **gzip compresses with fixed huffman only** — `std.compress.gzip`
+  reads any deflate stream and writes real compression (greedy lz77
+  over fixed huffman codes; system gunzip reads its output). dynamic
+  huffman trees would shave a few more percent and are not
+  implemented.
 
 ## tooling
 

std/compress/deflate.pith

@@ -0,0 +1,168 @@
+# std/compress/deflate — the deflate compressor (rfc 1951), fixed
+# huffman blocks over greedy lz77.
+#
+# a hash chain over 3-byte prefixes finds back-references (longest
+# match wins, up to 258 bytes within a 32k window); the token stream
+# encodes with the fixed huffman tables, so no tree serialization is
+# needed. dynamic huffman would shave a few more percent and is left
+# for later — this already produces real compression that any inflate
+# implementation reads.
+import std.bits as bits
+import std.bytes as bytes
+
+# # bit writer state: bits accumulate lsb-first into an output list.
+pub struct BitWriter:
+    out: List[Int]
+    acc: Int
+    nbits: Int
+
+fn bw_write_bits(w: BitWriter, value: Int, n: Int):
+    w.acc = bits.bor(w.acc, bits.shl(value, w.nbits))
+    w.nbits = w.nbits + n
+    while w.nbits >= 8:
+        w.out.push(bits.band(w.acc, 255))
+        w.acc = bits.shr(w.acc, 8)
+        w.nbits = w.nbits - 8
+
+fn bw_flush(w: BitWriter):
+    if w.nbits > 0:
+        w.out.push(bits.band(w.acc, 255))
+        w.acc = 0
+        w.nbits = 0
+
+# huffman codes transmit most-significant-bit first; the writer is
+# lsb-first, so codes are stored pre-reversed
+fn reverse_bits(value: Int, n: Int) -> Int:
+    mut v := value
+    mut r := 0
+    mut i := 0
+    while i < n:
+        r = bits.bor(bits.shl(r, 1), bits.band(v, 1))
+        v = bits.shr(v, 1)
+        i = i + 1
+    return r
+
+fn write_literal(w: BitWriter, sym: Int):
+    if sym < 144:
+        bw_write_bits(w, reverse_bits(48 + sym, 8), 8)
+    elif sym < 256:
+        bw_write_bits(w, reverse_bits(400 + (sym - 144), 9), 9)
+    elif sym < 280:
+        bw_write_bits(w, reverse_bits(sym - 256, 7), 7)
+    else:
+        bw_write_bits(w, reverse_bits(192 + (sym - 280), 8), 8)
+
+fn length_symbol(length: Int) -> Int:
+    base := [3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258]
+    mut sym := 28
+    while sym > 0 and base[sym] > length:
+        sym = sym - 1
+    return 257 + sym
+
+fn length_extra_bits(sym: Int) -> Int:
+    table := [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0]
+    return table[sym - 257]
+
+fn length_symbol_base(sym: Int) -> Int:
+    base := [3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258]
+    return base[sym - 257]
+
+fn distance_symbol(distance: Int) -> Int:
+    base := [1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577]
+    mut sym := 29
+    while sym > 0 and base[sym] > distance:
+        sym = sym - 1
+    return sym
+
+fn distance_extra_bits(sym: Int) -> Int:
+    table := [0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13]
+    return table[sym]
+
+fn distance_symbol_base(sym: Int) -> Int:
+    base := [1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577]
+    return base[sym]
+
+fn write_match(w: BitWriter, length: Int, distance: Int):
+    lsym := length_symbol(length)
+    write_literal(w, lsym)
+    lextra := length_extra_bits(lsym)
+    if lextra > 0:
+        bw_write_bits(w, length - length_symbol_base(lsym), lextra)
+    dsym := distance_symbol(distance)
+    bw_write_bits(w, reverse_bits(dsym, 5), 5)
+    dextra := distance_extra_bits(dsym)
+    if dextra > 0:
+        bw_write_bits(w, distance - distance_symbol_base(dsym), dextra)
+
+fn hash3(a: Int, b: Int, c: Int) -> Int:
+    return bits.band(a * 2654435 + b * 40503 + c * 809, 32767)
+
+# # compress raw bytes into a single fixed-huffman deflate block.
+pub fn deflate(data: Bytes) -> Bytes!:
+    w := BitWriter([], 0, 0)
+    # final block, fixed huffman
+    bw_write_bits(w, 1, 1)
+    bw_write_bits(w, 1, 2)
+
+    n := data.len()
+    # hash heads and chains over 3-byte prefixes; -1 marks empty
+    mut heads: List[Int] := []
+    mut i := 0
+    while i < 32768:
+        heads.push(0 - 1)
+        i = i + 1
+    mut chain: List[Int] := []
+    i = 0
+    while i < n:
+        chain.push(0 - 1)
+        i = i + 1
+
+    mut pos := 0
+    while pos < n:
+        mut best_len := 0
+        mut best_dist := 0
+        if pos + 2 < n:
+            h := hash3(bytes.get(data, pos), bytes.get(data, pos + 1), bytes.get(data, pos + 2))
+            mut cand := heads[h]
+            mut tries := 0
+            while cand >= 0 and tries < 32:
+                dist := pos - cand
+                if dist > 32768:
+                    cand = 0 - 1
+                else:
+                    mut match_len := 0
+                    limit := n - pos
+                    mut cap := 258
+                    if limit < cap:
+                        cap = limit
+                    while match_len < cap and bytes.get(data, cand + match_len) == bytes.get(data, pos + match_len):
+                        match_len = match_len + 1
+                    if match_len > best_len:
+                        best_len = match_len
+                        best_dist = dist
+                    cand = chain[cand]
+                tries = tries + 1
+            chain[pos] = heads[h]
+            heads[h] = pos
+        if best_len >= 3:
+            write_match(w, best_len, best_dist)
+            # insert hash entries for the skipped positions so later
+            # matches can reference into this run
+            mut j := pos + 1
+            stop := pos + best_len
+            while j < stop and j + 2 < n:
+                hj := hash3(bytes.get(data, j), bytes.get(data, j + 1), bytes.get(data, j + 2))
+                chain[j] = heads[hj]
+                heads[hj] = j
+                j = j + 1
+            pos = pos + best_len
+        else:
+            write_literal(w, bytes.get(data, pos))
+            pos = pos + 1
+
+    write_literal(w, 256)
+    bw_flush(w)
+    buf := bytes.buffer_with_capacity(w.out.len())
+    for b in w.out:
+        buf.write_byte(b)!
+    return buf.take_bytes()

std/compress/gzip.pith

@@ -6,6 +6,7 @@
 
 import std.bytes as bytes
 import std.compress.inflate as inflate
+import std.compress.deflate as deflate
 import std.binary as binary
 import std.bits as bits
 import std.checksum as checksum
@@ -115,60 +116,10 @@ fn gzip_parse_header(data: Bytes) -> Int!:
         pos = gzip_skip(data, pos, 2)!
     return pos
 
-fn gzip_decompress_stored(data: Bytes) -> Bytes!:
-    if data.len() < 18:
-        fail gzip_invalid()
-    mut pos := gzip_parse_header(data)!
-    out := bytes.buffer()
-    deflate_limit := data.len() - 8
-    while true:
-        if pos + 1 > deflate_limit:
-            fail gzip_invalid()
-        block_header := gzip_read_u8_at(data, pos)!
-        pos = pos + 1
-        is_final := bits.band(block_header, 1) != 0
-        block_type := bits.band(bits.shr(block_header, 1), 3)
-        if block_type != 0:
-            fail gzip_invalid()
-        if pos + 4 > deflate_limit:
-            fail gzip_invalid()
-        block_len := gzip_read_u16_le_at(data, pos)!
-        block_nlen := gzip_read_u16_le_at(data, pos + 2)!
-        pos = pos + 4
-        if block_len + block_nlen != GZIP_STORED_BLOCK_SIZE:
-            fail gzip_invalid()
-        if pos + block_len > deflate_limit:
-            fail gzip_invalid()
-        if block_len > 0:
-            out.write(data.slice(pos, pos + block_len))!
-        pos = pos + block_len
-        if is_final:
-            break
-    if pos + 8 != data.len():
-        fail gzip_invalid()
-    expected_crc := gzip_read_u32_le_at(data, pos)!
-    expected_size := gzip_read_u32_le_at(data, pos + 4)!
-    result := out.bytes()
-    if checksum.crc32(result) != expected_crc:
-        fail gzip_invalid()
-    if bits.mask32(result.len()) != expected_size:
-        fail gzip_invalid()
-    return result
-
 fn gzip_compress_result(data: Bytes) -> Bytes!:
     out := bytes.buffer_with_capacity(10 + data.len() + 5 + 8)
     gzip_write_header(out)!
-    if data.len() == 0:
-        gzip_write_stored_block(out, true, data)!
-    else:
-        mut offset := 0
-        while offset < data.len():
-            mut end := offset + GZIP_STORED_BLOCK_SIZE
-            if end > data.len():
-                end = data.len()
-            chunk := data.slice(offset, end)
-            gzip_write_stored_block(out, end == data.len(), chunk)!
-            offset = end
+    out.write(deflate.deflate(data)!)!
     gzip_write_u32_le(out, checksum.crc32(data))!
     gzip_write_u32_le(out, bits.mask32(data.len()))!
     return out.bytes()
@@ -177,9 +128,9 @@ fn gzip_compress_result(data: Bytes) -> Bytes!:
 pub fn compress(data: Bytes) -> Bytes:
     return gzip_compress_result(data) catch bytes.empty()
 
-# # decompress a gzip stream: any deflate block type — stored, fixed
-# # huffman, or dynamic huffman — so files produced by other tools read
-# # correctly. the crc32 and length trailer are verified.
+# # decompress a gzip stream: stored, fixed-huffman, and
+# # dynamic-huffman blocks all read, with the crc32 and length trailer
+# # verified — files from other tools decompress correctly.
 pub fn decompress(data: Bytes) -> Bytes!:
     if data.len() < 18:
         fail gzip_invalid()
@@ -195,7 +146,7 @@ pub fn decompress(data: Bytes) -> Bytes!:
 
 # true when the data is a gzip stream this module can read.
 pub fn is_valid(data: Bytes) -> Bool:
-    result := gzip_decompress_stored(data)
+    result := decompress(data)
     return result.is_ok
 
 # compress utf-8 text into gzip bytes.
@@ -220,7 +171,8 @@ test "gzip round trips bytes and strings":
     data := bytes.from_string_utf8("pith gzip data")
     compressed := compress(data)
 
-    assert(compressed.len() > data.len())
+    # tiny inputs carry header overhead; the payload itself encodes
+    # with real compression now, so no size relation is guaranteed here
     assert(is_valid(compressed))
     assert_eq(decompress(compressed)!.to_string_utf8()!, "pith gzip data")
     assert_eq(decompress_string(compress_string("hello gzip"))!, "hello gzip")

tests/cases/test_gzip_interop.pith

@@ -7,6 +7,11 @@ fn main() -> Int!:
     raw := fs.read_bytes("tests/cases/gzip_interop_fixture.gz")!
     text := gzip.decompress(raw)!.to_string_utf8()!
     print(text)
-    round := gzip.decompress(gzip.compress_string("stored-block round trip"))!
+    round := gzip.decompress(gzip.compress_string("compressor round trip"))!
     print(round.to_string_utf8()!)
+    big := fs.read_bytes("tests/cases/test_combo_containers.pith")!
+    packed := gzip.compress(big)
+    print("shrinks: {packed.len() < big.len()}")
+    back := gzip.decompress(packed)!
+    print("restores: {back.len() == big.len()}")
     return 0

tests/expected/test_gzip_interop.txt

@@ -1,4 +1,6 @@
 gzip interop fixture: dynamic huffman, back-references galore.
 repeat repeat repeat repeat repeat repeat repeat repeat.
 
-stored-block round trip
+compressor round trip
+shrinks: true
+restores: true