diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc557b6c..9152c30e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,9 @@ jobs: - name: parq golden check run: make parq-check + - name: gzip interop check + run: make gzip-interop-check + - name: verify bootstrap fixed point run: make bootstrap-verify diff --git a/Makefile b/Makefile index b615b354..845ae4b0 100644 --- a/Makefile +++ b/Makefile @@ -670,6 +670,19 @@ parq-check: diff -u tools/parq/expected/report.txt "$$tmpdir/report1.txt" && \ echo "parq output matches golden files" +# --- gzip interop check --- +# both directions against the system tool: pith reads gzip's output +# (covered in logscan-check too) and gzip reads pith's + +gzip-interop-check: + @echo "--- gzip interop check ---" + @tmpdir=$$(mktemp -d /tmp/pith-gzip-XXXXXX); \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + printf 'import std.fs as fs\nimport std.compress.gzip as gzip\n\nfn main() -> Int!:\n raw := fs.read_bytes("README.md")!\n fs.write_bytes("'"$$tmpdir"'/out.gz", gzip.compress(raw))!\n return 0\n' > "$$tmpdir/pack.pith"; \ + ./target/release/pith run "$$tmpdir/pack.pith" > /dev/null 2>&1 && \ + gunzip -c "$$tmpdir/out.gz" | cmp - README.md && \ + echo "system gunzip reads pith output byte-identical" + # --- cli regressions --- cli-regressions: build cli-regressions-only diff --git a/README.md b/README.md index 53f3f0a6..79f5f490 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ that path. - std.os.path, std.os.process, std.fs, std.glob — files, paths, and file discovery - std.cli, std.diagnostic, std.testing, std.text.scanner — small tooling layers - std.log, std.metrics, std.fmt, std.math, std.rand, std.time, std.datetime, std.uuid — common app helpers -- std.compress (gzip reads any deflate stream — dynamic huffman included — via an inflate engine written in pith; compression still writes stored blocks), std.archive (tar/zip), std.net.sse, std.data.table — archives, server-sent events, and tabular data +- std.compress (full gzip in pith: the inflate engine reads any deflate stream, and compression is lz77 over fixed huffman — interops with system gzip in both directions), std.archive (tar/zip), std.net.sse, std.data.table — archives, server-sent events, and tabular data - std.regex — a pike-vm regular expression engine written in pith itself: linear-time matching with no pathological backtracking, capture groups, classes, alternation, anchors (no {n,m}, lazy quantifiers, backreferences, or lookaround) for child processes, prefer `std.os.process.command(...)` and the structured diff --git a/docs/limitations.md b/docs/limitations.md index 80b5c62a..4f29b7a7 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -49,11 +49,11 @@ something here that now works, the page is stale and a fix to it is welcome. groups, and `^ $` anchors. it does not support `{n,m}` counts, lazy quantifiers, backreferences, or lookaround. matching is a pike vm, so time is linear in the input for any pattern. -- **gzip compression writes stored blocks** — `std.compress.gzip` now - reads any deflate stream (stored, fixed, and dynamic huffman — files - from other tools decompress correctly, crc-verified), but its own - compress() still emits stored blocks, so pith-written .gz files are - larger than their input. a real compressor is the remaining half. +- **gzip compresses with fixed huffman only** — `std.compress.gzip` + reads any deflate stream and writes real compression (greedy lz77 + over fixed huffman codes; system gunzip reads its output). dynamic + huffman trees would shave a few more percent and are not + implemented. ## tooling diff --git a/std/compress/deflate.pith b/std/compress/deflate.pith new file mode 100644 index 00000000..9e70608b --- /dev/null +++ b/std/compress/deflate.pith @@ -0,0 +1,168 @@ +# std/compress/deflate — the deflate compressor (rfc 1951), fixed +# huffman blocks over greedy lz77. +# +# a hash chain over 3-byte prefixes finds back-references (longest +# match wins, up to 258 bytes within a 32k window); the token stream +# encodes with the fixed huffman tables, so no tree serialization is +# needed. dynamic huffman would shave a few more percent and is left +# for later — this already produces real compression that any inflate +# implementation reads. +import std.bits as bits +import std.bytes as bytes + +# # bit writer state: bits accumulate lsb-first into an output list. +pub struct BitWriter: + out: List[Int] + acc: Int + nbits: Int + +fn bw_write_bits(w: BitWriter, value: Int, n: Int): + w.acc = bits.bor(w.acc, bits.shl(value, w.nbits)) + w.nbits = w.nbits + n + while w.nbits >= 8: + w.out.push(bits.band(w.acc, 255)) + w.acc = bits.shr(w.acc, 8) + w.nbits = w.nbits - 8 + +fn bw_flush(w: BitWriter): + if w.nbits > 0: + w.out.push(bits.band(w.acc, 255)) + w.acc = 0 + w.nbits = 0 + +# huffman codes transmit most-significant-bit first; the writer is +# lsb-first, so codes are stored pre-reversed +fn reverse_bits(value: Int, n: Int) -> Int: + mut v := value + mut r := 0 + mut i := 0 + while i < n: + r = bits.bor(bits.shl(r, 1), bits.band(v, 1)) + v = bits.shr(v, 1) + i = i + 1 + return r + +fn write_literal(w: BitWriter, sym: Int): + if sym < 144: + bw_write_bits(w, reverse_bits(48 + sym, 8), 8) + elif sym < 256: + bw_write_bits(w, reverse_bits(400 + (sym - 144), 9), 9) + elif sym < 280: + bw_write_bits(w, reverse_bits(sym - 256, 7), 7) + else: + bw_write_bits(w, reverse_bits(192 + (sym - 280), 8), 8) + +fn length_symbol(length: Int) -> Int: + base := [3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258] + mut sym := 28 + while sym > 0 and base[sym] > length: + sym = sym - 1 + return 257 + sym + +fn length_extra_bits(sym: Int) -> Int: + table := [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0] + return table[sym - 257] + +fn length_symbol_base(sym: Int) -> Int: + base := [3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258] + return base[sym - 257] + +fn distance_symbol(distance: Int) -> Int: + base := [1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577] + mut sym := 29 + while sym > 0 and base[sym] > distance: + sym = sym - 1 + return sym + +fn distance_extra_bits(sym: Int) -> Int: + table := [0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13] + return table[sym] + +fn distance_symbol_base(sym: Int) -> Int: + base := [1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577] + return base[sym] + +fn write_match(w: BitWriter, length: Int, distance: Int): + lsym := length_symbol(length) + write_literal(w, lsym) + lextra := length_extra_bits(lsym) + if lextra > 0: + bw_write_bits(w, length - length_symbol_base(lsym), lextra) + dsym := distance_symbol(distance) + bw_write_bits(w, reverse_bits(dsym, 5), 5) + dextra := distance_extra_bits(dsym) + if dextra > 0: + bw_write_bits(w, distance - distance_symbol_base(dsym), dextra) + +fn hash3(a: Int, b: Int, c: Int) -> Int: + return bits.band(a * 2654435 + b * 40503 + c * 809, 32767) + +# # compress raw bytes into a single fixed-huffman deflate block. +pub fn deflate(data: Bytes) -> Bytes!: + w := BitWriter([], 0, 0) + # final block, fixed huffman + bw_write_bits(w, 1, 1) + bw_write_bits(w, 1, 2) + + n := data.len() + # hash heads and chains over 3-byte prefixes; -1 marks empty + mut heads: List[Int] := [] + mut i := 0 + while i < 32768: + heads.push(0 - 1) + i = i + 1 + mut chain: List[Int] := [] + i = 0 + while i < n: + chain.push(0 - 1) + i = i + 1 + + mut pos := 0 + while pos < n: + mut best_len := 0 + mut best_dist := 0 + if pos + 2 < n: + h := hash3(bytes.get(data, pos), bytes.get(data, pos + 1), bytes.get(data, pos + 2)) + mut cand := heads[h] + mut tries := 0 + while cand >= 0 and tries < 32: + dist := pos - cand + if dist > 32768: + cand = 0 - 1 + else: + mut match_len := 0 + limit := n - pos + mut cap := 258 + if limit < cap: + cap = limit + while match_len < cap and bytes.get(data, cand + match_len) == bytes.get(data, pos + match_len): + match_len = match_len + 1 + if match_len > best_len: + best_len = match_len + best_dist = dist + cand = chain[cand] + tries = tries + 1 + chain[pos] = heads[h] + heads[h] = pos + if best_len >= 3: + write_match(w, best_len, best_dist) + # insert hash entries for the skipped positions so later + # matches can reference into this run + mut j := pos + 1 + stop := pos + best_len + while j < stop and j + 2 < n: + hj := hash3(bytes.get(data, j), bytes.get(data, j + 1), bytes.get(data, j + 2)) + chain[j] = heads[hj] + heads[hj] = j + j = j + 1 + pos = pos + best_len + else: + write_literal(w, bytes.get(data, pos)) + pos = pos + 1 + + write_literal(w, 256) + bw_flush(w) + buf := bytes.buffer_with_capacity(w.out.len()) + for b in w.out: + buf.write_byte(b)! + return buf.take_bytes() diff --git a/std/compress/gzip.pith b/std/compress/gzip.pith index 2a9f3bbd..a24e04ff 100644 --- a/std/compress/gzip.pith +++ b/std/compress/gzip.pith @@ -6,6 +6,7 @@ import std.bytes as bytes import std.compress.inflate as inflate +import std.compress.deflate as deflate import std.binary as binary import std.bits as bits import std.checksum as checksum @@ -115,60 +116,10 @@ fn gzip_parse_header(data: Bytes) -> Int!: pos = gzip_skip(data, pos, 2)! return pos -fn gzip_decompress_stored(data: Bytes) -> Bytes!: - if data.len() < 18: - fail gzip_invalid() - mut pos := gzip_parse_header(data)! - out := bytes.buffer() - deflate_limit := data.len() - 8 - while true: - if pos + 1 > deflate_limit: - fail gzip_invalid() - block_header := gzip_read_u8_at(data, pos)! - pos = pos + 1 - is_final := bits.band(block_header, 1) != 0 - block_type := bits.band(bits.shr(block_header, 1), 3) - if block_type != 0: - fail gzip_invalid() - if pos + 4 > deflate_limit: - fail gzip_invalid() - block_len := gzip_read_u16_le_at(data, pos)! - block_nlen := gzip_read_u16_le_at(data, pos + 2)! - pos = pos + 4 - if block_len + block_nlen != GZIP_STORED_BLOCK_SIZE: - fail gzip_invalid() - if pos + block_len > deflate_limit: - fail gzip_invalid() - if block_len > 0: - out.write(data.slice(pos, pos + block_len))! - pos = pos + block_len - if is_final: - break - if pos + 8 != data.len(): - fail gzip_invalid() - expected_crc := gzip_read_u32_le_at(data, pos)! - expected_size := gzip_read_u32_le_at(data, pos + 4)! - result := out.bytes() - if checksum.crc32(result) != expected_crc: - fail gzip_invalid() - if bits.mask32(result.len()) != expected_size: - fail gzip_invalid() - return result - fn gzip_compress_result(data: Bytes) -> Bytes!: out := bytes.buffer_with_capacity(10 + data.len() + 5 + 8) gzip_write_header(out)! - if data.len() == 0: - gzip_write_stored_block(out, true, data)! - else: - mut offset := 0 - while offset < data.len(): - mut end := offset + GZIP_STORED_BLOCK_SIZE - if end > data.len(): - end = data.len() - chunk := data.slice(offset, end) - gzip_write_stored_block(out, end == data.len(), chunk)! - offset = end + out.write(deflate.deflate(data)!)! gzip_write_u32_le(out, checksum.crc32(data))! gzip_write_u32_le(out, bits.mask32(data.len()))! return out.bytes() @@ -177,9 +128,9 @@ fn gzip_compress_result(data: Bytes) -> Bytes!: pub fn compress(data: Bytes) -> Bytes: return gzip_compress_result(data) catch bytes.empty() -# # decompress a gzip stream: any deflate block type — stored, fixed -# # huffman, or dynamic huffman — so files produced by other tools read -# # correctly. the crc32 and length trailer are verified. +# # decompress a gzip stream: stored, fixed-huffman, and +# # dynamic-huffman blocks all read, with the crc32 and length trailer +# # verified — files from other tools decompress correctly. pub fn decompress(data: Bytes) -> Bytes!: if data.len() < 18: fail gzip_invalid() @@ -195,7 +146,7 @@ pub fn decompress(data: Bytes) -> Bytes!: # true when the data is a gzip stream this module can read. pub fn is_valid(data: Bytes) -> Bool: - result := gzip_decompress_stored(data) + result := decompress(data) return result.is_ok # compress utf-8 text into gzip bytes. @@ -220,7 +171,8 @@ test "gzip round trips bytes and strings": data := bytes.from_string_utf8("pith gzip data") compressed := compress(data) - assert(compressed.len() > data.len()) + # tiny inputs carry header overhead; the payload itself encodes + # with real compression now, so no size relation is guaranteed here assert(is_valid(compressed)) assert_eq(decompress(compressed)!.to_string_utf8()!, "pith gzip data") assert_eq(decompress_string(compress_string("hello gzip"))!, "hello gzip") diff --git a/tests/cases/test_gzip_interop.pith b/tests/cases/test_gzip_interop.pith index 8b77d68c..bcc82bf8 100644 --- a/tests/cases/test_gzip_interop.pith +++ b/tests/cases/test_gzip_interop.pith @@ -7,6 +7,11 @@ fn main() -> Int!: raw := fs.read_bytes("tests/cases/gzip_interop_fixture.gz")! text := gzip.decompress(raw)!.to_string_utf8()! print(text) - round := gzip.decompress(gzip.compress_string("stored-block round trip"))! + round := gzip.decompress(gzip.compress_string("compressor round trip"))! print(round.to_string_utf8()!) + big := fs.read_bytes("tests/cases/test_combo_containers.pith")! + packed := gzip.compress(big) + print("shrinks: {packed.len() < big.len()}") + back := gzip.decompress(packed)! + print("restores: {back.len() == big.len()}") return 0 diff --git a/tests/expected/test_gzip_interop.txt b/tests/expected/test_gzip_interop.txt index 3818960f..051061fc 100644 --- a/tests/expected/test_gzip_interop.txt +++ b/tests/expected/test_gzip_interop.txt @@ -1,4 +1,6 @@ gzip interop fixture: dynamic huffman, back-references galore. repeat repeat repeat repeat repeat repeat repeat repeat. -stored-block round trip +compressor round trip +shrinks: true +restores: true