diff --git a/Makefile b/Makefile index 2cbf9451..b615b354 100644 --- a/Makefile +++ b/Makefile @@ -628,7 +628,8 @@ logscan-check: ./tools/logscan/logscan tools/logscan/sample/access.log "$$tmpdir/paths.csv" | grep -v "^csv written" > "$$tmpdir/report.txt"; \ diff -u tools/logscan/expected/report.txt "$$tmpdir/report.txt" && \ diff -u tools/logscan/expected/paths.csv "$$tmpdir/paths.csv" && \ - ./tools/logscan/logscan tools/logscan/sample/access.log.gz | grep -v "^csv written" > "$$tmpdir/report_gz.txt" && \ + gzip -c tools/logscan/sample/access.log > "$$tmpdir/sys.gz" && \ + ./tools/logscan/logscan "$$tmpdir/sys.gz" | grep -v "^csv written" > "$$tmpdir/report_gz.txt" && \ diff -u tools/logscan/expected/report.txt "$$tmpdir/report_gz.txt" && \ echo "logscan output matches golden files" diff --git a/README.md b/README.md index 9216c89f..53f3f0a6 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ that path. - std.os.path, std.os.process, std.fs, std.glob — files, paths, and file discovery - std.cli, std.diagnostic, std.testing, std.text.scanner — small tooling layers - std.log, std.metrics, std.fmt, std.math, std.rand, std.time, std.datetime, std.uuid — common app helpers -- std.compress (gzip/zlib containers, stored blocks only — see limitations), std.archive (tar/zip), std.net.sse, std.data.table — archives, server-sent events, and tabular data +- std.compress (gzip reads any deflate stream — dynamic huffman included — via an inflate engine written in pith; compression still writes stored blocks), std.archive (tar/zip), std.net.sse, std.data.table — archives, server-sent events, and tabular data - std.regex — a pike-vm regular expression engine written in pith itself: linear-time matching with no pathological backtracking, capture groups, classes, alternation, anchors (no {n,m}, lazy quantifiers, backreferences, or lookaround) for child processes, prefer `std.os.process.command(...)` and the structured diff --git a/docs/limitations.md b/docs/limitations.md index 8b264737..80b5c62a 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -49,10 +49,11 @@ something here that now works, the page is stale and a fix to it is welcome. groups, and `^ $` anchors. it does not support `{n,m}` counts, lazy quantifiers, backreferences, or lookaround. matching is a pike vm, so time is linear in the input for any pattern. -- **gzip is stored-blocks only** — `std.compress.gzip` round-trips its own - output but implements no huffman coding: its "compressed" data is larger - than the input, and it cannot read gzip files produced by other tools. - full deflate is an open stdlib project, alongside regex. +- **gzip compression writes stored blocks** — `std.compress.gzip` now + reads any deflate stream (stored, fixed, and dynamic huffman — files + from other tools decompress correctly, crc-verified), but its own + compress() still emits stored blocks, so pith-written .gz files are + larger than their input. a real compressor is the remaining half. ## tooling diff --git a/std/compress/gzip.pith b/std/compress/gzip.pith index b6d96373..2a9f3bbd 100644 --- a/std/compress/gzip.pith +++ b/std/compress/gzip.pith @@ -5,6 +5,7 @@ # slice favors compatibility over smaller files. import std.bytes as bytes +import std.compress.inflate as inflate import std.binary as binary import std.bits as bits import std.checksum as checksum @@ -176,9 +177,21 @@ fn gzip_compress_result(data: Bytes) -> Bytes!: pub fn compress(data: Bytes) -> Bytes: return gzip_compress_result(data) catch bytes.empty() -# decompress a gzip stream created with stored deflate blocks. +# # decompress a gzip stream: any deflate block type — stored, fixed +# # huffman, or dynamic huffman — so files produced by other tools read +# # correctly. the crc32 and length trailer are verified. pub fn decompress(data: Bytes) -> Bytes!: - return gzip_decompress_stored(data) + if data.len() < 18: + fail gzip_invalid() + start := gzip_parse_header(data)! + payload := inflate.inflate(data, start)! + expected_crc := gzip_read_u32_le_at(data, data.len() - 8)! + expected_len := gzip_read_u32_le_at(data, data.len() - 4)! + if payload.len() != expected_len: + fail "gzip: length trailer mismatch" + if checksum.crc32(payload) != expected_crc: + fail "gzip: crc mismatch" + return payload # true when the data is a gzip stream this module can read. pub fn is_valid(data: Bytes) -> Bool: diff --git a/std/compress/inflate.pith b/std/compress/inflate.pith new file mode 100644 index 00000000..fe85eee3 --- /dev/null +++ b/std/compress/inflate.pith @@ -0,0 +1,232 @@ +# std/compress/inflate — the deflate decompressor (rfc 1951). +# +# a bit reader walks the stream lsb-first; huffman codes decode +# bit-by-bit against canonical first-code tables (no lookup tables to +# build, a few comparisons per symbol); back-references copy from a +# List[Int] window that doubles as the output. +import std.bits as bits +import std.bytes as bytes + +# # reader state over the deflate stream: byte position and bit offset. +pub struct BitReader: + data: Bytes + pos: Int + bit: Int + +# # a canonical huffman decoder: per-length symbol counts and the +# # symbols ordered by (length, symbol) — enough to decode bit-by-bit. +pub struct Huffman: + counts: List[Int] + symbols: List[Int] + +fn br_read_bit(r: BitReader) -> Int!: + if r.pos >= r.data.len(): + fail "inflate: unexpected end of stream" + b := bytes.get(r.data, r.pos) + v := bits.band(bits.shr(b, r.bit), 1) + r.bit = r.bit + 1 + if r.bit == 8: + r.bit = 0 + r.pos = r.pos + 1 + return v + +fn br_read_bits(r: BitReader, n: Int) -> Int!: + mut v := 0 + mut i := 0 + while i < n: + v = bits.bor(v, bits.shl(br_read_bit(r)!, i)) + i = i + 1 + return v + +fn br_align(r: BitReader): + if r.bit != 0: + r.bit = 0 + r.pos = r.pos + 1 + +# build a canonical decoder from code lengths (0 = unused symbol) +fn huffman_from_lengths(lengths: List[Int]) -> Huffman: + mut counts: List[Int] := [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + for l in lengths: + if l > 0: + counts[l] = counts[l] + 1 + mut symbols: List[Int] := [] + mut length := 1 + while length < 16: + mut sym := 0 + while sym < lengths.len(): + if lengths[sym] == length: + symbols.push(sym) + sym = sym + 1 + length = length + 1 + return Huffman(counts, symbols) + +fn huffman_decode(r: BitReader, h: Huffman) -> Int!: + mut code := 0 + mut first := 0 + mut index := 0 + mut length := 1 + while length < 16: + code = bits.bor(code, br_read_bit(r)!) + count := h.counts[length] + if code - first < count: + return h.symbols[index + (code - first)] + index = index + count + first = bits.shl(first + count, 1) + code = bits.shl(code, 1) + length = length + 1 + fail "inflate: invalid huffman code" + +# length and distance tables (rfc 1951 section 3.2.5) +fn length_base(sym: Int) -> Int: + table := [3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258] + return table[sym - 257] + +fn length_extra(sym: Int) -> Int: + table := [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0] + return table[sym - 257] + +fn dist_base(sym: Int) -> Int: + table := [1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577] + return table[sym] + +fn dist_extra(sym: Int) -> Int: + table := [0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13] + return table[sym] + +fn fixed_literal_huffman() -> Huffman: + mut lengths: List[Int] := [] + mut i := 0 + while i < 288: + if i < 144: + lengths.push(8) + elif i < 256: + lengths.push(9) + elif i < 280: + lengths.push(7) + else: + lengths.push(8) + i = i + 1 + return huffman_from_lengths(lengths) + +fn fixed_distance_huffman() -> Huffman: + mut lengths: List[Int] := [] + mut i := 0 + while i < 30: + lengths.push(5) + i = i + 1 + return huffman_from_lengths(lengths) + +# the code-length alphabet arrives in this fixed order +fn clen_order(i: Int) -> Int: + order := [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15] + return order[i] + +struct HuffmanPair: + lit: Huffman + dist: Huffman + +fn read_dynamic_huffmans(r: BitReader) -> HuffmanPair!: + hlit := br_read_bits(r, 5)! + 257 + hdist := br_read_bits(r, 5)! + 1 + hclen := br_read_bits(r, 4)! + 4 + mut clen_lengths: List[Int] := [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + mut i := 0 + while i < hclen: + clen_lengths[clen_order(i)] = br_read_bits(r, 3)! + i = i + 1 + clen_tree := huffman_from_lengths(clen_lengths) + + # literal and distance lengths share one run-length-coded sequence + mut lengths: List[Int] := [] + while lengths.len() < hlit + hdist: + sym := huffman_decode(r, clen_tree)! + if sym < 16: + lengths.push(sym) + elif sym == 16: + if lengths.len() == 0: + fail "inflate: repeat with no previous length" + prev := lengths[lengths.len() - 1] + mut rep := br_read_bits(r, 2)! + 3 + while rep > 0: + lengths.push(prev) + rep = rep - 1 + elif sym == 17: + mut rep := br_read_bits(r, 3)! + 3 + while rep > 0: + lengths.push(0) + rep = rep - 1 + else: + mut rep := br_read_bits(r, 7)! + 11 + while rep > 0: + lengths.push(0) + rep = rep - 1 + mut lit_lengths: List[Int] := [] + mut dist_lengths: List[Int] := [] + i = 0 + while i < hlit: + lit_lengths.push(lengths[i]) + i = i + 1 + while i < hlit + hdist: + dist_lengths.push(lengths[i]) + i = i + 1 + return HuffmanPair(huffman_from_lengths(lit_lengths), huffman_from_lengths(dist_lengths)) + +fn inflate_block(r: BitReader, out: List[Int], lit: Huffman, dist: Huffman) -> Bool!: + while true: + sym := huffman_decode(r, lit)! + if sym < 256: + out.push(sym) + elif sym == 256: + return true + else: + if sym > 285: + fail "inflate: invalid length symbol" + mut length := length_base(sym) + br_read_bits(r, length_extra(sym))! + dsym := huffman_decode(r, dist)! + if dsym > 29: + fail "inflate: invalid distance symbol" + distance := dist_base(dsym) + br_read_bits(r, dist_extra(dsym))! + if distance > out.len(): + fail "inflate: distance beyond output" + mut src := out.len() - distance + while length > 0: + out.push(out[src]) + src = src + 1 + length = length - 1 + return true + +# # decompress a raw deflate stream starting at `start`; returns the +# # payload and leaves nothing implicit — bad streams fail with a +# # message naming what was malformed. +pub fn inflate(data: Bytes, start: Int) -> Bytes!: + r := BitReader(data, start, 0) + mut out: List[Int] := [] + mut final_block := false + while not final_block: + final_block = br_read_bits(r, 1)! == 1 + btype := br_read_bits(r, 2)! + if btype == 0: + br_align(r) + if r.pos + 4 > data.len(): + fail "inflate: truncated stored block" + block_len := bytes.get(data, r.pos) + bits.shl(bytes.get(data, r.pos + 1), 8) + nlen := bytes.get(data, r.pos + 2) + bits.shl(bytes.get(data, r.pos + 3), 8) + if bits.band(block_len + nlen, 65535) != 65535: + fail "inflate: stored block length check failed" + r.pos = r.pos + 4 + mut i := 0 + while i < block_len: + out.push(bytes.get(data, r.pos + i)) + i = i + 1 + r.pos = r.pos + block_len + elif btype == 1: + inflate_block(r, out, fixed_literal_huffman(), fixed_distance_huffman())! + elif btype == 2: + trees := read_dynamic_huffmans(r)! + inflate_block(r, out, trees.lit, trees.dist)! + else: + fail "inflate: reserved block type" + buf := bytes.buffer_with_capacity(out.len()) + for b in out: + buf.write_byte(b)! + return buf.take_bytes() diff --git a/tests/cases/gzip_interop_fixture.gz b/tests/cases/gzip_interop_fixture.gz new file mode 100644 index 00000000..b1d50319 Binary files /dev/null and b/tests/cases/gzip_interop_fixture.gz differ diff --git a/tests/cases/test_gzip_interop.pith b/tests/cases/test_gzip_interop.pith new file mode 100644 index 00000000..8b77d68c --- /dev/null +++ b/tests/cases/test_gzip_interop.pith @@ -0,0 +1,12 @@ +# the committed fixture was produced by system gzip -9 (dynamic +# huffman blocks); reading it proves interop with the outside world. +import std.fs as fs +import std.compress.gzip as gzip + +fn main() -> Int!: + raw := fs.read_bytes("tests/cases/gzip_interop_fixture.gz")! + text := gzip.decompress(raw)!.to_string_utf8()! + print(text) + round := gzip.decompress(gzip.compress_string("stored-block round trip"))! + print(round.to_string_utf8()!) + return 0 diff --git a/tests/expected/test_gzip_interop.txt b/tests/expected/test_gzip_interop.txt new file mode 100644 index 00000000..3818960f --- /dev/null +++ b/tests/expected/test_gzip_interop.txt @@ -0,0 +1,4 @@ +gzip interop fixture: dynamic huffman, back-references galore. +repeat repeat repeat repeat repeat repeat repeat repeat. + +stored-block round trip