From fc6c4b3db708bd8a8bcbaf701a8d0e18115c5a74 Mon Sep 17 00:00:00 2001 From: WonderBeat Date: Tue, 30 Sep 2025 17:35:30 +0200 Subject: [PATCH 1/7] Update xorfilter to use rawRemap and improve test assertions - ZIG 0.15 compatible - Replace std.fmt.format with writer.print for better performance - Add rawRemap function to MeasuredAllocator to support memory remapping - Update xorTest to calculate expected size instead of hardcoding it - Switch Zig setup in CI to use mlugg/setup-zig action --- .github/workflows/ci.yml | 7 +++---- build.zig | 10 ++++++---- build.zig.zon | 15 +++++++++++++++ src/MeasuredAllocator.zig | 12 +++++++++--- src/benchmark.zig | 25 +++++++++++++++---------- src/xorfilter.zig | 33 ++++++++++++++++++++------------- 6 files changed, 68 insertions(+), 34 deletions(-) create mode 100644 build.zig.zon diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b02421f..a3d43332 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,10 +8,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: Setup Zig - run: | - sudo apt install xz-utils - sudo sh -c 'wget -c https://pkg.machengine.org/zig/zig-linux-x86_64-0.14.0-dev.2577+271452d22.tar.xz -O - | tar -xJ --strip-components=1 -C /usr/local/bin' + - uses: mlugg/setup-zig@v2 + with: + version: 0.15.1 - name: build run: zig build - name: test diff --git a/build.zig b/build.zig index 314ae828..b78d3def 100644 --- a/build.zig +++ b/build.zig @@ -6,13 +6,15 @@ pub fn build(b: *std.Build) void { _ = b.addModule("fastfilter", .{ .root_source_file = b.path("src/main.zig"), + .optimize = optimize, }); const main_tests = b.addTest(.{ - .name = "fastfilter-tests", - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, + .root_module = b.addModule("test", .{ + .root_source_file = b.path("src/main.zig"), + .target = target, + .optimize = optimize, + }), }); const run_main_tests = b.addRunArtifact(main_tests); diff --git a/build.zig.zon b/build.zig.zon new file mode 100644 index 00000000..9cd957d1 --- /dev/null +++ b/build.zig.zon @@ -0,0 +1,15 @@ +.{ + .name = .fastfilter, + .version = "1.0.1", + + .dependencies = .{}, + .fingerprint = 0x809515533cb3d42b, + + .minimum_zig_version = "0.15.1", + .paths = .{ + "build.zig", + "build.zig.zon", + "src/main.zig", + "src", + }, +} diff --git a/src/MeasuredAllocator.zig b/src/MeasuredAllocator.zig index ebf07a6d..25bd7664 100644 --- a/src/MeasuredAllocator.zig +++ b/src/MeasuredAllocator.zig @@ -37,11 +37,17 @@ pub fn allocator(self: *MeasuredAllocator) Allocator { .alloc = allocFn, .resize = resizeFn, .free = freeFn, + .remap = remapFn, }, }; } -fn allocFn(ptr: *anyopaque, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8 { +fn remapFn(ptr: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + const self = @as(*MeasuredAllocator, @ptrCast(@alignCast(ptr))); + return self.parent_allocator.rawRemap(memory, alignment, new_len, ret_addr); +} + +fn allocFn(ptr: *anyopaque, len: usize, ptr_align: mem.Alignment, ret_addr: usize) ?[*]u8 { const self = @as(*MeasuredAllocator, @ptrCast(@alignCast(ptr))); const result = self.parent_allocator.rawAlloc(len, ptr_align, ret_addr); if (result) |_| { @@ -51,7 +57,7 @@ fn allocFn(ptr: *anyopaque, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8 { return result; } -fn resizeFn(ptr: *anyopaque, buf: []u8, buf_align: u8, new_len: usize, ret_addr: usize) bool { +fn resizeFn(ptr: *anyopaque, buf: []u8, buf_align: mem.Alignment, new_len: usize, ret_addr: usize) bool { const self = @as(*MeasuredAllocator, @ptrCast(@alignCast(ptr))); if (self.parent_allocator.rawResize(buf, buf_align, new_len, ret_addr)) { self.state.current_memory_usage_bytes -= buf.len - new_len; @@ -61,7 +67,7 @@ fn resizeFn(ptr: *anyopaque, buf: []u8, buf_align: u8, new_len: usize, ret_addr: return false; } -fn freeFn(ptr: *anyopaque, buf: []u8, buf_align: u8, ret_addr: usize) void { +fn freeFn(ptr: *anyopaque, buf: []u8, buf_align: mem.Alignment, ret_addr: usize) void { const self = @as(*MeasuredAllocator, @ptrCast(@alignCast(ptr))); self.parent_allocator.rawFree(buf, buf_align, ret_addr); self.state.current_memory_usage_bytes -= buf.len; diff --git a/src/benchmark.zig b/src/benchmark.zig index f0b0384b..abdf803c 100644 --- a/src/benchmark.zig +++ b/src/benchmark.zig @@ -9,18 +9,18 @@ const MeasuredAllocator = @import("MeasuredAllocator.zig"); fn formatTime(writer: anytype, comptime spec: []const u8, start: u64, end: u64, division: usize) !void { const ns = @as(f64, @floatFromInt((end - start) / division)); if (ns <= time.ns_per_ms) { - try std.fmt.format(writer, spec, .{ ns, "ns " }); + try writer.print(spec, .{ ns, "ns " }); return; } if (ns <= time.ns_per_s) { - try std.fmt.format(writer, spec, .{ ns / @as(f64, @floatFromInt(time.ns_per_ms)), "ms " }); + try writer.print(spec, .{ ns / @as(f64, @floatFromInt(time.ns_per_ms)), "ms " }); return; } if (ns <= time.ns_per_min) { - try std.fmt.format(writer, spec, .{ ns / @as(f64, @floatFromInt(time.ns_per_s)), "s " }); + try writer.print(spec, .{ ns / @as(f64, @floatFromInt(time.ns_per_s)), "s " }); return; } - try std.fmt.format(writer, spec, .{ ns / @as(f64, @floatFromInt(time.ns_per_min)), "min" }); + try writer.print(spec, .{ ns / @as(f64, @floatFromInt(time.ns_per_min)), "min" }); return; } @@ -29,17 +29,17 @@ fn formatBytes(writer: anytype, comptime spec: []const u8, bytes: u64) !void { const mib = 1024 * kib; const gib = 1024 * mib; if (bytes < kib) { - try std.fmt.format(writer, spec, .{ bytes, "B " }); + try writer.print(spec, .{ bytes, "B " }); } if (bytes < mib) { - try std.fmt.format(writer, spec, .{ bytes / kib, "KiB" }); + try writer.print(spec, .{ bytes / kib, "KiB" }); return; } if (bytes < gib) { - try std.fmt.format(writer, spec, .{ bytes / mib, "MiB" }); + try writer.print(spec, .{ bytes / mib, "MiB" }); return; } - try std.fmt.format(writer, spec, .{ bytes / gib, "GiB" }); + try writer.print(spec, .{ bytes / gib, "GiB" }); return; } @@ -52,7 +52,6 @@ fn bench(algorithm: []const u8, Filter: anytype, size: usize, trials: usize) !vo var buildMA = MeasuredAllocator.init(allocator); const buildAllocator = buildMA.allocator(); - const stdout = std.io.getStdOut().writer(); var timer = try Timer.start(); // Initialize filter. @@ -97,6 +96,10 @@ fn bench(algorithm: []const u8, Filter: anytype, size: usize, trials: usize) !vo @panic("sizeInBytes reporting wrong numbers?"); } + var stdout_buffer: [1024]u8 = undefined; + var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); + const stdout = &stdout_writer.interface; + try stdout.print("| {s: <12} ", .{algorithm}); try stdout.print("| {: <10} ", .{keys.len}); try stdout.print("| ", .{}); @@ -139,7 +142,9 @@ pub fn main() !void { } } - const stdout = std.io.getStdOut().writer(); + var stdout_buffer: [1024]u8 = undefined; + var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); + const stdout = &stdout_writer.interface; try stdout.print("| Algorithm | # of keys | populate | contains(k) | false+ prob. | bits per entry | peak populate | filter total |\n", .{}); try stdout.print("|--------------|------------|------------|-------------|--------------|----------------|---------------|--------------|\n", .{}); try bench("binaryfuse8", xorfilter.BinaryFuse(u8), 1_000_000, num_trials); diff --git a/src/xorfilter.zig b/src/xorfilter.zig index f05c7554..9ab015ce 100644 --- a/src/xorfilter.zig +++ b/src/xorfilter.zig @@ -39,9 +39,11 @@ pub fn Xor(comptime T: type) type { pub fn init(allocator: Allocator, size: usize) !Self { var capacity = @as(usize, @intFromFloat(32 + 1.23 * @as(f64, @floatFromInt(size)))); capacity = capacity / 3 * 3; + const fingerprints = try allocator.alloc(T, capacity); + @memset(fingerprints, 0); return Self{ .seed = 0, - .fingerprints = try allocator.alloc(T, capacity), + .fingerprints = fingerprints, .blockLength = capacity / 3, }; } @@ -334,7 +336,7 @@ const Keyindex = struct { index: u32, }; -fn xorTest(T: anytype, size: usize, size_in_bytes: usize) !void { +fn xorTest(T: anytype, size: usize) !void { const allocator = std.heap.page_allocator; var filter = try Xor(T).init(allocator, size); defer filter.deinit(allocator); @@ -347,14 +349,19 @@ fn xorTest(T: anytype, size: usize, size_in_bytes: usize) !void { try filter.populate(allocator, keys[0..]); - try testing.expect(filter.contain(1) == true); - try testing.expect(filter.contain(5) == true); - try testing.expect(filter.contain(9) == true); - try testing.expect(filter.contain(1234) == true); - try testing.expectEqual(@as(usize, size_in_bytes), filter.sizeInBytes()); + try testing.expect(filter.contain(1)); + try testing.expect(filter.contain(5)); + try testing.expect(filter.contain(9)); + try testing.expect(filter.contain(1234)); + + var capacity = @as(usize, @intFromFloat(32 + 1.23 * @as(f64, @floatFromInt(size)))); + capacity = capacity / 3 * 3; + const blockLength = capacity / 3; + const expected_size = 3 * blockLength * @sizeOf(T) + @sizeOf(Xor(T)); + try testing.expectEqual(expected_size, filter.sizeInBytes()); for (keys) |key| { - try testing.expect(filter.contain(key) == true); + try testing.expect(filter.contain(key)); } var random_matches: u64 = 0; @@ -378,15 +385,15 @@ fn xorTest(T: anytype, size: usize, size_in_bytes: usize) !void { } test "xor8" { - try xorTest(u8, 10000, 12370); + try xorTest(u8, 10000); } test "xor16" { - try xorTest(u16, 10000, 24700); + try xorTest(u16, 10000); } test "xor20" { - try xorTest(u20, 10000, 49360); + try xorTest(u20, 10000); } test "xor32" { @@ -396,7 +403,7 @@ test "xor32" { // // If you have a really beefy machine, it would be cool to try this test with a huge amount of // keys and higher `trials` in `xorTest`. - try xorTest(u32, 1000000, 4920160); + try xorTest(u32, 1000000); } test "xor64" { @@ -406,5 +413,5 @@ test "xor64" { // // If you have a really beefy machine, it would be cool to try this test with a huge amount of // keys and higher `trials` in `xorTest`. - try xorTest(u64, 1000000, 9840280); + try xorTest(u64, 1000000); } From df3496c9e2ea0255dafc550d9e57ff4426c83c0a Mon Sep 17 00:00:00 2001 From: Denis Golovachev Date: Tue, 21 Oct 2025 13:00:20 +0300 Subject: [PATCH 2/7] Update CI badge link in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fcbd53de..4e08fd5d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # fastfilter: Binary fuse & xor filters for Zig Hexops logo -[![CI](https://github.com/hexops/fastfilter/workflows/CI/badge.svg)](https://github.com/hexops/fastfilter/actions) +[![CI](https://github.com/WonderBeat/fastfilter/workflows/CI/badge.svg)](https://github.com/hexops/WonderBeat/actions) comparison From 274a0e73b2a9673fcee721571f08302880900106 Mon Sep 17 00:00:00 2001 From: Denis Golovachev Date: Tue, 21 Oct 2025 13:01:45 +0300 Subject: [PATCH 3/7] Fix README formatting and links --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4e08fd5d..ea0b0ab8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# fastfilter: Binary fuse & xor filters for Zig Hexops logo +# fastfilter: Binary fuse & xor filters for Zig -[![CI](https://github.com/WonderBeat/fastfilter/workflows/CI/badge.svg)](https://github.com/hexops/WonderBeat/actions) +[![CI](https://github.com/WonderBeat/fastfilter/workflows/CI/badge.svg)](https://github.com/WonderBeat/fastfilter/actions) comparison From bbf122a6cf591006af91751939a2fd5f8ae869f8 Mon Sep 17 00:00:00 2001 From: WonderBeat Date: Thu, 23 Oct 2025 12:24:24 +0300 Subject: [PATCH 4/7] =?UTF-8?q?Add=20GitHub=20Release=20Drafter=20configur?= =?UTF-8?q?ation=20=F0=9F=93=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Introduce .github/release-drafter.yml for automated changelog generation - Configure release drafter with standard categories and version resolution rules --- .github/release-drafter.yml | 42 +++++++++++++++++++++++++++++++++++++ .github/workflows/draft.yml | 14 +++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 .github/release-drafter.yml create mode 100644 .github/workflows/draft.yml diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 00000000..af227480 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,42 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +template: | + # What's Changed + + $CHANGES + + **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION + +categories: + - title: 'Breaking' + label: 'type: breaking' + - title: 'New' + label: 'type: feature' + - title: 'Bug Fixes' + label: 'type: bug' + - title: 'Maintenance' + label: 'type: maintenance' + - title: 'Documentation' + label: 'type: docs' + - title: 'Other changes' + - title: 'Dependency Updates' + label: 'type: dependencies' + collapse-after: 5 + +version-resolver: + major: + labels: + - 'type: breaking' + minor: + labels: + - 'type: feature' + patch: + labels: + - 'type: bug' + - 'type: maintenance' + - 'type: docs' + - 'type: dependencies' + - 'type: security' + +exclude-labels: + - 'skip-changelog' diff --git a/.github/workflows/draft.yml b/.github/workflows/draft.yml new file mode 100644 index 00000000..cf6d8307 --- /dev/null +++ b/.github/workflows/draft.yml @@ -0,0 +1,14 @@ +name: Release Drafter + +on: + push: + branches: + - master + +jobs: + update_release_draft: + runs-on: ubuntu-latest + steps: + - uses: release-drafter/release-drafter@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 642062e30f09830307c2de69c3006e23ef7d4b7c Mon Sep 17 00:00:00 2001 From: Denis Golovachev Date: Thu, 23 Oct 2025 12:27:31 +0300 Subject: [PATCH 5/7] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ea0b0ab8..63a7d064 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,9 @@ If you intend to use a xor filter with datasets of 100m+ keys, there is a possib The API is generally finalized, but we may make some adjustments as Zig changes or we learn of more idiomatic ways to express things. We will release v1.0 once Zig v1.0 is released. +### **v0.12.0** +- Updated to the latest version of Zig 0.15.1 + ### **v0.11.0** - fastfilter is now available via the Zig package manager. From a9ab5ea4180994e531706484e9513496312251ad Mon Sep 17 00:00:00 2001 From: WonderBeat Date: Thu, 23 Oct 2025 12:28:17 +0300 Subject: [PATCH 6/7] fix branch target --- .github/workflows/draft.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/draft.yml b/.github/workflows/draft.yml index cf6d8307..19f55562 100644 --- a/.github/workflows/draft.yml +++ b/.github/workflows/draft.yml @@ -4,6 +4,7 @@ on: push: branches: - master + - main jobs: update_release_draft: From a0b82a92f0ffbf150103d8a2033e80e24d7d5b74 Mon Sep 17 00:00:00 2001 From: WonderBeat Date: Thu, 23 Oct 2025 12:28:17 +0300 Subject: [PATCH 7/7] fix branch target --- .github/workflows/draft.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/draft.yml b/.github/workflows/draft.yml index 19f55562..dea1bfd5 100644 --- a/.github/workflows/draft.yml +++ b/.github/workflows/draft.yml @@ -9,6 +9,9 @@ on: jobs: update_release_draft: runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write steps: - uses: release-drafter/release-drafter@master env: