diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 00000000..af227480 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,42 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +template: | + # What's Changed + + $CHANGES + + **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION + +categories: + - title: 'Breaking' + label: 'type: breaking' + - title: 'New' + label: 'type: feature' + - title: 'Bug Fixes' + label: 'type: bug' + - title: 'Maintenance' + label: 'type: maintenance' + - title: 'Documentation' + label: 'type: docs' + - title: 'Other changes' + - title: 'Dependency Updates' + label: 'type: dependencies' + collapse-after: 5 + +version-resolver: + major: + labels: + - 'type: breaking' + minor: + labels: + - 'type: feature' + patch: + labels: + - 'type: bug' + - 'type: maintenance' + - 'type: docs' + - 'type: dependencies' + - 'type: security' + +exclude-labels: + - 'skip-changelog' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b02421f..a3d43332 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,10 +8,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: Setup Zig - run: | - sudo apt install xz-utils - sudo sh -c 'wget -c https://pkg.machengine.org/zig/zig-linux-x86_64-0.14.0-dev.2577+271452d22.tar.xz -O - | tar -xJ --strip-components=1 -C /usr/local/bin' + - uses: mlugg/setup-zig@v2 + with: + version: 0.15.1 - name: build run: zig build - name: test diff --git a/.github/workflows/draft.yml b/.github/workflows/draft.yml new file mode 100644 index 00000000..dea1bfd5 --- /dev/null +++ b/.github/workflows/draft.yml @@ -0,0 +1,18 @@ +name: Release Drafter + +on: + push: + branches: + - master + - main + +jobs: + update_release_draft: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - uses: release-drafter/release-drafter@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index fcbd53de..63a7d064 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# fastfilter: Binary fuse & xor filters for Zig Hexops logo +# fastfilter: Binary fuse & xor filters for Zig -[![CI](https://github.com/hexops/fastfilter/workflows/CI/badge.svg)](https://github.com/hexops/fastfilter/actions) +[![CI](https://github.com/WonderBeat/fastfilter/workflows/CI/badge.svg)](https://github.com/WonderBeat/fastfilter/actions) comparison @@ -147,6 +147,9 @@ If you intend to use a xor filter with datasets of 100m+ keys, there is a possib The API is generally finalized, but we may make some adjustments as Zig changes or we learn of more idiomatic ways to express things. We will release v1.0 once Zig v1.0 is released. +### **v0.12.0** +- Updated to the latest version of Zig 0.15.1 + ### **v0.11.0** - fastfilter is now available via the Zig package manager. diff --git a/build.zig b/build.zig index 314ae828..b78d3def 100644 --- a/build.zig +++ b/build.zig @@ -6,13 +6,15 @@ pub fn build(b: *std.Build) void { _ = b.addModule("fastfilter", .{ .root_source_file = b.path("src/main.zig"), + .optimize = optimize, }); const main_tests = b.addTest(.{ - .name = "fastfilter-tests", - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, + .root_module = b.addModule("test", .{ + .root_source_file = b.path("src/main.zig"), + .target = target, + .optimize = optimize, + }), }); const run_main_tests = b.addRunArtifact(main_tests); diff --git a/build.zig.zon b/build.zig.zon new file mode 100644 index 00000000..9cd957d1 --- /dev/null +++ b/build.zig.zon @@ -0,0 +1,15 @@ +.{ + .name = .fastfilter, + .version = "1.0.1", + + .dependencies = .{}, + .fingerprint = 0x809515533cb3d42b, + + .minimum_zig_version = "0.15.1", + .paths = .{ + "build.zig", + "build.zig.zon", + "src/main.zig", + "src", + }, +} diff --git a/src/MeasuredAllocator.zig b/src/MeasuredAllocator.zig index ebf07a6d..25bd7664 100644 --- a/src/MeasuredAllocator.zig +++ b/src/MeasuredAllocator.zig @@ -37,11 +37,17 @@ pub fn allocator(self: *MeasuredAllocator) Allocator { .alloc = allocFn, .resize = resizeFn, .free = freeFn, + .remap = remapFn, }, }; } -fn allocFn(ptr: *anyopaque, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8 { +fn remapFn(ptr: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + const self = @as(*MeasuredAllocator, @ptrCast(@alignCast(ptr))); + return self.parent_allocator.rawRemap(memory, alignment, new_len, ret_addr); +} + +fn allocFn(ptr: *anyopaque, len: usize, ptr_align: mem.Alignment, ret_addr: usize) ?[*]u8 { const self = @as(*MeasuredAllocator, @ptrCast(@alignCast(ptr))); const result = self.parent_allocator.rawAlloc(len, ptr_align, ret_addr); if (result) |_| { @@ -51,7 +57,7 @@ fn allocFn(ptr: *anyopaque, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8 { return result; } -fn resizeFn(ptr: *anyopaque, buf: []u8, buf_align: u8, new_len: usize, ret_addr: usize) bool { +fn resizeFn(ptr: *anyopaque, buf: []u8, buf_align: mem.Alignment, new_len: usize, ret_addr: usize) bool { const self = @as(*MeasuredAllocator, @ptrCast(@alignCast(ptr))); if (self.parent_allocator.rawResize(buf, buf_align, new_len, ret_addr)) { self.state.current_memory_usage_bytes -= buf.len - new_len; @@ -61,7 +67,7 @@ fn resizeFn(ptr: *anyopaque, buf: []u8, buf_align: u8, new_len: usize, ret_addr: return false; } -fn freeFn(ptr: *anyopaque, buf: []u8, buf_align: u8, ret_addr: usize) void { +fn freeFn(ptr: *anyopaque, buf: []u8, buf_align: mem.Alignment, ret_addr: usize) void { const self = @as(*MeasuredAllocator, @ptrCast(@alignCast(ptr))); self.parent_allocator.rawFree(buf, buf_align, ret_addr); self.state.current_memory_usage_bytes -= buf.len; diff --git a/src/benchmark.zig b/src/benchmark.zig index f0b0384b..abdf803c 100644 --- a/src/benchmark.zig +++ b/src/benchmark.zig @@ -9,18 +9,18 @@ const MeasuredAllocator = @import("MeasuredAllocator.zig"); fn formatTime(writer: anytype, comptime spec: []const u8, start: u64, end: u64, division: usize) !void { const ns = @as(f64, @floatFromInt((end - start) / division)); if (ns <= time.ns_per_ms) { - try std.fmt.format(writer, spec, .{ ns, "ns " }); + try writer.print(spec, .{ ns, "ns " }); return; } if (ns <= time.ns_per_s) { - try std.fmt.format(writer, spec, .{ ns / @as(f64, @floatFromInt(time.ns_per_ms)), "ms " }); + try writer.print(spec, .{ ns / @as(f64, @floatFromInt(time.ns_per_ms)), "ms " }); return; } if (ns <= time.ns_per_min) { - try std.fmt.format(writer, spec, .{ ns / @as(f64, @floatFromInt(time.ns_per_s)), "s " }); + try writer.print(spec, .{ ns / @as(f64, @floatFromInt(time.ns_per_s)), "s " }); return; } - try std.fmt.format(writer, spec, .{ ns / @as(f64, @floatFromInt(time.ns_per_min)), "min" }); + try writer.print(spec, .{ ns / @as(f64, @floatFromInt(time.ns_per_min)), "min" }); return; } @@ -29,17 +29,17 @@ fn formatBytes(writer: anytype, comptime spec: []const u8, bytes: u64) !void { const mib = 1024 * kib; const gib = 1024 * mib; if (bytes < kib) { - try std.fmt.format(writer, spec, .{ bytes, "B " }); + try writer.print(spec, .{ bytes, "B " }); } if (bytes < mib) { - try std.fmt.format(writer, spec, .{ bytes / kib, "KiB" }); + try writer.print(spec, .{ bytes / kib, "KiB" }); return; } if (bytes < gib) { - try std.fmt.format(writer, spec, .{ bytes / mib, "MiB" }); + try writer.print(spec, .{ bytes / mib, "MiB" }); return; } - try std.fmt.format(writer, spec, .{ bytes / gib, "GiB" }); + try writer.print(spec, .{ bytes / gib, "GiB" }); return; } @@ -52,7 +52,6 @@ fn bench(algorithm: []const u8, Filter: anytype, size: usize, trials: usize) !vo var buildMA = MeasuredAllocator.init(allocator); const buildAllocator = buildMA.allocator(); - const stdout = std.io.getStdOut().writer(); var timer = try Timer.start(); // Initialize filter. @@ -97,6 +96,10 @@ fn bench(algorithm: []const u8, Filter: anytype, size: usize, trials: usize) !vo @panic("sizeInBytes reporting wrong numbers?"); } + var stdout_buffer: [1024]u8 = undefined; + var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); + const stdout = &stdout_writer.interface; + try stdout.print("| {s: <12} ", .{algorithm}); try stdout.print("| {: <10} ", .{keys.len}); try stdout.print("| ", .{}); @@ -139,7 +142,9 @@ pub fn main() !void { } } - const stdout = std.io.getStdOut().writer(); + var stdout_buffer: [1024]u8 = undefined; + var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); + const stdout = &stdout_writer.interface; try stdout.print("| Algorithm | # of keys | populate | contains(k) | false+ prob. | bits per entry | peak populate | filter total |\n", .{}); try stdout.print("|--------------|------------|------------|-------------|--------------|----------------|---------------|--------------|\n", .{}); try bench("binaryfuse8", xorfilter.BinaryFuse(u8), 1_000_000, num_trials); diff --git a/src/xorfilter.zig b/src/xorfilter.zig index f05c7554..9ab015ce 100644 --- a/src/xorfilter.zig +++ b/src/xorfilter.zig @@ -39,9 +39,11 @@ pub fn Xor(comptime T: type) type { pub fn init(allocator: Allocator, size: usize) !Self { var capacity = @as(usize, @intFromFloat(32 + 1.23 * @as(f64, @floatFromInt(size)))); capacity = capacity / 3 * 3; + const fingerprints = try allocator.alloc(T, capacity); + @memset(fingerprints, 0); return Self{ .seed = 0, - .fingerprints = try allocator.alloc(T, capacity), + .fingerprints = fingerprints, .blockLength = capacity / 3, }; } @@ -334,7 +336,7 @@ const Keyindex = struct { index: u32, }; -fn xorTest(T: anytype, size: usize, size_in_bytes: usize) !void { +fn xorTest(T: anytype, size: usize) !void { const allocator = std.heap.page_allocator; var filter = try Xor(T).init(allocator, size); defer filter.deinit(allocator); @@ -347,14 +349,19 @@ fn xorTest(T: anytype, size: usize, size_in_bytes: usize) !void { try filter.populate(allocator, keys[0..]); - try testing.expect(filter.contain(1) == true); - try testing.expect(filter.contain(5) == true); - try testing.expect(filter.contain(9) == true); - try testing.expect(filter.contain(1234) == true); - try testing.expectEqual(@as(usize, size_in_bytes), filter.sizeInBytes()); + try testing.expect(filter.contain(1)); + try testing.expect(filter.contain(5)); + try testing.expect(filter.contain(9)); + try testing.expect(filter.contain(1234)); + + var capacity = @as(usize, @intFromFloat(32 + 1.23 * @as(f64, @floatFromInt(size)))); + capacity = capacity / 3 * 3; + const blockLength = capacity / 3; + const expected_size = 3 * blockLength * @sizeOf(T) + @sizeOf(Xor(T)); + try testing.expectEqual(expected_size, filter.sizeInBytes()); for (keys) |key| { - try testing.expect(filter.contain(key) == true); + try testing.expect(filter.contain(key)); } var random_matches: u64 = 0; @@ -378,15 +385,15 @@ fn xorTest(T: anytype, size: usize, size_in_bytes: usize) !void { } test "xor8" { - try xorTest(u8, 10000, 12370); + try xorTest(u8, 10000); } test "xor16" { - try xorTest(u16, 10000, 24700); + try xorTest(u16, 10000); } test "xor20" { - try xorTest(u20, 10000, 49360); + try xorTest(u20, 10000); } test "xor32" { @@ -396,7 +403,7 @@ test "xor32" { // // If you have a really beefy machine, it would be cool to try this test with a huge amount of // keys and higher `trials` in `xorTest`. - try xorTest(u32, 1000000, 4920160); + try xorTest(u32, 1000000); } test "xor64" { @@ -406,5 +413,5 @@ test "xor64" { // // If you have a really beefy machine, it would be cool to try this test with a huge amount of // keys and higher `trials` in `xorTest`. - try xorTest(u64, 1000000, 9840280); + try xorTest(u64, 1000000); }