From 81969c18541905ba16d2e58a73dc9053def3b9e3 Mon Sep 17 00:00:00 2001 From: bomanaps Date: Mon, 27 Apr 2026 16:36:09 +0100 Subject: [PATCH 1/9] feat: add zig gossipsub interop --- gossipsub-interop/Makefile | 2 + gossipsub-interop/experiment.py | 1 + gossipsub-interop/run-shadow.sh | 29 ++++++ gossipsub-interop/run.py | 10 +- gossipsub-interop/zig-libp2p/.gitignore | 4 + gossipsub-interop/zig-libp2p/build.zig | 66 +++++++++++++ gossipsub-interop/zig-libp2p/build.zig.zon | 15 +++ gossipsub-interop/zig-libp2p/src/main.zig | 106 +++++++++++++++++++++ 8 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 gossipsub-interop/run-shadow.sh create mode 100644 gossipsub-interop/zig-libp2p/.gitignore create mode 100644 gossipsub-interop/zig-libp2p/build.zig create mode 100644 gossipsub-interop/zig-libp2p/build.zig.zon create mode 100644 gossipsub-interop/zig-libp2p/src/main.zig diff --git a/gossipsub-interop/Makefile b/gossipsub-interop/Makefile index 90a60bf7b..c8396ea07 100644 --- a/gossipsub-interop/Makefile +++ b/gossipsub-interop/Makefile @@ -26,6 +26,8 @@ binaries: nim-libp2p/gossipsub-bin cd go-libp2p && go build -linkshared -o gossipsub-bin cd rust-libp2p && cargo build cd jvm-libp2p && ./gradlew installDist + zig build --build-file $(CURDIR)/../../eth-p2p-z/build.zig --prefix $(CURDIR)/zig-libp2p/zig-out -Doptimize=ReleaseFast + cp $(CURDIR)/zig-libp2p/zig-out/bin/gossipsub-bin $(CURDIR)/zig-libp2p/gossipsub-bin # Clean all generated shadow simulation files clean: diff --git a/gossipsub-interop/experiment.py b/gossipsub-interop/experiment.py index 7ecca96fb..211418488 100644 --- a/gossipsub-interop/experiment.py +++ b/gossipsub-interop/experiment.py @@ -346,6 +346,7 @@ def scenario( "rust": "rust-libp2p/target/debug/rust-libp2p-gossip", "nim": "nim-libp2p/gossipsub-bin", "jvm": "jvm-libp2p/build/install/jvm-libp2p-gossip/bin/jvm-libp2p-gossip", + "zig": "zig-libp2p/gossipsub-bin", } diff --git a/gossipsub-interop/run-shadow.sh b/gossipsub-interop/run-shadow.sh new file mode 100644 index 000000000..f8272d7dc --- /dev/null +++ b/gossipsub-interop/run-shadow.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Run a Shadow gossipsub experiment inside the Docker container. +# +# Expected mounts: +# /work → gossipsub-interop directory (this repo) +# /eth-p2p-z → eth-p2p-z source directory +# /root/.cache/zig → host Zig global package cache (optional, speeds up build) +# +# Usage: +# bash /work/run-shadow.sh [run.py args...] +# Example: +# bash /work/run-shadow.sh --node_count 4 --composition all-zig --scenario subnet-blob-msg +set -euo pipefail + +echo "==> Building gossipsub-bin for linux/x86_64..." +# -Dcpu overrides the auto-detected CPU (e.g. 'athlon-xp' under Rosetta/QEMU +# which newer LLVM doesn't recognise) while keeping native mode so Zig still +# uses system headers (needed for zlib.h etc). +zig build \ + --build-file /eth-p2p-z/build.zig \ + --prefix /work/zig-libp2p/zig-out \ + -Doptimize=ReleaseFast \ + -Dcpu=x86_64 + +cp /work/zig-libp2p/zig-out/bin/gossipsub-bin /work/zig-libp2p/gossipsub-bin +echo " gossipsub-bin: $(du -sh /work/zig-libp2p/gossipsub-bin | cut -f1)" + +echo "==> Running Shadow experiment..." +uv run /work/run.py --skip_build true "$@" diff --git a/gossipsub-interop/run.py b/gossipsub-interop/run.py index 90dfe111b..cba6991e5 100644 --- a/gossipsub-interop/run.py +++ b/gossipsub-interop/run.py @@ -38,6 +38,13 @@ def main(): "Nodes are split evenly across them.", ) parser.add_argument("--output_dir", type=str, required=False) + parser.add_argument( + "--skip_build", + type=bool, + required=False, + default=False, + help="If set, skip running 'make binaries' (binary already built)", + ) args = parser.parse_args() shadow_outputs_dir = os.path.join(os.getcwd(), "shadow-outputs") @@ -97,7 +104,8 @@ def main(): if args.dry_run: return - subprocess.run(["make", "binaries"], check=True) + if not args.skip_build: + subprocess.run(["make", "binaries"], check=True) subprocess.run( ["shadow", "--progress", "true", "-d", args.output_dir, "shadow.yaml"], diff --git a/gossipsub-interop/zig-libp2p/.gitignore b/gossipsub-interop/zig-libp2p/.gitignore new file mode 100644 index 000000000..f204361ff --- /dev/null +++ b/gossipsub-interop/zig-libp2p/.gitignore @@ -0,0 +1,4 @@ +gossipsub-interop/zig-libp2p/.zig-cache/ +zig-pkg/ +.zig-cache/ +zig-out/ diff --git a/gossipsub-interop/zig-libp2p/build.zig b/gossipsub-interop/zig-libp2p/build.zig new file mode 100644 index 000000000..68e80a0a5 --- /dev/null +++ b/gossipsub-interop/zig-libp2p/build.zig @@ -0,0 +1,66 @@ +const std = @import("std"); + +/// NOTE: Due to a Zig 0.15 limitation, translate-c steps in transitive +/// path dependencies (boringssl/ssl.zig) do not execute in a standalone +/// project's local cache. +/// +/// Build via the Makefile target instead: +/// make binaries +/// +/// Which runs: +/// zig build --build-file ../../eth-p2p-z/build.zig --prefix zig-libp2p/zig-out +/// cp zig-libp2p/zig-out/bin/gossipsub-bin zig-libp2p/gossipsub-bin +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + // eth-p2p-z as a path dependency (swap for a git URL when publishing) + const libp2p_dep = b.dependency("libp2p", .{ + .target = target, + .optimize = optimize, + }); + const libp2p_module = libp2p_dep.module("zig-libp2p"); + + // lsquic: C artifact needed for final linking + const lsquic_dep = libp2p_dep.builder.dependency("lsquic", .{ + .target = target, + .optimize = optimize, + }); + const lsquic_artifact = lsquic_dep.artifact("lsquic"); + + // peer_id: main.zig uses @import("peer_id") directly for PeerId + keys + const multiaddr_dep = libp2p_dep.builder.dependency("multiaddr", .{ + .target = target, + .optimize = optimize, + }); + const peer_id_dep = multiaddr_dep.builder.dependency("peer_id", .{ + .target = target, + .optimize = optimize, + }); + const peer_id_module = peer_id_dep.module("peer-id"); + + const exe_module = b.createModule(.{ + .root_source_file = b.path("src/main.zig"), + .target = target, + .optimize = optimize, + }); + exe_module.addImport("zig-libp2p", libp2p_module); + exe_module.addImport("peer_id", peer_id_module); + + const exe = b.addExecutable(.{ + .name = "gossipsub-bin", + .root_module = exe_module, + }); + exe.linkLibrary(lsquic_artifact); + exe.linkSystemLibrary(switch (target.result.os.tag) { + .windows => "zlib1", + else => "z", + }); + + b.installArtifact(exe); + + const run_cmd = b.addRunArtifact(exe); + run_cmd.step.dependOn(b.getInstallStep()); + if (b.args) |args| run_cmd.addArgs(args); + b.step("run", "Run gossipsub-bin").dependOn(&run_cmd.step); +} diff --git a/gossipsub-interop/zig-libp2p/build.zig.zon b/gossipsub-interop/zig-libp2p/build.zig.zon new file mode 100644 index 000000000..e7424a71f --- /dev/null +++ b/gossipsub-interop/zig-libp2p/build.zig.zon @@ -0,0 +1,15 @@ +.{ + .name = .gossipsub_bin, + .version = "0.1.0", + .fingerprint = 0xa3892b19cdfabd6c, + .dependencies = .{ + .libp2p = .{ + .path = "../../../eth-p2p-z", + }, + }, + .paths = .{ + "build.zig", + "build.zig.zon", + "src", + }, +} diff --git a/gossipsub-interop/zig-libp2p/src/main.zig b/gossipsub-interop/zig-libp2p/src/main.zig new file mode 100644 index 000000000..e1156c826 --- /dev/null +++ b/gossipsub-interop/zig-libp2p/src/main.zig @@ -0,0 +1,106 @@ +/// Phase 2.1 — Binary Skeleton +/// +/// Starts up, derives a deterministic PeerID from the Shadow node hostname, +/// logs it to stdout in the same JSON format as the Go binary, then exits. +/// +/// TODO Phase 2.3: wire TCP + TLS + Yamux host listening on port 9000 +/// TODO Phase 2.2: parse params file and run experiment instructions + +pub const std_options = @import("zig-libp2p").std_options; + +const std = @import("std"); +const libp2p = @import("zig-libp2p"); +const keys_mod = @import("peer_id").keys; +const PeerId = @import("peer_id").PeerId; + +pub fn main() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + // ------------------------------------------------------------------ // + // 1. Parse --params flag + // ------------------------------------------------------------------ // + const args = try std.process.argsAlloc(allocator); + defer std.process.argsFree(allocator, args); + + var params_path: ?[]const u8 = null; + var i: usize = 1; + while (i < args.len) : (i += 1) { + if (std.mem.eql(u8, args[i], "--params") and i + 1 < args.len) { + params_path = args[i + 1]; + i += 1; + } + } + if (params_path == null) { + std.log.err("--params flag is required", .{}); + std.process.exit(1); + } + + // ------------------------------------------------------------------ // + // 2. Hostname → node ID + // Shadow sets hostnames as "node0", "node1", … + // ------------------------------------------------------------------ // + var hostname_buf: [std.posix.HOST_NAME_MAX]u8 = undefined; + const hostname = try std.posix.gethostname(hostname_buf[0..]); + + var node_id: u64 = 0; + if (std.mem.startsWith(u8, hostname, "node")) { + node_id = std.fmt.parseInt(u64, hostname[4..], 10) catch blk: { + std.log.warn("could not parse node ID from hostname '{s}', defaulting to 0", .{hostname}); + break :blk 0; + }; + } + + // ------------------------------------------------------------------ // + // 3. Deterministic ED25519 key — matches Go's nodePrivKey(id): + // seed = [32]u8{0}, seed[0..8] = LE u64(node_id) + // privkey = ed25519.NewKeyFromSeed(seed) + // ------------------------------------------------------------------ // + const Ed25519 = std.crypto.sign.Ed25519; + var seed: [Ed25519.KeyPair.seed_length]u8 = [_]u8{0} ** Ed25519.KeyPair.seed_length; + std.mem.writeInt(u64, seed[0..8], node_id, .little); + const ed_kp = try Ed25519.KeyPair.generateDeterministic(seed); + + // ------------------------------------------------------------------ // + // 4. Derive libp2p PeerID from the ED25519 public key + // ------------------------------------------------------------------ // + var pub_key = keys_mod.PublicKey{ + .type = .ED25519, + .data = &ed_kp.public_key.bytes, + }; + const peer_id = try PeerId.fromPublicKey(allocator, &pub_key); + + const b58_len = peer_id.toBase58Len(); + const b58_buf = try allocator.alloc(u8, b58_len); + defer allocator.free(b58_buf); + const peer_id_str = try peer_id.toBase58(b58_buf); + + // ------------------------------------------------------------------ // + // 5. Log PeerID to stdout — matches Go slog JSON format: + // {"time":"...","level":"INFO","msg":"PeerID","id":"","node_id":N} + // ------------------------------------------------------------------ // + const now_s = std.time.timestamp(); + const stdout_file: std.fs.File = .{ .handle = std.posix.STDOUT_FILENO }; + var stdout_buf: [512]u8 = undefined; + var stdout_writer = stdout_file.writer(&stdout_buf); + try stdout_writer.interface.print( + "{{\"time\":\"{d}\",\"level\":\"INFO\",\"msg\":\"PeerID\",\"id\":\"{s}\",\"node_id\":{d}}}\n", + .{ now_s, peer_id_str, node_id }, + ); + try stdout_writer.interface.flush(); + + // ------------------------------------------------------------------ // + // 6. Initialise event loop (proves the runtime starts cleanly) + // TCP listener + gossipsub host wired in Phase 2.3 + // ------------------------------------------------------------------ // + var loop: libp2p.thread_event_loop.ThreadEventLoop = undefined; + try loop.init(allocator); + defer { + loop.close(); + loop.deinit(); + } + + std.log.info("Phase 2.1 skeleton started — node_id={d} peer_id={s}", .{ node_id, peer_id_str }); + // Phase 2.2 will read params_path and execute experiment instructions here +} From dd07fc76b755f8b9a2776682b4549e631ce53795 Mon Sep 17 00:00:00 2001 From: bomanaps Date: Mon, 27 Apr 2026 16:42:23 +0100 Subject: [PATCH 2/9] feat: add gossipsub shadow Dockerfile --- gossipsub-interop/Dockerfile | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 gossipsub-interop/Dockerfile diff --git a/gossipsub-interop/Dockerfile b/gossipsub-interop/Dockerfile new file mode 100644 index 000000000..36bba75e3 --- /dev/null +++ b/gossipsub-interop/Dockerfile @@ -0,0 +1,36 @@ +# Shadow network simulator + Zig build environment +# Runs on linux/amd64 (Shadow requirement). +# +# Uses the pre-built shadowsim/shadow-ci image so we skip the ~1h Shadow +# Rust compilation that was previously needed under QEMU on Apple Silicon. +# +# Build: +# docker build --platform linux/amd64 -t gossipsub-shadow . +# +# Run (from gossipsub-interop directory): +# docker run --rm --platform linux/amd64 \ +# -v $(pwd):/work \ +# -v /path/to/eth-p2p-z:/eth-p2p-z \ +# -v $HOME/.cache/zig:/root/.cache/zig \ +# gossipsub-shadow \ +# bash /work/run-shadow.sh --node_count 4 --composition all-zig --scenario subnet-blob-msg + +FROM --platform=linux/amd64 shadowsim/shadow-ci:ubuntu-22.04-gcc-release + +# ── Extra system libraries needed by the Zig/C build ───────────────────── +# zlib1g-dev: required by lsquic (zlib.h) +RUN apt-get update && apt-get install -y \ + zlib1g-dev \ + git \ + && rm -rf /var/lib/apt/lists/* + +# ── Zig 0.15.2 ──────────────────────────────────────────────────────────── +RUN curl -fL https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz \ + | tar -xJ -C /usr/local \ + && ln -s /usr/local/zig-x86_64-linux-0.15.2/zig /usr/local/bin/zig + +# ── uv (Python package manager for experiment scripts) ──────────────────── +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +ENV PATH="/root/.local/bin:${PATH}" + +WORKDIR /work From ee232db2e034d90704f360a8f9ee6ed0b1b1cfcd Mon Sep 17 00:00:00 2001 From: bomanaps Date: Mon, 27 Apr 2026 17:08:09 +0100 Subject: [PATCH 3/9] fix: use cd instead of --build-file for zig build --- gossipsub-interop/run-shadow.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gossipsub-interop/run-shadow.sh b/gossipsub-interop/run-shadow.sh index f8272d7dc..2c286256b 100644 --- a/gossipsub-interop/run-shadow.sh +++ b/gossipsub-interop/run-shadow.sh @@ -16,8 +16,7 @@ echo "==> Building gossipsub-bin for linux/x86_64..." # -Dcpu overrides the auto-detected CPU (e.g. 'athlon-xp' under Rosetta/QEMU # which newer LLVM doesn't recognise) while keeping native mode so Zig still # uses system headers (needed for zlib.h etc). -zig build \ - --build-file /eth-p2p-z/build.zig \ +cd /eth-p2p-z && zig build \ --prefix /work/zig-libp2p/zig-out \ -Doptimize=ReleaseFast \ -Dcpu=x86_64 From 865917b4e12e4ddc45e903efa036b1486fce32f7 Mon Sep 17 00:00:00 2001 From: bomanaps Date: Mon, 27 Apr 2026 17:40:03 +0100 Subject: [PATCH 4/9] fix: collapse multi-line f-string for Python 3.11 compat --- gossipsub-interop/run.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gossipsub-interop/run.py b/gossipsub-interop/run.py index cba6991e5..40e78b92c 100644 --- a/gossipsub-interop/run.py +++ b/gossipsub-interop/run.py @@ -64,9 +64,7 @@ def main(): timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") composition_label = "-".join(args.composition) - args.output_dir = f"{args.scenario}-{args.node_count}-{composition_label}-{ - args.seed - }-{timestamp}-{git_describe}.data" + args.output_dir = f"{args.scenario}-{args.node_count}-{composition_label}-{args.seed}-{timestamp}-{git_describe}.data" if not os.path.isabs(args.output_dir): args.output_dir = os.path.join(shadow_outputs_dir, args.output_dir) From 2fc3a11b62e016c58bfe0355ac1044e71dae9296 Mon Sep 17 00:00:00 2001 From: bomanaps Date: Mon, 27 Apr 2026 17:50:17 +0100 Subject: [PATCH 5/9] fix: use subshell for cd to preserve CWD for uv run --- gossipsub-interop/run-shadow.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gossipsub-interop/run-shadow.sh b/gossipsub-interop/run-shadow.sh index 2c286256b..3f96b68e6 100644 --- a/gossipsub-interop/run-shadow.sh +++ b/gossipsub-interop/run-shadow.sh @@ -16,10 +16,10 @@ echo "==> Building gossipsub-bin for linux/x86_64..." # -Dcpu overrides the auto-detected CPU (e.g. 'athlon-xp' under Rosetta/QEMU # which newer LLVM doesn't recognise) while keeping native mode so Zig still # uses system headers (needed for zlib.h etc). -cd /eth-p2p-z && zig build \ +(cd /eth-p2p-z && zig build \ --prefix /work/zig-libp2p/zig-out \ -Doptimize=ReleaseFast \ - -Dcpu=x86_64 + -Dcpu=x86_64) cp /work/zig-libp2p/zig-out/bin/gossipsub-bin /work/zig-libp2p/gossipsub-bin echo " gossipsub-bin: $(du -sh /work/zig-libp2p/gossipsub-bin | cut -f1)" From 4686c077c1e981a7d4e4107bd48dadcca1aedf3f Mon Sep 17 00:00:00 2001 From: bomanaps Date: Thu, 7 May 2026 20:54:06 +0100 Subject: [PATCH 6/9] ci: run zig gossipsub interop tests --- .github/workflows/gossipsub-interop-pr.yml | 40 +++++++++++++++++++--- gossipsub-interop/Makefile | 6 +++- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gossipsub-interop-pr.yml b/.github/workflows/gossipsub-interop-pr.yml index 48643e9c5..b274ce62e 100644 --- a/.github/workflows/gossipsub-interop-pr.yml +++ b/.github/workflows/gossipsub-interop-pr.yml @@ -15,13 +15,22 @@ jobs: test: runs-on: ubuntu-22.04 steps: - - name: Checkout code + - name: Checkout test-plans uses: actions/checkout@v4 + with: + path: test-plans + + - name: Checkout eth-p2p-z + uses: actions/checkout@v4 + with: + repository: zen-eth/eth-p2p-z + ref: feat/gossipsub + path: eth-p2p-z - name: Set up Go uses: actions/setup-go@v5 with: - go-version-file: gossipsub-interop/go-libp2p/go.mod + go-version-file: test-plans/gossipsub-interop/go-libp2p/go.mod cache: true - name: Set up Rust @@ -32,6 +41,21 @@ jobs: curl https://nim-lang.org/choosenim/init.sh -sSf | sh -s -- -y echo "$HOME/.nimble/bin" >> $GITHUB_PATH + - name: Set up Zig 0.15.2 + run: | + curl -fL https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz \ + | sudo tar -xJ -C /usr/local + sudo ln -sf /usr/local/zig-x86_64-linux-0.15.2/zig /usr/local/bin/zig + zig version + + - name: Cache Zig global packages + uses: actions/cache@v4 + with: + path: ~/.cache/zig + key: zig-0.15.2-${{ hashFiles('eth-p2p-z/build.zig.zon') }} + restore-keys: | + zig-0.15.2- + - name: Install uv run: | curl -LsSf https://astral.sh/uv/install.sh | sh @@ -52,7 +76,8 @@ jobs: netbase \ python3 \ python3-networkx \ - xz-utils + xz-utils \ + zlib1g-dev # Build and install Shadow v3.3.0 from source git clone --depth 1 --branch v3.3.0 https://github.com/shadow/shadow.git shadow-simulator @@ -63,14 +88,19 @@ jobs: echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Run gossipsub interop tests - working-directory: gossipsub-interop + working-directory: test-plans/gossipsub-interop run: make test continue-on-error: true + - name: Run gossipsub interop tests (Zig) + working-directory: test-plans/gossipsub-interop + run: make test-zig + continue-on-error: true + - name: Upload Shadow output if: always() uses: actions/upload-artifact@v4 with: name: shadow-output - path: gossipsub-interop/shadow-outputs/ + path: test-plans/gossipsub-interop/shadow-outputs/ retention-days: 5 diff --git a/gossipsub-interop/Makefile b/gossipsub-interop/Makefile index c8396ea07..b71144619 100644 --- a/gossipsub-interop/Makefile +++ b/gossipsub-interop/Makefile @@ -121,5 +121,9 @@ test-rust-only: @uv run run.py --node_count 2 --composition rust --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ +test-zig: + # Testing all-zig subnet-blob-msg + @echo "Testing all-zig subnet-blob-msg" + @uv run run.py --node_count 4 --composition zig --scenario "subnet-blob-msg" && uv run checks/subnet_blob_msg.py latest/ -.PHONY: binaries all clean test test-rust-only test-go test-subnet-blob test-partial-messages +.PHONY: binaries all clean test test-rust-only test-go test-subnet-blob test-partial-messages test-zig From 4884ebe752b2afb9948910bb1771e591cc7db004 Mon Sep 17 00:00:00 2001 From: bomanaps Date: Sat, 9 May 2026 01:06:24 +0100 Subject: [PATCH 7/9] address review feedback --- .github/workflows/gossipsub-interop-pr.yml | 18 ++-- gossipsub-interop/run-shadow.sh | 28 ------ gossipsub-interop/run.py | 10 +- gossipsub-interop/zig-libp2p/build.zig | 66 ------------- gossipsub-interop/zig-libp2p/build.zig.zon | 15 --- gossipsub-interop/zig-libp2p/src/main.zig | 106 --------------------- 6 files changed, 8 insertions(+), 235 deletions(-) delete mode 100644 gossipsub-interop/run-shadow.sh delete mode 100644 gossipsub-interop/zig-libp2p/build.zig delete mode 100644 gossipsub-interop/zig-libp2p/build.zig.zon delete mode 100644 gossipsub-interop/zig-libp2p/src/main.zig diff --git a/.github/workflows/gossipsub-interop-pr.yml b/.github/workflows/gossipsub-interop-pr.yml index b274ce62e..58fc65101 100644 --- a/.github/workflows/gossipsub-interop-pr.yml +++ b/.github/workflows/gossipsub-interop-pr.yml @@ -15,22 +15,20 @@ jobs: test: runs-on: ubuntu-22.04 steps: - - name: Checkout test-plans + - name: Checkout code uses: actions/checkout@v4 - with: - path: test-plans - name: Checkout eth-p2p-z uses: actions/checkout@v4 with: repository: zen-eth/eth-p2p-z ref: feat/gossipsub - path: eth-p2p-z + path: ../eth-p2p-z - name: Set up Go uses: actions/setup-go@v5 with: - go-version-file: test-plans/gossipsub-interop/go-libp2p/go.mod + go-version-file: gossipsub-interop/go-libp2p/go.mod cache: true - name: Set up Rust @@ -52,9 +50,7 @@ jobs: uses: actions/cache@v4 with: path: ~/.cache/zig - key: zig-0.15.2-${{ hashFiles('eth-p2p-z/build.zig.zon') }} - restore-keys: | - zig-0.15.2- + key: zig-0.15.2 - name: Install uv run: | @@ -88,12 +84,12 @@ jobs: echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Run gossipsub interop tests - working-directory: test-plans/gossipsub-interop + working-directory: gossipsub-interop run: make test continue-on-error: true - name: Run gossipsub interop tests (Zig) - working-directory: test-plans/gossipsub-interop + working-directory: gossipsub-interop run: make test-zig continue-on-error: true @@ -102,5 +98,5 @@ jobs: uses: actions/upload-artifact@v4 with: name: shadow-output - path: test-plans/gossipsub-interop/shadow-outputs/ + path: gossipsub-interop/shadow-outputs/ retention-days: 5 diff --git a/gossipsub-interop/run-shadow.sh b/gossipsub-interop/run-shadow.sh deleted file mode 100644 index 3f96b68e6..000000000 --- a/gossipsub-interop/run-shadow.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# Run a Shadow gossipsub experiment inside the Docker container. -# -# Expected mounts: -# /work → gossipsub-interop directory (this repo) -# /eth-p2p-z → eth-p2p-z source directory -# /root/.cache/zig → host Zig global package cache (optional, speeds up build) -# -# Usage: -# bash /work/run-shadow.sh [run.py args...] -# Example: -# bash /work/run-shadow.sh --node_count 4 --composition all-zig --scenario subnet-blob-msg -set -euo pipefail - -echo "==> Building gossipsub-bin for linux/x86_64..." -# -Dcpu overrides the auto-detected CPU (e.g. 'athlon-xp' under Rosetta/QEMU -# which newer LLVM doesn't recognise) while keeping native mode so Zig still -# uses system headers (needed for zlib.h etc). -(cd /eth-p2p-z && zig build \ - --prefix /work/zig-libp2p/zig-out \ - -Doptimize=ReleaseFast \ - -Dcpu=x86_64) - -cp /work/zig-libp2p/zig-out/bin/gossipsub-bin /work/zig-libp2p/gossipsub-bin -echo " gossipsub-bin: $(du -sh /work/zig-libp2p/gossipsub-bin | cut -f1)" - -echo "==> Running Shadow experiment..." -uv run /work/run.py --skip_build true "$@" diff --git a/gossipsub-interop/run.py b/gossipsub-interop/run.py index 40e78b92c..f1a7da2b2 100644 --- a/gossipsub-interop/run.py +++ b/gossipsub-interop/run.py @@ -38,13 +38,6 @@ def main(): "Nodes are split evenly across them.", ) parser.add_argument("--output_dir", type=str, required=False) - parser.add_argument( - "--skip_build", - type=bool, - required=False, - default=False, - help="If set, skip running 'make binaries' (binary already built)", - ) args = parser.parse_args() shadow_outputs_dir = os.path.join(os.getcwd(), "shadow-outputs") @@ -102,8 +95,7 @@ def main(): if args.dry_run: return - if not args.skip_build: - subprocess.run(["make", "binaries"], check=True) + subprocess.run(["make", "binaries"], check=True) subprocess.run( ["shadow", "--progress", "true", "-d", args.output_dir, "shadow.yaml"], diff --git a/gossipsub-interop/zig-libp2p/build.zig b/gossipsub-interop/zig-libp2p/build.zig deleted file mode 100644 index 68e80a0a5..000000000 --- a/gossipsub-interop/zig-libp2p/build.zig +++ /dev/null @@ -1,66 +0,0 @@ -const std = @import("std"); - -/// NOTE: Due to a Zig 0.15 limitation, translate-c steps in transitive -/// path dependencies (boringssl/ssl.zig) do not execute in a standalone -/// project's local cache. -/// -/// Build via the Makefile target instead: -/// make binaries -/// -/// Which runs: -/// zig build --build-file ../../eth-p2p-z/build.zig --prefix zig-libp2p/zig-out -/// cp zig-libp2p/zig-out/bin/gossipsub-bin zig-libp2p/gossipsub-bin -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - // eth-p2p-z as a path dependency (swap for a git URL when publishing) - const libp2p_dep = b.dependency("libp2p", .{ - .target = target, - .optimize = optimize, - }); - const libp2p_module = libp2p_dep.module("zig-libp2p"); - - // lsquic: C artifact needed for final linking - const lsquic_dep = libp2p_dep.builder.dependency("lsquic", .{ - .target = target, - .optimize = optimize, - }); - const lsquic_artifact = lsquic_dep.artifact("lsquic"); - - // peer_id: main.zig uses @import("peer_id") directly for PeerId + keys - const multiaddr_dep = libp2p_dep.builder.dependency("multiaddr", .{ - .target = target, - .optimize = optimize, - }); - const peer_id_dep = multiaddr_dep.builder.dependency("peer_id", .{ - .target = target, - .optimize = optimize, - }); - const peer_id_module = peer_id_dep.module("peer-id"); - - const exe_module = b.createModule(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - exe_module.addImport("zig-libp2p", libp2p_module); - exe_module.addImport("peer_id", peer_id_module); - - const exe = b.addExecutable(.{ - .name = "gossipsub-bin", - .root_module = exe_module, - }); - exe.linkLibrary(lsquic_artifact); - exe.linkSystemLibrary(switch (target.result.os.tag) { - .windows => "zlib1", - else => "z", - }); - - b.installArtifact(exe); - - const run_cmd = b.addRunArtifact(exe); - run_cmd.step.dependOn(b.getInstallStep()); - if (b.args) |args| run_cmd.addArgs(args); - b.step("run", "Run gossipsub-bin").dependOn(&run_cmd.step); -} diff --git a/gossipsub-interop/zig-libp2p/build.zig.zon b/gossipsub-interop/zig-libp2p/build.zig.zon deleted file mode 100644 index e7424a71f..000000000 --- a/gossipsub-interop/zig-libp2p/build.zig.zon +++ /dev/null @@ -1,15 +0,0 @@ -.{ - .name = .gossipsub_bin, - .version = "0.1.0", - .fingerprint = 0xa3892b19cdfabd6c, - .dependencies = .{ - .libp2p = .{ - .path = "../../../eth-p2p-z", - }, - }, - .paths = .{ - "build.zig", - "build.zig.zon", - "src", - }, -} diff --git a/gossipsub-interop/zig-libp2p/src/main.zig b/gossipsub-interop/zig-libp2p/src/main.zig deleted file mode 100644 index e1156c826..000000000 --- a/gossipsub-interop/zig-libp2p/src/main.zig +++ /dev/null @@ -1,106 +0,0 @@ -/// Phase 2.1 — Binary Skeleton -/// -/// Starts up, derives a deterministic PeerID from the Shadow node hostname, -/// logs it to stdout in the same JSON format as the Go binary, then exits. -/// -/// TODO Phase 2.3: wire TCP + TLS + Yamux host listening on port 9000 -/// TODO Phase 2.2: parse params file and run experiment instructions - -pub const std_options = @import("zig-libp2p").std_options; - -const std = @import("std"); -const libp2p = @import("zig-libp2p"); -const keys_mod = @import("peer_id").keys; -const PeerId = @import("peer_id").PeerId; - -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // ------------------------------------------------------------------ // - // 1. Parse --params flag - // ------------------------------------------------------------------ // - const args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, args); - - var params_path: ?[]const u8 = null; - var i: usize = 1; - while (i < args.len) : (i += 1) { - if (std.mem.eql(u8, args[i], "--params") and i + 1 < args.len) { - params_path = args[i + 1]; - i += 1; - } - } - if (params_path == null) { - std.log.err("--params flag is required", .{}); - std.process.exit(1); - } - - // ------------------------------------------------------------------ // - // 2. Hostname → node ID - // Shadow sets hostnames as "node0", "node1", … - // ------------------------------------------------------------------ // - var hostname_buf: [std.posix.HOST_NAME_MAX]u8 = undefined; - const hostname = try std.posix.gethostname(hostname_buf[0..]); - - var node_id: u64 = 0; - if (std.mem.startsWith(u8, hostname, "node")) { - node_id = std.fmt.parseInt(u64, hostname[4..], 10) catch blk: { - std.log.warn("could not parse node ID from hostname '{s}', defaulting to 0", .{hostname}); - break :blk 0; - }; - } - - // ------------------------------------------------------------------ // - // 3. Deterministic ED25519 key — matches Go's nodePrivKey(id): - // seed = [32]u8{0}, seed[0..8] = LE u64(node_id) - // privkey = ed25519.NewKeyFromSeed(seed) - // ------------------------------------------------------------------ // - const Ed25519 = std.crypto.sign.Ed25519; - var seed: [Ed25519.KeyPair.seed_length]u8 = [_]u8{0} ** Ed25519.KeyPair.seed_length; - std.mem.writeInt(u64, seed[0..8], node_id, .little); - const ed_kp = try Ed25519.KeyPair.generateDeterministic(seed); - - // ------------------------------------------------------------------ // - // 4. Derive libp2p PeerID from the ED25519 public key - // ------------------------------------------------------------------ // - var pub_key = keys_mod.PublicKey{ - .type = .ED25519, - .data = &ed_kp.public_key.bytes, - }; - const peer_id = try PeerId.fromPublicKey(allocator, &pub_key); - - const b58_len = peer_id.toBase58Len(); - const b58_buf = try allocator.alloc(u8, b58_len); - defer allocator.free(b58_buf); - const peer_id_str = try peer_id.toBase58(b58_buf); - - // ------------------------------------------------------------------ // - // 5. Log PeerID to stdout — matches Go slog JSON format: - // {"time":"...","level":"INFO","msg":"PeerID","id":"","node_id":N} - // ------------------------------------------------------------------ // - const now_s = std.time.timestamp(); - const stdout_file: std.fs.File = .{ .handle = std.posix.STDOUT_FILENO }; - var stdout_buf: [512]u8 = undefined; - var stdout_writer = stdout_file.writer(&stdout_buf); - try stdout_writer.interface.print( - "{{\"time\":\"{d}\",\"level\":\"INFO\",\"msg\":\"PeerID\",\"id\":\"{s}\",\"node_id\":{d}}}\n", - .{ now_s, peer_id_str, node_id }, - ); - try stdout_writer.interface.flush(); - - // ------------------------------------------------------------------ // - // 6. Initialise event loop (proves the runtime starts cleanly) - // TCP listener + gossipsub host wired in Phase 2.3 - // ------------------------------------------------------------------ // - var loop: libp2p.thread_event_loop.ThreadEventLoop = undefined; - try loop.init(allocator); - defer { - loop.close(); - loop.deinit(); - } - - std.log.info("Phase 2.1 skeleton started — node_id={d} peer_id={s}", .{ node_id, peer_id_str }); - // Phase 2.2 will read params_path and execute experiment instructions here -} From 84550ad39dc94aeeae5d26053a8abd39d586207a Mon Sep 17 00:00:00 2001 From: bomanaps Date: Sat, 9 May 2026 01:59:15 +0100 Subject: [PATCH 8/9] build zig with epoll backend for shadow compat --- gossipsub-interop/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gossipsub-interop/Makefile b/gossipsub-interop/Makefile index b71144619..736dcf32d 100644 --- a/gossipsub-interop/Makefile +++ b/gossipsub-interop/Makefile @@ -26,7 +26,7 @@ binaries: nim-libp2p/gossipsub-bin cd go-libp2p && go build -linkshared -o gossipsub-bin cd rust-libp2p && cargo build cd jvm-libp2p && ./gradlew installDist - zig build --build-file $(CURDIR)/../../eth-p2p-z/build.zig --prefix $(CURDIR)/zig-libp2p/zig-out -Doptimize=ReleaseFast + zig build --build-file $(CURDIR)/../../eth-p2p-z/build.zig --prefix $(CURDIR)/zig-libp2p/zig-out -Doptimize=ReleaseFast -Dlibxev-backend=epoll cp $(CURDIR)/zig-libp2p/zig-out/bin/gossipsub-bin $(CURDIR)/zig-libp2p/gossipsub-bin # Clean all generated shadow simulation files From 47e5731e36624393d5a65c1ee88dbd66db49007c Mon Sep 17 00:00:00 2001 From: bomanaps Date: Fri, 29 May 2026 21:12:07 +0100 Subject: [PATCH 9/9] add zig go interop target --- gossipsub-interop/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gossipsub-interop/Makefile b/gossipsub-interop/Makefile index 736dcf32d..b3d2dc033 100644 --- a/gossipsub-interop/Makefile +++ b/gossipsub-interop/Makefile @@ -126,4 +126,8 @@ test-zig: @echo "Testing all-zig subnet-blob-msg" @uv run run.py --node_count 4 --composition zig --scenario "subnet-blob-msg" && uv run checks/subnet_blob_msg.py latest/ -.PHONY: binaries all clean test test-rust-only test-go test-subnet-blob test-partial-messages test-zig +test-zig-go: + @echo "Testing zig+go subnet-blob-msg" + @uv run run.py --node_count 16 --composition zig go --scenario "subnet-blob-msg" && uv run checks/subnet_blob_msg.py latest/ + +.PHONY: binaries all clean test test-rust-only test-go test-subnet-blob test-partial-messages test-zig test-zig-go