feat: add callgrind measure

.github/scripts/callgrind.mjs

@@ -0,0 +1,169 @@
+// Run `$BENCHMARKS` under Valgrind's Callgrind with the `callgrind` measure
+// for the `callgrind` workflow; see `callgrind.yml`.
+
+import { strict as assert } from "node:assert";
+import { spawnSync } from "node:child_process";
+import * as fs from "node:fs";
+
+/// Run a command with inherited stdio, throwing if it fails.
+function exec(file, args) {
+  const result = spawnSync(file, args, { stdio: "inherit" });
+  assert.equal(
+    result.status,
+    0,
+    `\`${file} ${args.join(" ")}\` exited with ${result.status}`,
+  );
+}
+
+/// Shorten a benchmark's wasm path to a human-friendly name, e.g.
+/// `benchmarks/bz2/benchmark.wasm` to `bz2` and
+/// `benchmarks/cm-online-stats/cm-online-stats.wasm` to `cm-online-stats`.
+function shortName(wasm) {
+  const name = wasm
+    .replace(/^.*benchmarks\//, "")
+    .replace(/\.wasm$/, "")
+    .replace(/\/benchmark$/, "");
+  const parts = name.split("/");
+  return parts.length === 2 && parts[0] === parts[1] ? parts[0] : name;
+}
+
+/// Run the benchmarks, render a summary table, check the results, and (with
+/// at least three iterations) generate an HTML report. Everything is left in
+/// the `callgrind-results` directory for upload as a workflow artifact.
+async function main() {
+  const benchmarks = (process.env.BENCHMARKS || process.env.DEFAULT_BENCHMARKS)
+    .split(/\s+/)
+    .filter(Boolean);
+  const iterations = parseInt(process.env.ITERATIONS || "1", 10);
+
+  // `sightglass-cli` launches each benchmark process under Valgrind's
+  // Callgrind itself when the `callgrind` measure is selected. (Valgrind is
+  // installed by the workflow before sightglass is even built: the Valgrind
+  // headers must be present at build time.)
+  const run = spawnSync(
+    "target/debug/sightglass-cli",
+    [
+      "benchmark",
+      "--engine",
+      "engines/wasmtime/libengine.so",
+      "--measure",
+      "callgrind",
+      "--processes",
+      "1",
+      "--iterations-per-process",
+      String(iterations),
+      "--small-workloads",
+      "--raw",
+      "--output-format",
+      "json",
+      "--",
+      ...benchmarks,
+    ],
+    {
+      stdio: ["ignore", "pipe", "inherit"],
+      encoding: "utf8",
+      maxBuffer: 64 * 1024 * 1024,
+    },
+  );
+  console.log(run.stdout);
+  assert.equal(run.status, 0, `sightglass-cli exited with ${run.status}`);
+
+  fs.mkdirSync("callgrind-results");
+  fs.writeFileSync("callgrind-results/results.json", run.stdout);
+  for (const file of fs.readdirSync(".")) {
+    if (file.startsWith("callgrind.out.")) {
+      fs.renameSync(file, `callgrind-results/${file}`);
+    }
+  }
+
+  // Group the measurements by benchmark.
+  const byWasm = new Map();
+  for (const m of JSON.parse(run.stdout)) {
+    if (m.event !== "callgrind-ir") {
+      continue;
+    }
+    if (!byWasm.has(m.wasm)) {
+      byWasm.set(m.wasm, []);
+    }
+    byWasm.get(m.wasm).push(m);
+  }
+
+  // Display each benchmark by its short name, unless two benchmarks share one.
+  const displayNames = new Map();
+  {
+    const occurrences = new Map();
+    for (const wasm of byWasm.keys()) {
+      const name = shortName(wasm);
+      occurrences.set(name, (occurrences.get(name) ?? 0) + 1);
+    }
+    for (const wasm of byWasm.keys()) {
+      const name = shortName(wasm);
+      displayNames.set(wasm, occurrences.get(name) === 1 ? name : wasm);
+    }
+  }
+
+  // A phase's instruction counts across iterations; deterministic counts
+  // collapse to a single value.
+  const phaseCounts = (ms, phase) => {
+    const counts = new Set(
+      ms.filter((m) => m.phase === phase).map((m) => m.count),
+    );
+    return [...counts].join(" / ");
+  };
+
+  // Render a table of the results on the workflow run's summary page. This
+  // happens before the checks below so that a failing run still reports what
+  // it measured.
+  const summary = [
+    "### Callgrind instruction counts (`callgrind-ir`)",
+    "",
+    `Engine: \`wasmtime@${process.env.WASMTIME_SHA}\``,
+    "",
+    "| Benchmark | Compilation | Instantiation | Execution |",
+    "| --- | ---: | ---: | ---: |",
+    ...[...byWasm.entries()].map(([wasm, ms]) => {
+      const phases = ["Compilation", "Instantiation", "Execution"]
+        .map((phase) => phaseCounts(ms, phase))
+        .join(" | ");
+      return `| ${displayNames.get(wasm)} | ${phases} |`;
+    }),
+    "",
+    "Distinct counts across iterations are shown separated by ` / `; the raw" +
+      " per-phase Callgrind dumps and the JSON measurements are available in" +
+      " the `callgrind-results` artifact.",
+    "",
+  ].join("\n");
+  fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary);
+
+  // Every benchmark must have recorded an instruction count for each of its
+  // three phases on every iteration.
+  assert.ok(byWasm.size >= 1, "no benchmarks were measured");
+  for (const [wasm, ms] of byWasm) {
+    assert.equal(
+      ms.length,
+      3 * iterations,
+      `wrong number of instruction counts for ${wasm}`,
+    );
+  }
+
+  // With enough samples, also generate an HTML report.
+  if (iterations >= 3) {
+    exec("target/debug/sightglass-cli", [
+      "report",
+      "-i",
+      "json",
+      "--event",
+      "callgrind-ir",
+      "--phase",
+      "execution",
+      "-o",
+      "callgrind-results/report.html",
+      "callgrind-results/results.json",
+    ]);
+  }
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});

.github/scripts/resolve-wasmtime-ref.mjs

@@ -0,0 +1,33 @@
+// Resolve `WASMTIME_REF` (a branch, a tag, or a full commit hash; `main` by
+// default) to a full commit hash and emit it as the step's `sha` output. Used
+// by the `callgrind` workflow; see `callgrind.yml`.
+
+import { strict as assert } from "node:assert";
+import { execFileSync } from "node:child_process";
+import * as fs from "node:fs";
+
+const WASMTIME_REPOSITORY = "https://github.com/bytecodealliance/wasmtime/";
+
+async function main() {
+  const ref = process.env.WASMTIME_REF || "main";
+  let sha;
+  if (/^[0-9a-f]{40}$/.test(ref)) {
+    sha = ref;
+  } else {
+    const refs = execFileSync("git", ["ls-remote", WASMTIME_REPOSITORY, ref], {
+      encoding: "utf8",
+    });
+    sha = refs.split("\n")[0]?.split("\t")[0];
+  }
+  assert.match(
+    sha ?? "",
+    /^[0-9a-f]{40}$/,
+    `unable to resolve wasmtime ref \`${ref}\``,
+  );
+  fs.appendFileSync(process.env.GITHUB_OUTPUT, `sha=${sha}\n`);
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});

.github/workflows/callgrind.yml

@@ -0,0 +1,127 @@
+name: callgrind
+
+# Run benchmarks with the `callgrind` measure,
+# recording deterministic per-phase instruction counts against the latest
+# wasmtime commit. The JSON measurements and the raw per-phase Callgrind dump
+# files are uploaded as a workflow artifact, and a summary table is rendered on
+# the workflow run page.
+
+on:
+  push:
+    branches: [ main ]
+  schedule:
+    # Run nightly at 04:23 UTC.
+    - cron: "23 4 * * *"
+  workflow_dispatch:
+    inputs:
+      benchmarks:
+        description: >-
+          Space-separated benchmark `.wasm` (or `.suite`) paths to run.
+          Defaults to the default suite -- minus spidermonkey, whose large
+          module takes too long to compile under Callgrind's (serialized)
+          simulation -- plus the component-model benchmark.
+        required: false
+        default: ""
+      iterations:
+        description: >-
+          Iterations per process. The instruction counts are deterministic so
+          `1` is usually enough; use `3` or more to also generate an HTML
+          report in the artifact.
+        required: false
+        default: "1"
+      wasmtime-ref:
+        description: >-
+          The wasmtime revision to benchmark: a branch, a tag, or a full
+          commit hash. Defaults to `main`.
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_LOG: info
+  DEFAULT_BENCHMARKS: >-
+    benchmarks/noop/benchmark.wasm
+    benchmarks/bz2/benchmark.wasm
+    benchmarks/pulldown-cmark/benchmark.wasm
+    benchmarks/cm-online-stats/cm-online-stats.wasm
+
+jobs:
+  callgrind:
+    # Don't run scheduled jobs on forks.
+    if: github.event_name != 'schedule' || github.repository == 'bytecodealliance/sightglass'
+    runs-on: ubuntu-latest
+    # The default benchmark set takes well under an hour, but leave headroom
+    # for `workflow_dispatch` runs over larger benchmark sets or with more
+    # iterations.
+    timeout-minutes: 120
+    steps:
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      with:
+        persist-credentials: false
+    - run: rustup update
+
+    # Benchmark against the requested wasmtime revision (the latest commit on
+    # `main` by default), caching the built engine by commit hash so that
+    # back-to-back runs do not rebuild it.
+    - name: Resolve the wasmtime revision
+      id: wasmtime
+      env:
+        WASMTIME_REF: ${{ inputs.wasmtime-ref }}
+      run: node .github/scripts/resolve-wasmtime-ref.mjs
+    - name: Download cached wasmtime engine
+      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+      id: wasmtime-cache
+      with:
+        path: engines/wasmtime/*
+        key: callgrind-wasmtime-${{ runner.os }}-${{ steps.wasmtime.outputs.sha }}
+    - name: Build wasmtime engine
+      if: steps.wasmtime-cache.outputs.cache-hit != 'true'
+      working-directory: ./engines/wasmtime
+      env:
+        REVISION: ${{ steps.wasmtime.outputs.sha }}
+      run: |
+        rustc build.rs
+        ./build
+
+    # Cache the cargo registry and build artifacts for the sightglass-cli
+    # build. `cargo build` still runs below to pick up any source changes; a
+    # warm cache makes it incremental.
+    - name: Cache sightglass-cli build
+      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          target
+        key: callgrind-sightglass-${{ runner.os }}-${{ hashFiles('Cargo.lock') }}
+        restore-keys: |
+          callgrind-sightglass-${{ runner.os }}-
+
+    # Valgrind must be installed BEFORE building sightglass: the Valgrind
+    # client-request support in sightglass-recorder is only compiled in when
+    # the Valgrind headers are present at build time.
+    - name: Install valgrind
+      run: |
+        sudo apt-get update -q
+        sudo apt-get install -y -q valgrind
+
+    - name: Build sightglass-cli
+      run: cargo build
+
+    - name: Benchmark with the callgrind measure
+      env:
+        BENCHMARKS: ${{ inputs.benchmarks }}
+        ITERATIONS: ${{ inputs.iterations }}
+        WASMTIME_SHA: ${{ steps.wasmtime.outputs.sha }}
+      run: node .github/scripts/callgrind.mjs
+
+    - name: Upload callgrind results
+      if: always()
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      with:
+        name: callgrind-results
+        path: callgrind-results/
+        if-no-files-found: warn