Add objdump parser benchmark with aggregated JSON output

.github/workflows/benchmark.yml

@@ -0,0 +1,68 @@
+name: Benchmark
+
+# Runs automatically on every push to any branch and on pull requests, producing
+# a benchmark summary artifact (the main-branch runs can be promoted to a
+# baseline for future comparisons). Also exposes a manual trigger for ad-hoc reruns.
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      runs:
+        description: "Number of benchmark process runs to average"
+        required: false
+        default: "5"
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Install Conan
+        id: conan
+        uses: turtlebrowser/get-conan@v1.2
+        with:
+          version: 2.8.1
+
+      - name: Configure conan
+        run: |
+          conan profile detect
+          sed -i 's/compiler\.cppstd=.*/compiler.cppstd=20/g' ~/.conan2/profiles/default
+          conan install --build=fmt/11.0.0 .
+
+      - name: Build (Release)
+        run: |
+          CURDIR=$PWD
+          mkdir -p build-release
+          cd build-release
+          cmake "-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=$CURDIR/conan_provider.cmake" -DCMAKE_C_COMPILER=/usr/bin/gcc-10 -DCMAKE_CXX_COMPILER=/usr/bin/g++-10 -DCMAKE_BUILD_TYPE=Release ..
+          cmake --build .
+
+      - name: Run benchmarks
+        env:
+          BENCH_RUNS: ${{ github.event.inputs.runs }}
+          BENCH_LABEL: ${{ runner.os }}/${{ runner.arch }}
+        run: |
+          ./scripts/run-benchmarks.sh
+
+      - name: Show summary
+        run: cat bench-results/bench-summary.json
+
+      - name: Compare against baseline
+        run: |
+          python3 scripts/compare_bench.py \
+            --baseline bench-results/baseline.json \
+            --current bench-results/bench-summary.json \
+            --threshold "${BENCH_THRESHOLD:-10}"
+        env:
+          BENCH_THRESHOLD: "10"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: bench-summary
+          path: |
+            bench-results/bench-summary.json
+            bench-results/run-*.xml

.github/workflows/build.yml

@@ -12,11 +12,11 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
 
       - name: Install Conan
         id: conan
-        uses: turtlebrowser/get-conan@main
+        uses: turtlebrowser/get-conan@v1.2
         with:
           version: 2.8.1
 
@@ -42,12 +42,12 @@ jobs:
         run: |
           $GITHUB_WORKSPACE/build/src/asm-parser $GITHUB_WORKSPACE/resources/example_intel.asm > ./example_intel.json
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         with:
           path: build/src/asm-parser
 
       - name: Release
-        uses: fnkr/github-action-ghr@v1
+        uses: fnkr/github-action-ghr@v1.3
         if: startsWith(github.ref, 'refs/tags/')
         env:
           GHR_COMPRESS: xz

.gitignore

@@ -32,6 +32,10 @@
 *.app
 
 build/
+build-release/
+# ignore generated benchmark output, but keep the committed baseline
+/bench-results/*
+!/bench-results/baseline.json
 CMakeUserPresets.json
 
 # CLion stuff

bench-results/baseline.json

@@ -0,0 +1,90 @@
+{
+  "schema": "asm-parser-bench/1",
+  "generated_utc": "2026-06-27T14:08:48.248475+00:00",
+  "commit": "c3440213b27ecae1583337c4b446991da21341a0",
+  "ref": "HEAD",
+  "label": "Linux/X64",
+  "run_count": 5,
+  "benchmarks": [
+    {
+      "name": "parse example.asm",
+      "runs": 5,
+      "samples_per_run": 100,
+      "mean_ns": 155782.8,
+      "mean_ms": 0.1557828,
+      "stddev_across_runs_ns": 1852.4474891343075,
+      "stddev_across_runs_ms": 0.0018524474891343074,
+      "cv_percent": 1.1891219628446192,
+      "min_ms": 0.154184,
+      "max_ms": 0.158972,
+      "mean_within_run_stddev_ms": 0.016918420000000003,
+      "per_run_mean_ms": [
+        0.155571,
+        0.155049,
+        0.154184,
+        0.155138,
+        0.158972
+      ]
+    },
+    {
+      "name": "parse example_intel.asm",
+      "runs": 5,
+      "samples_per_run": 100,
+      "mean_ns": 7681760.0,
+      "mean_ms": 7.68176,
+      "stddev_across_runs_ns": 8631.949374272303,
+      "stddev_across_runs_ms": 0.008631949374272302,
+      "cv_percent": 0.11236942281810812,
+      "min_ms": 7.67117,
+      "max_ms": 7.69446,
+      "mean_within_run_stddev_ms": 0.08032122,
+      "per_run_mean_ms": [
+        7.67117,
+        7.69446,
+        7.67717,
+        7.68252,
+        7.68348
+      ]
+    },
+    {
+      "name": "parse gcc12_sort_object_reloc.asm",
+      "runs": 5,
+      "samples_per_run": 100,
+      "mean_ns": 6399328.0,
+      "mean_ms": 6.399328,
+      "stddev_across_runs_ns": 95071.93550149276,
+      "stddev_across_runs_ms": 0.09507193550149276,
+      "cv_percent": 1.4856549859843526,
+      "min_ms": 6.33027,
+      "max_ms": 6.56389,
+      "mean_within_run_stddev_ms": 0.33537957999999995,
+      "per_run_mean_ms": [
+        6.56389,
+        6.37586,
+        6.38739,
+        6.33027,
+        6.33923
+      ]
+    },
+    {
+      "name": "parse gcc12_bin_fmt_O2_flto.asm",
+      "runs": 5,
+      "samples_per_run": 100,
+      "mean_ns": 117524800.0,
+      "mean_ms": 117.5248,
+      "stddev_across_runs_ns": 272434.76283323317,
+      "stddev_across_runs_ms": 0.27243476283323315,
+      "cv_percent": 0.231810445823548,
+      "min_ms": 117.144,
+      "max_ms": 117.791,
+      "mean_within_run_stddev_ms": 0.9706758000000001,
+      "per_run_mean_ms": [
+        117.604,
+        117.791,
+        117.735,
+        117.35,
+        117.144
+      ]
+    }
+  ]
+}

scripts/aggregate_bench.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""Aggregate one or more Catch2 (v2) XML benchmark runs into a summary JSON.
+
+Each input XML is one process run of the benchmark (which itself takes many
+samples). This script combines the per-run means for each benchmark into
+run-to-run statistics suitable for storing as a CI artifact / baseline.
+
+Usage:
+    aggregate_bench.py run-*.xml -o bench-summary.json
+    aggregate_bench.py run-1.xml run-2.xml --commit "$(git rev-parse HEAD)"
+
+All timing values in the Catch2 XML are nanoseconds; the summary keeps raw
+nanoseconds and adds millisecond convenience values.
+"""
+import argparse
+import datetime
+import json
+import statistics
+import sys
+import xml.etree.ElementTree as ET
+
+
+def parse_run(path):
+    """Return {benchmark_name: {mean_ns, stddev_ns, samples}} for one XML run."""
+    root = ET.parse(path).getroot()
+    out = {}
+    for bench in root.iter("BenchmarkResults"):
+        mean = bench.find("mean")
+        stddev = bench.find("standardDeviation")
+        out[bench.attrib["name"]] = {
+            "mean_ns": float(mean.attrib["value"]) if mean is not None else None,
+            "stddev_ns": float(stddev.attrib["value"]) if stddev is not None else None,
+            "samples": int(bench.attrib["samples"]),
+        }
+    return out
+
+
+def aggregate(paths):
+    runs = [parse_run(p) for p in paths]
+    # Preserve first-seen order of benchmark names.
+    names = []
+    for run in runs:
+        for name in run:
+            if name not in names:
+                names.append(name)
+
+    benchmarks = []
+    for name in names:
+        means = [run[name]["mean_ns"] for run in runs if name in run]
+        within = [run[name]["stddev_ns"] for run in runs if name in run]
+        samples = next(run[name]["samples"] for run in runs if name in run)
+
+        mean_ns = statistics.fmean(means)
+        across_ns = statistics.stdev(means) if len(means) > 1 else 0.0
+        benchmarks.append({
+            "name": name,
+            "runs": len(means),
+            "samples_per_run": samples,
+            "mean_ns": mean_ns,
+            "mean_ms": mean_ns / 1e6,
+            "stddev_across_runs_ns": across_ns,
+            "stddev_across_runs_ms": across_ns / 1e6,
+            "cv_percent": (across_ns / mean_ns * 100.0) if mean_ns else 0.0,
+            "min_ms": min(means) / 1e6,
+            "max_ms": max(means) / 1e6,
+            "mean_within_run_stddev_ms": (statistics.fmean(within) / 1e6) if within else None,
+            "per_run_mean_ms": [m / 1e6 for m in means],
+        })
+    return benchmarks
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("xml", nargs="+", help="Catch2 XML run file(s)")
+    ap.add_argument("-o", "--output", help="JSON output file (default: stdout)")
+    ap.add_argument("--commit", default="", help="git commit SHA to record")
+    ap.add_argument("--ref", default="", help="git ref/branch to record")
+    ap.add_argument("--label", default="", help="free-form label (e.g. runner/build)")
+    args = ap.parse_args()
+
+    summary = {
+        "schema": "asm-parser-bench/1",
+        "generated_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+        "commit": args.commit,
+        "ref": args.ref,
+        "label": args.label,
+        "run_count": len(args.xml),
+        "benchmarks": aggregate(args.xml),
+    }
+    text = json.dumps(summary, indent=2)
+
+    if args.output:
+        with open(args.output, "w") as f:
+            f.write(text + "\n")
+        print(f"Wrote {args.output}", file=sys.stderr)
+    else:
+        print(text)
+
+
+if __name__ == "__main__":
+    main()