diff --git a/BENCHMARKING.md b/BENCHMARKING.md index 31bd4d7..a9727fc 100644 --- a/BENCHMARKING.md +++ b/BENCHMARKING.md @@ -1,7 +1,7 @@ # Benchmarking phpcs-changed `benchmark.sh` measures the wall-clock time of running `phpcs-changed --git-staged` -against a set of changed PHP files, and compares the current branch to `trunk`. +against a set of changed PHP files, comparing the **current branch** to `trunk`. It uses [hyperfine](https://github.com/sharkdp/hyperfine) for statistical measurement and a self-contained throwaway git repo for reproducible test inputs. @@ -66,14 +66,10 @@ The worktree is created on the first run and reused on subsequent runs. ### What is being compared -| Branch | phpcs invocations per run | -|--------|--------------------------| -| trunk | up to `2 × N_FILES` — one per file per version (modified + unmodified) | -| feature branch | 1 — all file versions batched into a single phpcs call | - -The dominant cost in each phpcs invocation is **process startup** (~250–400 ms -on typical hardware). The batch approach eliminates `2 × N_FILES − 1` of those -startups. +The script runs the same `phpcs-changed --git-staged` command against the same +set of test files using two builds: the current branch and `trunk`. Hyperfine +runs both commands back-to-back the same number of times, so any difference in +wall-clock time reflects a real performance difference between the two builds. ## Interpreting results @@ -81,22 +77,14 @@ Hyperfine reports mean time, standard deviation, and a relative speedup ratio. Example output for `N_FILES=10`: ``` -Benchmark 1: batch (a75e2c2): 1 phpcs call +Benchmark 1: my-feature-branch (a75e2c2) Time (mean ± σ): 2.44 s ± 0.05 s -Benchmark 2: trunk (5c9f6b2): 20 phpcs calls +Benchmark 2: trunk (5c9f6b2) Time (mean ± σ): 5.74 s ± 0.11 s -Summary: batch ran 2.36 ± 0.06 times faster than trunk +Summary: my-feature-branch ran 2.36 ± 0.06 times faster than trunk ``` -The speedup scales roughly linearly with `N_FILES`. A project with 20 changed -files should see approximately twice the speedup of a 10-file project. - -The batch variant carries its own overhead (temp directory creation, one file -write per file version, one phpcs invocation on all files). This overhead -appears as a baseline cost that does not scale with `N_FILES`, so the -crossover point where batching wins is low — roughly 2 or more files. - ## Artifacts The following files are created outside the git index and are gitignored: diff --git a/benchmark.sh b/benchmark.sh index 6da6087..633d140 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# benchmark.sh — compare batch-phpcs-invocations branch vs trunk +# benchmark.sh — compare the current branch vs trunk # # Usage: # ./benchmark.sh @@ -17,7 +17,7 @@ set -euo pipefail # ── Paths ────────────────────────────────────────────────────────────────── SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BATCH_BIN="$SCRIPT_DIR/bin/phpcs-changed" +CURRENT_BIN="$SCRIPT_DIR/bin/phpcs-changed" TRUNK_WORKTREE="$SCRIPT_DIR/.bench-trunk" TRUNK_BIN="$TRUNK_WORKTREE/bin/phpcs-changed" PHPCS="$SCRIPT_DIR/vendor/bin/phpcs" @@ -51,8 +51,8 @@ git -C "$BENCH_REPO" config user.email "bench@example.com" git -C "$BENCH_REPO" config user.name "Benchmark" # Create initial PHP files with PSR2 violations (uppercase TRUE / FALSE). -# Both the committed and staged versions have violations so trunk will always -# scan both file versions — giving it 2×N_FILES phpcs invocations. +# Both the committed and staged versions have violations so both builds must +# scan both file versions — this is the worst-case and ensures a fair comparison. echo "→ Preparing $N_FILES PHP test files …" FILES=() for i in $(seq 1 "$N_FILES"); do @@ -98,16 +98,17 @@ done # ── 3. Run hyperfine ────────────────────────────────────────────────────── FILES_STR="${FILES[*]}" -BATCH_SHA="$(git -C "$SCRIPT_DIR" rev-parse --short HEAD)" +CURRENT_BRANCH="$(git -C "$SCRIPT_DIR" branch --show-current)" +CURRENT_SHA="$(git -C "$SCRIPT_DIR" rev-parse --short HEAD)" TRUNK_SHA="$(git -C "$TRUNK_WORKTREE" rev-parse --short HEAD)" -BATCH_CMD="php '$BATCH_BIN' --git-staged --phpcs-path='$PHPCS' --standard=PSR2 --always-exit-zero $FILES_STR" +CURRENT_CMD="php '$CURRENT_BIN' --git-staged --phpcs-path='$PHPCS' --standard=PSR2 --always-exit-zero $FILES_STR" TRUNK_CMD="php '$TRUNK_BIN' --git-staged --phpcs-path='$PHPCS' --standard=PSR2 --always-exit-zero $FILES_STR" echo "" printf "Benchmark: %d staged PHP files, --standard=PSR2\n" "$N_FILES" -printf " batch %s — 1 phpcs invocation (all files in one shot)\n" "$BATCH_SHA" -printf " trunk %s — up to %d phpcs invocations (2 per file)\n" "$TRUNK_SHA" "$((N_FILES * 2))" +printf " %-30s %s\n" "$CURRENT_BRANCH" "$CURRENT_SHA" +printf " %-30s %s\n" "trunk" "$TRUNK_SHA" echo "" cd "$BENCH_REPO" @@ -115,9 +116,9 @@ cd "$BENCH_REPO" hyperfine \ --warmup "$WARMUP" \ --runs "$RUNS" \ - -n "batch ($BATCH_SHA): 1 phpcs call" \ - -n "trunk ($TRUNK_SHA): $((N_FILES * 2)) phpcs calls" \ - "$BATCH_CMD" \ + -n "$CURRENT_BRANCH ($CURRENT_SHA)" \ + -n "trunk ($TRUNK_SHA)" \ + "$CURRENT_CMD" \ "$TRUNK_CMD" \ --export-markdown "$RESULTS_FILE"