Improve performance metrics

.github/workflows/README.md

@@ -8,11 +8,12 @@ This folder contains the repository's GitHub Actions workflows.
 - `ci-full.yml`: broader matrix CI for pull requests to `develop` and `master`.
 - `docs.yml`: Documenter build and deployment workflow.
 - `TagBot.yml`: release-tag automation.
+- `nightly.yml`: scheduled benchmarking on non-Linux platforms.
 
 ### Disabled
 
 - `format.yml.disabled`: disabled formatter workflow kept for reference.
 - `CompatHelper.yml.disabled`: dependency update automation. 
-- `nightly.yml.disabled`: scheduled regression sweep on non-Linux platforms.
+
 
 See [`../../docs/src/ci-testing.md`](../../docs/src/ci-testing.md) for the user-facing workflow overview.

.github/workflows/ci-full.yml

@@ -14,7 +14,7 @@ jobs:
     name: Julia ${{ matrix.julia-version }} – ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         julia-version: ['1.10', '1.11']
         os: [ubuntu-latest, macos-latest, windows-latest]
@@ -46,7 +46,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     needs: test
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
 

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     name: Julia ${{ matrix.julia-version }} – Linux
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         julia-version: ['1.12']
 

.github/workflows/nightly.yml.disabled → .github/workflows/nightly.yml

@@ -14,12 +14,10 @@ jobs:
     name: Nightly – Julia ${{ matrix.julia-version }} – ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         include:
-          - os: macos-latest
-            julia-version: '1.12'
-          - os: windows-latest
+          - os: ubuntu-latest
             julia-version: '1.12'
 
     steps:
@@ -41,5 +39,5 @@ jobs:
       - name: Install dependencies
         run: julia --project=. -e 'using Pkg; Pkg.instantiate()'
 
-      - name: Run unit tests
-        run: julia --project=. -e 'using Pkg; Pkg.test()'
+      - name: Run benchmark
+        run: make ci

Makefile

@@ -15,7 +15,9 @@ QMCPY_PYTHON_AUTO := $(shell \
 	done)
 PYTHON ?= $(if $(QMCPY_PYTHON_AUTO),$(QMCPY_PYTHON_AUTO),python)
 BENCH_COVERAGE ?= 0
+BENCH_BLAS_THREADS ?= 1
 JULIA_BENCH_COVERAGE_FLAG := $(if $(filter 1,$(BENCH_COVERAGE)),--code-coverage=user,)
+BENCH_THREAD_ENV := QMC_BENCH_BLAS_THREADS=$(BENCH_BLAS_THREADS) OPENBLAS_NUM_THREADS=$(BENCH_BLAS_THREADS) MKL_NUM_THREADS=$(BENCH_BLAS_THREADS) OMP_NUM_THREADS=$(BENCH_BLAS_THREADS) NUMEXPR_NUM_THREADS=$(BENCH_BLAS_THREADS)
 
 define RUN_TIMED
 	@start=$$(date +%s); \
@@ -136,7 +138,7 @@ notebook-%:
 # Override output label with: make bench LABEL=gaus or make bench gaus
 # Saves results to benchmark/results/latest.json (default) or <LABEL>.json
 bench:
-	$(call RUN_TIMED,BENCH_COVERAGE=$(BENCH_COVERAGE) julia $(JULIA_BENCH_COVERAGE_FLAG) benchmark/runbenchmarks.jl $(LABEL),bench)
+	$(call RUN_TIMED,$(BENCH_THREAD_ENV) BENCH_COVERAGE=$(BENCH_COVERAGE) julia $(JULIA_BENCH_COVERAGE_FLAG) benchmark/runbenchmarks.jl $(LABEL),bench)
 
 check-qmcpy-python:
 	@$(PYTHON) -c "import qmcpy" >/dev/null 2>&1 || \
@@ -152,7 +154,7 @@ check-qmcpy-python:
 REV ?= HEAD
 LABEL ?=
 bench-compare:
-	$(call RUN_TIMED,BENCH_COVERAGE=$(BENCH_COVERAGE) julia $(JULIA_BENCH_COVERAGE_FLAG) benchmark/compare.jl $(REV) $(LABEL),bench-compare)
+	$(call RUN_TIMED,$(BENCH_THREAD_ENV) BENCH_COVERAGE=$(BENCH_COVERAGE) julia $(JULIA_BENCH_COVERAGE_FLAG) benchmark/compare.jl $(REV) $(LABEL),bench-compare)
 
 # Side-by-side Julia vs QMCPy comparison.
 # Runs both the Julia and QMCPy benchmark harnesses for the requested labels.
@@ -163,35 +165,40 @@ bench-compare:
 # If LABEL is set, it also becomes the default input label for both sides.
 JL_LABEL ?= $(if $(LABEL),$(LABEL),latest)
 PY_LABEL ?= $(JL_LABEL)
+BENCH_COMPARE_OUT := $(if $(LABEL),benchmark/results/compare_$(LABEL).md,benchmark/results/compare_head.md)
+BENCH_COMPARE_PY_OUT := $(if $(LABEL),benchmark/results/compare_python_$(LABEL).md,benchmark/results/compare_python.md)
 bench-compare-py: bench check-qmcpy-python
-	$(call RUN_TIMED,$(PYTHON) benchmark/benchmark_qmcpy.py $(PY_LABEL) && BENCH_COVERAGE=$(BENCH_COVERAGE) julia $(JULIA_BENCH_COVERAGE_FLAG) benchmark/compare_py.jl $(JL_LABEL) $(PY_LABEL) $(LABEL),bench-compare-py)
+	$(call RUN_TIMED,$(BENCH_THREAD_ENV) $(PYTHON) benchmark/benchmark_qmcpy.py $(PY_LABEL) && $(BENCH_THREAD_ENV) BENCH_COVERAGE=$(BENCH_COVERAGE) julia $(JULIA_BENCH_COVERAGE_FLAG) benchmark/compare_py.jl $(JL_LABEL) $(PY_LABEL) $(LABEL),bench-compare-py)
 
 # Explicit labeled Julia-vs-QMCPy comparison flow.
 # Usage: make bench-compare-py-label LABEL=base
 bench-compare-py-label: check-qmcpy-python
-	$(call RUN_TIMED,BENCH_COVERAGE=$(BENCH_COVERAGE) julia $(JULIA_BENCH_COVERAGE_FLAG) benchmark/runbenchmarks.jl $(LABEL) && $(PYTHON) benchmark/benchmark_qmcpy.py $(LABEL) && BENCH_COVERAGE=$(BENCH_COVERAGE) julia $(JULIA_BENCH_COVERAGE_FLAG) benchmark/compare_py.jl $(LABEL) $(LABEL) $(LABEL),bench-compare-py-label)
+	$(call RUN_TIMED,$(BENCH_THREAD_ENV) BENCH_COVERAGE=$(BENCH_COVERAGE) julia $(JULIA_BENCH_COVERAGE_FLAG) benchmark/runbenchmarks.jl $(LABEL) && $(BENCH_THREAD_ENV) $(PYTHON) benchmark/benchmark_qmcpy.py $(LABEL) && $(BENCH_THREAD_ENV) BENCH_COVERAGE=$(BENCH_COVERAGE) julia $(JULIA_BENCH_COVERAGE_FLAG) benchmark/compare_py.jl $(LABEL) $(LABEL) $(LABEL),bench-compare-py-label)
 
 # Run the labeled Julia-only comparison and Julia-vs-QMCPy comparison in one task.
+# This target does not define a third ratio; inspect:
+#   - $(BENCH_COMPARE_OUT)      with ratio = reference ÷ local
+#   - $(BENCH_COMPARE_PY_OUT)   with ratio = Python ÷ Julia
 # Usage: make bench-all LABEL=base
 bench-all:
-	$(call RUN_TIMED,$(MAKE) bench-compare BENCH_COVERAGE=$(BENCH_COVERAGE) LABEL=$(LABEL) && $(MAKE) bench-compare-py BENCH_COVERAGE=$(BENCH_COVERAGE) LABEL=$(LABEL),bench-all)
+	$(call RUN_TIMED,$(MAKE) bench-compare BENCH_COVERAGE=$(BENCH_COVERAGE) BENCH_BLAS_THREADS=$(BENCH_BLAS_THREADS) LABEL=$(LABEL) && $(MAKE) bench-compare-py BENCH_COVERAGE=$(BENCH_COVERAGE) BENCH_BLAS_THREADS=$(BENCH_BLAS_THREADS) LABEL=$(LABEL) && printf '\n[bench-all] wrote %s (ratio = reference ÷ local) and %s (ratio = Python ÷ Julia)\n' "$(BENCH_COMPARE_OUT)" "$(BENCH_COMPARE_PY_OUT)",bench-all)
 
 # Run the benchmark suite with coverage enabled and produce an lcov report over
 # both src/ and benchmark/ coverage files.
 bench-coverage:
-	$(call RUN_TIMED,find src benchmark -name '*.cov' -delete && rm -f lcov.info && $(MAKE) bench BENCH_COVERAGE=1 && julia --project=. devtools/process_coverage.jl src benchmark && find src benchmark -name '*.cov' -delete,bench-coverage)
+	$(call RUN_TIMED,find src benchmark -name '*.cov' -delete && rm -f lcov.info && $(MAKE) bench BENCH_COVERAGE=1 BENCH_BLAS_THREADS=$(BENCH_BLAS_THREADS) && julia --project=. devtools/process_coverage.jl src benchmark && find src benchmark -name '*.cov' -delete,bench-coverage)
 
 # Run the Julia-vs-Julia comparison with coverage enabled and produce an lcov report.
 bench-compare-coverage:
-	$(call RUN_TIMED,find src benchmark -name '*.cov' -delete && rm -f lcov.info && $(MAKE) bench-compare BENCH_COVERAGE=1 REV=$(REV) LABEL=$(LABEL) && julia --project=. devtools/process_coverage.jl src benchmark && find src benchmark -name '*.cov' -delete,bench-compare-coverage)
+	$(call RUN_TIMED,find src benchmark -name '*.cov' -delete && rm -f lcov.info && $(MAKE) bench-compare BENCH_COVERAGE=1 BENCH_BLAS_THREADS=$(BENCH_BLAS_THREADS) REV=$(REV) LABEL=$(LABEL) && julia --project=. devtools/process_coverage.jl src benchmark && find src benchmark -name '*.cov' -delete,bench-compare-coverage)
 
 # Run the Julia-vs-QMCPy comparison with coverage enabled and produce an lcov report.
 bench-compare-py-coverage:
-	$(call RUN_TIMED,find src benchmark -name '*.cov' -delete && rm -f lcov.info && $(MAKE) bench-compare-py BENCH_COVERAGE=1 LABEL=$(LABEL) JL_LABEL=$(JL_LABEL) PY_LABEL=$(PY_LABEL) && julia --project=. devtools/process_coverage.jl src benchmark && find src benchmark -name '*.cov' -delete,bench-compare-py-coverage)
+	$(call RUN_TIMED,find src benchmark -name '*.cov' -delete && rm -f lcov.info && $(MAKE) bench-compare-py BENCH_COVERAGE=1 BENCH_BLAS_THREADS=$(BENCH_BLAS_THREADS) LABEL=$(LABEL) JL_LABEL=$(JL_LABEL) PY_LABEL=$(PY_LABEL) && julia --project=. devtools/process_coverage.jl src benchmark && find src benchmark -name '*.cov' -delete,bench-compare-py-coverage)
 
 # Run the full labeled benchmark workflow with coverage enabled and produce an lcov report.
 bench-all-coverage:
-	$(call RUN_TIMED,find src benchmark -name '*.cov' -delete && rm -f lcov.info && $(MAKE) bench-all BENCH_COVERAGE=1 LABEL=$(LABEL) && julia --project=. devtools/process_coverage.jl src benchmark && find src benchmark -name '*.cov' -delete,bench-all-coverage)
+	$(call RUN_TIMED,find src benchmark -name '*.cov' -delete && rm -f lcov.info && $(MAKE) bench-all BENCH_COVERAGE=1 BENCH_BLAS_THREADS=$(BENCH_BLAS_THREADS) LABEL=$(LABEL) && julia --project=. devtools/process_coverage.jl src benchmark && find src benchmark -name '*.cov' -delete,bench-all-coverage)
 
 # Compare two saved Julia benchmark-result labels and decide which one is better.
 # Usage: make bench-compare-labels LABEL_A=a LABEL_B=b [OUT_LABEL=report]
@@ -203,7 +210,7 @@ bench-compare-labels:
 # Combination of above targets
 # ============================================================================
 
-# Run the local full-check pipeline: format code, collect unit-test,
+# Run the full-check pipeline: format code, collect unit-test,
 # then collect full benchmark. Pass LABEL=... through to the benchmark step.
-local-ci:
-	$(call RUN_TIMED,$(MAKE) format && $(MAKE) test && $(MAKE) bench-all LABEL=$(LABEL),local-ci)
+ci:
+	$(call RUN_TIMED,$(MAKE) format && $(MAKE) test && $(MAKE) bench-all LABEL=$(LABEL),local-ci)

README.md

@@ -25,18 +25,23 @@ println("Exact:    $(keister_exact(3))")
 
 ## Components
 
-QMC.jl provides four building blocks that snap together:
+QMC.jl provides four core building blocks, plus related kernel types:
 
 | Component | Types |
 |---|---|
 | **Discrete Distribution** | `IIDStdUniform`, `Lattice`, `DigitalNetB2`, `Halton`, `Kronecker`, `DigitalNetAnyBases`, `Faure` |
-| **True Measure** | `Uniform`, `Gaussian`, `BrownianMotion`, `Lebesgue`, `GeometricBrownianMotion`, `StudentT`, `Triangular`, `Kumaraswamy`, `JohnsonsSU`, `BernoulliCont`, `AcceptanceRejection`, `DistributionsWrapper`, `MaternGP`, `UniformTriangle`, `ZeroInflatedExpUniform` |
+| **True Measure** | `Uniform`, `Gaussian`, `BrownianMotion`, `Lebesgue`, `GeometricBrownianMotion`, `StudentT`, `Triangular`, `Kumaraswamy`, `JohnsonsSU`, `BernoulliCont`, `AcceptanceRejection`, `AcceptanceRejectionReal`, `DistributionsWrapper`, `MaternGP`, `UniformTriangle`, `ZeroInflatedExpUniform` |
 | **Integrand** | `Keister`, `Genz`, `AsianOption`, `FinancialOption`, `FinancialOptionML`, `BoxIntegral`, `Linear0`, `Sin1D`, `Ishigami`, `Hartmann6D`, `Multimodal2D`, `FourBranch2D`, `SensitivityIndices`, `BayesianLRCoeffs`, `UMBridgeWrapper`, `CustomFun` |
 | **Stopping Criterion** | `CubMCCLT`, `CubMCCLTVec`, `CubMCG`, `CubQMCLatticeG`, `CubQMCNetG`, `CubQMCNetGRep`, `CubQMCBayesLatticeG`, `CubQMCBayesNetG`, `CubQMCRepStudentT`, `CubMLMC`, `CubMLMCCont`, `CubMLQMC`, `CubMLQMCCont`, `PFGPCI` |
-| **Kernel** | `KernelShiftInvar`, `KernelDigShiftInvar`, `KernelMatern12`, `KernelMatern32`, `KernelMatern52`, `KernelGaussian`, `KernelMultiTask`, `SumKernel`, `ProductKernel` |
+| **Kernel** | `KernelShiftInvar`, `KernelDigShiftInvar`, `KernelMatern12`, `KernelMatern32`, `KernelMatern52`, `KernelGaussian`, `KernelRationalQuadratic`, `KernelSquaredExponential`, `KernelMultiTask`, `SumKernel`, `ProductKernel` |
 
 Additional utilities include periodization transforms, iteration diagnostics (`IterationLog`), resume/checkpoint support, Walsh-Hadamard and BRO-FFT transforms, and a LatNetBuilder linker.
 
+Advanced status notes: `PFGPCI` is currently exported as a parity placeholder,
+but `integrate(::PFGPCI)` intentionally errors until a Julia GP backend is
+implemented. Likewise, `gpu_fwht!` is currently a CPU-fallback stub rather than
+a real GPU acceleration path.
+
 ## Installation
 
 ### Prerequisites

benchmark/README.md

@@ -43,6 +43,14 @@ This writes:
 benchmark/results/latest.json
 ```
 
+By default the benchmark targets now pin BLAS-style thread env vars to `1` for
+more stable dense-transform timings on both the Julia and QMCPy sides. Override
+that when needed:
+
+```bash
+make bench BENCH_BLAS_THREADS=4
+```
+
 To save to a labeled file instead:
 
 ```bash
@@ -151,6 +159,12 @@ benchmark/results/qmcpy_base.json
 benchmark/results/compare_python_base.md
 ```
 
+The Julia memory sidecar and the QMCPy JSON now also record lightweight
+benchmark metadata such as generation time, thread configuration, and the
+integrate timing sample/repeat counts. `compare_python*.md` surfaces that
+metadata in its header so stale or mismatched artifact pairings are easier to
+spot.
+
 ### Coverage Caveat
 
 Do not interpret coverage-enabled benchmark runs as representative performance comparisons.
@@ -195,6 +209,19 @@ If one memory ratio is `< 1` and the other is `> 1`, that is normal rather than
 
 If `<label>_memory.json` is missing because the Julia benchmarks were generated before this feature was added, `compare_python*.md` will show Julia RSS delta as `n/a` until that label is rerun with `make bench` or `make bench-compare-py`.
 
+The heaviest adaptive integrate rows are intentionally timed with more samples
+than the leaf transform/evaluate rows (`samples=9` in Julia, `repeat=9` in
+QMCPy) because those rows were the main source of weighted-ratio swings.
+
+The QMCPy `StudentT` transform rows now also use a dedicated higher-stability
+setting (`repeat=21`, `warmup=3`), and the Julia `StudentT` rows use
+`samples=9`. Those settings are recorded in the report header so unusually large
+cross-run changes in that family are easier to audit. The aggregate summary in
+`compare_python*.md` now also breaks out `StudentT` timing separately
+(`all matched rows`, `excluding StudentT`, and `StudentT only`) so the headline
+cross-language ratio is easier to interpret when that transform family dominates
+the total.
+
 The Python harness also mirrors most of the newer Julia-only benchmark rows:
 
 - large-`d` `Gaussian(diag)` and `Gaussian(dense)` transforms
@@ -274,6 +301,14 @@ benchmark/results/qmcpy_base.json
 benchmark/results/compare_python_base.md
 ```
 
+`bench-all` does not create a third combined ratio. It just runs both report
+generators, so you need to read each report with its own convention:
+
+- `compare_<label>.md`: `ratio = reference ÷ local`
+- `compare_python_<label>.md`: `ratio = Python ÷ Julia`
+
+So in both of those reports, `ratio > 1` means the local Julia side is faster.
+
 ## Output Files
 
 Common generated files:

benchmark/benchmark_qmcpy.py

@@ -44,6 +44,7 @@
 import tracemalloc
 import warnings
 import statistics
+from datetime import datetime
 from pathlib import Path
 
 import numpy as np
@@ -55,6 +56,17 @@
 LARGE_DIMS = [50, 200]
 LARGE_N = [1024, 4096]
 SEED = 42
+DEFAULT_REPEAT = 7
+INTEGRATE_REPEAT = 9
+STUDENT_T_REPEAT = 21
+STUDENT_T_WARMUP_RUNS = 3
+THREAD_ENV_KEYS = (
+    "QMC_BENCH_BLAS_THREADS",
+    "OPENBLAS_NUM_THREADS",
+    "MKL_NUM_THREADS",
+    "OMP_NUM_THREADS",
+    "NUMEXPR_NUM_THREADS",
+)
 
 
 def current_rss_kib():
@@ -90,11 +102,17 @@ def measure_memory(fn):
     return metrics
 
 
-def bench(make_call, *, repeat=7, warmup=True):
+def bench(make_call, *, repeat=DEFAULT_REPEAT, warmup=True):
     """Return median seconds plus approximate memory metrics per call."""
+    if isinstance(warmup, bool):
+        warmup_runs = 1 if warmup else 0
+    else:
+        warmup_runs = int(warmup)
+        if warmup_runs < 0:
+            raise ValueError("warmup must be bool or non-negative integer")
     fn = make_call()
-    if warmup:
-        fn()  # discard first call (cache/allocation warm-up)
+    for _ in range(warmup_runs):
+        fn()  # discard first call(s) to reduce cache/allocation warm-up noise
     mem = measure_memory(fn)
     # Pick an inner count so each timing sample is long enough to be stable.
     timer = timeit.Timer(fn)
@@ -134,12 +152,25 @@ def record(results, group, name, make_call, **kw):
         print(f"  {name:<45s}  ERR: {type(e).__name__}: {e}")
 
 
+def active_thread_env():
+    return {key: os.environ[key] for key in THREAD_ENV_KEYS if key in os.environ}
+
+
 def main():
     label = sys.argv[1] if len(sys.argv) > 1 else "latest"
     results = {"gen_samples": {}, "transform": {}, "evaluate": {}, "integrate": {}}
 
     print("\n\nQMCPy Benchmarks (qmcpy %s)" % getattr(qp, "__version__", "?"))
     print("=" * 70)
+    thread_env = active_thread_env()
+    if thread_env:
+        print(
+            "Thread env: "
+            + ", ".join(f"{key}={value}" for key, value in sorted(thread_env.items()))
+        )
+    print(
+        f"StudentT config: repeat={STUDENT_T_REPEAT}, warmup_runs={STUDENT_T_WARMUP_RUNS}"
+    )
 
     # 1. Discrete distribution sampling --------------------------------------
     # [C] = qmctoolscl C kernel (not a language comparison)
@@ -191,7 +222,14 @@ def make_student_t(n=n):
             tm = qp.StudentT(dd, loc=np.zeros(dim), shape=np.eye(dim), df=2.0)
             x = dd(n)
             return lambda: tm._transform(x)
-        record(results, "transform", f"StudentT d=10 n={n}", make_student_t)
+        record(
+            results,
+            "transform",
+            f"StudentT d=10 n={n}",
+            make_student_t,
+            repeat=STUDENT_T_REPEAT,
+            warmup=STUDENT_T_WARMUP_RUNS,
+        )
 
         def make_johnsons_su(n=n):
             dim = 10
@@ -336,7 +374,7 @@ def _scalar(x):
 
     for name, make_sc, warm in integrate_cases:
         record(results, "integrate", name, integrate_call(make_sc),
-               repeat=3, warmup=warm)
+               repeat=INTEGRATE_REPEAT, warmup=warm)
         # One-shot accuracy measurement: solution value + tolerances used. Kept
         # separate from the timed runs and never aborts the run on failure.
         try:
@@ -358,6 +396,16 @@ def _scalar(x):
     outfile = resdir / f"qmcpy_{label}.json"
     payload = {
         "qmcpy_version": getattr(qp, "__version__", "?"),
+        "python_version": sys.version.split()[0],
+        "label": label,
+        "generated_at": datetime.now().isoformat(timespec="seconds"),
+        "thread_env": thread_env,
+        "benchmark_config": {
+            "default_repeat": DEFAULT_REPEAT,
+            "integrate_repeat": INTEGRATE_REPEAT,
+            "student_t_repeat": STUDENT_T_REPEAT,
+            "student_t_warmup_runs": STUDENT_T_WARMUP_RUNS,
+        },
         "results": results,
     }
     outfile.write_text(json.dumps(payload, indent=2))