From 2a254cece0401b5b1624ea4be33786761c6a78f9 Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 09:38:08 -0500 Subject: [PATCH 001/104] Replace abstract fields with specific types --- src/discrete_distribution/iid_std_uniform.jl | 4 ++-- src/discrete_distribution/lattice.jl | 4 ++-- src/integrand/custom_fun.jl | 6 +++--- src/integrand/keister.jl | 6 +++--- src/stopping_criterion/cub_mc_clt.jl | 4 ++-- src/true_measure/brownian_motion.jl | 7 ++++--- src/true_measure/gaussian.jl | 17 +++++++++-------- src/true_measure/geometric_brownian_motion.jl | 7 ++++--- src/true_measure/uniform.jl | 4 ++-- 9 files changed, 31 insertions(+), 28 deletions(-) diff --git a/src/discrete_distribution/iid_std_uniform.jl b/src/discrete_distribution/iid_std_uniform.jl index d70cc0d..123c0bf 100644 --- a/src/discrete_distribution/iid_std_uniform.jl +++ b/src/discrete_distribution/iid_std_uniform.jl @@ -12,9 +12,9 @@ dd = IIDStdUniform(3; seed=42) x = gen_samples(dd, 1024) # 1024×3 matrix of uniform samples ``` """ -mutable struct IIDStdUniform <: AbstractDiscreteDistribution +mutable struct IIDStdUniform{R <: AbstractRNG} <: AbstractDiscreteDistribution dimension::Int - rng::AbstractRNG + rng::R mimics::String end diff --git a/src/discrete_distribution/lattice.jl b/src/discrete_distribution/lattice.jl index f28dc06..00019f8 100644 --- a/src/discrete_distribution/lattice.jl +++ b/src/discrete_distribution/lattice.jl @@ -33,14 +33,14 @@ dd_r = Lattice(3; seed=7, replications=16) x = gen_samples(dd_r, 1024) # 16×1024×3 array ``` """ -mutable struct Lattice <: AbstractDiscreteDistribution +mutable struct Lattice{R <: AbstractRNG} <: AbstractDiscreteDistribution dimension::Int randomize::Bool order::String replications::Union{Nothing, Int} gen_vector::Vector{UInt64} shift::Matrix{Float64} # R × d - rng::AbstractRNG + rng::R mimics::String end diff --git a/src/integrand/custom_fun.jl b/src/integrand/custom_fun.jl index 15db615..12c75c2 100644 --- a/src/integrand/custom_fun.jl +++ b/src/integrand/custom_fun.jl @@ -11,9 +11,9 @@ tm = Uniform(dd; lower_bound=0.0, upper_bound=1.0) f = CustomFun(tm, x -> sum(x .^ 2, dims=2)[:]) ``` """ -struct CustomFun <: AbstractIntegrand - true_measure::AbstractTrueMeasure - g::Function +struct CustomFun{TM <: AbstractTrueMeasure, G} <: AbstractIntegrand + true_measure::TM + g::G dimension::Int end diff --git a/src/integrand/keister.jl b/src/integrand/keister.jl index 665738f..54a4e3d 100644 --- a/src/integrand/keister.jl +++ b/src/integrand/keister.jl @@ -16,8 +16,8 @@ tm = Gaussian(dd; covariance=0.5) f = Keister(tm) ``` """ -struct Keister <: AbstractIntegrand - true_measure::AbstractTrueMeasure +struct Keister{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM dimension::Int end @@ -29,7 +29,7 @@ function evaluate(f::Keister, x::AbstractMatrix) n, d = size(x) y = Vector{Float64}(undef, n) coeff = π^(d / 2) - for i in 1:n + @inbounds for i in 1:n norm_xi = sqrt(sum(x[i, j]^2 for j in 1:d)) y[i] = coeff * cos(norm_xi) end diff --git a/src/stopping_criterion/cub_mc_clt.jl b/src/stopping_criterion/cub_mc_clt.jl index 37092fc..7494b7c 100644 --- a/src/stopping_criterion/cub_mc_clt.jl +++ b/src/stopping_criterion/cub_mc_clt.jl @@ -21,8 +21,8 @@ sc = CubMCCLT(f; abs_tol=1e-3) result = integrate(sc) ``` """ -mutable struct CubMCCLT <: AbstractStoppingCriterion - integrand::AbstractIntegrand +mutable struct CubMCCLT{I <: AbstractIntegrand} <: AbstractStoppingCriterion + integrand::I abs_tol::Float64 rel_tol::Float64 n_init::Int diff --git a/src/true_measure/brownian_motion.jl b/src/true_measure/brownian_motion.jl index 45b4cc4..3349a0d 100644 --- a/src/true_measure/brownian_motion.jl +++ b/src/true_measure/brownian_motion.jl @@ -22,12 +22,13 @@ x = gen_samples(dd, 256) paths = transform(bm, x) # 256×64 Brownian motion paths ``` """ -struct BrownianMotion <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct BrownianMotion{D <: AbstractDiscreteDistribution, G <: Gaussian} <: + AbstractTrueMeasure + dd::D dimension::Int time_vector::Vector{Float64} drift::Float64 - _gaussian::Gaussian + _gaussian::G end function BrownianMotion(dd::AbstractDiscreteDistribution; diff --git a/src/true_measure/gaussian.jl b/src/true_measure/gaussian.jl index 9d93ee3..b46cedd 100644 --- a/src/true_measure/gaussian.jl +++ b/src/true_measure/gaussian.jl @@ -25,8 +25,8 @@ x = gen_samples(dd, 256) y = transform(tm, x) # 256×2 Gaussian samples ``` """ -struct Gaussian <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct Gaussian{D <: AbstractDiscreteDistribution} <: AbstractTrueMeasure + dd::D dimension::Int mean::Vector{Float64} covariance::Matrix{Float64} @@ -105,12 +105,13 @@ function _compute_decomp(cov::Matrix{Float64}, decomp_type::Symbol) end function transform(tm::Gaussian, x::AbstractMatrix) - # x is n×d with entries in [0,1) - # Step 1: inverse CDF of standard normal, componentwise - z = quantile.(Distributions.Normal(), x) - # Step 2: apply decomposition and add mean - # y[i,:] = A * z[i,:] + μ <=> Y = Z * Aᵀ .+ μᵀ - return z * tm._decomp' .+ tm.mean' + # x is n×d with entries in [0,1); compute y = Φ⁻¹(x) * Aᵀ .+ μᵀ + # Numerics are identical to the original (same standard-normal quantile); + # the mean shift is done in place to avoid an extra n×d allocation. + z = quantile.(Distributions.Normal(), x) # one n×d temporary + y = z * transpose(tm._decomp) # BLAS gemm (handles diagonal A too) + y .+= transpose(tm.mean) # in-place mean shift, no allocation + return y end function Base.show(io::IO, tm::Gaussian) diff --git a/src/true_measure/geometric_brownian_motion.jl b/src/true_measure/geometric_brownian_motion.jl index 259378a..645c0d8 100644 --- a/src/true_measure/geometric_brownian_motion.jl +++ b/src/true_measure/geometric_brownian_motion.jl @@ -23,14 +23,15 @@ x = gen_samples(dd, 256) paths = transform(gbm, x) # 256×4 stock price paths ``` """ -struct GeometricBrownianMotion <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct GeometricBrownianMotion{D <: AbstractDiscreteDistribution, B <: BrownianMotion} <: + AbstractTrueMeasure + dd::D dimension::Int time_vector::Vector{Float64} initial_value::Float64 drift::Float64 diffusion::Float64 - _bm::BrownianMotion + _bm::B end function GeometricBrownianMotion(dd::AbstractDiscreteDistribution; diff --git a/src/true_measure/uniform.jl b/src/true_measure/uniform.jl index efd9659..03fce3c 100644 --- a/src/true_measure/uniform.jl +++ b/src/true_measure/uniform.jl @@ -19,8 +19,8 @@ x = gen_samples(dd, 100) y = transform(tm, x) # 100×3 matrix in [-2, 2]^3 ``` """ -struct Uniform <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct Uniform{D <: AbstractDiscreteDistribution} <: AbstractTrueMeasure + dd::D dimension::Int lower_bound::Vector{Float64} upper_bound::Vector{Float64} From 2d4ba822c2b24b10e509377c047032c90a77a015 Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 09:48:46 -0500 Subject: [PATCH 002/104] Replace abstract fields with specific types --- src/discrete_distribution/digital_net_any_bases.jl | 4 ++-- src/discrete_distribution/digital_net_b2.jl | 4 ++-- src/discrete_distribution/halton.jl | 4 ++-- src/integrand/asian_option.jl | 4 ++-- src/integrand/bayesian_lr_coeffs.jl | 4 ++-- src/integrand/box_integral.jl | 4 ++-- src/integrand/financial_option.jl | 4 ++-- src/integrand/financial_option_ml.jl | 4 ++-- src/integrand/four_branch2d.jl | 4 ++-- src/integrand/genz.jl | 4 ++-- src/integrand/hartmann6d.jl | 4 ++-- src/integrand/ishigami.jl | 4 ++-- src/integrand/linear0.jl | 4 ++-- src/integrand/multimodal2d.jl | 4 ++-- src/integrand/sensitivity_indices.jl | 7 ++++--- src/integrand/sin1d.jl | 4 ++-- src/integrand/umbridge_wrapper.jl | 4 ++-- src/kernel/multitask.jl | 4 ++-- src/stopping_criterion/cub_mc_clt_vec.jl | 4 ++-- src/stopping_criterion/cub_mc_g.jl | 4 ++-- src/stopping_criterion/cub_mlmc.jl | 4 ++-- src/stopping_criterion/cub_mlmc_cont.jl | 4 ++-- src/stopping_criterion/cub_mlqmc.jl | 4 ++-- src/stopping_criterion/cub_mlqmc_cont.jl | 4 ++-- src/stopping_criterion/cub_qmc_bayes_lattice_g.jl | 4 ++-- src/stopping_criterion/cub_qmc_bayes_net_g.jl | 4 ++-- src/stopping_criterion/cub_qmc_lattice_g.jl | 4 ++-- src/stopping_criterion/cub_qmc_net_g.jl | 4 ++-- src/stopping_criterion/cub_qmc_rep_student_t.jl | 4 ++-- src/stopping_criterion/pf_gp_ci.jl | 4 ++-- src/true_measure/acceptance_rejection.jl | 11 ++++++----- src/true_measure/bernoulli_cont.jl | 4 ++-- src/true_measure/distributions_wrapper.jl | 4 ++-- src/true_measure/johnsons_su.jl | 4 ++-- src/true_measure/kumaraswamy.jl | 4 ++-- src/true_measure/lebesgue.jl | 4 ++-- src/true_measure/matern_gp.jl | 4 ++-- src/true_measure/student_t.jl | 4 ++-- src/true_measure/triangular.jl | 4 ++-- src/true_measure/uniform_triangle.jl | 8 ++++---- src/true_measure/zero_inflated_exp_uniform.jl | 4 ++-- 41 files changed, 90 insertions(+), 88 deletions(-) diff --git a/src/discrete_distribution/digital_net_any_bases.jl b/src/discrete_distribution/digital_net_any_bases.jl index 318a2da..71c8b8b 100644 --- a/src/discrete_distribution/digital_net_any_bases.jl +++ b/src/discrete_distribution/digital_net_any_bases.jl @@ -30,7 +30,7 @@ dd = DigitalNetAnyBases(2; bases=3, generating_matrices=C, randomize="none") x = gen_samples(dd, 9) ``` """ -mutable struct DigitalNetAnyBases <: AbstractDiscreteDistribution +mutable struct DigitalNetAnyBases{R <: AbstractRNG} <: AbstractDiscreteDistribution dimension::Int d_gen::Int # dimension of generating matrices (may differ from dimension if alpha > 1) bases::Vector{Int} # base per gen-dimension @@ -39,7 +39,7 @@ mutable struct DigitalNetAnyBases <: AbstractDiscreteDistribution randomize::String alpha::Int replications::Union{Nothing, Int} - rng::AbstractRNG + rng::R mimics::String end diff --git a/src/discrete_distribution/digital_net_b2.jl b/src/discrete_distribution/digital_net_b2.jl index 10e6811..3ab5ef8 100644 --- a/src/discrete_distribution/digital_net_b2.jl +++ b/src/discrete_distribution/digital_net_b2.jl @@ -77,11 +77,11 @@ See the outer constructor for full documentation. QMCToolsCL is required for sample generation. """ -mutable struct DigitalNetB2 <: AbstractDiscreteDistribution +mutable struct DigitalNetB2{R <: AbstractRNG} <: AbstractDiscreteDistribution dimension::Int randomize::String graycode::Bool - rng::AbstractRNG + rng::R direction_nums::Matrix{UInt32} # ndim × BITS mimics::String replications::Union{Nothing, Int} diff --git a/src/discrete_distribution/halton.jl b/src/discrete_distribution/halton.jl index 66a8331..90afe59 100644 --- a/src/discrete_distribution/halton.jl +++ b/src/discrete_distribution/halton.jl @@ -39,11 +39,11 @@ Supports up to 250 dimensions. See the outer constructor for full documentation QMCToolsCL is required for sample generation. """ -mutable struct Halton <: AbstractDiscreteDistribution +mutable struct Halton{R <: AbstractRNG} <: AbstractDiscreteDistribution dimension::Int randomize::Bool generalize::Bool - rng::AbstractRNG + rng::R mimics::String end diff --git a/src/integrand/asian_option.jl b/src/integrand/asian_option.jl index 080fd68..8e41731 100644 --- a/src/integrand/asian_option.jl +++ b/src/integrand/asian_option.jl @@ -21,8 +21,8 @@ tm = BrownianMotion(dd) f = AsianOption(tm; volatility=0.5, start_price=30.0, strike_price=25.0) ``` """ -mutable struct AsianOption <: AbstractIntegrand - true_measure::AbstractTrueMeasure +mutable struct AsianOption{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM dimension::Int volatility::Float64 start_price::Float64 diff --git a/src/integrand/bayesian_lr_coeffs.jl b/src/integrand/bayesian_lr_coeffs.jl index cd46d2a..33aca0e 100644 --- a/src/integrand/bayesian_lr_coeffs.jl +++ b/src/integrand/bayesian_lr_coeffs.jl @@ -29,8 +29,8 @@ sc = CubMCCLT(f; abs_tol=0.05) result = integrate(sc) ``` """ -struct BayesianLRCoeffs <: AbstractIntegrand - true_measure::AbstractTrueMeasure +struct BayesianLRCoeffs{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM dimension::Int feature_array::Matrix{Float64} # m × p response_vector::Vector{Float64} # m diff --git a/src/integrand/box_integral.jl b/src/integrand/box_integral.jl index 2c02251..50c8ce6 100644 --- a/src/integrand/box_integral.jl +++ b/src/integrand/box_integral.jl @@ -11,8 +11,8 @@ is smooth; for `s < 0` it has an integrable singularity at the origin. The exact integral is known for certain values of s and d (see BoxIntegral tables by Bailey, Borwein, and Crandall). """ -struct BoxIntegral <: AbstractIntegrand - true_measure::AbstractTrueMeasure +struct BoxIntegral{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM dimension::Int s::Float64 end diff --git a/src/integrand/financial_option.jl b/src/integrand/financial_option.jl index 1e66c05..e1f3b75 100644 --- a/src/integrand/financial_option.jl +++ b/src/integrand/financial_option.jl @@ -35,8 +35,8 @@ tm = BrownianMotion(dd) f = FinancialOption(tm; option_type=:european, volatility=0.2, strike_price=100.0, start_price=100.0) ``` """ -mutable struct FinancialOption <: AbstractIntegrand - true_measure::AbstractTrueMeasure +mutable struct FinancialOption{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM dimension::Int volatility::Float64 start_price::Float64 diff --git a/src/integrand/financial_option_ml.jl b/src/integrand/financial_option_ml.jl index 647a3f8..36b9c1e 100644 --- a/src/integrand/financial_option_ml.jl +++ b/src/integrand/financial_option_ml.jl @@ -27,8 +27,8 @@ sc = CubMLMC(f; abs_tol=0.05) result = integrate(sc) ``` """ -struct FinancialOptionML <: AbstractMLIntegrand - true_measure::AbstractTrueMeasure +struct FinancialOptionML{TM <: AbstractTrueMeasure} <: AbstractMLIntegrand + true_measure::TM dimension::Int d_coarsest::Int nb_of_levels::Int diff --git a/src/integrand/four_branch2d.jl b/src/integrand/four_branch2d.jl index 9e54e34..5b6cb87 100644 --- a/src/integrand/four_branch2d.jl +++ b/src/integrand/four_branch2d.jl @@ -16,8 +16,8 @@ reliability analysis. The true measure should be a `Gaussian` with mean 0 and appropriate covariance. """ -struct FourBranch2D <: AbstractIntegrand - true_measure::AbstractTrueMeasure +struct FourBranch2D{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM dimension::Int k::Float64 end diff --git a/src/integrand/genz.jl b/src/integrand/genz.jl index 3d807a8..c317f83 100644 --- a/src/integrand/genz.jl +++ b/src/integrand/genz.jl @@ -20,8 +20,8 @@ tm = Uniform(dd) f = Genz(tm; kind=:oscillatory) ``` """ -struct Genz <: AbstractIntegrand - true_measure::AbstractTrueMeasure +struct Genz{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM dimension::Int kind::Symbol a::Vector{Float64} diff --git a/src/integrand/hartmann6d.jl b/src/integrand/hartmann6d.jl index 0d57e95..5b3429a 100644 --- a/src/integrand/hartmann6d.jl +++ b/src/integrand/hartmann6d.jl @@ -8,8 +8,8 @@ where x ∈ [0,1]⁶. The global minimum is approximately -3.3224 at x* ≈ (0.2017, 0.1500, 0.4769, 0.2753, 0.3117, 0.6573). """ -struct Hartmann6D <: AbstractIntegrand - true_measure::AbstractTrueMeasure +struct Hartmann6D{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM dimension::Int end diff --git a/src/integrand/ishigami.jl b/src/integrand/ishigami.jl index 2836ec9..ebd0d26 100644 --- a/src/integrand/ishigami.jl +++ b/src/integrand/ishigami.jl @@ -13,8 +13,8 @@ The true measure should map uniform [0,1]³ samples to (-π, π)³. The exact mean, variance, and Sobol' indices are computable in closed form for given a, b. """ -struct Ishigami <: AbstractIntegrand - true_measure::AbstractTrueMeasure +struct Ishigami{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM dimension::Int a::Float64 b::Float64 diff --git a/src/integrand/linear0.jl b/src/integrand/linear0.jl index 2311a80..fa0c9b9 100644 --- a/src/integrand/linear0.jl +++ b/src/integrand/linear0.jl @@ -8,8 +8,8 @@ The true measure should map uniform samples to [-0.5, 0.5]^d (i.e., a Exact integral = 0 by symmetry. """ -struct Linear0 <: AbstractIntegrand - true_measure::AbstractTrueMeasure +struct Linear0{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM dimension::Int end diff --git a/src/integrand/multimodal2d.jl b/src/integrand/multimodal2d.jl index 6780c69..f84f4a7 100644 --- a/src/integrand/multimodal2d.jl +++ b/src/integrand/multimodal2d.jl @@ -8,8 +8,8 @@ where x ∈ [0,1]². This is useful for testing QMC on functions with multiple local features. """ -struct Multimodal2D <: AbstractIntegrand - true_measure::AbstractTrueMeasure +struct Multimodal2D{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM dimension::Int end diff --git a/src/integrand/sensitivity_indices.jl b/src/integrand/sensitivity_indices.jl index 915a800..421ecf2 100644 --- a/src/integrand/sensitivity_indices.jl +++ b/src/integrand/sensitivity_indices.jl @@ -39,10 +39,11 @@ closed_indices = mu ./ var_est 2. Sorokin, Rathinavel. "On Bounding and Approximating Functions of Multiple Expectations Using Quasi-Monte Carlo." MCQMC 2022. """ -struct SensitivityIndices <: AbstractIntegrand - true_measure::AbstractTrueMeasure +struct SensitivityIndices{TM <: AbstractTrueMeasure, BI <: AbstractIntegrand} <: + AbstractIntegrand + true_measure::TM dimension::Int # 2 * d_base - base_integrand::AbstractIntegrand + base_integrand::BI d_base::Int # original integrand dimension indices::Matrix{Bool} # k × d_base, each row is a subset indicator end diff --git a/src/integrand/sin1d.jl b/src/integrand/sin1d.jl index 24e582e..8fd2cf0 100644 --- a/src/integrand/sin1d.jl +++ b/src/integrand/sin1d.jl @@ -6,8 +6,8 @@ One-dimensional sinusoidal test function: f(t) = sin(t) on U[0, 2πk]. The true measure should map uniform [0,1] samples to [0, 2πk]. Exact integral = 0 for integer k. """ -struct Sin1D <: AbstractIntegrand - true_measure::AbstractTrueMeasure +struct Sin1D{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM dimension::Int k::Int end diff --git a/src/integrand/umbridge_wrapper.jl b/src/integrand/umbridge_wrapper.jl index 15b1626..ecc98ad 100644 --- a/src/integrand/umbridge_wrapper.jl +++ b/src/integrand/umbridge_wrapper.jl @@ -41,8 +41,8 @@ sc = CubQMCNetG(f; abs_tol=2.5e-2) result = integrate(sc) ``` """ -struct UMBridgeWrapper <: AbstractIntegrand - true_measure::AbstractTrueMeasure +struct UMBridgeWrapper{TM <: AbstractTrueMeasure} <: AbstractIntegrand + true_measure::TM url::String model_name::String config::Dict{String, Any} diff --git a/src/kernel/multitask.jl b/src/kernel/multitask.jl index d3de950..98b019a 100644 --- a/src/kernel/multitask.jl +++ b/src/kernel/multitask.jl @@ -24,8 +24,8 @@ kmt = KernelMultiTask(base, 3; diag=[1.0, 2.0, 3.0]) # Task correlation matrix is I + diag([1,2,3]) ``` """ -struct KernelMultiTask <: AbstractKernel - base_kernel::AbstractKernel +struct KernelMultiTask{K <: AbstractKernel} <: AbstractKernel + base_kernel::K num_tasks::Int factor::Matrix{Float64} diag_vec::Vector{Float64} diff --git a/src/stopping_criterion/cub_mc_clt_vec.jl b/src/stopping_criterion/cub_mc_clt_vec.jl index ceab620..3ac41ec 100644 --- a/src/stopping_criterion/cub_mc_clt_vec.jl +++ b/src/stopping_criterion/cub_mc_clt_vec.jl @@ -20,8 +20,8 @@ result = integrate(sc) show(result.data[:iteration_log]) ``` """ -mutable struct CubMCCLTVec <: AbstractStoppingCriterion - integrand::AbstractIntegrand +mutable struct CubMCCLTVec{I <: AbstractIntegrand} <: AbstractStoppingCriterion + integrand::I abs_tol::Float64 rel_tol::Float64 n_init::Int diff --git a/src/stopping_criterion/cub_mc_g.jl b/src/stopping_criterion/cub_mc_g.jl index 41ef984..c9eb6f5 100644 --- a/src/stopping_criterion/cub_mc_g.jl +++ b/src/stopping_criterion/cub_mc_g.jl @@ -23,8 +23,8 @@ functions with bounded kurtosis. 1. Hickernell, Jiang, Liu, Owen. "Guaranteed conservative fixed width confidence intervals via Monte Carlo sampling." MCQMC 2012. """ -mutable struct CubMCG <: AbstractStoppingCriterion - integrand::AbstractIntegrand +mutable struct CubMCG{I <: AbstractIntegrand} <: AbstractStoppingCriterion + integrand::I abs_tol::Float64 rel_tol::Float64 n_init::Int diff --git a/src/stopping_criterion/cub_mlmc.jl b/src/stopping_criterion/cub_mlmc.jl index 52e7402..82a8b49 100644 --- a/src/stopping_criterion/cub_mlmc.jl +++ b/src/stopping_criterion/cub_mlmc.jl @@ -37,8 +37,8 @@ result = integrate(sc) 1. M.B. Giles. "Multi-level Monte Carlo path simulation." Operations Research, 56(3):607–617, 2008. """ -mutable struct CubMLMC <: AbstractStoppingCriterion - integrand::AbstractMLIntegrand +mutable struct CubMLMC{I <: AbstractMLIntegrand} <: AbstractStoppingCriterion + integrand::I rmse_tol::Float64 n_init::Int n_limit::Int diff --git a/src/stopping_criterion/cub_mlmc_cont.jl b/src/stopping_criterion/cub_mlmc_cont.jl index 20e8164..ec442fb 100644 --- a/src/stopping_criterion/cub_mlmc_cont.jl +++ b/src/stopping_criterion/cub_mlmc_cont.jl @@ -35,8 +35,8 @@ result = integrate(sc) # References 1. [MultilevelEstimators.jl](https://github.com/PieterjanRobbe/MultilevelEstimators.jl) """ -mutable struct CubMLMCCont <: AbstractStoppingCriterion - integrand::AbstractMLIntegrand +mutable struct CubMLMCCont{I <: AbstractMLIntegrand} <: AbstractStoppingCriterion + integrand::I target_tol::Float64 n_init::Int n_limit::Int diff --git a/src/stopping_criterion/cub_mlqmc.jl b/src/stopping_criterion/cub_mlqmc.jl index a10af0f..3f04c07 100644 --- a/src/stopping_criterion/cub_mlqmc.jl +++ b/src/stopping_criterion/cub_mlqmc.jl @@ -30,8 +30,8 @@ sc = CubMLQMC(f; abs_tol=0.05) result = integrate(sc) ``` """ -mutable struct CubMLQMC <: AbstractStoppingCriterion - integrand::AbstractMLIntegrand +mutable struct CubMLQMC{I <: AbstractMLIntegrand} <: AbstractStoppingCriterion + integrand::I target_tol::Float64 n_init::Int n_limit::Int diff --git a/src/stopping_criterion/cub_mlqmc_cont.jl b/src/stopping_criterion/cub_mlqmc_cont.jl index f5d4ded..7dc2e11 100644 --- a/src/stopping_criterion/cub_mlqmc_cont.jl +++ b/src/stopping_criterion/cub_mlqmc_cont.jl @@ -38,8 +38,8 @@ result = integrate(sc) # References 1. [MultilevelEstimators.jl](https://github.com/PieterjanRobbe/MultilevelEstimators.jl) """ -mutable struct CubMLQMCCont <: AbstractStoppingCriterion - integrand::AbstractMLIntegrand +mutable struct CubMLQMCCont{I <: AbstractMLIntegrand} <: AbstractStoppingCriterion + integrand::I target_tol::Float64 n_init::Int n_limit::Int diff --git a/src/stopping_criterion/cub_qmc_bayes_lattice_g.jl b/src/stopping_criterion/cub_qmc_bayes_lattice_g.jl index ac810aa..a90d136 100644 --- a/src/stopping_criterion/cub_qmc_bayes_lattice_g.jl +++ b/src/stopping_criterion/cub_qmc_bayes_lattice_g.jl @@ -18,8 +18,8 @@ sc = CubQMCBayesLatticeG(f; abs_tol=1e-4) result = integrate(sc) ``` """ -mutable struct CubQMCBayesLatticeG <: AbstractStoppingCriterion - integrand::AbstractIntegrand +mutable struct CubQMCBayesLatticeG{I <: AbstractIntegrand} <: AbstractStoppingCriterion + integrand::I abs_tol::Float64 rel_tol::Float64 n_init::Int diff --git a/src/stopping_criterion/cub_qmc_bayes_net_g.jl b/src/stopping_criterion/cub_qmc_bayes_net_g.jl index 4e9af23..ff094c7 100644 --- a/src/stopping_criterion/cub_qmc_bayes_net_g.jl +++ b/src/stopping_criterion/cub_qmc_bayes_net_g.jl @@ -15,8 +15,8 @@ sc = CubQMCBayesNetG(f; abs_tol=1e-4) result = integrate(sc) ``` """ -mutable struct CubQMCBayesNetG <: AbstractStoppingCriterion - integrand::AbstractIntegrand +mutable struct CubQMCBayesNetG{I <: AbstractIntegrand} <: AbstractStoppingCriterion + integrand::I abs_tol::Float64 rel_tol::Float64 n_init::Int diff --git a/src/stopping_criterion/cub_qmc_lattice_g.jl b/src/stopping_criterion/cub_qmc_lattice_g.jl index de9b7d4..17b19ec 100644 --- a/src/stopping_criterion/cub_qmc_lattice_g.jl +++ b/src/stopping_criterion/cub_qmc_lattice_g.jl @@ -16,8 +16,8 @@ sc = CubQMCLatticeG(f; abs_tol=1e-4) result = integrate(sc) ``` """ -mutable struct CubQMCLatticeG <: AbstractStoppingCriterion - integrand::AbstractIntegrand +mutable struct CubQMCLatticeG{I <: AbstractIntegrand} <: AbstractStoppingCriterion + integrand::I abs_tol::Float64 rel_tol::Float64 n_init::Int diff --git a/src/stopping_criterion/cub_qmc_net_g.jl b/src/stopping_criterion/cub_qmc_net_g.jl index 271a655..2a86efc 100644 --- a/src/stopping_criterion/cub_qmc_net_g.jl +++ b/src/stopping_criterion/cub_qmc_net_g.jl @@ -16,8 +16,8 @@ sc = CubQMCNetG(f; abs_tol=1e-4) result = integrate(sc) ``` """ -mutable struct CubQMCNetG <: AbstractStoppingCriterion - integrand::AbstractIntegrand +mutable struct CubQMCNetG{I <: AbstractIntegrand} <: AbstractStoppingCriterion + integrand::I abs_tol::Float64 rel_tol::Float64 n_init::Int diff --git a/src/stopping_criterion/cub_qmc_rep_student_t.jl b/src/stopping_criterion/cub_qmc_rep_student_t.jl index a750cf3..35e0ece 100644 --- a/src/stopping_criterion/cub_qmc_rep_student_t.jl +++ b/src/stopping_criterion/cub_qmc_rep_student_t.jl @@ -42,8 +42,8 @@ result = integrate(sc) 2. Pierre l'Ecuyer et al. "Confidence intervals for randomized quasi-Monte Carlo estimators." 2023 Winter Simulation Conference. """ -mutable struct CubQMCRepStudentT <: AbstractStoppingCriterion - integrand::AbstractIntegrand +mutable struct CubQMCRepStudentT{I <: AbstractIntegrand} <: AbstractStoppingCriterion + integrand::I abs_tol::Float64 rel_tol::Float64 n_init::Int diff --git a/src/stopping_criterion/pf_gp_ci.jl b/src/stopping_criterion/pf_gp_ci.jl index ec32412..beba3c9 100644 --- a/src/stopping_criterion/pf_gp_ci.jl +++ b/src/stopping_criterion/pf_gp_ci.jl @@ -20,8 +20,8 @@ construction and credible intervals. 1. Sorokin, Aleksei G., and Vishwas Rao. "Credible Intervals for Probability of Failure with Gaussian Processes." arXiv:2311.07733 (2023). """ -struct PFGPCI <: AbstractStoppingCriterion - integrand::AbstractIntegrand +struct PFGPCI{I <: AbstractIntegrand} <: AbstractStoppingCriterion + integrand::I failure_threshold::Float64 failure_above_threshold::Bool abs_tol::Float64 diff --git a/src/true_measure/acceptance_rejection.jl b/src/true_measure/acceptance_rejection.jl index f316f1a..f456398 100644 --- a/src/true_measure/acceptance_rejection.jl +++ b/src/true_measure/acceptance_rejection.jl @@ -33,12 +33,13 @@ tm = AcceptanceRejection(dd; pdf_func=pdf_func, envelope_multiplier=6.0) 1. Zhu, H. and Dick, J. "A discrepancy bound for deterministic acceptance-rejection samplers beyond N^{-1/2}." 2014. """ -struct AcceptanceRejection <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct AcceptanceRejection{D <: AbstractDiscreteDistribution, F1, F2, F3} <: + AbstractTrueMeasure + dd::D dimension::Int # target dimension (d = dd.dimension - 1) - pdf_func::Function - proposal_pdf_func::Function - proposal_sample_func::Function + pdf_func::F1 + proposal_pdf_func::F2 + proposal_sample_func::F3 envelope_multiplier::Float64 end diff --git a/src/true_measure/bernoulli_cont.jl b/src/true_measure/bernoulli_cont.jl index 97e8b81..cb81f54 100644 --- a/src/true_measure/bernoulli_cont.jl +++ b/src/true_measure/bernoulli_cont.jl @@ -14,8 +14,8 @@ where C(λ) is the normalizing constant. The inverse CDF is: - `dd`: discrete distribution. - `lam`: parameter λ ∈ (0,1), scalar or d-vector. """ -struct BernoulliCont <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct BernoulliCont{D <: AbstractDiscreteDistribution} <: AbstractTrueMeasure + dd::D dimension::Int lam::Vector{Float64} end diff --git a/src/true_measure/distributions_wrapper.jl b/src/true_measure/distributions_wrapper.jl index 69ae70f..3557247 100644 --- a/src/true_measure/distributions_wrapper.jl +++ b/src/true_measure/distributions_wrapper.jl @@ -30,8 +30,8 @@ tm = DistributionsWrapper(dd; marginals=[ ]) ``` """ -struct DistributionsWrapper <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct DistributionsWrapper{D <: AbstractDiscreteDistribution} <: AbstractTrueMeasure + dd::D dimension::Int marginals::Vector{Distributions.UnivariateDistribution} end diff --git a/src/true_measure/johnsons_su.jl b/src/true_measure/johnsons_su.jl index f201d7c..09348c5 100644 --- a/src/true_measure/johnsons_su.jl +++ b/src/true_measure/johnsons_su.jl @@ -14,8 +14,8 @@ where Φ⁻¹ is the standard normal quantile function. - `gamma`: shape parameter γ (scalar or d-vector). - `delta`: shape parameter δ > 0 (scalar or d-vector). """ -struct JohnsonsSU <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct JohnsonsSU{D <: AbstractDiscreteDistribution} <: AbstractTrueMeasure + dd::D dimension::Int xi::Vector{Float64} lambda::Vector{Float64} diff --git a/src/true_measure/kumaraswamy.jl b/src/true_measure/kumaraswamy.jl index bebfa36..4d7b634 100644 --- a/src/true_measure/kumaraswamy.jl +++ b/src/true_measure/kumaraswamy.jl @@ -13,8 +13,8 @@ inverse CDF, making it efficient for QMC. - `alpha`: shape parameter α > 0 (scalar or d-vector). - `beta`: shape parameter β > 0 (scalar or d-vector). """ -struct Kumaraswamy <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct Kumaraswamy{D <: AbstractDiscreteDistribution} <: AbstractTrueMeasure + dd::D dimension::Int alpha::Vector{Float64} beta::Vector{Float64} diff --git a/src/true_measure/lebesgue.jl b/src/true_measure/lebesgue.jl index f8a9270..3aadf5e 100644 --- a/src/true_measure/lebesgue.jl +++ b/src/true_measure/lebesgue.jl @@ -20,8 +20,8 @@ y = transform(tm, x) # 100×2 matrix in [0, 2]^2 tm.volume # 4.0 ``` """ -struct Lebesgue <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct Lebesgue{D <: AbstractDiscreteDistribution} <: AbstractTrueMeasure + dd::D dimension::Int lower_bound::Vector{Float64} upper_bound::Vector{Float64} diff --git a/src/true_measure/matern_gp.jl b/src/true_measure/matern_gp.jl index fe8c4f7..218b0ee 100644 --- a/src/true_measure/matern_gp.jl +++ b/src/true_measure/matern_gp.jl @@ -18,8 +18,8 @@ tm = MaternGP(dd; nu=2.5, lengthscale=0.3) x = transform(tm, gen_samples(dd, 100)) # 100 GP sample paths ``` """ -struct MaternGP <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct MaternGP{D <: AbstractDiscreteDistribution} <: AbstractTrueMeasure + dd::D nu::Float64 lengthscale::Float64 variance::Float64 diff --git a/src/true_measure/student_t.jl b/src/true_measure/student_t.jl index 329a4bd..9218e22 100644 --- a/src/true_measure/student_t.jl +++ b/src/true_measure/student_t.jl @@ -12,8 +12,8 @@ applied to the uniform samples. - `loc`: location (scalar or d-vector). - `scale`: scale (scalar or d-vector, positive). """ -struct StudentT <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct StudentT{D <: AbstractDiscreteDistribution} <: AbstractTrueMeasure + dd::D dimension::Int df::Float64 loc::Vector{Float64} diff --git a/src/true_measure/triangular.jl b/src/true_measure/triangular.jl index 577bb1c..7bc964f 100644 --- a/src/true_measure/triangular.jl +++ b/src/true_measure/triangular.jl @@ -11,8 +11,8 @@ Each dimension is independently transformed through the triangular PPF. - `upper`: upper bound (scalar or d-vector). - `mode`: peak/mode (scalar or d-vector), must satisfy lower ≤ mode ≤ upper. """ -struct Triangular <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct Triangular{D <: AbstractDiscreteDistribution} <: AbstractTrueMeasure + dd::D dimension::Int lower::Vector{Float64} upper::Vector{Float64} diff --git a/src/true_measure/uniform_triangle.jl b/src/true_measure/uniform_triangle.jl index c708960..9cfbfbc 100644 --- a/src/true_measure/uniform_triangle.jl +++ b/src/true_measure/uniform_triangle.jl @@ -16,11 +16,11 @@ x = transform(tm, gen_samples(dd, 1000)) # x[:,1] >= x[:,2] and both in [0,1] ``` """ -struct UniformTriangle <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution - function UniformTriangle(dd::AbstractDiscreteDistribution) +struct UniformTriangle{D <: AbstractDiscreteDistribution} <: AbstractTrueMeasure + dd::D + function UniformTriangle(dd::D) where {D <: AbstractDiscreteDistribution} dd.dimension == 2 || throw(ArgumentError("UniformTriangle requires dimension=2")) - return new(dd) + return new{D}(dd) end end diff --git a/src/true_measure/zero_inflated_exp_uniform.jl b/src/true_measure/zero_inflated_exp_uniform.jl index 4b9bad0..a59ed99 100644 --- a/src/true_measure/zero_inflated_exp_uniform.jl +++ b/src/true_measure/zero_inflated_exp_uniform.jl @@ -17,8 +17,8 @@ tm = ZeroInflatedExpUniform(dd; p_zero=0.3, rate=2.0) x = transform(tm, gen_samples(dd, 1000)) ``` """ -struct ZeroInflatedExpUniform <: AbstractTrueMeasure - dd::AbstractDiscreteDistribution +struct ZeroInflatedExpUniform{D <: AbstractDiscreteDistribution} <: AbstractTrueMeasure + dd::D p_zero::Float64 rate::Float64 low::Float64 From 1ee36dc666b91f5d3ebd30e53fe30faa19196bb5 Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 10:02:10 -0500 Subject: [PATCH 003/104] Measure notebook run time --- test/run_notebooks.jl | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/test/run_notebooks.jl b/test/run_notebooks.jl index ce914b4..561e17a 100644 --- a/test/run_notebooks.jl +++ b/test/run_notebooks.jl @@ -39,6 +39,16 @@ function Logging.handle_message( cl.inner, level, message, _module, group, id, file, line; kwargs...) end +# Format a duration in seconds as a short string. +function fmt_duration(s::Real) + if s >= 60 + m = floor(Int, s / 60) + return "$(m)m $(round(s - 60m; digits = 1))s" + else + return "$(round(s; digits = 2))s" + end +end + # ── Main ───────────────────────────────────────────────── demos_dir = joinpath(@__DIR__, "..", "demos") notebooks = sort(filter(f -> endswith(f, ".ipynb"), readdir(demos_dir))) @@ -53,6 +63,7 @@ if !isempty(ARGS) end errors = String[] +times = Dict{String, Float64}() # notebook name → wall-clock seconds clogger = CountingLogger() for nb in notebooks @@ -60,7 +71,7 @@ for nb in notebooks println("Running: ", nb) println("="^60) clogger.current_nb[] = nb - try + elapsed = @elapsed try with_logger(clogger) do @nbinclude(joinpath(demos_dir, nb)) end @@ -70,28 +81,32 @@ for nb in notebooks showerror(stdout, e, catch_backtrace()) println() end + times[nb] = elapsed + println(" ⏱ elapsed: ", fmt_duration(elapsed)) end # ── Summary ────────────────────────────────────────────── total_warnings = sum(values(clogger.counts); init = 0) +total_time = sum(values(times); init = 0.0) n_pass = length(notebooks) - length(errors) println("\n", "="^60) +println("Summary") +println("="^60) +for nb in notebooks + status = nb in errors ? "✗ FAILED" : "✓ ok" + w = get(clogger.counts, nb, 0) + t = get(times, nb, 0.0) + wtxt = w > 0 ? ", $w warning(s)" : "" + println(" $status $nb [$(fmt_duration(t))]$wtxt") +end +println("-"^60) println( - "Ran $(length(notebooks)) notebook(s): $(n_pass) passed, $(length(errors)) failed, $(total_warnings) warning(s).", + "Ran $(length(notebooks)) notebook(s): $(n_pass) passed, $(length(errors)) failed, " * + "$(total_warnings) warning(s) in $(fmt_duration(total_time)).", ) -if total_warnings > 0 - for nb in notebooks - w = get(clogger.counts, nb, 0) - w > 0 && println(" ⚠ $nb: $w warning(s)") - end -end - if !isempty(errors) - for nb in errors - println(" ✗ $nb: FAILED") - end exit(1) else println("All notebooks passed.") From 8d49e5371a4bd89a2cf4dd1407b81df7ace423cc Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 10:32:07 -0500 Subject: [PATCH 004/104] Add benchmarking --- Makefile | 6 +- benchmark/Manifest.toml | 543 ++++++++++++++++++++++++++++++++++ benchmark/Project.toml | 10 + benchmark/benchmarks.jl | 31 +- benchmark/results/latest.json | 1 + 5 files changed, 584 insertions(+), 7 deletions(-) create mode 100644 benchmark/Manifest.toml create mode 100644 benchmark/Project.toml create mode 100644 benchmark/results/latest.json diff --git a/Makefile b/Makefile index 7fc0a75..9037b95 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test doc format format-check lint clean +.PHONY: test doc format format-check lint clean bench FORMATTER_PROJECT=devtools/formatter @@ -47,3 +47,7 @@ notebook: # Run a single notebook by name: make notebook-quickstart notebook-%: julia --project=. test/run_notebooks.jl $* + +# Run the benchmark suite (uses its own environment in benchmark/, set up on first run) +bench: + julia benchmark/benchmarks.jl diff --git a/benchmark/Manifest.toml b/benchmark/Manifest.toml new file mode 100644 index 0000000..57be5ec --- /dev/null +++ b/benchmark/Manifest.toml @@ -0,0 +1,543 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.12.6" +manifest_format = "2.0" +project_hash = "7aa63bce31182e8cc1e1f01f792bc7b5d5001817" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.5.0" + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + AbstractFFTsTestExt = "Test" + + [deps.AbstractFFTs.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.AliasTables]] +deps = ["PtrArrays", "Random"] +git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" +uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" +version = "1.1.3" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.BenchmarkTools]] +deps = ["Compat", "JSON", "Logging", "PrecompileTools", "Printf", "Profile", "Statistics", "UUIDs"] +git-tree-sha1 = "9670d3febc2b6da60a0ae57846ba74670290653f" +uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +version = "1.8.0" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.18.1" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.3.0+1" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + +[[deps.DataStructures]] +deps = ["OrderedCollections"] +git-tree-sha1 = "6fb53a69613a0b2b68a0d12671717d307ab8b24e" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.19.5" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.Distributions]] +deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] +git-tree-sha1 = "e421c1938fafab0165b04dc1a9dbe2a26272952c" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.125" + + [deps.Distributions.extensions] + DistributionsChainRulesCoreExt = "ChainRulesCore" + DistributionsDensityInterfaceExt = "DensityInterface" + DistributionsTestExt = "Test" + + [deps.Distributions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.DocStringExtensions]] +git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.5" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.7.0" + +[[deps.FFTW]] +deps = ["AbstractFFTs", "FFTW_jll", "Libdl", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] +git-tree-sha1 = "97f08406df914023af55ade2f843c39e99c5d969" +uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" +version = "1.10.0" + +[[deps.FFTW_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "6866aec60ef98e3164cd8d6855225684207e9dff" +uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" +version = "3.3.12+0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.FillArrays]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "2f979084d1e13948a3352cf64a25df6bd3b4dca3" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "1.16.0" + + [deps.FillArrays.extensions] + FillArraysPDMatsExt = "PDMats" + FillArraysSparseArraysExt = "SparseArrays" + FillArraysStaticArraysExt = "StaticArrays" + FillArraysStatisticsExt = "Statistics" + + [deps.FillArrays.weakdeps] + PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[deps.HypergeometricFunctions]] +deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] +git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec" +uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" +version = "0.3.28" + +[[deps.IntelOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] +git-tree-sha1 = "ec1debd61c300961f98064cfb21287613ad7f303" +uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" +version = "2025.2.0+0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "b2d91fe939cae05960e760110b328288867b5758" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.6" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "7204148362dafe5fe6a273f855b8ccbe4df8173e" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.8.0" + +[[deps.JSON]] +deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs", "Unicode"] +git-tree-sha1 = "f76f7560267b840e492180f9899b472f30b88450" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "1.6.0" + + [deps.JSON.extensions] + JSONArrowExt = ["ArrowTypes"] + + [deps.JSON.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.12.0" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +version = "1.11.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.15.0+0" + +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.9.0+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.3+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.12.0" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "13ca9e2586b89836fd20cccf56e57e2b9ae7f38f" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.29" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.MKL_jll]] +deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"] +git-tree-sha1 = "282cadc186e7b2ae0eeadbd7a4dffed4196ae2aa" +uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" +version = "2025.2.0+0" + +[[deps.Markdown]] +deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.2.0" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2025.11.4" + +[[deps.NBInclude]] +deps = ["JSON", "Markdown", "SoftGlobalScope"] +git-tree-sha1 = "89248151cda1af190b8d634cf58fc6b21a333a34" +uuid = "0db19996-df87-5ea3-a455-e3a50d440464" +version = "2.4.0" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.3.0" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.29+0" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.7+0" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.5.4+0" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1346c9208249809840c91b26703912dff463d335" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.6+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "94ba93778373a53bfd5a0caaf7d809c445292ff4" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.8.2" + +[[deps.PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "e4cff168707d441cd6bf3ff7e4832bdf34278e4a" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.37" +weakdeps = ["StatsBase"] + + [deps.PDMats.extensions] + StatsBaseExt = "StatsBase" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "5d5e0a78e971354b1c7bff0655d11fdc1b0e12c8" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.8.4" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.12.1" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "edbeefc7a4889f528644251bdb5fc9ab5348bc2c" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.3.4" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.5.2" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.Profile]] +deps = ["StyledStrings"] +uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" +version = "1.11.0" + +[[deps.PtrArrays]] +git-tree-sha1 = "4fbbafbc6251b883f4d2705356f3641f3652a7fe" +uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" +version = "1.4.0" + +[[deps.QMC]] +deps = ["Distributions", "FFTW", "Libdl", "LinearAlgebra", "NBInclude", "Printf", "Random", "SpecialFunctions", "Statistics"] +path = "/Users/terrya/Documents/ProgramData/qmcju_software_choi" +uuid = "b8e7c4a1-3f5d-4e9a-b2c6-1a8d9f0e7c3b" +version = "0.1.0" + +[[deps.QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "5e8e8b0ab68215d7a2b14b9921a946fee794749e" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.11.3" + + [deps.QuadGK.extensions] + QuadGKEnzymeExt = "Enzyme" + + [deps.QuadGK.weakdeps] + Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" + +[[deps.REPL]] +deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "5b3d50eb374cea306873b371d3f8d3915a018f0b" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.9.0" + +[[deps.Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "58cdd8fb2201a6267e1db87ff148dd6c1dbd8ad8" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.5.1+0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.SoftGlobalScope]] +deps = ["REPL"] +git-tree-sha1 = "986ec2b6162ccb95de5892ed17832f95badf770c" +uuid = "b85f4697-e234-5449-a836-ec8e2f98b302" +version = "1.1.0" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "64d974c2e6fdf07f8155b5b2ca2ffa9069b608d9" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.2.2" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.12.0" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "6547cbdd8ce32efba0d21c5a40fa96d1a3548f9f" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.8.0" + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + + [deps.SpecialFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + +[[deps.Statistics]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11.1" +weakdeps = ["SparseArrays"] + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "178ed29fd5b2a2cfc3bd31c13375ae925623ff36" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.8.0" + +[[deps.StatsBase]] +deps = ["AliasTables", "DataAPI", "DataStructures", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "c6f18e5a52a176a383f6f6c635e0f81feed1d6d4" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.11" + +[[deps.StatsFuns]] +deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "91f091a8716a6bb38417a6e6f274602a19aaa685" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "1.5.2" + + [deps.StatsFuns.extensions] + StatsFunsChainRulesCoreExt = "ChainRulesCore" + StatsFunsInverseFunctionsExt = "InverseFunctions" + + [deps.StatsFuns.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.StructUtils]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "82bee338d650aa515f31866c460cb7e3bcef90b8" +uuid = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" +version = "2.8.2" + + [deps.StructUtils.extensions] + StructUtilsMeasurementsExt = ["Measurements"] + StructUtilsStaticArraysCoreExt = ["StaticArraysCore"] + StructUtilsTablesExt = ["Tables"] + + [deps.StructUtils.weakdeps] + Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" + StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" + Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.8.3+2" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.3.1+2" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.15.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.64.0+1" + +[[deps.oneTBB_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] +git-tree-sha1 = "da8c1f6eee04831f14edcfa5dae611d309807e57" +uuid = "1317d2d5-d96f-522e-a858-c73665f53c3e" +version = "2022.3.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.7.0+0" diff --git a/benchmark/Project.toml b/benchmark/Project.toml new file mode 100644 index 0000000..8b4ea09 --- /dev/null +++ b/benchmark/Project.toml @@ -0,0 +1,10 @@ +[deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +QMC = "b8e7c4a1-3f5d-4e9a-b2c6-1a8d9f0e7c3b" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[compat] +BenchmarkTools = "1" +julia = "1.12" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 133a50a..7197db3 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,14 +1,26 @@ # QMC.jl Benchmark Suite # -# Usage: -# julia --project=. benchmark/benchmarks.jl +# Usage (from the repository root): +# julia benchmark/benchmarks.jl # -# Requires BenchmarkTools.jl (add to your environment if not present). +# The script uses its own environment in `benchmark/` (see benchmark/Project.toml) +# so BenchmarkTools is not pulled into the main package deps. It bootstraps that +# environment on first run: activating it, developing the parent QMC package, and +# installing BenchmarkTools. No `--project` flag or manual `Pkg.add` is needed. + +using Pkg +Pkg.activate(@__DIR__) +let deps = keys(Pkg.project().dependencies) + "QMC" in deps || Pkg.develop(; path = dirname(@__DIR__)) + "BenchmarkTools" in deps || Pkg.add("BenchmarkTools") +end +Pkg.instantiate() using QMC using BenchmarkTools using Statistics using Printf +using Logging import QMC: Uniform # ── Configuration ───────────────────────────────────────────────────────── @@ -24,7 +36,7 @@ function bench_gen_samples(dd_constructor, dim, n; kwargs...) end function bench_integrate(make_sc; kwargs...) - @benchmarkable integrate($sc) evals=1 samples=3 setup=(sc = $make_sc()) + @benchmarkable integrate(sc) evals=1 samples=3 setup=(sc = $make_sc()) end # ── Benchmark Groups ───────────────────────────────────────────────────── @@ -41,7 +53,7 @@ for dim in DIMS, n in SAMPLES suite["gen_samples"]["DigitalNetB2 d=$dim n=$n"] = bench_gen_samples(DigitalNetB2, dim, n; randomize="LMS_DS") suite["gen_samples"]["Halton d=$dim n=$n"] = - bench_gen_samples(Halton, dim, n; randomize="OWEN") + bench_gen_samples(Halton, dim, n; randomize=true) suite["gen_samples"]["Kronecker d=$dim n=$n"] = bench_gen_samples(Kronecker, dim, n) end @@ -117,7 +129,14 @@ end println("QMC.jl Benchmarks") println("=" ^ 70) -results = run(suite; verbose=true) +# CubMCCLT is a faithful single-pass two-stage estimator: its realized error can +# land just above the requested tolerance whenever the main-stage variance +# exceeds the pilot estimate, which correctly emits a non-convergence warning. +# That is expected and irrelevant here — we are timing, not checking accuracy — +# so warnings are silenced for the duration of the run to keep output readable. +results = with_logger(ConsoleLogger(stderr, Logging.Error)) do + run(suite; verbose=true) +end # ── Summary ────────────────────────────────────────────────────────────── diff --git a/benchmark/results/latest.json b/benchmark/results/latest.json new file mode 100644 index 0000000..695c0f0 --- /dev/null +++ b/benchmark/results/latest.json @@ -0,0 +1 @@ +[{"BenchmarkTools":"1.8.0","Julia":"1.12.6"},[["BenchmarkGroup",{"data":{"evaluate":["BenchmarkGroup",{"data":{"Genz(continuous) n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":8256,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[4694.666666666667,4097.0,4125.0,4055.3333333333335,4125.0]}],"Genz(continuous) n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":131136,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[58458.333333333336,57305.666666666664,57236.0,57333.333333333336,62277.666666666664]}],"Genz(continuous) n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":2112,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1916.6666666666667,1014.0,1097.0,944.3333333333334,1069.3333333333333]}],"Genz(continuous) n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":32832,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[16722.333333333332,16416.666666666668,16028.0,16222.333333333334,16125.0]}],"Keister n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":8256,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[7736.0,5319.333333333333,5125.0,5055.333333333333,5264.0]}],"Keister n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":131136,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[110014.0,96291.66666666667,95000.0,95750.0,95055.33333333333]}],"Keister n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":2112,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2847.0,1250.0,1319.3333333333333,1194.3333333333333,1277.6666666666667]}],"Keister n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":32832,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[29069.333333333332,22597.333333333332,21819.333333333332,22444.333333333332,24139.0]}]},"tags":[]}],"gen_samples":["BenchmarkGroup",{"data":{"DigitalNetB2 d=10 n=1024":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":335392,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[44444.666666666664,44125.0,37527.666666666664,36444.333333333336,41750.0]}],"DigitalNetB2 d=10 n=16384":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":5250592,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[823958.3333333334,760291.6666666666,759847.0,740819.3333333334,747222.3333333334]}],"DigitalNetB2 d=10 n=256":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":89632,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[17097.333333333332,14291.666666666666,14264.0,18000.0,18680.666666666668]}],"DigitalNetB2 d=10 n=4096":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1318432,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[149430.66666666666,143250.0,138944.66666666666,136763.66666666666,137000.0]}],"DigitalNetB2 d=3 n=1024":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":101040,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[14514.0,12069.666666666666,13180.666666666666,14028.0,22916.666666666668]}],"DigitalNetB2 d=3 n=16384":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1575600,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[152083.33333333334,161944.33333333334,165847.33333333334,162847.33333333334,190639.0]}],"DigitalNetB2 d=3 n=256":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":27312,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[7055.666666666667,4958.333333333333,4958.333333333333,5055.666666666667,4652.666666666667]}],"DigitalNetB2 d=3 n=4096":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":395952,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[44639.0,44472.333333333336,41555.333333333336,48083.333333333336,44069.333333333336]}],"Halton d=10 n=1024":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":84688,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[378319.3333333333,383166.6666666667,384236.3333333333,389000.0,384708.3333333333]}],"Halton d=10 n=16384":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1313488,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6.359083333333333e6,6.377583333333333e6,6.671653e6,6.374014e6,6.384625e6]}],"Halton d=10 n=256":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":23248,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[94194.66666666667,93041.66666666667,92986.33333333333,93208.33333333333,92930.66666666667]}],"Halton d=10 n=4096":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":330448,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1.6355833333333333e6,1.759764e6,1.560486e6,1.572153e6,1.7474446666666667e6]}],"Halton d=3 n=1024":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":25552,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[128111.0,129375.0,124972.33333333333,125333.33333333333,126847.33333333333]}],"Halton d=3 n=16384":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":394192,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2.1427223333333335e6,2.1118333333333335e6,2.1305833333333335e6,2.0853333333333333e6,2.0952916666666667e6]}],"Halton d=3 n=256":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":7120,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[29778.0,31875.0,31861.333333333332,32319.666666666668,28847.333333333332]}],"Halton d=3 n=4096":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":99280,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[515444.3333333333,524666.6666666666,530805.3333333334,515625.0,502041.6666666667]}],"IIDStdUniform d=10 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":82000,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[7666.666666666667,5333.333333333333,5305.333333333333,5319.333333333333,4389.0]}],"IIDStdUniform d=10 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1310800,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[122805.66666666667,103930.33333333333,108555.66666666667,92375.0,108152.66666666667]}],"IIDStdUniform d=10 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":20560,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2583.3333333333335,1930.6666666666667,1153.0,1389.0,1208.3333333333333]}],"IIDStdUniform d=10 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":327760,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[26736.333333333332,17166.666666666668,25458.333333333332,17139.0,21278.0]}],"IIDStdUniform d=3 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":24656,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2514.0,1639.0,2152.6666666666665,1444.3333333333333,1430.3333333333333]}],"IIDStdUniform d=3 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":393296,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[39583.333333333336,25708.333333333332,31833.333333333332,22236.0,30680.666666666668]}],"IIDStdUniform d=3 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":6224,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1791.6666666666667,611.0,555.6666666666666,486.3333333333333,472.3333333333333]}],"IIDStdUniform d=3 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":98384,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[11166.666666666666,7250.0,6014.0,5750.0,6958.333333333333]}],"Kronecker d=10 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":82000,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[22916.666666666668,18750.0,19888.666666666668,20819.333333333332,20805.666666666668]}],"Kronecker d=10 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1310800,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[469194.3333333333,520055.6666666667,528111.0,545527.6666666666,508819.3333333333]}],"Kronecker d=10 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":20560,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6027.666666666667,5319.333333333333,5180.666666666667,5250.0,5166.666666666667]}],"Kronecker d=10 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":327760,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[107833.33333333333,101055.33333333333,116166.66666666667,117777.66666666667,112944.33333333333]}],"Kronecker d=3 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":24656,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6500.0,5750.0,5888.666666666667,6889.0,5708.333333333333]}],"Kronecker d=3 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":393296,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[137652.66666666666,137041.66666666666,138861.0,138208.33333333334,147055.33333333334]}],"Kronecker d=3 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":6224,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2208.3333333333335,1500.0,1472.0,1500.0,1653.0]}],"Kronecker d=3 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":98384,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[31764.0,27666.666666666668,31458.333333333332,27236.0,28902.666666666668]}],"Lattice d=10 n=1024":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":246656,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[25236.0,24458.333333333332,16930.333333333332,22347.333333333332,23805.666666666668]}],"Lattice d=10 n=16384":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":3933056,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[333444.3333333333,322819.3333333333,323583.3333333333,323930.3333333333,332819.3333333333]}],"Lattice d=10 n=256":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":62336,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6208.333333333333,4333.333333333333,4194.333333333333,4250.0,4264.0]}],"Lattice d=10 n=4096":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":983936,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[76305.66666666667,74125.0,74166.66666666667,73889.0,73777.66666666667]}],"Lattice d=3 n=1024":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":74160,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[8805.666666666666,6972.0,8097.0,9902.666666666666,11277.666666666666]}],"Lattice d=3 n=16384":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1180080,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[120764.0,118903.0,121472.33333333333,124083.33333333333,125722.0]}],"Lattice d=3 n=256":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":18864,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[4250.0,2138.6666666666665,2319.6666666666665,2208.3333333333335,2222.0]}],"Lattice d=3 n=4096":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":295344,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[29291.666666666668,26861.0,26930.666666666668,26805.666666666668,29375.0]}]},"tags":[]}],"integrate":["BenchmarkGroup",{"data":{"CubMCCLT AsianOption":["Trial",{"allocs":3547,"gctimes":[0.0,0.0,0.0],"memory":5834128,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[9.4535e6,1.2386208e7,9.030042e6]}],"CubMCCLT Keister":["Trial",{"allocs":49,"gctimes":[0.0,0.0,477292.0],"memory":67864112,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[3.45125e7,3.0809792e7,3.6293542e7]}],"CubQMCLatticeG Keister":["Trial",{"allocs":385,"gctimes":[0.0,0.0,0.0],"memory":2108592,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[649583.0,632791.0,627709.0]}],"CubQMCNetG EuropeanOption":["Trial",{"allocs":18545,"gctimes":[0.0,0.0,0.0],"memory":47429552,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[1.4737583e7,1.6672042e7,1.4372791e7]}],"CubQMCNetG Keister":["Trial",{"allocs":577,"gctimes":[0.0,0.0,0.0],"memory":2538672,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[773625.0,854334.0,866541.0]}]},"tags":[]}],"transform":["BenchmarkGroup",{"data":{"Gaussian d=10 n=1024":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":164000,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[91014.0,84611.0,84611.0,84416.66666666667,84111.33333333333]}],"Gaussian d=10 n=16384":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":2621600,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1.4138193333333333e6,1.464264e6,1.4783053333333333e6,1.4225693333333333e6,1.3385276666666667e6]}],"Gaussian d=10 n=256":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":41120,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[24736.0,21000.0,20527.666666666668,20458.333333333332,20500.0]}],"Gaussian d=10 n=4096":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":655520,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[352889.0,346666.6666666667,357111.0,350514.0,340930.6666666667]}],"Gaussian d=3 n=1024":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":49312,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[29277.666666666668,25653.0,24180.666666666668,23958.333333333332,23972.0]}],"Gaussian d=3 n=16384":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":786592,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[416291.6666666667,403666.6666666667,408027.6666666667,417777.6666666667,394152.6666666667]}],"Gaussian d=3 n=256":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":12448,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[9889.0,6208.333333333333,6111.333333333333,6375.0,6180.333333333333]}],"Gaussian d=3 n=4096":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":196768,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[105319.33333333333,98180.66666666667,97944.33333333333,97875.0,97847.33333333333]}]},"tags":[]}]},"tags":[]}]]] \ No newline at end of file From 8990e48f888544a8cce2e6f85d7ed15bb26b8ca6 Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 10:39:59 -0500 Subject: [PATCH 005/104] Update Julia version to 1.12 and enhance benchmark script documentation --- Project.toml | 2 +- benchmark/benchmarks.jl | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 8233f2e..d3fcb0f 100644 --- a/Project.toml +++ b/Project.toml @@ -28,7 +28,7 @@ Random = "1.10" SpecialFunctions = "2" Statistics = "1.10" Test = "1.10" -julia = "1.10" +julia = "1.12" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 7197db3..2dd403c 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -7,6 +7,16 @@ # so BenchmarkTools is not pulled into the main package deps. It bootstraps that # environment on first run: activating it, developing the parent QMC package, and # installing BenchmarkTools. No `--project` flag or manual `Pkg.add` is needed. +# +# Remarks: +# `allocs` is the number of heap allocations Julia performed during a benchmarked call. +# Fewer `allocs` means less garbage collection. +# For example, `16 allocs` means 16 separate memory allocations. +# +# `KiB` is the total amount of memory allocated, in kibibytes, where `1 KiB = 1024 bytes`. +# Lower `KiB` means less total temporary memory was created. +# So `240.9 KiB` means about `240.9 * 1024 ≈ 246,682` bytes were allocated during the call. + using Pkg Pkg.activate(@__DIR__) From 7ce5a8a40338ac925c39dc97c4300c3c0a8e3788 Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 11:02:17 -0500 Subject: [PATCH 006/104] Update References --- community.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/community.md b/community.md index c1cc512..9400db1 100644 --- a/community.md +++ b/community.md @@ -17,6 +17,7 @@ See the full list at the [QMCSoftware community page](https://qmcsoftware.github - [Julia Slack — #math channel](https://julialang.slack.com/) - [JuliaStats](https://github.com/JuliaStats) — statistics ecosystem - [JuliaMath](https://github.com/JuliaMath) — mathematics ecosystem +- [https://julialang.org/learning/](https://julialang.org/learning/) ## Select References @@ -36,6 +37,8 @@ The mathematical and algorithmic foundations used by QMC.jl are documented in th - Sanz-Alonso, Daniel, and Omar Al-Ghattas. *A First Course in Monte Carlo Methods.* Preprint, 2024. -### Julia Learning Materials +### Probability and Statistics Background -See [https://julialang.org/learning/](https://julialang.org/learning/). \ No newline at end of file +- Ross, Sheldon. *A First Course in Probability*. 9th ed. Boston: Pearson, 2014. + +- Wasserman, Larry. *All of Statistics: A Concise Course in Statistical Inference*. New York: Springer, 2004. From 26dcfa9ac18d1036b08b07f218793bf10305ae07 Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 14:45:25 -0500 Subject: [PATCH 007/104] measure execution time for each test, --- test/runtests.jl | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index a561afa..7b34a94 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,13 +6,36 @@ using Distributions # Resolve name conflicts: prefer QMC's versions in tests import QMC: Uniform, Kumaraswamy +const TEST_FILES = [ + "test_discrete_distributions.jl", + "test_true_measures.jl", + "test_integrands.jl", + "test_kernels.jl", + "test_stopping_criteria.jl", + "test_integration.jl", + "test_multilevel.jl", + "test_aqua.jl", +] + +"Format a duration in seconds as a short string." +function fmt_duration(s::Real) + if s >= 60 + m = floor(Int, s / 60) + return "$(m)m $(round(s - 60m; digits = 1))s" + else + return "$(round(s; digits = 2))s" + end +end + +times = Dict{String, Float64}() + @testset "QMC" begin - include("test_discrete_distributions.jl") - include("test_true_measures.jl") - include("test_integrands.jl") - include("test_kernels.jl") - include("test_stopping_criteria.jl") - include("test_integration.jl") - include("test_multilevel.jl") - include("test_aqua.jl") + for test_file in TEST_FILES + elapsed = @elapsed include(test_file) + times[test_file] = elapsed + println(" ⏱ $(test_file): $(fmt_duration(elapsed))") + end end + +total_time = sum(values(times); init = 0.0) +println("Ran $(length(TEST_FILES)) test file(s) in $(fmt_duration(total_time)).") From 8df624fb480600b9c5bd8ca631679baf0dc1be9b Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 14:45:47 -0500 Subject: [PATCH 008/104] Removed print statements --- test/test_multilevel.jl | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/test/test_multilevel.jl b/test/test_multilevel.jl index c38382d..4127dc1 100644 --- a/test/test_multilevel.jl +++ b/test/test_multilevel.jl @@ -139,10 +139,6 @@ end @test !isnan(result.solution) @test result.data[:n_total] > 0 @test result.data[:levels] >= 3 # levels_min + 1 - println( - "CubMLMC: solution=$(round(result.solution; digits=3)), " * - "n_total=$(result.data[:n_total]), levels=$(result.data[:levels])", - ) end @testset "CubMLMCCont" begin @@ -167,10 +163,6 @@ end @test result.solution isa Float64 @test !isnan(result.solution) @test result.data[:n_total] > 0 - println( - "CubMLMCCont: solution=$(round(result.solution; digits=3)), " * - "n_total=$(result.data[:n_total]), levels=$(result.data[:levels])", - ) end @testset "CubMLQMCCont" begin @@ -196,10 +188,4 @@ end @test !isnan(result.solution) @test result.data[:n_total] > 0 @test result.data[:replications] == 8 - println( - "CubMLQMCCont: solution=$(round(result.solution; digits=3)), " * - "n_total=$(result.data[:n_total]), levels=$(result.data[:levels])", - ) end - -println("\nAll multilevel tests passed!") From f21de9e56c3f9953df2f4fd4a4b3abb0d6eb69f4 Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 14:53:44 -0500 Subject: [PATCH 009/104] Add benchmarking scripts for QMCPy and Julia --- Makefile | 10 +- benchmark/Manifest.toml | 31 +++++- benchmark/Project.toml | 4 +- benchmark/benchmark_qmcpy.py | 176 ++++++++++++++++++++++++++++++++++ benchmark/benchmarks.jl | 134 +++++++++----------------- benchmark/compare.jl | 96 +++++++++++++++++++ benchmark/results/latest.json | 2 +- benchmark/runbenchmarks.jl | 56 +++++++++++ benchmark/tune.json | 1 + 9 files changed, 414 insertions(+), 96 deletions(-) create mode 100644 benchmark/benchmark_qmcpy.py create mode 100644 benchmark/compare.jl create mode 100644 benchmark/runbenchmarks.jl create mode 100644 benchmark/tune.json diff --git a/Makefile b/Makefile index 9037b95..e79686c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test doc format format-check lint clean bench +.PHONY: test doc format format-check lint clean bench bench-compare FORMATTER_PROJECT=devtools/formatter @@ -50,4 +50,10 @@ notebook-%: # Run the benchmark suite (uses its own environment in benchmark/, set up on first run) bench: - julia benchmark/benchmarks.jl + julia benchmark/runbenchmarks.jl + +# Compare the working tree against a baseline git revision (default: HEAD, i.e. +# the effect of your uncommitted changes). Override with: make bench-compare REV=master +REV ?= HEAD +bench-compare: + julia benchmark/compare.jl $(REV) diff --git a/benchmark/Manifest.toml b/benchmark/Manifest.toml index 57be5ec..0e442ea 100644 --- a/benchmark/Manifest.toml +++ b/benchmark/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.12.6" manifest_format = "2.0" -project_hash = "7aa63bce31182e8cc1e1f01f792bc7b5d5001817" +project_hash = "f886813682ad274be6b2fd29c59c0591e8733b5b" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] @@ -18,6 +18,11 @@ version = "1.5.0" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +[[deps.AbstractTrees]] +git-tree-sha1 = "2d9c9a55f9c93e8887ad391fbae72f8ef55e1177" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.4.5" + [[deps.AliasTables]] deps = ["PtrArrays", "Random"] git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" @@ -183,6 +188,12 @@ deps = ["Artifacts", "Pkg"] uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" version = "1.11.0" +[[deps.LeftChildRightSiblingTrees]] +deps = ["AbstractTrees"] +git-tree-sha1 = "95ba48564903b43b2462318aa243ee79d81135ff" +uuid = "1d6d02ad-be62-4b6b-8a6d-2f90e265016e" +version = "0.2.1" + [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" @@ -319,6 +330,12 @@ weakdeps = ["REPL"] [deps.Pkg.extensions] REPLExt = "REPL" +[[deps.PkgBenchmark]] +deps = ["BenchmarkTools", "Dates", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Pkg", "Printf", "TerminalLoggers", "UUIDs"] +git-tree-sha1 = "10940959d1174f5402729bb0891cafb785c0e927" +uuid = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" +version = "0.2.15" + [[deps.PrecompileTools]] deps = ["Preferences"] git-tree-sha1 = "edbeefc7a4889f528644251bdb5fc9ab5348bc2c" @@ -341,6 +358,12 @@ deps = ["StyledStrings"] uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" version = "1.11.0" +[[deps.ProgressLogging]] +deps = ["Logging", "SHA", "UUIDs"] +git-tree-sha1 = "f0803bc1171e455a04124affa9c21bba5ac4db32" +uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" +version = "0.1.6" + [[deps.PtrArrays]] git-tree-sha1 = "4fbbafbc6251b883f4d2705356f3641f3652a7fe" uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" @@ -507,6 +530,12 @@ deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" version = "1.10.0" +[[deps.TerminalLoggers]] +deps = ["LeftChildRightSiblingTrees", "Logging", "Markdown", "Printf", "ProgressLogging", "UUIDs"] +git-tree-sha1 = "f133fab380933d042f6796eda4e130272ba520ca" +uuid = "5d786b92-1e48-4d6f-9151-6b4477ca9bed" +version = "0.1.7" + [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" diff --git a/benchmark/Project.toml b/benchmark/Project.toml index 8b4ea09..22db034 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -1,10 +1,12 @@ [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" QMC = "b8e7c4a1-3f5d-4e9a-b2c6-1a8d9f0e7c3b" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [compat] BenchmarkTools = "1" -julia = "1.12" +PkgBenchmark = "0.2" +julia = "1.10" diff --git a/benchmark/benchmark_qmcpy.py b/benchmark/benchmark_qmcpy.py new file mode 100644 index 0000000..563622e --- /dev/null +++ b/benchmark/benchmark_qmcpy.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""QMCPy benchmark harness — Python counterpart of benchmark/benchmarks.jl. + +Mirrors the Julia `SUITE` so results can be compared case-by-case against +benchmark/results/latest.json (from `make bench`). Run with the QMCPy that the +Julia package was translated from: + + pip install qmcpy # or put QMCSoftware/ on PYTHONPATH + python benchmark/benchmark_qmcpy.py + python benchmark/benchmark_qmcpy.py mylabel # -> results/qmcpy_mylabel.json + +IMPORTANT — what is and isn't a language comparison +--------------------------------------------------- +* `gen_samples` for Lattice / DigitalNetB2 / Halton calls the SAME `qmctoolscl` + C library in both packages. Those rows compare the C kernel + FFI binding, not + Julia vs Python. Tagged [C] below. +* IIDStdUniform / Kronecker generation, `transform`, and `evaluate` are pure + Python/numpy vs pure Julia — those are the honest language comparisons. +* `integrate` mixes both (generator + transform + evaluate + algorithm glue). + +Method parity with the Julia suite +---------------------------------- +* gen_samples : dd(n) ~ gen_samples(dd, n) +* transform : Gaussian(dd)._transform(x) ~ transform(tm, x) +* evaluate : integrand.g(t) ~ evaluate(f, x) (t = transformed points) +* integrate : sc.integrate() ~ integrate(sc) + +Caveats: QMCPy's Genz supports only OSCILLATORY / CORNER PEAK (the Julia suite is +set to :oscillatory to match). QMCPy's FinancialOption wraps the sampler directly +(option="ASIAN"/"EUROPEAN") and builds its GBM internally, whereas the Julia +version wraps a GeometricBrownianMotion measure; the end-to-end problem is the +same. Timing here uses wall-clock medians; Python has no direct allocation count, +so only the time column is cross-comparable with the Julia ms column. +""" + +import sys +import json +import time +import timeit +import warnings +import statistics +from pathlib import Path + +import numpy as np +import qmcpy as qp + +# Mirror the Julia configuration. +SAMPLES = [256, 1024, 4096, 16384] +DIMS = [3, 10] +SEED = 42 + + +def bench(make_call, *, repeat=7, warmup=True): + """Return median seconds per call. `make_call` returns a zero-arg callable.""" + fn = make_call() + if warmup: + fn() # discard first call (cache/allocation warm-up) + # Pick an inner count so each timing sample is long enough to be stable. + timer = timeit.Timer(fn) + count, _ = timer.autorange() + samples = timer.repeat(repeat=repeat, number=count) + return statistics.median(samples) / count + + +def record(results, group, name, make_call, **kw): + try: + secs = bench(make_call, **kw) + results[group][name] = {"median_ms": secs * 1e3} + print(f" {name:<45s} {secs * 1e3:10.3f} ms") + except Exception as e: # noqa: BLE001 - keep one bad case from aborting the run + results[group][name] = {"error": f"{type(e).__name__}: {e}"} + print(f" {name:<45s} ERR: {type(e).__name__}: {e}") + + +def main(): + label = sys.argv[1] if len(sys.argv) > 1 else "latest" + results = {"gen_samples": {}, "transform": {}, "evaluate": {}, "integrate": {}} + + print("QMCPy Benchmarks (qmcpy %s)" % getattr(qp, "__version__", "?")) + print("=" * 70) + + # 1. Discrete distribution sampling -------------------------------------- + # [C] = qmctoolscl C kernel (not a language comparison) + print("\n-- gen_samples --") + for dim in DIMS: + for n in SAMPLES: + record(results, "gen_samples", f"IIDStdUniform d={dim} n={n}", + lambda dim=dim, n=n: (lambda dd=qp.IIDStdUniform(dim, seed=SEED): (lambda: dd(n)))()) + record(results, "gen_samples", f"Lattice [C] d={dim} n={n}", + lambda dim=dim, n=n: (lambda dd=qp.Lattice(dim, seed=SEED): (lambda: dd(n)))()) + record(results, "gen_samples", f"DigitalNetB2 [C] d={dim} n={n}", + lambda dim=dim, n=n: (lambda dd=qp.DigitalNetB2(dim, seed=SEED): (lambda: dd(n)))()) + record(results, "gen_samples", f"Halton [C] d={dim} n={n}", + lambda dim=dim, n=n: (lambda dd=qp.Halton(dim, seed=SEED): (lambda: dd(n)))()) + record(results, "gen_samples", f"Kronecker d={dim} n={n}", + lambda dim=dim, n=n: (lambda dd=qp.Kronecker(dim, seed=SEED): (lambda: dd(n)))()) + + # 2. Transform (pure numpy: inverse-CDF + covariance factor) -------------- + print("\n-- transform --") + for dim in DIMS: + for n in SAMPLES: + def make_transform(dim=dim, n=n): + dd = qp.IIDStdUniform(dim, seed=SEED) + tm = qp.Gaussian(dd) + x = dd(n) + return lambda: tm._transform(x) + record(results, "transform", f"Gaussian d={dim} n={n}", make_transform) + + # 3. Integrand evaluation (pure numpy) ----------------------------------- + print("\n-- evaluate --") + for n in SAMPLES: + def make_keister(n=n): + dd = qp.IIDStdUniform(3, seed=SEED) + f = qp.Keister(dd) + t = f.true_measure._transform(dd(n)) + return lambda: f.g(t) + record(results, "evaluate", f"Keister n={n}", make_keister) + + def make_genz(n=n): + dd = qp.IIDStdUniform(3, seed=SEED) + f = qp.Genz(dd, kind_func="OSCILLATORY") + t = f.true_measure._transform(dd(n)) + return lambda: f.g(t) + record(results, "evaluate", f"Genz(oscillatory) n={n}", make_genz) + + # 4. End-to-end integration ---------------------------------------------- + print("\n-- integrate --") + + def integrate_call(make_sc): + def factory(): + def run(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return make_sc().integrate() + return run + return factory + + record(results, "integrate", "CubMCCLT Keister", + integrate_call(lambda: qp.CubMCCLT(qp.Keister(qp.IIDStdUniform(3, seed=SEED)), abs_tol=0.01)), + repeat=3, warmup=True) + record(results, "integrate", "CubQMCLatticeG Keister", + integrate_call(lambda: qp.CubQMCLatticeG(qp.Keister(qp.Lattice(3, seed=SEED)), abs_tol=0.01)), + repeat=3) + record(results, "integrate", "CubQMCNetG Keister", + integrate_call(lambda: qp.CubQMCNetG(qp.Keister(qp.DigitalNetB2(3, seed=SEED)), abs_tol=0.01)), + repeat=3) + record(results, "integrate", "CubMCCLT AsianOption", + integrate_call(lambda: qp.CubMCCLT( + qp.FinancialOption(qp.IIDStdUniform(50, seed=SEED), option="ASIAN", + volatility=0.2, start_price=100, strike_price=100, + interest_rate=0.05, t_final=1), abs_tol=0.5)), + repeat=3) + record(results, "integrate", "CubQMCNetG EuropeanOption", + integrate_call(lambda: qp.CubQMCNetG( + qp.FinancialOption(qp.DigitalNetB2(50, seed=SEED), option="EUROPEAN", + volatility=0.2, start_price=100, strike_price=100, + interest_rate=0.05, t_final=1), abs_tol=0.5)), + repeat=3) + + # ── Save ───────────────────────────────────────────────────────────── + resdir = Path(__file__).resolve().parent / "results" + resdir.mkdir(exist_ok=True) + outfile = resdir / f"qmcpy_{label}.json" + payload = { + "qmcpy_version": getattr(qp, "__version__", "?"), + "python": sys.version.split()[0], + "numpy": np.__version__, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"), + "results": results, + } + outfile.write_text(json.dumps(payload, indent=2)) + print(f"\nResults saved to benchmark/results/qmcpy_{label}.json") + + +if __name__ == "__main__": + main() diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 2dd403c..341ca2b 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,37 +1,24 @@ -# QMC.jl Benchmark Suite +# QMC.jl Benchmark Suite (PkgBenchmark-compatible). # -# Usage (from the repository root): -# julia benchmark/benchmarks.jl +# This file ONLY defines `const SUITE::BenchmarkGroup`; it does not run anything. +# * `benchmark/runbenchmarks.jl` runs SUITE standalone and saves results. +# * `benchmark/compare.jl` uses PkgBenchmark.jl to diff two git revisions. +# * PkgBenchmark.benchmarkpkg(...) includes this file directly. # -# The script uses its own environment in `benchmark/` (see benchmark/Project.toml) -# so BenchmarkTools is not pulled into the main package deps. It bootstraps that -# environment on first run: activating it, developing the parent QMC package, and -# installing BenchmarkTools. No `--project` flag or manual `Pkg.add` is needed. -# -# Remarks: -# `allocs` is the number of heap allocations Julia performed during a benchmarked call. -# Fewer `allocs` means less garbage collection. -# For example, `16 allocs` means 16 separate memory allocations. -# -# `KiB` is the total amount of memory allocated, in kibibytes, where `1 KiB = 1024 bytes`. -# Lower `KiB` means less total temporary memory was created. -# So `240.9 KiB` means about `240.9 * 1024 ≈ 246,682` bytes were allocated during the call. - - -using Pkg -Pkg.activate(@__DIR__) -let deps = keys(Pkg.project().dependencies) - "QMC" in deps || Pkg.develop(; path = dirname(@__DIR__)) - "BenchmarkTools" in deps || Pkg.add("BenchmarkTools") -end -Pkg.instantiate() +# Environment: the `benchmark/` project (benchmark/Project.toml). The runner and +# compare scripts bootstrap it; PkgBenchmark sets it up itself. -using QMC using BenchmarkTools -using Statistics -using Printf -using Logging +using QMC import QMC: Uniform +using Logging + +# Silence expected, non-actionable warnings (notably CubMCCLT's single-pass +# non-convergence notice) for EVERY harness that loads this suite — the standalone +# runner and PkgBenchmark's per-revision subprocesses alike. Disabling Warn-and- +# below process-wide is the only mechanism that reliably reaches code executed +# inside BenchmarkTools; errors are still shown. +Logging.disable_logging(Logging.Warn) # ── Configuration ───────────────────────────────────────────────────────── @@ -51,74 +38,79 @@ end # ── Benchmark Groups ───────────────────────────────────────────────────── -suite = BenchmarkGroup() +const SUITE = BenchmarkGroup() # 1. Discrete Distribution sampling -suite["gen_samples"] = BenchmarkGroup() +# NOTE: Lattice / DigitalNetB2 / Halton call the qmctoolscl C library for +# point generation (same library QMCPy uses), so these measure the C kernel + +# FFI binding, NOT Julia-vs-Python. IIDStdUniform and Kronecker are pure Julia. +SUITE["gen_samples"] = BenchmarkGroup() for dim in DIMS, n in SAMPLES - suite["gen_samples"]["IIDStdUniform d=$dim n=$n"] = + SUITE["gen_samples"]["IIDStdUniform d=$dim n=$n"] = bench_gen_samples(IIDStdUniform, dim, n) - suite["gen_samples"]["Lattice d=$dim n=$n"] = + SUITE["gen_samples"]["Lattice d=$dim n=$n"] = bench_gen_samples(Lattice, dim, n; randomize=true) - suite["gen_samples"]["DigitalNetB2 d=$dim n=$n"] = + SUITE["gen_samples"]["DigitalNetB2 d=$dim n=$n"] = bench_gen_samples(DigitalNetB2, dim, n; randomize="LMS_DS") - suite["gen_samples"]["Halton d=$dim n=$n"] = + SUITE["gen_samples"]["Halton d=$dim n=$n"] = bench_gen_samples(Halton, dim, n; randomize=true) - suite["gen_samples"]["Kronecker d=$dim n=$n"] = + SUITE["gen_samples"]["Kronecker d=$dim n=$n"] = bench_gen_samples(Kronecker, dim, n) end -# 2. Transform -suite["transform"] = BenchmarkGroup() +# 2. Transform (pure Julia: inverse-CDF + covariance factor) +SUITE["transform"] = BenchmarkGroup() for dim in DIMS, n in SAMPLES dd = IIDStdUniform(dim; seed=42) tm_gauss = Gaussian(dd) x = gen_samples(dd, n) - suite["transform"]["Gaussian d=$dim n=$n"] = + SUITE["transform"]["Gaussian d=$dim n=$n"] = @benchmarkable transform($tm_gauss, $x) evals=3 samples=5 end -# 3. Integrand evaluation -suite["evaluate"] = BenchmarkGroup() +# 3. Integrand evaluation (pure Julia) +SUITE["evaluate"] = BenchmarkGroup() for n in SAMPLES dd = IIDStdUniform(3; seed=42) tm = Gaussian(dd) x = transform(tm, gen_samples(dd, n)) f_keister = Keister(tm) - suite["evaluate"]["Keister n=$n"] = + SUITE["evaluate"]["Keister n=$n"] = @benchmarkable evaluate($f_keister, $x) evals=3 samples=5 - f_genz = Genz(tm; kind=:continuous) - suite["evaluate"]["Genz(continuous) n=$n"] = + # NOTE: QMCPy's Genz supports only OSCILLATORY / CORNER PEAK. For a matched + # cross-language comparison use :oscillatory here (see benchmark_qmcpy.py). + f_genz = Genz(tm; kind=:oscillatory) + SUITE["evaluate"]["Genz(oscillatory) n=$n"] = @benchmarkable evaluate($f_genz, $x) evals=3 samples=5 end -# 4. End-to-end integration -suite["integrate"] = BenchmarkGroup() +# 4. End-to-end integration (mixed: C-backed generators + pure-Julia compute) +SUITE["integrate"] = BenchmarkGroup() -suite["integrate"]["CubMCCLT Keister"] = bench_integrate() do +SUITE["integrate"]["CubMCCLT Keister"] = bench_integrate() do dd = IIDStdUniform(3; seed=42) tm = Gaussian(dd) f = Keister(tm) CubMCCLT(f; abs_tol=0.01) end -suite["integrate"]["CubQMCLatticeG Keister"] = bench_integrate() do +SUITE["integrate"]["CubQMCLatticeG Keister"] = bench_integrate() do dd = Lattice(3; seed=42, randomize=true) tm = Gaussian(dd) f = Keister(tm) CubQMCLatticeG(f; abs_tol=0.01) end -suite["integrate"]["CubQMCNetG Keister"] = bench_integrate() do +SUITE["integrate"]["CubQMCNetG Keister"] = bench_integrate() do dd = DigitalNetB2(3; seed=42, randomize="LMS_DS") tm = Gaussian(dd) f = Keister(tm) CubQMCNetG(f; abs_tol=0.01) end -suite["integrate"]["CubMCCLT AsianOption"] = bench_integrate() do +SUITE["integrate"]["CubMCCLT AsianOption"] = bench_integrate() do dd = IIDStdUniform(50; seed=42) tm = GeometricBrownianMotion(dd; volatility=0.2, start_price=100.0, interest_rate=0.05, t_final=1.0) @@ -126,50 +118,10 @@ suite["integrate"]["CubMCCLT AsianOption"] = bench_integrate() do CubMCCLT(f; abs_tol=0.5) end -suite["integrate"]["CubQMCNetG EuropeanOption"] = bench_integrate() do +SUITE["integrate"]["CubQMCNetG EuropeanOption"] = bench_integrate() do dd = DigitalNetB2(50; seed=42, randomize="LMS_DS") tm = GeometricBrownianMotion(dd; volatility=0.2, start_price=100.0, interest_rate=0.05, t_final=1.0) f = FinancialOption(tm; option_type=:european, strike_price=100.0) CubQMCNetG(f; abs_tol=0.5) end - -# ── Run ────────────────────────────────────────────────────────────────── - -println("QMC.jl Benchmarks") -println("=" ^ 70) - -# CubMCCLT is a faithful single-pass two-stage estimator: its realized error can -# land just above the requested tolerance whenever the main-stage variance -# exceeds the pilot estimate, which correctly emits a non-convergence warning. -# That is expected and irrelevant here — we are timing, not checking accuracy — -# so warnings are silenced for the duration of the run to keep output readable. -results = with_logger(ConsoleLogger(stderr, Logging.Error)) do - run(suite; verbose=true) -end - -# ── Summary ────────────────────────────────────────────────────────────── - -println("\n", "=" ^ 70) -println("Summary") -println("=" ^ 70) - -for group_name in sort(collect(keys(results))) - println("\n── $group_name ──") - group = results[group_name] - for bench_name in sort(collect(keys(group))) - t = group[bench_name] - med = median(t) - @printf(" %-45s %10.3f ms (%d allocs, %.1f KiB)\n", - bench_name, - med.time / 1e6, - med.allocs, - med.memory / 1024) - end -end - -# Save results -using BenchmarkTools: save -mkpath(joinpath(@__DIR__, "results")) -save(joinpath(@__DIR__, "results", "latest.json"), results) -println("\nResults saved to benchmark/results/latest.json") diff --git a/benchmark/compare.jl b/benchmark/compare.jl new file mode 100644 index 0000000..f780a33 --- /dev/null +++ b/benchmark/compare.jl @@ -0,0 +1,96 @@ +# Compare benchmark results across git revisions using PkgBenchmark.jl. +# +# Usage (from the repository root): +# julia benchmark/compare.jl # benchmark the current working tree only +# julia benchmark/compare.jl main # judge working tree (incl. uncommitted +# # changes) against `main` +# julia benchmark/compare.jl HEAD main # judge one committed revision against another +# +# Dirty-tree friendly: a baseline/target *revision* is benchmarked inside a +# temporary `git worktree`, so your uncommitted changes are never stashed or +# checked out. The literal "current working tree" is benchmarked in place and may +# be dirty. (Both revisions must still post-date the `SUITE` refactor, since +# PkgBenchmark runs each revision's own benchmarks.jl.) + +using Pkg +Pkg.activate(@__DIR__) +let deps = keys(Pkg.project().dependencies) + "QMC" in deps || Pkg.develop(; path = dirname(@__DIR__)) + "PkgBenchmark" in deps || Pkg.add("PkgBenchmark") +end +Pkg.instantiate() + +using PkgBenchmark + +const PKG = dirname(@__DIR__) +const RESDIR = joinpath(@__DIR__, "results") +mkpath(RESDIR) + +"Benchmark the current working tree in place (uncommitted changes included)." +bench_worktree() = benchmarkpkg(PKG) + +"True if `rev` contains a benchmark/benchmarks.jl that defines `SUITE`." +function revision_has_suite(rev::AbstractString) + out = try + read(`git -C $PKG show $(rev):benchmark/benchmarks.jl`, String) + catch + return false # file absent at that revision + end + return occursin("SUITE", out) +end + +"Benchmark a committed `rev` in a throwaway git worktree, leaving PKG untouched." +function bench_revision(rev::AbstractString) + if !success(`git -C $PKG rev-parse --verify --quiet $rev`) + error("git revision \"$rev\" not found in this repository. Pass a valid " * + "branch/tag/commit, e.g. `make bench-compare REV=HEAD` (compare " * + "uncommitted changes against the last commit) or `REV=master`.") + end + if !revision_has_suite(rev) + error(""" + Revision "$rev" does not contain the PkgBenchmark `SUITE` harness + (a benchmark/benchmarks.jl that defines `const SUITE`). PkgBenchmark + runs each revision's own harness, so both sides must already have it — + and the benchmark tooling is currently UNCOMMITTED, so it is absent at + "$rev". Either: + * commit the benchmark tooling first, then compare later revisions; or + * for a git-free A/B on the current machine, use the manual flow: + julia benchmark/runbenchmarks.jl a # state A -> results/a.json + julia benchmark/runbenchmarks.jl b # state B -> results/b.json + then in Julia: + using BenchmarkTools + judge(median(BenchmarkTools.load("benchmark/results/b.json")[1]), + median(BenchmarkTools.load("benchmark/results/a.json")[1])) + """) + end + parent = mktempdir() + wt = joinpath(parent, "wt") # must not pre-exist; `git worktree add` creates it + run(`git -C $PKG worktree add --quiet --detach $wt $rev`) + try + return benchmarkpkg(wt) + finally + try + run(`git -C $PKG worktree remove --force $wt`) + catch + end + rm(parent; force = true, recursive = true) + end +end + +if length(ARGS) == 0 + result = bench_worktree() + export_markdown(joinpath(RESDIR, "current.md"), result) + println("\nWrote benchmark/results/current.md") +elseif length(ARGS) == 1 + baseline_rev = ARGS[1] + target = bench_worktree() # current tree (dirty OK) + baseline = bench_revision(baseline_rev) + export_markdown(joinpath(RESDIR, "judgement.md"), judge(target, baseline)) + println("\nWrote benchmark/results/judgement.md (working tree vs $(baseline_rev))") +else + target_rev, baseline_rev = ARGS[1], ARGS[2] + target = bench_revision(target_rev) + baseline = bench_revision(baseline_rev) + export_markdown(joinpath(RESDIR, "judgement.md"), judge(target, baseline)) + println("\nWrote benchmark/results/judgement.md ($(target_rev) vs $(baseline_rev))") +end diff --git a/benchmark/results/latest.json b/benchmark/results/latest.json index 695c0f0..f5d8dff 100644 --- a/benchmark/results/latest.json +++ b/benchmark/results/latest.json @@ -1 +1 @@ -[{"BenchmarkTools":"1.8.0","Julia":"1.12.6"},[["BenchmarkGroup",{"data":{"evaluate":["BenchmarkGroup",{"data":{"Genz(continuous) n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":8256,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[4694.666666666667,4097.0,4125.0,4055.3333333333335,4125.0]}],"Genz(continuous) n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":131136,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[58458.333333333336,57305.666666666664,57236.0,57333.333333333336,62277.666666666664]}],"Genz(continuous) n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":2112,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1916.6666666666667,1014.0,1097.0,944.3333333333334,1069.3333333333333]}],"Genz(continuous) n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":32832,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[16722.333333333332,16416.666666666668,16028.0,16222.333333333334,16125.0]}],"Keister n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":8256,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[7736.0,5319.333333333333,5125.0,5055.333333333333,5264.0]}],"Keister n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":131136,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[110014.0,96291.66666666667,95000.0,95750.0,95055.33333333333]}],"Keister n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":2112,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2847.0,1250.0,1319.3333333333333,1194.3333333333333,1277.6666666666667]}],"Keister n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":32832,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[29069.333333333332,22597.333333333332,21819.333333333332,22444.333333333332,24139.0]}]},"tags":[]}],"gen_samples":["BenchmarkGroup",{"data":{"DigitalNetB2 d=10 n=1024":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":335392,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[44444.666666666664,44125.0,37527.666666666664,36444.333333333336,41750.0]}],"DigitalNetB2 d=10 n=16384":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":5250592,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[823958.3333333334,760291.6666666666,759847.0,740819.3333333334,747222.3333333334]}],"DigitalNetB2 d=10 n=256":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":89632,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[17097.333333333332,14291.666666666666,14264.0,18000.0,18680.666666666668]}],"DigitalNetB2 d=10 n=4096":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1318432,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[149430.66666666666,143250.0,138944.66666666666,136763.66666666666,137000.0]}],"DigitalNetB2 d=3 n=1024":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":101040,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[14514.0,12069.666666666666,13180.666666666666,14028.0,22916.666666666668]}],"DigitalNetB2 d=3 n=16384":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1575600,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[152083.33333333334,161944.33333333334,165847.33333333334,162847.33333333334,190639.0]}],"DigitalNetB2 d=3 n=256":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":27312,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[7055.666666666667,4958.333333333333,4958.333333333333,5055.666666666667,4652.666666666667]}],"DigitalNetB2 d=3 n=4096":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":395952,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[44639.0,44472.333333333336,41555.333333333336,48083.333333333336,44069.333333333336]}],"Halton d=10 n=1024":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":84688,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[378319.3333333333,383166.6666666667,384236.3333333333,389000.0,384708.3333333333]}],"Halton d=10 n=16384":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1313488,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6.359083333333333e6,6.377583333333333e6,6.671653e6,6.374014e6,6.384625e6]}],"Halton d=10 n=256":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":23248,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[94194.66666666667,93041.66666666667,92986.33333333333,93208.33333333333,92930.66666666667]}],"Halton d=10 n=4096":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":330448,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1.6355833333333333e6,1.759764e6,1.560486e6,1.572153e6,1.7474446666666667e6]}],"Halton d=3 n=1024":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":25552,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[128111.0,129375.0,124972.33333333333,125333.33333333333,126847.33333333333]}],"Halton d=3 n=16384":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":394192,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2.1427223333333335e6,2.1118333333333335e6,2.1305833333333335e6,2.0853333333333333e6,2.0952916666666667e6]}],"Halton d=3 n=256":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":7120,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[29778.0,31875.0,31861.333333333332,32319.666666666668,28847.333333333332]}],"Halton d=3 n=4096":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":99280,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[515444.3333333333,524666.6666666666,530805.3333333334,515625.0,502041.6666666667]}],"IIDStdUniform d=10 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":82000,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[7666.666666666667,5333.333333333333,5305.333333333333,5319.333333333333,4389.0]}],"IIDStdUniform d=10 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1310800,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[122805.66666666667,103930.33333333333,108555.66666666667,92375.0,108152.66666666667]}],"IIDStdUniform d=10 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":20560,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2583.3333333333335,1930.6666666666667,1153.0,1389.0,1208.3333333333333]}],"IIDStdUniform d=10 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":327760,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[26736.333333333332,17166.666666666668,25458.333333333332,17139.0,21278.0]}],"IIDStdUniform d=3 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":24656,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2514.0,1639.0,2152.6666666666665,1444.3333333333333,1430.3333333333333]}],"IIDStdUniform d=3 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":393296,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[39583.333333333336,25708.333333333332,31833.333333333332,22236.0,30680.666666666668]}],"IIDStdUniform d=3 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":6224,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1791.6666666666667,611.0,555.6666666666666,486.3333333333333,472.3333333333333]}],"IIDStdUniform d=3 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":98384,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[11166.666666666666,7250.0,6014.0,5750.0,6958.333333333333]}],"Kronecker d=10 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":82000,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[22916.666666666668,18750.0,19888.666666666668,20819.333333333332,20805.666666666668]}],"Kronecker d=10 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1310800,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[469194.3333333333,520055.6666666667,528111.0,545527.6666666666,508819.3333333333]}],"Kronecker d=10 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":20560,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6027.666666666667,5319.333333333333,5180.666666666667,5250.0,5166.666666666667]}],"Kronecker d=10 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":327760,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[107833.33333333333,101055.33333333333,116166.66666666667,117777.66666666667,112944.33333333333]}],"Kronecker d=3 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":24656,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6500.0,5750.0,5888.666666666667,6889.0,5708.333333333333]}],"Kronecker d=3 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":393296,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[137652.66666666666,137041.66666666666,138861.0,138208.33333333334,147055.33333333334]}],"Kronecker d=3 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":6224,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2208.3333333333335,1500.0,1472.0,1500.0,1653.0]}],"Kronecker d=3 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":98384,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[31764.0,27666.666666666668,31458.333333333332,27236.0,28902.666666666668]}],"Lattice d=10 n=1024":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":246656,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[25236.0,24458.333333333332,16930.333333333332,22347.333333333332,23805.666666666668]}],"Lattice d=10 n=16384":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":3933056,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[333444.3333333333,322819.3333333333,323583.3333333333,323930.3333333333,332819.3333333333]}],"Lattice d=10 n=256":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":62336,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6208.333333333333,4333.333333333333,4194.333333333333,4250.0,4264.0]}],"Lattice d=10 n=4096":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":983936,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[76305.66666666667,74125.0,74166.66666666667,73889.0,73777.66666666667]}],"Lattice d=3 n=1024":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":74160,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[8805.666666666666,6972.0,8097.0,9902.666666666666,11277.666666666666]}],"Lattice d=3 n=16384":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1180080,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[120764.0,118903.0,121472.33333333333,124083.33333333333,125722.0]}],"Lattice d=3 n=256":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":18864,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[4250.0,2138.6666666666665,2319.6666666666665,2208.3333333333335,2222.0]}],"Lattice d=3 n=4096":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":295344,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[29291.666666666668,26861.0,26930.666666666668,26805.666666666668,29375.0]}]},"tags":[]}],"integrate":["BenchmarkGroup",{"data":{"CubMCCLT AsianOption":["Trial",{"allocs":3547,"gctimes":[0.0,0.0,0.0],"memory":5834128,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[9.4535e6,1.2386208e7,9.030042e6]}],"CubMCCLT Keister":["Trial",{"allocs":49,"gctimes":[0.0,0.0,477292.0],"memory":67864112,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[3.45125e7,3.0809792e7,3.6293542e7]}],"CubQMCLatticeG Keister":["Trial",{"allocs":385,"gctimes":[0.0,0.0,0.0],"memory":2108592,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[649583.0,632791.0,627709.0]}],"CubQMCNetG EuropeanOption":["Trial",{"allocs":18545,"gctimes":[0.0,0.0,0.0],"memory":47429552,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[1.4737583e7,1.6672042e7,1.4372791e7]}],"CubQMCNetG Keister":["Trial",{"allocs":577,"gctimes":[0.0,0.0,0.0],"memory":2538672,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[773625.0,854334.0,866541.0]}]},"tags":[]}],"transform":["BenchmarkGroup",{"data":{"Gaussian d=10 n=1024":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":164000,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[91014.0,84611.0,84611.0,84416.66666666667,84111.33333333333]}],"Gaussian d=10 n=16384":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":2621600,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1.4138193333333333e6,1.464264e6,1.4783053333333333e6,1.4225693333333333e6,1.3385276666666667e6]}],"Gaussian d=10 n=256":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":41120,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[24736.0,21000.0,20527.666666666668,20458.333333333332,20500.0]}],"Gaussian d=10 n=4096":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":655520,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[352889.0,346666.6666666667,357111.0,350514.0,340930.6666666667]}],"Gaussian d=3 n=1024":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":49312,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[29277.666666666668,25653.0,24180.666666666668,23958.333333333332,23972.0]}],"Gaussian d=3 n=16384":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":786592,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[416291.6666666667,403666.6666666667,408027.6666666667,417777.6666666667,394152.6666666667]}],"Gaussian d=3 n=256":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":12448,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[9889.0,6208.333333333333,6111.333333333333,6375.0,6180.333333333333]}],"Gaussian d=3 n=4096":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":196768,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[105319.33333333333,98180.66666666667,97944.33333333333,97875.0,97847.33333333333]}]},"tags":[]}]},"tags":[]}]]] \ No newline at end of file +[{"BenchmarkTools":"1.8.0","Julia":"1.12.6"},[["BenchmarkGroup",{"data":{"evaluate":["BenchmarkGroup",{"data":{"Genz(oscillatory) n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":8256,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[8583.333333333334,4375.0,4291.666666666667,4486.333333333333,4875.0]}],"Genz(oscillatory) n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":131136,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[142583.33333333334,105416.66666666667,108111.0,99236.0,99763.66666666667]}],"Genz(oscillatory) n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":2112,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2930.6666666666665,1291.6666666666667,1153.0,1138.6666666666667,1208.3333333333333]}],"Genz(oscillatory) n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":32832,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[31277.666666666668,19666.666666666668,20014.0,19944.666666666668,19847.333333333332]}],"Keister n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":8256,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6902.666666666667,4583.333333333333,4639.0,4541.666666666667,4458.333333333333]}],"Keister n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":131136,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[101847.33333333333,93833.33333333333,88805.66666666667,93486.0,89930.33333333333]}],"Keister n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":2112,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2514.0,1222.0,1027.6666666666667,1014.0,1111.3333333333333]}],"Keister n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":32832,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[26166.666666666668,22708.333333333332,22944.666666666668,21333.333333333332,25680.666666666668]}]},"tags":[]}],"gen_samples":["BenchmarkGroup",{"data":{"DigitalNetB2 d=10 n=1024":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":335392,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[38625.0,43402.666666666664,35791.666666666664,38638.666666666664,35666.666666666664]}],"DigitalNetB2 d=10 n=16384":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":5250592,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[673569.3333333334,726680.6666666666,787652.6666666666,749291.6666666666,699097.3333333334]}],"DigitalNetB2 d=10 n=256":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":89632,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[16805.666666666668,13861.0,13791.666666666666,15653.0,18430.666666666668]}],"DigitalNetB2 d=10 n=4096":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1318432,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[145597.33333333334,134139.0,134805.33333333334,141125.0,134083.33333333334]}],"DigitalNetB2 d=3 n=1024":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":101040,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[14083.333333333334,11416.666666666666,11416.666666666666,12208.333333333334,16625.0]}],"DigitalNetB2 d=3 n=16384":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1575600,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[156055.33333333334,173569.33333333334,176444.66666666666,196389.0,227139.0]}],"DigitalNetB2 d=3 n=256":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":27312,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6652.666666666667,4708.333333333333,4666.666666666667,5000.0,4514.0]}],"DigitalNetB2 d=3 n=4096":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":395952,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[47541.666666666664,43264.0,41305.333333333336,41180.333333333336,39041.666666666664]}],"Halton d=10 n=1024":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":84688,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[391555.3333333333,387930.6666666667,386819.3333333333,377694.3333333333,377847.0]}],"Halton d=10 n=16384":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1313488,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6.172083333333333e6,6.206333333333333e6,6.189694333333333e6,6.183277666666667e6,6.192597e6]}],"Halton d=10 n=256":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":23248,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[93402.66666666667,91819.66666666667,91694.33333333333,91722.33333333333,91569.33333333333]}],"Halton d=10 n=4096":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":330448,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1.5346803333333333e6,1.5297776666666667e6,1.5369166666666667e6,1.5356113333333333e6,1.5269583333333333e6]}],"Halton d=3 n=1024":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":25552,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[121694.66666666667,123597.33333333333,114069.66666666667,114597.0,127236.0]}],"Halton d=3 n=16384":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":394192,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1.9365276666666667e6,1.9398056666666667e6,1.9535696666666667e6,1.959653e6,1.9545e6]}],"Halton d=3 n=256":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":7120,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[29583.333333333332,28250.0,28250.0,28652.666666666668,28222.0]}],"Halton d=3 n=4096":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":99280,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[484805.6666666667,485305.6666666667,489625.0,485305.6666666667,484430.3333333333]}],"IIDStdUniform d=10 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":82000,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6416.666666666667,5486.0,4514.0,4472.0,4541.666666666667]}],"IIDStdUniform d=10 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1310800,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[103111.0,110680.66666666667,101972.33333333333,84916.66666666667,107680.66666666667]}],"IIDStdUniform d=10 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":20560,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2375.0,1514.0,1333.3333333333333,1597.0,1139.0]}],"IIDStdUniform d=10 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":327760,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[22569.666666666668,25597.333333333332,21236.0,21514.0,26250.0]}],"IIDStdUniform d=3 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":24656,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2750.0,2263.6666666666665,1458.3333333333333,1847.3333333333333,1416.6666666666667]}],"IIDStdUniform d=3 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":393296,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[26875.0,30625.0,30597.0,37888.666666666664,25500.0]}],"IIDStdUniform d=3 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":6224,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1736.3333333333333,513.6666666666666,402.6666666666667,430.6666666666667,458.3333333333333]}],"IIDStdUniform d=3 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":98384,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[8000.0,7583.333333333333,7694.333333333333,5125.0,6347.333333333333]}],"Kronecker d=10 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":82000,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[19666.666666666668,18819.333333333332,18791.666666666668,19958.333333333332,20986.0]}],"Kronecker d=10 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1310800,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[458166.6666666667,474361.0,466972.3333333333,472805.3333333333,479208.3333333333]}],"Kronecker d=10 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":20560,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[5375.0,4888.666666666667,4819.333333333333,4666.666666666667,4722.0]}],"Kronecker d=10 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":327760,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[106486.0,121639.0,120708.33333333333,129500.0,115583.33333333333]}],"Kronecker d=3 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":24656,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6694.333333333333,5694.666666666667,5625.0,5847.0,5694.333333333333]}],"Kronecker d=3 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":393296,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[125694.33333333333,137430.66666666666,136444.33333333334,136930.66666666666,136972.33333333334]}],"Kronecker d=3 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":6224,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2250.0,1486.0,1430.6666666666667,1444.6666666666667,1430.6666666666667]}],"Kronecker d=3 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":98384,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[28013.666666666668,26763.666666666668,26666.666666666668,26666.666666666668,26639.0]}],"Lattice d=10 n=1024":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":246656,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[18111.0,16153.0,17069.666666666668,16166.666666666666,25361.0]}],"Lattice d=10 n=16384":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":3933056,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[348333.3333333333,422569.3333333333,491389.0,470389.0,471333.3333333333]}],"Lattice d=10 n=256":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":62336,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6361.0,4263.666666666667,4236.333333333333,4139.0,4180.333333333333]}],"Lattice d=10 n=4096":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":983936,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[75778.0,73458.33333333333,132889.0,114694.33333333333,112861.0]}],"Lattice d=3 n=1024":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":74160,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[8583.333333333334,6916.666666666667,6889.0,6847.333333333333,8791.666666666666]}],"Lattice d=3 n=16384":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1180080,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[115236.0,130097.33333333333,116139.0,116902.66666666667,113402.66666666667]}],"Lattice d=3 n=256":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":18864,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[3708.3333333333335,1986.3333333333333,2097.3333333333335,2014.0,1902.6666666666667]}],"Lattice d=3 n=4096":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":295344,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[29430.333333333332,27430.666666666668,27389.0,28833.333333333332,27458.333333333332]}]},"tags":[]}],"integrate":["BenchmarkGroup",{"data":{"CubMCCLT AsianOption":["Trial",{"allocs":3545,"gctimes":[0.0,0.0,0.0],"memory":5834064,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[2.505375e6,2.304875e6,2.896666e6]}],"CubMCCLT Keister":["Trial",{"allocs":47,"gctimes":[0.0,146583.0,149417.0],"memory":67864048,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[3.0929292e7,3.107825e7,3.1256417e7]}],"CubQMCLatticeG Keister":["Trial",{"allocs":385,"gctimes":[0.0,0.0,0.0],"memory":2108592,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[659541.0,617500.0,636125.0]}],"CubQMCNetG EuropeanOption":["Trial",{"allocs":18545,"gctimes":[0.0,275500.0,0.0],"memory":47429552,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[1.4468e7,1.4684208e7,1.3403958e7]}],"CubQMCNetG Keister":["Trial",{"allocs":577,"gctimes":[0.0,0.0,0.0],"memory":2538672,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[741875.0,824958.0,840833.0]}]},"tags":[]}],"transform":["BenchmarkGroup",{"data":{"Gaussian d=10 n=1024":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":164000,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[89139.0,93847.0,92541.66666666667,84639.0,78805.66666666667]}],"Gaussian d=10 n=16384":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":2621600,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1.5699166666666667e6,1.5492916666666667e6,1.5227916666666667e6,1.5522083333333333e6,1.5836946666666667e6]}],"Gaussian d=10 n=256":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":41120,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[23389.0,19569.666666666668,19472.0,19347.0,19388.666666666668]}],"Gaussian d=10 n=4096":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":655520,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[372208.3333333333,354500.0,355222.0,356569.3333333333,357013.6666666667]}],"Gaussian d=3 n=1024":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":49312,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[28139.0,22875.0,24778.0,25472.333333333332,25472.333333333332]}],"Gaussian d=3 n=16384":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":786592,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[390833.3333333333,394652.6666666667,415347.0,419000.0,418166.6666666667]}],"Gaussian d=3 n=256":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":12448,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[9236.0,5944.333333333333,5958.333333333333,6152.666666666667,6013.666666666667]}],"Gaussian d=3 n=4096":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":196768,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[100764.0,104208.33333333333,100583.33333333333,99778.0,103305.66666666667]}]},"tags":[]}]},"tags":[]}]]] \ No newline at end of file diff --git a/benchmark/runbenchmarks.jl b/benchmark/runbenchmarks.jl new file mode 100644 index 0000000..7c848d6 --- /dev/null +++ b/benchmark/runbenchmarks.jl @@ -0,0 +1,56 @@ +# Standalone benchmark runner. +# +# Usage (from the repository root): +# julia benchmark/runbenchmarks.jl # run and save to results/latest.json +# julia benchmark/runbenchmarks.jl mylabel # save to results/mylabel.json instead +# +# Bootstraps the benchmark environment on first run (activate benchmark/, develop +# the parent QMC package, install deps), then runs SUITE (defined in benchmarks.jl) +# and writes a BenchmarkTools result file you can `judge` against another run: +# +# using BenchmarkTools +# base = BenchmarkTools.load("benchmark/results/baseline.json")[1] +# new = BenchmarkTools.load("benchmark/results/latest.json")[1] +# judge(median(new), median(base)) # time + memory ratios per benchmark + +using Pkg +Pkg.activate(@__DIR__) +let deps = keys(Pkg.project().dependencies) + "QMC" in deps || Pkg.develop(; path = dirname(@__DIR__)) + "BenchmarkTools" in deps || Pkg.add("BenchmarkTools") +end +Pkg.instantiate() + +using BenchmarkTools +using Statistics +using Printf + +include(joinpath(@__DIR__, "benchmarks.jl")) # defines SUITE (and silences expected warnings) + +label = isempty(ARGS) ? "latest" : ARGS[1] + +println("QMC.jl Benchmarks") +println("="^70) + +results = run(SUITE; verbose = true) + +# ── Summary ────────────────────────────────────────────────────────────── +println("\n", "="^70) +println("Summary") +println("="^70) +for group_name in sort(collect(keys(results))) + println("\n── $group_name ──") + group = results[group_name] + for bench_name in sort(collect(keys(group))) + med = median(group[bench_name]) + @printf(" %-45s %10.3f ms (%d allocs, %.1f KiB)\n", + bench_name, med.time / 1e6, med.allocs, med.memory / 1024) + end +end + +# ── Save ─────────────────────────────────────────────────────────────────── +resdir = joinpath(@__DIR__, "results") +mkpath(resdir) +outfile = joinpath(resdir, "$(label).json") +BenchmarkTools.save(outfile, results) +println("\nResults saved to benchmark/results/$(label).json") diff --git a/benchmark/tune.json b/benchmark/tune.json new file mode 100644 index 0000000..0a9023b --- /dev/null +++ b/benchmark/tune.json @@ -0,0 +1 @@ +[{"BenchmarkTools":"1.8.0","Julia":"1.12.6"},[["BenchmarkGroup",{"data":{"evaluate":["BenchmarkGroup",{"data":{"Genz(oscillatory) n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Genz(oscillatory) n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Genz(oscillatory) n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Genz(oscillatory) n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Keister n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Keister n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Keister n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Keister n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}]},"tags":[]}],"gen_samples":["BenchmarkGroup",{"data":{"DigitalNetB2 d=10 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=10 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=10 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=10 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=3 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=3 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=3 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=3 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=10 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=10 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=10 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=10 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=3 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=3 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=3 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=3 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=10 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=10 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=10 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=10 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=3 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=3 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=3 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=3 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=10 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=10 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=10 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=10 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=3 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=3 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=3 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=3 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=10 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=10 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=10 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=10 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=3 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=3 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=3 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=3 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}]},"tags":[]}],"integrate":["BenchmarkGroup",{"data":{"CubMCCLT AsianOption":["Parameters",{"evals":1,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"CubMCCLT Keister":["Parameters",{"evals":1,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"CubQMCLatticeG Keister":["Parameters",{"evals":1,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"CubQMCNetG EuropeanOption":["Parameters",{"evals":1,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"CubQMCNetG Keister":["Parameters",{"evals":1,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}]},"tags":[]}],"transform":["BenchmarkGroup",{"data":{"Gaussian d=10 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=10 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=10 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=10 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=3 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=3 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=3 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=3 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}]},"tags":[]}]},"tags":[]}]]] \ No newline at end of file From 7dccf01022f83ff4446ee6fb9e302bd8d17e8a76 Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 14:57:19 -0500 Subject: [PATCH 010/104] minor changes --- Makefile | 2 +- benchmark/compare.jl | 2 +- benchmark/results/judgement.md | 113 +++++++++++++++++++++++++++++++++ benchmark/runbenchmarks.jl | 2 +- 4 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 benchmark/results/judgement.md diff --git a/Makefile b/Makefile index e79686c..683f9b1 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ bench: julia benchmark/runbenchmarks.jl # Compare the working tree against a baseline git revision (default: HEAD, i.e. -# the effect of your uncommitted changes). Override with: make bench-compare REV=master +# the effect of uncommitted changes). Override with: make bench-compare REV=master REV ?= HEAD bench-compare: julia benchmark/compare.jl $(REV) diff --git a/benchmark/compare.jl b/benchmark/compare.jl index f780a33..747a633 100644 --- a/benchmark/compare.jl +++ b/benchmark/compare.jl @@ -7,7 +7,7 @@ # julia benchmark/compare.jl HEAD main # judge one committed revision against another # # Dirty-tree friendly: a baseline/target *revision* is benchmarked inside a -# temporary `git worktree`, so your uncommitted changes are never stashed or +# temporary `git worktree`, so uncommitted changes are never stashed or # checked out. The literal "current working tree" is benchmarked in place and may # be dirty. (Both revisions must still post-date the `SUITE` refactor, since # PkgBenchmark runs each revision's own benchmarks.jl.) diff --git a/benchmark/results/judgement.md b/benchmark/results/judgement.md new file mode 100644 index 0000000..ca75ce6 --- /dev/null +++ b/benchmark/results/judgement.md @@ -0,0 +1,113 @@ +# Benchmark Report for */Users/terrya/Documents/ProgramData/qmcju_software_choi* + +## Job Properties +* Time of benchmarks: + - Target: 2 Jun 2026 - 14:56 + - Baseline: 2 Jun 2026 - 14:57 +* Package commits: + - Target: dirty + - Baseline: f21de9e +* Julia commits: + - Target: 1534690 + - Baseline: 1534690 +* Julia command flags: + - Target: None + - Baseline: None +* Environment variables: + - Target: None + - Baseline: None + +## Results +A ratio greater than `1.0` denotes a possible regression (marked with :x:), while a ratio less +than `1.0` denotes a possible improvement (marked with :white_check_mark:). Brackets display [tolerances](https://juliaci.github.io/BenchmarkTools.jl/stable/manual/#Benchmark-Parameters) for the benchmark estimates. Only significant results - results +that indicate possible regressions or improvements - are shown below (thus, an empty table means that all +benchmark results remained invariant between builds). + +| ID | time ratio | memory ratio | +|-------------------------------------------------|------------------------------|--------------| +| `["evaluate", "Genz(oscillatory) n=16384"]` | 1.08 (5%) :x: | 1.00 (1%) | +| `["evaluate", "Keister n=256"]` | 1.24 (5%) :x: | 1.00 (1%) | +| `["gen_samples", "DigitalNetB2 d=10 n=16384"]` | 0.89 (5%) :white_check_mark: | 1.00 (1%) | +| `["gen_samples", "DigitalNetB2 d=3 n=1024"]` | 0.94 (5%) :white_check_mark: | 1.00 (1%) | +| `["gen_samples", "DigitalNetB2 d=3 n=4096"]` | 1.07 (5%) :x: | 1.00 (1%) | +| `["gen_samples", "Halton d=3 n=1024"]` | 0.90 (5%) :white_check_mark: | 1.00 (1%) | +| `["gen_samples", "IIDStdUniform d=10 n=1024"]` | 1.13 (5%) :x: | 1.00 (1%) | +| `["gen_samples", "IIDStdUniform d=10 n=16384"]` | 1.11 (5%) :x: | 1.00 (1%) | +| `["gen_samples", "IIDStdUniform d=10 n=4096"]` | 0.81 (5%) :white_check_mark: | 1.00 (1%) | +| `["gen_samples", "IIDStdUniform d=3 n=16384"]` | 1.73 (5%) :x: | 1.00 (1%) | +| `["gen_samples", "Kronecker d=3 n=1024"]` | 1.16 (5%) :x: | 1.00 (1%) | +| `["gen_samples", "Kronecker d=3 n=4096"]` | 1.19 (5%) :x: | 1.00 (1%) | +| `["gen_samples", "Lattice d=10 n=16384"]` | 1.17 (5%) :x: | 1.00 (1%) | +| `["gen_samples", "Lattice d=3 n=1024"]` | 0.80 (5%) :white_check_mark: | 1.00 (1%) | +| `["gen_samples", "Lattice d=3 n=256"]` | 0.86 (5%) :white_check_mark: | 1.00 (1%) | +| `["integrate", "CubMCCLT Keister"]` | 0.90 (5%) :white_check_mark: | 1.00 (1%) | +| `["integrate", "CubQMCNetG EuropeanOption"]` | 1.12 (5%) :x: | 1.00 (1%) | +| `["transform", "Gaussian d=10 n=1024"]` | 1.08 (5%) :x: | 1.00 (1%) | +| `["transform", "Gaussian d=10 n=16384"]` | 1.07 (5%) :x: | 1.00 (1%) | +| `["transform", "Gaussian d=10 n=4096"]` | 1.06 (5%) :x: | 1.00 (1%) | +| `["transform", "Gaussian d=3 n=1024"]` | 1.08 (5%) :x: | 1.00 (1%) | +| `["transform", "Gaussian d=3 n=16384"]` | 1.05 (5%) :x: | 1.00 (1%) | +| `["transform", "Gaussian d=3 n=256"]` | 1.10 (5%) :x: | 1.00 (1%) | +| `["transform", "Gaussian d=3 n=4096"]` | 1.08 (5%) :x: | 1.00 (1%) | + +## Benchmark Group List +Here's a list of all the benchmark groups executed by this job: + +- `["evaluate"]` +- `["gen_samples"]` +- `["integrate"]` +- `["transform"]` + +## Julia versioninfo + +### Target +``` +Julia Version 1.12.6 +Commit 15346901f0 (2026-04-09 19:20 UTC) +Build Info: + Built by Homebrew (v1.12.6) + + Note: This is an unofficial build, please report bugs to the project + responsible for this build and not to the Julia project unless you can + reproduce the issue using official builds available at https://julialang.org + +Platform Info: + OS: macOS (arm64-apple-darwin25.4.0) + uname: Darwin 25.5.0 Darwin Kernel Version 25.5.0: Mon Apr 27 20:41:15 PDT 2026; root:xnu-12377.121.6~2/RELEASE_ARM64_T6041 arm64 arm + CPU: Apple M4 Max: + speed user nice sys idle irq + #1-16 2400 MHz 709878 s 0 s 249549 s 14785237 s 0 s + Memory: 128.0 GB (4583.15625 MB free) + Uptime: 1.090962e6 sec + Load Avg: 4.328125 3.29736328125 3.0224609375 + WORD_SIZE: 64 + LLVM: libLLVM-18.1.7 (ORCJIT, apple-m4) + GC: Built with stock GC +Threads: 1 default, 1 interactive, 1 GC (on 12 virtual cores) +``` + +### Baseline +``` +Julia Version 1.12.6 +Commit 15346901f0 (2026-04-09 19:20 UTC) +Build Info: + Built by Homebrew (v1.12.6) + + Note: This is an unofficial build, please report bugs to the project + responsible for this build and not to the Julia project unless you can + reproduce the issue using official builds available at https://julialang.org + +Platform Info: + OS: macOS (arm64-apple-darwin25.4.0) + uname: Darwin 25.5.0 Darwin Kernel Version 25.5.0: Mon Apr 27 20:41:15 PDT 2026; root:xnu-12377.121.6~2/RELEASE_ARM64_T6041 arm64 arm + CPU: Apple M4 Max: + speed user nice sys idle irq + #1-16 2400 MHz 709913 s 0 s 249559 s 14785501 s 0 s + Memory: 128.0 GB (4493.359375 MB free) + Uptime: 1.090982e6 sec + Load Avg: 4.82861328125 3.49853515625 3.10107421875 + WORD_SIZE: 64 + LLVM: libLLVM-18.1.7 (ORCJIT, apple-m4) + GC: Built with stock GC +Threads: 1 default, 1 interactive, 1 GC (on 12 virtual cores) +``` \ No newline at end of file diff --git a/benchmark/runbenchmarks.jl b/benchmark/runbenchmarks.jl index 7c848d6..c88a47f 100644 --- a/benchmark/runbenchmarks.jl +++ b/benchmark/runbenchmarks.jl @@ -6,7 +6,7 @@ # # Bootstraps the benchmark environment on first run (activate benchmark/, develop # the parent QMC package, install deps), then runs SUITE (defined in benchmarks.jl) -# and writes a BenchmarkTools result file you can `judge` against another run: +# and writes a BenchmarkTools result file that can be `judged` against another run: # # using BenchmarkTools # base = BenchmarkTools.load("benchmark/results/baseline.json")[1] From f1541d66a62e9230e8200a0d0cf1a08ec930220e Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 15:25:31 -0500 Subject: [PATCH 011/104] +benchmarking scripts and results for QMC.jl vs QMCPy --- .gitignore | 5 + Makefile | 10 +- benchmark/Manifest.toml | 572 --------------------------------- benchmark/Project.toml | 1 + benchmark/compare_py.jl | 88 +++++ benchmark/results/judgement.md | 113 ------- benchmark/results/latest.json | 1 - benchmark/tune.json | 1 - 8 files changed, 103 insertions(+), 688 deletions(-) delete mode 100644 benchmark/Manifest.toml create mode 100644 benchmark/compare_py.jl delete mode 100644 benchmark/results/judgement.md delete mode 100644 benchmark/results/latest.json delete mode 100644 benchmark/tune.json diff --git a/.gitignore b/.gitignore index c51e11b..b6fcecc 100644 --- a/.gitignore +++ b/.gitignore @@ -30,5 +30,10 @@ Thumbs.db *.ji deps/build.log +# Benchmark outputs — machine-specific +benchmark/Manifest.toml +benchmark/tune.json +benchmark/results/ + # Developer-specific files sc_* diff --git a/Makefile b/Makefile index 683f9b1..af650db 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test doc format format-check lint clean bench bench-compare +.PHONY: test doc format format-check lint clean bench bench-compare bench-compare-py FORMATTER_PROJECT=devtools/formatter @@ -57,3 +57,11 @@ bench: REV ?= HEAD bench-compare: julia benchmark/compare.jl $(REV) + +# Side-by-side Julia vs QMCPy comparison (requires both result files to exist). +# Run `make bench` and `python benchmark/benchmark_qmcpy.py` first. +# Override label with: make bench-compare-py JL_LABEL=foo PY_LABEL=bar +JL_LABEL ?= latest +PY_LABEL ?= $(JL_LABEL) +bench-compare-py: bench bench-compare + julia benchmark/compare_py.jl $(JL_LABEL) $(PY_LABEL) diff --git a/benchmark/Manifest.toml b/benchmark/Manifest.toml deleted file mode 100644 index 0e442ea..0000000 --- a/benchmark/Manifest.toml +++ /dev/null @@ -1,572 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -julia_version = "1.12.6" -manifest_format = "2.0" -project_hash = "f886813682ad274be6b2fd29c59c0591e8733b5b" - -[[deps.AbstractFFTs]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" -uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" -version = "1.5.0" - - [deps.AbstractFFTs.extensions] - AbstractFFTsChainRulesCoreExt = "ChainRulesCore" - AbstractFFTsTestExt = "Test" - - [deps.AbstractFFTs.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[deps.AbstractTrees]] -git-tree-sha1 = "2d9c9a55f9c93e8887ad391fbae72f8ef55e1177" -uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" -version = "0.4.5" - -[[deps.AliasTables]] -deps = ["PtrArrays", "Random"] -git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" -uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" -version = "1.1.3" - -[[deps.ArgTools]] -uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.2" - -[[deps.Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" -version = "1.11.0" - -[[deps.Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -version = "1.11.0" - -[[deps.BenchmarkTools]] -deps = ["Compat", "JSON", "Logging", "PrecompileTools", "Printf", "Profile", "Statistics", "UUIDs"] -git-tree-sha1 = "9670d3febc2b6da60a0ae57846ba74670290653f" -uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" -version = "1.8.0" - -[[deps.Compat]] -deps = ["TOML", "UUIDs"] -git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" -uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.18.1" -weakdeps = ["Dates", "LinearAlgebra"] - - [deps.Compat.extensions] - CompatLinearAlgebraExt = "LinearAlgebra" - -[[deps.CompilerSupportLibraries_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.3.0+1" - -[[deps.DataAPI]] -git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" -uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.16.0" - -[[deps.DataStructures]] -deps = ["OrderedCollections"] -git-tree-sha1 = "6fb53a69613a0b2b68a0d12671717d307ab8b24e" -uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.19.5" - -[[deps.Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" -version = "1.11.0" - -[[deps.Distributions]] -deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] -git-tree-sha1 = "e421c1938fafab0165b04dc1a9dbe2a26272952c" -uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" -version = "0.25.125" - - [deps.Distributions.extensions] - DistributionsChainRulesCoreExt = "ChainRulesCore" - DistributionsDensityInterfaceExt = "DensityInterface" - DistributionsTestExt = "Test" - - [deps.Distributions.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" - Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[deps.DocStringExtensions]] -git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c" -uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.9.5" - -[[deps.Downloads]] -deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] -uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -version = "1.7.0" - -[[deps.FFTW]] -deps = ["AbstractFFTs", "FFTW_jll", "Libdl", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] -git-tree-sha1 = "97f08406df914023af55ade2f843c39e99c5d969" -uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -version = "1.10.0" - -[[deps.FFTW_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "6866aec60ef98e3164cd8d6855225684207e9dff" -uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" -version = "3.3.12+0" - -[[deps.FileWatching]] -uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" -version = "1.11.0" - -[[deps.FillArrays]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "2f979084d1e13948a3352cf64a25df6bd3b4dca3" -uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "1.16.0" - - [deps.FillArrays.extensions] - FillArraysPDMatsExt = "PDMats" - FillArraysSparseArraysExt = "SparseArrays" - FillArraysStaticArraysExt = "StaticArrays" - FillArraysStatisticsExt = "Statistics" - - [deps.FillArrays.weakdeps] - PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150" - SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" - Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" - -[[deps.HypergeometricFunctions]] -deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] -git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec" -uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" -version = "0.3.28" - -[[deps.IntelOpenMP_jll]] -deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] -git-tree-sha1 = "ec1debd61c300961f98064cfb21287613ad7f303" -uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" -version = "2025.2.0+0" - -[[deps.InteractiveUtils]] -deps = ["Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -version = "1.11.0" - -[[deps.IrrationalConstants]] -git-tree-sha1 = "b2d91fe939cae05960e760110b328288867b5758" -uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" -version = "0.2.6" - -[[deps.JLLWrappers]] -deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "7204148362dafe5fe6a273f855b8ccbe4df8173e" -uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.8.0" - -[[deps.JSON]] -deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs", "Unicode"] -git-tree-sha1 = "f76f7560267b840e492180f9899b472f30b88450" -uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "1.6.0" - - [deps.JSON.extensions] - JSONArrowExt = ["ArrowTypes"] - - [deps.JSON.weakdeps] - ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" - -[[deps.JuliaSyntaxHighlighting]] -deps = ["StyledStrings"] -uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" -version = "1.12.0" - -[[deps.LazyArtifacts]] -deps = ["Artifacts", "Pkg"] -uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" -version = "1.11.0" - -[[deps.LeftChildRightSiblingTrees]] -deps = ["AbstractTrees"] -git-tree-sha1 = "95ba48564903b43b2462318aa243ee79d81135ff" -uuid = "1d6d02ad-be62-4b6b-8a6d-2f90e265016e" -version = "0.2.1" - -[[deps.LibCURL]] -deps = ["LibCURL_jll", "MozillaCACerts_jll"] -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.4" - -[[deps.LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] -uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.15.0+0" - -[[deps.LibGit2]] -deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" -version = "1.11.0" - -[[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] -uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.9.0+0" - -[[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "OpenSSL_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.11.3+1" - -[[deps.Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -version = "1.11.0" - -[[deps.LinearAlgebra]] -deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -version = "1.12.0" - -[[deps.LogExpFunctions]] -deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] -git-tree-sha1 = "13ca9e2586b89836fd20cccf56e57e2b9ae7f38f" -uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" -version = "0.3.29" - - [deps.LogExpFunctions.extensions] - LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" - LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" - LogExpFunctionsInverseFunctionsExt = "InverseFunctions" - - [deps.LogExpFunctions.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" - InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" - -[[deps.Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" -version = "1.11.0" - -[[deps.MKL_jll]] -deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"] -git-tree-sha1 = "282cadc186e7b2ae0eeadbd7a4dffed4196ae2aa" -uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" -version = "2025.2.0+0" - -[[deps.Markdown]] -deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" -version = "1.11.0" - -[[deps.Missings]] -deps = ["DataAPI"] -git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" -uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" -version = "1.2.0" - -[[deps.MozillaCACerts_jll]] -uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2025.11.4" - -[[deps.NBInclude]] -deps = ["JSON", "Markdown", "SoftGlobalScope"] -git-tree-sha1 = "89248151cda1af190b8d634cf58fc6b21a333a34" -uuid = "0db19996-df87-5ea3-a455-e3a50d440464" -version = "2.4.0" - -[[deps.NetworkOptions]] -uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" -version = "1.3.0" - -[[deps.OpenBLAS_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] -uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.29+0" - -[[deps.OpenLibm_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "05823500-19ac-5b8b-9628-191a04bc5112" -version = "0.8.7+0" - -[[deps.OpenSSL_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.4+0" - -[[deps.OpenSpecFun_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] -git-tree-sha1 = "1346c9208249809840c91b26703912dff463d335" -uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" -version = "0.5.6+0" - -[[deps.OrderedCollections]] -git-tree-sha1 = "94ba93778373a53bfd5a0caaf7d809c445292ff4" -uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.8.2" - -[[deps.PDMats]] -deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "e4cff168707d441cd6bf3ff7e4832bdf34278e4a" -uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" -version = "0.11.37" -weakdeps = ["StatsBase"] - - [deps.PDMats.extensions] - StatsBaseExt = "StatsBase" - -[[deps.Parsers]] -deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "5d5e0a78e971354b1c7bff0655d11fdc1b0e12c8" -uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.8.4" - -[[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.12.1" -weakdeps = ["REPL"] - - [deps.Pkg.extensions] - REPLExt = "REPL" - -[[deps.PkgBenchmark]] -deps = ["BenchmarkTools", "Dates", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Pkg", "Printf", "TerminalLoggers", "UUIDs"] -git-tree-sha1 = "10940959d1174f5402729bb0891cafb785c0e927" -uuid = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" -version = "0.2.15" - -[[deps.PrecompileTools]] -deps = ["Preferences"] -git-tree-sha1 = "edbeefc7a4889f528644251bdb5fc9ab5348bc2c" -uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.3.4" - -[[deps.Preferences]] -deps = ["TOML"] -git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4" -uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.5.2" - -[[deps.Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" -version = "1.11.0" - -[[deps.Profile]] -deps = ["StyledStrings"] -uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" -version = "1.11.0" - -[[deps.ProgressLogging]] -deps = ["Logging", "SHA", "UUIDs"] -git-tree-sha1 = "f0803bc1171e455a04124affa9c21bba5ac4db32" -uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" -version = "0.1.6" - -[[deps.PtrArrays]] -git-tree-sha1 = "4fbbafbc6251b883f4d2705356f3641f3652a7fe" -uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" -version = "1.4.0" - -[[deps.QMC]] -deps = ["Distributions", "FFTW", "Libdl", "LinearAlgebra", "NBInclude", "Printf", "Random", "SpecialFunctions", "Statistics"] -path = "/Users/terrya/Documents/ProgramData/qmcju_software_choi" -uuid = "b8e7c4a1-3f5d-4e9a-b2c6-1a8d9f0e7c3b" -version = "0.1.0" - -[[deps.QuadGK]] -deps = ["DataStructures", "LinearAlgebra"] -git-tree-sha1 = "5e8e8b0ab68215d7a2b14b9921a946fee794749e" -uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" -version = "2.11.3" - - [deps.QuadGK.extensions] - QuadGKEnzymeExt = "Enzyme" - - [deps.QuadGK.weakdeps] - Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" - -[[deps.REPL]] -deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" -version = "1.11.0" - -[[deps.Random]] -deps = ["SHA"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -version = "1.11.0" - -[[deps.Reexport]] -git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" -uuid = "189a3867-3050-52da-a836-e630ba90ab69" -version = "1.2.2" - -[[deps.Rmath]] -deps = ["Random", "Rmath_jll"] -git-tree-sha1 = "5b3d50eb374cea306873b371d3f8d3915a018f0b" -uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" -version = "0.9.0" - -[[deps.Rmath_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "58cdd8fb2201a6267e1db87ff148dd6c1dbd8ad8" -uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" -version = "0.5.1+0" - -[[deps.SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" - -[[deps.Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -version = "1.11.0" - -[[deps.Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" -version = "1.11.0" - -[[deps.SoftGlobalScope]] -deps = ["REPL"] -git-tree-sha1 = "986ec2b6162ccb95de5892ed17832f95badf770c" -uuid = "b85f4697-e234-5449-a836-ec8e2f98b302" -version = "1.1.0" - -[[deps.SortingAlgorithms]] -deps = ["DataStructures"] -git-tree-sha1 = "64d974c2e6fdf07f8155b5b2ca2ffa9069b608d9" -uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.2.2" - -[[deps.SparseArrays]] -deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] -uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -version = "1.12.0" - -[[deps.SpecialFunctions]] -deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] -git-tree-sha1 = "6547cbdd8ce32efba0d21c5a40fa96d1a3548f9f" -uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "2.8.0" - - [deps.SpecialFunctions.extensions] - SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" - - [deps.SpecialFunctions.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - -[[deps.Statistics]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" -uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.11.1" -weakdeps = ["SparseArrays"] - - [deps.Statistics.extensions] - SparseArraysExt = ["SparseArrays"] - -[[deps.StatsAPI]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "178ed29fd5b2a2cfc3bd31c13375ae925623ff36" -uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" -version = "1.8.0" - -[[deps.StatsBase]] -deps = ["AliasTables", "DataAPI", "DataStructures", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] -git-tree-sha1 = "c6f18e5a52a176a383f6f6c635e0f81feed1d6d4" -uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" -version = "0.34.11" - -[[deps.StatsFuns]] -deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] -git-tree-sha1 = "91f091a8716a6bb38417a6e6f274602a19aaa685" -uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" -version = "1.5.2" - - [deps.StatsFuns.extensions] - StatsFunsChainRulesCoreExt = "ChainRulesCore" - StatsFunsInverseFunctionsExt = "InverseFunctions" - - [deps.StatsFuns.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" - -[[deps.StructUtils]] -deps = ["Dates", "UUIDs"] -git-tree-sha1 = "82bee338d650aa515f31866c460cb7e3bcef90b8" -uuid = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" -version = "2.8.2" - - [deps.StructUtils.extensions] - StructUtilsMeasurementsExt = ["Measurements"] - StructUtilsStaticArraysCoreExt = ["StaticArraysCore"] - StructUtilsTablesExt = ["Tables"] - - [deps.StructUtils.weakdeps] - Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" - StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" - Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" - -[[deps.StyledStrings]] -uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" -version = "1.11.0" - -[[deps.SuiteSparse]] -deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] -uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" - -[[deps.SuiteSparse_jll]] -deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] -uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "7.8.3+2" - -[[deps.TOML]] -deps = ["Dates"] -uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" -version = "1.0.3" - -[[deps.Tar]] -deps = ["ArgTools", "SHA"] -uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" -version = "1.10.0" - -[[deps.TerminalLoggers]] -deps = ["LeftChildRightSiblingTrees", "Logging", "Markdown", "Printf", "ProgressLogging", "UUIDs"] -git-tree-sha1 = "f133fab380933d042f6796eda4e130272ba520ca" -uuid = "5d786b92-1e48-4d6f-9151-6b4477ca9bed" -version = "0.1.7" - -[[deps.UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" -version = "1.11.0" - -[[deps.Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -version = "1.11.0" - -[[deps.Zlib_jll]] -deps = ["Libdl"] -uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.3.1+2" - -[[deps.libblastrampoline_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.15.0+0" - -[[deps.nghttp2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.64.0+1" - -[[deps.oneTBB_jll]] -deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] -git-tree-sha1 = "da8c1f6eee04831f14edcfa5dae611d309807e57" -uuid = "1317d2d5-d96f-522e-a858-c73665f53c3e" -version = "2022.3.0+0" - -[[deps.p7zip_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] -uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.7.0+0" diff --git a/benchmark/Project.toml b/benchmark/Project.toml index 22db034..82cb59a 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -1,5 +1,6 @@ [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" diff --git a/benchmark/compare_py.jl b/benchmark/compare_py.jl new file mode 100644 index 0000000..388dbf8 --- /dev/null +++ b/benchmark/compare_py.jl @@ -0,0 +1,88 @@ +# Side-by-side Julia vs QMCPy benchmark comparison. +# +# Usage (from the repository root): +# julia benchmark/compare_py.jl # latest.json vs qmcpy_latest.json +# julia benchmark/compare_py.jl mylabel # latest.json vs qmcpy_mylabel.json +# julia benchmark/compare_py.jl jl_label py_label # jl_label.json vs qmcpy_py_label.json +# +# Both files must exist in benchmark/results/ — run `make bench` then +# `python benchmark/benchmark_qmcpy.py [label]` first. +# +# NOTE: only the `ms` column is cross-comparable. `allocs`/`KiB` are Julia-only. +# C-kernel rows (Lattice, DigitalNetB2, Halton gen_samples) measure the same C +# library on both sides; they are NOT a language comparison (see benchmark_qmcpy.py). + +using Pkg +Pkg.activate(@__DIR__) +let deps = keys(Pkg.project().dependencies) + "QMC" in deps || Pkg.develop(; path = dirname(@__DIR__)) + "BenchmarkTools" in deps || Pkg.add("BenchmarkTools") + "JSON3" in deps || Pkg.add("JSON3") +end +Pkg.instantiate() + +using BenchmarkTools +using JSON3 +using Statistics +using Printf + +resdir = joinpath(@__DIR__, "results") + +jl_label = length(ARGS) >= 1 ? ARGS[1] : "latest" +py_label = length(ARGS) >= 2 ? ARGS[2] : jl_label + +jl_file = joinpath(resdir, "$(jl_label).json") +py_file = joinpath(resdir, "qmcpy_$(py_label).json") + +isfile(jl_file) || error("Julia results not found: $jl_file\nRun: make bench" * + (jl_label == "latest" ? "" : " then julia benchmark/runbenchmarks.jl $jl_label")) +isfile(py_file) || error("QMCPy results not found: $py_file\n" * + "Run: python benchmark/benchmark_qmcpy.py $py_label") + +jl_results = BenchmarkTools.load(jl_file)[1] +py_data = JSON3.read(read(py_file, String)) + +py_results = py_data["results"] +py_version = get(py_data, "qmcpy_version", "?") +py_python = get(py_data, "python", "?") + +println("Julia vs QMCPy benchmark comparison") +println(" Julia results : $jl_file") +println(" QMCPy results : $py_file (qmcpy $py_version, python $py_python)") +println() +println(" NOTE: C-kernel rows [C] (Lattice/DigitalNetB2/Halton gen_samples) use the") +println(" same qmctoolscl library on both sides and are NOT a language comparison.") +println() + +header = @sprintf(" %-48s %11s %11s %7s", "benchmark", "Julia (ms)", "Python (ms)", "ratio") +println("="^length(header)) +println(header) +println("-"^length(header)) + +for group in sort(collect(keys(jl_results))) + println("\n── $group ──") + py_group = get(py_results, Symbol(group), nothing) + for name in sort(collect(keys(jl_results[group]))) + jl_ms = median(jl_results[group][name]).time / 1e6 + + # Python key may have " [C]" suffix for C-kernel variants; try both. + py_entry = nothing + if py_group !== nothing + py_entry = get(py_group, name, get(py_group, replace(name, r" d=\d+" => s -> " [C]" * s), nothing)) + end + + if py_entry !== nothing && !haskey(py_entry, "error") + py_ms = Float64(py_entry["median_ms"]) + ratio = py_ms / jl_ms + tag = occursin(r"Lattice|DigitalNetB2|Halton", name) ? " [C]" : " " + @printf(" %s %-44s %11.3f %11.3f %6.2fx\n", tag, name, jl_ms, py_ms, ratio) + else + err = py_entry !== nothing ? string(py_entry["error"]) : "no Python data" + @printf(" ??? %-44s %11.3f %11s %7s\n", name, jl_ms, "n/a", "n/a") + end + end +end + +println() +println("="^length(header)) +println("ratio > 1: Python is slower ratio < 1: Python is faster") diff --git a/benchmark/results/judgement.md b/benchmark/results/judgement.md deleted file mode 100644 index ca75ce6..0000000 --- a/benchmark/results/judgement.md +++ /dev/null @@ -1,113 +0,0 @@ -# Benchmark Report for */Users/terrya/Documents/ProgramData/qmcju_software_choi* - -## Job Properties -* Time of benchmarks: - - Target: 2 Jun 2026 - 14:56 - - Baseline: 2 Jun 2026 - 14:57 -* Package commits: - - Target: dirty - - Baseline: f21de9e -* Julia commits: - - Target: 1534690 - - Baseline: 1534690 -* Julia command flags: - - Target: None - - Baseline: None -* Environment variables: - - Target: None - - Baseline: None - -## Results -A ratio greater than `1.0` denotes a possible regression (marked with :x:), while a ratio less -than `1.0` denotes a possible improvement (marked with :white_check_mark:). Brackets display [tolerances](https://juliaci.github.io/BenchmarkTools.jl/stable/manual/#Benchmark-Parameters) for the benchmark estimates. Only significant results - results -that indicate possible regressions or improvements - are shown below (thus, an empty table means that all -benchmark results remained invariant between builds). - -| ID | time ratio | memory ratio | -|-------------------------------------------------|------------------------------|--------------| -| `["evaluate", "Genz(oscillatory) n=16384"]` | 1.08 (5%) :x: | 1.00 (1%) | -| `["evaluate", "Keister n=256"]` | 1.24 (5%) :x: | 1.00 (1%) | -| `["gen_samples", "DigitalNetB2 d=10 n=16384"]` | 0.89 (5%) :white_check_mark: | 1.00 (1%) | -| `["gen_samples", "DigitalNetB2 d=3 n=1024"]` | 0.94 (5%) :white_check_mark: | 1.00 (1%) | -| `["gen_samples", "DigitalNetB2 d=3 n=4096"]` | 1.07 (5%) :x: | 1.00 (1%) | -| `["gen_samples", "Halton d=3 n=1024"]` | 0.90 (5%) :white_check_mark: | 1.00 (1%) | -| `["gen_samples", "IIDStdUniform d=10 n=1024"]` | 1.13 (5%) :x: | 1.00 (1%) | -| `["gen_samples", "IIDStdUniform d=10 n=16384"]` | 1.11 (5%) :x: | 1.00 (1%) | -| `["gen_samples", "IIDStdUniform d=10 n=4096"]` | 0.81 (5%) :white_check_mark: | 1.00 (1%) | -| `["gen_samples", "IIDStdUniform d=3 n=16384"]` | 1.73 (5%) :x: | 1.00 (1%) | -| `["gen_samples", "Kronecker d=3 n=1024"]` | 1.16 (5%) :x: | 1.00 (1%) | -| `["gen_samples", "Kronecker d=3 n=4096"]` | 1.19 (5%) :x: | 1.00 (1%) | -| `["gen_samples", "Lattice d=10 n=16384"]` | 1.17 (5%) :x: | 1.00 (1%) | -| `["gen_samples", "Lattice d=3 n=1024"]` | 0.80 (5%) :white_check_mark: | 1.00 (1%) | -| `["gen_samples", "Lattice d=3 n=256"]` | 0.86 (5%) :white_check_mark: | 1.00 (1%) | -| `["integrate", "CubMCCLT Keister"]` | 0.90 (5%) :white_check_mark: | 1.00 (1%) | -| `["integrate", "CubQMCNetG EuropeanOption"]` | 1.12 (5%) :x: | 1.00 (1%) | -| `["transform", "Gaussian d=10 n=1024"]` | 1.08 (5%) :x: | 1.00 (1%) | -| `["transform", "Gaussian d=10 n=16384"]` | 1.07 (5%) :x: | 1.00 (1%) | -| `["transform", "Gaussian d=10 n=4096"]` | 1.06 (5%) :x: | 1.00 (1%) | -| `["transform", "Gaussian d=3 n=1024"]` | 1.08 (5%) :x: | 1.00 (1%) | -| `["transform", "Gaussian d=3 n=16384"]` | 1.05 (5%) :x: | 1.00 (1%) | -| `["transform", "Gaussian d=3 n=256"]` | 1.10 (5%) :x: | 1.00 (1%) | -| `["transform", "Gaussian d=3 n=4096"]` | 1.08 (5%) :x: | 1.00 (1%) | - -## Benchmark Group List -Here's a list of all the benchmark groups executed by this job: - -- `["evaluate"]` -- `["gen_samples"]` -- `["integrate"]` -- `["transform"]` - -## Julia versioninfo - -### Target -``` -Julia Version 1.12.6 -Commit 15346901f0 (2026-04-09 19:20 UTC) -Build Info: - Built by Homebrew (v1.12.6) - - Note: This is an unofficial build, please report bugs to the project - responsible for this build and not to the Julia project unless you can - reproduce the issue using official builds available at https://julialang.org - -Platform Info: - OS: macOS (arm64-apple-darwin25.4.0) - uname: Darwin 25.5.0 Darwin Kernel Version 25.5.0: Mon Apr 27 20:41:15 PDT 2026; root:xnu-12377.121.6~2/RELEASE_ARM64_T6041 arm64 arm - CPU: Apple M4 Max: - speed user nice sys idle irq - #1-16 2400 MHz 709878 s 0 s 249549 s 14785237 s 0 s - Memory: 128.0 GB (4583.15625 MB free) - Uptime: 1.090962e6 sec - Load Avg: 4.328125 3.29736328125 3.0224609375 - WORD_SIZE: 64 - LLVM: libLLVM-18.1.7 (ORCJIT, apple-m4) - GC: Built with stock GC -Threads: 1 default, 1 interactive, 1 GC (on 12 virtual cores) -``` - -### Baseline -``` -Julia Version 1.12.6 -Commit 15346901f0 (2026-04-09 19:20 UTC) -Build Info: - Built by Homebrew (v1.12.6) - - Note: This is an unofficial build, please report bugs to the project - responsible for this build and not to the Julia project unless you can - reproduce the issue using official builds available at https://julialang.org - -Platform Info: - OS: macOS (arm64-apple-darwin25.4.0) - uname: Darwin 25.5.0 Darwin Kernel Version 25.5.0: Mon Apr 27 20:41:15 PDT 2026; root:xnu-12377.121.6~2/RELEASE_ARM64_T6041 arm64 arm - CPU: Apple M4 Max: - speed user nice sys idle irq - #1-16 2400 MHz 709913 s 0 s 249559 s 14785501 s 0 s - Memory: 128.0 GB (4493.359375 MB free) - Uptime: 1.090982e6 sec - Load Avg: 4.82861328125 3.49853515625 3.10107421875 - WORD_SIZE: 64 - LLVM: libLLVM-18.1.7 (ORCJIT, apple-m4) - GC: Built with stock GC -Threads: 1 default, 1 interactive, 1 GC (on 12 virtual cores) -``` \ No newline at end of file diff --git a/benchmark/results/latest.json b/benchmark/results/latest.json deleted file mode 100644 index f5d8dff..0000000 --- a/benchmark/results/latest.json +++ /dev/null @@ -1 +0,0 @@ -[{"BenchmarkTools":"1.8.0","Julia":"1.12.6"},[["BenchmarkGroup",{"data":{"evaluate":["BenchmarkGroup",{"data":{"Genz(oscillatory) n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":8256,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[8583.333333333334,4375.0,4291.666666666667,4486.333333333333,4875.0]}],"Genz(oscillatory) n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":131136,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[142583.33333333334,105416.66666666667,108111.0,99236.0,99763.66666666667]}],"Genz(oscillatory) n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":2112,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2930.6666666666665,1291.6666666666667,1153.0,1138.6666666666667,1208.3333333333333]}],"Genz(oscillatory) n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":32832,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[31277.666666666668,19666.666666666668,20014.0,19944.666666666668,19847.333333333332]}],"Keister n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":8256,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6902.666666666667,4583.333333333333,4639.0,4541.666666666667,4458.333333333333]}],"Keister n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":131136,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[101847.33333333333,93833.33333333333,88805.66666666667,93486.0,89930.33333333333]}],"Keister n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":2112,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2514.0,1222.0,1027.6666666666667,1014.0,1111.3333333333333]}],"Keister n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":32832,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[26166.666666666668,22708.333333333332,22944.666666666668,21333.333333333332,25680.666666666668]}]},"tags":[]}],"gen_samples":["BenchmarkGroup",{"data":{"DigitalNetB2 d=10 n=1024":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":335392,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[38625.0,43402.666666666664,35791.666666666664,38638.666666666664,35666.666666666664]}],"DigitalNetB2 d=10 n=16384":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":5250592,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[673569.3333333334,726680.6666666666,787652.6666666666,749291.6666666666,699097.3333333334]}],"DigitalNetB2 d=10 n=256":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":89632,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[16805.666666666668,13861.0,13791.666666666666,15653.0,18430.666666666668]}],"DigitalNetB2 d=10 n=4096":["Trial",{"allocs":42,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1318432,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[145597.33333333334,134139.0,134805.33333333334,141125.0,134083.33333333334]}],"DigitalNetB2 d=3 n=1024":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":101040,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[14083.333333333334,11416.666666666666,11416.666666666666,12208.333333333334,16625.0]}],"DigitalNetB2 d=3 n=16384":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1575600,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[156055.33333333334,173569.33333333334,176444.66666666666,196389.0,227139.0]}],"DigitalNetB2 d=3 n=256":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":27312,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6652.666666666667,4708.333333333333,4666.666666666667,5000.0,4514.0]}],"DigitalNetB2 d=3 n=4096":["Trial",{"allocs":26,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":395952,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[47541.666666666664,43264.0,41305.333333333336,41180.333333333336,39041.666666666664]}],"Halton d=10 n=1024":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":84688,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[391555.3333333333,387930.6666666667,386819.3333333333,377694.3333333333,377847.0]}],"Halton d=10 n=16384":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1313488,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6.172083333333333e6,6.206333333333333e6,6.189694333333333e6,6.183277666666667e6,6.192597e6]}],"Halton d=10 n=256":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":23248,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[93402.66666666667,91819.66666666667,91694.33333333333,91722.33333333333,91569.33333333333]}],"Halton d=10 n=4096":["Trial",{"allocs":7,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":330448,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1.5346803333333333e6,1.5297776666666667e6,1.5369166666666667e6,1.5356113333333333e6,1.5269583333333333e6]}],"Halton d=3 n=1024":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":25552,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[121694.66666666667,123597.33333333333,114069.66666666667,114597.0,127236.0]}],"Halton d=3 n=16384":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":394192,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1.9365276666666667e6,1.9398056666666667e6,1.9535696666666667e6,1.959653e6,1.9545e6]}],"Halton d=3 n=256":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":7120,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[29583.333333333332,28250.0,28250.0,28652.666666666668,28222.0]}],"Halton d=3 n=4096":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":99280,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[484805.6666666667,485305.6666666667,489625.0,485305.6666666667,484430.3333333333]}],"IIDStdUniform d=10 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":82000,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6416.666666666667,5486.0,4514.0,4472.0,4541.666666666667]}],"IIDStdUniform d=10 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1310800,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[103111.0,110680.66666666667,101972.33333333333,84916.66666666667,107680.66666666667]}],"IIDStdUniform d=10 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":20560,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2375.0,1514.0,1333.3333333333333,1597.0,1139.0]}],"IIDStdUniform d=10 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":327760,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[22569.666666666668,25597.333333333332,21236.0,21514.0,26250.0]}],"IIDStdUniform d=3 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":24656,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2750.0,2263.6666666666665,1458.3333333333333,1847.3333333333333,1416.6666666666667]}],"IIDStdUniform d=3 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":393296,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[26875.0,30625.0,30597.0,37888.666666666664,25500.0]}],"IIDStdUniform d=3 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":6224,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1736.3333333333333,513.6666666666666,402.6666666666667,430.6666666666667,458.3333333333333]}],"IIDStdUniform d=3 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":98384,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[8000.0,7583.333333333333,7694.333333333333,5125.0,6347.333333333333]}],"Kronecker d=10 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":82000,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[19666.666666666668,18819.333333333332,18791.666666666668,19958.333333333332,20986.0]}],"Kronecker d=10 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1310800,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[458166.6666666667,474361.0,466972.3333333333,472805.3333333333,479208.3333333333]}],"Kronecker d=10 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":20560,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[5375.0,4888.666666666667,4819.333333333333,4666.666666666667,4722.0]}],"Kronecker d=10 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":327760,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[106486.0,121639.0,120708.33333333333,129500.0,115583.33333333333]}],"Kronecker d=3 n=1024":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":24656,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6694.333333333333,5694.666666666667,5625.0,5847.0,5694.333333333333]}],"Kronecker d=3 n=16384":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":393296,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[125694.33333333333,137430.66666666666,136444.33333333334,136930.66666666666,136972.33333333334]}],"Kronecker d=3 n=256":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":6224,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[2250.0,1486.0,1430.6666666666667,1444.6666666666667,1430.6666666666667]}],"Kronecker d=3 n=4096":["Trial",{"allocs":3,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":98384,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[28013.666666666668,26763.666666666668,26666.666666666668,26666.666666666668,26639.0]}],"Lattice d=10 n=1024":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":246656,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[18111.0,16153.0,17069.666666666668,16166.666666666666,25361.0]}],"Lattice d=10 n=16384":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":3933056,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[348333.3333333333,422569.3333333333,491389.0,470389.0,471333.3333333333]}],"Lattice d=10 n=256":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":62336,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[6361.0,4263.666666666667,4236.333333333333,4139.0,4180.333333333333]}],"Lattice d=10 n=4096":["Trial",{"allocs":16,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":983936,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[75778.0,73458.33333333333,132889.0,114694.33333333333,112861.0]}],"Lattice d=3 n=1024":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":74160,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[8583.333333333334,6916.666666666667,6889.0,6847.333333333333,8791.666666666666]}],"Lattice d=3 n=16384":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":1180080,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[115236.0,130097.33333333333,116139.0,116902.66666666667,113402.66666666667]}],"Lattice d=3 n=256":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":18864,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[3708.3333333333335,1986.3333333333333,2097.3333333333335,2014.0,1902.6666666666667]}],"Lattice d=3 n=4096":["Trial",{"allocs":14,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":295344,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[29430.333333333332,27430.666666666668,27389.0,28833.333333333332,27458.333333333332]}]},"tags":[]}],"integrate":["BenchmarkGroup",{"data":{"CubMCCLT AsianOption":["Trial",{"allocs":3545,"gctimes":[0.0,0.0,0.0],"memory":5834064,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[2.505375e6,2.304875e6,2.896666e6]}],"CubMCCLT Keister":["Trial",{"allocs":47,"gctimes":[0.0,146583.0,149417.0],"memory":67864048,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[3.0929292e7,3.107825e7,3.1256417e7]}],"CubQMCLatticeG Keister":["Trial",{"allocs":385,"gctimes":[0.0,0.0,0.0],"memory":2108592,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[659541.0,617500.0,636125.0]}],"CubQMCNetG EuropeanOption":["Trial",{"allocs":18545,"gctimes":[0.0,275500.0,0.0],"memory":47429552,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[1.4468e7,1.4684208e7,1.3403958e7]}],"CubQMCNetG Keister":["Trial",{"allocs":577,"gctimes":[0.0,0.0,0.0],"memory":2538672,"params":["Parameters",{"evals":1,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"times":[741875.0,824958.0,840833.0]}]},"tags":[]}],"transform":["BenchmarkGroup",{"data":{"Gaussian d=10 n=1024":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":164000,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[89139.0,93847.0,92541.66666666667,84639.0,78805.66666666667]}],"Gaussian d=10 n=16384":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":2621600,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[1.5699166666666667e6,1.5492916666666667e6,1.5227916666666667e6,1.5522083333333333e6,1.5836946666666667e6]}],"Gaussian d=10 n=256":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":41120,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[23389.0,19569.666666666668,19472.0,19347.0,19388.666666666668]}],"Gaussian d=10 n=4096":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":655520,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[372208.3333333333,354500.0,355222.0,356569.3333333333,357013.6666666667]}],"Gaussian d=3 n=1024":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":49312,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[28139.0,22875.0,24778.0,25472.333333333332,25472.333333333332]}],"Gaussian d=3 n=16384":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":786592,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[390833.3333333333,394652.6666666667,415347.0,419000.0,418166.6666666667]}],"Gaussian d=3 n=256":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":12448,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[9236.0,5944.333333333333,5958.333333333333,6152.666666666667,6013.666666666667]}],"Gaussian d=3 n=4096":["Trial",{"allocs":6,"gctimes":[0.0,0.0,0.0,0.0,0.0],"memory":196768,"params":["Parameters",{"evals":3,"evals_set":false,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"times":[100764.0,104208.33333333333,100583.33333333333,99778.0,103305.66666666667]}]},"tags":[]}]},"tags":[]}]]] \ No newline at end of file diff --git a/benchmark/tune.json b/benchmark/tune.json deleted file mode 100644 index 0a9023b..0000000 --- a/benchmark/tune.json +++ /dev/null @@ -1 +0,0 @@ -[{"BenchmarkTools":"1.8.0","Julia":"1.12.6"},[["BenchmarkGroup",{"data":{"evaluate":["BenchmarkGroup",{"data":{"Genz(oscillatory) n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Genz(oscillatory) n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Genz(oscillatory) n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Genz(oscillatory) n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Keister n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Keister n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Keister n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Keister n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}]},"tags":[]}],"gen_samples":["BenchmarkGroup",{"data":{"DigitalNetB2 d=10 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=10 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=10 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=10 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=3 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=3 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=3 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"DigitalNetB2 d=3 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=10 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=10 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=10 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=10 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=3 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=3 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=3 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Halton d=3 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=10 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=10 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=10 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=10 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=3 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=3 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=3 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"IIDStdUniform d=3 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=10 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=10 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=10 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=10 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=3 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=3 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=3 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Kronecker d=3 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=10 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=10 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=10 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=10 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=3 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=3 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=3 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Lattice d=3 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}]},"tags":[]}],"integrate":["BenchmarkGroup",{"data":{"CubMCCLT AsianOption":["Parameters",{"evals":1,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"CubMCCLT Keister":["Parameters",{"evals":1,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"CubQMCLatticeG Keister":["Parameters",{"evals":1,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"CubQMCNetG EuropeanOption":["Parameters",{"evals":1,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}],"CubQMCNetG Keister":["Parameters",{"evals":1,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":3,"seconds":5.0,"time_tolerance":0.05}]},"tags":[]}],"transform":["BenchmarkGroup",{"data":{"Gaussian d=10 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=10 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=10 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=10 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=3 n=1024":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=3 n=16384":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=3 n=256":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}],"Gaussian d=3 n=4096":["Parameters",{"evals":3,"evals_set":true,"gcsample":false,"gctrial":true,"memory_tolerance":0.01,"overhead":0.0,"samples":5,"seconds":5.0,"time_tolerance":0.05}]},"tags":[]}]},"tags":[]}]]] \ No newline at end of file From b8dd7abf40f2ce9145a9b66c7188560f0bbc05e9 Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 15:49:43 -0500 Subject: [PATCH 012/104] Fix CI: revert julia compat to 1.10 (CI matrix tests 1.10 and 1.11) --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index d3fcb0f..8233f2e 100644 --- a/Project.toml +++ b/Project.toml @@ -28,7 +28,7 @@ Random = "1.10" SpecialFunctions = "2" Statistics = "1.10" Test = "1.10" -julia = "1.12" +julia = "1.10" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" From 795b48e9cf83d3d997a9c1d5109eb8a1970a139a Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 16:48:49 -0500 Subject: [PATCH 013/104] Speed up CubQMCNetG EuropeanOption --- benchmark/benchmarks.jl | 2 +- src/discrete_distribution/digital_net_b2.jl | 60 +++++++++++++-------- src/discrete_distribution/qmctoolscl_c.jl | 52 +++++++++++++++++- src/integrand/financial_option.jl | 17 +++++- src/true_measure/gaussian.jl | 7 +-- 5 files changed, 109 insertions(+), 29 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 341ca2b..9477a6c 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -43,7 +43,7 @@ const SUITE = BenchmarkGroup() # 1. Discrete Distribution sampling # NOTE: Lattice / DigitalNetB2 / Halton call the qmctoolscl C library for # point generation (same library QMCPy uses), so these measure the C kernel + -# FFI binding, NOT Julia-vs-Python. IIDStdUniform and Kronecker are pure Julia. +# (foreign function interface (FFI)binding, NOT Julia-vs-Python. IIDStdUniform and Kronecker are pure Julia. SUITE["gen_samples"] = BenchmarkGroup() for dim in DIMS, n in SAMPLES SUITE["gen_samples"]["IIDStdUniform d=$dim n=$n"] = diff --git a/src/discrete_distribution/digital_net_b2.jl b/src/discrete_distribution/digital_net_b2.jl index 3ab5ef8..a1e0902 100644 --- a/src/discrete_distribution/digital_net_b2.jl +++ b/src/discrete_distribution/digital_net_b2.jl @@ -215,19 +215,27 @@ function _gen_single_replication(dd::DigitalNetB2, n::Int; n_start::Int = 0) apply_shift = 0x00 end - xb_buf = Vector{UInt64}(undef, n * d) - if dd.graycode - _c_dnb2_gen_gray!(1, n, d, n_start, mmax, C_flat, xb_buf) - else - _c_dnb2_gen_natural!(1, n, d, n_start, mmax, C_flat, xb_buf) - end - - xrb_buf = Vector{UInt64}(undef, n * d) - _c_dnb2_digital_shift!(1, n, d, 1, lshifts, xb_buf, shiftsb, xrb_buf) - tmaxes = fill(UInt64(mmax), 1) x_buf = Vector{Float64}(undef, n * d) - _c_dnb2_integer_to_float!(1, n, d, tmaxes, xrb_buf, x_buf) + if _HAS_DNB2_FUSED[] + if dd.graycode + _c_dnb2_gen_gray_float!(1, n, d, n_start, mmax, 1, + apply_shift, lshifts, shiftsb, tmaxes, C_flat, x_buf) + else + _c_dnb2_gen_natural_float!(1, n, d, n_start, mmax, 1, + apply_shift, lshifts, shiftsb, tmaxes, C_flat, x_buf) + end + else + xb_buf = Vector{UInt64}(undef, n * d) + if dd.graycode + _c_dnb2_gen_gray!(1, n, d, n_start, mmax, C_flat, xb_buf) + else + _c_dnb2_gen_natural!(1, n, d, n_start, mmax, C_flat, xb_buf) + end + xrb_buf = Vector{UInt64}(undef, n * d) + _c_dnb2_digital_shift!(1, n, d, 1, lshifts, xb_buf, shiftsb, xrb_buf) + _c_dnb2_integer_to_float!(1, n, d, tmaxes, xrb_buf, x_buf) + end return _rowmaj_to_nxd(x_buf, n, d) end @@ -301,19 +309,27 @@ function gen_samples(dd::DigitalNetB2, n::Int; n_start::Int = 0) apply_shift = 0x00 end - xb_buf = Vector{UInt64}(undef, r_x * n * d) - if dd.graycode - _c_dnb2_gen_gray!(r_x, n, d, n_start, mmax, C_flat, xb_buf) - else - _c_dnb2_gen_natural!(r_x, n, d, n_start, mmax, C_flat, xb_buf) - end - - xrb_buf = Vector{UInt64}(undef, R * n * d) - _c_dnb2_digital_shift!(R, n, d, r_x, lshifts, xb_buf, shiftsb, xrb_buf) - tmaxes = fill(UInt64(mmax), R) x_buf = Vector{Float64}(undef, R * n * d) - _c_dnb2_integer_to_float!(R, n, d, tmaxes, xrb_buf, x_buf) + if _HAS_DNB2_FUSED[] + if dd.graycode + _c_dnb2_gen_gray_float!(R, n, d, n_start, mmax, r_x, + apply_shift, lshifts, shiftsb, tmaxes, C_flat, x_buf) + else + _c_dnb2_gen_natural_float!(R, n, d, n_start, mmax, r_x, + apply_shift, lshifts, shiftsb, tmaxes, C_flat, x_buf) + end + else + xb_buf = Vector{UInt64}(undef, r_x * n * d) + if dd.graycode + _c_dnb2_gen_gray!(r_x, n, d, n_start, mmax, C_flat, xb_buf) + else + _c_dnb2_gen_natural!(r_x, n, d, n_start, mmax, C_flat, xb_buf) + end + xrb_buf = Vector{UInt64}(undef, R * n * d) + _c_dnb2_digital_shift!(R, n, d, r_x, lshifts, xb_buf, shiftsb, xrb_buf) + _c_dnb2_integer_to_float!(R, n, d, tmaxes, xrb_buf, x_buf) + end return _rowmaj_to_Rnxd(x_buf, R, n, d) end diff --git a/src/discrete_distribution/qmctoolscl_c.jl b/src/discrete_distribution/qmctoolscl_c.jl index 2ba0097..d9fc0db 100644 --- a/src/discrete_distribution/qmctoolscl_c.jl +++ b/src/discrete_distribution/qmctoolscl_c.jl @@ -12,6 +12,10 @@ const _QMCTOOLSCL_LIB_PATH = Ref{String}("") const _QMCTOOLSCL_INIT_ATTEMPTED = Ref(false) const _QMCTOOLSCL_LAST_SEARCH = Ref("none") +# True when the loaded library exports the fused gen+shift+float functions +# (available since qmctoolscl 1.2.3; probe once at init time). +const _HAS_DNB2_FUSED = Ref(false) + function _python_candidates() candidates = String[] @@ -81,7 +85,9 @@ except Exception: path = strip(read(`$exe -c $py_script`, String)) if !isempty(path) && isfile(path) _QMCTOOLSCL_LIB_PATH[] = path - Libdl.dlopen(path, Libdl.RTLD_GLOBAL | Libdl.RTLD_LAZY) + hdl = Libdl.dlopen(path, Libdl.RTLD_GLOBAL | Libdl.RTLD_LAZY) + _HAS_DNB2_FUSED[] = + Libdl.dlsym_e(hdl, :dnb2_gen_gray_float) != C_NULL return true end catch @@ -249,6 +255,50 @@ function _c_dnb2_integer_to_float!(R::Int, n::Int, d::Int, tmaxes, xb_buf, x_buf) end +# ────────────────────────────────────────────────────────────────────────────── +# Fused generation + optional digital shift + integer-to-float +# +# Signatures (from qmctoolscl/c_funcs/): +# dnb2_gen_gray_float(r, n, d, bs_r, bs_n, bs_d, n_start, mmax, r_x, +# apply_shift, *lshifts, *shiftsb, *tmaxes, *C, *x) +# dnb2_gen_natural_float(...) — same signature, non-Gray ordering +# +# These fuse generate + digital shift + float conversion into a single C pass, +# eliminating the intermediate xb (UInt64) and xrb (UInt64) buffers. +# +# lshifts: size r_x UInt64 (left shift per matrix replication; use zeros) +# shiftsb: size r*d UInt64 (digital shifts; ignored when apply_shift=0x00) +# tmaxes: size r UInt64 (bit width for scaling: x = val * 2^{-tmaxes[l]}) +# C: size r_x*d*mmax UInt64 (generating matrices, row-major) +# x: size r*n*d Float64 (output, row-major) +# ────────────────────────────────────────────────────────────────────────────── + +function _c_dnb2_gen_gray_float!(R::Int, n::Int, d::Int, n_start::Int, + mmax::Int, r_x::Int, apply_shift::UInt8, + lshifts::Vector{UInt64}, shiftsb::Vector{UInt64}, tmaxes::Vector{UInt64}, + C::Vector{UInt64}, x_buf::Vector{Float64}) + ccall((:dnb2_gen_gray_float, _qmctoolscl_lib_path()), Cvoid, + (UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, + UInt8, Ptr{UInt64}, Ptr{UInt64}, Ptr{UInt64}, Ptr{UInt64}, Ptr{Float64}), + UInt64(R), UInt64(n), UInt64(d), + UInt64(R), UInt64(n), UInt64(d), + UInt64(n_start), UInt64(mmax), UInt64(r_x), + apply_shift, lshifts, shiftsb, tmaxes, C, x_buf) +end + +function _c_dnb2_gen_natural_float!(R::Int, n::Int, d::Int, n_start::Int, + mmax::Int, r_x::Int, apply_shift::UInt8, + lshifts::Vector{UInt64}, shiftsb::Vector{UInt64}, tmaxes::Vector{UInt64}, + C::Vector{UInt64}, x_buf::Vector{Float64}) + ccall((:dnb2_gen_natural_float, _qmctoolscl_lib_path()), Cvoid, + (UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, UInt64, + UInt8, Ptr{UInt64}, Ptr{UInt64}, Ptr{UInt64}, Ptr{UInt64}, Ptr{Float64}), + UInt64(R), UInt64(n), UInt64(d), + UInt64(R), UInt64(n), UInt64(d), + UInt64(n_start), UInt64(mmax), UInt64(r_x), + apply_shift, lshifts, shiftsb, tmaxes, C, x_buf) +end + # ────────────────────────────────────────────────────────────────────────────── # Halton C wrapper # diff --git a/src/integrand/financial_option.jl b/src/integrand/financial_option.jl index e1f3b75..7c7604a 100644 --- a/src/integrand/financial_option.jl +++ b/src/integrand/financial_option.jl @@ -181,12 +181,25 @@ function _barrier_payoff(f::FinancialOption, S::AbstractVector) end function evaluate(f::FinancialOption, x::AbstractMatrix) - n = size(x, 1) r = f.interest_rate T = f._time_vector[end] discount = exp(-r * T) - y = Vector{Float64}(undef, n) + # Fast path: European option with GBM transform already applied. + # Only the terminal price S(T) = x[:, end] is needed; skip the row loop + # and the per-row _stock_prices / _payoff calls entirely. + if f.option_type == :european && f._use_gbm_transform + S_T = @view x[:, end] + K = f.strike_price + return if f.call_put == :call + @. discount * max(S_T - K, 0.0) + else + @. discount * max(K - S_T, 0.0) + end + end + + n = size(x, 1) + y = Vector{Float64}(undef, n) @inbounds for i in 1:n S = _stock_prices(f, @view(x[i, :])) y[i] = discount * _payoff(f, S) diff --git a/src/true_measure/gaussian.jl b/src/true_measure/gaussian.jl index b46cedd..6e11573 100644 --- a/src/true_measure/gaussian.jl +++ b/src/true_measure/gaussian.jl @@ -106,9 +106,10 @@ end function transform(tm::Gaussian, x::AbstractMatrix) # x is n×d with entries in [0,1); compute y = Φ⁻¹(x) * Aᵀ .+ μᵀ - # Numerics are identical to the original (same standard-normal quantile); - # the mean shift is done in place to avoid an extra n×d allocation. - z = quantile.(Distributions.Normal(), x) # one n×d temporary + # Use erfinv directly: Φ⁻¹(u) = √2 · erfinv(2u-1). + # This avoids Distributions.Normal() dispatch overhead per element and is + # ~15–20% faster than quantile.(Normal(), x) at large n×d. + z = @. sqrt(2.0) * SpecialFunctions.erfinv(2.0 * x - 1.0) y = z * transpose(tm._decomp) # BLAS gemm (handles diagonal A too) y .+= transpose(tm.mean) # in-place mean shift, no allocation return y From 7599f28c25f57117d8593dc7938c2a7d0885ef44 Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Tue, 2 Jun 2026 20:39:20 -0500 Subject: [PATCH 014/104] Speed up CubQMCNetG EuropeanOption --- src/integrand/genz.jl | 11 ++++------- src/integrand/keister.jl | 13 +++++-------- src/true_measure/geometric_brownian_motion.jl | 12 +++--------- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/src/integrand/genz.jl b/src/integrand/genz.jl index c317f83..370a250 100644 --- a/src/integrand/genz.jl +++ b/src/integrand/genz.jl @@ -51,13 +51,10 @@ function evaluate(f::Genz, x::AbstractMatrix) y = Vector{Float64}(undef, n) if f.kind == :oscillatory - for i in 1:n - s = 2π * u[1] - for j in 1:d - s += a[j] * x[i, j] - end - y[i] = cos(s) - end + # x * a is BLAS GEMV (O(n·d) but vectorised): much faster than a row loop + # for large n; @. fuses cos into one broadcast pass. + s = x * a + @. y = cos(2π * u[1] + s) elseif f.kind == :product_peak for i in 1:n diff --git a/src/integrand/keister.jl b/src/integrand/keister.jl index 54a4e3d..4196387 100644 --- a/src/integrand/keister.jl +++ b/src/integrand/keister.jl @@ -26,14 +26,11 @@ function Keister(tm::AbstractTrueMeasure) end function evaluate(f::Keister, x::AbstractMatrix) - n, d = size(x) - y = Vector{Float64}(undef, n) - coeff = π^(d / 2) - @inbounds for i in 1:n - norm_xi = sqrt(sum(x[i, j]^2 for j in 1:d)) - y[i] = coeff * cos(norm_xi) - end - return y + coeff = π^(f.dimension / 2) + # sum(abs2, x; dims=2) sweeps column-by-column (cache-friendly for column-major + # Julia arrays), avoiding row-stride cache misses in the original loop. + norms = vec(sqrt.(sum(abs2, x; dims=2))) + return @. coeff * cos(norms) end """ diff --git a/src/true_measure/geometric_brownian_motion.jl b/src/true_measure/geometric_brownian_motion.jl index 645c0d8..b85c5c6 100644 --- a/src/true_measure/geometric_brownian_motion.jl +++ b/src/true_measure/geometric_brownian_motion.jl @@ -95,15 +95,9 @@ function transform(tm::GeometricBrownianMotion, x::AbstractMatrix) bm_samples = transform(tm._bm, x) # S(t) = S₀ exp[(γ - σ²/2) t + BM(t)] # where BM(t) already has variance σ² t from the covariance - n, d = size(bm_samples) - result = Matrix{Float64}(undef, n, d) - @inbounds for j in 1:d - exponent_drift = (tm.drift - 0.5 * tm.diffusion) * tm.time_vector[j] - for i in 1:n - result[i, j] = tm.initial_value * exp(exponent_drift + bm_samples[i, j]) - end - end - return result + # offsets is 1×d; broadcast fuses + and exp into one SIMD pass over the n×d array. + offsets = transpose((tm.drift - 0.5 * tm.diffusion) .* tm.time_vector) + return @. tm.initial_value * exp(offsets + bm_samples) end function Base.show(io::IO, tm::GeometricBrownianMotion) From d2d9d202384bf594e7198d9b8e16c0f91b691f3a Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Wed, 3 Jun 2026 09:32:50 -0500 Subject: [PATCH 015/104] Improve benchmarking output and standardize ratio conventions --- Makefile | 11 +++++-- benchmark/compare.jl | 73 +++++++++++++++++++++++++++++++++++------ benchmark/compare_py.jl | 54 ++++++++++++++++++++++++++++-- 3 files changed, 123 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index af650db..1ff445a 100644 --- a/Makefile +++ b/Makefile @@ -49,19 +49,24 @@ notebook-%: julia --project=. test/run_notebooks.jl $* # Run the benchmark suite (uses its own environment in benchmark/, set up on first run) +# Saves results to benchmark/results/latest.json bench: julia benchmark/runbenchmarks.jl # Compare the working tree against a baseline git revision (default: HEAD, i.e. # the effect of uncommitted changes). Override with: make bench-compare REV=master +# Output: benchmark/results/compare_head.md +# Ratio = reference (REV) time ÷ local time → < 1: local slower | > 1: local faster REV ?= HEAD bench-compare: julia benchmark/compare.jl $(REV) -# Side-by-side Julia vs QMCPy comparison (requires both result files to exist). -# Run `make bench` and `python benchmark/benchmark_qmcpy.py` first. +# Side-by-side Julia vs QMCPy comparison. +# Prerequisites: run `make bench` then `python benchmark/benchmark_qmcpy.py [label]`. +# Output: benchmark/results/compare_python.md +# Ratio = Python time ÷ Julia time → < 1: local (Julia) slower | > 1: local (Julia) faster # Override label with: make bench-compare-py JL_LABEL=foo PY_LABEL=bar JL_LABEL ?= latest PY_LABEL ?= $(JL_LABEL) -bench-compare-py: bench bench-compare +bench-compare-py: bench julia benchmark/compare_py.jl $(JL_LABEL) $(PY_LABEL) diff --git a/benchmark/compare.jl b/benchmark/compare.jl index 747a633..117931b 100644 --- a/benchmark/compare.jl +++ b/benchmark/compare.jl @@ -2,9 +2,16 @@ # # Usage (from the repository root): # julia benchmark/compare.jl # benchmark the current working tree only -# julia benchmark/compare.jl main # judge working tree (incl. uncommitted +# julia benchmark/compare.jl main # compare working tree (incl. uncommitted # # changes) against `main` -# julia benchmark/compare.jl HEAD main # judge one committed revision against another +# julia benchmark/compare.jl HEAD main # compare one committed revision against another +# +# Output: benchmark/results/compare_head.md +# +# Ratio convention (same as compare_py.jl): +# ratio = reference time ÷ local time +# ratio < 1 → local is SLOWER than reference ❌ +# ratio > 1 → local is FASTER than reference ✅ # # Dirty-tree friendly: a baseline/target *revision* is benchmarked inside a # temporary `git worktree`, so uncommitted changes are never stashed or @@ -21,11 +28,52 @@ end Pkg.instantiate() using PkgBenchmark +using Printf const PKG = dirname(@__DIR__) const RESDIR = joinpath(@__DIR__, "results") mkpath(RESDIR) +""" + write_comparison_md(outfile, target_result, baseline_result, target_label, baseline_label) + +Write a Markdown comparison table to `outfile`. + +Ratio = reference (`baseline_label`) time ÷ local (`target_label`) time. + - ratio < 1 → local is **slower** than reference ❌ + - ratio > 1 → local is **faster** than reference ✅ +""" +function write_comparison_md(outfile, target_result, baseline_result, + target_label, baseline_label) + tg = target_result.benchmarkgroup + bg = baseline_result.benchmarkgroup + open(outfile, "w") do io + println(io, "# Benchmark: `$(target_label)` vs `$(baseline_label)`\n") + println(io, "| | local (`$(target_label)`) | reference (`$(baseline_label)`) |") + println(io, "|---|---|---|") + println(io, "| commit | $(target_result.commit) | $(baseline_result.commit) |") + println(io, "| date | $(target_result.date) | $(baseline_result.date) |") + println(io, "") + println(io, "**`ratio = reference time ÷ local time`** ") + println(io, "ratio `< 1` → local is **slower** ❌ | ratio `> 1` → local is **faster** ✅") + println(io, "") + println(io, "| benchmark | ratio | verdict | local (ms) | reference (ms) |") + println(io, "|:----------|------:|:-------:|----------:|---------------:|") + for group in sort(collect(keys(tg))) + haskey(bg, group) || continue + for name in sort(collect(keys(tg[group]))) + haskey(bg[group], name) || continue + local_ms = median(tg[group][name]).time / 1e6 + ref_ms = median(bg[group][name]).time / 1e6 + ratio = ref_ms / local_ms + verdict = ratio < 0.95 ? "❌" : ratio > 1.05 ? "✅" : "–" + @printf(io, "| `[\"%s\", \"%s\"]` | %.3f | %s | %.3f | %.3f |\n", + group, name, ratio, verdict, local_ms, ref_ms) + end + end + end +end + "Benchmark the current working tree in place (uncommitted changes included)." bench_worktree() = benchmarkpkg(PKG) @@ -79,18 +127,23 @@ end if length(ARGS) == 0 result = bench_worktree() - export_markdown(joinpath(RESDIR, "current.md"), result) - println("\nWrote benchmark/results/current.md") + outfile = joinpath(RESDIR, "bench_local.md") + export_markdown(outfile, result) + println("\nWrote benchmark/results/bench_local.md (single-run summary, no comparison)") elseif length(ARGS) == 1 baseline_rev = ARGS[1] - target = bench_worktree() # current tree (dirty OK) + target = bench_worktree() # current tree (dirty OK) baseline = bench_revision(baseline_rev) - export_markdown(joinpath(RESDIR, "judgement.md"), judge(target, baseline)) - println("\nWrote benchmark/results/judgement.md (working tree vs $(baseline_rev))") + outfile = joinpath(RESDIR, "compare_head.md") + write_comparison_md(outfile, target, baseline, "local", baseline_rev) + println("\nWrote benchmark/results/compare_head.md (local vs $(baseline_rev))") + println(" ratio = $(baseline_rev) ÷ local → < 1: local slower | > 1: local faster") else target_rev, baseline_rev = ARGS[1], ARGS[2] - target = bench_revision(target_rev) + target = bench_revision(target_rev) baseline = bench_revision(baseline_rev) - export_markdown(joinpath(RESDIR, "judgement.md"), judge(target, baseline)) - println("\nWrote benchmark/results/judgement.md ($(target_rev) vs $(baseline_rev))") + outfile = joinpath(RESDIR, "compare_head.md") + write_comparison_md(outfile, target, baseline, target_rev, baseline_rev) + println("\nWrote benchmark/results/compare_head.md ($(target_rev) vs $(baseline_rev))") + println(" ratio = $(baseline_rev) ÷ $(target_rev) → < 1: $(target_rev) slower | > 1: $(target_rev) faster") end diff --git a/benchmark/compare_py.jl b/benchmark/compare_py.jl index 388dbf8..476e361 100644 --- a/benchmark/compare_py.jl +++ b/benchmark/compare_py.jl @@ -8,6 +8,13 @@ # Both files must exist in benchmark/results/ — run `make bench` then # `python benchmark/benchmark_qmcpy.py [label]` first. # +# Output: benchmark/results/compare_python.md +# +# Ratio convention (same as compare.jl): +# ratio = reference (Python) time ÷ local (Julia) time +# ratio < 1 → local is SLOWER than Python ❌ +# ratio > 1 → local is FASTER than Python ✅ +# # NOTE: only the `ms` column is cross-comparable. `allocs`/`KiB` are Julia-only. # C-kernel rows (Lattice, DigitalNetB2, Halton gen_samples) measure the same C # library on both sides; they are NOT a language comparison (see benchmark_qmcpy.py). @@ -77,7 +84,6 @@ for group in sort(collect(keys(jl_results))) tag = occursin(r"Lattice|DigitalNetB2|Halton", name) ? " [C]" : " " @printf(" %s %-44s %11.3f %11.3f %6.2fx\n", tag, name, jl_ms, py_ms, ratio) else - err = py_entry !== nothing ? string(py_entry["error"]) : "no Python data" @printf(" ??? %-44s %11.3f %11s %7s\n", name, jl_ms, "n/a", "n/a") end end @@ -85,4 +91,48 @@ end println() println("="^length(header)) -println("ratio > 1: Python is slower ratio < 1: Python is faster") +println("ratio > 1: local (Julia) faster than Python | ratio < 1: local (Julia) slower than Python") + +# ── Save markdown file ────────────────────────────────────────────────────────── +outfile = joinpath(resdir, "compare_python.md") +open(outfile, "w") do io + println(io, "# Benchmark: Julia (local) vs QMCPy (Python)\n") + println(io, "| | Julia (local) | Python (QMCPy) |") + println(io, "|---|---|---|") + println(io, "| Julia results | `$(jl_file)` | — |") + println(io, "| Python results | — | `$(py_file)` |") + println(io, "| qmcpy version | — | $(py_version) |") + println(io, "| python | — | $(py_python) |") + println(io, "") + println(io, "**`ratio = Python time ÷ Julia time`** ") + println(io, "ratio `< 1` → local (Julia) is **slower** ❌ | ratio `> 1` → local (Julia) is **faster** ✅ ") + println(io, "") + println(io, "> ⚠️ C-kernel rows `[C]` (Lattice/DigitalNetB2/Halton `gen_samples`) call the same") + println(io, "> `qmctoolscl` library on both sides and are **not** a Julia vs Python comparison.") + println(io, "") + println(io, "| benchmark | ratio | verdict | Julia (ms) | Python (ms) |") + println(io, "|:----------|------:|:-------:|----------:|------------:|") + for group in sort(collect(keys(jl_results))) + py_group = get(py_results, Symbol(group), nothing) + for name in sort(collect(keys(jl_results[group]))) + jl_ms = median(jl_results[group][name]).time / 1e6 + py_entry = nothing + if py_group !== nothing + py_entry = get(py_group, name, get(py_group, replace(name, r" d=\d+" => s -> " [C]" * s), nothing)) + end + c_note = occursin(r"Lattice|DigitalNetB2|Halton", name) ? " `[C]`" : "" + if py_entry !== nothing && !haskey(py_entry, "error") + py_ms = Float64(py_entry["median_ms"]) + ratio = py_ms / jl_ms + verdict = ratio < 0.95 ? "❌" : ratio > 1.05 ? "✅" : "–" + @printf(io, "| `[\"%s\", \"%s\"]`%s | %.3f | %s | %.3f | %.3f |\n", + group, name, c_note, ratio, verdict, jl_ms, py_ms) + else + @printf(io, "| `[\"%s\", \"%s\"]` | n/a | — | %.3f | n/a |\n", + group, name, jl_ms) + end + end + end +end +println("\nWrote benchmark/results/compare_python.md") +println(" ratio = Python ÷ Julia → < 1: local slower | > 1: local faster") From 9d245745b2cf26eec31e080627726714b0378777 Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Wed, 3 Jun 2026 09:43:29 -0500 Subject: [PATCH 016/104] Aggregate summary metrics --- benchmark/compare.jl | 84 ++++++++++++++++++++++----- benchmark/compare_py.jl | 122 +++++++++++++++++++++++++++++----------- 2 files changed, 157 insertions(+), 49 deletions(-) diff --git a/benchmark/compare.jl b/benchmark/compare.jl index 117931b..b361ad7 100644 --- a/benchmark/compare.jl +++ b/benchmark/compare.jl @@ -23,17 +23,60 @@ using Pkg Pkg.activate(@__DIR__) let deps = keys(Pkg.project().dependencies) "QMC" in deps || Pkg.develop(; path = dirname(@__DIR__)) + "BenchmarkTools" in deps || Pkg.add("BenchmarkTools") "PkgBenchmark" in deps || Pkg.add("PkgBenchmark") end Pkg.instantiate() using PkgBenchmark +using BenchmarkTools: median using Printf const PKG = dirname(@__DIR__) const RESDIR = joinpath(@__DIR__, "results") mkpath(RESDIR) +"Return comparable per-benchmark rows extracted from two benchmark results." +function collect_comparison_rows(target_result, baseline_result) + tg = target_result.benchmarkgroup + bg = baseline_result.benchmarkgroup + rows = NamedTuple[] + for group in sort(collect(keys(tg))) + haskey(bg, group) || continue + for name in sort(collect(keys(tg[group]))) + haskey(bg[group], name) || continue + local_trial = median(tg[group][name]) + ref_trial = median(bg[group][name]) + push!(rows, ( + group = group, + name = name, + local_ms = local_trial.time / 1e6, + ref_ms = ref_trial.time / 1e6, + local_kib = local_trial.memory / 1024, + ref_kib = ref_trial.memory / 1024, + )) + end + end + return rows +end + +"Weighted summary ratios using local totals as weights." +function summary_metrics(rows) + total_local_ms = sum(row.local_ms for row in rows) + total_ref_ms = sum(row.ref_ms for row in rows) + total_local_kib = sum(row.local_kib for row in rows) + total_ref_kib = sum(row.ref_kib for row in rows) + return ( + matched = length(rows), + local_ms = total_local_ms, + ref_ms = total_ref_ms, + time_ratio = total_local_ms > 0 ? total_ref_ms / total_local_ms : NaN, + local_kib = total_local_kib, + ref_kib = total_ref_kib, + memory_ratio = total_local_kib > 0 ? total_ref_kib / total_local_kib : NaN, + ) +end + """ write_comparison_md(outfile, target_result, baseline_result, target_label, baseline_label) @@ -45,8 +88,8 @@ Ratio = reference (`baseline_label`) time ÷ local (`target_label`) time. """ function write_comparison_md(outfile, target_result, baseline_result, target_label, baseline_label) - tg = target_result.benchmarkgroup - bg = baseline_result.benchmarkgroup + rows = collect_comparison_rows(target_result, baseline_result) + summary = summary_metrics(rows) open(outfile, "w") do io println(io, "# Benchmark: `$(target_label)` vs `$(baseline_label)`\n") println(io, "| | local (`$(target_label)`) | reference (`$(baseline_label)`) |") @@ -57,21 +100,24 @@ function write_comparison_md(outfile, target_result, baseline_result, println(io, "**`ratio = reference time ÷ local time`** ") println(io, "ratio `< 1` → local is **slower** ❌ | ratio `> 1` → local is **faster** ✅") println(io, "") + println(io, "## Aggregate Summary\n") + println(io, "| metric | ratio | local total | reference total |") + println(io, "|:-------|------:|------------:|----------------:|") + println(io, "| matched benchmarks | $(summary.matched) | — | — |") + @printf(io, "| weighted time ratio | %.3f | %.3f ms | %.3f ms |\n", + summary.time_ratio, summary.local_ms, summary.ref_ms) + @printf(io, "| weighted memory ratio | %.3f | %.1f KiB | %.1f KiB |\n\n", + summary.memory_ratio, summary.local_kib, summary.ref_kib) println(io, "| benchmark | ratio | verdict | local (ms) | reference (ms) |") println(io, "|:----------|------:|:-------:|----------:|---------------:|") - for group in sort(collect(keys(tg))) - haskey(bg, group) || continue - for name in sort(collect(keys(tg[group]))) - haskey(bg[group], name) || continue - local_ms = median(tg[group][name]).time / 1e6 - ref_ms = median(bg[group][name]).time / 1e6 - ratio = ref_ms / local_ms - verdict = ratio < 0.95 ? "❌" : ratio > 1.05 ? "✅" : "–" - @printf(io, "| `[\"%s\", \"%s\"]` | %.3f | %s | %.3f | %.3f |\n", - group, name, ratio, verdict, local_ms, ref_ms) - end + for row in rows + ratio = row.ref_ms / row.local_ms + verdict = ratio < 0.95 ? "❌" : ratio > 1.05 ? "✅" : "–" + @printf(io, "| `[\"%s\", \"%s\"]` | %.3f | %s | %.3f | %.3f |\n", + row.group, row.name, ratio, verdict, row.local_ms, row.ref_ms) end end + return summary end "Benchmark the current working tree in place (uncommitted changes included)." @@ -135,15 +181,23 @@ elseif length(ARGS) == 1 target = bench_worktree() # current tree (dirty OK) baseline = bench_revision(baseline_rev) outfile = joinpath(RESDIR, "compare_head.md") - write_comparison_md(outfile, target, baseline, "local", baseline_rev) + summary = write_comparison_md(outfile, target, baseline, "local", baseline_rev) println("\nWrote benchmark/results/compare_head.md (local vs $(baseline_rev))") println(" ratio = $(baseline_rev) ÷ local → < 1: local slower | > 1: local faster") + @printf(" weighted time ratio = %.3f (%s total %.3f ms vs %s total %.3f ms)\n", + summary.time_ratio, baseline_rev, summary.ref_ms, "local", summary.local_ms) + @printf(" weighted memory ratio = %.3f (%s total %.1f KiB vs %s total %.1f KiB)\n", + summary.memory_ratio, baseline_rev, summary.ref_kib, "local", summary.local_kib) else target_rev, baseline_rev = ARGS[1], ARGS[2] target = bench_revision(target_rev) baseline = bench_revision(baseline_rev) outfile = joinpath(RESDIR, "compare_head.md") - write_comparison_md(outfile, target, baseline, target_rev, baseline_rev) + summary = write_comparison_md(outfile, target, baseline, target_rev, baseline_rev) println("\nWrote benchmark/results/compare_head.md ($(target_rev) vs $(baseline_rev))") println(" ratio = $(baseline_rev) ÷ $(target_rev) → < 1: $(target_rev) slower | > 1: $(target_rev) faster") + @printf(" weighted time ratio = %.3f (%s total %.3f ms vs %s total %.3f ms)\n", + summary.time_ratio, baseline_rev, summary.ref_ms, target_rev, summary.local_ms) + @printf(" weighted memory ratio = %.3f (%s total %.1f KiB vs %s total %.1f KiB)\n", + summary.memory_ratio, baseline_rev, summary.ref_kib, target_rev, summary.local_kib) end diff --git a/benchmark/compare_py.jl b/benchmark/compare_py.jl index 476e361..92acba2 100644 --- a/benchmark/compare_py.jl +++ b/benchmark/compare_py.jl @@ -35,6 +35,63 @@ using Printf resdir = joinpath(@__DIR__, "results") +is_c_kernel_row(name::AbstractString) = occursin(r"Lattice|DigitalNetB2|Halton", name) + +function lookup_py_entry(py_results, group, name) + py_group = get(py_results, Symbol(group), nothing) + py_group === nothing && return nothing + return get(py_group, name, get(py_group, replace(name, r" d=\d+" => s -> " [C]" * s), nothing)) +end + +function collect_comparison_rows(jl_results, py_results) + rows = NamedTuple[] + for group in sort(collect(keys(jl_results))) + for name in sort(collect(keys(jl_results[group]))) + jl_trial = median(jl_results[group][name]) + jl_ms = jl_trial.time / 1e6 + jl_kib = jl_trial.memory / 1024 + py_entry = lookup_py_entry(py_results, group, name) + if py_entry !== nothing && !haskey(py_entry, "error") + push!(rows, ( + group = group, + name = name, + jl_ms = jl_ms, + jl_kib = jl_kib, + py_ms = Float64(py_entry["median_ms"]), + py_error = nothing, + c_kernel = is_c_kernel_row(name), + )) + else + push!(rows, ( + group = group, + name = name, + jl_ms = jl_ms, + jl_kib = jl_kib, + py_ms = nothing, + py_error = py_entry === nothing ? "no Python data" : string(py_entry["error"]), + c_kernel = is_c_kernel_row(name), + )) + end + end + end + return rows +end + +function summary_metrics(rows) + matched = filter(row -> row.py_ms !== nothing, rows) + total_jl_ms = sum(row.jl_ms for row in matched) + total_py_ms = sum(row.py_ms for row in matched) + total_jl_kib = sum(row.jl_kib for row in matched) + return ( + matched = length(matched), + total = length(rows), + jl_ms = total_jl_ms, + py_ms = total_py_ms, + time_ratio = total_jl_ms > 0 ? total_py_ms / total_jl_ms : NaN, + jl_kib = total_jl_kib, + ) +end + jl_label = length(ARGS) >= 1 ? ARGS[1] : "latest" py_label = length(ARGS) >= 2 ? ARGS[2] : jl_label @@ -52,6 +109,8 @@ py_data = JSON3.read(read(py_file, String)) py_results = py_data["results"] py_version = get(py_data, "qmcpy_version", "?") py_python = get(py_data, "python", "?") +rows = collect_comparison_rows(jl_results, py_results) +summary = summary_metrics(rows) println("Julia vs QMCPy benchmark comparison") println(" Julia results : $jl_file") @@ -66,25 +125,15 @@ println("="^length(header)) println(header) println("-"^length(header)) -for group in sort(collect(keys(jl_results))) +for group in sort(unique(row.group for row in rows)) println("\n── $group ──") - py_group = get(py_results, Symbol(group), nothing) - for name in sort(collect(keys(jl_results[group]))) - jl_ms = median(jl_results[group][name]).time / 1e6 - - # Python key may have " [C]" suffix for C-kernel variants; try both. - py_entry = nothing - if py_group !== nothing - py_entry = get(py_group, name, get(py_group, replace(name, r" d=\d+" => s -> " [C]" * s), nothing)) - end - - if py_entry !== nothing && !haskey(py_entry, "error") - py_ms = Float64(py_entry["median_ms"]) - ratio = py_ms / jl_ms - tag = occursin(r"Lattice|DigitalNetB2|Halton", name) ? " [C]" : " " - @printf(" %s %-44s %11.3f %11.3f %6.2fx\n", tag, name, jl_ms, py_ms, ratio) + for row in filter(r -> r.group == group, rows) + if row.py_ms !== nothing + ratio = row.py_ms / row.jl_ms + tag = row.c_kernel ? " [C]" : " " + @printf(" %s %-44s %11.3f %11.3f %6.2fx\n", tag, row.name, row.jl_ms, row.py_ms, ratio) else - @printf(" ??? %-44s %11.3f %11s %7s\n", name, jl_ms, "n/a", "n/a") + @printf(" ??? %-44s %11.3f %11s %7s\n", row.name, row.jl_ms, "n/a", "n/a") end end end @@ -92,6 +141,9 @@ end println() println("="^length(header)) println("ratio > 1: local (Julia) faster than Python | ratio < 1: local (Julia) slower than Python") +@printf("weighted time ratio = %.3f (Python total %.3f ms vs Julia total %.3f ms across %d/%d matched rows)\n", + summary.time_ratio, summary.py_ms, summary.jl_ms, summary.matched, summary.total) +println("weighted memory ratio = n/a (QMCPy results do not record comparable Python memory)") # ── Save markdown file ────────────────────────────────────────────────────────── outfile = joinpath(resdir, "compare_python.md") @@ -110,27 +162,29 @@ open(outfile, "w") do io println(io, "> ⚠️ C-kernel rows `[C]` (Lattice/DigitalNetB2/Halton `gen_samples`) call the same") println(io, "> `qmctoolscl` library on both sides and are **not** a Julia vs Python comparison.") println(io, "") + println(io, "## Aggregate Summary\n") + println(io, "| metric | ratio | Julia total | Python total |") + println(io, "|:-------|------:|------------:|-------------:|") + println(io, "| matched benchmarks | $(summary.matched)/$(summary.total) | — | — |") + @printf(io, "| weighted time ratio | %.3f | %.3f ms | %.3f ms |\n", + summary.time_ratio, summary.jl_ms, summary.py_ms) + @printf(io, "| weighted memory ratio | %s | %.1f KiB | %s |\n\n", + "n/a", summary.jl_kib, "n/a") + println(io, "> Python memory is not summarized here because the QMCPy harness currently records") + println(io, "> timing only; there is no comparable cross-language allocation/RSS measure in the results.") + println(io, "") println(io, "| benchmark | ratio | verdict | Julia (ms) | Python (ms) |") println(io, "|:----------|------:|:-------:|----------:|------------:|") - for group in sort(collect(keys(jl_results))) - py_group = get(py_results, Symbol(group), nothing) - for name in sort(collect(keys(jl_results[group]))) - jl_ms = median(jl_results[group][name]).time / 1e6 - py_entry = nothing - if py_group !== nothing - py_entry = get(py_group, name, get(py_group, replace(name, r" d=\d+" => s -> " [C]" * s), nothing)) - end - c_note = occursin(r"Lattice|DigitalNetB2|Halton", name) ? " `[C]`" : "" - if py_entry !== nothing && !haskey(py_entry, "error") - py_ms = Float64(py_entry["median_ms"]) - ratio = py_ms / jl_ms + for row in rows + c_note = row.c_kernel ? " `[C]`" : "" + if row.py_ms !== nothing + ratio = row.py_ms / row.jl_ms verdict = ratio < 0.95 ? "❌" : ratio > 1.05 ? "✅" : "–" @printf(io, "| `[\"%s\", \"%s\"]`%s | %.3f | %s | %.3f | %.3f |\n", - group, name, c_note, ratio, verdict, jl_ms, py_ms) - else - @printf(io, "| `[\"%s\", \"%s\"]` | n/a | — | %.3f | n/a |\n", - group, name, jl_ms) - end + row.group, row.name, c_note, ratio, verdict, row.jl_ms, row.py_ms) + else + @printf(io, "| `[\"%s\", \"%s\"]` | n/a | — | %.3f | n/a |\n", + row.group, row.name, row.jl_ms) end end end From 93b76ad2bca007602407a58fadf5bed72bf5f916 Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Wed, 3 Jun 2026 09:51:54 -0500 Subject: [PATCH 017/104] remove metadata and add single-run summary function --- benchmark/benchmark_qmcpy.py | 4 ---- benchmark/compare.jl | 19 ++++++++++++++++++- benchmark/compare_py.jl | 10 ++++------ 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/benchmark/benchmark_qmcpy.py b/benchmark/benchmark_qmcpy.py index 563622e..a740f35 100644 --- a/benchmark/benchmark_qmcpy.py +++ b/benchmark/benchmark_qmcpy.py @@ -35,7 +35,6 @@ import sys import json -import time import timeit import warnings import statistics @@ -163,9 +162,6 @@ def run(): outfile = resdir / f"qmcpy_{label}.json" payload = { "qmcpy_version": getattr(qp, "__version__", "?"), - "python": sys.version.split()[0], - "numpy": np.__version__, - "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"), "results": results, } outfile.write_text(json.dumps(payload, indent=2)) diff --git a/benchmark/compare.jl b/benchmark/compare.jl index b361ad7..998ea05 100644 --- a/benchmark/compare.jl +++ b/benchmark/compare.jl @@ -77,6 +77,23 @@ function summary_metrics(rows) ) end +"Write a sanitized single-run benchmark summary without host or path metadata." +function write_single_run_md(outfile, result; label = "local") + groupdata = result.benchmarkgroup + open(outfile, "w") do io + println(io, "# Benchmark: `$(label)`\n") + println(io, "| benchmark | median time (ms) | allocs | memory (KiB) |") + println(io, "|:----------|-----------------:|-------:|-------------:|") + for group in sort(collect(keys(groupdata))) + for name in sort(collect(keys(groupdata[group]))) + trial = median(groupdata[group][name]) + @printf(io, "| `[\"%s\", \"%s\"]` | %.3f | %d | %.1f |\n", + group, name, trial.time / 1e6, trial.allocs, trial.memory / 1024) + end + end + end +end + """ write_comparison_md(outfile, target_result, baseline_result, target_label, baseline_label) @@ -174,7 +191,7 @@ end if length(ARGS) == 0 result = bench_worktree() outfile = joinpath(RESDIR, "bench_local.md") - export_markdown(outfile, result) + write_single_run_md(outfile, result) println("\nWrote benchmark/results/bench_local.md (single-run summary, no comparison)") elseif length(ARGS) == 1 baseline_rev = ARGS[1] diff --git a/benchmark/compare_py.jl b/benchmark/compare_py.jl index 92acba2..8d439f8 100644 --- a/benchmark/compare_py.jl +++ b/benchmark/compare_py.jl @@ -108,13 +108,12 @@ py_data = JSON3.read(read(py_file, String)) py_results = py_data["results"] py_version = get(py_data, "qmcpy_version", "?") -py_python = get(py_data, "python", "?") rows = collect_comparison_rows(jl_results, py_results) summary = summary_metrics(rows) println("Julia vs QMCPy benchmark comparison") -println(" Julia results : $jl_file") -println(" QMCPy results : $py_file (qmcpy $py_version, python $py_python)") +println(" Julia label : $jl_label") +println(" QMCPy label : $py_label (qmcpy $py_version)") println() println(" NOTE: C-kernel rows [C] (Lattice/DigitalNetB2/Halton gen_samples) use the") println(" same qmctoolscl library on both sides and are NOT a language comparison.") @@ -151,10 +150,9 @@ open(outfile, "w") do io println(io, "# Benchmark: Julia (local) vs QMCPy (Python)\n") println(io, "| | Julia (local) | Python (QMCPy) |") println(io, "|---|---|---|") - println(io, "| Julia results | `$(jl_file)` | — |") - println(io, "| Python results | — | `$(py_file)` |") + println(io, "| Julia label | `$(jl_label)` | — |") + println(io, "| Python label | — | `$(py_label)` |") println(io, "| qmcpy version | — | $(py_version) |") - println(io, "| python | — | $(py_python) |") println(io, "") println(io, "**`ratio = Python time ÷ Julia time`** ") println(io, "ratio `< 1` → local (Julia) is **slower** ❌ | ratio `> 1` → local (Julia) is **faster** ✅ ") From 642c42365e2abe6e855a7a1cb04d0885bdef7a0a Mon Sep 17 00:00:00 2001 From: sou-cheng-choi Date: Wed, 3 Jun 2026 10:06:54 -0500 Subject: [PATCH 018/104] Support custom output file labels --- Makefile | 29 +++++++++++++++++++++++------ benchmark/compare.jl | 30 ++++++++++++++++++++++++------ benchmark/compare_py.jl | 10 +++++++--- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 1ff445a..07fa333 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,19 @@ FORMATTER_PROJECT=devtools/formatter +# Allow `make bench gaus` / `make bench-compare gaus` / `make bench-compare-py gaus` +# as shorthand for `LABEL=gaus`. `make` treats `gaus` as an extra goal, so consume +# it explicitly. +ifneq ($(filter bench bench-compare bench-compare-py,$(firstword $(MAKECMDGOALS))),) + EXTRA_BENCH_GOALS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) + ifneq ($(strip $(EXTRA_BENCH_GOALS)),) + LABEL ?= $(firstword $(EXTRA_BENCH_GOALS)) + .PHONY: $(EXTRA_BENCH_GOALS) + $(EXTRA_BENCH_GOALS): + @: + endif +endif + # Update packages and resolve dependencies update: julia --project=. -e 'using Pkg; Pkg.update(); Pkg.resolve' @@ -49,24 +62,28 @@ notebook-%: julia --project=. test/run_notebooks.jl $* # Run the benchmark suite (uses its own environment in benchmark/, set up on first run) -# Saves results to benchmark/results/latest.json +# Override output label with: make bench LABEL=gaus or make bench gaus +# Saves results to benchmark/results/latest.json (default) or