From 8e2ec8d2091e3b342bfe1c04ad67c97c177fefb3 Mon Sep 17 00:00:00 2001 From: RyanAugust Date: Thu, 30 Apr 2026 16:55:40 +0000 Subject: [PATCH 1/4] sensitivity analysis --- README.md | 8 ++- src/pysimmmulator/command_line.py | 6 +-- src/pysimmmulator/param_handlers.py | 19 +++++++ src/pysimmmulator/simulate.py | 79 ++++++++++++++++++++--------- tests/test_edge_cases.py | 11 ++-- tests/test_exogenous_factors.py | 6 +-- tests/test_geo_pipeline.py | 38 +++++++------- tests/test_multisim.py | 3 +- tests/test_sensitivity.py | 63 +++++++++++++++++++++++ tests/test_sim.py | 7 +-- tests/test_viz.py | 4 +- 11 files changed, 183 insertions(+), 61 deletions(-) create mode 100644 tests/test_sensitivity.py diff --git a/README.md b/README.md index f7ce7b6..252af62 100644 --- a/README.md +++ b/README.md @@ -24,14 +24,18 @@ PySiMMMulator's simulator can either be run on a step-by-step basis, or can be r ### Run via config -Run using this method, you'll be returned both a dataframe of for MMM input as well as the "True ROI" values for each of your channels. These true values are critical to validating your MMM model. +Run using this method, you'll be returned a `SimulationResult` object containing both a dataframe for MMM input as well as the "True ROI" values for each of your channels, and associated metadata. These true values are critical to validating your MMM model. ```python from pysimmmulator import load_config, Simulate cfg = load_config(config_path="./my_config.yaml") simmm = Simulate() -mmm_input_df, channel_roi = simmm.run_with_config(config=cfg) +result = simmm.run_with_config(config=cfg) + +# Access results +mmm_input_df = result.df +channel_roi = result.channel_roi ``` ### Run via CLI diff --git a/src/pysimmmulator/command_line.py b/src/pysimmmulator/command_line.py index 3e774ea..9495cfb 100644 --- a/src/pysimmmulator/command_line.py +++ b/src/pysimmmulator/command_line.py @@ -18,13 +18,13 @@ def run_with_config(config_path, output_path): cfg = load_config(config_path) logger.debug("config loaded successfully") sim = Simulate() - (mmm_input_df, channel_roi) = sim.run_with_config(config=cfg) + result = sim.run_with_config(config=cfg) logger.debug("sim run successfully, saving results") # save to current directory. Should be an optional argument for this os.makedirs(output_path, exist_ok=True) - mmm_input_df.to_csv(os.path.join(output_path, "mmm_input_df.csv"), index=False) - pd.DataFrame.from_dict(channel_roi, orient="index", columns=["true_roi"]).to_csv(os.path.join(output_path, "channel_roi.csv")) + result.df.to_csv(os.path.join(output_path, "mmm_input_df.csv"), index=True) + pd.DataFrame.from_dict(result.channel_roi, orient="index", columns=["true_roi"]).to_csv(os.path.join(output_path, "channel_roi.csv")) def main(): arg_parser = argparse.ArgumentParser() diff --git a/src/pysimmmulator/param_handlers.py b/src/pysimmmulator/param_handlers.py index c5273b8..341da58 100644 --- a/src/pysimmmulator/param_handlers.py +++ b/src/pysimmmulator/param_handlers.py @@ -1,6 +1,7 @@ from typing import Optional, Union from dataclasses import dataclass import datetime +import pandas as pd @dataclass class BasicParameters: @@ -283,3 +284,21 @@ class GeoParameters: def __post_init__(self): assert self.total_population > 0, "Total population must be greater than 0" + +@dataclass +class SimulationResult: + """Object for holding the results and metadata of a simulation run. + + Args: + df (pd.DataFrame): Final simulation DataFrame. + channel_roi (dict): Ground-truth ROI values per channel. + config (dict): The configuration dictionary used for the run. + random_state (object): The bit generator state of the RNG used. + """ + df: pd.DataFrame + channel_roi: dict + config: dict + random_state: object + + def __repr__(self): + return f"SimulationResult(rows={len(self.df)}, channels={list(self.channel_roi.keys())})" diff --git a/src/pysimmmulator/simulate.py b/src/pysimmmulator/simulate.py index f3a4d68..eaa3036 100644 --- a/src/pysimmmulator/simulate.py +++ b/src/pysimmmulator/simulate.py @@ -7,6 +7,7 @@ AdstockParameters, OutputParameters, GeoParameters, + SimulationResult, ) from .visualize import Visualize @@ -503,18 +504,18 @@ def finalize_output(self, mmm_df: pd.DataFrame, params: OutputParameters) -> pd. logger.info(f"You have completed running step 9: Finalization of output dataframe at the {params.aggregation_level} level") return final_df - def run_with_config(self, config: dict) -> tuple[pd.DataFrame, dict]: + def run_with_config(self, config: dict) -> SimulationResult: """Orchestrates the full simulation pipeline using a configuration dictionary. - This method handles parameter instantiation, baseline simulation, media and CVR - simulation, adstock/saturation, conversion calculation, and optional + This method handles parameter instantiation, baseline simulation, media and CVR + simulation, adstock/saturation, conversion calculation, and optional geographic distribution. Args: config (dict): Complete configuration dictionary. Returns: - tuple[pd.DataFrame, dict]: Finalized simulation DataFrame and a dictionary - of ground-truth ROI values per channel.""" + SimulationResult: Object containing the output DataFrame, ground-truth ROI, + configuration used, and random state metadata.""" from .load_parameters import create_all_parameters params = create_all_parameters(config) self.basic_params = params["basic_params"] @@ -533,41 +534,73 @@ def run_with_config(self, config: dict) -> tuple[pd.DataFrame, dict]: channel_roi = self.calculate_channel_roi(mmm_df=mmm_df) final_df = self.finalize_output(mmm_df=mmm_df, params=params["output_params"]) - return (final_df, channel_roi) + return SimulationResult( + df=final_df, + channel_roi=channel_roi, + config=config, + random_state=self._report_random_state() + ) class Multisim(Simulate): """Provides capability to generate multiple runs on a single configuration""" - def __init__(self): - super(Multisim, self).__init__() - self.final_frames = [] - self.rois = [] + def __init__(self, random_seed=None): + super(Multisim, self).__init__(random_seed=random_seed) + self.results = [] - def stash_outputs(self, final_df: pd.DataFrame, channel_roi: dict): + def stash_outputs(self, result: SimulationResult): """Stores the outputs of a single simulation run. Args: - final_df (pd.DataFrame): Final simulation DataFrame. - channel_roi (dict): Ground-truth ROI values.""" - self.final_frames.append(final_df) - self.rois.append(channel_roi) + result (SimulationResult): The result object from run_with_config.""" + self.results.append(result) @property def get_data(self): - """Provides the iterable generator for simulation final dataframes and channel ground truth ROI values + """Provides the list of SimulationResult objects generated. Returns: - data (iterable): iterable of final sim dataframes and channel ROI values""" - return self.data + results (list[SimulationResult]): List of simulation results.""" + return self.results + + def _apply_sensitivity(self, config: dict, sensitivity_config: dict) -> dict: + """Recursively applies sensitivity ranges to a configuration. - def run(self, config: dict, runs: int) -> None: + Args: + config (dict): The base configuration to copy and update. + sensitivity_config (dict): Configuration specifying ranges [low, high] for parameters. + Returns: + dict: A new configuration with sampled values.""" + import copy + new_config = copy.deepcopy(config) + + def recursive_update(target, source): + for key, value in source.items(): + if isinstance(value, dict) and key in target and isinstance(target[key], dict): + recursive_update(target[key], value) + elif isinstance(value, list) and len(value) == 2 and all(isinstance(x, (int, float)) for x in value): + # Sample from Uniform distribution + target[key] = self.rng.uniform(low=value[0], high=value[1]) + else: + target[key] = value + + recursive_update(new_config, sensitivity_config) + return new_config + + def run(self, config: dict, runs: int, sensitivity_config: dict = None) -> None: """Executes multiple simulation runs. Args: config (dict): Simulation configuration. - runs (int): Number of runs to execute.""" + runs (int): Number of runs to execute. + sensitivity_config (dict): Optional configuration for Monte Carlo sensitivity analysis. + """ for run in range(runs): - frame, roi = self.run_with_config(config=config) - self.stash_outputs(final_df=frame, channel_roi=roi) + current_config = config + if sensitivity_config: + current_config = self._apply_sensitivity(config, sensitivity_config) + + result = self.run_with_config(config=current_config) + self.stash_outputs(result=result) logger.info(f"{run + 1}/{runs} completed") - self.data = zip(self.final_frames, self.rois) logger.info(f"{runs} runs complete and stored") + diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index 1703b16..df26948 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -154,7 +154,7 @@ def test_visualize_empty_columns(): def test_multisim_get_data_coverage(): from pysimmmulator.simulate import Multisim ms = Multisim() - ms.data = "test_data" + ms.results = "test_data" assert ms.get_data == "test_data" def test_reproducibility(): @@ -168,10 +168,11 @@ def test_reproducibility(): seed = 42 sim1 = Simulate(random_seed=seed) - df1, roi1 = sim1.run_with_config(config) + result1 = sim1.run_with_config(config) sim2 = Simulate(random_seed=seed) - df2, roi2 = sim2.run_with_config(config) + result2 = sim2.run_with_config(config) + + pd.testing.assert_frame_equal(result1.df, result2.df) + assert result1.channel_roi == result2.channel_roi - pd.testing.assert_frame_equal(df1, df2) - assert roi1 == roi2 diff --git a/tests/test_exogenous_factors.py b/tests/test_exogenous_factors.py index 526d990..cb31b4f 100644 --- a/tests/test_exogenous_factors.py +++ b/tests/test_exogenous_factors.py @@ -96,9 +96,9 @@ def test_us_retail_example_run(): from pysimmmulator.load_parameters import load_config, create_all_parameters cfg = load_config("examples/us_retail_exogenous_config.yaml") sim = Simulate() - df, roi = sim.run_with_config(cfg) - assert len(df) > 0 - assert "baseline_sales" not in df.columns # it's aggregated in total_revenue + result = sim.run_with_config(cfg) + assert len(result.df) > 0 + assert "baseline_sales" not in result.df.columns # it's aggregated in total_revenue # We can check specific dates in the internal baseline if we run it manually params = create_all_parameters(cfg) diff --git a/tests/test_geo_pipeline.py b/tests/test_geo_pipeline.py index 6f80427..2048ea3 100644 --- a/tests/test_geo_pipeline.py +++ b/tests/test_geo_pipeline.py @@ -10,11 +10,11 @@ def base_config(): def test_run_with_config_no_geo(base_config): sim = Simulate() - final_df, channel_roi = sim.run_with_config(base_config) + result = sim.run_with_config(base_config) - assert final_df.index.name == "date" - assert "geo_name" not in final_df.index.names - assert len(final_df) > 0 + assert result.df.index.name == "date" + assert "geo_name" not in result.df.index.names + assert len(result.df) > 0 def test_run_with_config_with_geo(base_config): base_config["geo_params"] = { @@ -24,11 +24,11 @@ def test_run_with_config_with_geo(base_config): } sim = Simulate() - final_df, channel_roi = sim.run_with_config(base_config) + result = sim.run_with_config(base_config) - assert "geo_name" in final_df.index.names - assert "date" in final_df.index.names - geos = final_df.index.get_level_values("geo_name").unique() + assert "geo_name" in result.df.index.names + assert "date" in result.df.index.names + geos = result.df.index.get_level_values("geo_name").unique() assert len(geos) == 5 def test_run_with_config_weekly_geo(base_config): @@ -39,11 +39,11 @@ def test_run_with_config_weekly_geo(base_config): } sim = Simulate() - final_df, channel_roi = sim.run_with_config(base_config) + result = sim.run_with_config(base_config) - assert "geo_name" in final_df.index.names - assert "week_start" in final_df.index.names - geos = final_df.index.get_level_values("geo_name").unique() + assert "geo_name" in result.df.index.names + assert "week_start" in result.df.index.names + geos = result.df.index.get_level_values("geo_name").unique() assert len(geos) == 3 def test_run_with_config_single_geo(base_config): @@ -53,11 +53,11 @@ def test_run_with_config_single_geo(base_config): } sim = Simulate() - final_df, channel_roi = sim.run_with_config(base_config) + result = sim.run_with_config(base_config) - assert "geo_name" in final_df.index.names - assert "date" in final_df.index.names - geos = final_df.index.get_level_values("geo_name").unique() + assert "geo_name" in result.df.index.names + assert "date" in result.df.index.names + geos = result.df.index.get_level_values("geo_name").unique() assert len(geos) == 1 def test_geo_visualization(base_config): @@ -67,14 +67,14 @@ def test_geo_visualization(base_config): } sim = Simulate() - final_df, _ = sim.run_with_config(base_config) + result = sim.run_with_config(base_config) # Test plotting with multi-indexed geo data try: - sim.plot_spend(final_df, agg="weekly") + sim.plot_spend(result.df, agg="weekly") assert os.path.exists("Spend_by_channel.png") - sim.plot_revenue(final_df, agg="monthly") + sim.plot_revenue(result.df, agg="monthly") assert os.path.exists("Revenue_by_channel.png") finally: # Cleanup diff --git a/tests/test_multisim.py b/tests/test_multisim.py index a7b9214..c6bd716 100644 --- a/tests/test_multisim.py +++ b/tests/test_multisim.py @@ -1,4 +1,5 @@ from pysimmmulator import load_parameters, Multisim +import pytest def test_initiate_msim(): Multisim() @@ -8,4 +9,4 @@ def test_multiple_runs(): cfg = load_parameters.load_config(config_path="./examples/example_config.yaml") msim = Multisim() msim.run(config=cfg, runs=10) - assert len(msim.final_frames) == 10 + assert len(msim.get_data) == 10 diff --git a/tests/test_sensitivity.py b/tests/test_sensitivity.py new file mode 100644 index 0000000..97dd943 --- /dev/null +++ b/tests/test_sensitivity.py @@ -0,0 +1,63 @@ +import pytest +import numpy as np +from pysimmmulator import load_config, Multisim + +def test_multisim_sensitivity_variation(): + cfg = load_config("./examples/example_config.yaml") + + # Define ranges for error_std and one channel's lambda + sensitivity_config = { + "baseline_params": { + "error_std": [10.0, 500.0] + }, + "adstock_params": { + "adstock": { + "Amazon": { + "params": { + "lambda": [0.1, 0.9] + } + } + } + } + } + + msim = Multisim(random_seed=42) + msim.run(config=cfg, runs=5, sensitivity_config=sensitivity_config) + + results = msim.get_data + assert len(results) == 5 + + # Verify that parameters actually varied + error_stds = [r.config["baseline_params"]["error_std"] for r in results] + lambdas = [r.config["adstock_params"]["adstock"]["Amazon"]["params"]["lambda"] for r in results] + + # Check uniqueness + assert len(set(error_stds)) == 5 + assert len(set(lambdas)) == 5 + + # Check bounds + for val in error_stds: + assert 10.0 <= val <= 500.0 + for val in lambdas: + assert 0.1 <= val <= 0.9 + +def test_multisim_sensitivity_reproducibility(): + cfg = load_config("./examples/example_config.yaml") + sensitivity_config = { + "baseline_params": { + "error_std": [50.0, 100.0] + } + } + + seed = 123 + msim1 = Multisim(random_seed=seed) + msim1.run(cfg, runs=3, sensitivity_config=sensitivity_config) + results1 = msim1.get_data + + msim2 = Multisim(random_seed=seed) + msim2.run(cfg, runs=3, sensitivity_config=sensitivity_config) + results2 = msim2.get_data + + for r1, r2 in zip(results1, results2): + assert r1.config["baseline_params"]["error_std"] == r2.config["baseline_params"]["error_std"] + np.testing.assert_array_equal(r1.df.values, r2.df.values) diff --git a/tests/test_sim.py b/tests/test_sim.py index 400652e..cbac693 100644 --- a/tests/test_sim.py +++ b/tests/test_sim.py @@ -104,8 +104,9 @@ def test_run_with_config_weekly(config): sim.run_with_config(config=config) -def test_run_with_high_frequency(config): +def test_run_with_high_frequency(params): + config = load_parameters.load_config(config_path="./examples/example_config.yaml") config["basic_params"]["frequency_of_campaigns"] = 50 sim = Simulate() - final_df, _ = sim.run_with_config(config=config) - assert final_df.dropna().shape[0] > final_df.shape[0] - 5 + result = sim.run_with_config(config=config) + assert result.df.dropna().shape[0] > result.df.shape[0] - 5 diff --git a/tests/test_viz.py b/tests/test_viz.py index 633c7b6..14f3155 100644 --- a/tests/test_viz.py +++ b/tests/test_viz.py @@ -6,8 +6,8 @@ def sim_output(): cfg = pysimmm.load_parameters.load_config(config_path="./examples/example_config.yaml") sim = pysimmm.Simulate() - final_df, _ = sim.run_with_config(config=cfg) - return sim, final_df + result = sim.run_with_config(config=cfg) + return sim, result.df def test_viz_clicks_daily(sim_output): sim, df = sim_output From 347457a4ca2247249847c31874719aba29d4d928 Mon Sep 17 00:00:00 2001 From: RyanAugust Date: Thu, 30 Apr 2026 18:06:39 +0000 Subject: [PATCH 2/4] docs update --- README.md | 60 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 252af62..6515006 100644 --- a/README.md +++ b/README.md @@ -51,21 +51,61 @@ pysimmm -i example_config.yaml -o . Alternatively you may run each of the stages independently, which allows for easier debugging and in-run adjustments. Due to the stateless architecture, each stage returns its results which are then passed to the next stage. ```python -from pysimmmulator import load_config, Simulate, define_basic_params +from pysimmmulator import load_config, Simulate, define_basic_params, create_all_parameters cfg = load_config("./my_config.yaml") -basic_params = define_basic_params(**cfg["basic_params"]) -simmm = Simulate(basic_params) - -baseline_df = simmm.simulate_baseline(**cfg["baseline_params"]) -spend_df = simmm.simulate_ad_spend(baseline_sales_df=baseline_df, **cfg["ad_spend_params"]) -spend_df = simmm.simulate_media(spend_df=spend_df, **cfg["media_params"]) -spend_df = simmm.simulate_cvr(spend_df=spend_df, **cfg["cvr_params"]) -mmm_df = simmm.simulate_decay_returns(spend_df=spend_df, **cfg["adstock_params"]) +params = create_all_parameters(cfg) +simmm = Simulate(params["basic_params"]) + +baseline_df = simmm.simulate_baseline(params["baseline_params"]) +spend_df = simmm.simulate_ad_spend(baseline_sales_df=baseline_df, params=params["ad_spend_params"]) +spend_df = simmm.simulate_media(spend_df=spend_df, params=params["media_params"]) +spend_df = simmm.simulate_cvr(spend_df=spend_df, params=params["cvr_params"]) +mmm_df = simmm.simulate_decay_returns(spend_df=spend_df, params=params["adstock_params"]) mmm_df = simmm.calculate_conversions(mmm_df=mmm_df) mmm_df = simmm.consolidate_dataframe(mmm_df=mmm_df, baseline_sales_df=baseline_df) channel_roi = simmm.calculate_channel_roi(mmm_df=mmm_df) -final_df = simmm.finalize_output(mmm_df=mmm_df, **cfg["output_params"]) +final_df = simmm.finalize_output(mmm_df=mmm_df, params=params["output_params"]) +``` + +### Exogenous Factors + +PySiMMMulator supports the inclusion of external shocks, holidays, and promotions. These can be specified as either multipliers or additive impacts within the `baseline_params` block. + +```yaml +baseline_params: + ... + exogenous_factors: + - name: "Black Friday" + dates: ["2023-11-24"] + impact: 3.5 + type: "multiplier" + - name: "Christmas Peak" + start_date: "2023-12-20" + end_date: "2023-12-24" + impact: 2.0 + type: "multiplier" +``` + +### Automated Sensitivity Analysis (Monte Carlo) + +The `Multisim` class enables Monte Carlo simulations by allowing you to define uncertainty ranges for any configuration parameter. This helps researchers understand how sensitive an MMM is to data volatility. + +```python +from pysimmmulator import Multisim, load_config + +base_cfg = load_config("my_config.yaml") +sensitivity_config = { + "baseline_params": { + "error_std": [20.0, 150.0] # sample noise level for each run + } +} + +msim = Multisim(random_seed=42) +msim.run(config=base_cfg, runs=100, sensitivity_config=sensitivity_config) + +# results is a list of SimulationResult objects +results = msim.get_data ``` ### Geographic distribution From f277109691a46b9d31b635381cf6d8242c954497 Mon Sep 17 00:00:00 2001 From: RyanAugust Date: Thu, 30 Apr 2026 18:16:07 +0000 Subject: [PATCH 3/4] linting --- src/pysimmmulator/simulate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pysimmmulator/simulate.py b/src/pysimmmulator/simulate.py index eaa3036..2133ea2 100644 --- a/src/pysimmmulator/simulate.py +++ b/src/pysimmmulator/simulate.py @@ -598,7 +598,6 @@ def run(self, config: dict, runs: int, sensitivity_config: dict = None) -> None: current_config = config if sensitivity_config: current_config = self._apply_sensitivity(config, sensitivity_config) - result = self.run_with_config(config=current_config) self.stash_outputs(result=result) logger.info(f"{run + 1}/{runs} completed") From bff5b9e7e628dbee0b42dd4ded3587de9393485d Mon Sep 17 00:00:00 2001 From: RyanAugust Date: Thu, 30 Apr 2026 18:22:15 +0000 Subject: [PATCH 4/4] ruff done --- src/pysimmmulator/simulate.py | 6 +++--- tests/test_multisim.py | 1 - tests/test_sensitivity.py | 19 +++++++++---------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/pysimmmulator/simulate.py b/src/pysimmmulator/simulate.py index 2133ea2..0681448 100644 --- a/src/pysimmmulator/simulate.py +++ b/src/pysimmmulator/simulate.py @@ -507,14 +507,14 @@ def finalize_output(self, mmm_df: pd.DataFrame, params: OutputParameters) -> pd. def run_with_config(self, config: dict) -> SimulationResult: """Orchestrates the full simulation pipeline using a configuration dictionary. - This method handles parameter instantiation, baseline simulation, media and CVR - simulation, adstock/saturation, conversion calculation, and optional + This method handles parameter instantiation, baseline simulation, media and CVR + simulation, adstock/saturation, conversion calculation, and optional geographic distribution. Args: config (dict): Complete configuration dictionary. Returns: - SimulationResult: Object containing the output DataFrame, ground-truth ROI, + SimulationResult: Object containing the output DataFrame, ground-truth ROI, configuration used, and random state metadata.""" from .load_parameters import create_all_parameters params = create_all_parameters(config) diff --git a/tests/test_multisim.py b/tests/test_multisim.py index c6bd716..c9dbbfc 100644 --- a/tests/test_multisim.py +++ b/tests/test_multisim.py @@ -1,5 +1,4 @@ from pysimmmulator import load_parameters, Multisim -import pytest def test_initiate_msim(): Multisim() diff --git a/tests/test_sensitivity.py b/tests/test_sensitivity.py index 97dd943..4b3bcab 100644 --- a/tests/test_sensitivity.py +++ b/tests/test_sensitivity.py @@ -1,10 +1,9 @@ -import pytest import numpy as np from pysimmmulator import load_config, Multisim def test_multisim_sensitivity_variation(): cfg = load_config("./examples/example_config.yaml") - + # Define ranges for error_std and one channel's lambda sensitivity_config = { "baseline_params": { @@ -20,21 +19,21 @@ def test_multisim_sensitivity_variation(): } } } - + msim = Multisim(random_seed=42) msim.run(config=cfg, runs=5, sensitivity_config=sensitivity_config) - + results = msim.get_data assert len(results) == 5 - + # Verify that parameters actually varied error_stds = [r.config["baseline_params"]["error_std"] for r in results] lambdas = [r.config["adstock_params"]["adstock"]["Amazon"]["params"]["lambda"] for r in results] - + # Check uniqueness assert len(set(error_stds)) == 5 assert len(set(lambdas)) == 5 - + # Check bounds for val in error_stds: assert 10.0 <= val <= 500.0 @@ -48,16 +47,16 @@ def test_multisim_sensitivity_reproducibility(): "error_std": [50.0, 100.0] } } - + seed = 123 msim1 = Multisim(random_seed=seed) msim1.run(cfg, runs=3, sensitivity_config=sensitivity_config) results1 = msim1.get_data - + msim2 = Multisim(random_seed=seed) msim2.run(cfg, runs=3, sensitivity_config=sensitivity_config) results2 = msim2.get_data - + for r1, r2 in zip(results1, results2): assert r1.config["baseline_params"]["error_std"] == r2.config["baseline_params"]["error_std"] np.testing.assert_array_equal(r1.df.values, r2.df.values)