From dba2bd5e530db95de2332e439b3ddec1c8dad708 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 18 Jan 2026 18:05:24 +0100 Subject: [PATCH 1/8] add support for unified search space --- src/hyperactive/base/_optimizer.py | 7 + .../opt/_adapters/_base_optuna_adapter.py | 40 +++- src/hyperactive/opt/_adapters/_gfo.py | 21 ++ .../opt/gfo/_bayesian_optimization.py | 2 + .../opt/gfo/_differential_evolution.py | 2 + src/hyperactive/opt/gfo/_direct_algorithm.py | 2 + src/hyperactive/opt/gfo/_downhill_simplex.py | 2 + .../opt/gfo/_evolution_strategy.py | 2 + src/hyperactive/opt/gfo/_forest_optimizer.py | 2 + src/hyperactive/opt/gfo/_genetic_algorithm.py | 2 + src/hyperactive/opt/gfo/_grid_search.py | 2 + src/hyperactive/opt/gfo/_hillclimbing.py | 2 + .../opt/gfo/_lipschitz_optimization.py | 2 + .../opt/gfo/_parallel_tempering.py | 2 + .../opt/gfo/_particle_swarm_optimization.py | 2 + src/hyperactive/opt/gfo/_pattern_search.py | 2 + src/hyperactive/opt/gfo/_powells_method.py | 2 + .../opt/gfo/_random_restart_hill_climbing.py | 2 + src/hyperactive/opt/gfo/_random_search.py | 2 + .../opt/gfo/_repulsing_hillclimbing.py | 2 + .../opt/gfo/_simulated_annealing.py | 2 + .../opt/gfo/_spiral_optimization.py | 2 + .../opt/gfo/_stochastic_hillclimbing.py | 2 + .../gfo/_tree_structured_parzen_estimators.py | 2 + src/hyperactive/opt/gridsearch/_sk.py | 30 +++ .../opt/optuna/_cmaes_optimizer.py | 2 + src/hyperactive/opt/optuna/_gp_optimizer.py | 2 + src/hyperactive/opt/optuna/_grid_optimizer.py | 17 +- .../opt/optuna/_nsga_ii_optimizer.py | 2 + .../opt/optuna/_nsga_iii_optimizer.py | 2 + src/hyperactive/opt/optuna/_qmc_optimizer.py | 2 + .../opt/optuna/_random_optimizer.py | 2 + src/hyperactive/opt/optuna/_tpe_optimizer.py | 2 + .../tests/test_unified_search_space.py | 218 ++++++++++++++++++ 34 files changed, 380 insertions(+), 9 deletions(-) create mode 100644 src/hyperactive/tests/test_unified_search_space.py diff --git a/src/hyperactive/base/_optimizer.py b/src/hyperactive/base/_optimizer.py index 791ef91f..ed44cc94 100644 --- a/src/hyperactive/base/_optimizer.py +++ b/src/hyperactive/base/_optimizer.py @@ -18,6 +18,13 @@ class BaseOptimizer(BaseObject): "info:compute": "middle", # "low", "middle", "high" # see here for explanation of the tags: # https://simonblanke.github.io/gradient-free-optimizers-documentation/1.5/optimizers/ # noqa: E501 + # search space capabilities (conservative defaults) + "capability:discrete": True, # supports discrete lists + "capability:continuous": False, # supports continuous ranges + "capability:categorical": True, # supports categorical choices + "capability:log_scale": False, # supports log-scale sampling + "capability:conditions": False, # supports conditional params + "capability:constraints": False, # supports constraint functions } def __init__(self): diff --git a/src/hyperactive/opt/_adapters/_base_optuna_adapter.py b/src/hyperactive/opt/_adapters/_base_optuna_adapter.py index 8dd3171f..38645944 100644 --- a/src/hyperactive/opt/_adapters/_base_optuna_adapter.py +++ b/src/hyperactive/opt/_adapters/_base_optuna_adapter.py @@ -12,10 +12,16 @@ class _BaseOptunaAdapter(BaseOptimizer): _tags = { "python_dependencies": ["optuna"], "info:name": "Optuna-based optimizer", + # Search space capabilities + "capability:discrete": True, + "capability:continuous": True, + "capability:categorical": True, + "capability:log_scale": True, } def __init__( self, + unified_space=None, param_space=None, n_trials=100, initialize=None, @@ -25,6 +31,7 @@ def __init__( experiment=None, **optimizer_kwargs, ): + self.unified_space = unified_space self.param_space = param_space self.n_trials = n_trials self.initialize = initialize @@ -35,6 +42,34 @@ def __init__( self.optimizer_kwargs = optimizer_kwargs super().__init__() + def get_search_config(self): + """Get the search configuration. + + Returns + ------- + dict with str keys + The search configuration dictionary. + """ + search_config = super().get_search_config() + + # Resolve: unified_space is converted to param_space + unified_space = search_config.pop("unified_space", None) + param_space = search_config.get("param_space") + + # Validate: only one should be set + if unified_space is not None and param_space is not None: + raise ValueError( + "Provide either 'unified_space' or 'param_space', not both. " + "Use 'unified_space' for simple dict[str, list] format, " + "or 'param_space' for native Optuna format with ranges/distributions." + ) + + # Use unified_space if param_space is not set + if unified_space is not None: + search_config["param_space"] = unified_space + + return search_config + def _get_optimizer(self): """Get the Optuna optimizer to use. @@ -109,7 +144,7 @@ def _objective(self, trial): float The objective value """ - params = self._suggest_params(trial, self.param_space) + params = self._suggest_params(trial, self._resolved_param_space) score = self.experiment(params) # Handle early stopping based on max_score @@ -157,6 +192,9 @@ def _solve(self, experiment, param_space, n_trials, **kwargs): """ import optuna + # Store resolved param_space for use in _objective + self._resolved_param_space = param_space + # Create optimizer with random state if provided optimizer = self._get_optimizer() diff --git a/src/hyperactive/opt/_adapters/_gfo.py b/src/hyperactive/opt/_adapters/_gfo.py index 5edd81c9..822e5cdc 100644 --- a/src/hyperactive/opt/_adapters/_gfo.py +++ b/src/hyperactive/opt/_adapters/_gfo.py @@ -23,6 +23,11 @@ class _BaseGFOadapter(BaseOptimizer): _tags = { "authors": "SimonBlanke", "python_dependencies": ["gradient-free-optimizers>=1.5.0"], + # Search space capabilities + "capability:discrete": True, + "capability:continuous": True, + "capability:categorical": True, + "capability:constraints": True, } def __init__(self): @@ -55,6 +60,22 @@ def get_search_config(self): search_config["initialize"] = self._initialize del search_config["verbose"] + # Resolve: unified_space is converted to search_space + unified_space = search_config.pop("unified_space", None) + search_space = search_config.get("search_space") + + # Validate: only one should be set + if unified_space is not None and search_space is not None: + raise ValueError( + "Provide either 'unified_space' or 'search_space', not both. " + "Use 'unified_space' for simple dict[str, list] format, " + "or 'search_space' for native GFO format." + ) + + # Use unified_space if search_space is not set + if unified_space is not None: + search_config["search_space"] = unified_space + search_config = self._handle_gfo_defaults(search_config) search_config["search_space"] = self._to_dict_np(search_config["search_space"]) diff --git a/src/hyperactive/opt/gfo/_bayesian_optimization.py b/src/hyperactive/opt/gfo/_bayesian_optimization.py index c579d2eb..b0b52a76 100644 --- a/src/hyperactive/opt/gfo/_bayesian_optimization.py +++ b/src/hyperactive/opt/gfo/_bayesian_optimization.py @@ -87,6 +87,7 @@ class BayesianOptimizer(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -101,6 +102,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p diff --git a/src/hyperactive/opt/gfo/_differential_evolution.py b/src/hyperactive/opt/gfo/_differential_evolution.py index 72cd8e09..c7473cd7 100644 --- a/src/hyperactive/opt/gfo/_differential_evolution.py +++ b/src/hyperactive/opt/gfo/_differential_evolution.py @@ -81,6 +81,7 @@ class DifferentialEvolution(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -93,6 +94,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.population = population diff --git a/src/hyperactive/opt/gfo/_direct_algorithm.py b/src/hyperactive/opt/gfo/_direct_algorithm.py index da2703f2..61cf05e0 100644 --- a/src/hyperactive/opt/gfo/_direct_algorithm.py +++ b/src/hyperactive/opt/gfo/_direct_algorithm.py @@ -83,6 +83,7 @@ class DirectAlgorithm(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -96,6 +97,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.warm_start_smbo = warm_start_smbo diff --git a/src/hyperactive/opt/gfo/_downhill_simplex.py b/src/hyperactive/opt/gfo/_downhill_simplex.py index df0f47b1..225dbbdd 100644 --- a/src/hyperactive/opt/gfo/_downhill_simplex.py +++ b/src/hyperactive/opt/gfo/_downhill_simplex.py @@ -83,6 +83,7 @@ class DownhillSimplexOptimizer(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -96,6 +97,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.alpha = alpha diff --git a/src/hyperactive/opt/gfo/_evolution_strategy.py b/src/hyperactive/opt/gfo/_evolution_strategy.py index 0080f383..9f1a498e 100644 --- a/src/hyperactive/opt/gfo/_evolution_strategy.py +++ b/src/hyperactive/opt/gfo/_evolution_strategy.py @@ -87,6 +87,7 @@ class EvolutionStrategy(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -101,6 +102,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.population = population diff --git a/src/hyperactive/opt/gfo/_forest_optimizer.py b/src/hyperactive/opt/gfo/_forest_optimizer.py index e4c2b68a..9551cd0b 100644 --- a/src/hyperactive/opt/gfo/_forest_optimizer.py +++ b/src/hyperactive/opt/gfo/_forest_optimizer.py @@ -89,6 +89,7 @@ class ForestOptimizer(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -105,6 +106,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.warm_start_smbo = warm_start_smbo diff --git a/src/hyperactive/opt/gfo/_genetic_algorithm.py b/src/hyperactive/opt/gfo/_genetic_algorithm.py index 7dc08269..bd3850c5 100644 --- a/src/hyperactive/opt/gfo/_genetic_algorithm.py +++ b/src/hyperactive/opt/gfo/_genetic_algorithm.py @@ -87,6 +87,7 @@ class GeneticAlgorithm(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -102,6 +103,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.population = population diff --git a/src/hyperactive/opt/gfo/_grid_search.py b/src/hyperactive/opt/gfo/_grid_search.py index 222af122..203a0505 100644 --- a/src/hyperactive/opt/gfo/_grid_search.py +++ b/src/hyperactive/opt/gfo/_grid_search.py @@ -79,6 +79,7 @@ class GridSearch(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -90,6 +91,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.step_size = step_size diff --git a/src/hyperactive/opt/gfo/_hillclimbing.py b/src/hyperactive/opt/gfo/_hillclimbing.py index 5a0a3074..13672ea2 100644 --- a/src/hyperactive/opt/gfo/_hillclimbing.py +++ b/src/hyperactive/opt/gfo/_hillclimbing.py @@ -87,6 +87,7 @@ class HillClimbing(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -99,6 +100,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.epsilon = epsilon diff --git a/src/hyperactive/opt/gfo/_lipschitz_optimization.py b/src/hyperactive/opt/gfo/_lipschitz_optimization.py index 3940fb44..2216b23f 100644 --- a/src/hyperactive/opt/gfo/_lipschitz_optimization.py +++ b/src/hyperactive/opt/gfo/_lipschitz_optimization.py @@ -83,6 +83,7 @@ class LipschitzOptimizer(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -96,6 +97,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.warm_start_smbo = warm_start_smbo diff --git a/src/hyperactive/opt/gfo/_parallel_tempering.py b/src/hyperactive/opt/gfo/_parallel_tempering.py index 39e4c678..8d2ee989 100644 --- a/src/hyperactive/opt/gfo/_parallel_tempering.py +++ b/src/hyperactive/opt/gfo/_parallel_tempering.py @@ -82,6 +82,7 @@ class ParallelTempering(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -93,6 +94,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.population = population diff --git a/src/hyperactive/opt/gfo/_particle_swarm_optimization.py b/src/hyperactive/opt/gfo/_particle_swarm_optimization.py index 59d5785d..9109eabe 100644 --- a/src/hyperactive/opt/gfo/_particle_swarm_optimization.py +++ b/src/hyperactive/opt/gfo/_particle_swarm_optimization.py @@ -87,6 +87,7 @@ class ParticleSwarmOptimizer(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -101,6 +102,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.population = population diff --git a/src/hyperactive/opt/gfo/_pattern_search.py b/src/hyperactive/opt/gfo/_pattern_search.py index 4cc6a9b2..df1155d7 100644 --- a/src/hyperactive/opt/gfo/_pattern_search.py +++ b/src/hyperactive/opt/gfo/_pattern_search.py @@ -82,6 +82,7 @@ class PatternSearch(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -94,6 +95,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.n_positions = n_positions diff --git a/src/hyperactive/opt/gfo/_powells_method.py b/src/hyperactive/opt/gfo/_powells_method.py index 8f8502b5..4569efad 100644 --- a/src/hyperactive/opt/gfo/_powells_method.py +++ b/src/hyperactive/opt/gfo/_powells_method.py @@ -82,6 +82,7 @@ class PowellsMethod(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -92,6 +93,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.iters_p_dim = iters_p_dim diff --git a/src/hyperactive/opt/gfo/_random_restart_hill_climbing.py b/src/hyperactive/opt/gfo/_random_restart_hill_climbing.py index 02a8a830..e15a5e1f 100644 --- a/src/hyperactive/opt/gfo/_random_restart_hill_climbing.py +++ b/src/hyperactive/opt/gfo/_random_restart_hill_climbing.py @@ -77,6 +77,7 @@ class RandomRestartHillClimbing(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -90,6 +91,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.epsilon = epsilon diff --git a/src/hyperactive/opt/gfo/_random_search.py b/src/hyperactive/opt/gfo/_random_search.py index de0b68c9..2c6d01bc 100644 --- a/src/hyperactive/opt/gfo/_random_search.py +++ b/src/hyperactive/opt/gfo/_random_search.py @@ -74,6 +74,7 @@ class RandomSearch(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -82,6 +83,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.search_space = search_space self.initialize = initialize diff --git a/src/hyperactive/opt/gfo/_repulsing_hillclimbing.py b/src/hyperactive/opt/gfo/_repulsing_hillclimbing.py index f988dcb5..c23eb3c9 100644 --- a/src/hyperactive/opt/gfo/_repulsing_hillclimbing.py +++ b/src/hyperactive/opt/gfo/_repulsing_hillclimbing.py @@ -89,6 +89,7 @@ class RepulsingHillClimbing(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -102,6 +103,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.epsilon = epsilon diff --git a/src/hyperactive/opt/gfo/_simulated_annealing.py b/src/hyperactive/opt/gfo/_simulated_annealing.py index cb2d54dc..b2b4a201 100644 --- a/src/hyperactive/opt/gfo/_simulated_annealing.py +++ b/src/hyperactive/opt/gfo/_simulated_annealing.py @@ -86,6 +86,7 @@ class SimulatedAnnealing(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -100,6 +101,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.epsilon = epsilon diff --git a/src/hyperactive/opt/gfo/_spiral_optimization.py b/src/hyperactive/opt/gfo/_spiral_optimization.py index 09e24e8c..bfa519b8 100644 --- a/src/hyperactive/opt/gfo/_spiral_optimization.py +++ b/src/hyperactive/opt/gfo/_spiral_optimization.py @@ -84,6 +84,7 @@ class SpiralOptimization(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -95,6 +96,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.population = population diff --git a/src/hyperactive/opt/gfo/_stochastic_hillclimbing.py b/src/hyperactive/opt/gfo/_stochastic_hillclimbing.py index d10404a4..4a4cdb40 100644 --- a/src/hyperactive/opt/gfo/_stochastic_hillclimbing.py +++ b/src/hyperactive/opt/gfo/_stochastic_hillclimbing.py @@ -89,6 +89,7 @@ class StochasticHillClimbing(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -102,6 +103,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.epsilon = epsilon diff --git a/src/hyperactive/opt/gfo/_tree_structured_parzen_estimators.py b/src/hyperactive/opt/gfo/_tree_structured_parzen_estimators.py index 219f1b00..b5d60614 100644 --- a/src/hyperactive/opt/gfo/_tree_structured_parzen_estimators.py +++ b/src/hyperactive/opt/gfo/_tree_structured_parzen_estimators.py @@ -86,6 +86,7 @@ class TreeStructuredParzenEstimators(_BaseGFOadapter): def __init__( self, + unified_space=None, search_space=None, initialize=None, constraints=None, @@ -100,6 +101,7 @@ def __init__( verbose=False, experiment=None, ): + self.unified_space = unified_space self.random_state = random_state self.rand_rest_p = rand_rest_p self.warm_start_smbo = warm_start_smbo diff --git a/src/hyperactive/opt/gridsearch/_sk.py b/src/hyperactive/opt/gridsearch/_sk.py index 835bb4ae..3afcc0ee 100644 --- a/src/hyperactive/opt/gridsearch/_sk.py +++ b/src/hyperactive/opt/gridsearch/_sk.py @@ -105,6 +105,7 @@ class GridSearchSk(BaseOptimizer): def __init__( self, + unified_space=None, param_grid=None, error_score=np.nan, backend="None", @@ -112,6 +113,7 @@ def __init__( experiment=None, ): self.experiment = experiment + self.unified_space = unified_space self.param_grid = param_grid self.error_score = error_score self.backend = backend @@ -119,6 +121,34 @@ def __init__( super().__init__() + def get_search_config(self): + """Get the search configuration. + + Returns + ------- + dict with str keys + The search configuration dictionary. + """ + search_config = super().get_search_config() + + # Resolve: unified_space is converted to param_grid + unified_space = search_config.pop("unified_space", None) + param_grid = search_config.get("param_grid") + + # Validate: only one should be set + if unified_space is not None and param_grid is not None: + raise ValueError( + "Provide either 'unified_space' or 'param_grid', not both. " + "Use 'unified_space' for simple dict[str, list] format, " + "or 'param_grid' for native sklearn format." + ) + + # Use unified_space if param_grid is not set + if unified_space is not None: + search_config["param_grid"] = unified_space + + return search_config + def _check_param_grid(self, param_grid): """_check_param_grid from sklearn 1.0.2, before it was removed.""" if hasattr(param_grid, "items"): diff --git a/src/hyperactive/opt/optuna/_cmaes_optimizer.py b/src/hyperactive/opt/optuna/_cmaes_optimizer.py index 3c3c5788..3f5b3f59 100644 --- a/src/hyperactive/opt/optuna/_cmaes_optimizer.py +++ b/src/hyperactive/opt/optuna/_cmaes_optimizer.py @@ -66,6 +66,7 @@ class CmaEsOptimizer(_BaseOptunaAdapter): def __init__( self, + unified_space=None, param_space=None, n_trials=100, initialize=None, @@ -82,6 +83,7 @@ def __init__( self.n_startup_trials = n_startup_trials super().__init__( + unified_space=unified_space, param_space=param_space, n_trials=n_trials, initialize=initialize, diff --git a/src/hyperactive/opt/optuna/_gp_optimizer.py b/src/hyperactive/opt/optuna/_gp_optimizer.py index 9b800469..646c8ee5 100644 --- a/src/hyperactive/opt/optuna/_gp_optimizer.py +++ b/src/hyperactive/opt/optuna/_gp_optimizer.py @@ -64,6 +64,7 @@ class GPOptimizer(_BaseOptunaAdapter): def __init__( self, + unified_space=None, param_space=None, n_trials=100, initialize=None, @@ -78,6 +79,7 @@ def __init__( self.deterministic_objective = deterministic_objective super().__init__( + unified_space=unified_space, param_space=param_space, n_trials=n_trials, initialize=initialize, diff --git a/src/hyperactive/opt/optuna/_grid_optimizer.py b/src/hyperactive/opt/optuna/_grid_optimizer.py index b8bb75aa..9def1cd9 100644 --- a/src/hyperactive/opt/optuna/_grid_optimizer.py +++ b/src/hyperactive/opt/optuna/_grid_optimizer.py @@ -62,18 +62,17 @@ class GridOptimizer(_BaseOptunaAdapter): def __init__( self, + unified_space=None, param_space=None, n_trials=100, initialize=None, random_state=None, early_stopping=None, max_score=None, - search_space=None, experiment=None, ): - self.search_space = search_space - super().__init__( + unified_space=unified_space, param_space=param_space, n_trials=n_trials, initialize=initialize, @@ -93,11 +92,13 @@ def _get_optimizer(self): """ import optuna - # Convert param_space to Optuna search space format if needed - search_space = self.search_space - if search_space is None and self.param_space is not None: - search_space = {} - for key, space in self.param_space.items(): + # Convert param_space to Optuna grid search space format + # Use _resolved_param_space which is set in _solve before this is called + search_space = {} + param_space = getattr(self, "_resolved_param_space", self.param_space) + + if param_space is not None: + for key, space in param_space.items(): if isinstance(space, list): search_space[key] = space elif isinstance(space, (tuple,)) and len(space) == 2: diff --git a/src/hyperactive/opt/optuna/_nsga_ii_optimizer.py b/src/hyperactive/opt/optuna/_nsga_ii_optimizer.py index 003daab7..efff2d43 100644 --- a/src/hyperactive/opt/optuna/_nsga_ii_optimizer.py +++ b/src/hyperactive/opt/optuna/_nsga_ii_optimizer.py @@ -66,6 +66,7 @@ class NSGAIIOptimizer(_BaseOptunaAdapter): def __init__( self, + unified_space=None, param_space=None, n_trials=100, initialize=None, @@ -82,6 +83,7 @@ def __init__( self.crossover_prob = crossover_prob super().__init__( + unified_space=unified_space, param_space=param_space, n_trials=n_trials, initialize=initialize, diff --git a/src/hyperactive/opt/optuna/_nsga_iii_optimizer.py b/src/hyperactive/opt/optuna/_nsga_iii_optimizer.py index 32af691f..17a00b3c 100644 --- a/src/hyperactive/opt/optuna/_nsga_iii_optimizer.py +++ b/src/hyperactive/opt/optuna/_nsga_iii_optimizer.py @@ -66,6 +66,7 @@ class NSGAIIIOptimizer(_BaseOptunaAdapter): def __init__( self, + unified_space=None, param_space=None, n_trials=100, initialize=None, @@ -82,6 +83,7 @@ def __init__( self.crossover_prob = crossover_prob super().__init__( + unified_space=unified_space, param_space=param_space, n_trials=n_trials, initialize=initialize, diff --git a/src/hyperactive/opt/optuna/_qmc_optimizer.py b/src/hyperactive/opt/optuna/_qmc_optimizer.py index ea705c39..f3ae4599 100644 --- a/src/hyperactive/opt/optuna/_qmc_optimizer.py +++ b/src/hyperactive/opt/optuna/_qmc_optimizer.py @@ -64,6 +64,7 @@ class QMCOptimizer(_BaseOptunaAdapter): def __init__( self, + unified_space=None, param_space=None, n_trials=100, initialize=None, @@ -78,6 +79,7 @@ def __init__( self.scramble = scramble super().__init__( + unified_space=unified_space, param_space=param_space, n_trials=n_trials, initialize=initialize, diff --git a/src/hyperactive/opt/optuna/_random_optimizer.py b/src/hyperactive/opt/optuna/_random_optimizer.py index 77bb0fe2..6c93fd1d 100644 --- a/src/hyperactive/opt/optuna/_random_optimizer.py +++ b/src/hyperactive/opt/optuna/_random_optimizer.py @@ -60,6 +60,7 @@ class RandomOptimizer(_BaseOptunaAdapter): def __init__( self, + unified_space=None, param_space=None, n_trials=100, initialize=None, @@ -69,6 +70,7 @@ def __init__( experiment=None, ): super().__init__( + unified_space=unified_space, param_space=param_space, n_trials=n_trials, initialize=initialize, diff --git a/src/hyperactive/opt/optuna/_tpe_optimizer.py b/src/hyperactive/opt/optuna/_tpe_optimizer.py index d8658dc9..b1ff3508 100644 --- a/src/hyperactive/opt/optuna/_tpe_optimizer.py +++ b/src/hyperactive/opt/optuna/_tpe_optimizer.py @@ -66,6 +66,7 @@ class TPEOptimizer(_BaseOptunaAdapter): def __init__( self, + unified_space=None, param_space=None, n_trials=100, initialize=None, @@ -82,6 +83,7 @@ def __init__( self.weights = weights super().__init__( + unified_space=unified_space, param_space=param_space, n_trials=n_trials, initialize=initialize, diff --git a/src/hyperactive/tests/test_unified_search_space.py b/src/hyperactive/tests/test_unified_search_space.py new file mode 100644 index 00000000..c0426298 --- /dev/null +++ b/src/hyperactive/tests/test_unified_search_space.py @@ -0,0 +1,218 @@ +"""Tests for unified_space parameter across all backends.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +import pytest +from sklearn.datasets import load_iris +from sklearn.svm import SVC + +from hyperactive.experiment.integrations import SklearnCvExperiment + + +@pytest.fixture +def sklearn_experiment(): + """Create a sklearn experiment fixture for testing.""" + X, y = load_iris(return_X_y=True) + return SklearnCvExperiment(estimator=SVC(), X=X, y=y, cv=3) + + +@pytest.fixture +def simple_search_space(): + """Simple unified search space format: dict[str, list].""" + return {"C": [0.1, 1, 10], "gamma": [0.01, 0.1, 1]} + + +class TestUnifiedSearchSpaceGFO: + """Test unified_space parameter for GFO optimizers.""" + + def test_gfo_random_search_accepts_unified_space( + self, sklearn_experiment, simple_search_space + ): + """GFO RandomSearch should accept unified_space.""" + from hyperactive.opt.gfo import RandomSearch + + opt = RandomSearch( + unified_space=simple_search_space, + n_iter=5, + experiment=sklearn_experiment, + ) + best_params = opt.solve() + + assert isinstance(best_params, dict) + assert "C" in best_params + assert "gamma" in best_params + + def test_gfo_search_space_still_works( + self, sklearn_experiment, simple_search_space + ): + """Backward compatibility: search_space (native GFO) should still work.""" + from hyperactive.opt.gfo import RandomSearch + + opt = RandomSearch( + search_space=simple_search_space, + n_iter=5, + experiment=sklearn_experiment, + ) + best_params = opt.solve() + + assert isinstance(best_params, dict) + assert "C" in best_params + assert "gamma" in best_params + + def test_gfo_raises_when_both_provided( + self, sklearn_experiment, simple_search_space + ): + """GFO should raise when both unified_space and search_space are given.""" + from hyperactive.opt.gfo import RandomSearch + + opt = RandomSearch( + unified_space=simple_search_space, + search_space=simple_search_space, + n_iter=5, + experiment=sklearn_experiment, + ) + + with pytest.raises(ValueError, match="Provide either 'unified_space' or"): + opt.solve() + + +class TestUnifiedSearchSpaceOptuna: + """Test unified_space parameter for Optuna optimizers.""" + + @pytest.mark.parametrize( + "optimizer_cls", + [ + pytest.param("TPEOptimizer", id="tpe"), + pytest.param("RandomOptimizer", id="random"), + pytest.param("GridOptimizer", id="grid"), + ], + ) + def test_optuna_accepts_unified_space( + self, sklearn_experiment, simple_search_space, optimizer_cls + ): + """Optuna optimizers should accept unified_space.""" + import hyperactive.opt.optuna as optuna_module + + OptCls = getattr(optuna_module, optimizer_cls) + + opt = OptCls( + unified_space=simple_search_space, + n_trials=5, + experiment=sklearn_experiment, + ) + best_params = opt.solve() + + assert isinstance(best_params, dict) + assert "C" in best_params + assert "gamma" in best_params + + def test_optuna_param_space_still_works(self, sklearn_experiment): + """Backward compatibility: param_space should still work.""" + from hyperactive.opt.optuna import TPEOptimizer + + # Native Optuna format with ranges + param_space = {"C": (0.1, 10), "gamma": [0.01, 0.1, 1]} + + opt = TPEOptimizer( + param_space=param_space, + n_trials=5, + experiment=sklearn_experiment, + ) + best_params = opt.solve() + + assert isinstance(best_params, dict) + assert "C" in best_params + assert "gamma" in best_params + + def test_optuna_raises_when_both_provided(self, sklearn_experiment): + """Optuna should raise when both unified_space and param_space are given.""" + from hyperactive.opt.optuna import TPEOptimizer + + unified_space = {"C": [0.1, 1], "gamma": [0.01]} + param_space = {"C": (0.1, 10), "gamma": [0.01, 0.1, 1]} + + opt = TPEOptimizer( + unified_space=unified_space, + param_space=param_space, + n_trials=5, + experiment=sklearn_experiment, + ) + + with pytest.raises(ValueError, match="Provide either 'unified_space' or"): + opt.solve() + + +class TestUnifiedSearchSpaceGridSearch: + """Test unified_space parameter for GridSearchSk.""" + + def test_gridsearch_accepts_unified_space( + self, sklearn_experiment, simple_search_space + ): + """GridSearchSk should accept unified_space.""" + from hyperactive.opt import GridSearchSk + + opt = GridSearchSk( + unified_space=simple_search_space, + experiment=sklearn_experiment, + ) + best_params = opt.solve() + + assert isinstance(best_params, dict) + assert "C" in best_params + assert "gamma" in best_params + + def test_gridsearch_param_grid_still_works(self, sklearn_experiment): + """Backward compatibility: param_grid should still work.""" + from hyperactive.opt import GridSearchSk + + param_grid = {"C": [0.1, 1, 10], "gamma": [0.01, 0.1]} + + opt = GridSearchSk( + param_grid=param_grid, + experiment=sklearn_experiment, + ) + best_params = opt.solve() + + assert isinstance(best_params, dict) + assert "C" in best_params + assert "gamma" in best_params + + def test_gridsearch_raises_when_both_provided( + self, sklearn_experiment, simple_search_space + ): + """GridSearchSk should raise when both unified_space and param_grid given.""" + from hyperactive.opt import GridSearchSk + + opt = GridSearchSk( + unified_space=simple_search_space, + param_grid=simple_search_space, + experiment=sklearn_experiment, + ) + + with pytest.raises(ValueError, match="Provide either 'unified_space' or"): + opt.solve() + + +class TestCapabilityTags: + """Test capability tags for search space features.""" + + def test_gfo_capability_tags(self): + """GFO optimizers should have correct capability tags.""" + from hyperactive.opt.gfo import RandomSearch + + opt = RandomSearch.create_test_instance() + + assert opt.get_tag("capability:discrete") is True + assert opt.get_tag("capability:continuous") is True + assert opt.get_tag("capability:categorical") is True + assert opt.get_tag("capability:constraints") is True + + def test_optuna_capability_tags(self): + """Optuna optimizers should have correct capability tags.""" + from hyperactive.opt.optuna import TPEOptimizer + + opt = TPEOptimizer.create_test_instance() + + assert opt.get_tag("capability:discrete") is True + assert opt.get_tag("capability:continuous") is True + assert opt.get_tag("capability:categorical") is True + assert opt.get_tag("capability:log_scale") is True From 7c161014f625b62b19186a2bd6dc55b08947cbc3 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 18 Jan 2026 19:59:50 +0100 Subject: [PATCH 2/8] add search-space adapter --- src/hyperactive/base/_optimizer.py | 78 ++++++++ .../opt/_adapters/_search_space_adapter.py | 184 ++++++++++++++++++ 2 files changed, 262 insertions(+) create mode 100644 src/hyperactive/opt/_adapters/_search_space_adapter.py diff --git a/src/hyperactive/base/_optimizer.py b/src/hyperactive/base/_optimizer.py index ed44cc94..d46dc9f5 100644 --- a/src/hyperactive/base/_optimizer.py +++ b/src/hyperactive/base/_optimizer.py @@ -86,10 +86,88 @@ def solve(self): experiment = self.get_experiment() search_config = self.get_search_config() + # Adapt search space for backend capabilities (e.g., categorical encoding) + experiment, search_config, adapter = self._adapt_search_space( + experiment, search_config + ) + + # Run optimization best_params = self._solve(experiment, **search_config) + + # Decode results if adapter was used + if adapter is not None: + best_params = adapter.decode(best_params) + self.best_params_ = best_params return best_params + def _adapt_search_space(self, experiment, search_config): + """Adapt search space and experiment for backend capabilities. + + If the backend doesn't support certain search space features + (e.g., categorical values), this method encodes the search space + and wraps the experiment to handle encoding/decoding transparently. + + Parameters + ---------- + experiment : BaseExperiment + The experiment to optimize. + search_config : dict + The search configuration containing the search space. + + Returns + ------- + experiment : BaseExperiment + The experiment, possibly wrapped for decoding. + search_config : dict + The search config, possibly with encoded search space. + adapter : SearchSpaceAdapter or None + The adapter if encoding was applied, None otherwise. + """ + from hyperactive.opt._adapters._search_space_adapter import SearchSpaceAdapter + + search_space_key = self._detect_search_space_key(search_config) + + # No search space found - pass through unchanged + if not search_space_key or not search_config.get(search_space_key): + return experiment, search_config, None + + # Create adapter with backend capabilities + capabilities = { + "categorical": self.get_tag("capability:categorical"), + "continuous": self.get_tag("capability:continuous"), + } + adapter = SearchSpaceAdapter(search_config[search_space_key], capabilities) + + # Backend supports all features - pass through unchanged + if not adapter.needs_encoding: + return experiment, search_config, None + + # Encoding needed - transform search space and wrap experiment + encoded_config = search_config.copy() + encoded_config[search_space_key] = adapter.encode() + wrapped_experiment = adapter.wrap_experiment(experiment) + + return wrapped_experiment, encoded_config, adapter + + def _detect_search_space_key(self, search_config): + """Find which key holds the search space in the config. + + Parameters + ---------- + search_config : dict + The search configuration dictionary. + + Returns + ------- + str or None + The key name for search space, or None if not found. + """ + for key in ["search_space", "param_space", "param_grid"]: + if key in search_config and search_config[key] is not None: + return key + return None + def _solve(self, experiment, *args, **kwargs): """Run the optimization search process. diff --git a/src/hyperactive/opt/_adapters/_search_space_adapter.py b/src/hyperactive/opt/_adapters/_search_space_adapter.py new file mode 100644 index 00000000..62c2c08e --- /dev/null +++ b/src/hyperactive/opt/_adapters/_search_space_adapter.py @@ -0,0 +1,184 @@ +"""Search space adapter for optimizer backends.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +__all__ = ["SearchSpaceAdapter"] + + +class SearchSpaceAdapter: + """Adapts search spaces for optimizer backends. + + Handles encoding/decoding of categorical dimensions when the backend + doesn't support them natively. + + Parameters + ---------- + search_space : dict[str, list] + The search space as a dictionary mapping parameter names to value lists. + capabilities : dict + Backend capability tags, e.g., {"categorical": True, "continuous": False}. + + Attributes + ---------- + needs_encoding : bool + Whether any dimensions require encoding. + categorical_mapping : dict + Mapping of {param_name: {index: original_value}} for encoded dimensions. + + Examples + -------- + >>> space = {"kernel": ["rbf", "linear"], "C": [0.1, 1, 10]} + >>> adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + >>> encoded = adapter.encode() + >>> encoded + {"kernel": [0, 1], "C": [0.1, 1, 10]} + >>> adapter.decode({"kernel": 1, "C": 1}) + {"kernel": "linear", "C": 1} + """ + + def __init__(self, search_space: dict, capabilities: dict): + self._original_space = search_space + self._capabilities = capabilities + self._categorical_mapping = {} + self._needs_encoding = self._check_needs_encoding() + + @property + def needs_encoding(self) -> bool: + """Whether encoding is needed.""" + return self._needs_encoding + + @property + def categorical_mapping(self) -> dict: + """Mapping of encoded categorical dimensions.""" + return self._categorical_mapping + + def _check_needs_encoding(self) -> bool: + """Check if encoding is needed based on capabilities and search space.""" + # If backend supports categorical natively, no encoding needed + if self._capabilities.get("categorical", True): + return False + # Check if search space contains categorical values + return self._has_categorical_values() + + def _has_categorical_values(self) -> bool: + """Detect if any dimension contains string values.""" + for values in self._original_space.values(): + if self._is_categorical(values): + return True + return False + + def _is_categorical(self, values) -> bool: + """Check if a dimension's values are categorical (contain strings).""" + if hasattr(values, "__iter__") and not isinstance(values, str): + return any(isinstance(v, str) for v in values) + return False + + def encode(self) -> dict: + """Encode the search space for the backend. + + Categorical dimensions (containing strings) are converted to + integer indices. The mapping is stored for later decoding. + + Returns + ------- + dict + Encoded search space with categorical values as integers. + """ + if not self._needs_encoding: + return self._original_space + + self._categorical_mapping = {} + encoded = {} + + for name, values in self._original_space.items(): + if self._is_categorical(values): + # Store mapping: {index: original_value} + self._categorical_mapping[name] = {i: v for i, v in enumerate(values)} + # Replace with integer indices + encoded[name] = list(range(len(values))) + else: + encoded[name] = values + + return encoded + + def decode(self, params: dict) -> dict: + """Decode backend results to original format. + + Integer indices for categorical dimensions are converted back + to their original string values. + + Parameters + ---------- + params : dict + Parameter dictionary from the optimizer, potentially with + encoded categorical values. + + Returns + ------- + dict + Parameters with original categorical values restored. + """ + if not self._categorical_mapping: + return params + + decoded = params.copy() + for name, mapping in self._categorical_mapping.items(): + if name in decoded: + val = decoded[name] + # Handle numpy types + if hasattr(val, "item"): + val = val.item() + decoded[name] = mapping[int(val)] + + return decoded + + def wrap_experiment(self, experiment): + """Wrap experiment to decode params before scoring. + + During optimization, the backend calls experiment.score(params) + with encoded values. This wrapper decodes them first. + + Parameters + ---------- + experiment : BaseExperiment + The original experiment. + + Returns + ------- + experiment + Wrapped experiment that decodes params, or original if no encoding. + """ + if not self._categorical_mapping: + return experiment + + return _DecodingExperimentWrapper(experiment, self._categorical_mapping) + + +class _DecodingExperimentWrapper: + """Wrapper that decodes params before passing to experiment.""" + + def __init__(self, experiment, categorical_mapping): + self._experiment = experiment + self._mapping = categorical_mapping + + def _decode(self, params): + decoded = params.copy() + for name, mapping in self._mapping.items(): + if name in decoded: + val = decoded[name] + if hasattr(val, "item"): + val = val.item() + decoded[name] = mapping[int(val)] + return decoded + + def score(self, params): + return self._experiment.score(self._decode(params)) + + def __call__(self, params): + return self.score(params) + + def evaluate(self, params): + return self._experiment.evaluate(self._decode(params)) + + def __getattr__(self, name): + # Forward all other attributes to wrapped experiment + return getattr(self._experiment, name) From f38e5844fa124e4fc239a83f31b3e0d88a80e4e2 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 18 Jan 2026 20:00:06 +0100 Subject: [PATCH 3/8] small fixes --- src/hyperactive/opt/_adapters/_gfo.py | 2 +- .../opt/optuna/_cmaes_optimizer.py | 3 ++ src/hyperactive/opt/random_search.py | 36 ++++++++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/hyperactive/opt/_adapters/_gfo.py b/src/hyperactive/opt/_adapters/_gfo.py index 822e5cdc..d51622e3 100644 --- a/src/hyperactive/opt/_adapters/_gfo.py +++ b/src/hyperactive/opt/_adapters/_gfo.py @@ -26,7 +26,7 @@ class _BaseGFOadapter(BaseOptimizer): # Search space capabilities "capability:discrete": True, "capability:continuous": True, - "capability:categorical": True, + "capability:categorical": False, # GFO only supports numeric values "capability:constraints": True, } diff --git a/src/hyperactive/opt/optuna/_cmaes_optimizer.py b/src/hyperactive/opt/optuna/_cmaes_optimizer.py index 3f5b3f59..6df992d9 100644 --- a/src/hyperactive/opt/optuna/_cmaes_optimizer.py +++ b/src/hyperactive/opt/optuna/_cmaes_optimizer.py @@ -62,6 +62,9 @@ class CmaEsOptimizer(_BaseOptunaAdapter): "info:explore_vs_exploit": "mixed", "info:compute": "high", "python_dependencies": ["optuna", "cmaes"], + # CMA-ES only works with continuous parameters + "capability:categorical": False, + "capability:discrete": False, } def __init__( diff --git a/src/hyperactive/opt/random_search.py b/src/hyperactive/opt/random_search.py index 2ae97bbc..455d2797 100644 --- a/src/hyperactive/opt/random_search.py +++ b/src/hyperactive/opt/random_search.py @@ -17,7 +17,11 @@ class RandomSearchSk(BaseOptimizer): Parameters ---------- - param_distributions : dict[str, list | scipy.stats.rv_frozen] + unified_space : dict[str, list], default=None + Unified search space format: dict mapping parameter names to lists. + Use this for portable search space definitions across backends. + Mutually exclusive with ``param_distributions``. + param_distributions : dict[str, list | scipy.stats.rv_frozen], default=None Search space specification. Discrete lists are sampled uniformly; scipy distribution objects are sampled via their ``rvs`` method. @@ -107,6 +111,7 @@ class RandomSearchSk(BaseOptimizer): def __init__( self, + unified_space=None, param_distributions=None, n_iter=10, random_state=None, @@ -116,6 +121,7 @@ def __init__( experiment=None, ): self.experiment = experiment + self.unified_space = unified_space self.param_distributions = param_distributions self.n_iter = n_iter self.random_state = random_state @@ -125,6 +131,34 @@ def __init__( super().__init__() + def get_search_config(self): + """Get the search configuration. + + Returns + ------- + dict with str keys + The search configuration dictionary. + """ + search_config = super().get_search_config() + + # Resolve: unified_space is converted to param_distributions + unified_space = search_config.pop("unified_space", None) + param_distributions = search_config.get("param_distributions") + + # Validate: only one should be set + if unified_space is not None and param_distributions is not None: + raise ValueError( + "Provide either 'unified_space' or 'param_distributions', not both. " + "Use 'unified_space' for simple dict[str, list] format, " + "or 'param_distributions' for native sklearn format with distributions." + ) + + # Use unified_space if param_distributions is not set + if unified_space is not None: + search_config["param_distributions"] = unified_space + + return search_config + @staticmethod def _is_distribution(obj) -> bool: """Return True if *obj* looks like a scipy frozen distribution.""" From a61c4bf54247f24fd972874d6e7b53071fc425b0 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 18 Jan 2026 20:00:17 +0100 Subject: [PATCH 4/8] add search-space adapter tests --- .../tests/test_search_space_adapter.py | 368 ++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 src/hyperactive/tests/test_search_space_adapter.py diff --git a/src/hyperactive/tests/test_search_space_adapter.py b/src/hyperactive/tests/test_search_space_adapter.py new file mode 100644 index 00000000..56e34c9a --- /dev/null +++ b/src/hyperactive/tests/test_search_space_adapter.py @@ -0,0 +1,368 @@ +"""Tests for SearchSpaceAdapter encoding/decoding.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +import pytest + +from hyperactive.opt._adapters._search_space_adapter import SearchSpaceAdapter + + +class TestSearchSpaceAdapter: + """Tests for SearchSpaceAdapter encoding/decoding.""" + + def test_encode_categorical_to_integers(self): + """Categorical strings are encoded to integer indices.""" + space = {"kernel": ["rbf", "linear", "poly"]} + adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + + encoded = adapter.encode() + + assert encoded == {"kernel": [0, 1, 2]} + assert adapter.categorical_mapping == { + "kernel": {0: "rbf", 1: "linear", 2: "poly"} + } + + def test_decode_integers_to_categorical(self): + """Integer indices are decoded back to original strings.""" + space = {"kernel": ["rbf", "linear"]} + adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + adapter.encode() + + decoded = adapter.decode({"kernel": 1}) + + assert decoded == {"kernel": "linear"} + + def test_no_encoding_when_supported(self): + """No encoding when backend supports categorical.""" + space = {"kernel": ["rbf", "linear"]} + adapter = SearchSpaceAdapter(space, capabilities={"categorical": True}) + + assert adapter.needs_encoding is False + assert adapter.encode() is space # Same object, not copied + + def test_mixed_dimensions(self): + """Categorical and numeric dimensions coexist.""" + space = {"kernel": ["rbf", "linear"], "C": [0.1, 1, 10]} + adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + + encoded = adapter.encode() + + assert encoded == {"kernel": [0, 1], "C": [0.1, 1, 10]} + assert "kernel" in adapter.categorical_mapping + assert "C" not in adapter.categorical_mapping + + def test_wrapped_experiment_decodes(self): + """Wrapped experiment receives decoded params.""" + space = {"kernel": ["rbf", "linear"]} + adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + adapter.encode() + + received_params = [] + + class MockExperiment: + def score(self, params): + received_params.append(params.copy()) + return 1.0 + + wrapped = adapter.wrap_experiment(MockExperiment()) + wrapped.score({"kernel": 1}) + + assert received_params[0] == {"kernel": "linear"} + + def test_numpy_float_handling(self): + """Numpy float indices are converted correctly.""" + np = pytest.importorskip("numpy") + + space = {"kernel": ["rbf", "linear"]} + adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + adapter.encode() + + decoded = adapter.decode({"kernel": np.float64(0.0)}) + + assert decoded == {"kernel": "rbf"} + + def test_numpy_int_handling(self): + """Numpy integer indices are converted correctly.""" + np = pytest.importorskip("numpy") + + space = {"kernel": ["rbf", "linear", "poly"]} + adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + adapter.encode() + + decoded = adapter.decode({"kernel": np.int64(2)}) + + assert decoded == {"kernel": "poly"} + + def test_no_encoding_for_numeric_only_space(self): + """No encoding needed when space contains only numeric values.""" + space = {"C": [0.1, 1, 10], "gamma": [0.01, 0.1, 1]} + adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + + assert adapter.needs_encoding is False + assert adapter.encode() is space + + def test_wrapped_experiment_callable(self): + """Wrapped experiment is callable like original.""" + space = {"kernel": ["rbf", "linear"]} + adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + adapter.encode() + + class MockExperiment: + def score(self, params): + return 1.0 if params["kernel"] == "linear" else 0.5 + + wrapped = adapter.wrap_experiment(MockExperiment()) + + # Call via __call__ + result = wrapped({"kernel": 1}) + assert result == 1.0 + + def test_wrapped_experiment_evaluate(self): + """Wrapped experiment.evaluate() also decodes.""" + space = {"kernel": ["rbf", "linear"]} + adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + adapter.encode() + + received_params = [] + + class MockExperiment: + def score(self, params): + return 1.0 + + def evaluate(self, params): + received_params.append(params.copy()) + return {"accuracy": 0.95} + + wrapped = adapter.wrap_experiment(MockExperiment()) + wrapped.evaluate({"kernel": 0}) + + assert received_params[0] == {"kernel": "rbf"} + + def test_wrapped_experiment_forwards_attributes(self): + """Wrapped experiment forwards attribute access to original.""" + space = {"kernel": ["rbf", "linear"]} + adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + adapter.encode() + + class MockExperiment: + custom_attr = "test_value" + + def score(self, params): + return 1.0 + + wrapped = adapter.wrap_experiment(MockExperiment()) + + assert wrapped.custom_attr == "test_value" + + def test_decode_preserves_non_categorical_params(self): + """Decode preserves parameters that weren't encoded.""" + space = {"kernel": ["rbf", "linear"], "C": [0.1, 1, 10]} + adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + adapter.encode() + + decoded = adapter.decode({"kernel": 1, "C": 1, "extra": "value"}) + + assert decoded == {"kernel": "linear", "C": 1, "extra": "value"} + + def test_default_capability_is_categorical_supported(self): + """Default capability assumes categorical is supported.""" + space = {"kernel": ["rbf", "linear"]} + adapter = SearchSpaceAdapter(space, capabilities={}) + + assert adapter.needs_encoding is False + + def test_multiple_categorical_dimensions(self): + """Multiple categorical dimensions are all encoded.""" + space = { + "kernel": ["rbf", "linear"], + "solver": ["lbfgs", "sgd", "adam"], + } + adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + + encoded = adapter.encode() + + assert encoded == {"kernel": [0, 1], "solver": [0, 1, 2]} + assert adapter.categorical_mapping["kernel"] == {0: "rbf", 1: "linear"} + assert adapter.categorical_mapping["solver"] == {0: "lbfgs", 1: "sgd", 2: "adam"} + + decoded = adapter.decode({"kernel": 0, "solver": 2}) + assert decoded == {"kernel": "rbf", "solver": "adam"} + + +class TestCategoricalEncodingIntegration: + """Integration tests for categorical encoding in optimizers.""" + + @pytest.fixture + def sklearn_experiment(self): + """Create a sklearn experiment fixture for testing.""" + from sklearn.datasets import load_iris + from sklearn.svm import SVC + + from hyperactive.experiment.integrations import SklearnCvExperiment + + X, y = load_iris(return_X_y=True) + return SklearnCvExperiment(estimator=SVC(), X=X, y=y, cv=3) + + def test_cmaes_has_categorical_false_tag(self): + """CMA-ES optimizer should have categorical capability set to False.""" + from hyperactive.opt.optuna import CmaEsOptimizer + + opt = CmaEsOptimizer.create_test_instance() + assert opt.get_tag("capability:categorical") is False + + def test_optuna_optimizers_have_categorical_true_tag(self): + """Optuna TPE/Random/Grid optimizers should support categorical.""" + from hyperactive.opt.optuna import GridOptimizer, RandomOptimizer, TPEOptimizer + + for OptCls in [TPEOptimizer, RandomOptimizer, GridOptimizer]: + opt = OptCls.create_test_instance() + assert opt.get_tag("capability:categorical") is True + + def test_gfo_optimizers_have_categorical_false_tag(self): + """GFO optimizers should have categorical tag set to False.""" + from hyperactive.opt.gfo import RandomSearch + + opt = RandomSearch.create_test_instance() + # GFO does not support categorical natively - adapter handles encoding + assert opt.get_tag("capability:categorical") is False + + +# All GFO optimizers +_GFO_OPTIMIZERS = [ + "RandomSearch", + "HillClimbing", + "StochasticHillClimbing", + "RepulsingHillClimbing", + "SimulatedAnnealing", + "RandomRestartHillClimbing", + "ParallelTempering", + "ParticleSwarmOptimizer", + "SpiralOptimization", + "GeneticAlgorithm", + "EvolutionStrategy", + "DifferentialEvolution", + "BayesianOptimizer", + "TreeStructuredParzenEstimators", + "ForestOptimizer", + "GridSearch", + "PatternSearch", + "DirectAlgorithm", + "LipschitzOptimizer", + "PowellsMethod", + "DownhillSimplexOptimizer", +] + +_OPTUNA_OPTIMIZERS = [ + "TPEOptimizer", + "RandomOptimizer", + "GridOptimizer", + "GPOptimizer", + "QMCOptimizer", + # CmaEsOptimizer excluded - only supports continuous ranges + # NSGAIIOptimizer, NSGAIIIOptimizer excluded - multi-objective +] + +_SKLEARN_OPTIMIZERS = [ + "GridSearchSk", + "RandomSearchSk", +] + + +class TestAllOptimizersWithCategoricalSearchSpace: + """Test that all optimizers work with categorical search spaces.""" + + @pytest.fixture + def categorical_search_space(self): + """Search space with categorical and numeric values.""" + return { + "x": [0, 1, 2, 3, 4], + "option": ["a", "b", "c"], + } + + @pytest.fixture + def function_experiment(self): + """Simple function experiment for fast testing.""" + from hyperactive.experiment.func import FunctionExperiment + + def objective(params): + # Simple objective: x value + bonus for option + bonus = {"a": 0, "b": 1, "c": 2} + return params["x"] + bonus[params["option"]] + + return FunctionExperiment(objective) + + @pytest.mark.parametrize("optimizer_name", _GFO_OPTIMIZERS) + def test_gfo_optimizer_with_categorical( + self, optimizer_name, categorical_search_space, function_experiment + ): + """GFO optimizers should handle categorical search spaces.""" + from hyperactive.opt import gfo + + OptCls = getattr(gfo, optimizer_name) + + opt = OptCls( + search_space=categorical_search_space, + n_iter=3, + experiment=function_experiment, + ) + best_params = opt.solve() + + # Verify result contains original categorical string values + assert isinstance(best_params, dict) + assert "x" in best_params + assert "option" in best_params + assert best_params["option"] in ["a", "b", "c"] + assert isinstance(best_params["option"], str) + + @pytest.mark.parametrize("optimizer_name", _OPTUNA_OPTIMIZERS) + def test_optuna_optimizer_with_categorical( + self, optimizer_name, categorical_search_space, function_experiment + ): + """Optuna optimizers should handle categorical search spaces.""" + from hyperactive.opt import optuna + + OptCls = getattr(optuna, optimizer_name) + + opt = OptCls( + unified_space=categorical_search_space, + n_trials=3, + experiment=function_experiment, + ) + best_params = opt.solve() + + # Verify result contains original categorical string values + assert isinstance(best_params, dict) + assert "x" in best_params + assert "option" in best_params + assert best_params["option"] in ["a", "b", "c"] + assert isinstance(best_params["option"], str) + + @pytest.mark.parametrize("optimizer_name", _SKLEARN_OPTIMIZERS) + def test_sklearn_optimizer_with_categorical(self, optimizer_name): + """Sklearn-based optimizers should handle categorical search spaces.""" + from sklearn.datasets import load_iris + from sklearn.neighbors import KNeighborsClassifier + + from hyperactive.experiment.integrations import SklearnCvExperiment + from hyperactive.opt.gridsearch import GridSearchSk + from hyperactive.opt.random_search import RandomSearchSk + + X, y = load_iris(return_X_y=True) + exp = SklearnCvExperiment(estimator=KNeighborsClassifier(), X=X, y=y, cv=2) + + sklearn_space = { + "n_neighbors": [1, 3, 5], + "weights": ["uniform", "distance"], + } + + if optimizer_name == "GridSearchSk": + opt = GridSearchSk(unified_space=sklearn_space, experiment=exp) + else: + opt = RandomSearchSk(unified_space=sklearn_space, n_iter=3, experiment=exp) + + best_params = opt.solve() + + # Verify result contains original categorical string values + assert isinstance(best_params, dict) + assert "weights" in best_params + assert best_params["weights"] in ["uniform", "distance"] + assert isinstance(best_params["weights"], str) From 7396fc9c6f327a34d4414d768ce0db89f14d4680 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 18 Jan 2026 20:04:07 +0100 Subject: [PATCH 5/8] add tests --- src/hyperactive/tests/test_all_objects.py | 7 +++++++ src/hyperactive/tests/test_unified_search_space.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/hyperactive/tests/test_all_objects.py b/src/hyperactive/tests/test_all_objects.py index 559af35d..74c2ee5d 100644 --- a/src/hyperactive/tests/test_all_objects.py +++ b/src/hyperactive/tests/test_all_objects.py @@ -50,6 +50,13 @@ class PackageConfig: "info:local_vs_global", # "local", "mixed", "global" "info:explore_vs_exploit", # "explore", "exploit", "mixed" "info:compute", # "low", "middle", "high" + # search space capabilities + "capability:discrete", # supports discrete lists + "capability:continuous", # supports continuous ranges + "capability:categorical", # supports categorical choices + "capability:log_scale", # supports log-scale sampling + "capability:conditions", # supports conditional params + "capability:constraints", # supports constraint functions ] diff --git a/src/hyperactive/tests/test_unified_search_space.py b/src/hyperactive/tests/test_unified_search_space.py index c0426298..44a014f5 100644 --- a/src/hyperactive/tests/test_unified_search_space.py +++ b/src/hyperactive/tests/test_unified_search_space.py @@ -203,7 +203,7 @@ def test_gfo_capability_tags(self): assert opt.get_tag("capability:discrete") is True assert opt.get_tag("capability:continuous") is True - assert opt.get_tag("capability:categorical") is True + assert opt.get_tag("capability:categorical") is False # GFO only numeric assert opt.get_tag("capability:constraints") is True def test_optuna_capability_tags(self): From 187e88c89664b4e8710bf08339fed5704de14aa1 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 18 Jan 2026 20:08:43 +0100 Subject: [PATCH 6/8] move adapter code to utils --- src/hyperactive/base/_optimizer.py | 77 ++----------------- .../opt/_adapters/_adapter_utils.py | 71 +++++++++++++++++ 2 files changed, 79 insertions(+), 69 deletions(-) create mode 100644 src/hyperactive/opt/_adapters/_adapter_utils.py diff --git a/src/hyperactive/base/_optimizer.py b/src/hyperactive/base/_optimizer.py index d46dc9f5..3ef66b2a 100644 --- a/src/hyperactive/base/_optimizer.py +++ b/src/hyperactive/base/_optimizer.py @@ -83,12 +83,18 @@ def solve(self): The dict ``best_params`` can be used in ``experiment.score`` or ``experiment.evaluate`` directly. """ + from hyperactive.opt._adapters._adapter_utils import adapt_search_space + experiment = self.get_experiment() search_config = self.get_search_config() # Adapt search space for backend capabilities (e.g., categorical encoding) - experiment, search_config, adapter = self._adapt_search_space( - experiment, search_config + capabilities = { + "categorical": self.get_tag("capability:categorical"), + "continuous": self.get_tag("capability:continuous"), + } + experiment, search_config, adapter = adapt_search_space( + experiment, search_config, capabilities ) # Run optimization @@ -101,73 +107,6 @@ def solve(self): self.best_params_ = best_params return best_params - def _adapt_search_space(self, experiment, search_config): - """Adapt search space and experiment for backend capabilities. - - If the backend doesn't support certain search space features - (e.g., categorical values), this method encodes the search space - and wraps the experiment to handle encoding/decoding transparently. - - Parameters - ---------- - experiment : BaseExperiment - The experiment to optimize. - search_config : dict - The search configuration containing the search space. - - Returns - ------- - experiment : BaseExperiment - The experiment, possibly wrapped for decoding. - search_config : dict - The search config, possibly with encoded search space. - adapter : SearchSpaceAdapter or None - The adapter if encoding was applied, None otherwise. - """ - from hyperactive.opt._adapters._search_space_adapter import SearchSpaceAdapter - - search_space_key = self._detect_search_space_key(search_config) - - # No search space found - pass through unchanged - if not search_space_key or not search_config.get(search_space_key): - return experiment, search_config, None - - # Create adapter with backend capabilities - capabilities = { - "categorical": self.get_tag("capability:categorical"), - "continuous": self.get_tag("capability:continuous"), - } - adapter = SearchSpaceAdapter(search_config[search_space_key], capabilities) - - # Backend supports all features - pass through unchanged - if not adapter.needs_encoding: - return experiment, search_config, None - - # Encoding needed - transform search space and wrap experiment - encoded_config = search_config.copy() - encoded_config[search_space_key] = adapter.encode() - wrapped_experiment = adapter.wrap_experiment(experiment) - - return wrapped_experiment, encoded_config, adapter - - def _detect_search_space_key(self, search_config): - """Find which key holds the search space in the config. - - Parameters - ---------- - search_config : dict - The search configuration dictionary. - - Returns - ------- - str or None - The key name for search space, or None if not found. - """ - for key in ["search_space", "param_space", "param_grid"]: - if key in search_config and search_config[key] is not None: - return key - return None - def _solve(self, experiment, *args, **kwargs): """Run the optimization search process. diff --git a/src/hyperactive/opt/_adapters/_adapter_utils.py b/src/hyperactive/opt/_adapters/_adapter_utils.py new file mode 100644 index 00000000..16b78094 --- /dev/null +++ b/src/hyperactive/opt/_adapters/_adapter_utils.py @@ -0,0 +1,71 @@ +"""Utility functions for search space adaptation.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +from ._search_space_adapter import SearchSpaceAdapter + +__all__ = ["adapt_search_space", "detect_search_space_key"] + + +def detect_search_space_key(search_config): + """Find which key holds the search space in the config. + + Parameters + ---------- + search_config : dict + The search configuration dictionary. + + Returns + ------- + str or None + The key name for search space, or None if not found. + """ + for key in ["search_space", "param_space", "param_grid", "param_distributions"]: + if key in search_config and search_config[key] is not None: + return key + return None + + +def adapt_search_space(experiment, search_config, capabilities): + """Adapt search space and experiment for backend capabilities. + + If the backend doesn't support certain search space features + (e.g., categorical values), this function encodes the search space + and wraps the experiment to handle encoding/decoding transparently. + + Parameters + ---------- + experiment : BaseExperiment + The experiment to optimize. + search_config : dict + The search configuration containing the search space. + capabilities : dict + Backend capabilities, e.g., {"categorical": True, "continuous": False}. + + Returns + ------- + experiment : BaseExperiment + The experiment, possibly wrapped for decoding. + search_config : dict + The search config, possibly with encoded search space. + adapter : SearchSpaceAdapter or None + The adapter if encoding was applied, None otherwise. + """ + search_space_key = detect_search_space_key(search_config) + + # No search space found - pass through unchanged + if not search_space_key or not search_config.get(search_space_key): + return experiment, search_config, None + + # Create adapter with backend capabilities + adapter = SearchSpaceAdapter(search_config[search_space_key], capabilities) + + # Backend supports all features - pass through unchanged + if not adapter.needs_encoding: + return experiment, search_config, None + + # Encoding needed - transform search space and wrap experiment + encoded_config = search_config.copy() + encoded_config[search_space_key] = adapter.encode() + wrapped_experiment = adapter.wrap_experiment(experiment) + + return wrapped_experiment, encoded_config, adapter From 0a3318d7a27dfa5530a5869eea091a0132ede2d8 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 18 Jan 2026 20:08:56 +0100 Subject: [PATCH 7/8] move test files --- .../tests/test_search_space_adapter.py | 157 +----------------- .../tests/test_unified_search_space.py | 140 ++++++++++++++++ 2 files changed, 142 insertions(+), 155 deletions(-) diff --git a/src/hyperactive/tests/test_search_space_adapter.py b/src/hyperactive/tests/test_search_space_adapter.py index 56e34c9a..95f5af11 100644 --- a/src/hyperactive/tests/test_search_space_adapter.py +++ b/src/hyperactive/tests/test_search_space_adapter.py @@ -188,19 +188,8 @@ def test_multiple_categorical_dimensions(self): assert decoded == {"kernel": "rbf", "solver": "adam"} -class TestCategoricalEncodingIntegration: - """Integration tests for categorical encoding in optimizers.""" - - @pytest.fixture - def sklearn_experiment(self): - """Create a sklearn experiment fixture for testing.""" - from sklearn.datasets import load_iris - from sklearn.svm import SVC - - from hyperactive.experiment.integrations import SklearnCvExperiment - - X, y = load_iris(return_X_y=True) - return SklearnCvExperiment(estimator=SVC(), X=X, y=y, cv=3) +class TestCapabilityTags: + """Tests for capability tags related to categorical encoding.""" def test_cmaes_has_categorical_false_tag(self): """CMA-ES optimizer should have categorical capability set to False.""" @@ -224,145 +213,3 @@ def test_gfo_optimizers_have_categorical_false_tag(self): opt = RandomSearch.create_test_instance() # GFO does not support categorical natively - adapter handles encoding assert opt.get_tag("capability:categorical") is False - - -# All GFO optimizers -_GFO_OPTIMIZERS = [ - "RandomSearch", - "HillClimbing", - "StochasticHillClimbing", - "RepulsingHillClimbing", - "SimulatedAnnealing", - "RandomRestartHillClimbing", - "ParallelTempering", - "ParticleSwarmOptimizer", - "SpiralOptimization", - "GeneticAlgorithm", - "EvolutionStrategy", - "DifferentialEvolution", - "BayesianOptimizer", - "TreeStructuredParzenEstimators", - "ForestOptimizer", - "GridSearch", - "PatternSearch", - "DirectAlgorithm", - "LipschitzOptimizer", - "PowellsMethod", - "DownhillSimplexOptimizer", -] - -_OPTUNA_OPTIMIZERS = [ - "TPEOptimizer", - "RandomOptimizer", - "GridOptimizer", - "GPOptimizer", - "QMCOptimizer", - # CmaEsOptimizer excluded - only supports continuous ranges - # NSGAIIOptimizer, NSGAIIIOptimizer excluded - multi-objective -] - -_SKLEARN_OPTIMIZERS = [ - "GridSearchSk", - "RandomSearchSk", -] - - -class TestAllOptimizersWithCategoricalSearchSpace: - """Test that all optimizers work with categorical search spaces.""" - - @pytest.fixture - def categorical_search_space(self): - """Search space with categorical and numeric values.""" - return { - "x": [0, 1, 2, 3, 4], - "option": ["a", "b", "c"], - } - - @pytest.fixture - def function_experiment(self): - """Simple function experiment for fast testing.""" - from hyperactive.experiment.func import FunctionExperiment - - def objective(params): - # Simple objective: x value + bonus for option - bonus = {"a": 0, "b": 1, "c": 2} - return params["x"] + bonus[params["option"]] - - return FunctionExperiment(objective) - - @pytest.mark.parametrize("optimizer_name", _GFO_OPTIMIZERS) - def test_gfo_optimizer_with_categorical( - self, optimizer_name, categorical_search_space, function_experiment - ): - """GFO optimizers should handle categorical search spaces.""" - from hyperactive.opt import gfo - - OptCls = getattr(gfo, optimizer_name) - - opt = OptCls( - search_space=categorical_search_space, - n_iter=3, - experiment=function_experiment, - ) - best_params = opt.solve() - - # Verify result contains original categorical string values - assert isinstance(best_params, dict) - assert "x" in best_params - assert "option" in best_params - assert best_params["option"] in ["a", "b", "c"] - assert isinstance(best_params["option"], str) - - @pytest.mark.parametrize("optimizer_name", _OPTUNA_OPTIMIZERS) - def test_optuna_optimizer_with_categorical( - self, optimizer_name, categorical_search_space, function_experiment - ): - """Optuna optimizers should handle categorical search spaces.""" - from hyperactive.opt import optuna - - OptCls = getattr(optuna, optimizer_name) - - opt = OptCls( - unified_space=categorical_search_space, - n_trials=3, - experiment=function_experiment, - ) - best_params = opt.solve() - - # Verify result contains original categorical string values - assert isinstance(best_params, dict) - assert "x" in best_params - assert "option" in best_params - assert best_params["option"] in ["a", "b", "c"] - assert isinstance(best_params["option"], str) - - @pytest.mark.parametrize("optimizer_name", _SKLEARN_OPTIMIZERS) - def test_sklearn_optimizer_with_categorical(self, optimizer_name): - """Sklearn-based optimizers should handle categorical search spaces.""" - from sklearn.datasets import load_iris - from sklearn.neighbors import KNeighborsClassifier - - from hyperactive.experiment.integrations import SklearnCvExperiment - from hyperactive.opt.gridsearch import GridSearchSk - from hyperactive.opt.random_search import RandomSearchSk - - X, y = load_iris(return_X_y=True) - exp = SklearnCvExperiment(estimator=KNeighborsClassifier(), X=X, y=y, cv=2) - - sklearn_space = { - "n_neighbors": [1, 3, 5], - "weights": ["uniform", "distance"], - } - - if optimizer_name == "GridSearchSk": - opt = GridSearchSk(unified_space=sklearn_space, experiment=exp) - else: - opt = RandomSearchSk(unified_space=sklearn_space, n_iter=3, experiment=exp) - - best_params = opt.solve() - - # Verify result contains original categorical string values - assert isinstance(best_params, dict) - assert "weights" in best_params - assert best_params["weights"] in ["uniform", "distance"] - assert isinstance(best_params["weights"], str) diff --git a/src/hyperactive/tests/test_unified_search_space.py b/src/hyperactive/tests/test_unified_search_space.py index 44a014f5..d2a531e5 100644 --- a/src/hyperactive/tests/test_unified_search_space.py +++ b/src/hyperactive/tests/test_unified_search_space.py @@ -216,3 +216,143 @@ def test_optuna_capability_tags(self): assert opt.get_tag("capability:continuous") is True assert opt.get_tag("capability:categorical") is True assert opt.get_tag("capability:log_scale") is True + + +# All GFO optimizers +_GFO_OPTIMIZERS = [ + "RandomSearch", + "HillClimbing", + "StochasticHillClimbing", + "RepulsingHillClimbing", + "SimulatedAnnealing", + "RandomRestartHillClimbing", + "ParallelTempering", + "ParticleSwarmOptimizer", + "SpiralOptimization", + "GeneticAlgorithm", + "EvolutionStrategy", + "DifferentialEvolution", + "BayesianOptimizer", + "TreeStructuredParzenEstimators", + "ForestOptimizer", + "GridSearch", + "PatternSearch", + "DirectAlgorithm", + "LipschitzOptimizer", + "PowellsMethod", + "DownhillSimplexOptimizer", +] + +_OPTUNA_OPTIMIZERS = [ + "TPEOptimizer", + "RandomOptimizer", + "GridOptimizer", + "GPOptimizer", + "QMCOptimizer", + # CmaEsOptimizer excluded - only supports continuous ranges + # NSGAIIOptimizer, NSGAIIIOptimizer excluded - multi-objective +] + +_SKLEARN_OPTIMIZERS = [ + "GridSearchSk", + "RandomSearchSk", +] + + +class TestAllOptimizersWithCategoricalSearchSpace: + """Test that all optimizers work with categorical search spaces.""" + + @pytest.fixture + def categorical_search_space(self): + """Search space with categorical and numeric values.""" + return { + "x": [0, 1, 2, 3, 4], + "option": ["a", "b", "c"], + } + + @pytest.fixture + def function_experiment(self): + """Simple function experiment for fast testing.""" + from hyperactive.experiment.func import FunctionExperiment + + def objective(params): + # Simple objective: x value + bonus for option + bonus = {"a": 0, "b": 1, "c": 2} + return params["x"] + bonus[params["option"]] + + return FunctionExperiment(objective) + + @pytest.mark.parametrize("optimizer_name", _GFO_OPTIMIZERS) + def test_gfo_optimizer_with_categorical( + self, optimizer_name, categorical_search_space, function_experiment + ): + """GFO optimizers should handle categorical search spaces.""" + from hyperactive.opt import gfo + + OptCls = getattr(gfo, optimizer_name) + + opt = OptCls( + search_space=categorical_search_space, + n_iter=3, + experiment=function_experiment, + ) + best_params = opt.solve() + + # Verify result contains original categorical string values + assert isinstance(best_params, dict) + assert "x" in best_params + assert "option" in best_params + assert best_params["option"] in ["a", "b", "c"] + assert isinstance(best_params["option"], str) + + @pytest.mark.parametrize("optimizer_name", _OPTUNA_OPTIMIZERS) + def test_optuna_optimizer_with_categorical( + self, optimizer_name, categorical_search_space, function_experiment + ): + """Optuna optimizers should handle categorical search spaces.""" + from hyperactive.opt import optuna + + OptCls = getattr(optuna, optimizer_name) + + opt = OptCls( + unified_space=categorical_search_space, + n_trials=3, + experiment=function_experiment, + ) + best_params = opt.solve() + + # Verify result contains original categorical string values + assert isinstance(best_params, dict) + assert "x" in best_params + assert "option" in best_params + assert best_params["option"] in ["a", "b", "c"] + assert isinstance(best_params["option"], str) + + @pytest.mark.parametrize("optimizer_name", _SKLEARN_OPTIMIZERS) + def test_sklearn_optimizer_with_categorical(self, optimizer_name): + """Sklearn-based optimizers should handle categorical search spaces.""" + from sklearn.neighbors import KNeighborsClassifier + + from hyperactive.opt.gridsearch import GridSearchSk + from hyperactive.opt.random_search import RandomSearchSk + + X, y = load_iris(return_X_y=True) + exp = SklearnCvExperiment(estimator=KNeighborsClassifier(), X=X, y=y, cv=2) + + sklearn_space = { + "n_neighbors": [1, 3, 5], + "weights": ["uniform", "distance"], + } + + if optimizer_name == "GridSearchSk": + opt = GridSearchSk(unified_space=sklearn_space, experiment=exp) + else: + opt = RandomSearchSk(unified_space=sklearn_space, n_iter=3, experiment=exp) + + best_params = opt.solve() + + # Verify result contains original categorical string values + assert isinstance(best_params, dict) + assert "weights" in best_params + assert best_params["weights"] in ["uniform", "distance"] + assert isinstance(best_params["weights"], str) From d59871481584ae81fb30af7390106f0bb57f6b6a Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 18 Jan 2026 20:47:20 +0100 Subject: [PATCH 8/8] add support for continuous dimensions --- .../opt/_adapters/_adapter_utils.py | 23 +- .../opt/_adapters/_base_optuna_adapter.py | 60 +++- src/hyperactive/opt/_adapters/_gfo.py | 10 +- .../opt/_adapters/_search_space_adapter.py | 277 ++++++++++++++++-- src/hyperactive/opt/optuna/_grid_optimizer.py | 2 + .../tests/test_search_space_adapter.py | 194 ++++++++++++ .../tests/test_unified_search_space.py | 206 ++++++++++++- 7 files changed, 728 insertions(+), 44 deletions(-) diff --git a/src/hyperactive/opt/_adapters/_adapter_utils.py b/src/hyperactive/opt/_adapters/_adapter_utils.py index 16b78094..9be11ce7 100644 --- a/src/hyperactive/opt/_adapters/_adapter_utils.py +++ b/src/hyperactive/opt/_adapters/_adapter_utils.py @@ -29,8 +29,11 @@ def adapt_search_space(experiment, search_config, capabilities): """Adapt search space and experiment for backend capabilities. If the backend doesn't support certain search space features - (e.g., categorical values), this function encodes the search space - and wraps the experiment to handle encoding/decoding transparently. + (e.g., categorical values, continuous ranges), this function: + - Validates the search space format + - Encodes categorical dimensions (strings to integers) + - Discretizes continuous dimensions (tuples to lists) + - Wraps the experiment to decode parameters during scoring Parameters ---------- @@ -46,9 +49,14 @@ def adapt_search_space(experiment, search_config, capabilities): experiment : BaseExperiment The experiment, possibly wrapped for decoding. search_config : dict - The search config, possibly with encoded search space. + The search config, possibly with encoded/discretized search space. adapter : SearchSpaceAdapter or None - The adapter if encoding was applied, None otherwise. + The adapter if adaptation was applied, None otherwise. + + Raises + ------ + ValueError, TypeError + If the search space format is invalid. """ search_space_key = detect_search_space_key(search_config) @@ -59,11 +67,14 @@ def adapt_search_space(experiment, search_config, capabilities): # Create adapter with backend capabilities adapter = SearchSpaceAdapter(search_config[search_space_key], capabilities) + # Validate search space format + adapter.validate() + # Backend supports all features - pass through unchanged - if not adapter.needs_encoding: + if not adapter.needs_adaptation: return experiment, search_config, None - # Encoding needed - transform search space and wrap experiment + # Adaptation needed - transform search space and wrap experiment encoded_config = search_config.copy() encoded_config[search_space_key] = adapter.encode() wrapped_experiment = adapter.wrap_experiment(experiment) diff --git a/src/hyperactive/opt/_adapters/_base_optuna_adapter.py b/src/hyperactive/opt/_adapters/_base_optuna_adapter.py index 38645944..3581445a 100644 --- a/src/hyperactive/opt/_adapters/_base_optuna_adapter.py +++ b/src/hyperactive/opt/_adapters/_base_optuna_adapter.py @@ -117,13 +117,9 @@ def _suggest_params(self, trial, param_space): for key, space in param_space.items(): if hasattr(space, "suggest"): # optuna distribution object params[key] = trial._suggest(space, key) - elif isinstance(space, tuple) and len(space) == 2: - # Tuples are treated as ranges (low, high) - low, high = space - if isinstance(low, int) and isinstance(high, int): - params[key] = trial.suggest_int(key, low, high) - else: - params[key] = trial.suggest_float(key, low, high, log=False) + elif isinstance(space, tuple): + # Tuples are continuous ranges in unified format + params[key] = self._suggest_continuous(trial, key, space) elif isinstance(space, list): # Lists are treated as categorical choices params[key] = trial.suggest_categorical(key, space) @@ -131,6 +127,56 @@ def _suggest_params(self, trial, param_space): raise ValueError(f"Invalid parameter space for key '{key}': {space}") return params + def _suggest_continuous(self, trial, key, space): + """Suggest a continuous parameter from a tuple specification. + + Handles unified tuple formats: + - (low, high) - linear scale + - (low, high, "log") - log scale + - (low, high, n_points) - linear scale (n_points ignored for Optuna) + - (low, high, n_points, "log") - log scale (n_points ignored for Optuna) + + Parameters + ---------- + trial : optuna.Trial + The Optuna trial object + key : str + The parameter name + space : tuple + The continuous range specification + + Returns + ------- + float or int + The suggested value + """ + if len(space) < 2: + raise ValueError( + f"Parameter '{key}': continuous range needs at least 2 values " + f"(low, high), got {len(space)}." + ) + + low, high = space[0], space[1] + log_scale = False + + # Parse optional arguments + if len(space) == 3: + third = space[2] + if isinstance(third, str) and third.lower() == "log": + log_scale = True + # If third is int/float, it's n_points - ignore for Optuna + elif len(space) == 4: + # (low, high, n_points, "log") + fourth = space[3] + if isinstance(fourth, str) and fourth.lower() == "log": + log_scale = True + + # Suggest based on type + if isinstance(low, int) and isinstance(high, int): + return trial.suggest_int(key, low, high, log=log_scale) + else: + return trial.suggest_float(key, low, high, log=log_scale) + def _objective(self, trial): """Objective function for Optuna optimization. diff --git a/src/hyperactive/opt/_adapters/_gfo.py b/src/hyperactive/opt/_adapters/_gfo.py index d51622e3..ae52fdd5 100644 --- a/src/hyperactive/opt/_adapters/_gfo.py +++ b/src/hyperactive/opt/_adapters/_gfo.py @@ -25,7 +25,7 @@ class _BaseGFOadapter(BaseOptimizer): "python_dependencies": ["gradient-free-optimizers>=1.5.0"], # Search space capabilities "capability:discrete": True, - "capability:continuous": True, + "capability:continuous": False, # GFO needs lists, not (low, high) tuples "capability:categorical": False, # GFO only supports numeric values "capability:constraints": True, } @@ -78,7 +78,9 @@ def get_search_config(self): search_config = self._handle_gfo_defaults(search_config) - search_config["search_space"] = self._to_dict_np(search_config["search_space"]) + # Note: _to_dict_np is called in _solve(), after SearchSpaceAdapter processes + # continuous tuples. If we convert here, tuples like (1e-4, 1e-1, "log") + # would become numpy arrays with strings before the adapter can discretize them. return search_config @@ -151,6 +153,10 @@ def _solve(self, experiment, **search_config): n_iter = search_config.pop("n_iter", 100) max_time = search_config.pop("max_time", None) + # Convert search_space lists to numpy arrays (GFO requirement) + # This must happen after SearchSpaceAdapter has processed continuous tuples + search_config["search_space"] = self._to_dict_np(search_config["search_space"]) + gfo_cls = self._get_gfo_class() gfopt = gfo_cls(**search_config) diff --git a/src/hyperactive/opt/_adapters/_search_space_adapter.py b/src/hyperactive/opt/_adapters/_search_space_adapter.py index 62c2c08e..9f11849d 100644 --- a/src/hyperactive/opt/_adapters/_search_space_adapter.py +++ b/src/hyperactive/opt/_adapters/_search_space_adapter.py @@ -1,64 +1,236 @@ """Search space adapter for optimizer backends.""" # copyright: hyperactive developers, MIT License (see LICENSE file) +import numpy as np + __all__ = ["SearchSpaceAdapter"] +# Default number of points for discretizing continuous dimensions +DEFAULT_N_POINTS = 100 + class SearchSpaceAdapter: """Adapts search spaces for optimizer backends. - Handles encoding/decoding of categorical dimensions when the backend - doesn't support them natively. + Handles: + - Encoding/decoding of categorical dimensions (strings to integers) + - Discretization of continuous dimensions (tuples to lists) Parameters ---------- - search_space : dict[str, list] - The search space as a dictionary mapping parameter names to value lists. + search_space : dict[str, list | tuple] + The search space as a dictionary mapping parameter names to: + - list: discrete values (categorical if contains strings, numeric otherwise) + - tuple: continuous range, formats supported: + - (low, high) - linear scale, 100 points + - (low, high, "log") - log scale, 100 points + - (low, high, n_points) - linear scale, n_points + - (low, high, n_points, "log") - log scale, n_points capabilities : dict Backend capability tags, e.g., {"categorical": True, "continuous": False}. Attributes ---------- - needs_encoding : bool - Whether any dimensions require encoding. + needs_adaptation : bool + Whether any dimensions require encoding or discretization. categorical_mapping : dict Mapping of {param_name: {index: original_value}} for encoded dimensions. Examples -------- - >>> space = {"kernel": ["rbf", "linear"], "C": [0.1, 1, 10]} - >>> adapter = SearchSpaceAdapter(space, capabilities={"categorical": False}) + >>> space = {"kernel": ["rbf", "linear"], "C": (0.01, 10.0)} + >>> adapter = SearchSpaceAdapter(space, {"categorical": False, "continuous": False}) + >>> adapter.validate() # Raises if invalid >>> encoded = adapter.encode() >>> encoded - {"kernel": [0, 1], "C": [0.1, 1, 10]} - >>> adapter.decode({"kernel": 1, "C": 1}) - {"kernel": "linear", "C": 1} + {"kernel": [0, 1], "C": [0.01, 0.11, 0.21, ...]} # 100 points + >>> adapter.decode({"kernel": 1, "C": 0.5}) + {"kernel": "linear", "C": 0.5} """ def __init__(self, search_space: dict, capabilities: dict): self._original_space = search_space self._capabilities = capabilities self._categorical_mapping = {} - self._needs_encoding = self._check_needs_encoding() + self._continuous_dims = set() + @property + def needs_adaptation(self) -> bool: + """Whether any adaptation (encoding or discretization) is needed.""" + return self._needs_categorical_encoding() or self._needs_discretization() + + # Keep old property for backwards compatibility @property def needs_encoding(self) -> bool: - """Whether encoding is needed.""" - return self._needs_encoding + """Whether encoding is needed (deprecated, use needs_adaptation).""" + return self.needs_adaptation @property def categorical_mapping(self) -> dict: """Mapping of encoded categorical dimensions.""" return self._categorical_mapping - def _check_needs_encoding(self) -> bool: - """Check if encoding is needed based on capabilities and search space.""" - # If backend supports categorical natively, no encoding needed + # ------------------------------------------------------------------------- + # Validation + # ------------------------------------------------------------------------- + + def validate(self): + """Validate the search space format. + + Raises + ------ + ValueError + If the search space contains invalid definitions. + TypeError + If values have unexpected types. + """ + if not isinstance(self._original_space, dict): + raise TypeError( + f"Search space must be a dict, got {type(self._original_space).__name__}" + ) + + if not self._original_space: + raise ValueError("Search space cannot be empty") + + for name, values in self._original_space.items(): + self._validate_dimension(name, values) + + def _validate_dimension(self, name: str, values): + """Validate a single dimension.""" + if isinstance(values, tuple): + self._validate_continuous(name, values) + elif isinstance(values, (list, np.ndarray)): + self._validate_discrete(name, values) + else: + raise TypeError( + f"Parameter '{name}': expected list (discrete) or tuple (continuous), " + f"got {type(values).__name__}. " + f"Use [a, b, c] for discrete values or (low, high) for continuous ranges." + ) + + def _validate_continuous(self, name: str, values: tuple): + """Validate a continuous dimension (tuple).""" + if len(values) < 2: + raise ValueError( + f"Parameter '{name}': continuous range needs at least 2 values " + f"(low, high), got {len(values)}. " + f"Example: (0.01, 10.0) or (1e-5, 1e-1, 'log')" + ) + + if len(values) > 4: + raise ValueError( + f"Parameter '{name}': continuous range has too many values " + f"({len(values)}). Supported formats:\n" + f" (low, high) - linear, 100 points\n" + f" (low, high, 'log') - log scale, 100 points\n" + f" (low, high, n_points) - linear, n_points\n" + f" (low, high, n_points, 'log') - log scale, n_points" + ) + + low, high = values[0], values[1] + + # Check low and high are numeric + if not isinstance(low, (int, float)) or not isinstance(high, (int, float)): + raise TypeError( + f"Parameter '{name}': low and high must be numeric, " + f"got low={type(low).__name__}, high={type(high).__name__}" + ) + + # Check low < high + if low >= high: + raise ValueError( + f"Parameter '{name}': low ({low}) must be less than high ({high})" + ) + + # Parse and validate optional arguments + n_points, log_scale = self._parse_continuous_options(name, values) + + # Check log scale with non-positive values + if log_scale and low <= 0: + raise ValueError( + f"Parameter '{name}': log scale requires positive values, " + f"but low={low}. Use linear scale or adjust the range." + ) + + # Check n_points + if n_points < 2: + raise ValueError( + f"Parameter '{name}': n_points must be at least 2, got {n_points}" + ) + + def _validate_discrete(self, name: str, values): + """Validate a discrete dimension (list).""" + if len(values) == 0: + raise ValueError( + f"Parameter '{name}': discrete list cannot be empty. " + f"Provide at least one value." + ) + + def _parse_continuous_options(self, name: str, values: tuple): + """Parse optional n_points and log_scale from tuple. + + Returns + ------- + n_points : int + log_scale : bool + """ + n_points = DEFAULT_N_POINTS + log_scale = False + + if len(values) == 2: + # (low, high) + pass + elif len(values) == 3: + # (low, high, "log") or (low, high, n_points) + third = values[2] + if isinstance(third, str): + if third.lower() != "log": + raise ValueError( + f"Parameter '{name}': unknown scale '{third}'. " + f"Use 'log' for logarithmic scale." + ) + log_scale = True + elif isinstance(third, (int, float)): + n_points = int(third) + else: + raise TypeError( + f"Parameter '{name}': third value must be 'log' or n_points (int), " + f"got {type(third).__name__}" + ) + elif len(values) == 4: + # (low, high, n_points, "log") + third, fourth = values[2], values[3] + if not isinstance(third, (int, float)): + raise TypeError( + f"Parameter '{name}': n_points must be numeric, " + f"got {type(third).__name__}" + ) + n_points = int(third) + if not isinstance(fourth, str) or fourth.lower() != "log": + raise ValueError( + f"Parameter '{name}': fourth value must be 'log', " + f"got '{fourth}'" + ) + log_scale = True + + return n_points, log_scale + + # ------------------------------------------------------------------------- + # Detection + # ------------------------------------------------------------------------- + + def _needs_categorical_encoding(self) -> bool: + """Check if categorical encoding is needed.""" if self._capabilities.get("categorical", True): return False - # Check if search space contains categorical values return self._has_categorical_values() + def _needs_discretization(self) -> bool: + """Check if continuous discretization is needed.""" + if self._capabilities.get("continuous", False): + return False + return self._has_continuous_values() + def _has_categorical_values(self) -> bool: """Detect if any dimension contains string values.""" for values in self._original_space.values(): @@ -66,51 +238,100 @@ def _has_categorical_values(self) -> bool: return True return False + def _has_continuous_values(self) -> bool: + """Detect if any dimension is continuous (tuple).""" + for values in self._original_space.values(): + if self._is_continuous(values): + return True + return False + def _is_categorical(self, values) -> bool: - """Check if a dimension's values are categorical (contain strings).""" + """Check if a dimension's values are categorical (list containing strings).""" + if isinstance(values, tuple): + return False # Tuples are continuous, not categorical if hasattr(values, "__iter__") and not isinstance(values, str): return any(isinstance(v, str) for v in values) return False + def _is_continuous(self, values) -> bool: + """Check if a dimension is continuous (tuple).""" + return isinstance(values, tuple) + + # ------------------------------------------------------------------------- + # Encoding / Discretization + # ------------------------------------------------------------------------- + def encode(self) -> dict: """Encode the search space for the backend. - Categorical dimensions (containing strings) are converted to - integer indices. The mapping is stored for later decoding. + - Categorical dimensions (lists with strings) are converted to integers + - Continuous dimensions (tuples) are discretized to lists Returns ------- dict - Encoded search space with categorical values as integers. + Encoded search space ready for the backend. """ - if not self._needs_encoding: + if not self.needs_adaptation: return self._original_space self._categorical_mapping = {} + self._continuous_dims = set() encoded = {} + needs_cat_encoding = self._needs_categorical_encoding() + needs_discretize = self._needs_discretization() + for name, values in self._original_space.items(): - if self._is_categorical(values): - # Store mapping: {index: original_value} + if self._is_continuous(values) and needs_discretize: + # Discretize continuous dimension + encoded[name] = self._discretize(name, values) + self._continuous_dims.add(name) + elif self._is_categorical(values) and needs_cat_encoding: + # Encode categorical dimension self._categorical_mapping[name] = {i: v for i, v in enumerate(values)} - # Replace with integer indices encoded[name] = list(range(len(values))) else: encoded[name] = values return encoded + def _discretize(self, name: str, values: tuple) -> list: + """Convert continuous range to discrete list of values. + + Parameters + ---------- + name : str + Parameter name (for error messages). + values : tuple + Continuous range specification. + + Returns + ------- + list + Discretized values. + """ + low, high = values[0], values[1] + n_points, log_scale = self._parse_continuous_options(name, values) + + if log_scale: + return np.geomspace(low, high, n_points).tolist() + else: + return np.linspace(low, high, n_points).tolist() + def decode(self, params: dict) -> dict: """Decode backend results to original format. Integer indices for categorical dimensions are converted back to their original string values. + Note: Continuous dimensions don't need decoding - the discretized + float values are already valid. + Parameters ---------- params : dict - Parameter dictionary from the optimizer, potentially with - encoded categorical values. + Parameter dictionary from the optimizer. Returns ------- diff --git a/src/hyperactive/opt/optuna/_grid_optimizer.py b/src/hyperactive/opt/optuna/_grid_optimizer.py index 9def1cd9..bf84557b 100644 --- a/src/hyperactive/opt/optuna/_grid_optimizer.py +++ b/src/hyperactive/opt/optuna/_grid_optimizer.py @@ -58,6 +58,8 @@ class GridOptimizer(_BaseOptunaAdapter): "info:explore_vs_exploit": "explore", "info:compute": "low", "python_dependencies": ["optuna"], + # Grid search needs discrete values, cannot sample from continuous ranges + "capability:continuous": False, } def __init__( diff --git a/src/hyperactive/tests/test_search_space_adapter.py b/src/hyperactive/tests/test_search_space_adapter.py index 95f5af11..b0501eca 100644 --- a/src/hyperactive/tests/test_search_space_adapter.py +++ b/src/hyperactive/tests/test_search_space_adapter.py @@ -188,6 +188,200 @@ def test_multiple_categorical_dimensions(self): assert decoded == {"kernel": "rbf", "solver": "adam"} +class TestSearchSpaceValidation: + """Tests for search space validation.""" + + def test_validate_empty_space_raises(self): + """Empty search space raises ValueError.""" + adapter = SearchSpaceAdapter({}, capabilities={}) + + with pytest.raises(ValueError, match="cannot be empty"): + adapter.validate() + + def test_validate_non_dict_raises(self): + """Non-dict search space raises TypeError.""" + adapter = SearchSpaceAdapter(["C", "gamma"], capabilities={}) + + with pytest.raises(TypeError, match="must be a dict"): + adapter.validate() + + def test_validate_invalid_type_raises(self): + """Invalid dimension type raises TypeError.""" + adapter = SearchSpaceAdapter({"C": 0.1}, capabilities={}) + + with pytest.raises(TypeError, match="expected list.*or tuple"): + adapter.validate() + + def test_validate_empty_list_raises(self): + """Empty discrete list raises ValueError.""" + adapter = SearchSpaceAdapter({"C": []}, capabilities={}) + + with pytest.raises(ValueError, match="cannot be empty"): + adapter.validate() + + def test_validate_continuous_too_few_values_raises(self): + """Continuous tuple with < 2 values raises ValueError.""" + adapter = SearchSpaceAdapter({"C": (0.1,)}, capabilities={}) + + with pytest.raises(ValueError, match="at least 2 values"): + adapter.validate() + + def test_validate_continuous_too_many_values_raises(self): + """Continuous tuple with > 4 values raises ValueError.""" + adapter = SearchSpaceAdapter({"C": (0.1, 10, 50, "log", "extra")}, capabilities={}) + + with pytest.raises(ValueError, match="too many values"): + adapter.validate() + + def test_validate_continuous_low_ge_high_raises(self): + """Continuous with low >= high raises ValueError.""" + adapter = SearchSpaceAdapter({"C": (10, 0.1)}, capabilities={}) + + with pytest.raises(ValueError, match="must be less than"): + adapter.validate() + + def test_validate_continuous_log_non_positive_raises(self): + """Log scale with non-positive low raises ValueError.""" + adapter = SearchSpaceAdapter({"C": (-1, 10, "log")}, capabilities={}) + + with pytest.raises(ValueError, match="log scale requires positive"): + adapter.validate() + + def test_validate_continuous_n_points_too_small_raises(self): + """n_points < 2 raises ValueError.""" + adapter = SearchSpaceAdapter({"C": (0.1, 10, 1)}, capabilities={}) + + with pytest.raises(ValueError, match="n_points must be at least 2"): + adapter.validate() + + def test_validate_continuous_invalid_scale_raises(self): + """Invalid scale string raises ValueError.""" + adapter = SearchSpaceAdapter({"C": (0.1, 10, "linear")}, capabilities={}) + + with pytest.raises(ValueError, match="unknown scale"): + adapter.validate() + + def test_validate_valid_space_passes(self): + """Valid search space passes validation.""" + space = { + "C": (0.1, 10), + "gamma": (1e-5, 1e-1, "log"), + "kernel": ["rbf", "linear"], + "n_estimators": [10, 50, 100], + } + adapter = SearchSpaceAdapter(space, capabilities={}) + + # Should not raise + adapter.validate() + + +class TestContinuousDiscretization: + """Tests for continuous dimension discretization.""" + + def test_discretize_linear_default_points(self): + """Linear discretization with default 100 points.""" + np = pytest.importorskip("numpy") + + space = {"C": (0.1, 10.0)} + adapter = SearchSpaceAdapter(space, capabilities={"continuous": False}) + adapter.validate() + + encoded = adapter.encode() + + assert "C" in encoded + assert len(encoded["C"]) == 100 + assert encoded["C"][0] == pytest.approx(0.1) + assert encoded["C"][-1] == pytest.approx(10.0) + + def test_discretize_log_scale(self): + """Log scale discretization.""" + np = pytest.importorskip("numpy") + + space = {"lr": (1e-5, 1e-1, "log")} + adapter = SearchSpaceAdapter(space, capabilities={"continuous": False}) + adapter.validate() + + encoded = adapter.encode() + + assert len(encoded["lr"]) == 100 + assert encoded["lr"][0] == pytest.approx(1e-5) + assert encoded["lr"][-1] == pytest.approx(1e-1) + # Log scale should have geometric progression + # Ratio between consecutive values should be constant + ratio1 = encoded["lr"][1] / encoded["lr"][0] + ratio50 = encoded["lr"][51] / encoded["lr"][50] + assert ratio1 == pytest.approx(ratio50, rel=1e-5) + + def test_discretize_custom_n_points(self): + """Discretization with custom n_points.""" + space = {"C": (0.1, 10.0, 10)} + adapter = SearchSpaceAdapter(space, capabilities={"continuous": False}) + adapter.validate() + + encoded = adapter.encode() + + assert len(encoded["C"]) == 10 + + def test_discretize_custom_n_points_log(self): + """Discretization with custom n_points and log scale.""" + space = {"lr": (1e-4, 1e-1, 20, "log")} + adapter = SearchSpaceAdapter(space, capabilities={"continuous": False}) + adapter.validate() + + encoded = adapter.encode() + + assert len(encoded["lr"]) == 20 + assert encoded["lr"][0] == pytest.approx(1e-4) + assert encoded["lr"][-1] == pytest.approx(1e-1) + + def test_no_discretization_when_supported(self): + """No discretization when backend supports continuous.""" + space = {"C": (0.1, 10.0)} + adapter = SearchSpaceAdapter(space, capabilities={"continuous": True}) + + assert adapter.needs_adaptation is False + assert adapter.encode() is space + + def test_mixed_discrete_and_continuous(self): + """Mixed discrete and continuous dimensions.""" + space = { + "C": (0.1, 10.0, 5), + "kernel": ["rbf", "linear"], + "n_estimators": [10, 50, 100], + } + adapter = SearchSpaceAdapter( + space, capabilities={"categorical": False, "continuous": False} + ) + adapter.validate() + + encoded = adapter.encode() + + # Continuous discretized + assert len(encoded["C"]) == 5 + # Categorical encoded + assert encoded["kernel"] == [0, 1] + # Numeric discrete unchanged + assert encoded["n_estimators"] == [10, 50, 100] + + def test_needs_adaptation_continuous_only(self): + """needs_adaptation True when only continuous needs discretization.""" + space = {"C": (0.1, 10.0), "gamma": [0.01, 0.1, 1]} + adapter = SearchSpaceAdapter( + space, capabilities={"categorical": True, "continuous": False} + ) + + assert adapter.needs_adaptation is True + + def test_needs_adaptation_categorical_only(self): + """needs_adaptation True when only categorical needs encoding.""" + space = {"kernel": ["rbf", "linear"], "C": [0.1, 1, 10]} + adapter = SearchSpaceAdapter( + space, capabilities={"categorical": False, "continuous": True} + ) + + assert adapter.needs_adaptation is True + + class TestCapabilityTags: """Tests for capability tags related to categorical encoding.""" diff --git a/src/hyperactive/tests/test_unified_search_space.py b/src/hyperactive/tests/test_unified_search_space.py index d2a531e5..375e3ce0 100644 --- a/src/hyperactive/tests/test_unified_search_space.py +++ b/src/hyperactive/tests/test_unified_search_space.py @@ -202,7 +202,7 @@ def test_gfo_capability_tags(self): opt = RandomSearch.create_test_instance() assert opt.get_tag("capability:discrete") is True - assert opt.get_tag("capability:continuous") is True + assert opt.get_tag("capability:continuous") is False # GFO needs lists, not tuples assert opt.get_tag("capability:categorical") is False # GFO only numeric assert opt.get_tag("capability:constraints") is True @@ -356,3 +356,207 @@ def test_sklearn_optimizer_with_categorical(self, optimizer_name): assert "weights" in best_params assert best_params["weights"] in ["uniform", "distance"] assert isinstance(best_params["weights"], str) + + +class TestAllOptimizersWithContinuousSearchSpace: + """Test that all optimizers work with continuous search spaces (tuples).""" + + @pytest.fixture + def continuous_search_space(self): + """Search space with continuous dimensions (tuples).""" + return { + "x": (0.0, 10.0), # linear, default 100 points + "y": (1e-4, 1e-1, "log"), # log scale, 100 points + } + + @pytest.fixture + def continuous_search_space_custom_points(self): + """Search space with custom n_points.""" + return { + "x": (0.0, 10.0, 20), # linear, 20 points + "y": (1e-4, 1e-1, 15, "log"), # log scale, 15 points + } + + @pytest.fixture + def mixed_search_space(self): + """Search space with continuous, discrete, and categorical dimensions.""" + return { + "x": (0.0, 5.0, 10), # continuous + "n": [1, 2, 3, 4, 5], # discrete numeric + "option": ["a", "b", "c"], # categorical + } + + @pytest.fixture + def continuous_function_experiment(self): + """Simple function experiment for continuous optimization.""" + from hyperactive.experiment.func import FunctionExperiment + + def objective(params): + # Simple objective: maximize when x and y are in middle of range + x = params["x"] + y = params["y"] + return -(x - 5.0) ** 2 - (y - 0.01) ** 2 + + return FunctionExperiment(objective) + + @pytest.fixture + def mixed_function_experiment(self): + """Function experiment for mixed search space.""" + from hyperactive.experiment.func import FunctionExperiment + + def objective(params): + bonus = {"a": 0, "b": 1, "c": 2} + return params["x"] + params["n"] + bonus[params["option"]] + + return FunctionExperiment(objective) + + @pytest.mark.parametrize("optimizer_name", _GFO_OPTIMIZERS) + def test_gfo_optimizer_with_continuous( + self, optimizer_name, continuous_search_space, continuous_function_experiment + ): + """GFO optimizers should handle continuous search spaces via discretization.""" + from hyperactive.opt import gfo + + OptCls = getattr(gfo, optimizer_name) + + opt = OptCls( + search_space=continuous_search_space, + n_iter=3, + experiment=continuous_function_experiment, + ) + best_params = opt.solve() + + assert isinstance(best_params, dict) + assert "x" in best_params + assert "y" in best_params + # Values should be within the specified ranges + assert 0.0 <= best_params["x"] <= 10.0 + assert 1e-4 <= best_params["y"] <= 1e-1 + # Values should be floats + assert isinstance(best_params["x"], float) + assert isinstance(best_params["y"], float) + + @pytest.mark.parametrize("optimizer_name", _GFO_OPTIMIZERS) + def test_gfo_optimizer_with_custom_n_points( + self, + optimizer_name, + continuous_search_space_custom_points, + continuous_function_experiment, + ): + """GFO optimizers should handle custom n_points in continuous dimensions.""" + from hyperactive.opt import gfo + + OptCls = getattr(gfo, optimizer_name) + + opt = OptCls( + search_space=continuous_search_space_custom_points, + n_iter=3, + experiment=continuous_function_experiment, + ) + best_params = opt.solve() + + assert isinstance(best_params, dict) + assert 0.0 <= best_params["x"] <= 10.0 + assert 1e-4 <= best_params["y"] <= 1e-1 + + @pytest.mark.parametrize("optimizer_name", _GFO_OPTIMIZERS) + def test_gfo_optimizer_with_mixed_space( + self, optimizer_name, mixed_search_space, mixed_function_experiment + ): + """GFO optimizers should handle mixed continuous/discrete/categorical.""" + from hyperactive.opt import gfo + + OptCls = getattr(gfo, optimizer_name) + + opt = OptCls( + search_space=mixed_search_space, + n_iter=3, + experiment=mixed_function_experiment, + ) + best_params = opt.solve() + + assert isinstance(best_params, dict) + # Continuous + assert 0.0 <= best_params["x"] <= 5.0 + assert isinstance(best_params["x"], float) + # Discrete + assert best_params["n"] in [1, 2, 3, 4, 5] + # Categorical (should be decoded back to string) + assert best_params["option"] in ["a", "b", "c"] + assert isinstance(best_params["option"], str) + + @pytest.mark.parametrize("optimizer_name", _OPTUNA_OPTIMIZERS) + def test_optuna_optimizer_with_continuous( + self, optimizer_name, continuous_search_space, continuous_function_experiment + ): + """Optuna optimizers should handle continuous search spaces natively.""" + from hyperactive.opt import optuna + + OptCls = getattr(optuna, optimizer_name) + + opt = OptCls( + unified_space=continuous_search_space, + n_trials=3, + experiment=continuous_function_experiment, + ) + best_params = opt.solve() + + assert isinstance(best_params, dict) + assert "x" in best_params + assert "y" in best_params + assert 0.0 <= best_params["x"] <= 10.0 + assert 1e-4 <= best_params["y"] <= 1e-1 + + @pytest.mark.parametrize("optimizer_name", _OPTUNA_OPTIMIZERS) + def test_optuna_optimizer_with_mixed_space( + self, optimizer_name, mixed_search_space, mixed_function_experiment + ): + """Optuna optimizers should handle mixed continuous/discrete/categorical.""" + from hyperactive.opt import optuna + + OptCls = getattr(optuna, optimizer_name) + + opt = OptCls( + unified_space=mixed_search_space, + n_trials=3, + experiment=mixed_function_experiment, + ) + best_params = opt.solve() + + assert isinstance(best_params, dict) + # Continuous + assert 0.0 <= best_params["x"] <= 5.0 + # Discrete + assert best_params["n"] in [1, 2, 3, 4, 5] + # Categorical + assert best_params["option"] in ["a", "b", "c"] + assert isinstance(best_params["option"], str) + + @pytest.mark.parametrize("optimizer_name", _SKLEARN_OPTIMIZERS) + def test_sklearn_optimizer_with_continuous(self, optimizer_name): + """Sklearn-based optimizers should handle continuous via discretization.""" + from sklearn.neighbors import KNeighborsClassifier + + from hyperactive.opt.gridsearch import GridSearchSk + from hyperactive.opt.random_search import RandomSearchSk + + X, y = load_iris(return_X_y=True) + exp = SklearnCvExperiment(estimator=KNeighborsClassifier(), X=X, y=y, cv=2) + + # Continuous search space for sklearn + sklearn_space = { + "n_neighbors": [1, 3, 5, 7], # discrete (sklearn needs this) + "leaf_size": (10, 50, 5), # continuous, 5 points + } + + if optimizer_name == "GridSearchSk": + opt = GridSearchSk(unified_space=sklearn_space, experiment=exp) + else: + opt = RandomSearchSk(unified_space=sklearn_space, n_iter=3, experiment=exp) + + best_params = opt.solve() + + assert isinstance(best_params, dict) + assert "n_neighbors" in best_params + assert "leaf_size" in best_params + assert 10 <= best_params["leaf_size"] <= 50