diff --git a/examples/hyperactive_intro.ipynb b/examples/hyperactive_intro.ipynb index 00e1710b..7cf2c2f2 100644 --- a/examples/hyperactive_intro.ipynb +++ b/examples/hyperactive_intro.ipynb @@ -392,7 +392,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "e9a07a73", "metadata": {}, "outputs": [ @@ -408,7 +408,7 @@ } ], "source": [ - "from hyperactive.opt import GridSearch\n", + "from hyperactive.opt import GridSearchSk as GridSearch\n", "\n", "param_grid = {\n", " \"C\": [0.01, 0.1, 1, 10],\n", @@ -492,7 +492,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "9a13b4f3", "metadata": {}, "outputs": [ @@ -516,7 +516,7 @@ ], "source": [ "import numpy as np\n", - "from hyperactive.opt.gfo import HillClimbing\n", + "from hyperactive.opt import HillClimbing\n", "\n", "hillclimbing_config = {\n", " \"search_space\": {\n", @@ -532,7 +532,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "5aa7ca80", "metadata": {}, "outputs": [ @@ -556,7 +556,7 @@ ], "source": [ "import numpy as np\n", - "from hyperactive.opt.gfo import HillClimbing\n", + "from hyperactive.opt import HillClimbing\n", "\n", "hill_climbing_config = {\n", " \"search_space\": {\n", @@ -602,7 +602,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "4bdf2d49", "metadata": {}, "outputs": [], @@ -610,7 +610,7 @@ "# 1. defining the tuned estimator\n", "from sklearn.svm import SVC\n", "from hyperactive.integrations.sklearn import OptCV\n", - "from hyperactive.opt import GridSearch\n", + "from hyperactive.opt import GridSearchSk as GridSearch\n", "\n", "param_grid = {\"kernel\": [\"linear\", \"rbf\"], \"C\": [1, 10]}\n", "tuned_svc = OptCV(SVC(), optimizer=GridSearch(param_grid))\n", @@ -1084,7 +1084,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "f606284b", "metadata": {}, "outputs": [ @@ -1100,7 +1100,7 @@ "# 1. defining the tuned estimator\n", "from sklearn.svm import SVC\n", "from hyperactive.integrations.sklearn import OptCV\n", - "from hyperactive.opt.gfo import HillClimbing\n", + "from hyperactive.opt import HillClimbing\n", "\n", "# picking the optimizer is the only part that changes!\n", "hill_climbing_config = {\n", diff --git a/examples/integrations/sklearn_example.py b/examples/integrations/sklearn_example.py index c74a5711..45eb429a 100644 --- a/examples/integrations/sklearn_example.py +++ b/examples/integrations/sklearn_example.py @@ -1,7 +1,6 @@ from sklearn import svm, datasets from hyperactive.integrations import HyperactiveSearchCV -from sklearn.model_selection import GridSearchCV from hyperactive.optimizers import RandomSearchOptimizer iris = datasets.load_iris() diff --git a/src/hyperactive/base/_optimizer.py b/src/hyperactive/base/_optimizer.py index 2c51a8a8..f1dc0fa9 100644 --- a/src/hyperactive/base/_optimizer.py +++ b/src/hyperactive/base/_optimizer.py @@ -10,6 +10,13 @@ class BaseOptimizer(BaseObject): _tags = { "object_type": "optimizer", "python_dependencies": None, + # properties of the optimizer + "info:name": None, # str + "info:local_vs_global": "mixed", # "local", "mixed", "global" + "info:explore_vs_exploit": "mixed", # "explore", "exploit", "mixed" + "info:compute": "middle", # "low", "middle", "high" + # see here for explanation of the tags: + # https://simonblanke.github.io/gradient-free-optimizers-documentation/1.5/optimizers/ # noqa: E501 } def __init__(self): @@ -18,6 +25,9 @@ def __init__(self): search_config = self.get_params() self._experiment = search_config.pop("experiment", None) + if self.get_tag("info:name") is None: + self.set_tags(**{"info:name": self.__class__.__name__}) + def get_search_config(self): """Get the search configuration. diff --git a/src/hyperactive/base/tests/test_endtoend.py b/src/hyperactive/base/tests/test_endtoend.py index 7a0a9e36..4c5a1933 100644 --- a/src/hyperactive/base/tests/test_endtoend.py +++ b/src/hyperactive/base/tests/test_endtoend.py @@ -26,7 +26,7 @@ def test_endtoend_hillclimbing(): # 2. set up the HillClimbing optimizer import numpy as np - from hyperactive.opt.gfo import HillClimbing + from hyperactive.opt import HillClimbing hillclimbing_config = { "search_space": { diff --git a/src/hyperactive/integrations/sklearn/opt_cv.py b/src/hyperactive/integrations/sklearn/opt_cv.py index 167ff528..eb83cf90 100644 --- a/src/hyperactive/integrations/sklearn/opt_cv.py +++ b/src/hyperactive/integrations/sklearn/opt_cv.py @@ -44,7 +44,7 @@ class OptCV(BaseEstimator, _BestEstimator_, Checks): 1. defining the tuned estimator: >>> from sklearn.svm import SVC >>> from hyperactive.integrations.sklearn import OptCV - >>> from hyperactive.opt import GridSearch + >>> from hyperactive.opt import GridSearchSk as GridSearch >>> >>> param_grid = {"kernel": ["linear", "rbf"], "C": [1, 10]} >>> tuned_svc = OptCV(SVC(), GridSearch(param_grid)) diff --git a/src/hyperactive/opt/__init__.py b/src/hyperactive/opt/__init__.py index 0d1c0123..8014b642 100644 --- a/src/hyperactive/opt/__init__.py +++ b/src/hyperactive/opt/__init__.py @@ -1,10 +1,14 @@ """Individual optimization algorithms.""" # copyright: hyperactive developers, MIT License (see LICENSE file) -from hyperactive.opt.gfo import HillClimbing -from hyperactive.opt.gridsearch import GridSearch +from hyperactive.opt.gridsearch import GridSearchSk +from hyperactive.opt.hillclimbing import HillClimbing +from hyperactive.opt.hillclimbing_repulsing import HillClimbingRepulsing +from hyperactive.opt.hillclimbing_stochastic import HillClimbingStochastic __all__ = [ - "GridSearch", + "GridSearchSk", "HillClimbing", + "HillClimbingRepulsing", + "HillClimbingStochastic", ] diff --git a/src/hyperactive/opt/_adapters/__init__.py b/src/hyperactive/opt/_adapters/__init__.py new file mode 100644 index 00000000..d279503a --- /dev/null +++ b/src/hyperactive/opt/_adapters/__init__.py @@ -0,0 +1,2 @@ +"""Adapters for individual packages.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) diff --git a/src/hyperactive/opt/gfo/hillclimbing.py b/src/hyperactive/opt/_adapters/_gfo.py similarity index 51% rename from src/hyperactive/opt/gfo/hillclimbing.py rename to src/hyperactive/opt/_adapters/_gfo.py index ae16a745..aca57da3 100644 --- a/src/hyperactive/opt/gfo/hillclimbing.py +++ b/src/hyperactive/opt/_adapters/_gfo.py @@ -1,120 +1,48 @@ -"""Hill climbing optimizer from gfo.""" +"""Adapter for gfo package.""" # copyright: hyperactive developers, MIT License (see LICENSE file) -from gradient_free_optimizers import HillClimbingOptimizer from hyperactive.base import BaseOptimizer from skbase.utils.stdout_mute import StdoutMute +__all__ = ["_BaseGFOadapter"] -class HillClimbing(BaseOptimizer): - """Hill climbing optimizer. - - Parameters - ---------- - search_space : dict[str, list] - The search space to explore. A dictionary with parameter - names as keys and a numpy array as values. - Optional, can be passed later in ``add_search``. - initialize : dict[str, int], default={"grid": 4, "random": 2, "vertices": 4} - The method to generate initial positions. A dictionary with - the following key literals and the corresponding value type: - {"grid": int, "vertices": int, "random": int, "warm_start": list[dict]} - constraints : list[callable], default=[] - A list of constraints, where each constraint is a callable. - The callable returns `True` or `False` dependend on the input parameters. - random_state : None, int, default=None - If None, create a new random state. If int, create a new random state - seeded with the value. - rand_rest_p : float, default=0.1 - The probability of a random iteration during the the search process. - epsilon : float, default=0.01 - The step-size for the climbing. - distribution : str, default="normal" - The type of distribution to sample from. - n_neighbours : int, default=10 - The number of neighbours to sample and evaluate before moving to the best - of those neighbours. - n_iter : int, default=100 - The number of iterations to run the optimizer. - verbose : bool, default=False - If True, print the progress of the optimization process. - experiment : BaseExperiment, optional - The experiment to optimize parameters for. - Optional, can be passed later in ``add_search``. - - Examples - -------- - Hill climbing applied to scikit-learn parameter tuning: - - 1. defining the experiment to optimize: - >>> from hyperactive.experiment.integrations import SklearnCvExperiment - >>> from sklearn.datasets import load_iris - >>> from sklearn.svm import SVC - >>> - >>> X, y = load_iris(return_X_y=True) - >>> - >>> sklearn_exp = SklearnCvExperiment( - ... estimator=SVC(), - ... X=X, - ... y=y, - ... ) - - 2. setting up the hill climbing optimizer: - >>> from hyperactive.opt import HillClimbing - >>> import numpy as np - >>> - >>> hillclimbing_config = { - ... "search_space": { - ... "C": np.array([0.01, 0.1, 1, 10]), - ... "gamma": np.array([0.0001, 0.01, 0.1, 1, 10]), - ... }, - ... "n_iter": 100, - ... } - >>> hillclimbing = HillClimbing(experiment=sklearn_exp, **hillclimbing_config) - - 3. running the hill climbing search: - >>> best_params = hillclimbing.run() - - Best parameters can also be accessed via the attributes: - >>> best_params = hillclimbing.best_params_ + +class _BaseGFOadapter(BaseOptimizer): + """Adapter base class for gradient-free-optimizers. + + * default tag setting + * default _run method + * default get_search_config + * default get_test_params + * Handles defaults for "initialize" parameter + * extension interface: _get_gfo_class, docstring, tags """ _tags = { + "authors": "SimonBlanke", "python_dependencies": ["gradient-free-optimizers>=1.5.0"], } - def __init__( - self, - search_space=None, - initialize=None, - constraints=None, - random_state=None, - rand_rest_p=0.1, - epsilon=0.01, - distribution="normal", - n_neighbours=10, - n_iter=100, - verbose=False, - experiment=None, - ): - self.random_state = random_state - self.rand_rest_p = rand_rest_p - self.epsilon = epsilon - self.distribution = distribution - self.n_neighbours = n_neighbours - self.search_space = search_space - self.initialize = initialize - self.constraints = constraints - self.n_iter = n_iter - self.experiment = experiment - self.verbose = verbose + def __init__(self): super().__init__() - if initialize is None: + if self.initialize is None: self._initialize = {"grid": 4, "random": 2, "vertices": 4} else: - self._initialize = initialize + self._initialize = self.initialize + + def _get_gfo_class(self): + """Get the GFO class to use. + + Returns + ------- + class + The GFO class to use. One of the concrete GFO classes + """ + raise NotImplementedError( + "This method should be implemented in a subclass." + ) def get_search_config(self): """Get the search configuration. @@ -131,14 +59,12 @@ def get_search_config(self): def _run(self, experiment, **search_config): """Run the optimization search process. - Parameters ---------- experiment : BaseExperiment The experiment to optimize parameters for. search_config : dict with str keys identical to return of ``get_search_config``. - Returns ------- dict with str keys @@ -148,7 +74,8 @@ def _run(self, experiment, **search_config): n_iter = search_config.pop("n_iter", 100) max_time = search_config.pop("max_time", None) - hcopt = HillClimbingOptimizer(**search_config) + gfo_cls = self._get_gfo_class() + hcopt = gfo_cls(**search_config) with StdoutMute(active=not self.verbose): hcopt.search( diff --git a/src/hyperactive/opt/gfo/__init__.py b/src/hyperactive/opt/gfo/__init__.py deleted file mode 100644 index e40b831e..00000000 --- a/src/hyperactive/opt/gfo/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Optimizers from Gradient free optimizers package.""" -# copyright: hyperactive developers, MIT License (see LICENSE file) - -from hyperactive.opt.gfo.hillclimbing import HillClimbing - -__all__ = ["HillClimbing"] diff --git a/src/hyperactive/opt/gridsearch/__init__.py b/src/hyperactive/opt/gridsearch/__init__.py new file mode 100644 index 00000000..b34758fd --- /dev/null +++ b/src/hyperactive/opt/gridsearch/__init__.py @@ -0,0 +1,6 @@ +"""Grid search with sklearn style grid and backends.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +from hyperactive.opt.gridsearch._sk import GridSearchSk + +__all__ = ["GridSearchSk"] diff --git a/src/hyperactive/opt/gridsearch.py b/src/hyperactive/opt/gridsearch/_sk.py similarity index 96% rename from src/hyperactive/opt/gridsearch.py rename to src/hyperactive/opt/gridsearch/_sk.py index 766cb6b3..84e97625 100644 --- a/src/hyperactive/opt/gridsearch.py +++ b/src/hyperactive/opt/gridsearch/_sk.py @@ -10,8 +10,8 @@ from hyperactive.base import BaseOptimizer -class GridSearch(BaseOptimizer): - """Grid search optimizer. +class GridSearchSk(BaseOptimizer): + """Grid search optimizer, with backend selection and sklearn style parameter grid. Parameters ---------- @@ -22,7 +22,7 @@ class GridSearch(BaseOptimizer): The score to assign if an error occurs during the evaluation of a parameter set. experiment : BaseExperiment, optional The experiment to optimize parameters for. - Optional, can be passed later in ``add_search``. + Optional, can be passed later via ``set_params``. Example ------- @@ -42,7 +42,7 @@ class GridSearch(BaseOptimizer): ... ) 2. setting up the grid search optimizer: - >>> from hyperactive.opt import GridSearch + >>> from hyperactive.opt import GridSearchSk as GridSearch >>> param_grid = { ... "C": [0.01, 0.1, 1, 10], ... "gamma": [0.0001, 0.01, 0.1, 1, 10], diff --git a/src/hyperactive/opt/hillclimbing/__init__.py b/src/hyperactive/opt/hillclimbing/__init__.py new file mode 100644 index 00000000..4acd2fbf --- /dev/null +++ b/src/hyperactive/opt/hillclimbing/__init__.py @@ -0,0 +1,6 @@ +"""Hill climbing optimizer.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +from hyperactive.opt.hillclimbing._hillclimbing import HillClimbing + +__all__ = ["HillClimbing"] diff --git a/src/hyperactive/opt/hillclimbing/_hillclimbing.py b/src/hyperactive/opt/hillclimbing/_hillclimbing.py new file mode 100644 index 00000000..a6eabc81 --- /dev/null +++ b/src/hyperactive/opt/hillclimbing/_hillclimbing.py @@ -0,0 +1,125 @@ +"""Hill climbing optimizer from gfo.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +from hyperactive.opt._adapters._gfo import _BaseGFOadapter + + +class HillClimbing(_BaseGFOadapter): + """Hill climbing optimizer. + + Parameters + ---------- + search_space : dict[str, list] + The search space to explore. A dictionary with parameter + names as keys and a numpy array as values. + Optional, can be passed later via ``set_params``. + initialize : dict[str, int], default={"grid": 4, "random": 2, "vertices": 4} + The method to generate initial positions. A dictionary with + the following key literals and the corresponding value type: + {"grid": int, "vertices": int, "random": int, "warm_start": list[dict]} + constraints : list[callable], default=[] + A list of constraints, where each constraint is a callable. + The callable returns `True` or `False` dependend on the input parameters. + random_state : None, int, default=None + If None, create a new random state. If int, create a new random state + seeded with the value. + rand_rest_p : float, default=0.1 + The probability of a random iteration during the the search process. + epsilon : float, default=0.01 + The step-size for the climbing. + distribution : str, default="normal" + The type of distribution to sample from. + n_neighbours : int, default=10 + The number of neighbours to sample and evaluate before moving to the best + of those neighbours. + n_iter : int, default=100 + The number of iterations to run the optimizer. + verbose : bool, default=False + If True, print the progress of the optimization process. + experiment : BaseExperiment, optional + The experiment to optimize parameters for. + Optional, can be passed later via ``set_params``. + + Examples + -------- + Hill climbing applied to scikit-learn parameter tuning: + + 1. defining the experiment to optimize: + >>> from hyperactive.experiment.integrations import SklearnCvExperiment + >>> from sklearn.datasets import load_iris + >>> from sklearn.svm import SVC + >>> + >>> X, y = load_iris(return_X_y=True) + >>> + >>> sklearn_exp = SklearnCvExperiment( + ... estimator=SVC(), + ... X=X, + ... y=y, + ... ) + + 2. setting up the hill climbing optimizer: + >>> from hyperactive.opt import HillClimbing + >>> import numpy as np + >>> + >>> hillclimbing_config = { + ... "search_space": { + ... "C": np.array([0.01, 0.1, 1, 10]), + ... "gamma": np.array([0.0001, 0.01, 0.1, 1, 10]), + ... }, + ... "n_iter": 100, + ... } + >>> hillclimbing = HillClimbing(experiment=sklearn_exp, **hillclimbing_config) + + 3. running the hill climbing search: + >>> best_params = hillclimbing.run() + + Best parameters can also be accessed via the attributes: + >>> best_params = hillclimbing.best_params_ + """ + + _tags = { + "info:name": "Hill Climbing", + "info:local_vs_global": "local", # "local", "mixed", "global" + "info:explore_vs_exploit": "exploit", # "explore", "exploit", "mixed" + "info:compute": "low", # "low", "middle", "high" + } + + def __init__( + self, + search_space=None, + initialize=None, + constraints=None, + random_state=None, + rand_rest_p=0.1, + epsilon=0.01, + distribution="normal", + n_neighbours=10, + n_iter=100, + verbose=False, + experiment=None, + ): + self.random_state = random_state + self.rand_rest_p = rand_rest_p + self.epsilon = epsilon + self.distribution = distribution + self.n_neighbours = n_neighbours + self.search_space = search_space + self.initialize = initialize + self.constraints = constraints + self.n_iter = n_iter + self.experiment = experiment + self.verbose = verbose + + super().__init__() + + def _get_gfo_class(self): + """Get the GFO class to use. + + Returns + ------- + class + The GFO class to use. One of the concrete GFO classes + """ + from gradient_free_optimizers import HillClimbingOptimizer + + return HillClimbingOptimizer diff --git a/src/hyperactive/opt/hillclimbing_repulsing/__init__.py b/src/hyperactive/opt/hillclimbing_repulsing/__init__.py new file mode 100644 index 00000000..f6bd0b16 --- /dev/null +++ b/src/hyperactive/opt/hillclimbing_repulsing/__init__.py @@ -0,0 +1,8 @@ +"""Hill climbing optimizer.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +from hyperactive.opt.hillclimbing_repulsing._hillclimbing_repulsing import ( + HillClimbingRepulsing, +) + +__all__ = ["HillClimbingRepulsing"] diff --git a/src/hyperactive/opt/hillclimbing_repulsing/_hillclimbing_repulsing.py b/src/hyperactive/opt/hillclimbing_repulsing/_hillclimbing_repulsing.py new file mode 100644 index 00000000..7b50c62a --- /dev/null +++ b/src/hyperactive/opt/hillclimbing_repulsing/_hillclimbing_repulsing.py @@ -0,0 +1,154 @@ +"""Hill climbing optimizer from gfo.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +from hyperactive.opt._adapters._gfo import _BaseGFOadapter + + +class HillClimbingRepulsing(_BaseGFOadapter): + """Repulsing hill climbing optimizer. + + Parameters + ---------- + search_space : dict[str, list] + The search space to explore. A dictionary with parameter + names as keys and a numpy array as values. + Optional, can be passed later via ``set_params``. + initialize : dict[str, int], default={"grid": 4, "random": 2, "vertices": 4} + The method to generate initial positions. A dictionary with + the following key literals and the corresponding value type: + {"grid": int, "vertices": int, "random": int, "warm_start": list[dict]} + constraints : list[callable], default=[] + A list of constraints, where each constraint is a callable. + The callable returns `True` or `False` dependend on the input parameters. + random_state : None, int, default=None + If None, create a new random state. If int, create a new random state + seeded with the value. + rand_rest_p : float, default=0.1 + The probability of a random iteration during the the search process. + epsilon : float, default=0.01 + The step-size for the climbing. + distribution : str, default="normal" + The type of distribution to sample from. + n_neighbours : int, default=10 + The number of neighbours to sample and evaluate before moving to the best + of those neighbours. + repulsion_factor : float, default=5 + The factor to control the repulsion of the hill climbing process. + n_iter : int, default=100 + The number of iterations to run the optimizer. + verbose : bool, default=False + If True, print the progress of the optimization process. + experiment : BaseExperiment, optional + The experiment to optimize parameters for. + Optional, can be passed later via ``set_params``. + + Examples + -------- + Hill climbing applied to scikit-learn parameter tuning: + + 1. defining the experiment to optimize: + >>> from hyperactive.experiment.integrations import SklearnCvExperiment + >>> from sklearn.datasets import load_iris + >>> from sklearn.svm import SVC + >>> + >>> X, y = load_iris(return_X_y=True) + >>> + >>> sklearn_exp = SklearnCvExperiment( + ... estimator=SVC(), + ... X=X, + ... y=y, + ... ) + + 2. setting up the hill climbing optimizer: + >>> from hyperactive.opt import HillClimbingRepulsing + >>> import numpy as np + >>> + >>> hc_config = { + ... "search_space": { + ... "C": np.array([0.01, 0.1, 1, 10]), + ... "gamma": np.array([0.0001, 0.01, 0.1, 1, 10]), + ... }, + ... "n_iter": 100, + ... } + >>> hillclimbing = HillClimbingRepulsing(experiment=sklearn_exp, **hc_config) + + 3. running the hill climbing search: + >>> best_params = hillclimbing.run() + + Best parameters can also be accessed via the attributes: + >>> best_params = hillclimbing.best_params_ + """ + + _tags = { + "info:name": "Repulsing Hill Climbing", + "info:local_vs_global": "mixed", # "local", "mixed", "global" + "info:explore_vs_exploit": "exploit", # "explore", "exploit", "mixed" + "info:compute": "low", # "low", "middle", "high" + } + + def __init__( + self, + search_space=None, + initialize=None, + constraints=None, + random_state=None, + rand_rest_p=0.1, + epsilon=0.01, + distribution="normal", + n_neighbours=10, + repulsion_factor=5, + n_iter=100, + verbose=False, + experiment=None, + ): + self.random_state = random_state + self.rand_rest_p = rand_rest_p + self.epsilon = epsilon + self.distribution = distribution + self.n_neighbours = n_neighbours + self.search_space = search_space + self.initialize = initialize + self.constraints = constraints + self.repulsion_factor = repulsion_factor + self.n_iter = n_iter + self.experiment = experiment + self.verbose = verbose + + super().__init__() + + def _get_gfo_class(self): + """Get the GFO class to use. + + Returns + ------- + class + The GFO class to use. One of the concrete GFO classes + """ + from gradient_free_optimizers import RepulsingHillClimbingOptimizer + + return RepulsingHillClimbingOptimizer + + @classmethod + def get_test_params(cls, parameter_set="default"): + """Get the test parameters for the optimizer. + + Returns + ------- + dict with str keys + The test parameters dictionary. + """ + import numpy as np + + params = super().get_test_params() + experiment = params[0]["experiment"] + more_params = { + "experiment": experiment, + "repulsion_factor": 7, + "search_space": { + "C": np.array([0.01, 0.1, 1, 10]), + "gamma": np.array([0.0001, 0.01, 0.1, 1, 10]), + }, + "n_iter": 100, + } + params.append(more_params) + return params diff --git a/src/hyperactive/opt/hillclimbing_stochastic/__init__.py b/src/hyperactive/opt/hillclimbing_stochastic/__init__.py new file mode 100644 index 00000000..f7d1e78b --- /dev/null +++ b/src/hyperactive/opt/hillclimbing_stochastic/__init__.py @@ -0,0 +1,8 @@ +"""Hill climbing optimizer.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +from hyperactive.opt.hillclimbing_stochastic._hillclimbing_stochastic import ( + HillClimbingStochastic, +) + +__all__ = ["HillClimbingStochastic"] diff --git a/src/hyperactive/opt/hillclimbing_stochastic/_hillclimbing_stochastic.py b/src/hyperactive/opt/hillclimbing_stochastic/_hillclimbing_stochastic.py new file mode 100644 index 00000000..c44ad9bd --- /dev/null +++ b/src/hyperactive/opt/hillclimbing_stochastic/_hillclimbing_stochastic.py @@ -0,0 +1,154 @@ +"""Hill climbing optimizer from gfo.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +from hyperactive.opt._adapters._gfo import _BaseGFOadapter + + +class HillClimbingStochastic(_BaseGFOadapter): + """Stochastic hill climbing optimizer. + + Parameters + ---------- + search_space : dict[str, list] + The search space to explore. A dictionary with parameter + names as keys and a numpy array as values. + Optional, can be passed later via ``set_params``. + initialize : dict[str, int], default={"grid": 4, "random": 2, "vertices": 4} + The method to generate initial positions. A dictionary with + the following key literals and the corresponding value type: + {"grid": int, "vertices": int, "random": int, "warm_start": list[dict]} + constraints : list[callable], default=[] + A list of constraints, where each constraint is a callable. + The callable returns `True` or `False` dependend on the input parameters. + random_state : None, int, default=None + If None, create a new random state. If int, create a new random state + seeded with the value. + rand_rest_p : float, default=0.1 + The probability of a random iteration during the the search process. + epsilon : float, default=0.01 + The step-size for the climbing. + distribution : str, default="normal" + The type of distribution to sample from. + n_neighbours : int, default=10 + The number of neighbours to sample and evaluate before moving to the best + of those neighbours. + p_accept : float, default=0.5 + The probability of accepting a transition in the hill climbing process. + n_iter : int, default=100 + The number of iterations to run the optimizer. + verbose : bool, default=False + If True, print the progress of the optimization process. + experiment : BaseExperiment, optional + The experiment to optimize parameters for. + Optional, can be passed later via ``set_params``. + + Examples + -------- + Hill climbing applied to scikit-learn parameter tuning: + + 1. defining the experiment to optimize: + >>> from hyperactive.experiment.integrations import SklearnCvExperiment + >>> from sklearn.datasets import load_iris + >>> from sklearn.svm import SVC + >>> + >>> X, y = load_iris(return_X_y=True) + >>> + >>> sklearn_exp = SklearnCvExperiment( + ... estimator=SVC(), + ... X=X, + ... y=y, + ... ) + + 2. setting up the hill climbing optimizer: + >>> from hyperactive.opt import HillClimbingStochastic + >>> import numpy as np + >>> + >>> hc_config = { + ... "search_space": { + ... "C": np.array([0.01, 0.1, 1, 10]), + ... "gamma": np.array([0.0001, 0.01, 0.1, 1, 10]), + ... }, + ... "n_iter": 100, + ... } + >>> hillclimbing = HillClimbingStochastic(experiment=sklearn_exp, **hc_config) + + 3. running the hill climbing search: + >>> best_params = hillclimbing.run() + + Best parameters can also be accessed via the attributes: + >>> best_params = hillclimbing.best_params_ + """ + + _tags = { + "info:name": "Hill Climbing", + "info:local_vs_global": "local", # "local", "mixed", "global" + "info:explore_vs_exploit": "exploit", # "explore", "exploit", "mixed" + "info:compute": "low", # "low", "middle", "high" + } + + def __init__( + self, + search_space=None, + initialize=None, + constraints=None, + random_state=None, + rand_rest_p=0.1, + epsilon=0.01, + distribution="normal", + n_neighbours=10, + p_accept=0.5, + n_iter=100, + verbose=False, + experiment=None, + ): + self.random_state = random_state + self.rand_rest_p = rand_rest_p + self.epsilon = epsilon + self.distribution = distribution + self.n_neighbours = n_neighbours + self.search_space = search_space + self.initialize = initialize + self.constraints = constraints + self.p_accept = p_accept + self.n_iter = n_iter + self.experiment = experiment + self.verbose = verbose + + super().__init__() + + def _get_gfo_class(self): + """Get the GFO class to use. + + Returns + ------- + class + The GFO class to use. One of the concrete GFO classes + """ + from gradient_free_optimizers import StochasticHillClimbingOptimizer + + return StochasticHillClimbingOptimizer + + @classmethod + def get_test_params(cls, parameter_set="default"): + """Get the test parameters for the optimizer. + + Returns + ------- + dict with str keys + The test parameters dictionary. + """ + import numpy as np + + params = super().get_test_params() + experiment = params[0]["experiment"] + more_params = { + "experiment": experiment, + "p_accept": 0.33, + "search_space": { + "C": np.array([0.01, 0.1, 1, 10]), + "gamma": np.array([0.0001, 0.01, 0.1, 1, 10]), + }, + "n_iter": 100, + } + params.append(more_params) + return params diff --git a/src/hyperactive/tests/test_all_objects.py b/src/hyperactive/tests/test_all_objects.py index 41306360..84257db0 100644 --- a/src/hyperactive/tests/test_all_objects.py +++ b/src/hyperactive/tests/test_all_objects.py @@ -40,10 +40,15 @@ class PackageConfig: # general tags "object_type", "python_dependencies", + "authors", + "maintainers", # experiments "property:randomness", # optimizers - # none yet + "info:name", # str + "info:local_vs_global", # "local", "mixed", "global" + "info:explore_vs_exploit", # "explore", "exploit", "mixed" + "info:compute", # "low", "middle", "high" ] @@ -243,3 +248,53 @@ def test_opt_run(self, object_instance): msg = "Optimizer best_params_ must equal the best_params returned by run." if not object_instance.best_params_ == best_params: raise ValueError(msg) + + def test_gfo_integration(self, object_instance): + """Integration test for optimizer end-to-end, for GFO optimizers only. + + Runs the optimizer on the sklearn tuning experiment. + """ + from hyperactive.opt._adapters._gfo import _BaseGFOadapter + if not isinstance(object_instance, _BaseGFOadapter): + return None + + optimizer = object_instance + + # 1. define the experiment + from hyperactive.experiment.integrations import SklearnCvExperiment + from sklearn.datasets import load_iris + from sklearn.svm import SVC + from sklearn.metrics import accuracy_score + from sklearn.model_selection import KFold + + X, y = load_iris(return_X_y=True) + + sklearn_exp = SklearnCvExperiment( + estimator=SVC(), + scoring=accuracy_score, + cv=KFold(n_splits=3, shuffle=True), + X=X, + y=y, + ) + + # 2. set up the optimizer + import numpy as np + + _config = { + "search_space": { + "C": np.array([0.01, 0.1, 1, 10]), + "gamma": np.array([0.0001, 0.01, 0.1, 1, 10]), + }, + "n_iter": 100, + "experiment": sklearn_exp, + } + optimizer = optimizer.clone().set_params(**_config) + + # 3. run the HillClimbing optimizer + optimizer.run() + + best_params = optimizer.best_params_ + assert best_params is not None, "Best parameters should not be None" + assert isinstance(best_params, dict), "Best parameters should be a dictionary" + assert "C" in best_params, "Best parameters should contain 'C'" + assert "gamma" in best_params, "Best parameters should contain 'gamma'" diff --git a/tests/integrations/sklearn/test_parametrize_with_checks.py b/tests/integrations/sklearn/test_parametrize_with_checks.py index c1097c4c..68eca80d 100644 --- a/tests/integrations/sklearn/test_parametrize_with_checks.py +++ b/tests/integrations/sklearn/test_parametrize_with_checks.py @@ -1,7 +1,7 @@ from sklearn import svm from hyperactive.integrations import HyperactiveSearchCV, OptCV -from hyperactive.opt import GridSearch +from hyperactive.opt import GridSearchSk as GridSearch from hyperactive.optimizers import RandomSearchOptimizer from sklearn.model_selection import KFold