diff --git a/examples/hyperactive_intro.ipynb b/examples/hyperactive_intro.ipynb new file mode 100644 index 00000000..5a047b32 --- /dev/null +++ b/examples/hyperactive_intro.ipynb @@ -0,0 +1,1723 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "af034c62", + "metadata": {}, + "source": [ + "## hyperactive - unified interfaces for optimizers and experiments" + ] + }, + { + "cell_type": "markdown", + "id": "830f7eb7", + "metadata": {}, + "source": [ + "### \"experiment\" = optimization problem" + ] + }, + { + "cell_type": "markdown", + "id": "668f9ccc", + "metadata": {}, + "source": [ + "\"experiment\" classes model optimization functions and ML experiments under one API\n", + "\n", + "Examples below:\n", + "1. simple objective function - parabola function\n", + "2. ML cross-validation experiment in sklearn" + ] + }, + { + "cell_type": "markdown", + "id": "2380a84b", + "metadata": {}, + "source": [ + "#### user defined objective function" + ] + }, + { + "cell_type": "markdown", + "id": "21f50e9e", + "metadata": {}, + "source": [ + "simple objective definition:\n", + "\n", + "* function with single dict argument\n", + "* keys are variable names\n", + "* function evaluates variables and returns float\n", + "* maximization is assumed later" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "8c428229", + "metadata": {}, + "outputs": [], + "source": [ + "def sphere(opt):\n", + " x = opt[\"x\"]\n", + " y = opt[\"y\"]\n", + "\n", + " return -x**2 - y**2" + ] + }, + { + "cell_type": "markdown", + "id": "a242a2fc", + "metadata": {}, + "source": [ + "to evaluate:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ccb460f1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "-13" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sphere({\"x\": 2, \"y\": 3})" + ] + }, + { + "cell_type": "markdown", + "id": "173f1923", + "metadata": {}, + "source": [ + "#### parametric objective functions - parametric" + ] + }, + { + "cell_type": "markdown", + "id": "fd01be6c", + "metadata": {}, + "source": [ + "parametric objective functions are classes:\n", + "\n", + "* construct with parameters\n", + "* call `evaluate` with `dict`\n", + "\n", + "`hyperactive` comes with predefined objective functions.\n", + "\n", + "These are parametric through the constructor\n", + "\n", + "Example: `Parabola` function, docstring outlines parametric form" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "c474de73", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;31mInit signature:\u001b[0m \u001b[0mParabola\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1.0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mc\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mDocstring:\u001b[0m Parabola class.\n", + "\u001b[1;31mInit docstring:\u001b[0m Construct BaseObject.\n", + "\u001b[1;31mFile:\u001b[0m c:\\workspace\\hyperactive\\src\\hyperactive\\experiment\\bench\\_parabola.py\n", + "\u001b[1;31mType:\u001b[0m type\n", + "\u001b[1;31mSubclasses:\u001b[0m " + ] + } + ], + "source": [ + "from hyperactive.experiment.bench import Parabola\n", + "\n", + "?Parabola" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "59dd4715", + "metadata": {}, + "outputs": [], + "source": [ + "parabola = Parabola(a=42, b=3, c=4)" + ] + }, + { + "cell_type": "markdown", + "id": "1de4894a", + "metadata": {}, + "source": [ + "parametric objectives are evaluated via `evaluate` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "97b41711", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(564.0)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "score, metadata = parabola.evaluate({\"x\": 2, \"y\": 3}) # also returns metadata\n", + "score" + ] + }, + { + "cell_type": "markdown", + "id": "4f561158", + "metadata": {}, + "source": [ + "instances of parametric objectives are also directly callable" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "797535db", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(-564.0)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parabola(x=2, y=3)\n", + "# output is always np.float64" + ] + }, + { + "cell_type": "markdown", + "id": "b8096712", + "metadata": {}, + "source": [ + "the \"experiment\" class has two sets of variables:\n", + "\n", + "* optimization variables = inputs of the objective - inspectable via `paramnames`\n", + "* parameters of the experiment = constant in the objective\n", + " * these are params of `__init__`\n", + " * and are inspectable via `get_params`" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "fc08e901", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['x', 'y']" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parabola.paramnames()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "61a62efa", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['a', 'b', 'c']" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(parabola.get_params())" + ] + }, + { + "cell_type": "markdown", + "id": "a1dbc553", + "metadata": {}, + "source": [ + "call via `score` method:\n", + "\n", + "* returns two objects: the value, and metadata (in a dict)\n", + "* input is a single dict, the `**` variant of a direct call" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "b127d406", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(np.float64(-564.0), {})" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parabola.score({\"x\": 2, \"y\": 3})" + ] + }, + { + "cell_type": "markdown", + "id": "98924abf", + "metadata": {}, + "source": [ + "#### sklearn cross-validation" + ] + }, + { + "cell_type": "markdown", + "id": "a625210e", + "metadata": {}, + "source": [ + "\"experiment\" can be more complicated - e.g., a cross-validation experiment\n", + "\n", + "this is a single tuning-evaluation step for an `sklearn` estimator" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "57110e86", + "metadata": {}, + "outputs": [], + "source": [ + "from hyperactive.experiment.integrations import SklearnCvExperiment\n", + "from sklearn.datasets import load_iris\n", + "from sklearn.svm import SVC\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.model_selection import KFold\n", + "\n", + "X, y = load_iris(return_X_y=True)\n", + "\n", + "sklearn_exp = SklearnCvExperiment(\n", + " estimator=SVC(),\n", + " scoring=accuracy_score,\n", + " cv=KFold(n_splits=3, shuffle=True),\n", + " X=X,\n", + " y=y,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "dcf183ae", + "metadata": {}, + "source": [ + "usage syntax same as for the simple parabola!" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "70a27348", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.9733333333333333)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sklearn_exp(C=1, gamma=0.3)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "1cfd28d4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['C',\n", + " 'break_ties',\n", + " 'cache_size',\n", + " 'class_weight',\n", + " 'coef0',\n", + " 'decision_function_shape',\n", + " 'degree',\n", + " 'gamma',\n", + " 'kernel',\n", + " 'max_iter',\n", + " 'probability',\n", + " 'random_state',\n", + " 'shrinking',\n", + " 'tol',\n", + " 'verbose']" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sklearn_exp.paramnames()" + ] + }, + { + "cell_type": "markdown", + "id": "165c5830", + "metadata": {}, + "source": [ + "`get_params` works like in `sklearn` and is nested\n", + "\n", + "note that similar parameters appear as in `paramnames`\n", + "\n", + "* parameters in `paramnames` are optimized over\n", + "* parameters in `get_params` are default values, if not set in `score`" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "b6160e3c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['X',\n", + " 'cv',\n", + " 'estimator',\n", + " 'scoring',\n", + " 'y',\n", + " 'estimator__C',\n", + " 'estimator__break_ties',\n", + " 'estimator__cache_size',\n", + " 'estimator__class_weight',\n", + " 'estimator__coef0',\n", + " 'estimator__decision_function_shape',\n", + " 'estimator__degree',\n", + " 'estimator__gamma',\n", + " 'estimator__kernel',\n", + " 'estimator__max_iter',\n", + " 'estimator__probability',\n", + " 'estimator__random_state',\n", + " 'estimator__shrinking',\n", + " 'estimator__tol',\n", + " 'estimator__verbose']" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(sklearn_exp.get_params().keys())" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "f74ca927", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(np.float64(0.98),\n", + " {'score_time': array([0. , 0.01113176, 0.00051761]),\n", + " 'fit_time': array([0. , 0. , 0.00100803]),\n", + " 'n_test_samples': 150})" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sklearn_exp.score({\"C\": 1, \"gamma\": 0.3})" + ] + }, + { + "cell_type": "markdown", + "id": "71073496", + "metadata": {}, + "source": [ + "### use of optimizers" + ] + }, + { + "cell_type": "markdown", + "id": "725230fc", + "metadata": {}, + "source": [ + "#### optimizing a custom objective" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "ab78b796", + "metadata": {}, + "outputs": [], + "source": [ + "def sphere(opt):\n", + " x = opt[\"x\"]\n", + " y = opt[\"y\"]\n", + "\n", + " return -x**2 - y**2" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "7104e5ec", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/plain": [ + "{'x': np.float64(0.10101010101010033), 'y': np.float64(0.10101010101010033)}" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "from hyperactive.opt import HillClimbing\n", + "\n", + "hillclimbing_config = {\n", + " \"search_space\": {\n", + " \"x\": np.linspace(-10, 10, 100),\n", + " \"y\": np.linspace(-10, 10, 100),\n", + " },\n", + " \"n_iter\": 1000,\n", + "}\n", + "hill_climbing = HillClimbing(**hillclimbing_config, experiment=sphere)\n", + "\n", + "hill_climbing.solve()" + ] + }, + { + "cell_type": "markdown", + "id": "300a1666", + "metadata": {}, + "source": [ + "#### Grid search & sklearn CV" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "5e2328c9", + "metadata": {}, + "outputs": [], + "source": [ + "from hyperactive.experiment.integrations import SklearnCvExperiment\n", + "from sklearn.datasets import load_iris\n", + "from sklearn.svm import SVC\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.model_selection import KFold\n", + "\n", + "X, y = load_iris(return_X_y=True)\n", + "\n", + "sklearn_exp = SklearnCvExperiment(\n", + " estimator=SVC(),\n", + " scoring=accuracy_score,\n", + " cv=KFold(n_splits=3, shuffle=True),\n", + " X=X,\n", + " y=y,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "e9a07a73", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'C': 0.01, 'gamma': 1}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from hyperactive.opt import GridSearchSk as GridSearch\n", + "\n", + "param_grid = {\n", + " \"C\": [0.01, 0.1, 1, 10],\n", + " \"gamma\": [0.0001, 0.01, 0.1, 1, 10],\n", + "}\n", + "grid_search = GridSearch(param_grid=param_grid, experiment=sklearn_exp)\n", + "\n", + "grid_search.solve()" + ] + }, + { + "cell_type": "markdown", + "id": "72fcf886", + "metadata": {}, + "source": [ + "#### hill climbing & sklearn CV" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "f9a4d922", + "metadata": {}, + "outputs": [], + "source": [ + "from hyperactive.experiment.integrations import SklearnCvExperiment\n", + "from sklearn.datasets import load_iris\n", + "from sklearn.svm import SVC\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.model_selection import KFold\n", + "\n", + "X, y = load_iris(return_X_y=True)\n", + "\n", + "sklearn_exp = SklearnCvExperiment(\n", + " estimator=SVC(),\n", + " scoring=accuracy_score,\n", + " cv=KFold(n_splits=3, shuffle=True),\n", + " X=X,\n", + " y=y,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "9a13b4f3", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/plain": [ + "{'C': np.float64(10.0), 'gamma': np.float64(0.1)}" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "from hyperactive.opt import HillClimbing\n", + "\n", + "hillclimbing_config = {\n", + " \"search_space\": {\n", + " \"C\": np.array([0.01, 0.1, 1, 10]),\n", + " \"gamma\": np.array([0.0001, 0.01, 0.1, 1, 10]),\n", + " },\n", + " \"n_iter\": 100,\n", + "}\n", + "hill_climbing = HillClimbing(**hillclimbing_config, experiment=sklearn_exp)\n", + "\n", + "hill_climbing.solve()" + ] + }, + { + "cell_type": "markdown", + "id": "7a933b41", + "metadata": {}, + "source": [ + "### full sklearn integration as estimator" + ] + }, + { + "cell_type": "markdown", + "id": "1f368679", + "metadata": {}, + "source": [ + "`OptCV` allows `sklearn` tuning via any tuning algorithm.\n", + "\n", + "Below, we show tuning via:\n", + "\n", + "* standard `GridSearch`\n", + "* `HillClimbing` from `gradient-free-optimizers`" + ] + }, + { + "cell_type": "markdown", + "id": "7171fc17", + "metadata": {}, + "source": [ + "##### `OptCV` tuning via `GridSearch`" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "4bdf2d49", + "metadata": {}, + "outputs": [], + "source": [ + "# 1. defining the tuned estimator\n", + "from sklearn.svm import SVC\n", + "from hyperactive.integrations.sklearn import OptCV\n", + "from hyperactive.opt import GridSearchSk as GridSearch\n", + "\n", + "param_grid = {\"kernel\": [\"linear\", \"rbf\"], \"C\": [1, 10]}\n", + "tuned_svc = OptCV(SVC(), optimizer=GridSearch(param_grid))\n", + "\n", + "# 2. fitting the tuned estimator = tuning the hyperparameters\n", + "from sklearn.datasets import load_iris\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "X, y = load_iris(return_X_y=True)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", + "\n", + "tuned_svc.fit(X_train, y_train)\n", + "\n", + "# 3. making predictions with the tuned estimator\n", + "y_pred = tuned_svc.predict(X_test)" + ] + }, + { + "cell_type": "markdown", + "id": "1a4bddc0", + "metadata": {}, + "source": [ + "best parameters and best estimator can be returned" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "fdd8c14a", + "metadata": {}, + "outputs": [], + "source": [ + "best_params = tuned_svc.best_params_" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "252efea6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SVC(C=1)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "SVC(C=1)" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tuned_svc.best_estimator_" + ] + }, + { + "cell_type": "markdown", + "id": "89470c7d", + "metadata": {}, + "source": [ + "##### `OptCV` tuning via `HillClimbing`" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "f606284b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "# 1. defining the tuned estimator\n", + "from sklearn.svm import SVC\n", + "from hyperactive.integrations.sklearn import OptCV\n", + "from hyperactive.opt import HillClimbing\n", + "\n", + "# picking the optimizer is the only part that changes!\n", + "hill_climbing_config = {\n", + " \"search_space\": {\n", + " \"C\": np.array([0.01, 0.1, 1, 10]),\n", + " \"gamma\": np.array([0.0001, 0.01, 0.1, 1, 10]),\n", + " },\n", + " \"n_iter\": 100,\n", + "}\n", + "hill_climbing = HillClimbing(**hill_climbing_config)\n", + "\n", + "tuned_svc = OptCV(SVC(), optimizer=hill_climbing)\n", + "\n", + "# 2. fitting the tuned estimator = tuning the hyperparameters\n", + "from sklearn.datasets import load_iris\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "X, y = load_iris(return_X_y=True)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", + "\n", + "tuned_svc.fit(X_train, y_train)\n", + "\n", + "# 3. making predictions with the tuned estimator\n", + "y_pred = tuned_svc.predict(X_test)" + ] + }, + { + "cell_type": "markdown", + "id": "1e7164df", + "metadata": {}, + "source": [ + "best parameters and best estimator - works as before!" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "14710f3d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'C': np.float64(10.0), 'gamma': np.float64(0.1)}" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tuned_svc.best_params_" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "0a55cdd6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SVC(C=np.float64(10.0), gamma=np.float64(0.1))
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "SVC(C=np.float64(10.0), gamma=np.float64(0.1))" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tuned_svc.best_estimator_" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sktime-skpro-skbase-313", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/extension_templates/experiments.py b/extension_templates/experiments.py index e9ecda17..35c6bac8 100644 --- a/extension_templates/experiments.py +++ b/extension_templates/experiments.py @@ -143,11 +143,13 @@ def _paramnames(self): Returns ------- - list of str + list of str, or None The parameter names of the search parameters. + If not known or arbitrary, return None. """ # for every instance, this should return the correct parameter names # i.e., the maximal set of keys of the dict expected by _score + # (if not known or arbitrary, return None) return ["score_param1", "score_param2"] # todo: implement this, mandatory diff --git a/src/hyperactive/base/_experiment.py b/src/hyperactive/base/_experiment.py index db5d2aeb..42d11d2b 100644 --- a/src/hyperactive/base/_experiment.py +++ b/src/hyperactive/base/_experiment.py @@ -35,8 +35,12 @@ def paramnames(self): Returns ------- - list of str + list of str, or None The parameter names of the search parameters. + + * If list of str, params in ``evaluate`` and ``score`` must match this list, + or a subset thereof. + * If None, arbitrary parameters can be passed to ``evaluate`` and ``score``. """ return self._paramnames() @@ -45,10 +49,11 @@ def _paramnames(self): Returns ------- - list of str + list of str, or None The parameter names of the search parameters. + If not known or arbitrary, return None. """ - raise NotImplementedError + return None def evaluate(self, params): """Evaluate the parameters. @@ -66,8 +71,11 @@ def evaluate(self, params): Additional metadata about the search. """ paramnames = self.paramnames() - if not set(params.keys()) <= set(paramnames): - raise ValueError("Parameters do not match.") + if paramnames is not None and not set(params.keys()) <= set(paramnames): + raise ValueError( + f"Parameters passed to {type(self)}.evaluate do not match: " + f"expected {paramnames}, got {list(params.keys())}." + ) res, metadata = self._evaluate(params) res = np.float64(res) return res, metadata diff --git a/src/hyperactive/base/_optimizer.py b/src/hyperactive/base/_optimizer.py index 7ab8c68a..91210fc4 100644 --- a/src/hyperactive/base/_optimizer.py +++ b/src/hyperactive/base/_optimizer.py @@ -49,7 +49,13 @@ def get_experiment(self): BaseExperiment The experiment to optimize parameters for. """ - return self._experiment + exp = self._experiment + exp_is_baseobj = isinstance(exp, BaseObject) + if not exp_is_baseobj or exp.get_tag("object_type") != "experiment": + from hyperactive.experiment.func import FunctionExperiment + + exp = FunctionExperiment(exp) # callable adapted to BaseExperiment + return exp def solve(self): """Run the optimization search process to maximize the experiment's score. diff --git a/src/hyperactive/base/tests/test_dynamic_exp.py b/src/hyperactive/base/tests/test_dynamic_exp.py new file mode 100644 index 00000000..a1a0af1f --- /dev/null +++ b/src/hyperactive/base/tests/test_dynamic_exp.py @@ -0,0 +1,32 @@ +"""Test that old function signature can be passed as experiment.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + + +def test_dynamic_exp(): + """Test that old function signature can be passed as experiment.""" + + # 1. define the experiment + def parabola(opt): + return opt["x"] ** 2 + opt["y"] ** 2 + + # 2. set up the HillClimbing optimizer + import numpy as np + + from hyperactive.opt import HillClimbing + + hillclimbing_config = { + "search_space": { + "x": np.array([0, 1, 2]), + "y": np.array([0, 1, 2]), + }, + } + hill_climbing = HillClimbing(**hillclimbing_config, experiment=parabola) + + # 3. run the HillClimbing optimizer + hill_climbing.solve() + + best_params = hill_climbing.best_params_ + assert best_params is not None, "Best parameters should not be None" + assert isinstance(best_params, dict), "Best parameters should be a dictionary" + assert "x" in best_params, "Best parameters should contain 'x'" + assert "y" in best_params, "Best parameters should contain 'y'" diff --git a/src/hyperactive/experiment/func.py b/src/hyperactive/experiment/func.py new file mode 100644 index 00000000..e744544f --- /dev/null +++ b/src/hyperactive/experiment/func.py @@ -0,0 +1,160 @@ +"""Dynamic experiment to allow passing of functions.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +from hyperactive.base import BaseExperiment + + +class FunctionExperiment(BaseExperiment): + """Experiment that wraps a function. + + Takes a callable that evaluates parameters; exposes it as the ``evaluate`` method. + + Assumes higher scores are better. + + Parameters + ---------- + func : callable + Function to evaluate parameters. + + * if ``parametrization="dict"``, should accept a single argument + which is a dictionary with string keys, and return a float. + * if ``parametrization="kwargs"``, should accept keyword arguments + with string keys, and return a float. + + parametrization : str, one of {"dict", "kwargs"}, default="dict" + The way parameters are passed to the function. + + Example + ------- + >>> from hyperactive.experiment.func import FunctionExperiment + + Parameterized by a dictionary: + >>> def parabola(opt): + ... return opt["x"]**2 + opt["y"]**2 + >>> para_exp = FunctionExperiment(parabola) + >>> params = {"x": 1, "y": 2} + >>> score, add_info = para_exp.score(params) + + Parameterized by keyword arguments: + >>> def parabola_kwargs(x, y): + ... return x**2 + y**2 + >>> para_exp_kwargs = FunctionExperiment(parabola_kwargs, parametrization="kwargs") + >>> score, add_info = para_exp_kwargs.score({"x": 1, "y": 2}) + """ # noqa: E501 + + def __init__(self, func, parametrization="dict"): + self.func = func + self.parametrization = parametrization + super().__init__() + + def _paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + if self.parametrization == "dict": + return None + elif self.parametrization == "kwargs": + import inspect + + sig = inspect.signature(self.func) + return list(sig.parameters.keys()) + + def _evaluate(self, params): + """Evaluate the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to evaluate. + + Returns + ------- + float + The value of the parameters as per evaluation. + dict + Additional metadata about the search. + """ + if self.parametrization == "dict": + loss = self.func(params) + elif self.parametrization == "kwargs": + loss = self.func(**params) + else: + raise ValueError( + f"Error in FunctionExperiment, " + f"unknown parametrization {self.parametrization}. " + "Use 'dict' or 'kwargs'." + ) + return loss, {} + + @classmethod + def get_test_params(cls, parameter_set="default"): + """Return testing parameter settings for the skbase object. + + ``get_test_params`` is a unified interface point to store + parameter settings for testing purposes. This function is also + used in ``create_test_instance`` and ``create_test_instances_and_names`` + to construct test instances. + + ``get_test_params`` should return a single ``dict``, or a ``list`` of ``dict``. + + Each ``dict`` is a parameter configuration for testing, + and can be used to construct an "interesting" test instance. + A call to ``cls(**params)`` should + be valid for all dictionaries ``params`` in the return of ``get_test_params``. + + The ``get_test_params`` need not return fixed lists of dictionaries, + it can also return dynamic or stochastic parameter settings. + + Parameters + ---------- + parameter_set : str, default="default" + Name of the set of test parameters to return, for use in tests. If no + special parameters are defined for a value, will return `"default"` set. + + Returns + ------- + params : dict or list of dict, default = {} + Parameters to create testing instances of the class + Each dict are parameters to construct an "interesting" test instance, i.e., + `MyClass(**params)` or `MyClass(**params[i])` creates a valid test instance. + `create_test_instance` uses the first (or only) dictionary in `params` + """ + + def _func1(x): + """Evaluate parameters. Used in tests.""" + return x["x"] ** 2 + x["y"] ** 2 + + def _func2(x): + """Evaluate parameters. Used in tests.""" + return x["x"] ** 2 - x["y"] ** 2 + 10 * x["x"] + 5 * x["z"] + + def _func3(x, y, z): + """Evaluate parameters. Used in tests.""" + return x**2 + y**2 - 3 * x + 2 * y + 1 + + params0 = {"func": _func1} + params1 = {"func": _func2} + params2 = {"func": _func3, "parametrization": "kwargs"} + return [params0, params1, params2] + + @classmethod + def _get_score_params(self): + """Return settings for testing score/evaluate functions. Used in tests only. + + Returns a list, the i-th element should be valid arguments for + self.evaluate and self.score, of an instance constructed with + self.get_test_params()[i]. + + Returns + ------- + list of dict + The parameters to be used for scoring. + """ + params0 = {"x": 0, "y": 0} + params1 = {"x": 1, "y": 1, "z": 2} + params2 = {"x": 3, "y": 4, "z": 5} + return [params0, params1, params2] diff --git a/src/hyperactive/tests/test_all_objects.py b/src/hyperactive/tests/test_all_objects.py index c78061c1..79cc256c 100644 --- a/src/hyperactive/tests/test_all_objects.py +++ b/src/hyperactive/tests/test_all_objects.py @@ -194,9 +194,10 @@ def test_paramnames(self, object_class): for inst, obj_param in zip(inst_params, obj_params): obj_inst = object_class(**inst) paramnames = obj_inst.paramnames() - assert set(obj_param.keys()) <= set( - paramnames - ), f"Parameter names do not match: {paramnames} != {obj_param}" + if paramnames is not None: + assert set(obj_param.keys()) <= set( + paramnames + ), f"Parameter names do not match: {paramnames} != {obj_param}" def test_score_function(self, object_class): """Test that substituting into score works as intended."""