diff --git a/examples/hyperactive_intro.ipynb b/examples/hyperactive_intro.ipynb new file mode 100644 index 00000000..5a047b32 --- /dev/null +++ b/examples/hyperactive_intro.ipynb @@ -0,0 +1,1723 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "af034c62", + "metadata": {}, + "source": [ + "## hyperactive - unified interfaces for optimizers and experiments" + ] + }, + { + "cell_type": "markdown", + "id": "830f7eb7", + "metadata": {}, + "source": [ + "### \"experiment\" = optimization problem" + ] + }, + { + "cell_type": "markdown", + "id": "668f9ccc", + "metadata": {}, + "source": [ + "\"experiment\" classes model optimization functions and ML experiments under one API\n", + "\n", + "Examples below:\n", + "1. simple objective function - parabola function\n", + "2. ML cross-validation experiment in sklearn" + ] + }, + { + "cell_type": "markdown", + "id": "2380a84b", + "metadata": {}, + "source": [ + "#### user defined objective function" + ] + }, + { + "cell_type": "markdown", + "id": "21f50e9e", + "metadata": {}, + "source": [ + "simple objective definition:\n", + "\n", + "* function with single dict argument\n", + "* keys are variable names\n", + "* function evaluates variables and returns float\n", + "* maximization is assumed later" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "8c428229", + "metadata": {}, + "outputs": [], + "source": [ + "def sphere(opt):\n", + " x = opt[\"x\"]\n", + " y = opt[\"y\"]\n", + "\n", + " return -x**2 - y**2" + ] + }, + { + "cell_type": "markdown", + "id": "a242a2fc", + "metadata": {}, + "source": [ + "to evaluate:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ccb460f1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "-13" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sphere({\"x\": 2, \"y\": 3})" + ] + }, + { + "cell_type": "markdown", + "id": "173f1923", + "metadata": {}, + "source": [ + "#### parametric objective functions - parametric" + ] + }, + { + "cell_type": "markdown", + "id": "fd01be6c", + "metadata": {}, + "source": [ + "parametric objective functions are classes:\n", + "\n", + "* construct with parameters\n", + "* call `evaluate` with `dict`\n", + "\n", + "`hyperactive` comes with predefined objective functions.\n", + "\n", + "These are parametric through the constructor\n", + "\n", + "Example: `Parabola` function, docstring outlines parametric form" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "c474de73", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;31mInit signature:\u001b[0m \u001b[0mParabola\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1.0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mc\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mDocstring:\u001b[0m Parabola class.\n", + "\u001b[1;31mInit docstring:\u001b[0m Construct BaseObject.\n", + "\u001b[1;31mFile:\u001b[0m c:\\workspace\\hyperactive\\src\\hyperactive\\experiment\\bench\\_parabola.py\n", + "\u001b[1;31mType:\u001b[0m type\n", + "\u001b[1;31mSubclasses:\u001b[0m " + ] + } + ], + "source": [ + "from hyperactive.experiment.bench import Parabola\n", + "\n", + "?Parabola" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "59dd4715", + "metadata": {}, + "outputs": [], + "source": [ + "parabola = Parabola(a=42, b=3, c=4)" + ] + }, + { + "cell_type": "markdown", + "id": "1de4894a", + "metadata": {}, + "source": [ + "parametric objectives are evaluated via `evaluate` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "97b41711", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(564.0)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "score, metadata = parabola.evaluate({\"x\": 2, \"y\": 3}) # also returns metadata\n", + "score" + ] + }, + { + "cell_type": "markdown", + "id": "4f561158", + "metadata": {}, + "source": [ + "instances of parametric objectives are also directly callable" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "797535db", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(-564.0)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parabola(x=2, y=3)\n", + "# output is always np.float64" + ] + }, + { + "cell_type": "markdown", + "id": "b8096712", + "metadata": {}, + "source": [ + "the \"experiment\" class has two sets of variables:\n", + "\n", + "* optimization variables = inputs of the objective - inspectable via `paramnames`\n", + "* parameters of the experiment = constant in the objective\n", + " * these are params of `__init__`\n", + " * and are inspectable via `get_params`" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "fc08e901", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['x', 'y']" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parabola.paramnames()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "61a62efa", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['a', 'b', 'c']" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(parabola.get_params())" + ] + }, + { + "cell_type": "markdown", + "id": "a1dbc553", + "metadata": {}, + "source": [ + "call via `score` method:\n", + "\n", + "* returns two objects: the value, and metadata (in a dict)\n", + "* input is a single dict, the `**` variant of a direct call" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "b127d406", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(np.float64(-564.0), {})" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parabola.score({\"x\": 2, \"y\": 3})" + ] + }, + { + "cell_type": "markdown", + "id": "98924abf", + "metadata": {}, + "source": [ + "#### sklearn cross-validation" + ] + }, + { + "cell_type": "markdown", + "id": "a625210e", + "metadata": {}, + "source": [ + "\"experiment\" can be more complicated - e.g., a cross-validation experiment\n", + "\n", + "this is a single tuning-evaluation step for an `sklearn` estimator" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "57110e86", + "metadata": {}, + "outputs": [], + "source": [ + "from hyperactive.experiment.integrations import SklearnCvExperiment\n", + "from sklearn.datasets import load_iris\n", + "from sklearn.svm import SVC\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.model_selection import KFold\n", + "\n", + "X, y = load_iris(return_X_y=True)\n", + "\n", + "sklearn_exp = SklearnCvExperiment(\n", + " estimator=SVC(),\n", + " scoring=accuracy_score,\n", + " cv=KFold(n_splits=3, shuffle=True),\n", + " X=X,\n", + " y=y,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "dcf183ae", + "metadata": {}, + "source": [ + "usage syntax same as for the simple parabola!" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "70a27348", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.9733333333333333)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sklearn_exp(C=1, gamma=0.3)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "1cfd28d4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['C',\n", + " 'break_ties',\n", + " 'cache_size',\n", + " 'class_weight',\n", + " 'coef0',\n", + " 'decision_function_shape',\n", + " 'degree',\n", + " 'gamma',\n", + " 'kernel',\n", + " 'max_iter',\n", + " 'probability',\n", + " 'random_state',\n", + " 'shrinking',\n", + " 'tol',\n", + " 'verbose']" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sklearn_exp.paramnames()" + ] + }, + { + "cell_type": "markdown", + "id": "165c5830", + "metadata": {}, + "source": [ + "`get_params` works like in `sklearn` and is nested\n", + "\n", + "note that similar parameters appear as in `paramnames`\n", + "\n", + "* parameters in `paramnames` are optimized over\n", + "* parameters in `get_params` are default values, if not set in `score`" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "b6160e3c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['X',\n", + " 'cv',\n", + " 'estimator',\n", + " 'scoring',\n", + " 'y',\n", + " 'estimator__C',\n", + " 'estimator__break_ties',\n", + " 'estimator__cache_size',\n", + " 'estimator__class_weight',\n", + " 'estimator__coef0',\n", + " 'estimator__decision_function_shape',\n", + " 'estimator__degree',\n", + " 'estimator__gamma',\n", + " 'estimator__kernel',\n", + " 'estimator__max_iter',\n", + " 'estimator__probability',\n", + " 'estimator__random_state',\n", + " 'estimator__shrinking',\n", + " 'estimator__tol',\n", + " 'estimator__verbose']" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(sklearn_exp.get_params().keys())" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "f74ca927", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(np.float64(0.98),\n", + " {'score_time': array([0. , 0.01113176, 0.00051761]),\n", + " 'fit_time': array([0. , 0. , 0.00100803]),\n", + " 'n_test_samples': 150})" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sklearn_exp.score({\"C\": 1, \"gamma\": 0.3})" + ] + }, + { + "cell_type": "markdown", + "id": "71073496", + "metadata": {}, + "source": [ + "### use of optimizers" + ] + }, + { + "cell_type": "markdown", + "id": "725230fc", + "metadata": {}, + "source": [ + "#### optimizing a custom objective" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "ab78b796", + "metadata": {}, + "outputs": [], + "source": [ + "def sphere(opt):\n", + " x = opt[\"x\"]\n", + " y = opt[\"y\"]\n", + "\n", + " return -x**2 - y**2" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "7104e5ec", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/plain": [ + "{'x': np.float64(0.10101010101010033), 'y': np.float64(0.10101010101010033)}" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "from hyperactive.opt import HillClimbing\n", + "\n", + "hillclimbing_config = {\n", + " \"search_space\": {\n", + " \"x\": np.linspace(-10, 10, 100),\n", + " \"y\": np.linspace(-10, 10, 100),\n", + " },\n", + " \"n_iter\": 1000,\n", + "}\n", + "hill_climbing = HillClimbing(**hillclimbing_config, experiment=sphere)\n", + "\n", + "hill_climbing.solve()" + ] + }, + { + "cell_type": "markdown", + "id": "300a1666", + "metadata": {}, + "source": [ + "#### Grid search & sklearn CV" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "5e2328c9", + "metadata": {}, + "outputs": [], + "source": [ + "from hyperactive.experiment.integrations import SklearnCvExperiment\n", + "from sklearn.datasets import load_iris\n", + "from sklearn.svm import SVC\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.model_selection import KFold\n", + "\n", + "X, y = load_iris(return_X_y=True)\n", + "\n", + "sklearn_exp = SklearnCvExperiment(\n", + " estimator=SVC(),\n", + " scoring=accuracy_score,\n", + " cv=KFold(n_splits=3, shuffle=True),\n", + " X=X,\n", + " y=y,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "e9a07a73", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'C': 0.01, 'gamma': 1}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from hyperactive.opt import GridSearchSk as GridSearch\n", + "\n", + "param_grid = {\n", + " \"C\": [0.01, 0.1, 1, 10],\n", + " \"gamma\": [0.0001, 0.01, 0.1, 1, 10],\n", + "}\n", + "grid_search = GridSearch(param_grid=param_grid, experiment=sklearn_exp)\n", + "\n", + "grid_search.solve()" + ] + }, + { + "cell_type": "markdown", + "id": "72fcf886", + "metadata": {}, + "source": [ + "#### hill climbing & sklearn CV" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "f9a4d922", + "metadata": {}, + "outputs": [], + "source": [ + "from hyperactive.experiment.integrations import SklearnCvExperiment\n", + "from sklearn.datasets import load_iris\n", + "from sklearn.svm import SVC\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.model_selection import KFold\n", + "\n", + "X, y = load_iris(return_X_y=True)\n", + "\n", + "sklearn_exp = SklearnCvExperiment(\n", + " estimator=SVC(),\n", + " scoring=accuracy_score,\n", + " cv=KFold(n_splits=3, shuffle=True),\n", + " X=X,\n", + " y=y,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "9a13b4f3", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/plain": [ + "{'C': np.float64(10.0), 'gamma': np.float64(0.1)}" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "from hyperactive.opt import HillClimbing\n", + "\n", + "hillclimbing_config = {\n", + " \"search_space\": {\n", + " \"C\": np.array([0.01, 0.1, 1, 10]),\n", + " \"gamma\": np.array([0.0001, 0.01, 0.1, 1, 10]),\n", + " },\n", + " \"n_iter\": 100,\n", + "}\n", + "hill_climbing = HillClimbing(**hillclimbing_config, experiment=sklearn_exp)\n", + "\n", + "hill_climbing.solve()" + ] + }, + { + "cell_type": "markdown", + "id": "7a933b41", + "metadata": {}, + "source": [ + "### full sklearn integration as estimator" + ] + }, + { + "cell_type": "markdown", + "id": "1f368679", + "metadata": {}, + "source": [ + "`OptCV` allows `sklearn` tuning via any tuning algorithm.\n", + "\n", + "Below, we show tuning via:\n", + "\n", + "* standard `GridSearch`\n", + "* `HillClimbing` from `gradient-free-optimizers`" + ] + }, + { + "cell_type": "markdown", + "id": "7171fc17", + "metadata": {}, + "source": [ + "##### `OptCV` tuning via `GridSearch`" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "4bdf2d49", + "metadata": {}, + "outputs": [], + "source": [ + "# 1. defining the tuned estimator\n", + "from sklearn.svm import SVC\n", + "from hyperactive.integrations.sklearn import OptCV\n", + "from hyperactive.opt import GridSearchSk as GridSearch\n", + "\n", + "param_grid = {\"kernel\": [\"linear\", \"rbf\"], \"C\": [1, 10]}\n", + "tuned_svc = OptCV(SVC(), optimizer=GridSearch(param_grid))\n", + "\n", + "# 2. fitting the tuned estimator = tuning the hyperparameters\n", + "from sklearn.datasets import load_iris\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "X, y = load_iris(return_X_y=True)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", + "\n", + "tuned_svc.fit(X_train, y_train)\n", + "\n", + "# 3. making predictions with the tuned estimator\n", + "y_pred = tuned_svc.predict(X_test)" + ] + }, + { + "cell_type": "markdown", + "id": "1a4bddc0", + "metadata": {}, + "source": [ + "best parameters and best estimator can be returned" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "fdd8c14a", + "metadata": {}, + "outputs": [], + "source": [ + "best_params = tuned_svc.best_params_" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "252efea6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SVC(C=1)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
SVC(C=1)
SVC(C=np.float64(10.0), gamma=np.float64(0.1))In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
SVC(C=np.float64(10.0), gamma=np.float64(0.1))