From bb4bc8d3b84ef64c63fbaa7cbcc8a02791fbf997 Mon Sep 17 00:00:00 2001 From: PKGuo <56018055+PKGuo@users.noreply.github.com> Date: Wed, 18 Nov 2020 00:34:59 +0100 Subject: [PATCH 1/6] Sklearnify NB3 --- notebooks/3_KernelMethods.ipynb | 152 ++++++++++++++++++-------------- 1 file changed, 87 insertions(+), 65 deletions(-) diff --git a/notebooks/3_KernelMethods.ipynb b/notebooks/3_KernelMethods.ipynb index fa773d3..d93330e 100644 --- a/notebooks/3_KernelMethods.ipynb +++ b/notebooks/3_KernelMethods.ipynb @@ -8,7 +8,7 @@ "\n", "In this notebook, we introduce kernel-based versions of the models covered in the [Linear Methods](1_LinearMethods.ipynb) and [PCovR](2_PrincipalCovariatesRegression.ipynb) notebooks.\n", "\n", - "As always, for each model, we first go step-by-step through the derivation, with equations, embedded links, and citations supplied where useful. Second, we will employ a \"Utility Class\" for the model, which can be found in the utilities folder and contains all necessary functions.\n", + "As always, for each model, we first go step-by-step through the derivation, with equations, embedded links, and citations supplied where useful. Second, we will employ a \"Sklearn/Skcosmo Class\" for the models, which can be found in the skcosmo/sklearn mudule and contains all necessary functions.\n", "\n", "**Note**: To display values of variables in the markdown cells, you should [enable the jupyter contrib nbextension](https://stackoverflow.com/questions/52812231/print-variable-in-jupyter-notebook-markdown-cell-python)." ] @@ -35,9 +35,13 @@ "from utilities.plotting import (\n", " plot_projection, plot_regression, check_mirrors, get_cmaps, table_from_dict\n", ")\n", - "from utilities.kernels import linear_kernel, gaussian_kernel, center_kernel\n", - "from utilities.classes import PCA, LR, KPCA, KRR, KPCovR\n", - "\n", + "from utilities.kernels import linear_kernel, gaussian_kernel,center_kernel # to be replaced with scalers from skcosmo\n", + "#from sklearn.metrics.pairwise import linear_kernel, rbf_kernel \n", + "#I found that the result of rbf_kernel(gamma = 1.0) in sklearn is not the same as that of gaussian_kernel(gamma = 1.0) in utilities, so I keep the original gaussian_kernel.\n", + "from utilities.classes import KPCovR # to be replaced with scalers from skcosmo\n", + "from sklearn.decomposition import PCA, KernelPCA\n", + "from sklearn.linear_model import Ridge\n", + "from sklearn.kernel_ridge import KernelRidge\n", "cmaps = get_cmaps()\n", "plt.style.use('../utilities/kernel_pcovr.mplstyle')\n", "dbl_fig=(2*plt.rcParams['figure.figsize'][0], plt.rcParams['figure.figsize'][1])" @@ -83,7 +87,7 @@ " k(\\mathbf{x}, \\mathbf{x}^{\\prime}) = \\exp{\\left(-\\gamma\\lVert \\mathbf{x} - \\mathbf{x}^{\\prime}\\rVert^2\\right)}.\n", "\\end{equation}\n", "\n", - "In this notebook we use the gaussian kernel (```gaussian_kernel(XA, XB, gamma=1.0)```), however, we have also included the linear kernel function (```linear_kernel(XA, XB)```) in ```utilities/kernels.py```. You can check rather easily that if you use a linear kernel all of the kernel methods reduce explicitly to linear regression or principal component analysis in the original feature space." + "In this notebook we use the gaussian kernel (```gaussian_kernel(XA, XB, gamma=1.0)```), however, we have also included the linear kernel function (```linear_kernel(XA, XB)```) in ```skcosmo```. You can check rather easily that if you use a linear kernel all of the kernel methods reduce explicitly to linear regression or principal component analysis in the original feature space." ] }, { @@ -590,10 +594,10 @@ "metadata": {}, "outputs": [], "source": [ - "lr = LR(regularization=regularization)\n", + "lr = Ridge(alpha=regularization)\n", "\n", "lr.fit(X_train, Y_train)\n", - "Y_lr = lr.transform(X_test)\n", + "Y_lr = lr.predict(X_test)\n", "\n", "table_from_dict([get_stats(x=X_test, \n", " yp=Y_lr,\n", @@ -623,11 +627,11 @@ "outputs": [], "source": [ "regularizations = np.linspace(-10, -2, 24)\n", - "krrs = [KRR(regularization=10**i, kernel_type=kernel_type) \n", + "krrs = [KernelRidge(alpha=10**i, kernel = 'rbf', gamma = 1.0) \n", " for i in regularizations]\n", "\n", "for k in krrs:\n", - " k.fit(X=X_train, K=K_train, Y=Y_train)" + " k.fit(X=X_train, y=Y_train)" ] }, { @@ -636,10 +640,8 @@ "metadata": {}, "outputs": [], "source": [ - "coeff_deter = np.array([k.statistics(X=X_test, \n", - " Y=Y_test, \n", - " K=K_test)['Coefficient of Determination
($R^2$)'] for k in krrs])\n", - "best_regularization = 10**regularizations[coeff_deter.argmax()]" + "ell = np.array([np.linalg.norm(Y_test - k.predict(X_test)) ** 2.0 / np.linalg.norm(Y_test) **2.0 for k in krrs])\n", + "best_regularization = 10**regularizations[ell.argmin()]" ] }, { @@ -655,15 +657,15 @@ "metadata": {}, "outputs": [], "source": [ - "plt.semilogx(10**regularizations, coeff_deter, marker='o', zorder=-1)\n", + "plt.semilogx(10**regularizations, ell, marker='o', zorder=-1)\n", "plt.scatter(best_regularization,\n", - " max(coeff_deter), marker='o', color='r',\n", + " min(ell), marker='o', color='r',\n", " )\n", "\n", "plt.xlabel(r\"$\\lambda$\")\n", - "plt.ylabel(r\"$R^2$\")\n", + "plt.ylabel(r\"$\\ell$\")\n", "\n", - "plt.title(r\"Effect of $\\lambda$ on $R^2$ for Kernel Ridge Regression\")\n", + "plt.title(r\"Effect of $\\lambda$ on $\\ell$ for Kernel Ridge Regression\")\n", "\n", "plt.show()" ] @@ -717,10 +719,11 @@ "\n", "regularization = 1e-12\n", "\n", - "krr = KRR(regularization=regularization, kernel_type=kernel_type)\n", + "krr = KernelRidge(alpha=regularization, kernel = 'rbf', gamma = 1.0)\n", "krr.fit(X_train, Y_train)\n", - "Yhat_train = krr.transform(X_train).reshape(-1,Y.shape[-1])\n", - "PKY = krr.PKY\n", + "Yhat_train = krr.predict(X_train).reshape(-1,Y.shape[-1])\n", + "PKY = np.linalg.solve(K_train + regularization*np.eye(n_train), \n", + " Y_train)\n", "\n", "K_pca = K_train/(np.trace(K_train)/n_train)\n", "K_lr = np.matmul(Yhat_train, Yhat_train.T)\n", @@ -829,7 +832,7 @@ "source": [ "fig, axes = plt.subplots(1,2, figsize=dbl_fig)\n", "\n", - "ref = KPCA(n_PC=2, kernel_type=kernel_type)\n", + "ref = KernelPCA(n_components=2, kernel = 'rbf', gamma = 1.0)\n", "ref.fit(X_train)\n", "xref = ref.transform(X_test)\n", "\n", @@ -889,9 +892,9 @@ "source": [ "fig, axes = plt.subplots(1,2, figsize=dbl_fig)\n", "\n", - "ref = KRR(regularization=regularization, kernel_type=kernel_type)\n", + "ref = KernelRidge(alpha=regularization, kernel = 'rbf', gamma = 1.0)\n", "ref.fit(X_train, Y_train)\n", - "yref = ref.transform(X_test)\n", + "yref = ref.predict(X_test)\n", "\n", "errors = np.concatenate((np.abs(Ypred-Y_test),np.abs(yref-Y_test)))\n", "\n", @@ -1121,9 +1124,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# The Utility Classes\n", + "# Implementation in `scikit-learn`\n", "\n", - "Classes from the utility module enable computing KPCA, KRR, and KPCovR with a scikit.learn-like syntax. " + "Classes from `sklearn` enable computing KPCA and KRR. Here we'll review the basics of their usage; readers are encouraged to refer to sklearn documentation for further detail." ] }, { @@ -1132,16 +1135,8 @@ "metadata": {}, "outputs": [], "source": [ - "from utilities.classes import KPCA, KRR, KPCovR" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Important Note**: In all kernel classes, the functions `fit`, `transform`, and `statistics` will either compute the designated kernel for the supplied $\\mathbf{X}$ data or use the provided precomputed kernel. The kernel is **always** a keyword argument (e.g. `model.fit(K=K)`), and the first argument, positionally, is $\\mathbf{X}$.\n", - "\n", - "In each demonstration, we will show the function signature using X, but we will also supply our precomputed kernel for computational efficiency." + "from sklearn.decomposition import KernelPCA\n", + "from sklearn.kernel_ridge import KernelRidge" ] }, { @@ -1157,7 +1152,7 @@ "metadata": {}, "outputs": [], "source": [ - "kpca = KPCA(n_PC=2, kernel_type=kernel_type)" + "kpca = KernelPCA(n_components=2, kernel = 'rbf', gamma = 1.0)" ] }, { @@ -1173,7 +1168,7 @@ "metadata": {}, "outputs": [], "source": [ - "kpca.fit(X_train, K=K_train)" + "kpca.fit(X_train)" ] }, { @@ -1189,8 +1184,8 @@ "metadata": {}, "outputs": [], "source": [ - "T_KPCA_train = kpca.transform(X_train, K=K_train)\n", - "T_KPCA_test= kpca.transform(X_test, K=K_test)" + "T_KPCA_train = kpca.transform(X_train)\n", + "T_KPCA_test= kpca.transform(X_test)" ] }, { @@ -1208,21 +1203,35 @@ "plt.show()" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Calling `kpca.statistics(X, Y)` will output the statistics of the projection of the kernel created from $\\mathbf{X}$ and the regression onto $\\mathbf{Y}$." - ] - }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "table_from_dict([kpca.statistics(X_train, Y_train, K=K_train), \n", - " kpca.statistics(X_test, Y_test, K=K_test)], \n", + "PTX = np.matmul(np.diagflat(1/((v_K[:n_PC]))) , \n", + " np.matmul(T_KPCA_train.T, X_train))\n", + "\n", + "Xr_train = np.matmul(T_KPCA_train, PTX)\n", + "Xr_test = np.matmul(T_KPCA_test, PTX)\n", + "\n", + "K_approx_train = np.matmul(T_KPCA_train,T_KPCA_train.T)\n", + "K_test_test = kernel_func(X_test, X_test)\n", + "K_test_test = center_kernel(K_test_test)\n", + "K_approx_test = np.matmul(T_KPCA_test,T_KPCA_test.T)\n", + "\n", + "table_from_dict([get_stats(x=X_train, \n", + " xr=Xr_train,\n", + " y=Y_train, \n", + " t=T_KPCA_train, \n", + " k=K_train, \n", + " kapprox=K_approx_train), \n", + " get_stats(x=X_test, \n", + " xr=Xr_test,\n", + " y=Y_test, \n", + " t=T_KPCA_test, \n", + " k=K_test_test, \n", + " kapprox=K_approx_test)], \n", " headers = [\"Training\", \"Testing\"], \n", " title=\"KPCA\")" ] @@ -1240,7 +1249,7 @@ "metadata": {}, "outputs": [], "source": [ - "krr = KRR(regularization=regularization, kernel_type=kernel_type)" + "krr = KernelRidge(alpha=regularization, kernel = 'rbf', gamma = 1.0)" ] }, { @@ -1256,14 +1265,14 @@ "metadata": {}, "outputs": [], "source": [ - "krr.fit(X_train, Y_train, K=K_train)" + "krr.fit(X_train, Y_train)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Calling `krr.transform(X)` will generate the desired kernel from input $\\mathbf{X}$ and compute and return the predicted $\\mathbf{Y}$ values from $\\hat{\\mathbf{Y}}_{KRR} = \\mathbf{K}\\mathbf{P}_{KY}$." + "Calling `krr.predict(X)` will generate the desired kernel from input $\\mathbf{X}$ and compute and return the predicted $\\mathbf{Y}$ values from $\\hat{\\mathbf{Y}}_{KRR} = \\mathbf{K}\\mathbf{P}_{KY}$." ] }, { @@ -1272,8 +1281,8 @@ "metadata": {}, "outputs": [], "source": [ - "Y_krr_train = krr.transform(X_train, K=K_train)\n", - "Y_krr_test = krr.transform(X_test, K=K_test)" + "Y_krr_train = krr.predict(X_train)\n", + "Y_krr_test = krr.predict(X_test)" ] }, { @@ -1291,11 +1300,25 @@ "plt.show()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "table_from_dict([get_stats(x=X_train, yp=Y_krr_train,y=Y_train, k=K_train), \n", + " get_stats(x=X_test, yp=Y_krr_test, y=Y_test, k=K_test)], \n", + " headers = [\"Training\", \"Testing\"], \n", + " title=\"Kernel Ridge Regression\")" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Calling `krr.statistics(X,Y)` will output the statistics of the regression of $\\mathbf{X}$ and $\\mathbf{Y}$." + "# Implementation in `skcosmo`\n", + "\n", + "Classes from `skcosmo` enable computing KPCovR. Here we'll review the basics of their usage; readers are encouraged to refer to skcosmo documentation for further detail." ] }, { @@ -1304,10 +1327,16 @@ "metadata": {}, "outputs": [], "source": [ - "table_from_dict([krr.statistics(X=X_train, Y=Y_train, K=K_train), \n", - " krr.statistics(X=X_test, Y=Y_test, K=K_test)], \n", - " headers = [\"Training\", \"Testing\"], \n", - " title=\"Kernel Ridge Regression\")" + "from utilities.classes import KPCovR # to be replaced with scalers from skcosmo" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Important Note**: In the class, the functions `fit`, `transform`, and `statistics` will either compute the designated kernel for the supplied $\\mathbf{X}$ data or use the provided precomputed kernel. The kernel is **always** a keyword argument (e.g. `model.fit(K=K)`), and the first argument, positionally, is $\\mathbf{X}$.\n", + "\n", + "We will show the function signature using X, but we will also supply our precomputed kernel for computational efficiency." ] }, { @@ -1394,13 +1423,6 @@ " ], \\\n", " headers = [r\"KPCovR ($\\alpha={}$)\".format(kp.alpha), \"KRR\", \"KPCA\"])" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From 97d3d609da6ad771ec6bf7a035c8221087e67714 Mon Sep 17 00:00:00 2001 From: PKGuo <56018055+PKGuo@users.noreply.github.com> Date: Wed, 18 Nov 2020 16:28:10 +0100 Subject: [PATCH 2/6] Modified NB3 as suggested by Rose --- notebooks/3_KernelMethods.ipynb | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/notebooks/3_KernelMethods.ipynb b/notebooks/3_KernelMethods.ipynb index d93330e..3cc8210 100644 --- a/notebooks/3_KernelMethods.ipynb +++ b/notebooks/3_KernelMethods.ipynb @@ -35,10 +35,10 @@ "from utilities.plotting import (\n", " plot_projection, plot_regression, check_mirrors, get_cmaps, table_from_dict\n", ")\n", - "from utilities.kernels import linear_kernel, gaussian_kernel,center_kernel # to be replaced with scalers from skcosmo\n", + "from utilities.kernels import linear_kernel, gaussian_kernel,center_kernel # to be replaced with functions from skcosmo\n", "#from sklearn.metrics.pairwise import linear_kernel, rbf_kernel \n", "#I found that the result of rbf_kernel(gamma = 1.0) in sklearn is not the same as that of gaussian_kernel(gamma = 1.0) in utilities, so I keep the original gaussian_kernel.\n", - "from utilities.classes import KPCovR # to be replaced with scalers from skcosmo\n", + "from utilities.classes import KPCovR # to be replaced with the class from skcosmo\n", "from sklearn.decomposition import PCA, KernelPCA\n", "from sklearn.linear_model import Ridge\n", "from sklearn.kernel_ridge import KernelRidge\n", @@ -99,6 +99,7 @@ "# Changing the kernels in this cell will change them throughout the notebook tutorial.\n", "# The function variable designates the kernel function to use throughout the notebook.\n", "# The string variable is used to pass to the utility classes. \n", + "kernel_params = {\"kernel\": \"rbf\", \"gamma\": 1.0}\n", "kernel_func = gaussian_kernel\n", "kernel_type = 'gaussian'" ] @@ -627,11 +628,11 @@ "outputs": [], "source": [ "regularizations = np.linspace(-10, -2, 24)\n", - "krrs = [KernelRidge(alpha=10**i, kernel = 'rbf', gamma = 1.0) \n", + "krrs = [KernelRidge(alpha=10**i, kernel = 'precomputed') \n", " for i in regularizations]\n", "\n", "for k in krrs:\n", - " k.fit(X=X_train, y=Y_train)" + " k.fit(X=K_train, y = Y_train)" ] }, { @@ -640,7 +641,7 @@ "metadata": {}, "outputs": [], "source": [ - "ell = np.array([np.linalg.norm(Y_test - k.predict(X_test)) ** 2.0 / np.linalg.norm(Y_test) **2.0 for k in krrs])\n", + "ell = np.array([np.linalg.norm(Y_test - k.predict(K_test)) ** 2.0 / np.linalg.norm(Y_test) **2.0 for k in krrs])\n", "best_regularization = 10**regularizations[ell.argmin()]" ] }, @@ -719,12 +720,12 @@ "\n", "regularization = 1e-12\n", "\n", - "krr = KernelRidge(alpha=regularization, kernel = 'rbf', gamma = 1.0)\n", + "krr = KernelRidge(alpha=regularization, **kernel_params)\n", "krr.fit(X_train, Y_train)\n", "Yhat_train = krr.predict(X_train).reshape(-1,Y.shape[-1])\n", "PKY = np.linalg.solve(K_train + regularization*np.eye(n_train), \n", " Y_train)\n", - "\n", + "#PKY = krr.dual_coef_\n", "K_pca = K_train/(np.trace(K_train)/n_train)\n", "K_lr = np.matmul(Yhat_train, Yhat_train.T)\n", " \n", @@ -832,7 +833,7 @@ "source": [ "fig, axes = plt.subplots(1,2, figsize=dbl_fig)\n", "\n", - "ref = KernelPCA(n_components=2, kernel = 'rbf', gamma = 1.0)\n", + "ref = KernelPCA(n_components=2, **kernel_params)\n", "ref.fit(X_train)\n", "xref = ref.transform(X_test)\n", "\n", @@ -892,7 +893,7 @@ "source": [ "fig, axes = plt.subplots(1,2, figsize=dbl_fig)\n", "\n", - "ref = KernelRidge(alpha=regularization, kernel = 'rbf', gamma = 1.0)\n", + "ref = KernelRidge(alpha=regularization, **kernel_params)\n", "ref.fit(X_train, Y_train)\n", "yref = ref.predict(X_test)\n", "\n", @@ -1152,7 +1153,9 @@ "metadata": {}, "outputs": [], "source": [ - "kpca = KernelPCA(n_components=2, kernel = 'rbf', gamma = 1.0)" + "#If you want, you can set kernel = 'precomputed' and supply kpca.fit(X) with X = K_train to improve computational efficiency.\n", + "#kernel_params['kernel'] = 'precomputed'\n", + "kpca = KernelPCA(n_components=2, **kernel_params)" ] }, { @@ -1249,7 +1252,9 @@ "metadata": {}, "outputs": [], "source": [ - "krr = KernelRidge(alpha=regularization, kernel = 'rbf', gamma = 1.0)" + "#If you want, you can set kernel = 'precomputed' and supply krr.fit(X,y) with X = K_train to improve computational efficiency.\n", + "#kernel_params['kernel'] = 'precomputed'\n", + "krr = KernelRidge(alpha=regularization, **kernel_params)" ] }, { @@ -1327,7 +1332,7 @@ "metadata": {}, "outputs": [], "source": [ - "from utilities.classes import KPCovR # to be replaced with scalers from skcosmo" + "from utilities.classes import KPCovR # to be replaced with the class from skcosmo" ] }, { From f2bcbae12f15ba361853276c3c78e3353f4298fa Mon Sep 17 00:00:00 2001 From: PKGuo <56018055+PKGuo@users.noreply.github.com> Date: Fri, 20 Nov 2020 16:30:04 +0100 Subject: [PATCH 3/6] Changed kernel functions --- notebooks/3_KernelMethods.ipynb | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/notebooks/3_KernelMethods.ipynb b/notebooks/3_KernelMethods.ipynb index 3cc8210..079a509 100644 --- a/notebooks/3_KernelMethods.ipynb +++ b/notebooks/3_KernelMethods.ipynb @@ -36,8 +36,7 @@ " plot_projection, plot_regression, check_mirrors, get_cmaps, table_from_dict\n", ")\n", "from utilities.kernels import linear_kernel, gaussian_kernel,center_kernel # to be replaced with functions from skcosmo\n", - "#from sklearn.metrics.pairwise import linear_kernel, rbf_kernel \n", - "#I found that the result of rbf_kernel(gamma = 1.0) in sklearn is not the same as that of gaussian_kernel(gamma = 1.0) in utilities, so I keep the original gaussian_kernel.\n", + "from sklearn.metrics.pairwise import linear_kernel, rbf_kernel \n", "from utilities.classes import KPCovR # to be replaced with the class from skcosmo\n", "from sklearn.decomposition import PCA, KernelPCA\n", "from sklearn.linear_model import Ridge\n", @@ -100,7 +99,7 @@ "# The function variable designates the kernel function to use throughout the notebook.\n", "# The string variable is used to pass to the utility classes. \n", "kernel_params = {\"kernel\": \"rbf\", \"gamma\": 1.0}\n", - "kernel_func = gaussian_kernel\n", + "kernel_func = rbf_kernel\n", "kernel_type = 'gaussian'" ] }, @@ -138,7 +137,7 @@ "metadata": {}, "outputs": [], "source": [ - "K_train_raw = kernel_func(X_train, X_train)" + "K_train_raw = kernel_func(X_train, X_train, gamma = 1.0)" ] }, { @@ -254,7 +253,7 @@ "metadata": {}, "outputs": [], "source": [ - "K_test = kernel_func(X_test, X_train)" + "K_test = kernel_func(X_test, X_train, gamma = 1.0)" ] }, { @@ -456,7 +455,7 @@ "source": [ "K_approx_train = np.matmul(T_KPCA_train,T_KPCA_train.T)\n", "\n", - "K_test_test = kernel_func(X_test, X_test)\n", + "K_test_test = kernel_func(X_test, X_test,gamma = 1.0)\n", "K_test_test = center_kernel(K_test_test)\n", "K_approx_test = np.matmul(T_KPCA_test,T_KPCA_test.T)\n", "\n", @@ -720,12 +719,10 @@ "\n", "regularization = 1e-12\n", "\n", - "krr = KernelRidge(alpha=regularization, **kernel_params)\n", - "krr.fit(X_train, Y_train)\n", - "Yhat_train = krr.predict(X_train).reshape(-1,Y.shape[-1])\n", - "PKY = np.linalg.solve(K_train + regularization*np.eye(n_train), \n", - " Y_train)\n", - "#PKY = krr.dual_coef_\n", + "krr = KernelRidge(alpha=regularization, kernel='precomputed')\n", + "krr.fit(K_train, Y_train)\n", + "Yhat_train = krr.predict(K_train).reshape(-1,Y.shape[-1])\n", + "PKY = krr.dual_coef_\n", "K_pca = K_train/(np.trace(K_train)/n_train)\n", "K_lr = np.matmul(Yhat_train, Yhat_train.T)\n", " \n", @@ -1219,7 +1216,7 @@ "Xr_test = np.matmul(T_KPCA_test, PTX)\n", "\n", "K_approx_train = np.matmul(T_KPCA_train,T_KPCA_train.T)\n", - "K_test_test = kernel_func(X_test, X_test)\n", + "K_test_test = kernel_func(X_test, X_test,gamma = 1.0)\n", "K_test_test = center_kernel(K_test_test)\n", "K_approx_test = np.matmul(T_KPCA_test,T_KPCA_test.T)\n", "\n", From f3cd1c981f77391724d732d0ec06ba1f99e4936a Mon Sep 17 00:00:00 2001 From: PKGuo <56018055+PKGuo@users.noreply.github.com> Date: Fri, 20 Nov 2020 22:54:02 +0100 Subject: [PATCH 4/6] Set kernel = 'precomputed' --- notebooks/3_KernelMethods.ipynb | 44 ++++++++++++++++----------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/notebooks/3_KernelMethods.ipynb b/notebooks/3_KernelMethods.ipynb index 079a509..cb71560 100644 --- a/notebooks/3_KernelMethods.ipynb +++ b/notebooks/3_KernelMethods.ipynb @@ -35,7 +35,7 @@ "from utilities.plotting import (\n", " plot_projection, plot_regression, check_mirrors, get_cmaps, table_from_dict\n", ")\n", - "from utilities.kernels import linear_kernel, gaussian_kernel,center_kernel # to be replaced with functions from skcosmo\n", + "from utilities.kernels import center_kernel # to be replaced the function from skcosmo\n", "from sklearn.metrics.pairwise import linear_kernel, rbf_kernel \n", "from utilities.classes import KPCovR # to be replaced with the class from skcosmo\n", "from sklearn.decomposition import PCA, KernelPCA\n", @@ -98,7 +98,7 @@ "# Changing the kernels in this cell will change them throughout the notebook tutorial.\n", "# The function variable designates the kernel function to use throughout the notebook.\n", "# The string variable is used to pass to the utility classes. \n", - "kernel_params = {\"kernel\": \"rbf\", \"gamma\": 1.0}\n", + "kernel_params = {\"kernel\": \"precomputed\"}\n", "kernel_func = rbf_kernel\n", "kernel_type = 'gaussian'" ] @@ -627,7 +627,7 @@ "outputs": [], "source": [ "regularizations = np.linspace(-10, -2, 24)\n", - "krrs = [KernelRidge(alpha=10**i, kernel = 'precomputed') \n", + "krrs = [KernelRidge(alpha=10**i, **kernel_params) \n", " for i in regularizations]\n", "\n", "for k in krrs:\n", @@ -719,7 +719,7 @@ "\n", "regularization = 1e-12\n", "\n", - "krr = KernelRidge(alpha=regularization, kernel='precomputed')\n", + "krr = KernelRidge(alpha=regularization, **kernel_params)\n", "krr.fit(K_train, Y_train)\n", "Yhat_train = krr.predict(K_train).reshape(-1,Y.shape[-1])\n", "PKY = krr.dual_coef_\n", @@ -831,12 +831,12 @@ "fig, axes = plt.subplots(1,2, figsize=dbl_fig)\n", "\n", "ref = KernelPCA(n_components=2, **kernel_params)\n", - "ref.fit(X_train)\n", - "xref = ref.transform(X_test)\n", + "ref.fit(K_train)\n", + "T_kpca = ref.transform(K_test)\n", "\n", "plot_projection(Y_test, T_kpcovr_test, fig=fig, ax=axes[0], \n", " title = r\"KPCovR ($\\alpha={}$)\".format(alpha), **cmaps)\n", - "plot_projection(Y_test, xref, fig=fig, ax=axes[1], title = \"KPCA\", **cmaps)\n", + "plot_projection(Y_test, T_kpca, fig=fig, ax=axes[1], title = \"KPCA\", **cmaps)\n", "\n", "fig.suptitle(r\"These will become increasingly different as $\\alpha \\to 0.0.$\", \n", " y=0.0, fontsize=plt.rcParams['font.size']+6)\n", @@ -891,8 +891,8 @@ "fig, axes = plt.subplots(1,2, figsize=dbl_fig)\n", "\n", "ref = KernelRidge(alpha=regularization, **kernel_params)\n", - "ref.fit(X_train, Y_train)\n", - "yref = ref.predict(X_test)\n", + "ref.fit(K_train, Y_train)\n", + "yref = ref.predict(K_test)\n", "\n", "errors = np.concatenate((np.abs(Ypred-Y_test),np.abs(yref-Y_test)))\n", "\n", @@ -1150,8 +1150,7 @@ "metadata": {}, "outputs": [], "source": [ - "#If you want, you can set kernel = 'precomputed' and supply kpca.fit(X) with X = K_train to improve computational efficiency.\n", - "#kernel_params['kernel'] = 'precomputed'\n", + "# We set kernel = 'precomputed' and supply kpca.fit(X) with X = K_train to improve computational efficiency.\n", "kpca = KernelPCA(n_components=2, **kernel_params)" ] }, @@ -1159,7 +1158,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Calling `kpca.fit(X)` will generate the kernel for $\\mathbf{X}$ and compute/internally store the eigenvectors/values." + "Calling `kpca.fit(K_train)` will generate the kernel for $\\mathbf{X}$ and compute/internally store the eigenvectors/values." ] }, { @@ -1168,14 +1167,14 @@ "metadata": {}, "outputs": [], "source": [ - "kpca.fit(X_train)" + "kpca.fit(K_train)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Calling `kpca.transform(X)` will compute and return the KPCA projection $\\mathbf{T}$." + "Calling `kpca.transform(K)` will compute and return the KPCA projection $\\mathbf{T}$." ] }, { @@ -1184,8 +1183,8 @@ "metadata": {}, "outputs": [], "source": [ - "T_KPCA_train = kpca.transform(X_train)\n", - "T_KPCA_test= kpca.transform(X_test)" + "T_KPCA_train = kpca.transform(K_train)\n", + "T_KPCA_test= kpca.transform(K_test)" ] }, { @@ -1249,8 +1248,7 @@ "metadata": {}, "outputs": [], "source": [ - "#If you want, you can set kernel = 'precomputed' and supply krr.fit(X,y) with X = K_train to improve computational efficiency.\n", - "#kernel_params['kernel'] = 'precomputed'\n", + "# We set kernel = 'precomputed' and supply krr.fit(X,y) with X = K_train to improve computational efficiency.\n", "krr = KernelRidge(alpha=regularization, **kernel_params)" ] }, @@ -1267,7 +1265,7 @@ "metadata": {}, "outputs": [], "source": [ - "krr.fit(X_train, Y_train)" + "krr.fit(K_train, Y_train)" ] }, { @@ -1283,8 +1281,8 @@ "metadata": {}, "outputs": [], "source": [ - "Y_krr_train = krr.predict(X_train)\n", - "Y_krr_test = krr.predict(X_test)" + "Y_krr_train = krr.predict(K_train)\n", + "Y_krr_test = krr.predict(K_test)" ] }, { @@ -1361,7 +1359,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Calling `kpcovr.fit(X,Y)` will compute the weights $\\mathbf{P}_{KY}$ and internally store them." + "Calling `kpcovr.fit(X, Y, K)` will compute the weights $\\mathbf{P}_{KY}$ and internally store them." ] }, { @@ -1377,7 +1375,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Calling `kpcovr.transform(X)` will return the latent space transformation $\\mathbf{T}$, the predicted properties, and the reconstructed input data." + "Calling `kpcovr.transform(X, K)` will return the latent space transformation $\\mathbf{T}$, the predicted properties, and the reconstructed input data." ] }, { From b67c014a89bd37df93ca715d3497510810fb16f5 Mon Sep 17 00:00:00 2001 From: PKGuo <56018055+PKGuo@users.noreply.github.com> Date: Fri, 20 Nov 2020 23:10:49 +0100 Subject: [PATCH 5/6] Modified the text --- notebooks/3_KernelMethods.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/3_KernelMethods.ipynb b/notebooks/3_KernelMethods.ipynb index cb71560..dbc2d1c 100644 --- a/notebooks/3_KernelMethods.ipynb +++ b/notebooks/3_KernelMethods.ipynb @@ -86,7 +86,7 @@ " k(\\mathbf{x}, \\mathbf{x}^{\\prime}) = \\exp{\\left(-\\gamma\\lVert \\mathbf{x} - \\mathbf{x}^{\\prime}\\rVert^2\\right)}.\n", "\\end{equation}\n", "\n", - "In this notebook we use the gaussian kernel (```gaussian_kernel(XA, XB, gamma=1.0)```), however, we have also included the linear kernel function (```linear_kernel(XA, XB)```) in ```skcosmo```. You can check rather easily that if you use a linear kernel all of the kernel methods reduce explicitly to linear regression or principal component analysis in the original feature space." + "In this notebook we use the gaussian kernel (```rbf_kernel(XA, XB, gamma=1.0)```), however, we have also included the linear kernel function (```linear_kernel(XA, XB)```) in ```sklearn```. You can check rather easily that if you use a linear kernel all of the kernel methods reduce explicitly to linear regression or principal component analysis in the original feature space." ] }, { From f4ebbb69474a51be160514d2dd5831c4b96b5ca3 Mon Sep 17 00:00:00 2001 From: PKGuo <56018055+PKGuo@users.noreply.github.com> Date: Tue, 24 Nov 2020 10:19:59 +0100 Subject: [PATCH 6/6] Use functools to avoid gamma = 1.0 --- notebooks/3_KernelMethods.ipynb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/notebooks/3_KernelMethods.ipynb b/notebooks/3_KernelMethods.ipynb index dbc2d1c..1c4a2fc 100644 --- a/notebooks/3_KernelMethods.ipynb +++ b/notebooks/3_KernelMethods.ipynb @@ -41,6 +41,7 @@ "from sklearn.decomposition import PCA, KernelPCA\n", "from sklearn.linear_model import Ridge\n", "from sklearn.kernel_ridge import KernelRidge\n", + "from functools import partial\n", "cmaps = get_cmaps()\n", "plt.style.use('../utilities/kernel_pcovr.mplstyle')\n", "dbl_fig=(2*plt.rcParams['figure.figsize'][0], plt.rcParams['figure.figsize'][1])" @@ -99,7 +100,7 @@ "# The function variable designates the kernel function to use throughout the notebook.\n", "# The string variable is used to pass to the utility classes. \n", "kernel_params = {\"kernel\": \"precomputed\"}\n", - "kernel_func = rbf_kernel\n", + "kernel_func = partial(rbf_kernel, gamma=1.0)\n", "kernel_type = 'gaussian'" ] }, @@ -137,7 +138,7 @@ "metadata": {}, "outputs": [], "source": [ - "K_train_raw = kernel_func(X_train, X_train, gamma = 1.0)" + "K_train_raw = kernel_func(X_train, X_train)" ] }, { @@ -253,7 +254,7 @@ "metadata": {}, "outputs": [], "source": [ - "K_test = kernel_func(X_test, X_train, gamma = 1.0)" + "K_test = kernel_func(X_test, X_train)" ] }, { @@ -455,7 +456,7 @@ "source": [ "K_approx_train = np.matmul(T_KPCA_train,T_KPCA_train.T)\n", "\n", - "K_test_test = kernel_func(X_test, X_test,gamma = 1.0)\n", + "K_test_test = kernel_func(X_test, X_test)\n", "K_test_test = center_kernel(K_test_test)\n", "K_approx_test = np.matmul(T_KPCA_test,T_KPCA_test.T)\n", "\n", @@ -1215,7 +1216,7 @@ "Xr_test = np.matmul(T_KPCA_test, PTX)\n", "\n", "K_approx_train = np.matmul(T_KPCA_train,T_KPCA_train.T)\n", - "K_test_test = kernel_func(X_test, X_test,gamma = 1.0)\n", + "K_test_test = kernel_func(X_test, X_test)\n", "K_test_test = center_kernel(K_test_test)\n", "K_approx_test = np.matmul(T_KPCA_test,T_KPCA_test.T)\n", "\n",