diff --git a/Makefile b/Makefile index 03fb97ea..3e13a4eb 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ install-no-extras-for-test: python -m pip install .[test] install-all-extras-for-test: - python -m pip install .[all_extras,test] + python -m pip install .[all_extras,sktime-integration,test] install-editable: pip install -e . diff --git a/pyproject.toml b/pyproject.toml index e9d58347..e1661c10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,9 @@ dependencies = [ sklearn-integration = [ "scikit-learn <1.8.0", ] +sktime-integration = [ + "sktime", +] build = [ "setuptools", "build", diff --git a/src/hyperactive/experiment/integrations/__init__.py b/src/hyperactive/experiment/integrations/__init__.py index 6e14af6a..1b600df2 100644 --- a/src/hyperactive/experiment/integrations/__init__.py +++ b/src/hyperactive/experiment/integrations/__init__.py @@ -2,5 +2,8 @@ # copyright: hyperactive developers, MIT License (see LICENSE file) from hyperactive.experiment.integrations.sklearn_cv import SklearnCvExperiment +from hyperactive.experiment.integrations.sktime_forecasting import ( + SktimeForecastingExperiment, +) -__all__ = ["SklearnCvExperiment"] +__all__ = ["SklearnCvExperiment", "SktimeForecastingExperiment"] diff --git a/src/hyperactive/experiment/integrations/sktime_forecasting.py b/src/hyperactive/experiment/integrations/sktime_forecasting.py new file mode 100644 index 00000000..98c53a44 --- /dev/null +++ b/src/hyperactive/experiment/integrations/sktime_forecasting.py @@ -0,0 +1,304 @@ +"""Experiment adapter for sktime backtesting experiments.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +import numpy as np + +from hyperactive.base import BaseExperiment + + +class SktimeForecastingExperiment(BaseExperiment): + """Experiment adapter for time backtesting experiments. + + This class is used to perform backtesting experiments using a given + sktime forecaster. It allows for hyperparameter tuning and evaluation of + the model's performance. + + The score returned is the summary backtesting score, + of applying ``sktime`` ``evaluate`` to ``estimator`` with the parameters given in + ``score`` ``params``. + + The backtesting performed is specified by the ``cv`` parameter, + and the scoring metric is specified by the ``scoring`` parameter. + The ``X`` and ``y`` parameters are the input data and target values, + which are used in fit/predict cross-validation. + + Parameters + ---------- + forecaster : sktime BaseForecaster descendant (concrete forecaster) + sktime forecaster to benchmark + + cv : sktime BaseSplitter descendant + determines split of ``y`` and possibly ``X`` into test and train folds + y is always split according to ``cv``, see above + if ``cv_X`` is not passed, ``X`` splits are subset to ``loc`` equal to ``y`` + if ``cv_X`` is passed, ``X`` is split according to ``cv_X`` + + y : sktime time series container + Target (endogeneous) time series used in the evaluation experiment + + X : sktime time series container, of same mtype as y + Exogenous time series used in the evaluation experiment + + strategy : {"refit", "update", "no-update_params"}, optional, default="refit" + defines the ingestion mode when the forecaster sees new data when window expands + "refit" = forecaster is refitted to each training window + "update" = forecaster is updated with training window data, in sequence provided + "no-update_params" = fit to first training window, re-used without fit or update + + scoring : subclass of sktime.performance_metrics.BaseMetric, + default=None. Used to get a score function that takes y_pred and y_test + arguments and accept y_train as keyword argument. + If None, then uses scoring = MeanAbsolutePercentageError(symmetric=True). + + error_score : "raise" or numeric, default=np.nan + Value to assign to the score if an exception occurs in estimator fitting. If set + to "raise", the exception is raised. If a numeric value is given, + FitFailedWarning is raised. + + cv_X : sktime BaseSplitter descendant, optional + determines split of ``X`` into test and train folds + default is ``X`` being split to identical ``loc`` indices as ``y`` + if passed, must have same number of splits as ``cv`` + + backend : string, by default "None". + Parallelization backend to use for runs. + Runs parallel evaluate if specified and ``strategy="refit"``. + + - "None": executes loop sequentially, simple list comprehension + - "loky", "multiprocessing" and "threading": uses ``joblib.Parallel`` loops + - "joblib": custom and 3rd party ``joblib`` backends, e.g., ``spark`` + - "dask": uses ``dask``, requires ``dask`` package in environment + - "dask_lazy": same as "dask", + but changes the return to (lazy) ``dask.dataframe.DataFrame``. + - "ray": uses ``ray``, requires ``ray`` package in environment + + Recommendation: Use "dask" or "loky" for parallel evaluate. + "threading" is unlikely to see speed ups due to the GIL and the serialization + backend (``cloudpickle``) for "dask" and "loky" is generally more robust + than the standard ``pickle`` library used in "multiprocessing". + + backend_params : dict, optional + additional parameters passed to the backend as config. + Directly passed to ``utils.parallel.parallelize``. + Valid keys depend on the value of ``backend``: + + - "None": no additional parameters, ``backend_params`` is ignored + - "loky", "multiprocessing" and "threading": default ``joblib`` backends + any valid keys for ``joblib.Parallel`` can be passed here, e.g., ``n_jobs``, + with the exception of ``backend`` which is directly controlled by ``backend``. + If ``n_jobs`` is not passed, it will default to ``-1``, other parameters + will default to ``joblib`` defaults. + - "joblib": custom and 3rd party ``joblib`` backends, e.g., ``spark``. + any valid keys for ``joblib.Parallel`` can be passed here, e.g., ``n_jobs``, + ``backend`` must be passed as a key of ``backend_params`` in this case. + If ``n_jobs`` is not passed, it will default to ``-1``, other parameters + will default to ``joblib`` defaults. + - "dask": any valid keys for ``dask.compute`` can be passed, + e.g., ``scheduler`` + + - "ray": The following keys can be passed: + + - "ray_remote_args": dictionary of valid keys for ``ray.init`` + - "shutdown_ray": bool, default=True; False prevents ``ray`` from shutting + down after parallelization. + - "logger_name": str, default="ray"; name of the logger to use. + - "mute_warnings": bool, default=False; if True, suppresses warnings + + Example + ------- + >>> from hyperactive.experiment.integrations import SktimeForecastingExperiment + >>> from sktime.datasets import load_airline + >>> from sktime.forecasting.naive import NaiveForecaster + >>> from sktime.performance_metrics.forecasting import MeanAbsolutePercentageError + >>> from sktime.split import ExpandingWindowSplitter + >>> + >>> y = load_airline() + >>> + >>> sktime_exp = SktimeForecastingExperiment( + ... forecaster=NaiveForecaster(strategy="last"), + ... scoring=MeanAbsolutePercentageError(), + ... cv=ExpandingWindowSplitter(initial_window=36, step_length=12, fh=12), + ... y=y, + ... ) + >>> params = {"strategy": "mean"} + >>> score, add_info = sktime_exp.score(params) + + For default choices of ``scoring``: + >>> sktime_exp = SktimeForecastingExperiment( + ... forecaster=NaiveForecaster(strategy="last"), + ... cv=ExpandingWindowSplitter(initial_window=36, step_length=12, fh=12), + ... y=y, + ... ) + >>> params = {"strategy": "mean"} + >>> score, add_info = sktime_exp.score(params) + + Quick call without metadata return or dictionary: + >>> score = sktime_exp(strategy="mean") + """ + + _tags = { + "authors": "fkiraly", + "maintainers": "fkiraly", + "python_dependencies": "sktime", # python dependencies + } + + def __init__( + self, + forecaster, + cv, + y, + X=None, + strategy="refit", + scoring=None, + error_score=np.nan, + cv_X=None, + backend=None, + backend_params=None, + ): + self.forecaster = forecaster + self.X = X + self.y = y + self.strategy = strategy + self.scoring = scoring + self.cv = cv + self.error_score = error_score + self.cv_X = cv_X + self.backend = backend + self.backend_params = backend_params + + super().__init__() + + if scoring is None: + from sktime.performance_metrics.forecasting import ( + MeanAbsolutePercentageError, + ) + + self._scoring = MeanAbsolutePercentageError(symmetric=True) + else: + self._scoring = scoring + + if scoring is None or scoring.get_tag("lower_is_better", False): + higher_or_lower_better = "lower" + else: + higher_or_lower_better = "higher" + self.set_tags(**{"property:higher_or_lower_is_better": higher_or_lower_better}) + + def _paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + return list(self.forecaster.get_params().keys()) + + def _evaluate(self, params): + """Evaluate the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to evaluate. + + Returns + ------- + float + The value of the parameters as per evaluation. + dict + Additional metadata about the search. + """ + from sktime.forecasting.model_evaluation import evaluate + + results = evaluate( + self.forecaster, + cv=self.cv, + y=self.y, + X=self.X, + strategy=self.strategy, + scoring=self._scoring, + error_score=self.error_score, + cv_X=self.cv_X, + backend=self.backend, + backend_params=self.backend_params, + ) + + result_name = f"test_{self._scoring.name}" + + res_float = results[result_name].mean() + + return res_float, {"results": results} + + @classmethod + def get_test_params(cls, parameter_set="default"): + """Return testing parameter settings for the skbase object. + + ``get_test_params`` is a unified interface point to store + parameter settings for testing purposes. This function is also + used in ``create_test_instance`` and ``create_test_instances_and_names`` + to construct test instances. + + ``get_test_params`` should return a single ``dict``, or a ``list`` of ``dict``. + + Each ``dict`` is a parameter configuration for testing, + and can be used to construct an "interesting" test instance. + A call to ``cls(**params)`` should + be valid for all dictionaries ``params`` in the return of ``get_test_params``. + + The ``get_test_params`` need not return fixed lists of dictionaries, + it can also return dynamic or stochastic parameter settings. + + Parameters + ---------- + parameter_set : str, default="default" + Name of the set of test parameters to return, for use in tests. If no + special parameters are defined for a value, will return `"default"` set. + + Returns + ------- + params : dict or list of dict, default = {} + Parameters to create testing instances of the class + Each dict are parameters to construct an "interesting" test instance, i.e., + `MyClass(**params)` or `MyClass(**params[i])` creates a valid test instance. + `create_test_instance` uses the first (or only) dictionary in `params` + """ + from sktime.datasets import load_airline, load_longley + from sktime.forecasting.naive import NaiveForecaster + from sktime.split import ExpandingWindowSplitter + + y = load_airline() + params0 = { + "forecaster": NaiveForecaster(strategy="last"), + "cv": ExpandingWindowSplitter(initial_window=36, step_length=12, fh=12), + "y": y, + } + + from sktime.performance_metrics.forecasting import MeanAbsolutePercentageError + + y, X = load_longley() + params1 = { + "forecaster": NaiveForecaster(strategy="last"), + "cv": ExpandingWindowSplitter(initial_window=3, step_length=3, fh=1), + "y": y, + "X": X, + "scoring": MeanAbsolutePercentageError(symmetric=False), + } + + return [params0, params1] + + @classmethod + def _get_score_params(self): + """Return settings for testing score/evaluate functions. Used in tests only. + + Returns a list, the i-th element should be valid arguments for + self.evaluate and self.score, of an instance constructed with + self.get_test_params()[i]. + + Returns + ------- + list of dict + The parameters to be used for scoring. + """ + val0 = {"strategy": "mean"} + val1 = {"strategy": "last"} + return [val0, val1] diff --git a/src/hyperactive/integrations/sktime/__init__.py b/src/hyperactive/integrations/sktime/__init__.py index 3fae604a..a88ca2f0 100644 --- a/src/hyperactive/integrations/sktime/__init__.py +++ b/src/hyperactive/integrations/sktime/__init__.py @@ -1,8 +1,5 @@ -"""Sktime integration package for Hyperactive. +"""Integrations for sktime with Hyperactive.""" -Author: Simon Blanke -Email: simon.blanke@yahoo.com -License: MIT License -""" +from hyperactive.integrations.sktime._forecasting import ForecastingOptCV -from .main import HyperactiveSearchCV as HyperactiveSearchCV +__all__ = ["ForecastingOptCV"] diff --git a/src/hyperactive/integrations/sktime/_forecasting.py b/src/hyperactive/integrations/sktime/_forecasting.py new file mode 100644 index 00000000..50f2649a --- /dev/null +++ b/src/hyperactive/integrations/sktime/_forecasting.py @@ -0,0 +1,359 @@ +# copyright: hyperactive developers, MIT License (see LICENSE file) + +import numpy as np +from skbase.utils.dependencies import _check_soft_dependencies + +if _check_soft_dependencies("sktime", severity="none"): + from sktime.forecasting.base._delegate import _DelegatedForecaster +else: + from skbase.base import BaseEstimator as _DelegatedForecaster + +from hyperactive.experiment.integrations.sktime_forecasting import ( + SktimeForecastingExperiment, +) + + +class ForecastingOptCV(_DelegatedForecaster): + """Tune an sktime forecaster via any optimizer in the hyperactive API. + + Parameters + ---------- + forecaster : sktime forecaster, BaseForecaster instance or interface compatible + The forecaster to tune, must implement the sktime forecaster interface. + + optimizer : hyperactive BaseOptimizer + The optimizer to be used for hyperparameter search. + + cv : sktime BaseSplitter descendant + determines split of ``y`` and possibly ``X`` into test and train folds + y is always split according to ``cv``, see above + if ``cv_X`` is not passed, ``X`` splits are subset to ``loc`` equal to ``y`` + if ``cv_X`` is passed, ``X`` is split according to ``cv_X`` + + strategy : {"refit", "update", "no-update_params"}, optional, default="refit" + defines the ingestion mode when the forecaster sees new data when window expands + "refit" = forecaster is refitted to each training window + "update" = forecaster is updated with training window data, in sequence provided + "no-update_params" = fit to first training window, re-used without fit or update + + update_behaviour : str, optional, default = "full_refit" + one of {"full_refit", "inner_only", "no_update"} + behaviour of the forecaster when calling update + "full_refit" = both tuning parameters and inner estimator refit on all data seen + "inner_only" = tuning parameters are not re-tuned, inner estimator is updated + "no_update" = neither tuning parameters nor inner estimator are updated + + scoring : sktime metric (BaseMetric), str, or callable, optional (default=None) + scoring metric to use in tuning the forecaster + + * sktime metric objects (BaseMetric) descendants can be searched + with the ``registry.all_estimators`` search utility, + for instance via ``all_estimators("metric", as_dataframe=True)`` + + * If callable, must have signature + ``(y_true: 1D np.ndarray, y_pred: 1D np.ndarray) -> float``, + assuming np.ndarrays being of the same length, and lower being better. + Metrics in sktime.performance_metrics.forecasting are all of this form. + + * If str, uses registry.resolve_alias to resolve to one of the above. + Valid strings are valid registry.craft specs, which include + string repr-s of any BaseMetric object, e.g., "MeanSquaredError()"; + and keys of registry.ALIAS_DICT referring to metrics. + + * If None, defaults to MeanAbsolutePercentageError() + + refit : bool, optional (default=True) + True = refit the forecaster with the best parameters on the entire data in fit + False = no refitting takes place. The forecaster cannot be used to predict. + This is to be used to tune the hyperparameters, and then use the estimator + as a parameter estimator, e.g., via get_fitted_params or PluginParamsForecaster. + + error_score : "raise" or numeric, default=np.nan + Value to assign to the score if an exception occurs in estimator fitting. If set + to "raise", the exception is raised. If a numeric value is given, + FitFailedWarning is raised. + + cv_X : sktime BaseSplitter descendant, optional + determines split of ``X`` into test and train folds + default is ``X`` being split to identical ``loc`` indices as ``y`` + if passed, must have same number of splits as ``cv`` + + backend : string, by default "None". + Parallelization backend to use for runs. + Runs parallel evaluate if specified and ``strategy="refit"``. + + - "None": executes loop sequentially, simple list comprehension + - "loky", "multiprocessing" and "threading": uses ``joblib.Parallel`` loops + - "joblib": custom and 3rd party ``joblib`` backends, e.g., ``spark`` + - "dask": uses ``dask``, requires ``dask`` package in environment + - "dask_lazy": same as "dask", + but changes the return to (lazy) ``dask.dataframe.DataFrame``. + - "ray": uses ``ray``, requires ``ray`` package in environment + + Recommendation: Use "dask" or "loky" for parallel evaluate. + "threading" is unlikely to see speed ups due to the GIL and the serialization + backend (``cloudpickle``) for "dask" and "loky" is generally more robust + than the standard ``pickle`` library used in "multiprocessing". + + backend_params : dict, optional + additional parameters passed to the backend as config. + Directly passed to ``utils.parallel.parallelize``. + Valid keys depend on the value of ``backend``: + + - "None": no additional parameters, ``backend_params`` is ignored + - "loky", "multiprocessing" and "threading": default ``joblib`` backends + any valid keys for ``joblib.Parallel`` can be passed here, e.g., ``n_jobs``, + with the exception of ``backend`` which is directly controlled by ``backend``. + If ``n_jobs`` is not passed, it will default to ``-1``, other parameters + will default to ``joblib`` defaults. + - "joblib": custom and 3rd party ``joblib`` backends, e.g., ``spark``. + any valid keys for ``joblib.Parallel`` can be passed here, e.g., ``n_jobs``, + ``backend`` must be passed as a key of ``backend_params`` in this case. + If ``n_jobs`` is not passed, it will default to ``-1``, other parameters + will default to ``joblib`` defaults. + - "dask": any valid keys for ``dask.compute`` can be passed, + e.g., ``scheduler`` + + - "ray": The following keys can be passed: + + - "ray_remote_args": dictionary of valid keys for ``ray.init`` + - "shutdown_ray": bool, default=True; False prevents ``ray`` from shutting + down after parallelization. + - "logger_name": str, default="ray"; name of the logger to use. + - "mute_warnings": bool, default=False; if True, suppresses warnings + + Example + ------- + Tuning sklearn SVC via grid search + + 1. defining the tuned estimator: + >>> from sklearn.svm import SVC + >>> from hyperactive.integrations.sklearn import OptCV + >>> from hyperactive.opt import GridSearchSk as GridSearch + >>> + >>> param_grid = {"kernel": ["linear", "rbf"], "C": [1, 10]} + >>> tuned_svc = OptCV(SVC(), GridSearch(param_grid)) + + 2. fitting the tuned estimator: + >>> from sklearn.datasets import load_iris + >>> from sklearn.model_selection import train_test_split + >>> X, y = load_iris(return_X_y=True) + >>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) + >>> + >>> tuned_svc.fit(X_train, y_train) + OptCV(...) + >>> y_pred = tuned_svc.predict(X_test) + + 3. obtaining best parameters and best estimator + >>> best_params = tuned_svc.best_params_ + >>> best_estimator = tuned_svc.best_estimator_ + """ + + _tags = { + "authors": "fkiraly", + "maintainers": "fkiraly", + "python_dependencies": "sktime", + } + + # attribute for _DelegatedForecaster, which then delegates + # all non-overridden methods are same as of getattr(self, _delegate_name) + # see further details in _DelegatedForecaster docstring + _delegate_name = "best_forecaster_" + + def __init__( + self, + forecaster, + optimizer, + cv, + strategy="refit", + update_behaviour="full_refit", + scoring=None, + refit=True, + error_score=np.nan, + cv_X=None, + backend=None, + backend_params=None, + ): + self.forecaster = forecaster + self.optimizer = optimizer + self.cv = cv + self.strategy = strategy + self.update_behaviour = update_behaviour + self.scoring = scoring + self.refit = refit + self.error_score = error_score + self.cv_X = cv_X + self.backend = backend + self.backend_params = backend_params + super().__init__() + + def _fit(self, y, X, fh): + """Fit to training data. + + Parameters + ---------- + y : pd.Series + Target time series to which to fit the forecaster. + fh : int, list or np.array, optional (default=None) + The forecasters horizon with the steps ahead to to predict. + X : pd.DataFrame, optional (default=None) + Exogenous variables are ignored + + Returns + ------- + self : returns an instance of self. + """ + from sktime.utils.validation.forecasting import check_scoring + + forecaster = self.forecaster.clone() + + scoring = check_scoring(self.scoring, obj=self) + # scoring_name = f"test_{scoring.name}" + + experiment = SktimeForecastingExperiment( + forecaster=forecaster, + scoring=scoring, + cv=self.cv, + X=X, + y=y, + strategy=self.strategy, + error_score=self.error_score, + cv_X=self.cv_X, + backend=self.backend, + backend_params=self.backend_params, + ) + + optimizer = self.optimizer.clone() + optimizer.set_params(experiment=experiment) + best_params = optimizer.run() + + self.best_params_ = best_params + self.best_forecaster_ = forecaster.set_params(**best_params) + + # Refit model with best parameters. + if self.refit: + self.best_forecaster_.fit(y=y, X=X, fh=fh) + + return self + + def _predict(self, fh, X): + """Forecast time series at future horizon. + + private _predict containing the core logic, called from predict + + State required: + Requires state to be "fitted". + + Accesses in self: + Fitted model attributes ending in "_" + self.cutoff + + Parameters + ---------- + fh : guaranteed to be ForecastingHorizon or None, optional (default=None) + The forecasting horizon with the steps ahead to to predict. + If not passed in _fit, guaranteed to be passed here + X : pd.DataFrame, optional (default=None) + Exogenous time series + + Returns + ------- + y_pred : pd.Series + Point predictions + """ + if not self.refit: + raise RuntimeError( + f"In {self.__class__.__name__}, refit must be True to make predictions," + f" but found refit=False. If refit=False, {self.__class__.__name__} can" + " be used only to tune hyper-parameters, as a parameter estimator." + ) + return super()._predict(fh=fh, X=X) + + def _update(self, y, X=None, update_params=True): + """Update time series to incremental training data. + + Parameters + ---------- + y : guaranteed to be of a type in self.get_tag("y_inner_mtype") + Time series with which to update the forecaster. + if self.get_tag("scitype:y")=="univariate": + guaranteed to have a single column/variable + if self.get_tag("scitype:y")=="multivariate": + guaranteed to have 2 or more columns + if self.get_tag("scitype:y")=="both": no restrictions apply + X : optional (default=None) + guaranteed to be of a type in self.get_tag("X_inner_mtype") + Exogeneous time series for the forecast + update_params : bool, optional (default=True) + whether model parameters should be updated + + Returns + ------- + self : reference to self + """ + update_behaviour = self.update_behaviour + + if update_behaviour == "full_refit": + super()._update(y=y, X=X, update_params=update_params) + elif update_behaviour == "inner_only": + self.best_forecaster_.update(y=y, X=X, update_params=update_params) + elif update_behaviour == "no_update": + self.best_forecaster_.update(y=y, X=X, update_params=False) + else: + raise ValueError( + 'update_behaviour must be one of "full_refit", "inner_only",' + f' or "no_update", but found {update_behaviour}' + ) + return self + + @classmethod + def get_test_params(cls, parameter_set="default"): + """Return testing parameter settings for the estimator. + + Parameters + ---------- + parameter_set : str, default="default" + Name of the set of test parameters to return, for use in tests. If no + special parameters are defined for a value, will return ``"default"`` set. + + Returns + ------- + params : dict or list of dict + """ + from sktime.forecasting.naive import NaiveForecaster + from sktime.forecasting.trend import PolynomialTrendForecaster + from sktime.performance_metrics.forecasting import ( + MeanAbsolutePercentageError, + mean_absolute_percentage_error, + ) + from sktime.split import SingleWindowSplitter + + from hyperactive.opt.gfo import HillClimbing + from hyperactive.opt.gridsearch import GridSearchSk + from hyperactive.opt.random_search import RandomSearchSk + + params_gridsearch = { + "forecaster": NaiveForecaster(strategy="mean"), + "cv": SingleWindowSplitter(fh=1), + "optimizer": GridSearchSk(param_grid={"window_length": [2, 5]}), + "scoring": MeanAbsolutePercentageError(symmetric=True), + } + params_randomsearch = { + "forecaster": PolynomialTrendForecaster(), + "cv": SingleWindowSplitter(fh=1), + "optimizer": RandomSearchSk(param_distributions={"degree": [1, 2]}), + "scoring": mean_absolute_percentage_error, + "update_behaviour": "inner_only", + } + params_hillclimb = { + "forecaster": NaiveForecaster(strategy="mean"), + "cv": SingleWindowSplitter(fh=1), + "optimizer": HillClimbing( + search_space={"window_length": [2, 5]}, + n_iter=10, + n_neighbours=5, + ), + "scoring": "MeanAbsolutePercentageError(symmetric=True)", + "update_behaviour": "no_update", + } + return [params_gridsearch, params_randomsearch, params_hillclimb] diff --git a/src/hyperactive/integrations/sktime/main.py b/src/hyperactive/integrations/sktime/main.py deleted file mode 100644 index 991f16de..00000000 --- a/src/hyperactive/integrations/sktime/main.py +++ /dev/null @@ -1,11 +0,0 @@ -"""main module for Hyperactive optimization.""" - -# Email: simon.blanke@yahoo.com -# License: MIT License - - -class HyperactiveSearchCV: - """HyperactiveSearchCV class.""" - - def __init__(self) -> None: - pass diff --git a/src/hyperactive/integrations/sktime/tests/__init__.py b/src/hyperactive/integrations/sktime/tests/__init__.py new file mode 100644 index 00000000..e78b4da3 --- /dev/null +++ b/src/hyperactive/integrations/sktime/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for integrations for sktime.""" diff --git a/src/hyperactive/integrations/sktime/tests/test_sktime_estimators.py b/src/hyperactive/integrations/sktime/tests/test_sktime_estimators.py new file mode 100644 index 00000000..1ac4bd77 --- /dev/null +++ b/src/hyperactive/integrations/sktime/tests/test_sktime_estimators.py @@ -0,0 +1,22 @@ +"""Integration tests for sktime tuners.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +import pytest +from skbase.utils.dependencies import _check_soft_dependencies + +if _check_soft_dependencies("sktime", severity="none"): + from hyperactive.integrations.sktime import ForecastingOptCV + + EST_TO_TEST = [ForecastingOptCV] +else: + EST_TO_TEST = [] + + +@pytest.mark.parametrize("estimator", EST_TO_TEST) +def test_sktime_estimator(estimator): + """Test sktime estimator via check_estimator.""" + from sktime.utils.estimator_checks import check_estimator + + check_estimator(estimator, raise_exceptions=True) + # The above line collects all API conformance tests in sktime and runs them. + # It will raise an error if the estimator is not API conformant. diff --git a/src/hyperactive/tests/test_all_objects.py b/src/hyperactive/tests/test_all_objects.py index dda0890e..76fd9d37 100644 --- a/src/hyperactive/tests/test_all_objects.py +++ b/src/hyperactive/tests/test_all_objects.py @@ -120,11 +120,14 @@ def _all_objects(self): if isclass(filter): obj_list = [obj for obj in obj_list if issubclass(obj, filter)] - # run_test_for_class selects the estimators to run - # based on whether they have changed, and whether they have all dependencies - # internally, uses the ONLY_CHANGED_MODULES flag, - # and checks the python env against python_dependencies tag - # obj_list = [obj for obj in obj_list if run_test_for_class(obj)] + # only run tests if all soft dependencies are present + def softdeps_present(obj): + """Check if the object has all dependencies present.""" + from skbase.utils.dependencies import _check_estimator_deps + + return _check_estimator_deps(obj, severity="none") + + obj_list = [obj for obj in obj_list if softdeps_present(obj)] return obj_list @@ -135,10 +138,35 @@ def _all_objects(self): class TestAllObjects(BaseFixtureGenerator, _TestAllObjects): """Generic tests for all objects in the package.""" + OBJECT_TYPES_IN_HYPERACTIVE = [ + "experiment", + "optimizer", + ] + def test_doctest_examples(self, object_class): """Runs doctests for estimator class.""" run_doctest(object_class, name=f"class {object_class.__name__}") + def test_valid_object_class_tags(self, object_class): + """Check that object class tags are in self.valid_tags.""" + # stepout for estimators with base classes in other packages + # e.g., sktime BaseForecaster, BaseClassifier, used in hyperactive.integrations + cls_type = object_class.get_class_tag("object_type", None) + if cls_type not in self.OBJECT_TYPES_IN_HYPERACTIVE: + return None + + super().test_valid_object_class_tags(object_class) + + def test_valid_object_tags(self, object_instance): + """Check that object tags are in self.valid_tags.""" + # stepout for estimators with base classes in other packages + # e.g., sktime BaseForecaster, BaseClassifier, used in hyperactive.integrations + obj_type = object_instance.get_tag("object_type", None) + if obj_type not in self.OBJECT_TYPES_IN_HYPERACTIVE: + return None + + super().test_valid_object_class_tags(object_instance) + class ExperimentFixtureGenerator(BaseFixtureGenerator): """Fixture generator for experiments.