[BUG] `TSCOptCV` fails with `HillClimbing` optimizer

[fkiraly]

TSCOptCV fails with the HillClimbing optimizer.

To reproduce currently, check_estimator can be called:

from hyperactive.integrations.sktime import TSCOptCV
from sktime.utils import check_estimator

check_estimator(TSCOptCV, raise_exceptions=True)

There is likely a more minimal example, traceback here

File C:\Workspace\sktime\sktime\classification\base.py:264, in BaseClassifier.fit(self, X, y)
    261 X = self._convert_X(X, X_mtype)
    263 # pass coerced and checked data to inner _fit
--> [264](file:///C:/Workspace/sktime/sktime/classification/base.py:264) self._fit(X, y)
    265 self.fit_time_ = int(round(time.time() * 1000)) - start
    267 # this should happen last: fitted state is set to True

File C:\Workspace\Hyperactive\src\hyperactive\integrations\sktime\_classification.py:253, in TSCOptCV._fit(self, X, y)
    251 optimizer = self.optimizer.clone()
    252 optimizer.set_params(experiment=experiment)
--> [253](file:///C:/Workspace/Hyperactive/src/hyperactive/integrations/sktime/_classification.py:253) best_params = optimizer.solve()
    255 self.best_params_ = best_params
    256 self.best_estimator_ = estimator.set_params(**best_params)

File C:\Workspace\Hyperactive\src\hyperactive\base\_optimizer.py:82, in BaseOptimizer.solve(self)
     79 experiment = self.get_experiment()
     80 search_config = self.get_search_config()
---> [82](file:///C:/Workspace/Hyperactive/src/hyperactive/base/_optimizer.py:82) best_params = self._solve(experiment, **search_config)
     83 self.best_params_ = best_params
     84 return best_params

File C:\Workspace\Hyperactive\src\hyperactive\opt\_adapters\_gfo.py:137, in _BaseGFOadapter._solve(self, experiment, **search_config)
    134 gfopt = gfo_cls(**search_config)
    136 with StdoutMute(active=not self.verbose):
--> [137](file:///C:/Workspace/Hyperactive/src/hyperactive/opt/_adapters/_gfo.py:137)     gfopt.search(
    138         objective_function=experiment.score,
    139         n_iter=n_iter,
    140         max_time=max_time,
    141     )
    142 best_params = gfopt.best_para
    143 return best_params

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\search.py:95, in Search.search(self, objective_function, n_iter, max_time, max_score, early_stopping, memory, memory_warm_start, verbosity)
     83 self.init_search(
     84     objective_function,
     85     n_iter,
   (...)     91     verbosity,
     92 )
     94 for nth_trial in range(n_iter):
---> [95](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/search.py:95)     self.search_step(nth_trial)
     96     if self.stop.check():
     97         break

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\search.py:183, in Search.search_step(self, nth_iter)
    180 self.nth_iter = nth_iter
    182 if self.nth_iter < self.n_inits_norm:
--> [183](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/search.py:183)     self._initialization()
    185 if self.nth_iter == self.n_init_search:
    186     self.finish_initialization()

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\_times_tracker.py:27, in TimesTracker.iter_time.<locals>.wrapper(self, *args, **kwargs)
     25 def wrapper(self, *args, **kwargs):
     26     t = time.time()
---> [27](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/_times_tracker.py:27)     res = func(self, *args, **kwargs)
     28     self.iter_times.append(time.time() - t)
     29     return res

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\search.py:40, in Search._initialization(self)
     36 self.best_score = self.p_bar.score_best
     38 init_pos = self.init_pos()
---> [40](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/search.py:40) score_new = self._score(init_pos)
     41 self.evaluate_init(score_new)
     43 self.pos_l.append(init_pos)

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\_times_tracker.py:18, in TimesTracker.eval_time.<locals>.wrapper(self, *args, **kwargs)
     16 def wrapper(self, *args, **kwargs):
     17     t = time.time()
---> [18](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/_times_tracker.py:18)     res = func(self, *args, **kwargs)
     19     self.eval_times.append(time.time() - t)
     20     return res

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\search.py:32, in Search._score(self, pos)
     30 @TimesTracker.eval_time
     31 def _score(self, pos):
---> [32](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/search.py:32)     return self.score(pos)

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\_results_manager.py:33, in ResultsManager.score.<locals>._wrapper(pos)
     31 value = self.conv.position2value(pos)
     32 para = self.conv.value2para(value)
---> [33](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/_results_manager.py:33) results_dict = self._obj_func_results(objective_function, para)
     35 self.results_list.append({**results_dict, **para})
     37 return results_dict["score"]

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\_results_manager.py:16, in ResultsManager._obj_func_results(self, objective_function, para)
     15 def _obj_func_results(self, objective_function, para):
---> [16](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/_results_manager.py:16)     results = objective_function(para)
     18     if isinstance(results, tuple):
     19         score = results[0]

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\_memory.py:50, in Memory.memory.<locals>.wrapper(para)
     48 def wrapper(para: List[float]) -> float:
     49     value = self.conv.para2value(para)
---> [50](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/_memory.py:50)     position = self.conv.value2position(value)
     51     pos_tuple = tuple(position)
     53     if pos_tuple in self.memory_dict:

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\optimizers\core_optimizer\converter.py:72, in Converter.returnNoneIfArgNone.<locals>.wrapper(self, *args)
     70         return None
     71 else:
---> [72](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/optimizers/core_optimizer/converter.py:72)     return func_(self, *args)

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\optimizers\core_optimizer\converter.py:89, in Converter.value2position(self, value)
     87 position = []
     88 for n, space_dim in enumerate(self.search_space_values):
---> [89](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/optimizers/core_optimizer/converter.py:89)     pos = np.abs(value[n] - np.array(space_dim)).argmin()
     90     position.append(int(pos))
     92 return np.array(position)

UFuncTypeError: ufunc 'subtract' did not contain a loop with signature matching types (dtype('<U13'), dtype('<U13')) -> None

[SimonBlanke]

I am not sure what kind of search-space or objective-function is passed to the GFO backend in "TSCOptCV", but from the error message it looks like there are categorical values in the search-space (like strings). But GFO cannot handle categorical values, yet. We abandoned native support for categorical values in the search-space in Hyperactive in v5.0.0 and decided to "outsource" it to GFO in the future.

[SimonBlanke]

I get a very different error when executing the script (which is likely fixed by #190):

FAILED: scoring value <function accuracy_score at 0x70af13831260> looks like it is a metric function rather than a scorer. A scorer should require an estimator as its first parameter. Please use `make_scorer` to convert a metric to a scorer.
Traceback (most recent call last):
  File "/home/me/workspace/open-source/hyper_stack/Hyperactive/tsc_opt_cv_error.py", line 4, in <module>
    check_estimator(TSCOptCV, raise_exceptions=True)
  File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sktime/utils/estimator_checks.py", line 116, in check_estimator
    test_cls_results = test_cls().run_tests(
                       ^^^^^^^^^^^^^^^^^^^^^
  File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sktime/tests/test_all_estimators.py", line 639, in run_tests
    raise err
  File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sktime/tests/test_all_estimators.py", line 629, in run_tests
    test_fun(**deepcopy(args))
  File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sktime/tests/test_all_estimators.py", line 1329, in test_fit_does_not_overwrite_hyper_params
    fitted_est = scenario.run(estimator_instance, method_sequence=["fit"])
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sktime/utils/_testing/scenarios.py", line 180, in run
    res = getattr(obj, methodname)(**args)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sktime/classification/base.py", line 273, in fit
    self._fit(X, y)
  File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/hyperactive/integrations/sktime/_classification.py", line 242, in _fit
    scoring = check_scoring(DummyClassifier(), self.scoring)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sklearn/utils/_param_validation.py", line 218, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sklearn/metrics/_scorer.py", line 1029, in check_scoring
    raise ValueError(
ValueError: scoring value <function accuracy_score at 0x70af13831260> looks like it is a metric function rather than a scorer. A scorer should require an estimator as its first parameter. Please use `make_scorer` to convert a metric to a scorer.

[fkiraly]

the reported error is after the fix applied in #190 which is now merged.

I have verified that a categorical search space was passed, which is probably the reason?

[fkiraly]

so, not a bug?

[SimonBlanke]

so, not a bug?

Does not look like it to me.