diff --git a/.gitignore b/.gitignore index 5bc0c596..bd8695dd 100644 Binary files a/.gitignore and b/.gitignore differ diff --git a/pyproject.toml b/pyproject.toml index af3dd76a..53a20e88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ test_parallel_backends = [ all_extras = [ "hyperactive[integrations]", "optuna<5", + "lightning", ] diff --git a/src/hyperactive/experiment/integrations/__init__.py b/src/hyperactive/experiment/integrations/__init__.py index 06c4c584..5b92eae9 100644 --- a/src/hyperactive/experiment/integrations/__init__.py +++ b/src/hyperactive/experiment/integrations/__init__.py @@ -1,4 +1,5 @@ """Integrations with packages for tuning.""" + # copyright: hyperactive developers, MIT License (see LICENSE file) from hyperactive.experiment.integrations.sklearn_cv import SklearnCvExperiment @@ -11,10 +12,14 @@ from hyperactive.experiment.integrations.sktime_forecasting import ( SktimeForecastingExperiment, ) +from hyperactive.experiment.integrations.torch_lightning_experiment import ( + TorchExperiment, +) __all__ = [ "SklearnCvExperiment", "SkproProbaRegExperiment", "SktimeClassificationExperiment", "SktimeForecastingExperiment", + "TorchExperiment", ] diff --git a/src/hyperactive/experiment/integrations/logs/lightning_logs/version_0/hparams.yaml b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_0/hparams.yaml new file mode 100644 index 00000000..0b42a18c --- /dev/null +++ b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_0/hparams.yaml @@ -0,0 +1,3 @@ +hidden_dim: 16 +input_dim: 10 +lr: 0.001 diff --git a/src/hyperactive/experiment/integrations/logs/lightning_logs/version_0/metrics.csv b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_0/metrics.csv new file mode 100644 index 00000000..0b8964a1 --- /dev/null +++ b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_0/metrics.csv @@ -0,0 +1,3 @@ +epoch,step,val_loss +0,4,0.6708395481109619 +1,9,0.6716761589050293 diff --git a/src/hyperactive/experiment/integrations/logs/lightning_logs/version_1/hparams.yaml b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_1/hparams.yaml new file mode 100644 index 00000000..0b42a18c --- /dev/null +++ b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_1/hparams.yaml @@ -0,0 +1,3 @@ +hidden_dim: 16 +input_dim: 10 +lr: 0.001 diff --git a/src/hyperactive/experiment/integrations/logs/lightning_logs/version_1/metrics.csv b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_1/metrics.csv new file mode 100644 index 00000000..155fff50 --- /dev/null +++ b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_1/metrics.csv @@ -0,0 +1,3 @@ +epoch,step,val_loss +0,4,0.6967241168022156 +1,9,0.6974167823791504 diff --git a/src/hyperactive/experiment/integrations/logs/lightning_logs/version_2/hparams.yaml b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_2/hparams.yaml new file mode 100644 index 00000000..0b42a18c --- /dev/null +++ b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_2/hparams.yaml @@ -0,0 +1,3 @@ +hidden_dim: 16 +input_dim: 10 +lr: 0.001 diff --git a/src/hyperactive/experiment/integrations/logs/lightning_logs/version_2/metrics.csv b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_2/metrics.csv new file mode 100644 index 00000000..a7d4ad3d --- /dev/null +++ b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_2/metrics.csv @@ -0,0 +1,3 @@ +epoch,step,val_loss +0,4,0.7234764099121094 +1,9,0.7209647297859192 diff --git a/src/hyperactive/experiment/integrations/logs/lightning_logs/version_3/hparams.yaml b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_3/hparams.yaml new file mode 100644 index 00000000..0b42a18c --- /dev/null +++ b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_3/hparams.yaml @@ -0,0 +1,3 @@ +hidden_dim: 16 +input_dim: 10 +lr: 0.001 diff --git a/src/hyperactive/experiment/integrations/logs/lightning_logs/version_3/metrics.csv b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_3/metrics.csv new file mode 100644 index 00000000..7d0390aa --- /dev/null +++ b/src/hyperactive/experiment/integrations/logs/lightning_logs/version_3/metrics.csv @@ -0,0 +1,3 @@ +epoch,step,val_loss +0,4,0.7359145283699036 +1,9,0.7361313700675964 diff --git a/src/hyperactive/experiment/integrations/sklearn_cv.py b/src/hyperactive/experiment/integrations/sklearn_cv.py index 2ecc6c6d..fa366de8 100644 --- a/src/hyperactive/experiment/integrations/sklearn_cv.py +++ b/src/hyperactive/experiment/integrations/sklearn_cv.py @@ -2,6 +2,9 @@ # copyright: hyperactive developers, MIT License (see LICENSE file) +from hyperactive.base import BaseExperiment +from hyperactive.experiment.integrations._skl_metrics import \ + _coerce_to_scorer_and_sign from sklearn import clone from sklearn.model_selection import cross_validate from sklearn.utils.validation import _num_samples @@ -10,7 +13,6 @@ from hyperactive.experiment.integrations._skl_cv import _coerce_cv from hyperactive.experiment.integrations._skl_metrics import _coerce_to_scorer_and_sign - class SklearnCvExperiment(BaseExperiment): """Experiment adapter for sklearn cross-validation experiments. diff --git a/src/hyperactive/experiment/integrations/sktime_classification.py b/src/hyperactive/experiment/integrations/sktime_classification.py index d2ca27dc..314fed76 100644 --- a/src/hyperactive/experiment/integrations/sktime_classification.py +++ b/src/hyperactive/experiment/integrations/sktime_classification.py @@ -3,9 +3,9 @@ # copyright: hyperactive developers, MIT License (see LICENSE file) import numpy as np - from hyperactive.base import BaseExperiment -from hyperactive.experiment.integrations._skl_metrics import _coerce_to_scorer_and_sign +from hyperactive.experiment.integrations._skl_metrics import \ + _coerce_to_scorer_and_sign class SktimeClassificationExperiment(BaseExperiment): @@ -222,7 +222,8 @@ def _evaluate(self, params): metric_func = getattr(self._scoring, "_metric_func", None) if metric_func is None: # very defensive fallback (should not happen due to _coerce_to_scorer) - from sklearn.metrics import accuracy_score as metric_func # type: ignore + from sklearn.metrics import \ + accuracy_score as metric_func # type: ignore results = evaluate( estimator, diff --git a/src/hyperactive/experiment/integrations/sktime_forecasting.py b/src/hyperactive/experiment/integrations/sktime_forecasting.py index f335f6b2..c2002575 100644 --- a/src/hyperactive/experiment/integrations/sktime_forecasting.py +++ b/src/hyperactive/experiment/integrations/sktime_forecasting.py @@ -1,8 +1,8 @@ """Experiment adapter for sktime backtesting experiments.""" + # copyright: hyperactive developers, MIT License (see LICENSE file) import numpy as np - from hyperactive.base import BaseExperiment @@ -169,9 +169,8 @@ def __init__( super().__init__() if scoring is None: - from sktime.performance_metrics.forecasting import ( - MeanAbsolutePercentageError, - ) + from sktime.performance_metrics.forecasting import \ + MeanAbsolutePercentageError self._scoring = MeanAbsolutePercentageError(symmetric=True) else: @@ -275,7 +274,8 @@ def get_test_params(cls, parameter_set="default"): "y": y, } - from sktime.performance_metrics.forecasting import MeanAbsolutePercentageError + from sktime.performance_metrics.forecasting import \ + MeanAbsolutePercentageError y, X = load_longley() params1 = { diff --git a/src/hyperactive/experiment/integrations/torch_lightning_experiment.py b/src/hyperactive/experiment/integrations/torch_lightning_experiment.py new file mode 100644 index 00000000..e8cebb4c --- /dev/null +++ b/src/hyperactive/experiment/integrations/torch_lightning_experiment.py @@ -0,0 +1,299 @@ +"""Experiment adapter for PyTorch Lightning experiments.""" + +# copyright: hyperactive developers, MIT License (see LICENSE file) + +__author__ = ["amitsubhashchejara"] + +import numpy as np +from hyperactive.base import BaseExperiment + + +class TorchExperiment(BaseExperiment): + """Experiment adapter for PyTorch Lightning experiments. + + This class is used to perform experiments using PyTorch Lightning modules. + It allows for hyperparameter tuning and evaluation of the model's performance + using specified metrics. + + The experiment trains a Lightning module with given hyperparameters and returns + the validation metric value for optimization. + + Parameters + ---------- + datamodule : L.LightningDataModule + A PyTorch Lightning DataModule that handles data loading and preparation. + lightning_module : type + A PyTorch Lightning Module class (not an instance) that will be instantiated + with hyperparameters during optimization. + trainer_kwargs : dict, optional (default=None) + A dictionary of keyword arguments to pass to the PyTorch Lightning Trainer. + objective_metric : str, optional (default='val_loss') + The metric used to evaluate the model's performance. This should correspond + to a metric logged in the LightningModule during validation. + + Examples + -------- + >>> from hyperactive.experiment.integrations import TorchExperiment + >>> import torch + >>> import lightning as L + >>> from torch import nn + >>> from torch.utils.data import DataLoader + >>> + >>> # Define a simple Lightning Module + >>> class SimpleLightningModule(L.LightningModule): + ... def __init__(self, input_dim=10, hidden_dim=16, lr=1e-3): + ... super().__init__() + ... self.save_hyperparameters() + ... self.model = nn.Sequential( + ... nn.Linear(input_dim, hidden_dim), + ... nn.ReLU(), + ... nn.Linear(hidden_dim, 2) + ... ) + ... self.lr = lr + ... + ... def forward(self, x): + ... return self.model(x) + ... + ... def training_step(self, batch, batch_idx): + ... x, y = batch + ... y_hat = self(x) + ... loss = nn.functional.cross_entropy(y_hat, y) + ... self.log("train_loss", loss) + ... return loss + ... + ... def validation_step(self, batch, batch_idx): + ... x, y = batch + ... y_hat = self(x) + ... val_loss = nn.functional.cross_entropy(y_hat, y) + ... self.log("val_loss", val_loss, on_epoch=True) + ... return val_loss + ... + ... def configure_optimizers(self): + ... return torch.optim.Adam(self.parameters(), lr=self.lr) + >>> + >>> # Create DataModule + >>> class RandomDataModule(L.LightningDataModule): + ... def __init__(self, batch_size=32): + ... super().__init__() + ... self.batch_size = batch_size + ... + ... def setup(self, stage=None): + ... dataset = torch.utils.data.TensorDataset( + ... torch.randn(100, 10), + ... torch.randint(0, 2, (100,)) + ... ) + ... self.train, self.val = torch.utils.data.random_split( + ... dataset, [80, 20] + ... ) + ... + ... def train_dataloader(self): + ... return DataLoader(self.train, batch_size=self.batch_size) + ... + ... def val_dataloader(self): + ... return DataLoader(self.val, batch_size=self.batch_size) + >>> + >>> datamodule = RandomDataModule(batch_size=16) + >>> datamodule.setup() + >>> + >>> # Create Experiment + >>> experiment = TorchExperiment( + ... datamodule=datamodule, + ... lightning_module=SimpleLightningModule, + ... trainer_kwargs={'max_epochs': 3}, + ... objective_metric="val_loss" + ... ) + >>> + >>> params = {"input_dim": 10, "hidden_dim": 16, "lr": 1e-3} + >>> + >>> val_result, metadata = experiment._evaluate(params) + """ + + _tags = { + "property:randomness": "random", + "property:higher_or_lower_is_better": "lower", + "authors": ["amitsubhashchejara"], + "python_dependencies": ["torch", "lightning"], + } + + def __init__( + self, + datamodule, + lightning_module, + trainer_kwargs=None, + objective_metric: str = "val_loss", + ): + + self.datamodule = datamodule + self.lightning_module = lightning_module + self.trainer_kwargs = trainer_kwargs or {} + self.objective_metric = objective_metric + + super().__init__() + + self._trainer_kwargs = { + "max_epochs": 10, + "enable_checkpointing": False, + "logger": False, + "enable_progress_bar": False, + "enable_model_summary": False, + } + if trainer_kwargs is not None: + self._trainer_kwargs.update(trainer_kwargs) + + def _paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str, or None + The parameter names of the search parameters. + If not known or arbitrary, return None. + """ + import inspect + + sig = inspect.signature(self.lightning_module.__init__) + return [p for p in sig.parameters.keys() if p != "self"] + + def _evaluate(self, params): + """Evaluate the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to evaluate. + + Returns + ------- + float + The value of the parameters as per evaluation. + dict + Additional metadata about the search. + """ + import lightning as L + + try: + model = self.lightning_module(**params) + trainer = L.Trainer(**self._trainer_kwargs) + trainer.fit(model, self.datamodule) + + val_result = trainer.callback_metrics.get(self.objective_metric) + metadata = {} + + if val_result is None: + available_metrics = list(trainer.callback_metrics.keys()) + raise ValueError( + f"Metric '{self.objective_metric}' not found. " + f"Available: {available_metrics}" + ) + if hasattr(val_result, "item"): + val_result = np.float64(val_result.detach().cpu().item()) + elif isinstance(val_result, (int, float)): + val_result = np.float64(val_result) + else: + val_result = np.float64(float(val_result)) + + return val_result, metadata + + except Exception as e: + print(f"Training failed with params {params}: {e}") + return np.float64(float("inf")), {} + + @classmethod + def get_test_params(cls, parameter_set="default"): + """Return testing parameter settings for the estimator. + + Parameters + ---------- + parameter_set : str, default="default" + Name of the set of test parameters to return, for use in tests. + + Returns + ------- + params : dict or list of dict, default = {} + Parameters to create testing instances of the class. + """ + import lightning as L + import torch + from torch import nn + from torch.utils.data import DataLoader + + class SimpleLightningModule(L.LightningModule): + def __init__(self, input_dim=10, hidden_dim=16, lr=1e-3): + super().__init__() + self.save_hyperparameters() + self.model = nn.Sequential( + nn.Linear(input_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, 2), + ) + self.lr = lr + + def forward(self, x): + return self.model(x) + + def training_step(self, batch, batch_idx): + x, y = batch + y_hat = self(x) + loss = nn.functional.cross_entropy(y_hat, y) + self.log("train_loss", loss) + return loss + + def validation_step(self, batch, batch_idx): + x, y = batch + y_hat = self(x) + val_loss = nn.functional.cross_entropy(y_hat, y) + self.log("val_loss", val_loss, on_epoch=True) + return val_loss + + def configure_optimizers(self): + return torch.optim.Adam(self.parameters(), lr=self.lr) + + class RandomDataModule(L.LightningDataModule): + def __init__(self, batch_size=32): + super().__init__() + self.batch_size = batch_size + + def setup(self, stage=None): + dataset = torch.utils.data.TensorDataset( + torch.randn(100, 10), torch.randint(0, 2, (100,)) + ) + self.train, self.val = torch.utils.data.random_split(dataset, [80, 20]) + + def train_dataloader(self): + return DataLoader(self.train, batch_size=self.batch_size) + + def val_dataloader(self): + return DataLoader(self.val, batch_size=self.batch_size) + + datamodule = RandomDataModule(batch_size=16) + + params = { + "datamodule": datamodule, + "lightning_module": SimpleLightningModule, + "trainer_kwargs": { + "max_epochs": 1, + "enable_progress_bar": False, + "enable_model_summary": False, + "logger": False, + }, + "objective_metric": "val_loss", + } + + return [params] + + @classmethod + def _get_score_params(cls): + """Return settings for testing score/evaluate functions. + + Returns a list, the i-th element should be valid arguments for + self.evaluate and self.score, of an instance constructed with + self.get_test_params()[i]. + + Returns + ------- + list of dict + The parameters to be used for scoring. + """ + score_params1 = {"input_dim": 10, "hidden_dim": 20, "lr": 0.001} + score_params2 = {"input_dim": 10, "hidden_dim": 16, "lr": 0.01} + return [score_params1, score_params2]