diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 238a8055..da52e7e4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,7 +83,7 @@ jobs: - name: Show dependencies run: python -m pip list - - name: Test with pytest + - name: Test core package run: | python -m pytest src/hyperactive -p no:warnings @@ -117,7 +117,7 @@ jobs: - name: Show dependencies run: python -m pip list - - name: Test with pytest + - name: Test core package run: | python -m pytest src/hyperactive -p no:warnings @@ -150,3 +150,54 @@ jobs: - name: Run sklearn integration tests for ${{ matrix.sklearn-version }} run: | python -m pytest -x -p no:warnings src/hyperactive/integrations/sklearn/ + + test-examples: + name: test-examples + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Check for example changes + id: check-examples + run: | + if [ "${{ github.event_name }}" == "push" ]; then + # For pushes, compare with previous commit + CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD | grep "^examples/" || true) + else + # For pull requests, compare with base branch + CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep "^examples/" || true) + fi + + if [ -n "$CHANGED_FILES" ]; then + echo "examples_changed=true" >> $GITHUB_OUTPUT + echo "Examples changed:" + echo "$CHANGED_FILES" + else + echo "examples_changed=false" >> $GITHUB_OUTPUT + echo "No example files changed" + fi + + - name: Install dependencies + if: steps.check-examples.outputs.examples_changed == 'true' + run: | + python -m pip install --upgrade pip + python -m pip install build + make install-all-extras-for-test + + - name: Show dependencies + if: steps.check-examples.outputs.examples_changed == 'true' + run: python -m pip list + + - name: Test examples + if: steps.check-examples.outputs.examples_changed == 'true' + run: | + python -m pytest examples/ -v -p no:warnings diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 00000000..3d0064d7 --- /dev/null +++ b/examples/__init__.py @@ -0,0 +1 @@ +"""Test package for Hyperactive library functionality.""" diff --git a/examples/gfo/README.md b/examples/gfo/README.md new file mode 100644 index 00000000..6afea084 --- /dev/null +++ b/examples/gfo/README.md @@ -0,0 +1,147 @@ +# GFO (Gradient-Free Optimization) Examples + +This directory contains examples demonstrating the GFO optimization backend in Hyperactive. GFO provides a collection of gradient-free optimization algorithms that don't require derivative information, making them suitable for black-box optimization problems like hyperparameter tuning. + +## Quick Start + +Run any example directly: +```bash +python random_search_example.py +python hill_climbing_example.py +python bayesian_optimization_example.py +# ... etc +``` + +## Available Algorithms + +| Algorithm | Type | Best For | Characteristics | +|-----------|------|----------|-----------------| +| [RandomSearch](random_search_example.py) | Random | Baselines, high-dim spaces | Simple, parallel-friendly | +| [GridSearch](grid_search_example.py) | Exhaustive | Small discrete spaces | Complete coverage | +| [HillClimbing](hill_climbing_example.py) | Local Search | Fast convergence | Exploits local structure | +| [StochasticHillClimbing](stochastic_hill_climbing_example.py) | Local Search | Plateau handling | Probabilistic moves | +| [RepulsingHillClimbing](repulsing_hill_climbing_example.py) | Local Search | Multi-optima discovery | Memory-based repulsion | +| [RandomRestartHillClimbing](random_restart_hill_climbing_example.py) | Multi-start | Multi-modal problems | Multiple hill climbs | +| [SimulatedAnnealing](simulated_annealing_example.py) | Probabilistic | Escaping local optima | Temperature cooling | +| [ParallelTempering](parallel_tempering_example.py) | MCMC | Complex landscapes | Multi-temperature chains | +| [PatternSearch](pattern_search_example.py) | Direct Search | Noisy functions | Systematic patterns | +| [PowellsMethod](powells_method_example.py) | Coordinate Descent | Smooth functions | Conjugate directions | +| [DownhillSimplexOptimizer](downhill_simplex_example.py) | Simplex | Low-dimensional | Nelder-Mead method | +| [DirectAlgorithm](direct_algorithm_example.py) | Global | Lipschitz functions | Space division | +| [LipschitzOptimizer](lipschitz_optimizer_example.py) | Global | Smooth functions | Theoretical guarantees | +| [SpiralOptimization](spiral_optimization_example.py) | Nature-inspired | Mixed problems | Spiral search patterns | +| [ParticleSwarmOptimizer](particle_swarm_example.py) | Swarm Intelligence | Continuous problems | Population-based | +| [GeneticAlgorithm](genetic_algorithm_example.py) | Evolutionary | Multi-modal problems | Selection/crossover/mutation | +| [EvolutionStrategy](evolution_strategy_example.py) | Evolutionary | Self-adaptive | Strategy parameter evolution | +| [DifferentialEvolution](differential_evolution_example.py) | Evolutionary | Continuous problems | Vector differences | +| [BayesianOptimizer](bayesian_optimization_example.py) | Probabilistic | Expensive evaluations | Gaussian Process | +| [TreeStructuredParzenEstimators](tree_structured_parzen_estimators_example.py) | Bayesian | Mixed parameters | TPE algorithm | +| [ForestOptimizer](forest_optimizer_example.py) | Surrogate-based | Non-linear problems | Random Forest surrogates | + +## Algorithm Categories + +### Simple Algorithms +- **RandomSearch**: Pure random sampling - excellent baseline +- **GridSearch**: Exhaustive enumeration of discrete grids + +### Local Search Methods +- **HillClimbing**: Greedy local search from starting points +- **SimulatedAnnealing**: Hill climbing with probabilistic escapes +- **StochasticHillClimbing**: Hill climbing with random moves + +### Population-Based Methods +- **ParticleSwarmOptimizer**: Swarm intelligence optimization +- **GeneticAlgorithm**: Evolutionary computation approach +- **DifferentialEvolution**: Evolution strategy for continuous spaces + +### Advanced Methods +- **BayesianOptimizer**: Gaussian Process-based optimization +- **TreeStructuredParzenEstimators**: TPE algorithm (similar to Optuna's TPE) +- **ForestOptimizer**: Random forest-based surrogate optimization + +## Choosing the Right Algorithm + +### Quick Decision Tree + +1. **Need a baseline?** → RandomSearch +2. **Small discrete space?** → GridSearch +3. **Expensive evaluations?** → BayesianOptimizer +4. **Continuous optimization?** → ParticleSwarmOptimizer or BayesianOptimizer +5. **Fast local optimization?** → HillClimbing +6. **General purpose?** → RandomSearch or BayesianOptimizer + +### Problem Characteristics + +**Function Properties:** +- Smooth, continuous → BayesianOptimizer, ParticleSwarmOptimizer +- Noisy, discontinuous → RandomSearch, GeneticAlgorithm +- Multi-modal → ParticleSwarmOptimizer, GeneticAlgorithm + +**Search Space:** +- Low dimensional (< 10) → GridSearch, BayesianOptimizer +- High dimensional (> 20) → RandomSearch, GeneticAlgorithm +- Mixed types → RandomSearch, TreeStructuredParzenEstimators + +**Computational Budget:** +- Low budget (< 50 trials) → RandomSearch, HillClimbing +- Medium budget (50-200) → BayesianOptimizer, ParticleSwarmOptimizer +- High budget (200+) → GridSearch, GeneticAlgorithm + +## Common Configuration + +All GFO algorithms support: +- **Random state**: `random_state=42` for reproducibility +- **Early stopping**: `early_stopping=10` trials without improvement +- **Max score**: `max_score=0.99` stop when target reached +- **Warm start**: `initialize={"warm_start": [points]}` initial solutions + +## Advanced Usage + +### Sequential Optimization +```python +# Phase 1: Global exploration +random_opt = RandomSearch(n_trials=20, ...) +initial_results = random_opt.solve() + +# Phase 2: Local refinement +hill_opt = HillClimbing( + n_trials=30, + initialize={"warm_start": [initial_results]} +) +final_results = hill_opt.solve() +``` + +### Parameter Space Design + +**Continuous Spaces:** +```python +param_space = { + "learning_rate": (0.001, 0.1), # Log scale recommended + "n_estimators": (10, 1000), # Integer range + "regularization": (0.0, 1.0), # Bounded continuous +} +``` + +**Discrete/Categorical:** +```python +param_space = { + "algorithm": ["adam", "sgd", "rmsprop"], # Categorical + "layers": [1, 2, 3, 4, 5], # Discrete integers + "activation": ["relu", "tanh", "sigmoid"], # Categorical +} +``` + +## Performance Tips + +1. **Start simple**: RandomSearch is often surprisingly competitive +2. **Use warm starts**: Provide good initial points when available +3. **Set early stopping**: Avoid wasted evaluations on poor runs +4. **Match algorithm to problem**: Consider function smoothness and dimensionality +5. **Reproducibility**: Always set `random_state` for consistent results + +## Further Reading + +- [Gradient-Free Optimization Overview](https://en.wikipedia.org/wiki/Derivative-free_optimization) +- [Bayesian Optimization Tutorial](https://arxiv.org/abs/1807.02811) +- [Evolutionary Algorithms Survey](https://ieeexplore.ieee.org/document/6900297) +- [Hyperparameter Optimization Review](https://arxiv.org/abs/1502.02127) diff --git a/examples/gfo/bayesian_optimization_example.py b/examples/gfo/bayesian_optimization_example.py new file mode 100644 index 00000000..49af9252 --- /dev/null +++ b/examples/gfo/bayesian_optimization_example.py @@ -0,0 +1,75 @@ +""" +Bayesian Optimization Example - Gaussian Process-based Optimization + +Bayesian optimization uses a probabilistic model (typically Gaussian Process) to +model the objective function and an acquisition function to decide where to sample +next. This approach is highly sample-efficient and particularly useful when +function evaluations are expensive. + +Characteristics: +- Sample-efficient optimization +- Uses Gaussian Process to model objective function +- Balances exploration vs exploitation via acquisition functions +- Excellent for expensive function evaluations +- Handles uncertainty quantification naturally +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import BayesianOptimizer + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space - discrete values for Bayesian optimization +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values (step 10) + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Bayesian Optimization +# Provide some initial good points to help the GP model initialization +warm_start_points = [ + { + "n_estimators": 100, + "max_depth": 10, + "min_samples_split": 5, + "min_samples_leaf": 2, + }, + { + "n_estimators": 50, + "max_depth": 15, + "min_samples_split": 3, + "min_samples_leaf": 1, + }, +] + +optimizer = BayesianOptimizer( + search_space=search_space, + n_iter=15, + random_state=42, + initialize={"warm_start": warm_start_points}, + experiment=experiment, +) + +# Run optimization +# Bayesian optimization builds a GP model of the objective function +# and uses acquisition functions (like Expected Improvement) to select +# the most promising points to evaluate next +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Bayesian optimization completed successfully") diff --git a/examples/gfo/differential_evolution_example.py b/examples/gfo/differential_evolution_example.py new file mode 100644 index 00000000..283df6df --- /dev/null +++ b/examples/gfo/differential_evolution_example.py @@ -0,0 +1,59 @@ +""" +Differential Evolution Example - Vector-Based Evolutionary Algorithm + +Differential Evolution is a population-based evolutionary algorithm that uses +difference vectors between population members to generate new candidate solutions. +It's particularly effective for continuous optimization problems and has robust +convergence properties across various problem types. + +Characteristics: +- Population-based with vector difference operations +- Self-adapting through mutation and crossover +- Robust convergence across different problem types +- Effective for continuous and mixed optimization +- Simple parameter control with good default behavior +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import DifferentialEvolution + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 5)), # Discrete integer values (broader range) + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Differential Evolution +optimizer = DifferentialEvolution( + search_space=search_space, + n_iter=50, + random_state=42, + experiment=experiment +) + +# Run optimization +# Differential evolution uses difference vectors between population members +# For each target vector, it creates a mutant vector using weighted differences +# from other population members, then applies crossover to generate trial vector +# The trial vector replaces the target if it has better fitness +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Differential evolution optimization completed successfully") diff --git a/examples/gfo/direct_algorithm_example.py b/examples/gfo/direct_algorithm_example.py new file mode 100644 index 00000000..a7587414 --- /dev/null +++ b/examples/gfo/direct_algorithm_example.py @@ -0,0 +1,56 @@ +""" +DIRECT Algorithm Example - Dividing Rectangles Global Optimization + +DIRECT (DIviding RECTangles) is a global optimization algorithm that systematically +divides the search space into rectangles and evaluates their potential for containing +the global optimum. It balances global search with local refinement by focusing +computational effort on the most promising regions. + +Characteristics: +- Global optimization through systematic space division +- Balances exploration of large regions with exploitation of promising areas +- Lipschitz constant estimation for convergence guarantees +- No user-specified parameters required +- Effective for global optimization of continuous functions +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import DirectAlgorithm + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space - DIRECT works well with moderate-sized discrete spaces +search_space = { + "n_estimators": list(range(10, 201, 20)), # Discrete integer values (reduced) + "max_depth": list(range(1, 21, 2)), # Discrete integer values (reduced) + "min_samples_split": list(range(2, 21, 2)), # Discrete integer values (reduced) + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure DIRECT Algorithm +optimizer = DirectAlgorithm( + search_space=search_space, n_iter=15, random_state=42, experiment=experiment +) + +# Run optimization +# DIRECT divides the search space into hyperrectangles and maintains +# a list of potentially optimal rectangles. It systematically subdivides +# the most promising rectangles, balancing global exploration with +# local exploitation based on function values and rectangle sizes +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("DIRECT algorithm optimization completed successfully") diff --git a/examples/gfo/downhill_simplex_example.py b/examples/gfo/downhill_simplex_example.py new file mode 100644 index 00000000..6b5632ee --- /dev/null +++ b/examples/gfo/downhill_simplex_example.py @@ -0,0 +1,64 @@ +""" +Downhill Simplex Example - Nelder-Mead Optimization + +Downhill Simplex (Nelder-Mead) maintains a simplex of n+1 points in n-dimensional +space and iteratively transforms this simplex through reflection, expansion, +contraction, and shrinkage operations. It's a robust derivative-free method +that's particularly effective for low-dimensional continuous optimization. + +Characteristics: +- Maintains a simplex of n+1 points in n-dimensional space +- Uses geometric transformations (reflect, expand, contract, shrink) +- Derivative-free optimization method +- Good convergence properties for smooth functions +- Effective for low to moderate dimensional problems +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import DownhillSimplexOptimizer + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 15)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Downhill Simplex Optimizer +warm_start_points = [ + {"n_estimators": 100, "max_depth": 10, "min_samples_split": 5, "min_samples_leaf": 2} +] + +optimizer = DownhillSimplexOptimizer( + search_space=search_space, + n_iter=35, + random_state=42, + initialize={"warm_start": warm_start_points}, + experiment=experiment +) + +# Run optimization +# Downhill simplex maintains a simplex (triangle in 2D, tetrahedron in 3D, etc.) +# and applies geometric transformations: reflection (flip worst point), +# expansion (extend good moves), contraction (reduce bad moves), +# and shrinkage (reduce entire simplex) to iteratively improve the simplex +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Downhill simplex optimization completed successfully") diff --git a/examples/gfo/evolution_strategy_example.py b/examples/gfo/evolution_strategy_example.py new file mode 100644 index 00000000..d26ef2a6 --- /dev/null +++ b/examples/gfo/evolution_strategy_example.py @@ -0,0 +1,59 @@ +""" +Evolution Strategy Example - Self-Adaptive Evolutionary Algorithm + +Evolution Strategy is a class of evolutionary algorithms that evolves both the +solution candidates and their associated strategy parameters (like mutation +strengths). This self-adaptive mechanism makes it particularly effective for +continuous optimization without requiring manual parameter tuning. + +Characteristics: +- Self-adaptive mutation parameters +- Emphasis on mutation over crossover +- Excellent for continuous optimization problems +- Automatic adaptation to problem landscape +- Robust performance across different problem types +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import EvolutionStrategy + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Evolution Strategy +optimizer = EvolutionStrategy( + search_space=search_space, + n_iter=45, + random_state=42, + experiment=experiment +) + +# Run optimization +# Evolution strategy maintains a population where both solutions and strategy +# parameters (mutation strengths) evolve together. This self-adaptation allows +# the algorithm to automatically adjust its search behavior to match the +# problem characteristics without manual parameter tuning +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Evolution strategy optimization completed successfully") diff --git a/examples/gfo/forest_optimizer_example.py b/examples/gfo/forest_optimizer_example.py new file mode 100644 index 00000000..dab05a75 --- /dev/null +++ b/examples/gfo/forest_optimizer_example.py @@ -0,0 +1,64 @@ +""" +Forest Optimizer Example - Random Forest-Based Surrogate Optimization + +Forest Optimizer uses Random Forest models as surrogate functions to approximate +the expensive objective function. It leverages the Random Forest's ability to +capture non-linear relationships and provide uncertainty estimates to guide +the search toward promising regions of the parameter space. + +Characteristics: +- Random Forest surrogate model for objective function approximation +- Uncertainty-aware search through forest prediction variance +- Good for capturing non-linear parameter relationships +- Robust to noise and handles mixed parameter types well +- Efficient for moderately expensive function evaluations +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import ForestOptimizer + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Forest Optimizer +warm_start_points = [ + {"n_estimators": 80, "max_depth": 8, "min_samples_split": 5, "min_samples_leaf": 3} +] + +optimizer = ForestOptimizer( + search_space=search_space, + n_iter=15, + random_state=42, + initialize={"warm_start": warm_start_points}, + experiment=experiment, +) + +# Run optimization +# Forest optimizer builds Random Forest surrogate models from observed data +# It uses the forest predictions to estimate objective values and uncertainties +# New points are selected based on acquisition functions that balance +# predicted performance with prediction uncertainty (exploration vs exploitation) +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Forest optimization completed successfully") diff --git a/examples/gfo/genetic_algorithm_example.py b/examples/gfo/genetic_algorithm_example.py new file mode 100644 index 00000000..3e968cc2 --- /dev/null +++ b/examples/gfo/genetic_algorithm_example.py @@ -0,0 +1,60 @@ +""" +Genetic Algorithm Example - Evolutionary Computation + +Genetic Algorithm is inspired by biological evolution, maintaining a population +of candidate solutions that evolve over generations through selection, crossover, +and mutation operations. It explores multiple regions simultaneously and can +handle complex, multi-modal optimization landscapes effectively. + +Characteristics: +- Population-based evolutionary approach +- Uses crossover (recombination) and mutation operators +- Selection pressure drives evolution toward better solutions +- Good for complex, multi-modal optimization problems +- Naturally parallel and robust to noise +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import GeneticAlgorithm + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Genetic Algorithm +# Population-based approach works well without specific warm start points +optimizer = GeneticAlgorithm( + search_space=search_space, + n_iter=50, + random_state=42, + experiment=experiment +) + +# Run optimization +# Genetic algorithm maintains a population of candidate solutions +# Each generation applies selection, crossover, and mutation operations +# Fitter individuals have higher probability of reproduction +# Over time, the population evolves toward better solutions +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Genetic algorithm optimization completed successfully") diff --git a/examples/gfo/grid_search_example.py b/examples/gfo/grid_search_example.py new file mode 100644 index 00000000..ba7ecfbc --- /dev/null +++ b/examples/gfo/grid_search_example.py @@ -0,0 +1,65 @@ +""" +Grid Search Example - Exhaustive Parameter Space Search + +Grid search systematically evaluates all possible combinations of parameters +from a predefined discrete grid. While computationally expensive, it guarantees +finding the optimal solution within the defined grid and provides comprehensive +coverage of the parameter space. + +Characteristics: +- Exhaustive search over discrete parameter grid +- Guaranteed to find optimal solution in the grid +- Systematic and reproducible results +- Computationally expensive for large grids +- Best for small discrete parameter spaces +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import GridSearch + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define discrete grid - grid search requires explicit discrete values +search_space = { + "n_estimators": [10, 50, 100, 200], # Discrete values only + "max_depth": [5, 10, 15, 20], # Discrete depth values + "min_samples_split": [2, 5, 10], # Small discrete set + "min_samples_leaf": [1, 2, 5], # Discrete leaf values +} + +# Calculate total combinations +total_combinations = 1 +for param_values in search_space.values(): + total_combinations *= len(param_values) + +print(f"Total parameter combinations: {total_combinations}") + +# Configure Grid Search optimizer +optimizer = GridSearch( + search_space=search_space, + n_iter=total_combinations, # Evaluate all combinations + random_state=42, + experiment=experiment +) + +# Run optimization +# Grid search will systematically evaluate every possible combination +# of parameters from the defined grid, ensuring complete coverage +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print(f"Evaluated {total_combinations} parameter combinations") +print("Grid search completed successfully") diff --git a/examples/gfo/hill_climbing_example.py b/examples/gfo/hill_climbing_example.py new file mode 100644 index 00000000..5c8c54dc --- /dev/null +++ b/examples/gfo/hill_climbing_example.py @@ -0,0 +1,63 @@ +""" +Hill Climbing Example - Local Search Optimization + +Hill climbing is a local search algorithm that starts from a random point and +iteratively moves to neighboring solutions with better objective values. It's +a simple but effective optimization strategy that can quickly find good local +optima, especially when started from reasonable initial points. + +Characteristics: +- Local search strategy (moves to better neighbors) +- Fast convergence to local optima +- Simple and interpretable algorithm +- Good for continuous optimization problems +- Can get stuck in local maxima +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import HillClimbing + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space - discrete values for hill climbing +search_space = { + "n_estimators": list(range(10, 201)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure HillClimbing optimizer +# Starting from a reasonable initial point can improve performance +warm_start_points = [ + {"n_estimators": 100, "max_depth": 10, "min_samples_split": 5, "min_samples_leaf": 2} +] + +optimizer = HillClimbing( + search_space=search_space, + n_iter=40, + random_state=42, + initialize={"warm_start": warm_start_points}, + experiment=experiment +) + +# Run optimization +# Hill climbing explores the neighborhood of current best solution +# and moves to better solutions when found, creating a local search pattern +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Hill climbing optimization completed successfully") diff --git a/examples/gfo/lipschitz_optimizer_example.py b/examples/gfo/lipschitz_optimizer_example.py new file mode 100644 index 00000000..9b0e3274 --- /dev/null +++ b/examples/gfo/lipschitz_optimizer_example.py @@ -0,0 +1,56 @@ +""" +Lipschitz Optimizer Example - Lipschitz Constant-Based Global Optimization + +Lipschitz Optimization assumes the objective function satisfies a Lipschitz +condition (bounded rate of change) and uses this information to eliminate +regions that cannot contain the global optimum. This provides theoretical +convergence guarantees and efficient global optimization. + +Characteristics: +- Uses Lipschitz constant to bound function behavior +- Theoretical convergence guarantees to global optimum +- Efficient elimination of non-promising regions +- No local optima trapping due to global search strategy +- Particularly effective for functions with known smoothness properties +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import LipschitzOptimizer + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 15)), # Discrete integer values + "max_depth": list(range(1, 21, 2)), # Discrete integer values + "min_samples_split": list(range(2, 21, 2)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Lipschitz Optimizer +optimizer = LipschitzOptimizer( + search_space=search_space, n_iter=15, random_state=42, experiment=experiment +) + +# Run optimization +# Lipschitz optimization estimates the Lipschitz constant and uses it to +# construct lower bounds on the objective function. It eliminates regions +# where the lower bound exceeds the current best value, focusing search +# on regions that could potentially contain the global optimum +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Lipschitz optimization completed successfully") diff --git a/examples/gfo/parallel_tempering_example.py b/examples/gfo/parallel_tempering_example.py new file mode 100644 index 00000000..4865568f --- /dev/null +++ b/examples/gfo/parallel_tempering_example.py @@ -0,0 +1,60 @@ +""" +Parallel Tempering Example - Multi-Temperature Markov Chain Monte Carlo + +Parallel Tempering runs multiple Markov chains at different temperatures +simultaneously, allowing high-temperature chains to explore globally while +low-temperature chains exploit locally. Chains can exchange information, +combining global exploration with local refinement in a principled way. + +Characteristics: +- Multiple parallel chains at different temperatures +- Information exchange between chains (replica exchange) +- High-temperature chains provide global exploration +- Low-temperature chains provide local exploitation +- Effective for complex multi-modal optimization landscapes +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import ParallelTempering + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Parallel Tempering +optimizer = ParallelTempering( + search_space=search_space, + n_iter=45, + random_state=42, + experiment=experiment +) + +# Run optimization +# Parallel tempering maintains multiple chains at different temperatures +# High-temperature chains can easily accept worse moves (exploration) +# Low-temperature chains focus on local improvements (exploitation) +# Chains periodically attempt to exchange states based on temperature differences +# This provides both global search capability and local refinement +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Parallel tempering optimization completed successfully") diff --git a/examples/gfo/particle_swarm_example.py b/examples/gfo/particle_swarm_example.py new file mode 100644 index 00000000..c3b1ec0c --- /dev/null +++ b/examples/gfo/particle_swarm_example.py @@ -0,0 +1,60 @@ +""" +Particle Swarm Optimization Example - Swarm Intelligence + +Particle Swarm Optimization (PSO) is inspired by social behavior of bird flocking +or fish schooling. It maintains a population of candidate solutions (particles) +that move through the search space influenced by their own best position and +the global best position found by the swarm. + +Characteristics: +- Population-based metaheuristic +- Inspired by swarm intelligence in nature +- Good balance of exploration and exploitation +- Effective for continuous optimization problems +- Particles share information about promising regions +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import ParticleSwarmOptimizer + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space - discrete values for PSO +search_space = { + "n_estimators": list(range(10, 201, 5)), # Discrete integer values (step 5) + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Particle Swarm Optimization +# The swarm will explore the space collectively, sharing information +# about promising regions through particle communication +optimizer = ParticleSwarmOptimizer( + search_space=search_space, + n_iter=50, + random_state=42, + experiment=experiment +) + +# Run optimization +# PSO maintains a swarm of particles that move through the search space +# Each particle remembers its personal best and is influenced by the global best +# This creates a balance between individual exploration and collective exploitation +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Particle swarm optimization completed successfully") diff --git a/examples/gfo/pattern_search_example.py b/examples/gfo/pattern_search_example.py new file mode 100644 index 00000000..234b368e --- /dev/null +++ b/examples/gfo/pattern_search_example.py @@ -0,0 +1,64 @@ +""" +Pattern Search Example - Direct Search with Systematic Patterns + +Pattern Search is a direct search method that explores the search space using +a systematic pattern of points around the current best solution. It's particularly +robust for noisy functions and doesn't require gradient information, making it +suitable for black-box optimization problems. + +Characteristics: +- Systematic pattern-based exploration around current solution +- Robust to function noise and discontinuities +- Derivative-free direct search method +- Adaptive step size based on search success +- Good for black-box optimization problems +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import PatternSearch + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Pattern Search +warm_start_points = [ + {"n_estimators": 80, "max_depth": 8, "min_samples_split": 8, "min_samples_leaf": 3} +] + +optimizer = PatternSearch( + search_space=search_space, + n_iter=35, + random_state=42, + initialize={"warm_start": warm_start_points}, + experiment=experiment +) + +# Run optimization +# Pattern search systematically evaluates points in a pattern around +# the current best solution. If a better point is found, it becomes the new +# center and the pattern is applied again. If no improvement is found, +# the step size is reduced and the search continues with finer resolution +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Pattern search optimization completed successfully") diff --git a/examples/gfo/powells_method_example.py b/examples/gfo/powells_method_example.py new file mode 100644 index 00000000..ffa2a066 --- /dev/null +++ b/examples/gfo/powells_method_example.py @@ -0,0 +1,64 @@ +""" +Powell's Method Example - Coordinate Descent Optimization + +Powell's Method is a derivative-free optimization algorithm that performs +sequential line searches along conjugate directions. It's particularly effective +for smooth, continuous functions and can achieve quadratic convergence on +quadratic functions without requiring gradient information. + +Characteristics: +- Sequential line searches along conjugate directions +- Derivative-free optimization method +- Good convergence properties for smooth functions +- Builds conjugate directions automatically +- Effective for continuous optimization problems +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import PowellsMethod + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Powell's Method +warm_start_points = [ + {"n_estimators": 100, "max_depth": 10, "min_samples_split": 5, "min_samples_leaf": 2} +] + +optimizer = PowellsMethod( + search_space=search_space, + n_iter=35, + random_state=42, + initialize={"warm_start": warm_start_points}, + experiment=experiment +) + +# Run optimization +# Powell's method performs line searches along coordinate directions initially, +# then constructs conjugate directions from successful search directions +# Each iteration consists of n line searches (where n is the dimension) +# followed by an additional line search along the overall displacement vector +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Powell's method optimization completed successfully") diff --git a/examples/gfo/random_restart_hill_climbing_example.py b/examples/gfo/random_restart_hill_climbing_example.py new file mode 100644 index 00000000..3713b28a --- /dev/null +++ b/examples/gfo/random_restart_hill_climbing_example.py @@ -0,0 +1,59 @@ +""" +Random Restart Hill Climbing Example - Multi-Start Local Search + +Random Restart Hill Climbing performs multiple hill climbing runs from different +random starting points. This strategy overcomes the main limitation of single-point +hill climbing by exploring multiple regions of the search space, significantly +increasing the chances of finding the global optimum. + +Characteristics: +- Multiple independent hill climbing runs +- Each restart explores a different region of search space +- Combines local search efficiency with global exploration +- Simple but effective approach for multi-modal problems +- Performance scales with number of restarts and problem difficulty +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import RandomRestartHillClimbing + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Random Restart Hill Climbing +optimizer = RandomRestartHillClimbing( + search_space=search_space, + n_iter=40, + random_state=42, + experiment=experiment +) + +# Run optimization +# Random restart hill climbing performs multiple hill climbing runs +# Each run starts from a different random point in the search space +# The algorithm keeps track of the best solution found across all runs +# This approach combines the efficiency of local search with better global coverage +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Random restart hill climbing optimization completed successfully") diff --git a/examples/gfo/random_search_example.py b/examples/gfo/random_search_example.py new file mode 100644 index 00000000..eb67c654 --- /dev/null +++ b/examples/gfo/random_search_example.py @@ -0,0 +1,57 @@ +""" +Random Search Example - Pure Random Sampling + +Random search is one of the simplest optimization algorithms that samples parameters +randomly from the defined search space. Despite its simplicity, it's surprisingly +effective for many hyperparameter optimization tasks and serves as an excellent +baseline for comparison with more sophisticated algorithms. + +Characteristics: +- No learning or adaptation between trials +- Uniform sampling from the search space +- Excellent baseline for comparison +- Works well in high-dimensional spaces +- Parallel-friendly (no dependencies between trials) +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import RandomSearch + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values (reduced for speed) + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure RandomSearch optimizer +optimizer = RandomSearch( + search_space=search_space, + n_iter=30, + random_state=42, + experiment=experiment +) + +# Run optimization +# Random search samples each parameter independently and uniformly +# from its defined range or choices, making it simple but effective +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Optimization completed successfully") diff --git a/examples/gfo/repulsing_hill_climbing_example.py b/examples/gfo/repulsing_hill_climbing_example.py new file mode 100644 index 00000000..eef33612 --- /dev/null +++ b/examples/gfo/repulsing_hill_climbing_example.py @@ -0,0 +1,65 @@ +""" +Repulsing Hill Climbing Example - Multi-Point Local Search with Memory + +Repulsing Hill Climbing extends hill climbing by maintaining memory of previously +visited good solutions and using this information to repel the search away from +already explored regions. This helps discover multiple local optima and provides +better coverage of the search space than traditional hill climbing. + +Characteristics: +- Memory-based search with repulsion mechanism +- Discovers multiple local optima through exploration +- Avoids re-exploring already visited good regions +- Better search space coverage than standard hill climbing +- Effective for problems with multiple attractive regions +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import RepulsingHillClimbing + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Repulsing Hill Climbing +# Multiple warm start points can help demonstrate the repulsion mechanism +warm_start_points = [ + {"n_estimators": 50, "max_depth": 5, "min_samples_split": 10, "min_samples_leaf": 5} +] + +optimizer = RepulsingHillClimbing( + search_space=search_space, + n_iter=40, + random_state=42, + initialize={"warm_start": warm_start_points}, + experiment=experiment +) + +# Run optimization +# Repulsing hill climbing maintains a memory of good solutions found +# and uses repulsion forces to push the search away from these areas +# This encourages exploration of new regions and discovery of multiple optima +# The repulsion mechanism helps avoid cycling between the same local optima +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Repulsing hill climbing optimization completed successfully") diff --git a/examples/gfo/simulated_annealing_example.py b/examples/gfo/simulated_annealing_example.py new file mode 100644 index 00000000..abc0a96f --- /dev/null +++ b/examples/gfo/simulated_annealing_example.py @@ -0,0 +1,64 @@ +""" +Simulated Annealing Example - Probabilistic Hill Climbing + +Simulated Annealing is inspired by the metallurgical process of annealing, where +metals are heated and slowly cooled to reduce defects. This algorithm starts with +high "temperature" allowing random moves, then gradually cools down to focus on +local improvements. The cooling schedule allows escaping local optima early on. + +Characteristics: +- Probabilistic acceptance of worse solutions (temperature-dependent) +- Gradually reduces exploration over time (cooling schedule) +- Can escape local optima unlike pure hill climbing +- Good balance between exploration and exploitation +- Single-point search with memory of current solution +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import SimulatedAnnealing + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Simulated Annealing +# Start with good initial point to demonstrate cooling behavior +warm_start_points = [ + {"n_estimators": 100, "max_depth": 10, "min_samples_split": 5, "min_samples_leaf": 2} +] + +optimizer = SimulatedAnnealing( + search_space=search_space, + n_iter=40, + random_state=42, + initialize={"warm_start": warm_start_points}, + experiment=experiment +) + +# Run optimization +# Simulated annealing uses a cooling schedule to gradually reduce temperature +# Early iterations: high probability of accepting worse solutions (exploration) +# Later iterations: low probability of accepting worse solutions (exploitation) +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Simulated annealing optimization completed successfully") diff --git a/examples/gfo/spiral_optimization_example.py b/examples/gfo/spiral_optimization_example.py new file mode 100644 index 00000000..6bdcf4c5 --- /dev/null +++ b/examples/gfo/spiral_optimization_example.py @@ -0,0 +1,64 @@ +""" +Spiral Optimization Example - Nature-Inspired Spiral Search + +Spiral Optimization is inspired by natural phenomena like spiral galaxies and +logarithmic spirals found in nature. It uses spiral-shaped search patterns +to systematically explore the search space, combining global exploration +with local exploitation through adaptive spiral parameters. + +Characteristics: +- Nature-inspired spiral search patterns +- Adaptive spiral parameters for exploration/exploitation balance +- Systematic coverage of search space through spiral trajectories +- Good for continuous and mixed optimization problems +- Unique geometric approach to search space exploration +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import SpiralOptimization + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Spiral Optimization +warm_start_points = [ + {"n_estimators": 100, "max_depth": 10, "min_samples_split": 5, "min_samples_leaf": 3} +] + +optimizer = SpiralOptimization( + search_space=search_space, + n_iter=40, + random_state=42, + initialize={"warm_start": warm_start_points}, + experiment=experiment +) + +# Run optimization +# Spiral optimization uses logarithmic or Archimedean spiral patterns +# to explore the search space. The spiral parameters (radius, angle, center) +# adapt based on search progress, allowing both global exploration through +# wide spirals and local exploitation through tight spirals around promising regions +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Spiral optimization completed successfully") diff --git a/examples/gfo/stochastic_hill_climbing_example.py b/examples/gfo/stochastic_hill_climbing_example.py new file mode 100644 index 00000000..c6ee85da --- /dev/null +++ b/examples/gfo/stochastic_hill_climbing_example.py @@ -0,0 +1,64 @@ +""" +Stochastic Hill Climbing Example - Randomized Local Search + +Stochastic Hill Climbing extends basic hill climbing by introducing randomness +in the neighbor selection process. Instead of always choosing the best neighbor, +it probabilistically selects from improving neighbors, which helps avoid getting +stuck in plateaus and provides better exploration of the search space. + +Characteristics: +- Probabilistic neighbor selection (not always greedy) +- Better plateau handling than deterministic hill climbing +- Maintains local search efficiency with improved exploration +- Good balance between exploitation and local exploration +- Simpler than simulated annealing but more flexible than pure hill climbing +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import StochasticHillClimbing + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Stochastic Hill Climbing +warm_start_points = [ + {"n_estimators": 80, "max_depth": 8, "min_samples_split": 5, "min_samples_leaf": 3} +] + +optimizer = StochasticHillClimbing( + search_space=search_space, + n_iter=40, + random_state=42, + initialize={"warm_start": warm_start_points}, + experiment=experiment +) + +# Run optimization +# Stochastic hill climbing uses randomness in neighbor selection +# Instead of always picking the best neighbor, it probabilistically +# selects from improving neighbors, helping to navigate plateaus +# and avoid getting trapped in shallow local optima +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Stochastic hill climbing optimization completed successfully") diff --git a/examples/gfo/tree_structured_parzen_estimators_example.py b/examples/gfo/tree_structured_parzen_estimators_example.py new file mode 100644 index 00000000..a9c81fd9 --- /dev/null +++ b/examples/gfo/tree_structured_parzen_estimators_example.py @@ -0,0 +1,69 @@ +""" +Tree-structured Parzen Estimators Example - Bayesian Optimization with TPE + +Tree-structured Parzen Estimators (TPE) is a Bayesian optimization algorithm +that models the distribution of good and bad parameter configurations separately. +It uses these models to suggest new configurations that maximize expected +improvement, making it highly efficient for expensive function evaluations. + +Characteristics: +- Bayesian optimization with probabilistic surrogate models +- Separate modeling of good and bad parameter regions +- Expected improvement acquisition function +- Efficient for expensive function evaluations +- Handles mixed parameter types (continuous, discrete, categorical) +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt.gfo import TreeStructuredParzenEstimators + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space +search_space = { + "n_estimators": list(range(10, 201, 10)), # Discrete integer values + "max_depth": list(range(1, 21)), # Discrete integer values + "min_samples_split": list(range(2, 21)), # Discrete integer values + "min_samples_leaf": list(range(1, 11)), # Discrete integer values +} + +# Configure Tree-structured Parzen Estimators +warm_start_points = [ + { + "n_estimators": 100, + "max_depth": 10, + "min_samples_split": 5, + "min_samples_leaf": 2, + } +] + +optimizer = TreeStructuredParzenEstimators( + search_space=search_space, + n_iter=15, + random_state=42, + initialize={"warm_start": warm_start_points}, + experiment=experiment, +) + +# Run optimization +# TPE builds probabilistic models P(x|y) where x are parameters and y are outcomes +# It maintains separate models for good (y > threshold) and bad (y <= threshold) trials +# New configurations are selected to maximize the ratio of good to bad model densities +# This Bayesian approach efficiently guides search toward promising parameter regions +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print("Tree-structured Parzen Estimators optimization completed successfully") diff --git a/examples/hyperactive_intro.ipynb b/examples/hyperactive_intro.ipynb deleted file mode 100644 index 222d5e30..00000000 --- a/examples/hyperactive_intro.ipynb +++ /dev/null @@ -1,1576 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "af034c62", - "metadata": {}, - "source": [ - "## hyperactive - unified interfaces for optimizers and experiments" - ] - }, - { - "cell_type": "code", - "id": "ojjewxbtma", - "source": [ - "\"\"\"Hyperactive tutorial demonstrating unified interfaces for optimizers.\"\"\"" - ], - "metadata": {}, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "id": "830f7eb7", - "metadata": {}, - "source": [ - "### \"experiment\" = optimization problem" - ] - }, - { - "cell_type": "markdown", - "id": "668f9ccc", - "metadata": {}, - "source": [ - "\"experiment\" classes model optimization functions and ML experiments under one API\n", - "\n", - "Examples below:\n", - "1. simple objective function - parabola function\n", - "2. ML cross-validation experiment in sklearn" - ] - }, - { - "cell_type": "markdown", - "id": "173f1923", - "metadata": {}, - "source": [ - "#### simple objective function - parabola" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "59dd4715", - "metadata": {}, - "outputs": [], - "source": "from hyperactive.experiment.bench import Parabola\n\nparabola = Parabola(42, 3, 4)" - }, - { - "cell_type": "markdown", - "id": "4f561158", - "metadata": {}, - "source": [ - "instances of the class behave like a function with arguments\n", - "\n", - "we interpret the function as an \"objective function\" or random experiment" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "797535db", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "np.float64(564.0)" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "parabola(x=2, y=3)\n", - "# output is always np.float64" - ] - }, - { - "cell_type": "markdown", - "id": "b8096712", - "metadata": {}, - "source": [ - "the \"experiment\" class has two sets of variables:\n", - "\n", - "* optimization variables = inputs of the objective - inspectable via `paramnames`\n", - "* parameters of the experiment = constant in the objective\n", - " * these are params of `__init__`\n", - " * and are inspectable via `get_params`" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "fc08e901", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['x', 'y']" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "parabola.paramnames()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "61a62efa", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['a', 'b', 'c']" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "list(parabola.get_params())" - ] - }, - { - "cell_type": "markdown", - "id": "a1dbc553", - "metadata": {}, - "source": [ - "call via `score` method:\n", - "\n", - "* returns two objects: the value, and metadata (in a dict)\n", - "* input is a single dict, the `**` variant of a direct call" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "b127d406", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(np.float64(564.0), {})" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "parabola.score({\"x\": 2, \"y\": 3})" - ] - }, - { - "cell_type": "markdown", - "id": "98924abf", - "metadata": {}, - "source": [ - "#### sklearn cross-validation" - ] - }, - { - "cell_type": "markdown", - "id": "a625210e", - "metadata": {}, - "source": [ - "\"experiment\" can be more complicated - e.g., a cross-validation experiment\n", - "\n", - "this is a single tuning-evaluation step for an `sklearn` estimator" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57110e86", - "metadata": {}, - "outputs": [], - "source": [ - "from sklearn.datasets import load_iris\n", - "from sklearn.metrics import accuracy_score\n", - "from sklearn.model_selection import KFold\n", - "from sklearn.svm import SVC\n", - "\n", - "from hyperactive.experiment.integrations import SklearnCvExperiment\n", - "\n", - "X, y = load_iris(return_X_y=True)\n", - "\n", - "sklearn_exp = SklearnCvExperiment(\n", - " estimator=SVC(),\n", - " scoring=accuracy_score,\n", - " cv=KFold(n_splits=3, shuffle=True),\n", - " X=X,\n", - " y=y,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "dcf183ae", - "metadata": {}, - "source": [ - "usage syntax same as for the simple parabola!" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "70a27348", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "np.float64(0.9666666666666667)" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sklearn_exp(C=1, gamma=0.3)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "1cfd28d4", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['C',\n", - " 'break_ties',\n", - " 'cache_size',\n", - " 'class_weight',\n", - " 'coef0',\n", - " 'decision_function_shape',\n", - " 'degree',\n", - " 'gamma',\n", - " 'kernel',\n", - " 'max_iter',\n", - " 'probability',\n", - " 'random_state',\n", - " 'shrinking',\n", - " 'tol',\n", - " 'verbose']" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sklearn_exp.paramnames()" - ] - }, - { - "cell_type": "markdown", - "id": "165c5830", - "metadata": {}, - "source": [ - "`get_params` works like in `sklearn` and is nested\n", - "\n", - "note that similar parameters appear as in `paramnames`\n", - "\n", - "* parameters in `paramnames` are optimized over\n", - "* parameters in `get_params` are default values, if not set in `score`" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "b6160e3c", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['X',\n", - " 'cv',\n", - " 'estimator',\n", - " 'scoring',\n", - " 'y',\n", - " 'estimator__C',\n", - " 'estimator__break_ties',\n", - " 'estimator__cache_size',\n", - " 'estimator__class_weight',\n", - " 'estimator__coef0',\n", - " 'estimator__decision_function_shape',\n", - " 'estimator__degree',\n", - " 'estimator__gamma',\n", - " 'estimator__kernel',\n", - " 'estimator__max_iter',\n", - " 'estimator__probability',\n", - " 'estimator__random_state',\n", - " 'estimator__shrinking',\n", - " 'estimator__tol',\n", - " 'estimator__verbose']" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "list(sklearn_exp.get_params().keys())" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "f74ca927", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(np.float64(0.98),\n", - " {'score_time': array([0.00099897, 0.00099969, 0. ]),\n", - " 'fit_time': array([0.00099897, 0.00100136, 0.00100017]),\n", - " 'n_test_samples': 150})" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sklearn_exp.score({\"C\": 1, \"gamma\": 0.3})" - ] - }, - { - "cell_type": "markdown", - "id": "71073496", - "metadata": {}, - "source": [ - "### use of optimizers" - ] - }, - { - "cell_type": "markdown", - "id": "300a1666", - "metadata": {}, - "source": [ - "#### Grid search & sklearn CV" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5e2328c9", - "metadata": {}, - "outputs": [], - "source": [ - "from sklearn.datasets import load_iris\n", - "from sklearn.metrics import accuracy_score\n", - "from sklearn.model_selection import KFold\n", - "from sklearn.svm import SVC\n", - "\n", - "from hyperactive.experiment.integrations import SklearnCvExperiment\n", - "\n", - "X, y = load_iris(return_X_y=True)\n", - "\n", - "sklearn_exp = SklearnCvExperiment(\n", - " estimator=SVC(),\n", - " scoring=accuracy_score,\n", - " cv=KFold(n_splits=3, shuffle=True),\n", - " X=X,\n", - " y=y,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e9a07a73", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'C': 0.01, 'gamma': 10}" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from hyperactive.opt import GridSearchSk as GridSearch\n", - "\n", - "param_grid = {\n", - " \"C\": [0.01, 0.1, 1, 10],\n", - " \"gamma\": [0.0001, 0.01, 0.1, 1, 10],\n", - "}\n", - "grid_search = GridSearch(param_grid=param_grid, experiment=sklearn_exp)\n", - "\n", - "grid_search.solve()" - ] - }, - { - "cell_type": "markdown", - "id": "80dc9cb3", - "metadata": {}, - "source": [ - "we can also pass the parameters later, in `add_search`:" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "2ad90597", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'C': 0.01, 'gamma': 0.01}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from hyperactive.opt import GridSearch\n", - "\n", - "grid_search = GridSearch()\n", - "\n", - "param_grid = {\n", - " \"C\": [0.01, 0.1, 1, 10],\n", - " \"gamma\": [0.0001, 0.01, 0.1, 1, 10],\n", - "}\n", - "\n", - "grid_search.add_search(sklearn_exp, param_grid=param_grid)\n", - "grid_search.solve()" - ] - }, - { - "cell_type": "markdown", - "id": "72fcf886", - "metadata": {}, - "source": [ - "#### hill climbing & sklearn CV" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f9a4d922", - "metadata": {}, - "outputs": [], - "source": [ - "from sklearn.datasets import load_iris\n", - "from sklearn.metrics import accuracy_score\n", - "from sklearn.model_selection import KFold\n", - "from sklearn.svm import SVC\n", - "\n", - "from hyperactive.experiment.integrations import SklearnCvExperiment\n", - "\n", - "X, y = load_iris(return_X_y=True)\n", - "\n", - "sklearn_exp = SklearnCvExperiment(\n", - " estimator=SVC(),\n", - " scoring=accuracy_score,\n", - " cv=KFold(n_splits=3, shuffle=True),\n", - " X=X,\n", - " y=y,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a13b4f3", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "from hyperactive.opt import HillClimbing\n", - "\n", - "hillclimbing_config = {\n", - " \"search_space\": {\n", - " \"C\": np.array([0.01, 0.1, 1, 10]),\n", - " \"gamma\": np.array([0.0001, 0.01, 0.1, 1, 10]),\n", - " },\n", - " \"n_iter\": 100,\n", - "}\n", - "hill_climbing = HillClimbing(**hillclimbing_config, experiment=sklearn_exp)\n", - "\n", - "hill_climbing.solve()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5aa7ca80", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "from hyperactive.opt import HillClimbing\n", - "\n", - "hill_climbing_config = {\n", - " \"search_space\": {\n", - " \"C\": np.array([0.01, 0.1, 1, 10]),\n", - " \"gamma\": np.array([0.0001, 0.01, 0.1, 1, 10]),\n", - " },\n", - " \"n_iter\": 100,\n", - "}\n", - "hill_climbing = HillClimbing()\n", - "hill_climbing.add_search(**hill_climbing_config, experiment=sklearn_exp)\n", - "\n", - "hill_climbing.solve()" - ] - }, - { - "cell_type": "markdown", - "id": "7a933b41", - "metadata": {}, - "source": [ - "### full sklearn integration as estimator" - ] - }, - { - "cell_type": "markdown", - "id": "1f368679", - "metadata": {}, - "source": [ - "`OptCV` allows `sklearn` tuning via any tuning algorithm.\n", - "\n", - "Below, we show tuning via:\n", - "\n", - "* standard `GridSearch`\n", - "* `HillClimbing` from `gradient-free-optimizers`" - ] - }, - { - "cell_type": "markdown", - "id": "7171fc17", - "metadata": {}, - "source": [ - "##### `OptCV` tuning via `GridSearch`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4bdf2d49", - "metadata": {}, - "outputs": [], - "source": [ - "# 1. defining the tuned estimator\n", - "from sklearn.datasets import load_iris\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.svm import SVC\n", - "\n", - "from hyperactive.integrations.sklearn import OptCV\n", - "from hyperactive.opt import GridSearchSk as GridSearch\n", - "\n", - "param_grid = {\"kernel\": [\"linear\", \"rbf\"], \"C\": [1, 10]}\n", - "tuned_svc = OptCV(SVC(), optimizer=GridSearch(param_grid))\n", - "\n", - "# 2. fitting the tuned estimator = tuning the hyperparameters\n", - "X, y = load_iris(return_X_y=True)\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", - "\n", - "tuned_svc.fit(X_train, y_train)\n", - "\n", - "# 3. making predictions with the tuned estimator\n", - "y_pred = tuned_svc.predict(X_test)" - ] - }, - { - "cell_type": "markdown", - "id": "1a4bddc0", - "metadata": {}, - "source": [ - "best parameters and best estimator can be returned" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "fdd8c14a", - "metadata": {}, - "outputs": [], - "source": [ - "best_params = tuned_svc.best_params_" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "252efea6", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SVC(C=1)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" - ], - "text/plain": [ - "SVC(C=1)" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tuned_svc.best_estimator_" - ] - }, - { - "cell_type": "markdown", - "id": "89470c7d", - "metadata": {}, - "source": [ - "##### `OptCV` tuning via `HillClimbing`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f606284b", - "metadata": {}, - "outputs": [], - "source": [ - "# 1. defining the tuned estimator\n", - "from sklearn.datasets import load_iris\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.svm import SVC\n", - "\n", - "from hyperactive.integrations.sklearn import OptCV\n", - "from hyperactive.opt import HillClimbing\n", - "\n", - "# picking the optimizer is the only part that changes!\n", - "hill_climbing_config = {\n", - " \"search_space\": {\n", - " \"C\": np.array([0.01, 0.1, 1, 10]),\n", - " \"gamma\": np.array([0.0001, 0.01, 0.1, 1, 10]),\n", - " },\n", - " \"n_iter\": 100,\n", - "}\n", - "hill_climbing = HillClimbing(**hill_climbing_config)\n", - "\n", - "tuned_svc = OptCV(SVC(), optimizer=hill_climbing)\n", - "\n", - "# 2. fitting the tuned estimator = tuning the hyperparameters\n", - "X, y = load_iris(return_X_y=True)\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", - "\n", - "tuned_svc.fit(X_train, y_train)\n", - "\n", - "# 3. making predictions with the tuned estimator\n", - "y_pred = tuned_svc.predict(X_test)" - ] - }, - { - "cell_type": "markdown", - "id": "1e7164df", - "metadata": {}, - "source": [ - "best parameters and best estimator - works as before!" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "14710f3d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'C': np.float64(1.0), 'gamma': np.float64(0.1)}" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tuned_svc.best_params_" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "0a55cdd6", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SVC(C=np.float64(1.0), gamma=np.float64(0.1))
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" - ], - "text/plain": [ - "SVC(C=np.float64(1.0), gamma=np.float64(0.1))" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tuned_svc.best_estimator_" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "sktime-skpro-skbase-313", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.7" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/integrations/sklearn_example.py b/examples/integrations/sklearn_example.py deleted file mode 100644 index 45eb429a..00000000 --- a/examples/integrations/sklearn_example.py +++ /dev/null @@ -1,16 +0,0 @@ -from sklearn import svm, datasets - -from hyperactive.integrations import HyperactiveSearchCV -from hyperactive.optimizers import RandomSearchOptimizer - -iris = datasets.load_iris() - - -svc = svm.SVC() -opt = RandomSearchOptimizer() -parameters = {"kernel": ["linear", "rbf"], "C": [1, 10]} - -search = HyperactiveSearchCV(svc, opt, parameters) -search.fit(iris.data, iris.target) - -print("\n search.get_params() \n", search.get_params()) diff --git a/examples/opt_strat_early_stop.py b/examples/opt_strat_early_stop.py deleted file mode 100644 index 2a36067d..00000000 --- a/examples/opt_strat_early_stop.py +++ /dev/null @@ -1,40 +0,0 @@ -import numpy as np - -from hyperactive import Hyperactive - - -from hyperactive.optimizers.strategies import CustomOptimizationStrategy -from hyperactive.optimizers import ( - HillClimbingOptimizer, - RandomSearchOptimizer, - BayesianOptimizer, -) - - -opt_strat = CustomOptimizationStrategy() -opt_strat.add_optimizer( - RandomSearchOptimizer(), duration=0.5, early_stopping={"n_iter_no_change": 10} -) -opt_strat.add_optimizer( - HillClimbingOptimizer(), duration=0.5, early_stopping={"n_iter_no_change": 10} -) - - -def objective_function(opt): - score = -opt["x1"] * opt["x1"] - return score, {"additional stuff": 1} - - -search_space = {"x1": list(np.arange(-100, 101, 1))} -n_iter = 100 -optimizer = opt_strat - -hyper = Hyperactive() -hyper.add_search( - objective_function, - search_space, - n_iter=n_iter, - n_jobs=1, - optimizer=optimizer, -) -hyper.run() diff --git a/examples/opt_strat_search_space_pruning.py b/examples/opt_strat_search_space_pruning.py deleted file mode 100644 index 313d3559..00000000 --- a/examples/opt_strat_search_space_pruning.py +++ /dev/null @@ -1,38 +0,0 @@ -import numpy as np - -from hyperactive import Hyperactive - - -from hyperactive.optimizers.strategies import CustomOptimizationStrategy -from hyperactive.optimizers import ( - HillClimbingOptimizer, - RandomSearchOptimizer, - BayesianOptimizer, -) - - -opt_strat = CustomOptimizationStrategy() -opt_strat.add_optimizer(RandomSearchOptimizer(), duration=0.5) -opt_strat.prune_search_space() -opt_strat.add_optimizer(HillClimbingOptimizer(), duration=0.5) - - -def objective_function(opt): - score = -opt["x1"] * opt["x1"] - return score, {"additional stuff": 1} - - -search_space = {"x1": list(np.arange(-100, 101, 1))} -n_iter = 100 -optimizer = opt_strat - -hyper = Hyperactive() -hyper.add_search( - objective_function, - search_space, - n_iter=n_iter, - n_jobs=1, - optimizer=optimizer, - # random_state=1, -) -hyper.run() diff --git a/examples/optimization_applications/GMM_Hyperactive_Example.ipynb b/examples/optimization_applications/GMM_Hyperactive_Example.ipynb deleted file mode 100644 index 2690fac3..00000000 --- a/examples/optimization_applications/GMM_Hyperactive_Example.ipynb +++ /dev/null @@ -1,191 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "id": "rqk3u3ki08k", - "source": "\"\"\"Gaussian Mixture Model optimization example using Hyperactive.\"\"\"", - "metadata": {}, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "54e01852", - "metadata": {}, - "outputs": [], - "source": [ - "import warnings\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "from sklearn import datasets\n", - "from sklearn.mixture import GaussianMixture\n", - "\n", - "warnings.filterwarnings(\"ignore\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "2b01502a", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiQAAAGgCAYAAACaOnwjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/av/WaAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAz3ElEQVR4nO3df3BV9Z3/8ddNQhKoJEuo+aFEiIhgiAhorUH5sRUEQb4yO9O1DD/UKqMMbMHpWE3XFtFqpK3TdccOpexorJGyy9QfxbKxiD+gEioQaROjVmv44ZrATqEJCATJPd8/mGTJj5vcc3LvOZ9zzvMxkz9y+VzO50fOzTv3nPt5RSzLsgQAAOChFK87AAAAQEECAAA8R0ECAAA8R0ECAAA8R0ECAAA8R0ECAAA8R0ECAAA8R0ECAAA8R0ECAAA8R0ECAAA816+C5IknnlAkEtHKlStjtqmoqFAkEun0lZmZ2Z/DAgCAgElz+sTdu3dr3bp1GjduXJ9ts7Ky9NFHH3V8H4lEbB0rGo3q888/1+DBg20/FwAAeMOyLB0/flwXXXSRUlJ6fw/EUUFy4sQJLViwQOvXr9ePfvSjPttHIhHl5+c7OZQk6fPPP1dhYaHj5wMAAO8cOnRIw4YN67WNo4Jk2bJlmjNnjqZPnx5XQXLixAkNHz5c0WhUEydO1OOPP66xY8fGbN/a2qrW1taO79sDiQ8dOqSsrCwnXQYAAC5raWlRYWGhBg8e3Gdb2wXJxo0bVVNTo927d8fVfvTo0XrmmWc0btw4NTc366c//akmTZqk999/P2a1VF5ertWrV3d7PCsri4IEAACfied2i4jV/vZDHA4dOqRrrrlGW7du7bh3ZNq0aRo/frz+7d/+La7/48svv9QVV1yh+fPn69FHH+2xTdd3SNorrObmZgoSAAB8oqWlRdnZ2XH9/rb1DsnevXt15MgRTZw4seOxtrY2bd++XU8//bRaW1uVmpra6/8xYMAATZgwQZ988knMNhkZGcrIyLDTNQAA4GO2CpIbb7xRtbW1nR678847NWbMGD3wwAN9FiPSuQKmtrZWs2fPttdTAAAQWLYKksGDB6ukpKTTY1/5ylc0dOjQjscXL16siy++WOXl5ZKkRx55RNddd50uu+wy/f3vf9dPfvITHThwQHfffXeChgAAAPzO8T4ksRw8eLDTZ42PHTumJUuWqKmpSUOGDNHVV1+tnTt3qri4ONGHBgAAPmXrplav2LkpBgAAmMHO72+ybAAAgOcoSAAAgOcSfg8JgORqi1p6t+Gojhw/rdzBmbq2KEepKWQ8AfA3ChLAR6rqGrV6c70am093PFaQnalVc4s1q6TAw54BQP9wyQbwiaq6Ri2trOlUjEhSU/NpLa2sUVVdo0c9A4D+oyABfKAtamn15nr19JG49sdWb65XW9T4D80BQI8oSAAfeLfhaLd3Rs5nSWpsPq13G4661ykASCAKEsAHjhyPXYw4aQcApqEgAXwgd3BmQtsBgGkoSAAfuLYoRwXZmYr14d6Izn3a5tqiHDe7BQAJQ0EC+EBqSkSr5p7Lf+palLR/v2puMfuRAPAtChLAJ2aVFGjtwonKz+58WSY/O1NrF05kHxIAvsbGaICPzCop0IzifHZqBRA4FCSAz6SmRFQ6cqjX3QCAhOKSDQAA8BwFCQAA8BwFCQAA8BwFCQAA8BwFCQAA8BwFCQAA8BwFCQAA8BwFCQAA8BwFCQAA8BwFCQAA8BwFCQAA8BwFCQAA8BwFCQAA8BwFCQAA8BwFCQAA8BwFCQAA8Fya1x0AgqQtaundhqM6cvy0cgdn6tqiHKWmRLzuFgAYj4IESJCqukat3lyvxubTHY8VZGdq1dxizSop8LBnAGA+LtkACVBV16illTWdihFJamo+raWVNaqqa/SoZwDgDxQkQD+1RS2t3lwvq4d/a39s9eZ6tUV7agEAkChIgH57t+Fot3dGzmdJamw+rXcbjrrXKQDwGQoSoJ+OHI9djDhpBwBhREEC9FPu4MyEtgOAMKIgAfrp2qIcFWRnKtaHeyM692mba4ty3OwWAPgKBQnQT6kpEa2aWyxJ3YqS9u9XzS1mPxIA6AUFCZAAs0oKtHbhROVnd74sk5+dqbULJ7IPCQD0gY3RgASZVVKgGcX57NQKAA5QkAAJlJoSUenIoV53AwB8h0s2AADAcxQkAADAc1yygW+RrAsAwUFBAl8iWRcAgoVLNvAdknUBIHgoSOArJOsCQDBRkMBXSNYFgGCiIIGvkKwLAMFEQQJfIVkXAIKJggS+QrIuAAQTBQl8hWRdAAgmChL4Dsm6ABA8bIwGXyJZFwCChYIEvkWyLgAEB5dsAACA5yhIAACA57hkA4QAycgATNevd0ieeOIJRSIRrVy5std2mzZt0pgxY5SZmakrr7xSW7Zs6c9hAdhQVdeoG9a8ofnrd2nFxn2av36XbljzBiGEAIziuCDZvXu31q1bp3HjxvXabufOnZo/f77uuusuvffee5o3b57mzZunuro6p4cGECeSkQH4haOC5MSJE1qwYIHWr1+vIUOG9Nr2qaee0qxZs3T//ffriiuu0KOPPqqJEyfq6aefdtRhAPEhGRmAnzgqSJYtW6Y5c+Zo+vTpfbatrq7u1m7mzJmqrq6O+ZzW1la1tLR0+gJgD8nIAPzE9k2tGzduVE1NjXbv3h1X+6amJuXl5XV6LC8vT01NTTGfU15ertWrV9vtGoDzkIwMwE9svUNy6NAhrVixQi+88IIyM5OXplpWVqbm5uaOr0OHDiXtWEBQkYwMwE9svUOyd+9eHTlyRBMnTux4rK2tTdu3b9fTTz+t1tZWpaamdnpOfn6+Dh8+3Omxw4cPKz8/P+ZxMjIylJGRYadrALpoT0Zuaj7d430kEZ3L/yEZGYAJbL1DcuONN6q2tlb79u3r+Lrmmmu0YMEC7du3r1sxIkmlpaXatm1bp8e2bt2q0tLS/vUcQK9IRgbgJ7beIRk8eLBKSko6PfaVr3xFQ4cO7Xh88eLFuvjii1VeXi5JWrFihaZOnaonn3xSc+bM0caNG7Vnzx798pe/TNAQAMTSnoy8enN9pxtc87MztWpuMcnIAIyR8J1aDx48qJSU/3vjZdKkSdqwYYMeeughff/739eoUaP08ssvdytsACQHycgA/CBiWZbxmxC0tLQoOztbzc3NysrK8ro7AAAgDnZ+fxOuBwAAPEdBAgAAPEfaL9CLM2ejer56vw4cPanhOYO0qHSE0tOo4wEg0ShIgBjKt9Rr/Y4GnR/18tiWD7RkcpHKZhd71zEACCAKEqAH5VvqtW57Q7fHo5Y6HqcoAYDE4b1noIszZ6Nav6N7MXK+9TsadOZs1KUeAUDwUZAAXTxfvb/TZZqeRK1z7QAAiUFBAnRx4OjJhLYDAPSNggToYnjOoIS2AwD0jYIE6GJR6Qj1tat6SuRcOwBAYlCQAF2kp6VoyeSiXtssmVzEfiQAkEB87BfoQftHervuQ5ISEfuQAEASEK4H9IKdWgHAOTu/v3mHBOhFelqK7pp8qdfdAIDA4089AADgOQoSAADgOS7ZwAinzrTp8S312v+3kxoxdJC+P7tYA9NTve6Wkdqilt5tOKojx08rd3Cmri3KUWpfn1MG4DtunOsmvZ5QkMBzS361W1vrj3R8v+Nj6fldBzWjOFfrF3/Nw56Zp6quUas316ux+XTHYwXZmVo1t1izSgo87BmARHLjXDft9YRLNvBU12LkfFvrj2jJr3a73CNzVdU1amllTacXD0lqaj6tpZU1qqpr9KhnABLJjXPdxNcTChJ45tSZtpjFSLut9Ud06kybSz0yV1vU0urN9erpM/rtj63eXK+2vlIBARjNjXPd1NcTChJ45vEt9QltF2TvNhzt9pfM+SxJjc2n9W7DUfc6BSDh3DjXTX09oSCBZ/b/Lb603HjbBdmR47FfPJy0A2AmN851U19PKEjgmRFD40vLjbddkOUOzkxoOwBmcuNcN/X1hIIEnvl+nHkw8bYLsmuLclSQnalYH8aL6Nzd8dcW5bjZLQAJ5sa5burrCQUJPDMwPVUzinN7bTOjOJf9SCSlpkS0au65wqzri0j796vmFrMfCeBzbpzrpr6eUJDAU+sXfy1mUcI+JJ3NKinQ2oUTlZ/d+W3U/OxMrV04kX1IgIBw41w38fWEtF8YgZ1a42fSzooAkicIO7Xa+f1NQQIAAJLCzu9vLtkAAADPUZAAAADPEa4HI5h6rZT7NQDAHRQk8JypqZamJWECQJBxyQaeMjXV0sQkTAAIMgoSeMbUVEtTkzABIMgoSOAZU1MtTU3CBIAgoyCBZ0xNtTQ1CRMAgoyCBJ4xNdXS1CRMAAgyChJ4xtRUS1OTMAEgyChI4BlTUy1NTcIEgCCjIIGnTE21NDEJEwCCjHA9GIGdWgEgeOz8/manVhghNSWi0pFDjTuGG/0CAHDJBgAAGICCBAAAeI5LNoYx8Z4F7r0A4BVeS8KDgsQgJqbLkpILwCu8loQLn7IxRHu6bNfFaP87wIuPmjrpk4njAOA/vJYEg53f39xDYgAT02VJyQXgFV5LwomCxAAmpsuSkgvAK7yWhBMFiQFMTJclJReAV3gtCScKEgOYmC5LSi4Ar/BaEk4UJAYwMV2WlFwAXuG1JJwoSAxgYrosKbkAvMJrSThRkBjCxHRZUnIBeIXXkvBhHxLDmLgrITu1AvAKryX+Zuf3NwUJAABICjZGAwAAvkJBAgAAPEe4HpLizNmonq/erwNHT2p4ziAtKh2h9LTY9a/d9pKZ15ZN7BMA+IGte0jWrl2rtWvXav/+/ZKksWPH6oc//KFuvvnmHttXVFTozjvv7PRYRkaGTp+2t7se95D4S/mWeq3f0aDzYyZSItKSyUUqm13c7/aSmSmgJvYJALyUtHtIhg0bpieeeEJ79+7Vnj179I1vfEO33nqr3n///ZjPycrKUmNjY8fXgQMH7BwSPlO+pV7rtncuLiQpaknrtjeofEt9v9pL/5cC2jXroqn5tJZW1qiqrjEhY7HDxD4BgJ/YKkjmzp2r2bNna9SoUbr88sv12GOP6YILLtCuXbtiPicSiSg/P7/jKy8vr9+dhpnOnI1q/Y6GXtus39GgM2ejjtpLZqaAmtgnAPAbxze1trW1aePGjfriiy9UWloas92JEyc0fPhwFRYW9vluSrvW1la1tLR0+oL5nq/e3+2djq6i1rl2TtpLZqaAmtgnAPAb2wVJbW2tLrjgAmVkZOjee+/VSy+9pOLinq/zjx49Ws8884xeeeUVVVZWKhqNatKkSfrss896PUZ5ebmys7M7vgoLC+12Ex44cPSkrXZ220tmpoCa2CcA8BvbBcno0aO1b98+/fGPf9TSpUt1++23q76++3V+SSotLdXixYs1fvx4TZ06VS+++KIuvPBCrVu3rtdjlJWVqbm5uePr0KFDdrsJDwzPGWSrnd32kpkpoCb2CQD8xnZBkp6erssuu0xXX321ysvLddVVV+mpp56K67kDBgzQhAkT9Mknn/TaLiMjQ1lZWZ2+YL5FpSPU1ydcUyLn2jlpL5mZAmpinwDAb/q9MVo0GlVra2tcbdva2lRbW6uCAj4CGUTpaSlaMrmo1zZLJhd17C9it71kZgqoiX0CAL+xVZCUlZVp+/bt2r9/v2pra1VWVqa33npLCxYskCQtXrxYZWVlHe0feeQR/f73v9enn36qmpoaLVy4UAcOHNDdd9+d2FHAGGWzi3XPlKJu73ykRKR7pnTfV8Rue8nMFFAT+wQAfmJrp9YjR45o8eLFamxsVHZ2tsaNG6fXXntNM2bMkCQdPHhQKSn/V+McO3ZMS5YsUVNTk4YMGaKrr75aO3fujHkTLIKhbHaxvnvTmLh3XrXbXjpXAMwozjdqV1QT+wQAfkHaLwAASArSfgEAgK9QkAAAAM+R9msYN9JinSTrJvsYTsYdlLkKCjfWw9SfEwD9xz0kBnEjLdZJsm6yj+Fk3EGZq6BwYz1M/TkBEJud398UJIZoT4vtuhjtf8cl4qOj7cm6scT6mG0yj+Fk3EGZq6BwYz1M/TkB0DtuavUZN9JinSTrJvsYTsYdlLkKCjfWw9SfEwCJRUFiADfSYp0k6yb7GE7GHZS5Cgo31sPUnxMAiUVBYgA30mKdJOsm+xhOxh2UuQoKN9bD1J8TAIlFQWIAN9JinSTrJvsYTsYdlLkKCjfWw9SfEwCJRUFiADfSYp0k6yb7GE7GHZS5Cgo31sPUnxMAiUVBYgA30mKdJOsm+xhOxh2UuQoKN9bD1J8TAInFK6oh3EiLdZKsm+xjOBl3UOYqKNxYD1N/TgAkDvuQGCYou4+yU2v4sFMrgK7YGA0AAHiOjdEAAICvUJAAAADPkfaLpLB73Z7r/EiWoNwHFJRxALFwDwkSzm7CKomsSJagJDYHZRwIH+4hgWfaE1a75og0NZ/W0soaVdU19qs9EK/2xOauuURRS1q3vUHlW+q96ZhNQRkH0BcKEiSM3YRVElmRLEFJbA7KOIB4UJAgYewmrJLIimQJSmJzUMYBxIOCBAljN2GVRFYkS1ASm4MyDiAeFCRIGLsJqySyIlmCktgclHEA8aAgQcLYTVglkRXJEpTE5qCMA4gHBQkSxm7CKomsSJagJDYHZRxAPPgpRkLZTVglkRXJEpTE5qCMA+gLG6MhKdipFaYIyg6nQRkHwoW0XwAA4Dl2agUAAL5CQQIAADxH2q8NbtznYPcYpl5X5p4Q/wvrGjo5p0ycKyd9MvHeLxPnFsnBPSRxciOR1u4xTE0AJb3X/8K6hk7OKRPnykmfTEzpNnFuYQ83tSZYeyJt14lqr9ET8fFUu8doTwCNxauPA7oxV0iusK6hk3PKxLly0ie7zzHxNRFm4qbWBHIjkdbuMUxNACW91//CuoZOzikT58pJn0xM6TZxbpF8FCR9cCOR1u4xTE0AJb3X/8K6hk7OKRPnykmfTEzpNnFukXwUJH1wI5HW7jFMTQAlvdf/wrqGTs4pE+fKSZ9MTOk2cW6RfBQkfXAjkdbuMUxNACW91//CuoZOzikT58pJn0xM6TZxbpF8FCR9cCOR1u4xTE0AJb3X/8K6hk7OKRPnykmfTEzpNnFukXwUJH1wI5HW7jFMTQAlvdf/wrqGTs4pE+fKSZ9MTOk2cW6RfBQkcXAjkdbuMUxNACW91//CuoZOzikT58pJn0xM6TZxbpFc7ENig4m7ErJTK5IlrGvITq3s1IrEYWM0AADgOTZGAwAAvkJBAgAAPEfar2FMvCZr6n0qQDKYes/CqTNtenxLvfb/7aRGDB2k788u1sD0VN8dA4iFe0gMYmJ6pqmJwkAymJouu+RXu7W1/ki3x2cU52r94q/55hgIH+4h8aH2ZMuu+Q1Nzae1tLJGVXWNrh+jPf20a8ZH1JLWbW9Q+Zb6fvcJMIUb56ATsQoFSdpaf0RLfrXbF8cA+kJBYgAT0zNNTRQGksHUdNlTZ9piFgrtttYf0akzbUYfA4gHBYkBTEzPNDVRGEgGU9NlH4/zXch423l1DCAeFCQGMDE909REYSAZTE2X3f+3+M6veNt5dQwgHhQkBjAxPdPURGEgGUxNlx0xNL7zK952Xh0DiAcFiQFMTM80NVEYSAZT02W/H+cn2eJt59UxgHhQkBjAxPRMUxOFgWQwNV12YHqqZhTn9tpmRnFuv/YKceMYQDz4bWIIE9MzTU0UBpLB1HTZ9Yu/FrNgSNQeIW4cA+gLG6MZhp1aAW+xUys7tSJxSPsFAACeY6dWAADgKxQkAADAc6FN+3VyndjUa8t22b0nJMxz5cY43FgPU/sVlPuT7N574cZcmXoO2u2XqeOwK8yvo/GydQ/J2rVrtXbtWu3fv1+SNHbsWP3whz/UzTffHPM5mzZt0g9+8APt379fo0aN0po1azR79mxbnUz0PSROEj1NTQG1y256b5jnyo1xuLEepvYrKEnSdlNy3ZgrU89Bu/0ydRx2hfl1NGk3tW7evFmpqakaNWqULMvSc889p5/85Cd67733NHbs2G7td+7cqSlTpqi8vFy33HKLNmzYoDVr1qimpkYlJSVJGVBf2hM9uw66vebs6eN9Tp5jovb03li6fpQ3zHPlxjjcWA9T+2X3GKbqLSVX6l6UuDFXpp6Ddvtl6jjsCvPrqJTEm1rnzp2r2bNna9SoUbr88sv12GOP6YILLtCuXbt6bP/UU09p1qxZuv/++3XFFVfo0Ucf1cSJE/X000/bOWzCOEn0NDUF1C676b1hnis3xuHGepjar6AkSdtNyXVjrkw9B+32y9Rx2BXm11EnHF+sbWtr08aNG/XFF1+otLS0xzbV1dWaPn16p8dmzpyp6urqXv/v1tZWtbS0dPpKBCeJnqamgNplN703zHPlxjjcWA9T+xWUJGm7KbluzJWp56Ddfpk6DrvC/DrqhO2CpLa2VhdccIEyMjJ077336qWXXlJxcc9vrTY1NSkvL6/TY3l5eWpqaur1GOXl5crOzu74KiwstNvNHjlJ9DQ1BdQuu+m9YZ4rN8bhxno44Ua/gpIkbTcl1425MvUctNsvU8dhV5hfR52wXZCMHj1a+/bt0x//+EctXbpUt99+u+rr4/tLIV5lZWVqbm7u+Dp06FBC/l8niZ6mpoDaZTe9N8xz5cY43FgPJ9zoV1CSpO2m5LoxV6aeg3b7Zeo47Arz66gTtguS9PR0XXbZZbr66qtVXl6uq666Sk899VSPbfPz83X48OFOjx0+fFj5+fm9HiMjI0NZWVmdvhLBSaKnqSmgdtlN7w3zXLkxDjfWw9R+BSVJ2m5KrhtzZeo5aLdfpo7DrjC/jjrR7w/8R6NRtba29vhvpaWl2rZtW6fHtm7dGvOek2RzkuhpagqoXXbTe8M8V26Mw431MLVfQUmStpuS68ZcmXoO2u2XqeOwK8yvo07YOuPLysq0fft27d+/X7W1tSorK9Nbb72lBQsWSJIWL16ssrKyjvYrVqxQVVWVnnzySX344Yd6+OGHtWfPHi1fvjyxo7DBSaKnqSmgdtlN7w3zXLkxDjfWw9R+BSVJ2m5KrhtzZeo5aLdfpo7DrjC/jtplax+Su+66S9u2bVNjY6Oys7M1btw4PfDAA5oxY4Ykadq0aRoxYoQqKio6nrNp0yY99NBDHRuj/fjHP/Z8YzQp3LvmsVNr/NiplZ1a48FOrfFjp9ZwvY6S9gsAADxH2i8AAPAVChIAAOC50Kb9OhGE63kwT1CuqbtxjdzUY9hl6hoGBfPrTxQkcQpK8iLMEpT0UzfSTE09hl2mrmFQML/+xU2tcQhS8iLMEZT0UzfSTE09hl2mrmFQML/m4abWBApz8iKSJyjpp26kmZp6DLtMXcOgYH79j4KkD2FOXkTyBCX91I00U1OPYZepaxgUzK//UZD0IczJi0ieoKSfupFmauox7DJ1DYOC+fU/CpI+hDl5EckTlPRTN9JMTT2GXaauYVAwv/5HQdKHMCcvInmCkn7qRpqpqcewy9Q1DArm1/8oSPoQ5uRFJE9Q0k/dSDM19Rh2mbqGQcH8+h8FSRzCmryI5ApK+qkbaaamHsMuU9cwKJhff2MfEhvY/Q/JYOJOok6YuouqifNr6hoGBfNrDtJ+AQCA59gYDQAA+AoFCQAA8BzheoDH7F7vPnM2quer9+vA0ZManjNIi0pHKD2t978tTL2mbncsbtxDEpT5NbFPpjJ1rkztV7JwDwngIbvJpOVb6rV+R4POj+NIiUhLJhepbHZxQo7hFrtjcSPtNyjza2KfTGXqXJnaL7u4qRXwAbvJpOVb6rVue0PM/++eKd1/aZqafmp3LG6k/QZlfk3sk6lMnStT++UEN7UChrObTHrmbFTrd8T+ZSlJ63c06MzZqONjuMXuWNxI+w3K/JrYJ1OZOlem9ssNFCSAB+wmkz5fvV99vf5ErXPtnB7DLXbH4kbab1Dm18Q+mcrUuTK1X26gIAE8YDeZ9MDRk3G1P7+dqemndsfiRtpvUObXxD6ZytS5MrVfbqAgATxgN5l0eM6guNqf387U9FO7Y3Ej7Tco82tin0xl6lyZ2i83UJAAHrCbTLqodIT6+rRfSuRcO6fHcIvdsbiR9huU+TWxT6Yyda5M7ZcbKEgAD9hNJk1PS9GSyUW9/p9LJhd12i/D1PRTu2NxI+03KPNrYp9MZepcmdovN1CQAB6xm0xaNrtY90wp6vaXfEqk54+kOjmGW+yOxY2036DMr4l9MpWpc2Vqv5KNfUgAj4V1J1GJnVqTycQ+mcrUuTK1X3awMRoAAPAcG6MBAABfoSABAACeI+0XvhWE66uSO+M4cfqs7vvP93Tw2CldMmSgfnbbBF2QmdjT39T1sNsvU8cBBB0FCXwpKEmYbozj/z29Q3/+rKXj+4+ajqvk4dc0bliWfrt8ckKOYep62O2XqeMAwoCbWuE7QUnCdGMcXYuRrhJRlJi6Hnb7Zeo4AD/jplYEVlCSMN0Yx4nTZ3stRiTpz5+16MTps46PYep62O2XqeMAwoSCBL4SlCRMN8Zx33++l9B2PTF1Pez2y9RxAGFCQQJfCUoSphvjOHjsVELb9cTU9bDbL1PHAYQJBQl8JShJmG6M45IhAxPariemrofdfpk6DiBMKEjgK0FJwnRjHD+7bUJC2/XE1PWw2y9TxwGECQUJfCUoSZhujOOCzDSNG9b7Xe3jhmX1az8SU9fDbr9MHQcQJhQk8J2gJGG6MY7fLp8csyhJ1D4kpq6H3X6ZOg4gLNiHBL4VlB012ak1udipFfAOab8AAMBzbIwGAAB8hYIEAAB4jnA9IIHcuP/AyTFMvS/C1H4BJgjb+UFBAiSIG0mxTo5haoKtqf0CTBDG84ObWoEEcCMp1skxTE2wNbVfgAmCdH5wUyvgIjeSYp0cw9QEW1P7BZggzOcHBQnQT24kxTo5hqkJtqb2CzBBmM8PChKgn9xIinVyDFMTbE3tF2CCMJ8fFCRAP7mRFOvkGKYm2JraL8AEYT4/KEiAfnIjKdbJMUxNsDW1X4AJwnx+UJAA/eRGUqyTY5iaYGtqvwAThPn8oCABEsCNpFgnxzA1wdbUfgEmCOv5wT4kQAKxU6s9pvYLMEEQzg/SfgEAgOfYGA0AAPgKBQkAAPAc4Xrok6n3RZjIjfs7gjJXbjlzNqrnq/frwNGTGp4zSItKRyg9zX9/i7HuCDpbBUl5eblefPFFffjhhxo4cKAmTZqkNWvWaPTo0TGfU1FRoTvvvLPTYxkZGTp9Oni7zAWRqQm2JnIjiTcoc+WW8i31Wr+jQefHfjy25QMtmVykstnF3nXMJtYdYWDrz4S3335by5Yt065du7R161Z9+eWXuummm/TFF1/0+rysrCw1NjZ2fB04cKBfnYY72hMnu+YqNDWf1tLKGlXVNfriGG5wMg67zwnKXLmlfEu91m3vXIxIUtSS1m1vUPmWem86ZhPrjrCwVZBUVVXpjjvu0NixY3XVVVepoqJCBw8e1N69e3t9XiQSUX5+fsdXXl5evzqN5DM1wdZEbiTxBmWu3HLmbFTrdzT02mb9jgadORt1qUfOsO4Ik35dSG1ubpYk5eT0voXtiRMnNHz4cBUWFurWW2/V+++/32v71tZWtbS0dPqCu0xNsDWRG0m8QZkrtzxfvb/bOyNdRa1z7UzGuiNMHBck0WhUK1eu1PXXX6+SkpKY7UaPHq1nnnlGr7zyiiorKxWNRjVp0iR99tlnMZ9TXl6u7Ozsjq/CwkKn3YRDpibYmsiNJN6gzJVbDhw9mdB2XmHdESaOC5Jly5aprq5OGzdu7LVdaWmpFi9erPHjx2vq1Kl68cUXdeGFF2rdunUxn1NWVqbm5uaOr0OHDjntJhwyNcHWRG4k8QZlrtwyPGdQQtt5hXVHmDgqSJYvX65XX31Vb775poYNG2bruQMGDNCECRP0ySefxGyTkZGhrKysTl9wl6kJtiZyI4k3KHPllkWlI9TXJ2JTIufamYx1R5jYKkgsy9Ly5cv10ksv6Y033lBRUZHtA7a1tam2tlYFBXxUzWSmJtiayI0k3qDMlVvS01K0ZHLvr09LJhcZvx8J644wsXU2Llu2TJWVldqwYYMGDx6spqYmNTU16dSpUx1tFi9erLKyso7vH3nkEf3+97/Xp59+qpqaGi1cuFAHDhzQ3XffnbhRIClMTbA1kRtJvEGZK7eUzS7WPVOKur1TkhKR7pnin31IWHeEha1wvUik5yr82Wef1R133CFJmjZtmkaMGKGKigpJ0n333acXX3xRTU1NGjJkiK6++mr96Ec/0oQJE+LuJOF63mKn1vixU6t52KkV8A5pvwAAwHOk/QIAAF+hIAEAAJ4j7Rd94tp1/IJyvwIAuI2CBL0iZTR+QUmWBQAv8KcbYiJlNH5BSZYFAK9QkKBHpIzGLyjJsgDgJQoS9IiU0fgFJVkWALxEQYIekTIav6AkywKAlyhI0CNSRuMXlGRZAPASBQl6RMpo/IKSLAsAXqIgQY9IGY1fUJJlAcBLvEIiJlJG4xeUZFkA8ArheugTO7XGj51aAeD/2Pn9zU6t6FNqSkSlI4d63Q1fSE9L0V2TL/W6GwDgO/zpBgAAPEdBAgAAPMclmyQK870XYR17WMdtMtYE8AcKkiQJc0puWMce1nGbjDUB/INP2SRBe0pu14lt/5ssyB+ZDevYwzpuk7EmgPfs/P7mHpIEC3NKbljHHtZxm4w1AfyHgiTBwpySG9axh3XcJmNNAP+hIEmwMKfkhnXsYR23yVgTwH8oSBIszCm5YR17WMdtMtYE8B8KkgQLc0puWMce1nGbjDUB/IeCJMHCnJIb1rGHddwmY00A/6EgSYIwp+SGdexhHbfJWBPAX9iHJInCvENkWMce1nGbjDUBvGPn9zcFCQAASAo2RgMAAL5CQQIAADxHuB6AhDhzNqrnq/frwNGTGp4zSItKRyg9LbF/83A/CBBcFCQA+q18S73W72jQ+dEwj235QEsmF6lsdnFCjkFyLxBsXLIB0C/lW+q1bnvnYkSSopa0bnuDyrfU9/sY7cm9XfNpmppPa2lljarqGvt9DADeoiAB4NiZs1Gt39HQa5v1Oxp05mzU8TFI7gXCgYIEgGPPV+/v9s5IV1HrXDunSO4FwoGCBIBjB46eTGi7npDcC4QDBQkAx4bnDEpou56Q3AuEAwUJAMcWlY5QX5+6TYmca+cUyb1AOFCQAHAsPS1FSyYX9dpmyeSifu1HQnIvEA4UJAD6pWx2se6ZUtTtnZKUiHTPlMTsQ0JyLxB8hOsBSAh2agXQlZ3f3+zUCiAh0tNSdNfkS5N6jNSUiEpHDk3qMQB4g0s2AADAcxQkAADAcxQkAADAcxQkAADAcxQkAADAcxQkAADAcxQkAADAcxQkAADAcxQkAADAcxQkAADAcxQkAADAcxQkAADAcxQkAADAcxQkAADAcxQkAADAcxQkAADAc2ledwDB1Ba19G7DUR05flq5gzN1bVGOUlMiXncLAGAoW++QlJeX62tf+5oGDx6s3NxczZs3Tx999FGfz9u0aZPGjBmjzMxMXXnlldqyZYvjDsN8VXWNumHNG5q/fpdWbNyn+et36YY1b6iqrtHrrgEADGWrIHn77be1bNky7dq1S1u3btWXX36pm266SV988UXM5+zcuVPz58/XXXfdpffee0/z5s3TvHnzVFdX1+/OwzxVdY1aWlmjxubTnR5vaj6tpZU1FCUAgB5FLMuynD75f//3f5Wbm6u3335bU6ZM6bHNbbfdpi+++EKvvvpqx2PXXXedxo8fr1/84hdxHaelpUXZ2dlqbm5WVlaW0+4iydqilm5Y80a3YqRdRFJ+dqb+8MA3uHwDACFg5/d3v25qbW5uliTl5OTEbFNdXa3p06d3emzmzJmqrq6O+ZzW1la1tLR0+oL53m04GrMYkSRLUmPzab3bcNS9TgEAfMFxQRKNRrVy5Updf/31KikpidmuqalJeXl5nR7Ly8tTU1NTzOeUl5crOzu746uwsNBpN+GiI8djFyNO2gEAwsNxQbJs2TLV1dVp48aNieyPJKmsrEzNzc0dX4cOHUr4MZB4uYMzE9oOABAejj72u3z5cr366qvavn27hg0b1mvb/Px8HT58uNNjhw8fVn5+fsznZGRkKCMjw0nX4KFri3JUkJ2ppubT6unGpPZ7SK4tin2JDwAQTrbeIbEsS8uXL9dLL72kN954Q0VFRX0+p7S0VNu2bev02NatW1VaWmqvpzBeakpEq+YWSzpXfJyv/ftVc4u5oRUA0I2tgmTZsmWqrKzUhg0bNHjwYDU1NampqUmnTp3qaLN48WKVlZV1fL9ixQpVVVXpySef1IcffqiHH35Ye/bs0fLlyxM3ChhjVkmB1i6cqPzszpdl8rMztXbhRM0qKfCoZwAAk9n62G8k0vNfts8++6zuuOMOSdK0adM0YsQIVVRUdPz7pk2b9NBDD2n//v0aNWqUfvzjH2v27Nlxd5KP/foPO7UCAOz8/u7XPiRuoSABAMB/XNuHBAAAIBEoSAAAgOcoSAAAgOcoSAAAgOcoSAAAgOcoSAAAgOcoSAAAgOcoSAAAgOcoSAAAgOccpf26rX0z2ZaWFo97AgAA4tX+ezueTeF9UZAcP35cklRYWOhxTwAAgF3Hjx9XdnZ2r218kWUTjUb1+eefa/DgwTED/kzV0tKiwsJCHTp0KHQ5PGEde1jHLTH2MI49rOOWGHs8Y7csS8ePH9dFF12klJTe7xLxxTskKSkpGjZsmNfd6JesrKzQ/cC2C+vYwzpuibGHcexhHbfE2Psae1/vjLTjplYAAOA5ChIAAOA5CpIky8jI0KpVq5SRkeF1V1wX1rGHddwSYw/j2MM6bomxJ3rsvripFQAABBvvkAAAAM9RkAAAAM9RkAAAAM9RkAAAAM9RkCTQE088oUgkopUrV8ZsU1FRoUgk0ukrMzPTvU4myMMPP9xtHGPGjOn1OZs2bdKYMWOUmZmpK6+8Ulu2bHGpt4lld+xBWXNJ+p//+R8tXLhQQ4cO1cCBA3XllVdqz549vT7nrbfe0sSJE5WRkaHLLrtMFRUV7nQ2weyO/a233uq27pFIRE1NTS72uv9GjBjR4ziWLVsW8zlBONftjjtI53lbW5t+8IMfqKioSAMHDtTIkSP16KOP9plH099z3Rc7tfrB7t27tW7dOo0bN67PtllZWfroo486vvfbdvjtxo4dq9dff73j+7S02D9OO3fu1Pz581VeXq5bbrlFGzZs0Lx581RTU6OSkhI3uptQdsYuBWPNjx07puuvv17/+I//qP/+7//WhRdeqI8//lhDhgyJ+ZyGhgbNmTNH9957r1544QVt27ZNd999twoKCjRz5kwXe98/Tsbe7qOPPuq0k2Vubm4yu5pwu3fvVltbW8f3dXV1mjFjhr75zW/22D4o57rdcUvBOM8lac2aNVq7dq2ee+45jR07Vnv27NGdd96p7Oxsfec73+nxOQk51y302/Hjx61Ro0ZZW7dutaZOnWqtWLEiZttnn33Wys7Odq1vybJq1Srrqquuirv9P//zP1tz5szp9NjXv/5165577klwz5LP7tiDsuYPPPCAdcMNN9h6zve+9z1r7NixnR677bbbrJkzZyaya0nnZOxvvvmmJck6duxYcjrlkRUrVlgjR460otFoj/8epHP9fH2NOyjnuWVZ1pw5c6xvf/vbnR77p3/6J2vBggUxn5OIc51LNgmwbNkyzZkzR9OnT4+r/YkTJzR8+HAVFhbq1ltv1fvvv5/kHibHxx9/rIsuukiXXnqpFixYoIMHD8ZsW11d3W1+Zs6cqerq6mR3MynsjF0Kxpr/9re/1TXXXKNvfvObys3N1YQJE7R+/fpenxOUdXcy9nbjx49XQUGBZsyYoXfeeSfJPU2uM2fOqLKyUt/+9rdj/vUflDU/XzzjloJxnkvSpEmTtG3bNv3lL3+RJP3pT3/SH/7wB918880xn5OIdacg6aeNGzeqpqZG5eXlcbUfPXq0nnnmGb3yyiuqrKxUNBrVpEmT9NlnnyW5p4n19a9/XRUVFaqqqtLatWvV0NCgyZMn6/jx4z22b2pqUl5eXqfH8vLyfHc9XbI/9qCs+aeffqq1a9dq1KhReu2117R06VJ95zvf0XPPPRfzObHWvaWlRadOnUp2lxPGydgLCgr0i1/8Qr/5zW/0m9/8RoWFhZo2bZpqampc7Hlivfzyy/r73/+uO+64I2abIJ3r7eIZd1DOc0l68MEH9a1vfUtjxozRgAEDNGHCBK1cuVILFiyI+ZyEnOv23sjB+Q4ePGjl5uZaf/rTnzoe6+uSTVdnzpyxRo4caT300ENJ6KF7jh07ZmVlZVn/8R//0eO/DxgwwNqwYUOnx37+859bubm5bnQvqfoae1d+XfMBAwZYpaWlnR77l3/5F+u6666L+ZxRo0ZZjz/+eKfHfve731mSrJMnTyaln8ngZOw9mTJlirVw4cJEds1VN910k3XLLbf02iaI53o84+7Kr+e5ZVnWr3/9a2vYsGHWr3/9a+vPf/6z9atf/crKycmxKioqYj4nEec675D0w969e3XkyBFNnDhRaWlpSktL09tvv61///d/V1paWqcbomJprz4/+eQTF3qcPP/wD/+gyy+/POY48vPzdfjw4U6PHT58WPn5+W50L6n6GntXfl3zgoICFRcXd3rsiiuu6PVyVax1z8rK0sCBA5PSz2RwMvaeXHvttb5b93YHDhzQ66+/rrvvvrvXdkE71+Mdd1d+Pc8l6f777+94l+TKK6/UokWLdN999/V6JSAR5zoFST/ceOONqq2t1b59+zq+rrnmGi1YsED79u1Tampqn/9HW1ubamtrVVBQ4EKPk+fEiRP661//GnMcpaWl2rZtW6fHtm7dqtLSUje6l1R9jb0rv6759ddf3+kTBJL0l7/8RcOHD4/5nKCsu5Ox92Tfvn2+W/d2zz77rHJzczVnzpxe2wVlzdvFO+6u/HqeS9LJkyeVktK5PEhNTVU0Go35nISse7/e10E3XS/ZLFq0yHrwwQc7vl+9erX12muvWX/961+tvXv3Wt/61reszMxM6/333/egt85997vftd566y2roaHBeuedd6zp06dbX/3qV60jR45YltV93O+8846VlpZm/fSnP7U++OADa9WqVdaAAQOs2tpar4bgmN2xB2XN3333XSstLc167LHHrI8//th64YUXrEGDBlmVlZUdbR588EFr0aJFHd9/+umn1qBBg6z777/f+uCDD6yf//znVmpqqlVVVeXFEBxzMvaf/exn1ssvv2x9/PHHVm1trbVixQorJSXFev31170YQr+0tbVZl1xyifXAAw90+7cgn+t2xh2U89yyLOv222+3Lr74YuvVV1+1GhoarBdffNH66le/an3ve9/raJOMc52CJMG6FiRTp061br/99o7vV65caV1yySVWenq6lZeXZ82ePduqqalxv6P9dNttt1kFBQVWenq6dfHFF1u33Xab9cknn3T8e9dxW5Zl/dd//Zd1+eWXW+np6dbYsWOt3/3udy73OjHsjj0oa25ZlrV582arpKTEysjIsMaMGWP98pe/7PTvt99+uzV16tROj7355pvW+PHjrfT0dOvSSy+1nn32Wfc6nEB2x75mzRpr5MiRVmZmppWTk2NNmzbNeuONN1zudWK89tprliTro48+6vZvQT7X7Yw7SOd5S0uLtWLFCuuSSy6xMjMzrUsvvdT613/9V6u1tbWjTTLO9Yhl9bH1GgAAQJJxDwkAAPAcBQkAAPAcBQkAAPAcBQkAAPAcBQkAAPAcBQkAAPAcBQkAAPAcBQkAAPAcBQkAAPAcBQkAAPAcBQkAAPAcBQkAAPDc/weew/xTKPPtGQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# load the iris dataset\n", - "iris = datasets.load_iris()\n", - "\n", - "# select first two columns\n", - "X = iris.data[:, :2]\n", - "\n", - "# turn it into a dataframe\n", - "d = pd.DataFrame(X)\n", - "\n", - "# plot the data\n", - "plt.scatter(d[0], d[1])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b38d7bd7", - "metadata": {}, - "outputs": [], - "source": [ - "def model(opt):\n", - " \"\"\"Gaussian Mixture Model clustering with visualization.\n", - "\n", - " Args:\n", - " opt: Dictionary with optimization parameters including n_components.\n", - "\n", - " Returns\n", - " -------\n", - " Lower bound of log-likelihood value.\n", - " \"\"\"\n", - " gmm = GaussianMixture(n_components=opt[\"n_components\"])\n", - " gmm.fit(d)\n", - " # Assign a label to each sample\n", - " labels = gmm.predict(d)\n", - " d[\"labels\"] = labels\n", - " d0 = d[d[\"labels\"] == 0]\n", - " d1 = d[d[\"labels\"] == 1]\n", - " d2 = d[d[\"labels\"] == 2]\n", - "\n", - " # plot three clusters in same plot\n", - " plt.scatter(d0[0], d0[1], c=\"r\")\n", - " plt.scatter(d1[0], d1[1], c=\"yellow\")\n", - " plt.scatter(d2[0], d2[1], c=\"g\")\n", - "\n", - " # print the converged log-likelihood value\n", - " return gmm.lower_bound_.mean()\n", - "\n", - " # print the number of iterations needed\n", - " # for the log-likelihood value to converge\n", - " return gmm.n_iter_.mean()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "74afb615", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Results: 'model' \n", - " Best score: 4.6114458394573585 \n", - " Best parameter set:\n", - " 'n_components' : 9 \n", - " Best iteration: 6 \n", - " \n", - " Random seed: 737808454 \n", - " \n", - " Evaluation time : 1.2670788764953613 sec [98.78 %]\n", - " Optimization time : 0.015654563903808594 sec [1.22 %]\n", - " Iteration time : 1.28273344039917 sec [15.59 iter/sec]\n", - " \n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiQAAAGgCAYAAACaOnwjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/av/WaAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAzklEQVR4nO3df3RU1b3//9ckgYSYH4gFEkKCNlCsCKIVNdor9IqlNe0X9HOtuhC01d7W4g+67qLCvd5r7S/sr0+53rZW7UfwR5VbFLUtKiJVWVWsqKWNWJWklIRA0KqZJASSMLO/fxwSkmEmmXNy5pwzM8/HWrPinHkf9t4zmZy35+yz3yFjjBEAAICPcvzuAAAAAAkJAADwHQkJAADwHQkJAADwHQkJAADwHQkJAADwHQkJAADwHQkJAADwHQkJAADwHQkJAADw3bASkttvv12hUEhLly5NGLNmzRqFQqEBj4KCguE0CwAAMkye0x23bdumu+66SzNmzBgytqSkRG+//Xbf81AoZKutaDSqvXv3qri42Pa+AADAH8YYtbe3a8KECcrJGfwciKOEpKOjQwsXLtQ999yj73znO0PGh0IhlZWVOWlKkrR3715VVlY63h8AAPinqalJEydOHDTGUUKyZMkS1dbWau7cuUklJB0dHZo0aZKi0ajOOOMMfe9739O0adMSxnd1damrq6vveW9B4qamJpWUlDjpMgAA8FhbW5sqKytVXFw8ZKzthGTt2rV6/fXXtW3btqTip06dqnvvvVczZsxQOBzWj370I5177rnasWNHwmxp5cqVuu22247ZXlJSQkICAECaSWa6Rcj0nn5IQlNTk84880xt2rSpb+7InDlzNHPmTK1atSqpf6Onp0cf//jHdcUVV+jb3/523JjYMyS9GVY4HCYhAQAgTbS1tam0tDSp47etMySvvfaa3n33XZ1xxhl92yKRiLZs2aKf/vSn6urqUm5u7qD/xogRI3T66aervr4+YUx+fr7y8/PtdA0AAKQxWwnJBRdcoLq6ugHbvvjFL+rkk0/WzTffPGQyIlkJTF1dnS666CJ7PQUAABnLVkJSXFysU089dcC24447TieccELf9sWLF6uiokIrV66UJH3rW9/SOeeco8mTJ6u1tVU//OEPtXv3bl177bUuDQEAAKQ7x+uQJNLY2DjgXuMPP/xQX/7yl9XS0qLjjz9en/jEJ/TSSy/plFNOcbtpAACQpmxNavWLnUkxAAAgGOwcv6llAwAAfEdCAgAAfEdCAqSbjrB08SelGVXWz46w3z0CgGFzfVIrgBQ6a7K0reHo87omqXi0NKtaeiXx2j4AEHScIQHSRWwy0t+2But1AEhTJCRAOugIJ05Gem1r4PINgLRFQgKkg0W17sYBQMCQkADpoKHR3TgACBgSEiAdVFe5GwcAAUNCAqSDBza4GwcAAUNCAqSDolLr1t7BzKq24gAgDZGQAOnilfrESQnrkABIcyQkQDp5pV5qb5UWnCdNr7R+treSjABIe6zUCqSbolLpsT/43QsAcBVnSAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAAAgO9ISAA3dYSliz8pzaiyfnaE/e4RAKSFPL87AGSMsyZL2xqOPq9rkopHS7OqpVfqfesWAKQDzpAAbohNRvrb1mC9DgBIiIQEGK6OcOJkpNe2Bi7fAMAgSEiA4VpU624cAGQhEhJguBoa3Y0DgCxEQgIMV3WVu3EAkIVISIDhemCDu3EAkIVISIDhKiq1bu0dzKxqKw4AEBcJCeCGV+oTJyWsQwIAQyIhAdzySr3U3iotOE+aXmn9bG8lGQGAJLBSK+CmolLpsT/43QsASDucIQEAAL4jIQEAAL4jIUH6orIuAGQM5pAgPVFZFwAyCmdIkH6orAsAGYeEBOmFyroAkJFISJBeqKwLABmJhATphcq6AJCRSEiQXqisCwAZiYQE6YXKugCQkUhIkF6orAsAGYmEBOmHyroAkHFISJCeqKwLABmFlVqRvqisCwAZgzMkAADAdyQkAADAdyQkQDagMjKAgBtWQnL77bcrFApp6dKlg8atW7dOJ598sgoKCjR9+nQ9+eSTw2kWgB1nTbYqIT/+olUV+fEXrecUIQQQII4Tkm3btumuu+7SjBkzBo176aWXdMUVV+iaa67Rn/70Jy1YsEALFizQG2+84bRpAMmiMjKANOEoIeno6NDChQt1zz336Pjjjx809r//+7/1mc98RsuWLdPHP/5xffvb39YZZ5yhn/70p446DCBJVEYGkEYcJSRLlixRbW2t5s6dO2Ts1q1bj4mbN2+etm7dmnCfrq4utbW1DXgAsInKyADSiO11SNauXavXX39d27ZtSyq+paVF48ePH7Bt/PjxamlpSbjPypUrddttt9ntGoD+qIwMII3YOkPS1NSkm266Sb/61a9UUFCQqj5pxYoVCofDfY+mpqaUtQVkLCojA0gjts6QvPbaa3r33Xd1xhln9G2LRCLasmWLfvrTn6qrq0u5ubkD9ikrK9P+/fsHbNu/f7/KysoStpOfn6/8/Hw7XQMQ64EN1t00ycQBgM9snSG54IILVFdXp+3bt/c9zjzzTC1cuFDbt28/JhmRpJqaGm3evHnAtk2bNqmmpmZ4PQcwOCojA0gjts6QFBcX69RTTx2w7bjjjtMJJ5zQt33x4sWqqKjQypUrJUk33XSTZs+erR//+Meqra3V2rVr9eqrr+ruu+92aQgAEnqlPvGtv1RGBhAgrq/U2tjYqH379vU9P/fcc/XQQw/p7rvv1mmnnaZHHnlEjz/++DGJDYAUoTIygDQQMsYYvzsxlLa2NpWWliocDqukpMTv7gAAgCTYOX5TywYAAPiOhAQAAPiOhAQYTEujVFYsFeRaP1tYRAwAUsH2Sq1A1jhuhNR5+Ojz/R1S+SSpME860ONfvwAgA3GGBIgnNhnpr/Ow9ToAwDUkJECslsbEyUivzsNcvgEAF5GQALFmTnM3DgAwJBISIFZrp7txAIAhkZAAsUYXuhsHABgSCQkQa/sOd+MAAEMiIQFilVVZt/YOpjDPigMAuIKEBIjnQE/ipIR1SADAdSQkQCIHeqR9u6XxRVJ+jvVz326SEQBIAVZqBQZTViW1tPvdCwDIeJwhAQAAviMhAQAAviMhQTDU10kjc6RQyPpZX+d3j4KrIyxd/ElpRpX1syPsd48ApEDkUFjNz3xS+zZXqfmZTypyyP3vuhdtJCtkjDG+tZ6ktrY2lZaWKhwOq6SkxO/uwG05ISneb2FIUjTwv57eOmuytK3h2O2zqqVX6r3vD4CUaN40WRM+1aBQv5me5rC097lqVVzoznfdizbsHL85QwJ/JUpGJGt7TsjL3gRbomREsrafNdnb/gBIieZNkzVhboOUG/NCrjRhboOaNw3/u+5FG3ZxhgT+qa+TpswYOm7nX6TJ01PfnyDrCEvFo4eOa2+VikpT3RsAKRI5FFZO3mgp17qCHcsYSREperhVuQXOvutetNGLMyRID6ec5m5cJltU624cgEBq2VKrUF78REGytofyrLggt+EECQn805Pkyblk4zJZQ6O7cQACKSc3ue9wsnF+teEECQn8MyLJ+SHJxmWy6iTr5iQbByCQopHkvsPJxvnVhhPMIYF/mEOSPOaQAFmBOSSAHyZPt27tHUxIJCOSlWTMqh48ZlY1yQiQ5nILSrX3Oeu7Hnu6oPf53ueqh5UoeNGGEyQk8FfUJE5KWIdkoFfqEyclrEMCZIyKC+u199lqKRLzQkTa+6w7a4R40YZdXLJBMNTXWXfT9Bhrzsibf+bMSCIdYetumoZGa87IAxs4MwJkoMihsFq21Cont1HRSJXKzt/g+lmLVLdh5/hNQgIAAFKCOSQAACCtkJAAAADfkZAgGLyoYOukDSrrAoAn8oYOAVIstmhcXZO15oabd444acOLfgEAJHGGBH7zooKtkzaorAsAnuIuG/jHi9VHnbTBqqgA4AruskF68KKCrZM2qKwLAJ4jIYF/vKhg66QNKusCgOdISOAfLyrYOmmDyroA4DnmkMA/zCEBgIzGHBKkBy8q2Dppg8q6AOA5EhL4y4sKtk7aoLIuAHiKhAT+e6Xeuvyx4DxpeqX1s73V3YO+kza86BcAQBJzSAAAQIowhwQAAKQVEhIAAOA7EpKgCWJ1WarkAvBLd4f0q4ul/zvD+tnd4XePkCLMIQmSRAXd/Lyrw0mfgjgOAOnnf86SfrBN2tNv20RJ35gl3fCKX72CDcwhSUdBrC5LlVwAfvmfs6SbYpIRSWqWtf1/zvKjV0ghzpAEQRBXBmWFUwB+6e6QqouPTUZ6hSRVSGpol0YWedgx2MUZknQTxOqyVMkF4Jd1ixInI5JkZL2+bpFHHYIXSEiCIIjVZamSC8Av+xNc9nUah7RAQhIEQawuS5VcAH4ZP0QtKbtxSAvMIQmCIM69YA4JAL/0ziFplnV5JhZzSNIGc0jSTRCry1IlF4BfRhZZt/ZKVvLRX+/zb8wiGckwJCRBEcTqslTJBeCXG16R/nuWdSakvwpZ21mHJONwySZoOsLWXSgNjdZciwc2+H9GwUmfgjgOAOmnu8O6m2Z/gzVn5NIHODOSRuwcv0lIAABASjCHBAAApBUSEgAA4DsSEqRGS6NUViwV5Fo/W4ZYDM1uvBTMisJB7BMApAFbCcmdd96pGTNmqKSkRCUlJaqpqdFTTz2VMH7NmjUKhUIDHgUFBcPuNALuuBFS+SRpf4fUFbV+lk+ytrsRL1lF+opHS4+/KNU1WT+LR/tbvC+IfQKANGErIZk4caJuv/12vfbaa3r11Vf1z//8z5o/f7527NiRcJ+SkhLt27ev77F79+5hdxoBdtwIqfNw/Nc6Dx+bZNiNl4JZUTiIfQKANJJnJ/jzn//8gOff/e53deedd+rll1/WtGnT4u4TCoVUVlbmvIdIHy2NiZOLXp2Hj1yeqbIfL1mXQBId+Htta7DivFzVNmh9AoA043gOSSQS0dq1a3XgwAHV1NQkjOvo6NCkSZNUWVk55NmUXl1dXWpraxvwQBqYGT8pTRhnN14KZkXhIPYJANKM7YSkrq5ORUVFys/P11e/+lU99thjOuWUU+LGTp06Vffee6+eeOIJPfjgg4pGozr33HO1Z89gdaWllStXqrS0tO9RWVlpt5vwQ2unvTi78VIwKwoHsU8AkGZsJyRTp07V9u3b9cc//lHXXXedrrrqKr355ptxY2tqarR48WLNnDlTs2fP1vr16zV27Fjdddddg7axYsUKhcPhvkdTU5PdbsIPowvtxdmNl4JZUTiIfQKANDPslVrnzp2r6urqIZOMXpdeeqny8vL08MMPJ90GK7WmiZZG6+6YoezbfXQOiZ14KZgVhYPYJwAIAE9Xao1Go+rq6koqNhKJqK6uTuXl5cNtFkFUViUVDjFPujDvaHJhN14KZkXhIPYJANKMrYRkxYoV2rJli/7+97+rrq5OK1as0PPPP6+FCxdKkhYvXqwVK1b0xX/rW9/SM888o7/97W96/fXXdeWVV2r37t269tpr3R0FguNAT+IkozDPen048VIwKwoHsU8AkEZs3fb77rvvavHixdq3b59KS0s1Y8YMbdy4URdeeKEkqbGxUTk5R3OcDz/8UF/+8pfV0tKi448/Xp/4xCf00ksvJZwEiwxxoMe6HDNzmjUhdXShtH3HwDMdw4mXrAN80CoKB7FPAJAmqPYLAABSgmq/AAAgrZCQAAAA39maQ4LUi/R0qGXrIuV0NihaWK2ymgeUO6LI3UY6WqT/nSmNapUOjpYu2y4Vuby8v902ujuk3yySOhqkomrp/3tAGjnEuJ3sY5cX71WmiByUdi2TojulnCnSST+Uckf534YX/QIwfCYNhMNhI8mEw2G/u5JSe56ZZaLNMsYcfUSbZfY8M8u9Ru4rNKZnYBumR9Z2v9q4b5YxjTHxjbK2J2zDwT6pHkc2e2u+MYdj3qvDsrb72YYX/QKQkJ3jNwlJQOx5ZpaJRmSikZiE5Mg2V5KS+wqNicp69P8D3bvNjQOt3Tbum2VMRNajf3zvtngJhpN9Uj2ObPbW/MHfKzcO/k7a8KJfAAZl5/jNXTYBEOnpUM57xVKZFIozq8dEJe2TouPanV++6WiRCsqlXEmhOK8bSRFJh/Y5vyRht43uDml/sVSh+LOZopL2SCprP3opxsk+qR5HNosclFRofRaDvVehTueXSZy04UW/AAyJu2zSTMvWRQpNiJ+MSNb2UIUV59j/zrRmDMX746wj2/OOxHnVxm8WSZVK/FuYI6nqSFwvJ/vY5cV7lSl2LUucuElH36tdy7xtw4t+AXAVCUkA5HQ2uBoX16hWd+PcaKMjyfH0j3Oyj11evFeZIrrT3Ti32vCiXwBcRUISANHCIeqg2IyL6+Bod+PcaKMoyfH0j3Oyj11evFeZImeKu3FuteFFvwC4ijkkAcAcEjGHJF0xhwTAIJhDkmZyRxRp745Zko4kH/30Pt/75qzhrUdSVCY9VHjkH415rff5Q4XDO8DabWNkkfScNW7FjLvv+fOzBiYWTvaxy4v3KlPkjpLq51v/nei9apg/vIO+kza86BcAV5GQBETFha9o7+ZZUkvMC/ukvZtnqeLCV4bfyOID0gOF1v8Z9heRtX3xAe/bWPyK9OAsqTkmfo+s7YvjjNvJPnZ58V5liqmPS+/MPzZBjMjaPvVxf9rwol8AXMMlm4BhpVZWak1brNQKIIad4zcJCQAASAnmkAAAgLRCQgIAAHxHQoLU6O6QHrlYWjPD+tnd4W48kKzuD6T66dKeE6yf3R/43SNnMmUcQALMIYH77j9L+tQ2a4n3Xk2ybteNdweM3XggWY1lUuX+gWuRGElN46Wq2FvaAixTxoGswxwS+Of+s6Qrt1kLl/VXIWv7/WcNLx5IVu9BPJ7K/dbr6SBTxgEMgYQE7unusM50SMf+ZvU+n7Pt6OUYu/FAsro/OHoQj12ptfd55f7gX/bIlHEASSAhgXvsVuL1onIvslPjbOuAPVi139CRuCDLlHEASSAhgXvsVuL1onIvslPBXnfj/JIp4wCSQEIC99itxOtF5V5kp0MT3I3zS6aMA0gCd9nAPXYr8XpRuRfZqfsDacQJ1n8nqvYrST3vSyPHeNUr+zJlHMha3GUDf9itxOtF5V5kp5FjrFtipcTVfpvGB/8gninjAJJAQgJ32a3E60XlXmSnqpajB/NY6bR+R6aMAxgCl2yQGnYr8XpRuRfZqfsD6y6Ugr3WXIuqF9LzjEKmjANZhWq/AADAd8whAQAAaYWEBAAA+C7P7w6kk0hPh1q2LlJOZ4OihdUqq3lAuSPcnedgt43uzhaFn5+pghGtOtQzWqVztmtkof+1Lbx4r5Ba3V1hvfZMrXIONio6qkqf+PQGjcwv9btbqedkrkbkoLRrmRTdKeVMkU76oZQ7ypv+utknu/t4Me7ug9L9y6TmnVLFFGnxD6WRPr+3SA2TBsLhsJFkwuGwb33Y88wsE22WMeboI9oss+eZWb610fp0oYn2xMT3yLQ+Xehan5zw4r1Cam1dV20O7xn4GR7eI7N1XbXPPUux3eONiQ4ct4nK2p7IW/ONORyzz2FZ2/3ipE929/Fi3CvnG1MuY9TvUS5rO9KCneM3CUkS9jwzy0QjMtFIzEH2yDY3DrR222h9utBEozLRmD+evdv8Skq8eK+QWlvXVQ/6GWZsUtKbjMRLSBIlJW/NH3wfP5ISJ32yu48X414535hQTDIiWdtCJCXpws7xm7tshhDp6VDOe8VSmRSKM+PGRCXtk6Lj2h1fkrDbRndni0aMLJdypVCc1RuNkRSRerr3eXr5xov3CqnV3RVW7j9GK6c88WcY2StFx7Zm1uUbJyuiRg5KKrRm4iXaJyIp1Ond5RsnfbK7jxfj7j4onVgo7UvwekhSuaRdnVy+CTjusnFRy9ZFCk2I/8dZsraHKqw4r9oIPz9Tobz4yYhkbQ/lWXFe8uK9Qmq99kytcisG/wzzJlpxGcVJVd1dy6TcIfbJOxLnFSd9sruPF+O+f1niZESykp69R+KQMUhIhpDTmVyl2WTj3GijYERrUvHJxrnFi/cKqZVzsNHVuLThpKpudGdy+yQb5wYnfbK7jxfjbk5y32TjkBZISIYQLUyu0myycW60cahndFLxyca5xYv3CqkVHVXlalzacFJVN2dKcvskG+cGJ32yu48X465Ict9k45AWmEMyBOaQJI85JOmPOSRiDslQ+3g5h6RFxxYVlJhDkkaYQ+Ki3BFF2rvDqkhrYirS9j7f++asYR1g7bYxsrBMbZsLrddjvqy9z9s2F3q+HokX7xVSa2R+qbZttc5gJfoMX325OrOSEclZVd3cUVL9/MH3aZjv7XokTvpkdx8vxj1ylHTjkTZik57e5zfMJxnJNCm+48cVft/2a0yCtTX2eLAOySBtpNU6JC6/V0iteOuQ9DRl8C2/vdxah6QngOuQDNUnu/t4Me5465BM4JbfdMJtvynCSq3JY6XW9MdKrazUykqtGC6q/QIAAN8xhwQAAKQVEhIAAOA7qv0GDPNUAH8Fdu7MwWbpg+lSUbvUUSyNqZNGVaRfG0ACzCEJkOZNZ2nCtG0K9Vt7yeyV9u6YpYoLX/GljfDG41RyQadC/VJXc9i6rbh03gFX+gQExcuPTNasmgbl9jsGR5qlbVurdc6/1PvXsffzpTHdA2+BNZI+GCmd0JU+bSDrMIckDTVvOksTLtgmxZ54KJMmXLBNzZvO8ryN8MbjVPLpTqtuRX+5UsmnOxXeeNyw+wQExcuPTNbZlzQop3zg9pxy6exLGvTyI5P96VhvohDPmG7r9XRoAxgCZ0gCgNVgAX8FdoXag81SwUTrvwdbQfbQHueXVrxoA1mLMyRphorCgL8CW+X4g+nJVSH+YHqw2wCSQEISAFQUBvwV2CrHRe3uxvnVBpAEEpIAoKIw4K/AVjnuKHY3zq82gCQwhyQAmEMC+Is5JMwhQWowhyTNUFEY8FdgqxyPqrBuu5USV9b9YOTwEgUv2gCSQEISEBUXvqK9m2dJLTEv7JP2bnZnHRK7bZTOO6C2ZwqlSEx8RGp7hnVIkFnO+Zd6/XF9taL7Bm6P7JX+uN7HdUhO6DqaMMRya40QL9oAhsAlm4BhpVbAX6zUykqtcA/VfgEAgO+YQwIAANIKCQkAAPBd1lb7dTJXw4v5HV6wOyckm9+rgx0faP2/z1Zkz17lTpygS773gkYVjXG1jQ9a6vX0Zaeo+P0etZ8wQp/53zc1pixx3RQv+iRJHW2Nqn9qmkYXdKr1UKEmf3aHikoSr8PhZO6F3TYC64M6aflp0i4jnRSSbv+zNGaQlU27w9ITtdJ7jdLYKmn+BmnkEPNUOhqlVdOkpk6pslBaukMqGuS9ctKGFyIHpV3LpOhOKWeKdNIPpdxR7sUHVfdB6f5lUvNOqWKKtPiH0sghxuFkn3RmbPj5z39upk+fboqLi01xcbE555xzzJNPPjnoPr/+9a/N1KlTTX5+vjn11FPNhg0b7DRpjDEmHA4bSSYcDtveN549z8wy0WYZY44+os0ye56Z5eo+QdT6dKGJ9sSMo0em9enCuPHZ/F7dd/l401wkY3T00Vwkc9/l411rY/1pIdMTGthGT0hm/Wkh3/pkjDHvPJIX9/fknUfy4sZvXVdtDu8ZGH94j8zWddWutRFYnx/4efQ9Pp/gz+vd1cZMjImdKGt7IjfkGZMbs0+urO1uteGFt+Ybc3jgZ24Oy9ruRnxQrZxvTHnM51Eua7ub+wSQneO3rYTkN7/5jdmwYYN55513zNtvv23+/d//3YwYMcK88cYbceNffPFFk5uba37wgx+YN99809xyyy1mxIgRpq6uzk6zriYke56ZZaIRmWgk5g/hkW3xDppO9gmi1qcLTTQqE43GjOPIttikJJvfq/suH28ikonEHGR6t7mRAKw/LWSi1jIXA9ro3RablHjRJ2OOJAqD/J7EJgxb11UP+pnHS0rsthFYiZKRREnJ3dXGhOLEhY484iUMN+QN3kZsUuKkDS+8Nd+YqKxH/wSjd1tskmE3PqhWzh/884iXYDjZJ6BSlpDEc/zxx5tf/vKXcV/7whe+YGprawdsO/vss81XvvIVW224lZAc7m430eZj/3AO+AO6R+Zwd/uw9gmirgP7TLTn2APAgANBj0zXgX3GmOx+rzrb3zfNRcce+PsnAHuKZTrb33fcxvv7dpqe0LHJSP+kpCck8/6+nZ71yRhj2sO7k/o9aQ/vNsYY03Wo1RzeM/hn3tMk03Wo1XEbgfX+XwZPFHof7//Fiu9qPfasRezBplJWXK/23ceeGYl95MqKc9qGFw53Wmc2EnzmJipjemTFOYkPqq7OY89yxH4eE2TFDWefALNz/HY8qTUSiWjt2rU6cOCAampq4sZs3bpVc+fOHbBt3rx52rp166D/dldXl9ra2gY83OCkqq4XlXi9YLd6bza/V+v/fbYmdCSe8Z0jqaLdinPq6ctOUZ4ZvMBqnrHivOqTJNU/NS2p35P6p6ZJclYl124bgbX8NHtxT9RKewaJM5KajsT1WjXt2IUJY0WOxDltwwu7lkm5GuIX/kick/igun+ZtG+Q142kvUfihrNPhrCdkNTV1amoqEj5+fn66le/qscee0ynnHJK3NiWlhaNHz9+wLbx48erpSV2qdCBVq5cqdLS0r5HZWWl3W7G5aSqrheVeL1gt3pvNr9XkT17XY2Lp/j9HltxXvRJkkYXdNqKc1Il124bgbXL2It7L8lKwf3jmpJ8D3rjnLThhehOe3F244OqOcn+9Y9zsk+GsJ2QTJ06Vdu3b9cf//hHXXfddbrqqqv05ptvutqpFStWKBwO9z2amppc+XedVNX1ohKvF+xW783m9yp34gRX4+JpP2GErTgv+iRJrYcKbcU5qZJrt43AOinR/74niBub5N1D/eMqk3wPeuOctOGFnCn24uzGB1VFkv3rH+dkn0wx3OtDF1xwgfnXf/3XuK9VVlaan/zkJwO2/dd//ZeZMWOGrTaYQzJ8zCFJHnNImEOSFKdzSOJNVnR7DomdNryQ7XNIBvs8Es0hsbNPgHkyh6RXNBpVV1f8wks1NTXavHnzgG2bNm1KOOck1ZxU1fWiEq8X7Fbvzeb3alTRGD37OetSY8ww+p5vrh0/rLU/xpRN1m9nWP/nHHviv/f5b2eE+tYj8aJPklRUUqX6J6zliRL9ntQ/kde3VoiTKrl22wisMdOlzw8R83kdXY9kZKn0X0fODsaeXOl9/p/VA9cKKaqSvjbEclFfyzu6HomTNryQO0qqn2/9d6Jf+Ib5R9cXsRsfVCNHSTceGUeiz+OG+QPXFnGyT6awk+ksX77cvPDCC2bXrl3mL3/5i1m+fLkJhULmmWeeMcYYs2jRIrN8+fK++BdffNHk5eWZH/3oR+avf/2rufXWW32/7deYBOtk7HGwtsYQ+wSRK+uQZMl7FW/Njz3FwVuHxO0+GePOOiQ9TaxDEle8NUIqlfp1SIZqwwvx1hXpkb11SAaLD6p4a4pMkP11SIbaJ4DsHL9tFde75pprtHnzZu3bt0+lpaWaMWOGbr75Zl144YWSpDlz5ujEE0/UmjVr+vZZt26dbrnlFv3973/XlClT9IMf/EAXXXSRraQpFcX1snn1UVZqTR4rtbJSa1JYqTV5rNSaVSu1Uu0XAAD4jmq/AAAgrZCQAAAA32VttV8nMmVeBILF7pwQr+aQ2OVkDondfbxow5O5F0Gd35EpMmXeSbZJ8QRbV7h9l40TmVLBFsFit3qvV9V+7XJS7dfuPl604UmV3KBW4s0UmVIhOEN4WlzPC34nJJlSwRbBYrd6r1fVfu1yUu3X7j5etOFJldygVuLNFJlSITiDpOy2X7/4eZdNpKdDOe8VS2XxC4iZqKR9UnRcO5dvkLSDHR/ow/ITVJagYF5U0r5iacze9zWqaIzteK90d4WV+4/RyilP/P2I7JWiY1v7LpPY3ceLNtQdlqpHJy5MF5I0UVJ9q/NLK160kc0iByUVWl+QeKv6G1mFCEOdXL7xEHfZuChTKtgiWOxW7/Wq2q9dTqr92t3HizY8qZIb1Eq8mSJTKgRnMRKSIWRKBVsEi93qvV5V+7XLSbVfu/t40YYnVXKDWok3U2RKheAsRkIyhEypYItgsVu916tqv3Y5qfZrdx8v2vCkSm5QK/FmikypEJzFmEMyBOaQIBWYQxLQOSTNOraYm+TuHJJUtpHNmEMSSMwhcVGmVLBFsNit3utVtV+7nFT7tbuPF214UiU3qJV4M0WmVAjOZim+48cVft/2a0zmVLBFsNit3utVtV+7nFT7tbuPF214UiU3qJV4M0WmVAjOENz2myKs1IpUYKVWVmplpVaXsVJrYFDtFwAA+I45JAAAIK2QkAAAAN9R7Rfwmd05IR+01Ovpy05R8fs9aj9hhD7zv29qTNlkV9vwSkdbo+qfmqbRBZ1qPVSoyZ/doaKSxOtwOJlDYnu+RkejtGqa1NQpVRZKS3dIRUOsDRLEOSFB7FNQBXXOSfdB6f5lUvNOqWKKtPiH0sgA9CtVUjzB1hVBuMsGSAW71XvXnxYyPTHF2XpCMutPC7nWhlfeeSTPRHti7lzrkXnnkby48U6q/dqurHtDnjG5MfG5sra71YYXgtinoApqdeCV840pj/kMy2VtTyNU+wXSgN3qvetPC5motYzGgPjebfGSkqBWCH7nkTwTjcpEY6qy9m6LTUqcVPu1XVn3hrxjY/s/4iUlQazeG8Q+BVVQqwOvnD/4Z5hGSQm3/QIBZ3fl1Q9a6lUyYYpyzSCLUIaktr07+y7fBHV11462Rh1XOEnKlUJxBmOOrKh5oHO3ikqqHK3UaruybkejNHqStZJnIrmSWncfvXwTxOq9QexTUAV1Zdfug9KJhdK+BK+HJJVL2tWZFpdvuMsGCDi71XufvuwU5SVIRqQjhUyNFee0Da/UPzVNobz4yYhkbQ/lWXGSs2q/tivrrpo2eDIiWa+vmua8DS8EsU9BFdTqwPcvS5yMSNZnuPdIXIYhIQF8YLd6b/H7PUnF948LaoXg0QWdtuKcVPu1XVm3Kbk+DYgLYvXeIPYpqIJaHbg5yfaSjUsjJCSAD+xW720/YURS8f3jglohuPVQoa04J9V+bVfWrUyuTwPigli9N4h9CqqgVgeuSLK9ZOPSCHNIAB8wh8SjOSTJVtYdzhySIFXvDWKfgiroc0halPgzZA4JALfYrd47pmyyfjvD+quZqJDpb2eEBqxHEtQKwUUlVap/wloCKfZ/h3qf1z+R17ceiZNqv7Yr6xZVSV8bYlmmr+UNXI8kiNV7g9inoApqdeCRo6Qbj/Qr0Wd4w/y0SEZsS/EdP67gtl9kKrvVe91ahyQIFYLdWIdkqGq/tivrurUOid/Ve4PYp6AKanXgeOuQTEivW36N4bZfIK2wUisrtaZEEPsUVKzUmjJU+wUAAL5jDgkAAEgrJCQAAMB3VPtF2jp4MKw1G2t1yDSqIFSlq+dt0KhR6XeN3Iv5HfXbn1bz/M9qfIe0v0iqeOIpTZ75GVfbcDS/wwt251Iw9wLwBXNIkJZWrZ+s/3NBgyr7HSeawtKjm6u19JJ6/zpm0/1XlGnu7/ZrQsfRbXuLpGc/N16LH25xpY2/jAtp+nsD7yA0kurGSjPedefr//IjkzWrpkG5FUe3RZqlbVurdc6/+Ph53DNZ+lbDwOXUJ8q6NfbLcfplNx7AoJhDgoy2av1k3XhxgypifrcrSqQbL27QqvWD33ESFPdfUaYr1+5XWcfA7WUd0pVr9+v+K8qG3UZvMhLP9Pes14fr5Ucm6+xLGpRTPnB7Trl09iUNevkRnz6PeyZLX2k4trZLs6zt90weXjwAV3GGBGnl4MGw/tE9WhUlUk6cY2nUSHvapLEjWwN9+caLVVTrtz+t6tM/KynxQpSS1PAn55dvHK2i6gW7VW+pkgukBGdIkLHWbKxVZWn8ZESytleVWnFB5kUl3ub5n1VIgxczDR2Jc8pRJV4v2K16S5VcwHckJEgrh0xyVUqTjfOLF5V4x3cMHWMnLh5HlXi9YLfqLVVyAd+RkCCtFISSq1KabJxfvKjEu7/I3bh4HFXi9YLdqrdUyQV8xxwSpBXmkCSPOSSjk696S5VcICWYQ4KMNWpUqR7dbFUzjcYcOHqfr99cHehkRPKmEu/kmZ9R3VjrvxMVM60bq2GtR+KoEq8X7Fa9pUou4DsSEqSdpZfU647HqtXcNnD7njbpjsfSZx2SxQ+36MHLx6sl5pLJvmLpwcvdWYdkxrumLymJ5dY6JOf8S73+uL5a0X0Dt0f2Sn9c7+M6JF+ul+6qlipitk+UtT12XRG78QBcxSUbpC1Wak0eK7WyUivgB6r9AgAA3zGHBAAApBUSEgAA4Duq/QIu8mI+iJM2vOiXI8zXABKKHAqrZUutcnIbFY1Uqez8DcotyNzvB3NIAJd4UbnXSRte9MsRKusCCTVvmqwJn2pQqN9pA3NY2vtctSouTJ/vB3NIAI95UbnXSRte9MsRKusCCTVvmqwJcxuk3JgXcqUJcxvUvCkzvx+cIQGGyYtVV5204UW/HKGyLpBQ5FBYOXmjpVwpFGeJZWMkRaTo4da0uHzDGRLAQ15U7nXShhf9coTKukBCLVtqFcqLn4xI1vZQnhWXaUhIgGHyonKvkza86JcjVNYFEsrJTbKCdpJx6YSEBBgmLyr3OmnDi345QmVdIKFoJMkK2knGpRPmkADDxBwSm6isCyTEHBIAjnlRuddJG170yxEq6wIJ5RaUau9zRypoxyTsvc/3PledFsmIXSQkgAu8qNzrpA0v+uUIlXWBhCourNfeZ6ulSMwLEWnvs+m1DokdXLIBXMRKrTaxUiuQUCas1Eq1XwAA4DvmkAAAgLRCQgIAAHxHtV8M6eDBsNZsrNUh06iCUJWunrdBo0a5ex3Tiza84GQcdvcJ7HyQoOpolFZNk5o6pcpCaekOqSgN13Bgvg0ynbHhe9/7njnzzDNNUVGRGTt2rJk/f7556623Bt1n9erVRtZqA32P/Px8O82acDhsJJlwOGxrPwzfTx6tNo2tMsYcfTS2yvzk0eq0asMLTsZhd5/7Lh9vmotkjI4+motk7rt8vPsDygQ35BmTO/D9MrmytqeTu6uNmRgzjomytgMBZuf4bSshmTdvnlm9erV54403zPbt281FF11kqqqqTEdHR8J9Vq9ebUpKSsy+ffv6Hi0tLXaaJSHxyU8erTaRqEwkOvCA2bvNjYTBiza84GQcdve57/LxJiKZiAYemHq3kZTEuCFv4AE89pEuScnd1caE4vQ/dORBUoIAs3P8HtZdNu+9957GjRunF154Qeeff37cmDVr1mjp0qVqbW112gx32fjg4MGw/tE9WhUlUk6c1QKjRtrTJo0d2er40ooXbXjByTjs7hPYVVeDqqNRGj3p2HUc+suV1Lo72JdvqIyMNOfZXTbhcFiSNGbM4H8AOzo6NGnSJFVWVmr+/PnasWPHoPFdXV1qa2sb8IC31mysVWVp/IOlZG2vKrXigtyGF5yMw+4+ga3cG1Srpg2ejEjW66umedEb56iMjCziOCGJRqNaunSpzjvvPJ166qkJ46ZOnap7771XTzzxhB588EFFo1Gde+652rMn8bds5cqVKi0t7XtUVlY67SYcOmSSqySZbJxfbXjByTjs7hPYyr1B1dTpbpxfqIyMLOI4IVmyZIneeOMNrV27dtC4mpoaLV68WDNnztTs2bO1fv16jR07VnfddVfCfVasWKFwONz3aGpqctpNOFQQSu40drJxfrXhBSfjsLtPYCv3BlVlobtxfqEyMrKIozkk119/vZ544glt2bJFJ510ku1GL730UuXl5enhhx9OKp45JN5jDknymEMSQJk2h4TKyEhTKZtDYozR9ddfr8cee0y///3vHSUjkUhEdXV1Ki8vt70vvDNqVKke3WxVnIzG/CHsfb5+c/WwEgUv2vCCk3HY3SewlXuDqqhK+toQyyx9LS/YyYhEZWRkFzu371x33XWmtLTUPP/88wNu4+3s7OyLWbRokVm+fHnf89tuu81s3LjRNDQ0mNdee81cfvnlpqCgwOzYsSPpdrnt1z/x1snY7cE6JG634QUn47C7T7x1SPYUc8tvQpm8Dkklt/wi+FJ2228oFP+WgNWrV+vqq6+WJM2ZM0cnnnii1qxZI0n6+te/rvXr16ulpUXHH3+8PvGJT+g73/mOTj/99KSTJi7Z+IuVWpPHSq0BxEqtgG+o9gsAAHxHtV8AAJBWSEgAAIDvqPaLIWXK/A4vfPBBox7YOk2jR3eqtbVQi2p2aMyYNJyvAAAeYw4JBrVq/WT9nwsaVNkv/2gKS49urtbSS+r961gA/fL5Ebr6/MPK63fe8XBUWrMlT9fO6fGvYwDgE+aQwBWr1k/WjRc3qCLmd6iiRLrx4gatWj/Zn44F0C+fH6FrZh9WbsyNaLkh6ZrZh/XL50f40zEASBOcIUFcmbKKqhc++KBRJaMnKTckxbsz3hgpYqS21t1cvgGQVThDgmHLlEq8Xnhg6zTl5cRPRiRre16OFQcAiI+EBHFlSiVeL4wenVzF2GTjACAbkZAgrkypxOuF1tbkKsYmGwcA2Yg5JIiLOSTJYw4JAMTHHBIMW6ZU4vXCmDFVWrPFWtInNr3vfb5mSx7JCAAMgoQECS29pF53PFat5raB2/e0SXc8xjok/V07p0f/74U8RWISkoiR/t8LrEMCAEPhkg2GxEqtyWOlVgA4imq/AADAd8whAQAAaYWEBAAA+I5qvymUzXMvsnXs2TruQOsOS0/USu81SmOrpPkbpJF8JkDQMIckRbK5Sm62jj1bxx1o90yWvtUg7em3baKk/6qWvsxnAqQac0h8ls1VcrN17Nk67kC7Z7L0lZhkRJKaZW2/h88ECBLOkLgsm1c4zdaxZ+u4A607LFWPPjYZ6RWSdaakvpXLN0AKcYbER9lcJTdbx56t4w60J2oTJyOSZCQ1HYkDEAgkJC7L5iq52Tr2bB13oL2X5HudbByAlCMhcVk2V8nN1rFn67gDbWyS73WycQBSjjkkLsvm+QTZOvZsHXeg9c4haZZ1eSYWc0gATzCHxEfZXCU3W8eereMOtJGl1q29kpV89Nf7/D+rSUaAACEhSYFsrpKbrWPP1nEH2pfrpbuqpYqY7RNlbWcdEiBQuGSTQtm8ame2jj1bxx1orNQK+IZqvwAAwHfMIQEAAGmFhAQAAPiOar8A3NHRKK2aJjV1SpWF0tIdUpHL63wwHwTIWCQkAIbvxhHSzw9Lkd4NHdI3J0lfy5Pu6HGnjWMq9zZJE0dTuRfIEFyyATA8N46Q/qd/MnJERNb2G0cMvw0q9wIZj4QEgHMdjdaZkcH8/LAV51R32DozEu9+wN5t326w4gCkLRISAM6tmnbsmZFYkSNxTlG5F8gKJCQAnGvqdDcuHir3AlmBhASAc5WF7sbFQ+VeICuQkABwbukOKXeImNwjcU7N32DVn4lTSVk6sr3ySByAtEVCAsC5oirr1t7BfC1veOuRULkXyAokJACG544e6Ya8Y8+U5Mra7sY6JFTuBTIexfUAuIOVWgHEsHP8ZqVWAO4oqpJuaU9tGyNLpUv/kNo2APiCSzYAAMB3JCQAAMB3JCQAAMB3JCQAAMB3JCQAAMB3JCQAAMB3JCQAAMB3JCQAAMB3JCQAAMB3JCQAAMB3JCQAAMB3JCQAAMB3JCQAAMB3JCQAAMB3JCQAAMB3JCQAAMB3eX53AJnp4MGw1mys1SHTqIJQla6et0GjRpX63S0AQEDZOkOycuVKzZo1S8XFxRo3bpwWLFigt99+e8j91q1bp5NPPlkFBQWaPn26nnzySccdRvCtWj9Z/+geresWvKivX9yk6xa8qH90j9aq9ZP97hoAIKBsJSQvvPCClixZopdfflmbNm1ST0+PPv3pT+vAgQMJ93nppZd0xRVX6JprrtGf/vQnLViwQAsWLNAbb7wx7M4jeFatn6wbL25QRcnA7RUl0o0XN5CUAADiChljjNOd33vvPY0bN04vvPCCzj///Lgxl112mQ4cOKDf/e53fdvOOecczZw5U7/4xS+SaqetrU2lpaUKh8MqKSkZegf44uDBsP7RPVoVJVJO6NjXo0ba0yaNHdnK5RsAyAJ2jt/DmtQaDoclSWPGjEkYs3XrVs2dO3fAtnnz5mnr1q0J9+nq6lJbW9uAB4JvzcZaVZbGT0Yka3tVqRUHAEB/jhOSaDSqpUuX6rzzztOpp56aMK6lpUXjx48fsG38+PFqaWlJuM/KlStVWlra96isrHTaTXjokGl0NQ4AkD0cJyRLlizRG2+8obVr17rZH0nSihUrFA6H+x5NTU2utwH3FYSqXI0DAGQPR7f9Xn/99frd736nLVu2aOLEiYPGlpWVaf/+/QO27d+/X2VlZQn3yc/PV35+vpOuwUdXz9ugpvDQc0iunrfB+84BAALN1hkSY4yuv/56PfbYY/r973+vk046ach9ampqtHnz5gHbNm3apJqaGns9ReCNGlWqRzdXS7KSj/56n6/fXM2EVgDAMWwlJEuWLNGDDz6ohx56SMXFxWppaVFLS4sOHjzYF7N48WKtWLGi7/lNN92kp59+Wj/+8Y/11ltv6Zvf/KZeffVVXX/99e6NAoGx9JJ63fFYtZpj5iHvaZPueKxaSy+p96djAIBAs3XbbygU//aJ1atX6+qrr5YkzZkzRyeeeKLWrFnT9/q6det0yy236O9//7umTJmiH/zgB7rooouS7iS3/aYfVmoFANg5fg9rHRKvkJAAAJB+PFuHBAAAwA0kJAAAwHckJAAAwHckJAAAwHckJAAAwHckJAAAwHckJAAAwHckJAAAwHckJAAAwHeOqv16rXcx2ba2tiEiAQBAUPQet5NZFD4tEpL29nZJUmVlpc89AQAAdrW3t6u0dPB6ZmlRyyYajWrv3r0qLi5OWOAvqNra2lRZWammpqasq8OTrWPP1nFLjD0bx56t45YYezJjN8aovb1dEyZMUE7O4LNE0uIMSU5OjiZOnOh3N4alpKQk635he2Xr2LN13BJjz8axZ+u4JcY+1NiHOjPSi0mtAADAdyQkAADAdyQkKZafn69bb71V+fn5fnfFc9k69mwdt8TYs3Hs2TpuibG7Pfa0mNQKAAAyG2dIAACA70hIAACA70hIAACA70hIAACA70hIXHT77bcrFApp6dKlCWPWrFmjUCg04FFQUOBdJ13yzW9+85hxnHzyyYPus27dOp188skqKCjQ9OnT9eSTT3rUW3fZHXumfOaS1NzcrCuvvFInnHCCRo0apenTp+vVV18ddJ/nn39eZ5xxhvLz8zV58mStWbPGm866zO7Yn3/++WM+91AopJaWFg97PXwnnnhi3HEsWbIk4T6Z8F23O+5M+p5HIhH953/+p0466SSNGjVK1dXV+va3vz1kPZrhftfTYqXWdLBt2zbdddddmjFjxpCxJSUlevvtt/uep9ty+L2mTZumZ599tu95Xl7iX6eXXnpJV1xxhVauXKnPfe5zeuihh7RgwQK9/vrrOvXUU73orqvsjF3KjM/8ww8/1HnnnadPfepTeuqppzR27Fjt3LlTxx9/fMJ9du3apdraWn31q1/Vr371K23evFnXXnutysvLNW/ePA97PzxOxt7r7bffHrCS5bhx41LZVddt27ZNkUik7/kbb7yhCy+8UJdeemnc+Ez5rtsdt5QZ33NJ+v73v68777xT9913n6ZNm6ZXX31VX/ziF1VaWqobb7wx7j6ufNcNhq29vd1MmTLFbNq0ycyePdvcdNNNCWNXr15tSktLPetbqtx6663mtNNOSzr+C1/4gqmtrR2w7eyzzzZf+cpXXO5Z6tkde6Z85jfffLP55Cc/aWufb3zjG2batGkDtl122WVm3rx5bnYt5ZyM/bnnnjOSzIcffpiaTvnkpptuMtXV1SYajcZ9PZO+6/0NNe5M+Z4bY0xtba350pe+NGDbJZdcYhYuXJhwHze+61yyccGSJUtUW1uruXPnJhXf0dGhSZMmqbKyUvPnz9eOHTtS3MPU2LlzpyZMmKCPfvSjWrhwoRobGxPGbt269Zj3Z968edq6dWuqu5kSdsYuZcZn/pvf/EZnnnmmLr30Uo0bN06nn3667rnnnkH3yZTP3cnYe82cOVPl5eW68MIL9eKLL6a4p6nV3d2tBx98UF/60pcS/t9/pnzm/SUzbikzvueSdO6552rz5s165513JEl//vOf9Yc//EGf/exnE+7jxudOQjJMa9eu1euvv66VK1cmFT916lTde++9euKJJ/Tggw8qGo3q3HPP1Z49e1LcU3edffbZWrNmjZ5++mndeeed2rVrl/7pn/5J7e3tceNbWlo0fvz4AdvGjx+fdtfTJftjz5TP/G9/+5vuvPNOTZkyRRs3btR1112nG2+8Uffdd1/CfRJ97m1tbTp48GCqu+waJ2MvLy/XL37xCz366KN69NFHVVlZqTlz5uj111/3sOfuevzxx9Xa2qqrr746YUwmfdd7JTPuTPmeS9Ly5ct1+eWX6+STT9aIESN0+umna+nSpVq4cGHCfVz5rts7kYP+Ghsbzbhx48yf//znvm1DXbKJ1d3dbaqrq80tt9ySgh5658MPPzQlJSXml7/8ZdzXR4wYYR566KEB2372s5+ZcePGedG9lBpq7LHS9TMfMWKEqampGbDthhtuMOecc07CfaZMmWK+973vDdi2YcMGI8l0dnampJ+p4GTs8Zx//vnmyiuvdLNrnvr0pz9tPve5zw0ak4nf9WTGHStdv+fGGPPwww+biRMnmocfftj85S9/Mffff78ZM2aMWbNmTcJ93Piuc4ZkGF577TW9++67OuOMM5SXl6e8vDy98MILuuOOO5SXlzdgQlQivdlnfX29Bz1OndGjR+tjH/tYwnGUlZVp//79A7bt379fZWVlXnQvpYYae6x0/czLy8t1yimnDNj28Y9/fNDLVYk+95KSEo0aNSol/UwFJ2OP56yzzkq7z73X7t279eyzz+raa68dNC7TvuvJjjtWun7PJWnZsmV9Z0mmT5+uRYsW6etf//qgVwLc+K6TkAzDBRdcoLq6Om3fvr3vceaZZ2rhwoXavn27cnNzh/w3IpGI6urqVF5e7kGPU6ejo0MNDQ0Jx1FTU6PNmzcP2LZp0ybV1NR40b2UGmrssdL1Mz/vvPMG3EEgSe+8844mTZqUcJ9M+dydjD2e7du3p93n3mv16tUaN26camtrB43LlM+8V7LjjpWu33NJ6uzsVE7OwPQgNzdX0Wg04T6ufO7DOq+DY8Reslm0aJFZvnx53/PbbrvNbNy40TQ0NJjXXnvNXH755aagoMDs2LHDh94692//9m/m+eefN7t27TIvvviimTt3rvnIRz5i3n33XWPMseN+8cUXTV5envnRj35k/vrXv5pbb73VjBgxwtTV1fk1BMfsjj1TPvNXXnnF5OXlme9+97tm586d5le/+pUpLCw0Dz74YF/M8uXLzaJFi/qe/+1vfzOFhYVm2bJl5q9//av52c9+ZnJzc83TTz/txxAcczL2n/zkJ+bxxx83O3fuNHV1deamm24yOTk55tlnn/VjCMMSiURMVVWVufnmm495LZO/63bGnSnfc2OMueqqq0xFRYX53e9+Z3bt2mXWr19vPvKRj5hvfOMbfTGp+K6TkLgsNiGZPXu2ueqqq/qeL1261FRVVZmRI0ea8ePHm4suusi8/vrr3nd0mC677DJTXl5uRo4caSoqKsxll11m6uvr+16PHbcxxvz61782H/vYx8zIkSPNtGnTzIYNGzzutTvsjj1TPnNjjPntb39rTj31VJOfn29OPvlkc/fddw94/aqrrjKzZ88esO25554zM2fONCNHjjQf/ehHzerVq73rsIvsjv373/++qa6uNgUFBWbMmDFmzpw55ve//73HvXbHxo0bjSTz9ttvH/NaJn/X7Yw7k77nbW1t5qabbjJVVVWmoKDAfPSjHzX/8R//Ybq6uvpiUvFdDxkzxNJrAAAAKcYcEgAA4DsSEgAA4DsSEgAA4DsSEgAA4DsSEgAA4DsSEgAA4DsSEgAA4DsSEgAA4DsSEgAA4DsSEgAA4DsSEgAA4DsSEgAA4Lv/H51atiF4m0dyAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from hyperactive import Hyperactive\n", - "\n", - "search_space = {\"n_components\": list(range(2, 10))}\n", - "\n", - "hyper = Hyperactive()\n", - "hyper.add_search(model, search_space, n_iter=20)\n", - "hyper.run()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "99435692", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.8" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/optimization_applications/constrained_optimization.py b/examples/optimization_applications/constrained_optimization.py deleted file mode 100644 index 1a1fa779..00000000 --- a/examples/optimization_applications/constrained_optimization.py +++ /dev/null @@ -1,38 +0,0 @@ -import numpy as np - -from hyperactive import Hyperactive - - -def convex_function(pos_new): - score = -(pos_new["x1"] * pos_new["x1"] + pos_new["x2"] * pos_new["x2"]) - return score - - -search_space = { - "x1": list(np.arange(-100, 101, 0.1)), - "x2": list(np.arange(-100, 101, 0.1)), -} - - -def constraint_1(para): - # reject parameters where x1 and x2 are higher than 2.5 at the same time - return not (para["x1"] > 2.5 and para["x2"] > 2.5) - - -# put one or more constraints inside a list -constraints_list = [constraint_1] - - -hyper = Hyperactive() -# pass list of constraints -hyper.add_search( - convex_function, - search_space, - n_iter=50, - constraints=constraints_list, -) -hyper.run() - -search_data = hyper.search_data(convex_function) - -print("\n search_data \n", search_data, "\n") diff --git a/examples/optimization_applications/ensemble_learning_example.py b/examples/optimization_applications/ensemble_learning_example.py deleted file mode 100644 index 428929ca..00000000 --- a/examples/optimization_applications/ensemble_learning_example.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -This example shows how you can search for the best models in each layer in a -stacking ensemble. - -We want to create a stacking ensemble with 3 layers: - - a top layer with one model - - a middle layer with multiple models - - a bottom layer with multiple models - -We also want to know how many models should be used in the middle and bottom layer. -For that we can use the helper function "get_combinations". It works as follows: - -input = [1, 2 , 3] -output = get_combinations(input, comb_len=2) -output: [[1, 2], [1, 3], [2, 3], [1, 2, 3]] - -Instead of numbers we insert models into "input". This way we get each combination -with more than 2 elements. Only 1 model per layer would not make much sense. - -The ensemble itself is created via the package "mlxtend" in the objective-function "stacking". -""" - -import itertools - -from sklearn.datasets import load_breast_cancer -from sklearn.model_selection import cross_val_score -from mlxtend.classifier import StackingClassifier - -from sklearn.ensemble import ( - GradientBoostingClassifier, - RandomForestClassifier, - ExtraTreesClassifier, -) - -from sklearn.neighbors import KNeighborsClassifier -from sklearn.neural_network import MLPClassifier -from sklearn.gaussian_process import GaussianProcessClassifier -from sklearn.tree import DecisionTreeClassifier -from sklearn.naive_bayes import GaussianNB - -from sklearn.linear_model import LogisticRegression -from sklearn.linear_model import RidgeClassifier - -from hyperactive import Hyperactive - -data = load_breast_cancer() -X, y = data.data, data.target - -# define models that are used in search space -gbc = GradientBoostingClassifier() -rfc = RandomForestClassifier() -etc = ExtraTreesClassifier() - -mlp = MLPClassifier() -gnb = GaussianNB() -gpc = GaussianProcessClassifier() -dtc = DecisionTreeClassifier() -knn = KNeighborsClassifier() - -lr = LogisticRegression() -rc = RidgeClassifier() - - -def stacking(opt): - lvl_1_ = opt["lvl_1"]() - lvl_0_ = opt["lvl_0"]() - top_ = opt["top"]() - - stack_lvl_0 = StackingClassifier(classifiers=lvl_0_, meta_classifier=top_) - stack_lvl_1 = StackingClassifier(classifiers=lvl_1_, meta_classifier=stack_lvl_0) - scores = cross_val_score(stack_lvl_1, X, y, cv=3) - - return scores.mean() - - -# helper function to create search space dimensions -def get_combinations(models, comb_len=2): - def _list_in_list_of_lists(list_, list_of_lists): - for list__ in list_of_lists: - if set(list_) == set(list__): - return True - - comb_list = [] - for i in range(0, len(models) + 1): - for subset in itertools.permutations(models, i): - if len(subset) < comb_len: - continue - if _list_in_list_of_lists(subset, comb_list): - continue - - comb_list.append(list(subset)) - - comb_list_f = [] - for comb_ in comb_list: - - def _func_(): - return comb_ - - _func_.__name__ = str(i) + "___" + str(comb_) - comb_list_f.append(_func_) - - return comb_list_f - - -def lr_f(): - return lr - - -def dtc_f(): - return dtc - - -def gnb_f(): - return gnb - - -def rc_f(): - return rc - - -models_0 = [gpc, dtc, mlp, gnb, knn] -models_1 = [gbc, rfc, etc] - -stack_lvl_0_clfs = get_combinations(models_0) -stack_lvl_1_clfs = get_combinations(models_1) - - -print("\n stack_lvl_0_clfs \n", stack_lvl_0_clfs, "\n") - - -search_space = { - "lvl_1": stack_lvl_1_clfs, - "lvl_0": stack_lvl_0_clfs, - "top": [lr_f, dtc_f, gnb_f, rc_f], -} - -""" -hyper = Hyperactive() -hyper.add_search(stacking, search_space, n_iter=3) -hyper.run() -""" diff --git a/examples/optimization_applications/feature_selection.py b/examples/optimization_applications/feature_selection.py deleted file mode 100644 index 02fa79d4..00000000 --- a/examples/optimization_applications/feature_selection.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -This example shows how to select the best features for a model -and dataset. - -The boston dataset has 13 features, therefore we have 13 search space -dimensions for the feature selection. - -The function "get_feature_indices" returns the list of features that -where selected. This can be used to select the subset of features in "x_new". -""" - -import numpy as np -import itertools -from sklearn.datasets import load_diabetes -from sklearn.model_selection import cross_val_score -from sklearn.neighbors import KNeighborsRegressor -from hyperactive import Hyperactive -from hyperactive.optimizers import EvolutionStrategyOptimizer - - -data = load_diabetes() -X, y = data.data, data.target - - -# helper function that returns the selected training data features by index -def get_feature_indices(opt): - feature_indices = [] - for key in opt.keys(): - if "feature" not in key: - continue - if opt[key] == 0: - continue - - nth_feature = int(key.rsplit(".", 1)[1]) - feature_indices.append(nth_feature) - - return feature_indices - - -def model(opt): - feature_indices = get_feature_indices(opt) - if len(feature_indices) == 0: - return 0 - - feature_idx_list = [idx for idx in feature_indices if idx is not None] - x_new = X[:, feature_idx_list] - - knr = KNeighborsRegressor(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, x_new, y, cv=5) - score = scores.mean() - - return score - - -# each feature is used for training (1) or not used for training (0) -search_space = { - "n_neighbors": list(range(1, 100)), - "feature.0": [1, 0], - "feature.1": [1, 0], - "feature.2": [1, 0], - "feature.3": [1, 0], - "feature.4": [1, 0], - "feature.5": [1, 0], - "feature.6": [1, 0], - "feature.7": [1, 0], - "feature.8": [1, 0], - "feature.9": [1, 0], -} - - -optimizer = EvolutionStrategyOptimizer(rand_rest_p=0.20) - -hyper = Hyperactive() -hyper.add_search( - model, - search_space, - n_iter=200, - initialize={"random": 15}, - optimizer=optimizer, -) -hyper.run() diff --git a/examples/optimization_applications/feature_transformation.py b/examples/optimization_applications/feature_transformation.py deleted file mode 100644 index ff589399..00000000 --- a/examples/optimization_applications/feature_transformation.py +++ /dev/null @@ -1,99 +0,0 @@ -""" -This example shows how you can search for useful feature -transformations for your dataset. This example is very similar to -"feature_selection". It adds the possibility to change the features -with the numpy functions in the search space. - -""" - -import numpy as np -import itertools -from sklearn.datasets import load_diabetes -from sklearn.model_selection import cross_val_score -from sklearn.neighbors import KNeighborsRegressor -from hyperactive import Hyperactive - -data = load_diabetes() -X, y = data.data, data.target - - -def get_feature_list(opt): - feature_list = [] - for key in opt.keys(): - if "feature" not in key: - continue - - nth_feature = int(key.rsplit(".", 1)[1]) - - if opt[key] == 0: - continue - elif opt[key] == 1: - feature = X[:, nth_feature] - feature_list.append(feature) - else: - feature = opt[key](X[:, nth_feature]) - feature_list.append(feature) - - return feature_list - - -def model(opt): - feature_list = get_feature_list(opt) - X_new = np.array(feature_list).T - - knr = KNeighborsRegressor(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X_new, y, cv=5) - score = scores.mean() - - return score - - -def log_f(*args, **kwargs): - return np.log(*args, **kwargs) - - -def square_f(*args, **kwargs): - return np.square(*args, **kwargs) - - -def sqrt_f(*args, **kwargs): - return np.sqrt(*args, **kwargs) - - -def sin_f(*args, **kwargs): - return np.sin(*args, **kwargs) - - -def cos_f(*args, **kwargs): - return np.cos(*args, **kwargs) - - -# features can be used (1), not used (0) or transformed for training -features_search_space = [ - 1, - 0, - log_f, - square_f, - sqrt_f, - sin_f, - cos_f, -] - -search_space = { - "n_neighbors": list(range(1, 100)), - "feature.0": features_search_space, - "feature.1": features_search_space, - "feature.2": features_search_space, - "feature.3": features_search_space, - "feature.4": features_search_space, - "feature.5": features_search_space, - "feature.6": features_search_space, - "feature.7": features_search_space, - "feature.8": features_search_space, - "feature.9": features_search_space, -} - - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=150) -hyper.run() diff --git a/examples/optimization_applications/hyperpara_optimize.py b/examples/optimization_applications/hyperpara_optimize.py deleted file mode 100644 index 7d414387..00000000 --- a/examples/optimization_applications/hyperpara_optimize.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -This example shows the original purpose of Hyperactive. -You can search for any number of hyperparameters and Hyperactive -will return the best one after the optimization run. - -""" - -import numpy as np -from sklearn.model_selection import cross_val_score -from sklearn.ensemble import GradientBoostingClassifier -from sklearn.datasets import load_wine -from hyperactive import Hyperactive - -data = load_wine() -X, y = data.data, data.target - - -def model(opt): - gbr = GradientBoostingClassifier( - n_estimators=opt["n_estimators"], - max_depth=opt["max_depth"], - min_samples_split=opt["min_samples_split"], - min_samples_leaf=opt["min_samples_leaf"], - criterion=opt["criterion"], - ) - scores = cross_val_score(gbr, X, y, cv=4) - - return scores.mean() - - -search_space = { - "n_estimators": list(range(10, 150, 5)), - "max_depth": list(range(2, 12)), - "min_samples_split": list(range(2, 25)), - "min_samples_leaf": list(range(1, 25)), - "criterion": ["friedman_mse", "squared_error", "absolute_error"], - "subsample": list(np.arange(0.1, 3, 0.1)), -} - - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=40) -hyper.run() diff --git a/examples/optimization_applications/memory.py b/examples/optimization_applications/memory.py deleted file mode 100644 index c95d6560..00000000 --- a/examples/optimization_applications/memory.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Hyperactive saves all positions it explores in a memory dictionary. If it encounters -this positions again Hyperactive will just read the score from the memory dictionary -instead of reevaluating the objective function. If there is a machine-/deep-learning -model within the objective function this memory saves you a lot of computation -time, because it is much faster to just look up the score in a dictionary instead -of retraining an entire machine learning model. - -You can also pass the search data to the "memory_warm_start"-parameter of the next -optimization run. This way the next optimization run has the memory of the -previous run, which (again) saves you a lot of computation time. -""" -import time -from sklearn.model_selection import cross_val_score -from sklearn.tree import DecisionTreeRegressor -from sklearn.datasets import load_diabetes -from hyperactive import Hyperactive - -data = load_diabetes() -X, y = data.data, data.target - - -def model(opt): - gbr = DecisionTreeRegressor( - max_depth=opt["max_depth"], - min_samples_split=opt["min_samples_split"], - ) - scores = cross_val_score(gbr, X, y, cv=10) - - return scores.mean() - - -search_space = { - "max_depth": list(range(10, 35)), - "min_samples_split": list(range(2, 22)), -} - -c_time1 = time.time() -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=100) -hyper.run() -d_time1 = time.time() - c_time1 -print("Optimization time 1:", round(d_time1, 2)) - -# Hyperactive collects the search data -search_data = hyper.search_data(model) - -# You can pass the search data to memory_warm_start to save time -c_time2 = time.time() -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=100, memory_warm_start=search_data) -# The next run will be faster, because Hyperactive knows parts of the search space -hyper.run() -d_time2 = time.time() - c_time2 -print("Optimization time 2:", round(d_time2, 2)) diff --git a/examples/optimization_applications/meta_data_collection.py b/examples/optimization_applications/meta_data_collection.py deleted file mode 100644 index 1a9c999d..00000000 --- a/examples/optimization_applications/meta_data_collection.py +++ /dev/null @@ -1,39 +0,0 @@ -import pandas as pd -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive - -data = load_iris() -X, y = data.data, data.target - - -def model1(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=10) - score = scores.mean() - - return score - - -search_space = {"n_neighbors": list(range(1, 50)), "leaf_size": list(range(5, 60, 5))} - - -hyper = Hyperactive() -hyper.add_search(model1, search_space, n_iter=500, memory=True) -hyper.run() - -search_data = hyper.search_data(model1) -# save the search data of a model for later use -search_data.to_csv("./model1.csv", index=False) - - -# load the search data and pass it to "memory_warm_start" -search_data_loaded = pd.read_csv("./model1.csv") - -hyper = Hyperactive() -hyper.add_search( - model1, search_space, n_iter=500, memory=True, memory_warm_start=search_data_loaded -) -hyper.run() diff --git a/examples/optimization_applications/meta_learning.py b/examples/optimization_applications/meta_learning.py deleted file mode 100644 index db4b92d6..00000000 --- a/examples/optimization_applications/meta_learning.py +++ /dev/null @@ -1,90 +0,0 @@ -import random -import numpy as pd -import pandas as pd - -from sklearn.datasets import load_iris -from sklearn.datasets import make_classification -from sklearn.neighbors import KNeighborsClassifier -from sklearn.ensemble import GradientBoostingRegressor -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 80)), -} - - -search_data_list = [] - -for i in range(25): - n_samples = random.randint(100, 1000) - n_features = random.randint(3, 20) - n_informative = n_features - random.randint(0, n_features - 2) - - X, y = make_classification( - n_samples=n_samples, - n_classes=2, - n_features=n_features, - n_informative=n_informative, - n_redundant=0, - random_state=i, - ) - - hyper = Hyperactive(verbosity=False) - hyper.add_search(model, search_space, n_iter=10) - hyper.run() - - search_data = hyper.search_data(model) - - search_data["size_X"] = X.size - search_data["itemsize_X"] = X.itemsize - search_data["ndim_X"] = X.ndim - - search_data["size_y"] = y.size - search_data["itemsize_y"] = y.itemsize - search_data["ndim_y"] = y.ndim - - search_data_list.append(search_data) - - -meta_data = pd.concat(search_data_list) - -X_meta = meta_data.drop(["score"], axis=1) -y_meta = meta_data["score"] - - -gbr = GradientBoostingRegressor() -gbr.fit(X_meta, y_meta) - -data = load_iris() -X_new, y_new = data.data, data.target - -X_meta_test = pd.DataFrame(range(1, 100), columns=["n_neighbors"]) - -X_meta_test["size_X"] = X_new.size -X_meta_test["itemsize_X"] = X_new.itemsize -X_meta_test["ndim_X"] = X_new.ndim - -X_meta_test["size_y"] = y_new.size -X_meta_test["itemsize_y"] = y_new.itemsize -X_meta_test["ndim_y"] = y_new.ndim - - -y_meta_pred = gbr.predict(X_meta_test) - -y_meta_pred_max_idx = y_meta_pred.argmax() -n_neighbors_best = search_space["n_neighbors"][y_meta_pred_max_idx] - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=200) -hyper.run() diff --git a/examples/optimization_applications/meta_optimization.py b/examples/optimization_applications/meta_optimization.py deleted file mode 100644 index 21e818b0..00000000 --- a/examples/optimization_applications/meta_optimization.py +++ /dev/null @@ -1,62 +0,0 @@ -import numpy as np -from hyperactive import Hyperactive -from hyperactive.optimizers import BayesianOptimizer - - -from gradient_free_optimizers import RandomRestartHillClimbingOptimizer - - -def meta_opt(opt_para): - scores = [] - - for i in range(33): - - def ackley_function(para): - x = para["x"] - y = para["y"] - loss1 = -20 * np.exp(-0.2 * np.sqrt(0.5 * (x * x + y * y))) - loss2 = -np.exp(0.5 * (np.cos(2 * np.pi * x) + np.cos(2 * np.pi * y))) - loss3 = np.exp(1) - loss4 = 20 - - loss = loss1 + loss2 + loss3 + loss4 - - return -loss - - dim_size = np.arange(-6, 6, 0.01) - - search_space = { - "x": dim_size, - "y": dim_size, - } - - opt = RandomRestartHillClimbingOptimizer( - search_space, - random_state=i, - epsilon=opt_para["epsilon"], - n_neighbours=opt_para["n_neighbours"], - n_iter_restart=opt_para["n_iter_restart"], - ) - opt.search( - ackley_function, - n_iter=100, - verbosity=False, - ) - - scores.append(opt.best_score) - - return np.array(scores).sum() - - -search_space = { - "epsilon": list(np.arange(0.01, 0.1, 0.01)), - "n_neighbours": list(range(1, 10)), - "n_iter_restart": list(range(2, 12)), -} - - -optimizer = BayesianOptimizer() - -hyper = Hyperactive() -hyper.add_search(meta_opt, search_space, n_iter=120, optimizer=optimizer) -hyper.run() diff --git a/examples/optimization_applications/model_selection.py b/examples/optimization_applications/model_selection.py deleted file mode 100644 index a69de293..00000000 --- a/examples/optimization_applications/model_selection.py +++ /dev/null @@ -1,77 +0,0 @@ -from sklearn.model_selection import cross_val_score - -from sklearn.svm import SVR -from sklearn.neighbors import KNeighborsRegressor -from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.tree import DecisionTreeRegressor -from sklearn.ensemble import ( - GradientBoostingRegressor, - RandomForestRegressor, - ExtraTreesRegressor, -) -from sklearn.neural_network import MLPRegressor - -from sklearn.datasets import load_diabetes -from hyperactive import Hyperactive - -data = load_diabetes() -X, y = data.data, data.target - - -def model(opt): - model_class = opt["regressor"]() - model = model_class() - scores = cross_val_score(model, X, y, cv=5) - - return scores.mean() - - -def SVR_f(): - return SVR - - -def KNeighborsRegressor_f(): - return KNeighborsRegressor - - -def GaussianProcessRegressor_f(): - return GaussianProcessRegressor - - -def DecisionTreeRegressor_f(): - return DecisionTreeRegressor - - -def GradientBoostingRegressor_f(): - return GradientBoostingRegressor - - -def RandomForestRegressor_f(): - return RandomForestRegressor - - -def ExtraTreesRegressor_f(): - return ExtraTreesRegressor - - -def MLPRegressor_f(): - return MLPRegressor - - -search_space = { - "regressor": [ - SVR_f, - KNeighborsRegressor_f, - GaussianProcessRegressor_f, - DecisionTreeRegressor_f, - GradientBoostingRegressor_f, - RandomForestRegressor_f, - ExtraTreesRegressor_f, - MLPRegressor_f, - ], -} - - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=50) -hyper.run() diff --git a/examples/optimization_applications/multiple_different_optimizers.py b/examples/optimization_applications/multiple_different_optimizers.py deleted file mode 100644 index 111bdbe4..00000000 --- a/examples/optimization_applications/multiple_different_optimizers.py +++ /dev/null @@ -1,85 +0,0 @@ -import numpy as np - -from sklearn.model_selection import cross_val_score -from sklearn.ensemble import GradientBoostingClassifier -from sklearn.ensemble import RandomForestClassifier -from sklearn.datasets import load_breast_cancer - -from hyperactive import Hyperactive -from hyperactive.optimizers import ( - HillClimbingOptimizer, - RandomRestartHillClimbingOptimizer, -) - -data = load_breast_cancer() -X, y = data.data, data.target - - -def model_rfc(opt): - rfc = RandomForestClassifier( - n_estimators=opt["n_estimators"], - criterion=opt["criterion"], - max_features=opt["max_features"], - min_samples_split=opt["min_samples_split"], - min_samples_leaf=opt["min_samples_leaf"], - bootstrap=opt["bootstrap"], - ) - scores = cross_val_score(rfc, X, y, cv=3) - - return scores.mean() - - -def model_gbc(opt): - gbc = GradientBoostingClassifier( - n_estimators=opt["n_estimators"], - learning_rate=opt["learning_rate"], - max_depth=opt["max_depth"], - min_samples_split=opt["min_samples_split"], - min_samples_leaf=opt["min_samples_leaf"], - subsample=opt["subsample"], - max_features=opt["max_features"], - ) - scores = cross_val_score(gbc, X, y, cv=3) - - return scores.mean() - - -search_space_rfc = { - "n_estimators": list(range(10, 200, 10)), - "criterion": ["gini", "entropy"], - "max_features": list(np.arange(0.05, 1.01, 0.05)), - "min_samples_split": list(range(2, 21)), - "min_samples_leaf": list(range(1, 21)), - "bootstrap": [True, False], -} - - -search_space_gbc = { - "n_estimators": list(range(10, 200, 10)), - "learning_rate": [1e-3, 1e-2, 1e-1, 0.5, 1.0], - "max_depth": list(range(1, 11)), - "min_samples_split": list(range(2, 21)), - "min_samples_leaf": list(range(1, 21)), - "subsample": list(np.arange(0.05, 1.01, 0.05)), - "max_features": list(np.arange(0.05, 1.01, 0.05)), -} - -optimizer1 = HillClimbingOptimizer() -optimizer2 = RandomRestartHillClimbingOptimizer() - - -hyper = Hyperactive() -hyper.add_search( - model_rfc, - search_space_rfc, - n_iter=50, - optimizer=optimizer1, -) -hyper.add_search( - model_gbc, - search_space_gbc, - n_iter=50, - optimizer=optimizer2, - n_jobs=2, -) -hyper.run(max_time=5) diff --git a/examples/optimization_applications/multiple_scores.py b/examples/optimization_applications/multiple_scores.py deleted file mode 100644 index 4407ef1c..00000000 --- a/examples/optimization_applications/multiple_scores.py +++ /dev/null @@ -1,51 +0,0 @@ -import time -from sklearn.model_selection import cross_val_score -from sklearn.ensemble import GradientBoostingRegressor -from sklearn.datasets import load_diabetes -from hyperactive import Hyperactive - -data = load_diabetes() -X, y = data.data, data.target - -""" -Hyperactive cannot handle multi objective optimization. -But we can achive something similar with a workaround. -The following example searches for the highest cv-score and the lowest training time. -It is possible by creating an objective/score from those two variables. -You can also return additional parameters to track the cv-score and training time separately. -""" - - -def model(opt): - gbr = GradientBoostingRegressor( - n_estimators=opt["n_estimators"], - max_depth=opt["max_depth"], - min_samples_split=opt["min_samples_split"], - ) - - c_time = time.time() - scores = cross_val_score(gbr, X, y, cv=3) - train_time = time.time() - c_time - - cv_score = scores.mean() - - # you can create a score that is a composition of two objectives - score = cv_score / train_time - - # instead of just returning the score you can also return the score + a dict - return score, {"training_time": train_time, "cv_score": cv_score} - - -search_space = { - "n_estimators": list(range(10, 150, 5)), - "max_depth": list(range(2, 12)), - "min_samples_split": list(range(2, 22)), -} - - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=20) -hyper.run() - -# The variables from the dict are collected in the results. -print("\n Results \n", hyper.search_data(model)) diff --git a/examples/optimization_applications/neural_architecture_search.py b/examples/optimization_applications/neural_architecture_search.py deleted file mode 100644 index 78fee799..00000000 --- a/examples/optimization_applications/neural_architecture_search.py +++ /dev/null @@ -1,93 +0,0 @@ -import numpy as np -from keras.models import Sequential -from keras.layers import ( - Dense, - Conv2D, - MaxPooling2D, - Flatten, - Activation, - Dropout, -) -from keras.datasets import cifar10 -from keras.utils import to_categorical - -from hyperactive import Hyperactive - -(X_train, y_train), (X_test, y_test) = cifar10.load_data() - -y_train = to_categorical(y_train, 10) -y_test = to_categorical(y_test, 10) - -# to make the example quick -X_train = X_train[0:1000] -y_train = y_train[0:1000] - -X_test = X_test[0:1000] -y_test = y_test[0:1000] - - -def conv1(nn): - nn.add(Conv2D(32, (3, 3))) - nn.add(Activation("relu")) - nn.add(MaxPooling2D(pool_size=(2, 2))) - return nn - - -def conv2(nn): - nn.add(Conv2D(32, (3, 3))) - nn.add(Activation("relu")) - return nn - - -def conv3(nn): - return nn - - -def cnn(opt): - nn = Sequential() - nn.add( - Conv2D( - opt["filters.0"], - (3, 3), - padding="same", - input_shape=X_train.shape[1:], - ) - ) - nn.add(Activation("relu")) - nn.add(Conv2D(opt["filters.0"], (3, 3))) - nn.add(Activation("relu")) - nn.add(MaxPooling2D(pool_size=(2, 2))) - nn.add(Dropout(0.25)) - - nn.add(Conv2D(opt["filters.0"], (3, 3), padding="same")) - nn.add(Activation("relu")) - nn = opt["conv_layer.0"](nn) - nn.add(Dropout(0.25)) - - nn.add(Flatten()) - nn.add(Dense(opt["neurons.0"])) - nn.add(Activation("relu")) - nn.add(Dropout(0.5)) - nn.add(Dense(10)) - nn.add(Activation("softmax")) - - nn.compile( - optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] - ) - nn.fit(X_train, y_train, epochs=5, batch_size=256) - - _, score = nn.evaluate(x=X_test, y=y_test) - - return score - - -search_space = { - "conv_layer.0": [conv1, conv2, conv3], - "filters.0": [16, 32, 64, 128], - "neurons.0": list(range(100, 1000, 100)), -} - - -hyper = Hyperactive() -hyper.add_search(cnn, search_space, n_iter=5) -hyper.run() diff --git a/examples/optimization_applications/pretrained_nas.py b/examples/optimization_applications/pretrained_nas.py deleted file mode 100644 index da8e4452..00000000 --- a/examples/optimization_applications/pretrained_nas.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -This script describes how to save time during the optimization by -using a pretrained model. It is similar to the transer learning example, -but here you do the training and model creation of the pretrained model -yourself. - -The problem is that most of the optimization time is "waisted" by -training the model. The time to find a new position to explore by -Hyperactive is very small compared to the training time of -neural networks. This means, that we can do more optimization -if we keep the training time as little as possible. - -The idea of pretrained neural architecture search is to pretrain a complete model one time. -In the next step we remove the layers that should be optimized -and make the remaining layers not-trainable. - -This results in a partial, pretrained, not-trainable model that will be -used during the Hyperactive optimization. - -You can now add layers to the partial model in the objective function -and add the parameters or layers that will be optimized by Hyperactive. - -With each iteration of the optimization run we are only training -the added layers of the model. This saves a lot of training time. - -""" - -import numpy as np -import keras -from keras.models import Sequential -from keras.layers import ( - Dense, - Conv2D, - MaxPooling2D, - Flatten, - Activation, - Dropout, -) -from keras.datasets import cifar10 -from keras.utils import to_categorical - -from hyperactive import Hyperactive - -(X_train, y_train), (X_test, y_test) = cifar10.load_data() - -y_train = to_categorical(y_train, 10) -y_test = to_categorical(y_test, 10) - -# to make the example quick -X_train = X_train[0:1000] -y_train = y_train[0:1000] - -X_test = X_test[0:1000] -y_test = y_test[0:1000] - - -# create model and train it -model = Sequential() -model.add(Conv2D(64, (3, 3), padding="same", input_shape=X_train.shape[1:])) -model.add(Activation("relu")) -model.add(Conv2D(32, (3, 3))) -model.add(Activation("relu")) -model.add(MaxPooling2D(pool_size=(2, 2))) -model.add(Dropout(0.25)) - -model.add(Conv2D(32, (3, 3), padding="same")) -model.add(Activation("relu")) -model.add(Dropout(0.25)) -model.add(Flatten()) -model.add(Dense(200)) -model.add(Activation("relu")) -model.add(Dropout(0.5)) -model.add(Dense(10)) -model.add(Activation("softmax")) - -model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"]) -model.fit(X_train, y_train, epochs=5, batch_size=500) - -model_pretrained = model -n_layers = len(model_pretrained.layers) - -# delete the last 9 layers -for i in range(n_layers - 9): - model_pretrained.pop() - -# set remaining layers to not-trainable -for layer in model_pretrained.layers: - layer.trainable = False - -model_pretrained.summary() - - -def cnn(opt): - model = keras.models.clone_model(model_pretrained) - - model.add(Flatten()) - model.add(Dense(opt["neurons.0"])) - model.add(Activation("relu")) - model.add(Dropout(0.5)) - model.add(Dense(10)) - model.add(Activation("softmax")) - - model.compile( - optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] - ) - model.fit(X_train, y_train, epochs=5, batch_size=500) - - model.summary() - - _, score = model.evaluate(x=X_test, y=y_test) - - return score - - -# conv 1, 2, 3 are functions that adds layers. We want to know which function is the best -def conv1(model): - model.add(Conv2D(64, (3, 3))) - model.add(Activation("relu")) - model.add(MaxPooling2D(pool_size=(2, 2))) - return model - - -def conv2(model): - model.add(Conv2D(64, (3, 3))) - model.add(Activation("relu")) - return model - - -def conv3(model): - return model - - -search_space = { - "conv_layer.0": [conv1, conv2, conv3], - "neurons.0": list(range(100, 1000, 100)), -} - - -hyper = Hyperactive() -hyper.add_search(cnn, search_space, n_iter=3) -hyper.run() diff --git a/examples/optimization_applications/search_space_example.py b/examples/optimization_applications/search_space_example.py deleted file mode 100644 index 3575392b..00000000 --- a/examples/optimization_applications/search_space_example.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Hyperactive is very versatile, because it can handle not just numerical or -string variables in the search space, but also functions. If you want to -search for the best list, numpy array, dataframed or class you can put them into a -function that returns them as shown in the example below. - -This enables many possibilities for more complex optimization applications. -Neural architecture search, feature engineering, ensemble optimization and many other applications are -only possible or much easier if you can put functions in the search space. -""" - -from hyperactive import Hyperactive - - -def function_0(): - # do stuff in function0 - return - - -def function_1(): - # do stuff in function1 - return - - -def function_2(): - # do stuff in function2 - return - - -def list1(): - return [1, 0, 0] - - -def list2(): - return [0, 1, 0] - - -def list3(): - return [0, 0, 1] - - -# Hyperactive can handle python objects in the search space -search_space = { - "int": list(range(1, 10)), - "float": [0.1, 0.01, 0.001], - "string": ["string1", "string2"], - "function": [function_0, function_1, function_2], - "list": [list1, list2, list3], -} - - -def objective_function(para): - # score must be a number - score = 1 - return score - - -hyper = Hyperactive() -hyper.add_search(objective_function, search_space, n_iter=20) -hyper.run() - -search_data = hyper.search_data(objective_function) - -print("\n Search Data: \n", search_data) diff --git a/examples/optimization_applications/sklearn_pipeline_example.py b/examples/optimization_applications/sklearn_pipeline_example.py deleted file mode 100644 index da4ae616..00000000 --- a/examples/optimization_applications/sklearn_pipeline_example.py +++ /dev/null @@ -1,48 +0,0 @@ -from sklearn.datasets import load_breast_cancer -from sklearn.model_selection import cross_val_score -from sklearn.feature_selection import SelectKBest, f_classif -from sklearn.ensemble import GradientBoostingClassifier -from sklearn.pipeline import Pipeline - -from hyperactive import Hyperactive - -data = load_breast_cancer() -X, y = data.data, data.target - - -def pipeline1(filter_, gbc): - return Pipeline([("filter_", filter_), ("gbc", gbc)]) - - -def pipeline2(filter_, gbc): - return gbc - - -def model(opt): - gbc = GradientBoostingClassifier( - n_estimators=opt["n_estimators"], - max_depth=opt["max_depth"], - min_samples_split=opt["min_samples_split"], - min_samples_leaf=opt["min_samples_leaf"], - ) - filter_ = SelectKBest(f_classif, k=opt["k"]) - model_ = opt["pipeline"](filter_, gbc) - - scores = cross_val_score(model_, X, y, cv=3) - - return scores.mean() - - -search_space = { - "k": list(range(2, 30)), - "n_estimators": list(range(10, 200, 10)), - "max_depth": list(range(2, 12)), - "min_samples_split": list(range(2, 12)), - "min_samples_leaf": list(range(1, 11)), - "pipeline": [pipeline1, pipeline2], -} - - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=30) -hyper.run() diff --git a/examples/optimization_applications/sklearn_preprocessing.py b/examples/optimization_applications/sklearn_preprocessing.py deleted file mode 100644 index 5027b346..00000000 --- a/examples/optimization_applications/sklearn_preprocessing.py +++ /dev/null @@ -1,49 +0,0 @@ -import numpy as np -from sklearn.datasets import load_breast_cancer -from sklearn.model_selection import cross_val_score -from sklearn.decomposition import PCA -from sklearn.feature_selection import SelectKBest, f_classif -from sklearn.ensemble import GradientBoostingClassifier -from hyperactive import Hyperactive - -data = load_breast_cancer() -X, y = data.data, data.target - - -def model(opt): - model = GradientBoostingClassifier( - n_estimators=opt["n_estimators"], - max_depth=opt["max_depth"], - ) - - X_pca = opt["decomposition"](X, opt) - X_mod = np.hstack((X, X_pca)) - - X_best = SelectKBest(f_classif, k=opt["k"]).fit_transform(X_mod, y) - scores = cross_val_score(model, X_best, y, cv=3) - - return scores.mean() - - -def pca(X_, opt): - X_ = PCA(n_components=opt["n_components"]).fit_transform(X_) - - return X_ - - -def none(X_, opt): - return X_ - - -search_space = { - "decomposition": [pca, none], - "k": list(range(2, 30)), - "n_components": list(range(1, 11)), - "n_estimators": list(range(10, 100, 3)), - "max_depth": list(range(2, 12)), -} - - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=20) -hyper.run() diff --git a/examples/optimization_applications/test_function.py b/examples/optimization_applications/test_function.py deleted file mode 100644 index debe7380..00000000 --- a/examples/optimization_applications/test_function.py +++ /dev/null @@ -1,26 +0,0 @@ -import numpy as np -from hyperactive import Hyperactive - - -def ackley_function(para): - x, y = para["x"], para["y"] - - loss = ( - -20 * np.exp(-0.2 * np.sqrt(0.5 * (x * x + y * y))) - - np.exp(0.5 * (np.cos(2 * np.pi * x) + np.cos(2 * np.pi * y))) - + np.exp(1) - + 20 - ) - - return -loss - - -search_space = { - "x": list(np.arange(-10, 10, 0.01)), - "y": list(np.arange(-10, 10, 0.01)), -} - - -hyper = Hyperactive() -hyper.add_search(ackley_function, search_space, n_iter=100000) -hyper.run() diff --git a/examples/optimization_applications/transfer_learning.py b/examples/optimization_applications/transfer_learning.py deleted file mode 100644 index 5d192fb3..00000000 --- a/examples/optimization_applications/transfer_learning.py +++ /dev/null @@ -1,47 +0,0 @@ -import numpy as np -from keras.models import Sequential -from keras import applications -from keras.layers import Dense, Flatten, Dropout, Activation -from keras.datasets import cifar10 -from keras.utils import to_categorical - -from hyperactive import Hyperactive - -(X_train, y_train), (X_test, y_test) = cifar10.load_data() - -y_train = to_categorical(y_train, 10) -y_test = to_categorical(y_test, 10) - -nn = applications.VGG19(weights="imagenet", include_top=False) - -for layer in nn.layers[:5]: - layer.trainable = False - - -def cnn(opt): - nn = Sequential() - - nn.add(Flatten()) - nn.add(Dense(opt["Dense.0"])) - nn.add(Activation("relu")) - nn.add(Dropout(opt["Dropout.0"])) - nn.add(Dense(10)) - nn.add(Activation("softmax")) - - nn.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"]) - nn.fit(X_train, y_train, epochs=5, batch_size=256) - - _, score = nn.evaluate(x=X_test, y=y_test) - - return score - - -search_space = { - "Dense.0": list(range(100, 1000, 100)), - "Dropout.0": list(np.arange(0.1, 0.9, 0.1)), -} - - -hyper = Hyperactive() -hyper.add_search(cnn, search_space, n_iter=5) -hyper.run() diff --git a/examples/optimization_techniques/bayesian_optimization.py b/examples/optimization_techniques/bayesian_optimization.py deleted file mode 100644 index 14fed901..00000000 --- a/examples/optimization_techniques/bayesian_optimization.py +++ /dev/null @@ -1,40 +0,0 @@ -import numpy as np - - -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive -from hyperactive.optimizers import BayesianOptimizer - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=100) -hyper.run() - -search_data = hyper.search_data(model) - - -optimizer = BayesianOptimizer(xi=0.03, warm_start_smbo=search_data, rand_rest_p=0.1) - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optimization_techniques/direct_algorithm.py b/examples/optimization_techniques/direct_algorithm.py deleted file mode 100644 index 2965e270..00000000 --- a/examples/optimization_techniques/direct_algorithm.py +++ /dev/null @@ -1,24 +0,0 @@ -import numpy as np - -from hyperactive import Hyperactive -from hyperactive.optimizers import DirectAlgorithm - - -def sphere_function(para): - x = para["x"] - y = para["y"] - - return -(x * x + y * y) - - -search_space = { - "x": list(np.arange(-10, 10, 0.1)), - "y": list(np.arange(-10, 10, 0.1)), -} - -opt = DirectAlgorithm() - - -hyper = Hyperactive() -hyper.add_search(sphere_function, search_space, n_iter=1500, optimizer=opt) -hyper.run() diff --git a/examples/optimization_techniques/downhill_simplex.py b/examples/optimization_techniques/downhill_simplex.py deleted file mode 100644 index afa8253d..00000000 --- a/examples/optimization_techniques/downhill_simplex.py +++ /dev/null @@ -1,29 +0,0 @@ -import numpy as np - -from hyperactive import Hyperactive -from hyperactive.optimizers import DownhillSimplexOptimizer - - -def sphere_function(para): - x = para["x"] - y = para["y"] - - return -(x * x + y * y) - - -search_space = { - "x": list(np.arange(-10, 10, 0.1)), - "y": list(np.arange(-10, 10, 0.1)), -} - -opt = DownhillSimplexOptimizer( - alpha=1.2, - gamma=1.1, - beta=0.8, - sigma=1, -) - - -hyper = Hyperactive() -hyper.add_search(sphere_function, search_space, n_iter=1500, optimizer=opt) -hyper.run() diff --git a/examples/optimization_techniques/ensemble_optimizer.py b/examples/optimization_techniques/ensemble_optimizer.py deleted file mode 100644 index a58c7d69..00000000 --- a/examples/optimization_techniques/ensemble_optimizer.py +++ /dev/null @@ -1,44 +0,0 @@ -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from sklearn.svm import SVR -from sklearn.tree import DecisionTreeRegressor -from sklearn.neural_network import MLPRegressor - -from hyperactive import Hyperactive -from hyperactive.optimizers import EnsembleOptimizer - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=100) -hyper.run() - -search_data = hyper.search_data(model) - -optimizer = EnsembleOptimizer( - estimators=[SVR(), DecisionTreeRegressor(), MLPRegressor()], - xi=0.02, - warm_start_smbo=search_data, - rand_rest_p=0.05, -) - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optimization_techniques/evolution_strategy.py b/examples/optimization_techniques/evolution_strategy.py deleted file mode 100644 index 148c5938..00000000 --- a/examples/optimization_techniques/evolution_strategy.py +++ /dev/null @@ -1,32 +0,0 @@ -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive -from hyperactive.optimizers import EvolutionStrategyOptimizer - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - - -optimizer = EvolutionStrategyOptimizer( - mutation_rate=0.5, crossover_rate=0.5, rand_rest_p=0.05 -) - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optimization_techniques/forest_optimization.py b/examples/optimization_techniques/forest_optimization.py deleted file mode 100644 index 873bd3ea..00000000 --- a/examples/optimization_techniques/forest_optimization.py +++ /dev/null @@ -1,40 +0,0 @@ -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive -from hyperactive.optimizers import ForestOptimizer - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=100) -hyper.run() - -search_data = hyper.search_data(model) - -optimizer = ForestOptimizer( - tree_regressor="random_forest", - xi=0.02, - warm_start_smbo=search_data, - rand_rest_p=0.05, -) - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optimization_techniques/grid_search.py b/examples/optimization_techniques/grid_search.py deleted file mode 100644 index c50e68ce..00000000 --- a/examples/optimization_techniques/grid_search.py +++ /dev/null @@ -1,26 +0,0 @@ -import numpy as np - -from hyperactive import Hyperactive -from hyperactive.optimizers import GridSearchOptimizer - - -def sphere_function(para): - x = para["x"] - y = para["y"] - - return -(x * x + y * y) - - -search_space = { - "x": list(np.arange(-10, 10, 0.1)), - "y": list(np.arange(-10, 10, 0.1)), -} - -opt = GridSearchOptimizer( - step_size=3, -) - - -hyper = Hyperactive() -hyper.add_search(sphere_function, search_space, n_iter=1500, optimizer=opt) -hyper.run() diff --git a/examples/optimization_techniques/grid_search_sk.py b/examples/optimization_techniques/grid_search_sk.py deleted file mode 100644 index ccb084c6..00000000 --- a/examples/optimization_techniques/grid_search_sk.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -GridSearchSk Optimizer Example - -This example demonstrates how to use the GridSearchSk optimizer with -loky backend for parallel execution. -""" - -import numpy as np -from hyperactive.opt import GridSearchSk -from hyperactive.experiment.bench import Sphere - -# Define the optimization problem using a benchmark sphere function -# The sphere function f(x,y) = x² + y² has its minimum at (0,0) -sphere_experiment = Sphere(n_dim=2) - -# Define parameter grid - creates a 9x9 = 81 point grid -param_grid = { - "x0": np.linspace(-2, 2, 9), - "x1": np.linspace(-2, 2, 9), -} - -# Grid search with parallel execution using loky backend -grid_search = GridSearchSk( - param_grid=param_grid, - backend="loky", # Use loky backend for parallelization - backend_params={ - "n_jobs": -1, # Use all available CPU cores - }, - experiment=sphere_experiment, -) - -best_params = grid_search.solve() -print(f"Best params: {best_params}, Score: {grid_search.best_score_:.6f}") diff --git a/examples/optimization_techniques/hill_climbing.py b/examples/optimization_techniques/hill_climbing.py deleted file mode 100644 index 3154e8a4..00000000 --- a/examples/optimization_techniques/hill_climbing.py +++ /dev/null @@ -1,32 +0,0 @@ -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive -from hyperactive.optimizers import HillClimbingOptimizer - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - - -optimizer = HillClimbingOptimizer( - epsilon=0.1, distribution="laplace", n_neighbours=4, rand_rest_p=0.1 -) - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optimization_techniques/lipschitz_optimization.py b/examples/optimization_techniques/lipschitz_optimization.py deleted file mode 100644 index fbbd5acc..00000000 --- a/examples/optimization_techniques/lipschitz_optimization.py +++ /dev/null @@ -1,25 +0,0 @@ -import numpy as np - -from hyperactive import Hyperactive -from hyperactive.optimizers import LipschitzOptimizer - - -def sphere_function(para): - x = para["x"] - y = para["y"] - - return -(x * x + y * y) - - -search_space = { - "x": list(np.arange(-10, 10, 0.1)), - "y": list(np.arange(-10, 10, 0.1)), -} - -opt = LipschitzOptimizer( - sampling={"random": 100000}, -) - -hyper = Hyperactive() -hyper.add_search(sphere_function, search_space, n_iter=100, optimizer=opt) -hyper.run() diff --git a/examples/optimization_techniques/parallel_tempering.py b/examples/optimization_techniques/parallel_tempering.py deleted file mode 100644 index d8990b11..00000000 --- a/examples/optimization_techniques/parallel_tempering.py +++ /dev/null @@ -1,30 +0,0 @@ -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive -from hyperactive.optimizers import ParallelTemperingOptimizer - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - - -optimizer = ParallelTemperingOptimizer(n_iter_swap=5, rand_rest_p=0.05) - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optimization_techniques/particle_swarm_optimization.py b/examples/optimization_techniques/particle_swarm_optimization.py deleted file mode 100644 index df129e75..00000000 --- a/examples/optimization_techniques/particle_swarm_optimization.py +++ /dev/null @@ -1,36 +0,0 @@ -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive -from hyperactive.optimizers import ParticleSwarmOptimizer - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - - -optimizer = ParticleSwarmOptimizer( - inertia=0.4, - cognitive_weight=0.7, - social_weight=0.7, - temp_weight=0.3, - rand_rest_p=0.05, -) - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optimization_techniques/pattern_search.py b/examples/optimization_techniques/pattern_search.py deleted file mode 100644 index f7bc4846..00000000 --- a/examples/optimization_techniques/pattern_search.py +++ /dev/null @@ -1,28 +0,0 @@ -import numpy as np - -from hyperactive import Hyperactive -from hyperactive.optimizers import PatternSearch - - -def sphere_function(para): - x = para["x"] - y = para["y"] - - return -(x * x + y * y) - - -search_space = { - "x": list(np.arange(-10, 10, 0.1)), - "y": list(np.arange(-10, 10, 0.1)), -} - -opt = PatternSearch( - n_positions=2, - pattern_size=0.5, - reduction=0.99, -) - - -hyper = Hyperactive() -hyper.add_search(sphere_function, search_space, n_iter=1500, optimizer=opt) -hyper.run() diff --git a/examples/optimization_techniques/powells_method.py b/examples/optimization_techniques/powells_method.py deleted file mode 100644 index c2f36edd..00000000 --- a/examples/optimization_techniques/powells_method.py +++ /dev/null @@ -1,25 +0,0 @@ -import numpy as np - -from hyperactive import Hyperactive -from hyperactive.optimizers import PowellsMethod - - -def sphere_function(para): - x = para["x"] - y = para["y"] - - return -(x * x + y * y) - - -search_space = { - "x": list(np.arange(-10, 10, 0.1)), - "y": list(np.arange(-10, 10, 0.1)), -} - -opt = PowellsMethod( - iters_p_dim=20, -) - -hyper = Hyperactive() -hyper.add_search(sphere_function, search_space, n_iter=1500, optimizer=opt) -hyper.run() diff --git a/examples/optimization_techniques/rand_rest_hill_climbing.py b/examples/optimization_techniques/rand_rest_hill_climbing.py deleted file mode 100644 index 43b2707c..00000000 --- a/examples/optimization_techniques/rand_rest_hill_climbing.py +++ /dev/null @@ -1,36 +0,0 @@ -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive -from hyperactive.optimizers import RandomRestartHillClimbingOptimizer - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - - -optimizer = RandomRestartHillClimbingOptimizer( - epsilon=0.1, - distribution="laplace", - n_neighbours=4, - rand_rest_p=0.1, - n_iter_restart=20, -) - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optimization_techniques/random_annealing.py b/examples/optimization_techniques/random_annealing.py deleted file mode 100644 index 27874628..00000000 --- a/examples/optimization_techniques/random_annealing.py +++ /dev/null @@ -1,37 +0,0 @@ -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive -from hyperactive.optimizers import RandomAnnealingOptimizer - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - - -optimizer = RandomAnnealingOptimizer( - epsilon=0.1, - distribution="laplace", - n_neighbours=4, - rand_rest_p=0.1, - annealing_rate=0.999, - start_temp=0.8, -) - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optimization_techniques/random_search.py b/examples/optimization_techniques/random_search.py deleted file mode 100644 index a61731ee..00000000 --- a/examples/optimization_techniques/random_search.py +++ /dev/null @@ -1,30 +0,0 @@ -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive -from hyperactive.optimizers import RandomSearchOptimizer - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - - -optimizer = RandomSearchOptimizer() - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optimization_techniques/random_search_sk.py b/examples/optimization_techniques/random_search_sk.py deleted file mode 100644 index ffdc91be..00000000 --- a/examples/optimization_techniques/random_search_sk.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -RandomSearchSk Optimizer Example - -This example demonstrates how to use the RandomSearchSk optimizer with -sklearn integration and threading backend for parallel execution. -""" - -import numpy as np -from scipy.stats import uniform, randint -from sklearn.datasets import load_iris -from sklearn.svm import SVC -from hyperactive.opt import RandomSearchSk -from hyperactive.experiment.integrations import SklearnCvExperiment - -# Load iris dataset and create sklearn experiment -X, y = load_iris(return_X_y=True) -sklearn_experiment = SklearnCvExperiment( - estimator=SVC(), - X=X, - y=y, -) - -# Define parameter distributions for random search -# Mix of discrete and continuous distributions -param_distributions = { - "C": uniform(loc=0.1, scale=10), # Continuous uniform distribution - "gamma": ["scale", "auto"] + list(np.logspace(-4, 1, 20)), # Mixed discrete/continuous - "kernel": ["rbf", "poly", "sigmoid"], # Discrete choices -} - -# Random search with threading backend -random_search = RandomSearchSk( - param_distributions=param_distributions, - n_iter=30, # Number of random samples - random_state=42, # For reproducible results - backend="threading", # Use threading backend for parallelization - backend_params={ - "n_jobs": 2, # Use 2 threads - }, - experiment=sklearn_experiment, -) - -best_params = random_search.run() -print(f"Best params: {best_params}, Score: {random_search.best_score_:.4f}") diff --git a/examples/optimization_techniques/repulsing_hill_climbing.py b/examples/optimization_techniques/repulsing_hill_climbing.py deleted file mode 100644 index 82e66f5c..00000000 --- a/examples/optimization_techniques/repulsing_hill_climbing.py +++ /dev/null @@ -1,37 +0,0 @@ -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive -from hyperactive.optimizers import RepulsingHillClimbingOptimizer - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - - -optimizer = RepulsingHillClimbingOptimizer( - epsilon=0.1, - distribution="laplace", - n_neighbours=4, - repulsion_factor=5, - rand_rest_p=0.1, -) - - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optimization_techniques/simulated_annealing.py b/examples/optimization_techniques/simulated_annealing.py deleted file mode 100644 index b5824579..00000000 --- a/examples/optimization_techniques/simulated_annealing.py +++ /dev/null @@ -1,39 +0,0 @@ -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive -from hyperactive.optimizers import SimulatedAnnealingOptimizer - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - - -optimizer = SimulatedAnnealingOptimizer( - epsilon=0.1, - distribution="laplace", - n_neighbours=4, - rand_rest_p=0.1, - p_accept=0.15, - norm_factor="adaptive", - annealing_rate=0.999, - start_temp=0.8, -) - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optimization_techniques/spiral_optimization.py b/examples/optimization_techniques/spiral_optimization.py deleted file mode 100644 index b5504843..00000000 --- a/examples/optimization_techniques/spiral_optimization.py +++ /dev/null @@ -1,26 +0,0 @@ -import numpy as np - -from hyperactive import Hyperactive -from hyperactive.optimizers import SpiralOptimization - - -def sphere_function(para): - x = para["x"] - y = para["y"] - - return -(x * x + y * y) - - -search_space = { - "x": list(np.arange(-25, 10, 0.1)), - "y": list(np.arange(-10, 15, 0.1)), -} - -opt = SpiralOptimization( - population=15, - decay_rate=0.99, -) - -hyper = Hyperactive() -hyper.add_search(sphere_function, search_space, n_iter=1500, optimizer=opt) -hyper.run() diff --git a/examples/optimization_techniques/stochastic_hill_climbing.py b/examples/optimization_techniques/stochastic_hill_climbing.py deleted file mode 100644 index 66e44e23..00000000 --- a/examples/optimization_techniques/stochastic_hill_climbing.py +++ /dev/null @@ -1,28 +0,0 @@ -import numpy as np - -from hyperactive import Hyperactive -from hyperactive.optimizers import StochasticHillClimbingOptimizer - - -def sphere_function(para): - x = para["x"] - y = para["y"] - - return -(x * x + y * y) - - -search_space = { - "x": list(np.arange(-10, 10, 0.1)), - "y": list(np.arange(-10, 10, 0.1)), -} - -opt = StochasticHillClimbingOptimizer( - epsilon=0.01, - n_neighbours=5, - distribution="laplace", - p_accept=0.05, -) - -hyper = Hyperactive() -hyper.add_search(sphere_function, search_space, n_iter=1500, optimizer=opt) -hyper.run() diff --git a/examples/optimization_techniques/tpe.py b/examples/optimization_techniques/tpe.py deleted file mode 100644 index c60eeee7..00000000 --- a/examples/optimization_techniques/tpe.py +++ /dev/null @@ -1,38 +0,0 @@ -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier -from sklearn.model_selection import cross_val_score - -from hyperactive import Hyperactive -from hyperactive.optimizers import TreeStructuredParzenEstimators - - -data = load_iris() -X, y = data.data, data.target - - -def model(opt): - knr = KNeighborsClassifier(n_neighbors=opt["n_neighbors"]) - scores = cross_val_score(knr, X, y, cv=5) - score = scores.mean() - - return score - - -search_space = { - "n_neighbors": list(range(1, 100)), -} - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=100) -hyper.run() - -search_data = hyper.search_data(model) - - -optimizer = TreeStructuredParzenEstimators( - gamma_tpe=0.5, warm_start_smbo=search_data, rand_rest_p=0.05 -) - -hyper = Hyperactive() -hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100) -hyper.run() diff --git a/examples/optuna/cmaes_sampler_example.py b/examples/optuna/cmaes_sampler_example.py index 52da75ed..4129380d 100644 --- a/examples/optuna/cmaes_sampler_example.py +++ b/examples/optuna/cmaes_sampler_example.py @@ -1,5 +1,5 @@ """ -CmaEsSampler Example - Covariance Matrix Adaptation Evolution Strategy +CmaEsOptimizer Example - Covariance Matrix Adaptation Evolution Strategy CMA-ES is a powerful evolution strategy particularly effective for continuous optimization problems. It adapts both the mean and covariance matrix of a @@ -23,143 +23,126 @@ from sklearn.metrics import mean_squared_error from hyperactive.experiment.integrations import SklearnCvExperiment -from hyperactive.opt.optuna import CmaEsSampler - - -def cmaes_theory(): - """Explain CMA-ES algorithm theory.""" - # CMA-ES Algorithm Theory: - # 1. Maintains a multivariate normal distribution N(μ, σ²C) - # - μ: mean vector (center of search) - # - σ: step size (global scaling) - # - C: covariance matrix (shape and orientation) - # - # 2. In each generation: - # - Sample λ offspring from N(μ, σ²C) - # - Evaluate all offspring - # - Select μ best solutions - # - Update μ, σ, and C based on selected solutions - # - # 3. Adaptive features: - # - Covariance matrix learns correlations between parameters - # - Step size adapts to local landscape - # - Handles rotated/scaled problems efficiently - - -def main(): - # === CmaEsSampler Example === - # Covariance Matrix Adaptation Evolution Strategy - - # Check if cmaes is available - try: - import cmaes - - cmaes_available = True - print(" CMA-ES package is available") - except ImportError: - cmaes_available = False - print("⚠ CMA-ES package not available. Install with: pip install cmaes") - print(" This example will demonstrate the interface but may fail at runtime.") +from hyperactive.opt.optuna import CmaEsOptimizer + + +# CMA-ES Algorithm Theory: +# 1. Maintains a multivariate normal distribution N(μ, σ²C) +# - μ: mean vector (center of search) +# - σ: step size (global scaling) +# - C: covariance matrix (shape and orientation) +# +# 2. In each generation: +# - Sample λ offspring from N(μ, σ²C) +# - Evaluate all offspring +# - Select μ best solutions +# - Update μ, σ, and C based on selected solutions +# +# 3. Adaptive features: +# - Covariance matrix learns correlations between parameters +# - Step size adapts to local landscape +# - Handles rotated/scaled problems efficiently + + +# === CmaEsOptimizer Example === +# Covariance Matrix Adaptation Evolution Strategy + +# Check if cmaes is available +try: + import cmaes + + cmaes_available = True + print(" CMA-ES package is available") +except ImportError: + cmaes_available = False + print("⚠ CMA-ES package not available. Install with: pip install cmaes") + print(" This example will demonstrate the interface but may fail at runtime.") +print() + + +# Create a continuous optimization problem +X, y = make_regression(n_samples=200, n_features=10, noise=0.1, random_state=42) +print(f"Dataset: Synthetic regression ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment - neural network with continuous parameters +estimator = MLPRegressor(random_state=42, max_iter=1000) +experiment = SklearnCvExperiment( + estimator=estimator, X=X, y=y, cv=3, scoring="neg_mean_squared_error" +) + +# Define search space - ONLY continuous parameters (CMA-ES limitation) +param_space = { + "alpha": (1e-6, 1e-1), # L2 regularization + "learning_rate_init": (1e-4, 1e-1), # Initial learning rate + "beta_1": (0.8, 0.99), # Adam beta1 parameter + "beta_2": (0.9, 0.999), # Adam beta2 parameter + "epsilon": (1e-9, 1e-6), # Adam epsilon parameter + # Note: No categorical parameters - CMA-ES doesn't support them +} + +# Search Space (Continuous parameters only): +# for param, space in param_space.items(): +# print(f" {param}: {space}") +# Note: CMA-ES only works with continuous parameters +# For mixed parameter types, consider TPESampler or GPOptimizer + +# Configure CmaEsOptimizer +optimizer = CmaEsOptimizer( + param_space=param_space, + n_trials=20, + random_state=42, + experiment=experiment, + sigma0=0.2, # Initial step size (exploration vs exploitation) + n_startup_trials=5, # Random trials before CMA-ES starts +) + +# CmaEsOptimizer Configuration: +# n_trials: configured above +# sigma0: initial step size +# n_startup_trials: random trials before CMA-ES starts +# Adaptive covariance matrix will be learned during optimization + +if not cmaes_available: + print("⚠ Skipping optimization due to missing 'cmaes' package") + print("Install with: pip install cmaes") + +# Run optimization +# Running CMA-ES optimization... +try: + best_params = optimizer.solve() + + # Results + print("\n=== Results ===") + print(f"Best parameters: {best_params}") + print(f"Best score: {optimizer.best_score_:.4f}") print() - cmaes_theory() - - # Create a continuous optimization problem - X, y = make_regression(n_samples=200, n_features=10, noise=0.1, random_state=42) - print( - f"Dataset: Synthetic regression ({X.shape[0]} samples, {X.shape[1]} features)" - ) - - # Create experiment - neural network with continuous parameters - estimator = MLPRegressor(random_state=42, max_iter=1000) - experiment = SklearnCvExperiment( - estimator=estimator, X=X, y=y, cv=3, scoring="neg_mean_squared_error" - ) - - # Define search space - ONLY continuous parameters (CMA-ES limitation) - param_space = { - "alpha": (1e-6, 1e-1), # L2 regularization - "learning_rate_init": (1e-4, 1e-1), # Initial learning rate - "beta_1": (0.8, 0.99), # Adam beta1 parameter - "beta_2": (0.9, 0.999), # Adam beta2 parameter - "epsilon": (1e-9, 1e-6), # Adam epsilon parameter - # Note: No categorical parameters - CMA-ES doesn't support them - } - - # Search Space (Continuous parameters only): - # for param, space in param_space.items(): - # print(f" {param}: {space}") - # Note: CMA-ES only works with continuous parameters - # For mixed parameter types, consider TPESampler or GPSampler - - # Configure CmaEsSampler - optimizer = CmaEsSampler( - param_space=param_space, - n_trials=40, - random_state=42, - experiment=experiment, - sigma0=0.2, # Initial step size (exploration vs exploitation) - n_startup_trials=5, # Random trials before CMA-ES starts - ) - - # CmaEsSampler Configuration: - # n_trials: configured above - # sigma0: initial step size - # n_startup_trials: random trials before CMA-ES starts - # Adaptive covariance matrix will be learned during optimization - - if not cmaes_available: - print("⚠ Skipping optimization due to missing 'cmaes' package") - print("Install with: pip install cmaes") - return None, None - - # Run optimization - # Running CMA-ES optimization... - try: - best_params = optimizer.solve() - - # Results - print("\n=== Results ===") - print(f"Best parameters: {best_params}") - print(f"Best score: {optimizer.best_score_:.4f}") - print() - - except ImportError as e: - print(f"CMA-ES failed: {e}") - print("Install the required package: pip install cmaes") - return None, None - - # CMA-ES Behavior Analysis: - # Evolution of search distribution: - # Initial: Spherical distribution (σ₀ * I) - # Early trials: Random exploration to gather information - # Mid-trials: Covariance matrix learns parameter correlations - # Later trials: Focused search along principal component directions - - # Adaptive Properties: - # Step size (σ) adapts to local topology - # Covariance matrix (C) learns parameter interactions - # Mean vector (μ) tracks promising regions - # Handles ill-conditioned and rotated problems - - # Best Use Cases: - # Continuous optimization problems - # Parameters with potential correlations - # Non-convex, multimodal functions - # When gradient information is unavailable - # Medium-dimensional problems (2-40 parameters) - - # Limitations: - # Only continuous parameters (no categorical/discrete) - # Requires additional 'cmaes' package - # Can be slower than TPE for simple problems - # Memory usage grows with parameter dimension - - if cmaes_available: - return best_params, optimizer.best_score_ - else: - return None, None - - -if __name__ == "__main__": - best_params, best_score = main() +except ImportError as e: + print(f"CMA-ES failed: {e}") + print("Install the required package: pip install cmaes") + +# CMA-ES Behavior Analysis: +# Evolution of search distribution: +# Initial: Spherical distribution (σ₀ * I) +# Early trials: Random exploration to gather information +# Mid-trials: Covariance matrix learns parameter correlations +# Later trials: Focused search along principal component directions + +# Adaptive Properties: +# Step size (σ) adapts to local topology +# Covariance matrix (C) learns parameter interactions +# Mean vector (μ) tracks promising regions +# Handles ill-conditioned and rotated problems + +# Best Use Cases: +# Continuous optimization problems +# Parameters with potential correlations +# Non-convex, multimodal functions +# When gradient information is unavailable +# Medium-dimensional problems (2-40 parameters) + +# Limitations: +# Only continuous parameters (no categorical/discrete) +# Requires additional 'cmaes' package +# Can be slower than TPE for simple problems +# Memory usage grows with parameter dimension diff --git a/examples/optuna/gp_sampler_example.py b/examples/optuna/gp_sampler_example.py index fc28f377..930c0b23 100644 --- a/examples/optuna/gp_sampler_example.py +++ b/examples/optuna/gp_sampler_example.py @@ -1,7 +1,7 @@ """ -GPSampler Example - Gaussian Process Bayesian Optimization +GPOptimizer Example - Gaussian Process Bayesian Optimization -The GPSampler uses Gaussian Processes to model the objective function and +The GPOptimizer uses Gaussian Processes to model the objective function and select promising parameter configurations. It's particularly effective for expensive function evaluations and provides uncertainty estimates. @@ -20,143 +20,130 @@ from sklearn.model_selection import cross_val_score from hyperactive.experiment.integrations import SklearnCvExperiment -from hyperactive.opt.optuna import GPSampler - - -def gaussian_process_theory(): - """Explain Gaussian Process theory for optimization.""" - # Gaussian Process Bayesian Optimization: - # - # 1. Surrogate Model: - # - GP models f(x) ~ N(μ(x), σ²(x)) - # - μ(x): predicted mean (expected objective value) - # - σ²(x): predicted variance (uncertainty estimate) - # - # 2. Acquisition Function: - # - Balances exploration vs exploitation - # - Common choices: Expected Improvement (EI), Upper Confidence Bound (UCB) - # - Selects next point to evaluate: x_next = argmax acquisition(x) - # - # 3. Iterative Process: - # - Fit GP to observed data (x_i, f(x_i)) - # - Optimize acquisition function to find x_next - # - Evaluate f(x_next) - # - Update dataset and repeat - # - # 4. Key Advantages: - # - Uncertainty-aware: explores uncertain regions - # - Sample efficient: good for expensive evaluations - # - Principled: grounded in Bayesian inference - - -def main(): - # === GPSampler Example === - # Gaussian Process Bayesian Optimization - - gaussian_process_theory() - - # Load dataset - classification problem - X, y = load_breast_cancer(return_X_y=True) - print( - f"Dataset: Breast cancer classification ({X.shape[0]} samples, {X.shape[1]} features)" - ) - - # Create experiment - estimator = SVC(random_state=42) - experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=5) - - # Define search space - mixed parameter types - param_space = { - "C": (0.01, 100), # Continuous - regularization - "gamma": (1e-6, 1e2), # Continuous - RBF parameter - "kernel": ["rbf", "poly", "sigmoid"], # Categorical - "degree": (2, 5), # Integer - polynomial degree - "coef0": (0.0, 1.0), # Continuous - kernel coefficient - } - - # Search Space (Mixed parameter types): - # for param, space in param_space.items(): - # print(f" {param}: {space}") - - # Configure GPSampler - optimizer = GPSampler( - param_space=param_space, - n_trials=25, # Fewer trials - GP is sample efficient - random_state=42, - experiment=experiment, - n_startup_trials=8, # Random initialization before GP modeling - deterministic_objective=False, # Set True if objective is noise-free - ) - - # GPSampler Configuration: - # n_trials: configured above - # n_startup_trials: random initialization - # deterministic_objective: configures noise handling - # Acquisition function: Expected Improvement (default) - - # Run optimization - # Running GP-based optimization... - best_params = optimizer.solve() - - # Results - print("\n=== Results ===") - print(f"Best parameters: {best_params}") - print(f"Best score: {optimizer.best_score_:.4f}") - print() - - # GP Optimization Phases: - # - # Phase 1 (Trials 1-8): Random Exploration - # Random sampling for initial GP training data - # Builds diverse set of observations - # No model assumptions yet - - # Phase 2 (Trials 9-25): GP-guided Search - # GP model learns from observed data - # Acquisition function balances: - # - Exploitation: areas with high predicted performance - # - Exploration: areas with high uncertainty - # Sequential decision making with uncertainty - - # GP Model Characteristics: - # Handles mixed parameter types (continuous, discrete, categorical) - # Provides uncertainty estimates for all predictions - # Automatically balances exploration vs exploitation - # Sample efficient - good for expensive evaluations - # Can incorporate prior knowledge through mean/kernel functions - - # Acquisition Function Behavior: - # High mean + low variance → exploitation - # Low mean + high variance → exploration - # Balanced trade-off prevents premature convergence - # Adapts exploration strategy based on observed data - - # Best Use Cases: - # Expensive objective function evaluations - # Small to medium parameter spaces (< 20 dimensions) - # When uncertainty quantification is valuable - # Mixed parameter types (continuous + categorical) - # Noisy objective functions (with appropriate kernel) - - # Limitations: - # Computational cost grows with number of observations - # Hyperparameter tuning for GP kernel - # May struggle in very high dimensions - # Assumes some smoothness in objective function - - # Comparison with TPESampler: - # GPSampler advantages: - # + Principled uncertainty quantification - # + Better for expensive evaluations - # + Can handle constraints naturally - # - # TPESampler advantages: - # + Faster computation - # + Better scalability to high dimensions - # + More robust hyperparameter defaults - - return best_params, optimizer.best_score_ - - -if __name__ == "__main__": - best_params, best_score = main() +from hyperactive.opt.optuna import GPOptimizer + + +# Gaussian Process Bayesian Optimization: +# +# 1. Surrogate Model: +# - GP models f(x) ~ N(μ(x), σ²(x)) +# - μ(x): predicted mean (expected objective value) +# - σ²(x): predicted variance (uncertainty estimate) +# +# 2. Acquisition Function: +# - Balances exploration vs exploitation +# - Common choices: Expected Improvement (EI), Upper Confidence Bound (UCB) +# - Selects next point to evaluate: x_next = argmax acquisition(x) +# +# 3. Iterative Process: +# - Fit GP to observed data (x_i, f(x_i)) +# - Optimize acquisition function to find x_next +# - Evaluate f(x_next) +# - Update dataset and repeat +# +# 4. Key Advantages: +# - Uncertainty-aware: explores uncertain regions +# - Sample efficient: good for expensive evaluations +# - Principled: grounded in Bayesian inference + + +# === GPOptimizer Example === +# Gaussian Process Bayesian Optimization + + +# Load dataset - classification problem +X, y = load_breast_cancer(return_X_y=True) +print( + f"Dataset: Breast cancer classification ({X.shape[0]} samples, {X.shape[1]} features)" +) + +# Create experiment +estimator = SVC(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=5) + +# Define search space - mixed parameter types +param_space = { + "C": (0.01, 100), # Continuous - regularization + "kernel": ["rbf", "poly", "sigmoid"], # Categorical +} + +# Search Space (Mixed parameter types): +# for param, space in param_space.items(): +# print(f" {param}: {space}") + +# Configure GPOptimizer +optimizer = GPOptimizer( + param_space=param_space, + n_trials=5, + random_state=42, + experiment=experiment, + n_startup_trials=5, # Random initialization before GP modeling + deterministic_objective=False, # Set True if objective is noise-free +) + +# GPOptimizer Configuration: +# n_trials: configured above +# n_startup_trials: random initialization +# deterministic_objective: configures noise handling +# Acquisition function: Expected Improvement (default) + +# Run optimization +# Running GP-based optimization... +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print(f"Best score: {optimizer.best_score_:.4f}") +print() + +# GP Optimization Phases: +# +# Phase 1 (Trials 1-8): Random Exploration +# Random sampling for initial GP training data +# Builds diverse set of observations +# No model assumptions yet + +# Phase 2 (Trials 9-25): GP-guided Search +# GP model learns from observed data +# Acquisition function balances: +# - Exploitation: areas with high predicted performance +# - Exploration: areas with high uncertainty +# Sequential decision making with uncertainty + +# GP Model Characteristics: +# Handles mixed parameter types (continuous, discrete, categorical) +# Provides uncertainty estimates for all predictions +# Automatically balances exploration vs exploitation +# Sample efficient - good for expensive evaluations +# Can incorporate prior knowledge through mean/kernel functions + +# Acquisition Function Behavior: +# High mean + low variance → exploitation +# Low mean + high variance → exploration +# Balanced trade-off prevents premature convergence +# Adapts exploration strategy based on observed data + +# Best Use Cases: +# Expensive objective function evaluations +# Small to medium parameter spaces (< 20 dimensions) +# When uncertainty quantification is valuable +# Mixed parameter types (continuous + categorical) +# Noisy objective functions (with appropriate kernel) + +# Limitations: +# Computational cost grows with number of observations +# Hyperparameter tuning for GP kernel +# May struggle in very high dimensions +# Assumes some smoothness in objective function + +# Comparison with TPESampler: +# GPOptimizer advantages: +# + Principled uncertainty quantification +# + Better for expensive evaluations +# + Can handle constraints naturally +# +# TPESampler advantages: +# + Faster computation +# + Better scalability to high dimensions +# + More robust hyperparameter defaults diff --git a/examples/optuna/grid_sampler_example.py b/examples/optuna/grid_sampler_example.py index a52ba46d..846f8f87 100644 --- a/examples/optuna/grid_sampler_example.py +++ b/examples/optuna/grid_sampler_example.py @@ -1,7 +1,7 @@ """ -GridSampler Example - Exhaustive Grid Search +GridOptimizer Example - Exhaustive Grid Search -The GridSampler performs exhaustive search over a discretized parameter grid. +The GridOptimizer performs exhaustive search over a discretized parameter grid. It systematically evaluates every combination of specified parameter values, ensuring complete coverage but potentially requiring many evaluations. @@ -20,170 +20,140 @@ from sklearn.model_selection import cross_val_score from hyperactive.experiment.integrations import SklearnCvExperiment -from hyperactive.opt.optuna import GridSampler - - -def grid_search_theory(): - """Explain grid search methodology.""" - # Grid Search Methodology: - # - # 1. Parameter Discretization: - # - Each continuous parameter divided into discrete levels - # - Categorical parameters use all specified values - # - Creates n₁ × n₂ × ... × nₖ total combinations - # - # 2. Systematic Evaluation: - # - Every combination evaluated exactly once - # - No randomness or learning involved - # - Order of evaluation is deterministic - # - # 3. Optimality Guarantees: - # - Finds global optimum within the discrete grid - # - Quality depends on grid resolution - # - May miss optimal values between grid points - # - # 4. Computational Complexity: - # - Exponential growth with number of parameters - # - Curse of dimensionality for many parameters - # - Embarrassingly parallel - - -def demonstrate_curse_of_dimensionality(): - """Show how grid search scales with dimensions.""" - # Grid Search Scaling (Curse of Dimensionality): - # - # scenarios = [ - # (2, 5, "2 parameters × 5 values each"), - # (3, 5, "3 parameters × 5 values each"), - # (4, 5, "4 parameters × 5 values each"), - # (5, 10, "5 parameters × 10 values each"), - # (10, 3, "10 parameters × 3 values each"), - # ] - # - # for n_params, n_values, description in scenarios: - # total_combinations = n_values ** n_params - # print(f" {description}: {total_combinations:,} combinations") - # - # → Grid search works best with small parameter spaces! - - -def main(): - # === GridSampler Example === - # Exhaustive Grid Search - - grid_search_theory() - demonstrate_curse_of_dimensionality() - - # Load dataset - simple classification - X, y = load_iris(return_X_y=True) - print(f"Dataset: Iris classification ({X.shape[0]} samples, {X.shape[1]} features)") - - # Create experiment - estimator = KNeighborsClassifier() - experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=5) - - # Define search space - DISCRETE values only for grid search - param_space = { - "n_neighbors": [1, 3, 5, 7, 11, 15, 21], # 7 values - "weights": ["uniform", "distance"], # 2 values - "metric": ["euclidean", "manhattan", "minkowski"], # 3 values - "p": [1, 2], # Only relevant for minkowski metric # 2 values - } - - # Total combinations: 7 × 2 × 3 × 2 = 84 combinations - total_combinations = 1 - for param, values in param_space.items(): - total_combinations *= len(values) - - # Search Space (Discrete grids only): - # for param, values in param_space.items(): - # print(f" {param}: {values} ({len(values)} values)") - # Total combinations: calculated above - - # Configure GridSampler - optimizer = GridSampler( - param_space=param_space, - n_trials=total_combinations, # Will evaluate all combinations - random_state=42, # For deterministic ordering - experiment=experiment, - ) - - # GridSampler Configuration: - # n_trials: matches total combinations - # search_space: automatically derived from param_space - # Systematic evaluation of every combination - - # Run optimization - # Running exhaustive grid search... - best_params = optimizer.solve() - - # Results - print("\n=== Results ===") - print(f"Best parameters: {best_params}") - print(f"Best score: {optimizer.best_score_:.4f}") - print() - - # Grid Search Characteristics: - # - # Exhaustive Coverage: - # - Evaluated all parameter combinations - # - Guaranteed to find best configuration within grid - # - No risk of missing good regions - - # Reproducibility: - # - Same grid → same results every time - # - Deterministic evaluation order - # - No randomness or hyperparameters - - # Interpretability: - # - Easy to understand methodology - # - Clear relationship between grid density and accuracy - # - Results easily visualized and analyzed - - # Grid Design Considerations: - # - # Parameter Value Selection: - # Include reasonable ranges for each parameter - # Use domain knowledge to choose meaningful values - # Consider logarithmic spacing for scale-sensitive parameters - # Start coarse, then refine around promising regions - # - # Computational Budget: - # Balance grid density with available compute - # Consider parallel evaluation to speed up - # Use coarse grids for initial exploration - # - # Best Use Cases: - # Small parameter spaces (< 6 parameters) - # Discrete/categorical parameters - # When exhaustive evaluation is feasible - # Baseline comparison for other methods - # When interpretability is crucial - # Parallel computing environments - # - # Limitations: - # Exponential scaling with parameter count - # May miss optimal values between grid points - # Inefficient for continuous parameters - # No adaptive learning or focusing - # Can waste evaluations in clearly bad regions - # - # Grid Search vs Other Methods: - # - # vs Random Search: - # + Systematic coverage guarantee - # + Reproducible results - # - Exponential scaling - # - Less efficient in high dimensions - # - # vs Bayesian Optimization: - # + No assumptions about objective function - # + Guaranteed to find grid optimum - # - Much less sample efficient - # - No learning from previous evaluations - - return best_params, optimizer.best_score_ - - -if __name__ == "__main__": - best_params, best_score = main() +from hyperactive.opt.optuna import GridOptimizer + + +# Grid Search Methodology: +# +# 1. Parameter Discretization: +# - Each continuous parameter divided into discrete levels +# - Categorical parameters use all specified values +# - Creates n₁ × n₂ × ... × nₖ total combinations +# +# 2. Systematic Evaluation: +# - Every combination evaluated exactly once +# - No randomness or learning involved +# - Order of evaluation is deterministic +# +# 3. Optimality Guarantees: +# - Finds global optimum within the discrete grid +# - Quality depends on grid resolution +# - May miss optimal values between grid points +# +# 4. Computational Complexity: +# - Exponential growth with number of parameters +# - Curse of dimensionality for many parameters +# - Embarrassingly parallel + + +# === GridOptimizer Example === +# Exhaustive Grid Search + + +# Load dataset - simple classification +X, y = load_iris(return_X_y=True) +print(f"Dataset: Iris classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = KNeighborsClassifier() +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space - DISCRETE values only for grid search +param_space = { + "n_neighbors": [1, 3, 5, 7, 11, 15, 21], # 7 values + "weights": ["uniform", "distance"], # 2 values + "metric": ["euclidean", "manhattan", "minkowski"], # 3 values + "p": [1, 2], # Only relevant for minkowski metric # 2 values +} + +# Total combinations: 7 × 2 × 3 × 2 = 84 combinations +total_combinations = 1 +for param, values in param_space.items(): + total_combinations *= len(values) + +# Search Space (Discrete grids only): +# for param, values in param_space.items(): +# print(f" {param}: {values} ({len(values)} values)") +# Total combinations: calculated above + +# Configure GridOptimizer +optimizer = GridOptimizer( + param_space=param_space, + n_trials=total_combinations, # Will evaluate all combinations + random_state=42, # For deterministic ordering + experiment=experiment, +) + +# GridOptimizer Configuration: +# n_trials: matches total combinations +# search_space: automatically derived from param_space +# Systematic evaluation of every combination + +# Run optimization +# Running exhaustive grid search... +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print(f"Best score: {optimizer.best_score_:.4f}") +print() + +# Grid Search Characteristics: +# +# Exhaustive Coverage: +# - Evaluated all parameter combinations +# - Guaranteed to find best configuration within grid +# - No risk of missing good regions + +# Reproducibility: +# - Same grid → same results every time +# - Deterministic evaluation order +# - No randomness or hyperparameters + +# Interpretability: +# - Easy to understand methodology +# - Clear relationship between grid density and accuracy +# - Results easily visualized and analyzed + +# Grid Design Considerations: +# +# Parameter Value Selection: +# Include reasonable ranges for each parameter +# Use domain knowledge to choose meaningful values +# Consider logarithmic spacing for scale-sensitive parameters +# Start coarse, then refine around promising regions +# +# Computational Budget: +# Balance grid density with available compute +# Consider parallel evaluation to speed up +# Use coarse grids for initial exploration +# +# Best Use Cases: +# Small parameter spaces (< 6 parameters) +# Discrete/categorical parameters +# When exhaustive evaluation is feasible +# Baseline comparison for other methods +# When interpretability is crucial +# Parallel computing environments +# +# Limitations: +# Exponential scaling with parameter count +# May miss optimal values between grid points +# Inefficient for continuous parameters +# No adaptive learning or focusing +# Can waste evaluations in clearly bad regions +# +# Grid Search vs Other Methods: +# +# vs Random Search: +# + Systematic coverage guarantee +# + Reproducible results +# - Exponential scaling +# - Less efficient in high dimensions +# +# vs Bayesian Optimization: +# + No assumptions about objective function +# + Guaranteed to find grid optimum +# - Much less sample efficient +# - No learning from previous evaluations diff --git a/examples/optuna/nsga_ii_sampler_example.py b/examples/optuna/nsga_ii_sampler_example.py index 1f2fd1e3..88f844ae 100644 --- a/examples/optuna/nsga_ii_sampler_example.py +++ b/examples/optuna/nsga_ii_sampler_example.py @@ -1,5 +1,5 @@ """ -NSGAIISampler Example - Multi-objective Optimization with NSGA-II +NSGAIIOptimizer Example - Multi-objective Optimization with NSGA-II NSGA-II (Non-dominated Sorting Genetic Algorithm II) is designed for multi-objective optimization problems where you want to optimize multiple @@ -23,7 +23,7 @@ from sklearn.model_selection import cross_val_score from hyperactive.experiment.integrations import SklearnCvExperiment -from hyperactive.opt.optuna import NSGAIISampler +from hyperactive.opt.optuna import NSGAIIOptimizer class MultiObjectiveExperiment: @@ -52,161 +52,147 @@ def __call__(self, **params): return [-accuracy, complexity / 10000] # Scale complexity for better balance -def nsga_ii_theory(): - """Explain NSGA-II algorithm theory.""" - # NSGA-II Algorithm (Multi-objective Optimization): - # - # 1. Core Concepts: - # - Pareto Dominance: Solution A dominates B if A is better in all objectives - # - Pareto Front: Set of non-dominated solutions - # - Trade-offs: Improving one objective may worsen another - # - # 2. NSGA-II Process: - # - Initialize population randomly - # - For each generation: - # a) Fast non-dominated sorting (rank solutions by dominance) - # b) Crowding distance calculation (preserve diversity) - # c) Selection based on rank and crowding distance - # d) Crossover and mutation to create offspring - # - # 3. Selection Criteria: - # - Primary: Non-domination rank (prefer better fronts) - # - Secondary: Crowding distance (prefer diverse solutions) - # - Elitist: Best solutions always survive - # - # 4. Output: - # - Set of Pareto-optimal solutions - # - User chooses final solution based on preferences - - -def main(): - # === NSGAIISampler Example === - # Multi-objective Optimization with NSGA-II - - nsga_ii_theory() - - # Load dataset - X, y = load_digits(return_X_y=True) - print(f"Dataset: Handwritten digits ({X.shape[0]} samples, {X.shape[1]} features)") - - # Create multi-objective experiment - experiment = MultiObjectiveExperiment(X, y) - - # Multi-objective Problem: - # Objective 1: Maximize classification accuracy - # Objective 2: Minimize model complexity - # → Trade-off between performance and simplicity - - # Define search space - param_space = { - "n_estimators": (10, 200), # Number of trees - "max_depth": (1, 20), # Tree depth (complexity) - "min_samples_split": (2, 20), # Minimum samples to split - "min_samples_leaf": (1, 10), # Minimum samples per leaf - "max_features": ["sqrt", "log2", None], # Feature sampling - } - - # Search Space: - # for param, space in param_space.items(): - # print(f" {param}: {space}") - - # Configure NSGAIISampler - optimizer = NSGAIISampler( - param_space=param_space, - n_trials=50, # Population evolves over multiple generations - random_state=42, - experiment=experiment, - population_size=20, # Population size for genetic algorithm - mutation_prob=0.1, # Mutation probability - crossover_prob=0.9, # Crossover probability - ) - - # NSGAIISampler Configuration: - # n_trials: configured above - # population_size: for genetic algorithm - # mutation_prob: mutation probability - # crossover_prob: crossover probability - # Selection: Non-dominated sorting + crowding distance - - # Note: This example demonstrates the interface. - # In practice, NSGA-II returns multiple Pareto-optimal solutions. - # For single-objective problems, consider TPE or GP samplers instead. - - # Run optimization - # Running NSGA-II multi-objective optimization... - - try: - best_params = optimizer.solve() - - # Results - print("\n=== Results ===") - print(f"Best parameters: {best_params}") - print(f"Best score: {optimizer.best_score_:.4f}") - print() - - # NSGA-II typically returns multiple solutions along Pareto front: - # High accuracy, high complexity models - # Medium accuracy, medium complexity models - # Lower accuracy, low complexity models - # User selects based on preferences/constraints - - except Exception as e: - print(f"Multi-objective optimization example: {e}") - print("Note: This demonstrates the interface for multi-objective problems.") - return None, None - - # NSGA-II Evolution Process: - # - # Generation 1: Random initialization - # Diverse population across parameter space - # Wide range of accuracy/complexity trade-offs - - # Generations 2-N: Evolutionary improvement - # Non-dominated sorting identifies best fronts - # Crowding distance maintains solution diversity - # Crossover combines good solutions - # Mutation explores new parameter regions - - # Final Population: Pareto front approximation - # Multiple non-dominated solutions - # Represents optimal trade-offs - # User chooses based on domain requirements - - # Key Advantages: - # Handles multiple conflicting objectives naturally - # Finds diverse set of optimal trade-offs - # No need to specify objective weights a priori - # Provides insight into objective relationships - # Robust to objective scaling differences - - # Best Use Cases: - # True multi-objective problems (accuracy vs speed, cost vs quality) - # When trade-offs between objectives are important - # Robustness analysis with multiple criteria - # When single objective formulation is unclear - - # Limitations: - # More complex than single-objective methods - # Requires more evaluations (population-based) - # May be overkill for single-objective problems - # Final solution selection still required - - # When to Use NSGA-II vs Single-objective Methods: - # Use NSGA-II when: - # Multiple objectives genuinely conflict - # Trade-off analysis is valuable - # Objective weights are unknown - # - # Use TPE/GP when: - # Single clear objective - # Computational budget is limited - # Faster convergence needed - - if "best_params" in locals(): - return best_params, optimizer.best_score_ - else: - return None, None - - -if __name__ == "__main__": - best_params, best_score = main() +# NSGA-II Algorithm (Multi-objective Optimization): +# +# 1. Core Concepts: +# - Pareto Dominance: Solution A dominates B if A is better in all objectives +# - Pareto Front: Set of non-dominated solutions +# - Trade-offs: Improving one objective may worsen another +# +# 2. NSGA-II Process: +# - Initialize population randomly +# - For each generation: +# a) Fast non-dominated sorting (rank solutions by dominance) +# b) Crowding distance calculation (preserve diversity) +# c) Selection based on rank and crowding distance +# d) Crossover and mutation to create offspring +# +# 3. Selection Criteria: +# - Primary: Non-domination rank (prefer better fronts) +# - Secondary: Crowding distance (prefer diverse solutions) +# - Elitist: Best solutions always survive +# +# 4. Output: +# - Set of Pareto-optimal solutions +# - User chooses final solution based on preferences + + +# === NSGAIIOptimizer Example === +# Multi-objective Optimization with NSGA-II + + +# Load dataset +X, y = load_digits(return_X_y=True) +print(f"Dataset: Handwritten digits ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create multi-objective experiment +experiment = MultiObjectiveExperiment(X, y) + +# Multi-objective Problem: +# Objective 1: Maximize classification accuracy +# Objective 2: Minimize model complexity +# → Trade-off between performance and simplicity + +# Define search space +param_space = { + "n_estimators": (10, 200), # Number of trees + "max_depth": (1, 20), # Tree depth (complexity) + "min_samples_split": (2, 20), # Minimum samples to split + "min_samples_leaf": (1, 10), # Minimum samples per leaf + "max_features": ["sqrt", "log2", None], # Feature sampling +} + +# Search Space: +# for param, space in param_space.items(): +# print(f" {param}: {space}") + +# Configure NSGAIIOptimizer +optimizer = NSGAIIOptimizer( + param_space=param_space, + n_trials=50, # Population evolves over multiple generations + random_state=42, + experiment=experiment, + population_size=20, # Population size for genetic algorithm + mutation_prob=0.1, # Mutation probability + crossover_prob=0.9, # Crossover probability +) + +# NSGAIIOptimizer Configuration: +# n_trials: configured above +# population_size: for genetic algorithm +# mutation_prob: mutation probability +# crossover_prob: crossover probability +# Selection: Non-dominated sorting + crowding distance + +# Note: This example demonstrates the interface. +# In practice, NSGA-II returns multiple Pareto-optimal solutions. +# For single-objective problems, consider TPE or GP samplers instead. + +# Run optimization +# Running NSGA-II multi-objective optimization... + +try: + best_params = optimizer.solve() + + # Results + print("\n=== Results ===") + print(f"Best parameters: {best_params}") + print(f"Best score: {optimizer.best_score_:.4f}") + print() + + # NSGA-II typically returns multiple solutions along Pareto front: + # High accuracy, high complexity models + # Medium accuracy, medium complexity models + # Lower accuracy, low complexity models + # User selects based on preferences/constraints + +except Exception as e: + print(f"Multi-objective optimization example: {e}") + print("Note: This demonstrates the interface for multi-objective problems.") + +# NSGA-II Evolution Process: +# +# Generation 1: Random initialization +# Diverse population across parameter space +# Wide range of accuracy/complexity trade-offs + +# Generations 2-N: Evolutionary improvement +# Non-dominated sorting identifies best fronts +# Crowding distance maintains solution diversity +# Crossover combines good solutions +# Mutation explores new parameter regions + +# Final Population: Pareto front approximation +# Multiple non-dominated solutions +# Represents optimal trade-offs +# User chooses based on domain requirements + +# Key Advantages: +# Handles multiple conflicting objectives naturally +# Finds diverse set of optimal trade-offs +# No need to specify objective weights a priori +# Provides insight into objective relationships +# Robust to objective scaling differences + +# Best Use Cases: +# True multi-objective problems (accuracy vs speed, cost vs quality) +# When trade-offs between objectives are important +# Robustness analysis with multiple criteria +# When single objective formulation is unclear + +# Limitations: +# More complex than single-objective methods +# Requires more evaluations (population-based) +# May be overkill for single-objective problems +# Final solution selection still required + +# When to Use NSGA-II vs Single-objective Methods: +# Use NSGA-II when: +# Multiple objectives genuinely conflict +# Trade-off analysis is valuable +# Objective weights are unknown +# +# Use TPE/GP when: +# Single clear objective +# Computational budget is limited +# Faster convergence needed diff --git a/examples/optuna/nsga_iii_sampler_example.py b/examples/optuna/nsga_iii_sampler_example.py index 8f97fd28..c0c67772 100644 --- a/examples/optuna/nsga_iii_sampler_example.py +++ b/examples/optuna/nsga_iii_sampler_example.py @@ -1,5 +1,5 @@ """ -NSGAIIISampler Example - Many-objective Optimization with NSGA-III +NSGAIIIOptimizer Example - Many-objective Optimization with NSGA-III NSGA-III is an extension of NSGA-II specifically designed for many-objective optimization problems (typically 3+ objectives). It uses reference points @@ -24,7 +24,7 @@ import time from hyperactive.experiment.integrations import SklearnCvExperiment -from hyperactive.opt.optuna import NSGAIIISampler +from hyperactive.opt.optuna import NSGAIIIOptimizer class ManyObjectiveExperiment: @@ -64,175 +64,161 @@ def __call__(self, **params): ] -def nsga_iii_theory(): - """Explain NSGA-III algorithm theory.""" - # NSGA-III Algorithm (Many-objective Optimization): - # - # 1. Many-objective Challenge: - # - With 3+ objectives, most solutions become non-dominated - # - Traditional Pareto ranking loses selection pressure - # - Crowding distance becomes less effective - # - Need structured diversity preservation - # - # 2. NSGA-III Innovations: - # - Reference points on normalized hyperplane - # - Associate solutions with reference points - # - Select solutions to maintain balanced distribution - # - Adaptive normalization for different objective scales - # - # 3. Reference Point Strategy: - # - Systematic placement on unit simplex - # - Each reference point guides search direction - # - Solutions clustered around reference points - # - Maintains diversity across objective space - # - # 4. Selection Mechanism: - # - Non-dominated sorting (like NSGA-II) - # - Reference point association - # - Niche count balancing - # - Preserve solutions near each reference point - - -def main(): - # === NSGAIIISampler Example === - # Many-objective Optimization with NSGA-III - - nsga_iii_theory() - - # Load dataset - X, y = load_breast_cancer(return_X_y=True) - print( - f"Dataset: Breast cancer classification ({X.shape[0]} samples, {X.shape[1]} features)" - ) - - # Create many-objective experiment - experiment = ManyObjectiveExperiment(X, y) - - # Many-objective Problem (4 objectives): - # Objective 1: Maximize classification accuracy - # Objective 2: Minimize model complexity (tree depth) - # Objective 3: Minimize training time - # Objective 4: Maximize interpretability (smaller trees) - # → Complex trade-offs between multiple conflicting goals - - # Define search space - param_space = { - "max_depth": (1, 20), # Tree depth - "min_samples_split": (2, 50), # Minimum samples to split - "min_samples_leaf": (1, 20), # Minimum samples per leaf - "max_leaf_nodes": (10, 200), # Maximum leaf nodes - "criterion": ["gini", "entropy"], # Split criterion - } - - # Search Space: - # for param, space in param_space.items(): - # print(f" {param}: {space}") - - # Configure NSGAIIISampler - optimizer = NSGAIIISampler( - param_space=param_space, - n_trials=60, # More trials needed for many objectives - random_state=42, - experiment=experiment, - population_size=24, # Larger population for many objectives - mutation_prob=0.1, # Mutation probability - crossover_prob=0.9, # Crossover probability - ) - - # NSGAIIISampler Configuration: - # n_trials: configured above - # population_size: larger for many objectives - # mutation_prob: mutation probability - # crossover_prob: crossover probability - # Selection: Reference point-based diversity preservation - - # Note: NSGA-III is designed for 3+ objectives. - # For 2 objectives, NSGA-II is typically preferred. - # This example demonstrates the interface for many-objective problems. - - # Run optimization - # Running NSGA-III many-objective optimization... - - try: - best_params = optimizer.solve() - - # Results - print("\n=== Results ===") - print(f"Best parameters: {best_params}") - print(f"Best score: {optimizer.best_score_:.4f}") - print() - - # NSGA-III produces a diverse set of solutions across 4D Pareto front: - # High accuracy, complex, slower models - # Balanced accuracy/complexity trade-offs - # Fast, simple, interpretable models - # Various combinations optimizing different objectives - - except Exception as e: - print(f"Many-objective optimization example: {e}") - print("Note: This demonstrates the interface for many-objective problems.") - return None, None - - # NSGA-III vs NSGA-II for Many Objectives: - # - # NSGA-II Limitations (3+ objectives): - # Most solutions become non-dominated - # Crowding distance loses effectiveness - # Selection pressure decreases - # Uneven distribution in objective space - - # NSGA-III Advantages: - # Reference points guide search directions - # Maintains diversity across all objectives - # Better selection pressure in many objectives - # Structured exploration of objective space - # Adaptive normalization handles different scales - - # Reference Point Mechanism: - # Systematic placement on normalized hyperplane - # Each point represents a different objective priority - # Solutions associated with nearest reference points - # Selection maintains balance across all points - # Prevents clustering in limited objective regions - - # Many-objective Problem Characteristics: - # - # Challenges: - # Exponential growth of non-dominated solutions - # Difficulty visualizing high-dimensional trade-offs - # User preference articulation becomes complex - # Increased computational requirements - - # Best Use Cases: - # Engineering design with multiple constraints - # Multi-criteria decision making (3+ criteria) - # Resource allocation problems - # System optimization with conflicting requirements - # When objective interactions are complex - # - # Algorithm Selection Guide: - # - # Use NSGA-III when: - # 3 or more objectives - # Objectives are truly conflicting - # Comprehensive trade-off analysis needed - # Reference point guidance is beneficial - # - # Use NSGA-II when: - # 2 objectives - # Simpler Pareto front structure - # Established performance for bi-objective problems - # - # Use single-objective methods when: - # Can formulate as weighted combination - # Clear primary objective with constraints - # Computational efficiency is critical - - if "best_params" in locals(): - return best_params, optimizer.best_score_ - else: - return None, None - - -if __name__ == "__main__": - best_params, best_score = main() +# NSGA-III Algorithm (Many-objective Optimization): +# +# 1. Many-objective Challenge: +# - With 3+ objectives, most solutions become non-dominated +# - Traditional Pareto ranking loses selection pressure +# - Crowding distance becomes less effective +# - Need structured diversity preservation +# +# 2. NSGA-III Innovations: +# - Reference points on normalized hyperplane +# - Associate solutions with reference points +# - Select solutions to maintain balanced distribution +# - Adaptive normalization for different objective scales +# +# 3. Reference Point Strategy: +# - Systematic placement on unit simplex +# - Each reference point guides search direction +# - Solutions clustered around reference points +# - Maintains diversity across objective space +# +# 4. Selection Mechanism: +# - Non-dominated sorting (like NSGA-II) +# - Reference point association +# - Niche count balancing +# - Preserve solutions near each reference point + + +# === NSGAIIIOptimizer Example === +# Many-objective Optimization with NSGA-III + + +# Load dataset +X, y = load_breast_cancer(return_X_y=True) +print( + f"Dataset: Breast cancer classification ({X.shape[0]} samples, {X.shape[1]} features)" +) + +# Create many-objective experiment +experiment = ManyObjectiveExperiment(X, y) + +# Many-objective Problem (4 objectives): +# Objective 1: Maximize classification accuracy +# Objective 2: Minimize model complexity (tree depth) +# Objective 3: Minimize training time +# Objective 4: Maximize interpretability (smaller trees) +# → Complex trade-offs between multiple conflicting goals + +# Define search space +param_space = { + "max_depth": (1, 20), # Tree depth + "min_samples_split": (2, 50), # Minimum samples to split + "min_samples_leaf": (1, 20), # Minimum samples per leaf + "max_leaf_nodes": (10, 200), # Maximum leaf nodes + "criterion": ["gini", "entropy"], # Split criterion +} + +# Search Space: +# for param, space in param_space.items(): +# print(f" {param}: {space}") + +# Configure NSGAIIIOptimizer +optimizer = NSGAIIIOptimizer( + param_space=param_space, + n_trials=60, + random_state=42, + experiment=experiment, + population_size=24, # Larger population for many objectives + mutation_prob=0.1, # Mutation probability + crossover_prob=0.9, # Crossover probability +) + +# NSGAIIIOptimizer Configuration: +# n_trials: configured above +# population_size: larger for many objectives +# mutation_prob: mutation probability +# crossover_prob: crossover probability +# Selection: Reference point-based diversity preservation + +# Note: NSGA-III is designed for 3+ objectives. +# For 2 objectives, NSGA-II is typically preferred. +# This example demonstrates the interface for many-objective problems. + +# Run optimization +# Running NSGA-III many-objective optimization... + +try: + best_params = optimizer.solve() + + # Results + print("\n=== Results ===") + print(f"Best parameters: {best_params}") + print(f"Best score: {optimizer.best_score_:.4f}") + print() + + # NSGA-III produces a diverse set of solutions across 4D Pareto front: + # High accuracy, complex, slower models + # Balanced accuracy/complexity trade-offs + # Fast, simple, interpretable models + # Various combinations optimizing different objectives + +except Exception as e: + print(f"Many-objective optimization example: {e}") + print("Note: This demonstrates the interface for many-objective problems.") + +# NSGA-III vs NSGA-II for Many Objectives: +# +# NSGA-II Limitations (3+ objectives): +# Most solutions become non-dominated +# Crowding distance loses effectiveness +# Selection pressure decreases +# Uneven distribution in objective space + +# NSGA-III Advantages: +# Reference points guide search directions +# Maintains diversity across all objectives +# Better selection pressure in many objectives +# Structured exploration of objective space +# Adaptive normalization handles different scales + +# Reference Point Mechanism: +# Systematic placement on normalized hyperplane +# Each point represents a different objective priority +# Solutions associated with nearest reference points +# Selection maintains balance across all points +# Prevents clustering in limited objective regions + +# Many-objective Problem Characteristics: +# +# Challenges: +# Exponential growth of non-dominated solutions +# Difficulty visualizing high-dimensional trade-offs +# User preference articulation becomes complex +# Increased computational requirements + +# Best Use Cases: +# Engineering design with multiple constraints +# Multi-criteria decision making (3+ criteria) +# Resource allocation problems +# System optimization with conflicting requirements +# When objective interactions are complex +# +# Algorithm Selection Guide: +# +# Use NSGA-III when: +# 3 or more objectives +# Objectives are truly conflicting +# Comprehensive trade-off analysis needed +# Reference point guidance is beneficial +# +# Use NSGA-II when: +# 2 objectives +# Simpler Pareto front structure +# Established performance for bi-objective problems +# +# Use single-objective methods when: +# Can formulate as weighted combination +# Clear primary objective with constraints +# Computational efficiency is critical diff --git a/examples/optuna/qmc_sampler_example.py b/examples/optuna/qmc_sampler_example.py index 1cd6d8c5..70b6b846 100644 --- a/examples/optuna/qmc_sampler_example.py +++ b/examples/optuna/qmc_sampler_example.py @@ -1,7 +1,7 @@ """ -QMCSampler Example - Quasi-Monte Carlo Sampling +QMCOptimizer Example - Quasi-Monte Carlo Sampling -The QMCSampler uses Quasi-Monte Carlo sequences (like Sobol or Halton) +The QMCOptimizer uses Quasi-Monte Carlo sequences (like Sobol or Halton) to generate low-discrepancy samples. These sequences provide better coverage of the parameter space compared to purely random sampling. @@ -25,190 +25,154 @@ from sklearn.model_selection import cross_val_score from hyperactive.experiment.integrations import SklearnCvExperiment -from hyperactive.opt.optuna import QMCSampler - - -def qmc_theory(): - """Explain Quasi-Monte Carlo theory.""" - # Quasi-Monte Carlo (QMC) Theory: - # - # 1. Low-Discrepancy Sequences: - # - Deterministic sequences that fill space uniformly - # - Better distribution than random sampling - # - Minimize gaps and clusters in parameter space - # - Convergence rate O(log^d(N)/N) vs O(1/√N) for random - # - # 2. Common QMC Sequences: - # - Sobol: Based on binary representations - # - Halton: Based on prime number bases - # - Faure: Generalization of Halton sequences - # - Each has different strengths for different dimensions - # - # 3. Space-Filling Properties: - # - Stratification: Even coverage of parameter regions - # - Low discrepancy: Uniform distribution approximation - # - Correlation breaking: Reduces clustering - # - # 4. Advantages over Random Sampling: - # - Better convergence for integration - # - More uniform exploration - # - Reproducible sequences - # - No unlucky clustering - - -def demonstrate_space_filling(): - """Demonstrate space-filling properties conceptually.""" - # Space-Filling Comparison: - # - # Random Sampling: - # Can have clusters and gaps - # Uneven coverage especially with few samples - # Variance in coverage quality - # Some regions may be under-explored - # - # Quasi-Monte Carlo (QMC): - # Systematic space filling - # Even coverage with fewer samples - # Consistent coverage quality - # All regions explored proportionally - # - # Grid Sampling: - # Perfect regular coverage - # Exponential scaling with dimensions - # May miss optimal points between grid lines - # - # → QMC provides balanced approach between random and grid - - -def main(): - # === QMCSampler Example === - # Quasi-Monte Carlo Low-Discrepancy Sampling - - qmc_theory() - demonstrate_space_filling() - - # Load dataset - X, y = load_wine(return_X_y=True) - print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") - - # Create experiment - estimator = LogisticRegression(random_state=42, max_iter=1000) - experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=5) - - # Define search space - param_space = { - "C": (0.001, 100), # Regularization strength - "l1_ratio": (0.0, 1.0), # Elastic net mixing parameter - "solver": ["liblinear", "saga"], # Solver algorithm - "penalty": ["l1", "l2", "elasticnet"], # Regularization type - } - - # Search Space: - # C: (0.001, 100) - Regularization strength - # l1_ratio: (0.0, 1.0) - Elastic net mixing parameter - # solver: ['liblinear', 'saga'] - Solver algorithm - # penalty: ['l1', 'l2', 'elasticnet'] - Regularization type - - # Configure QMCSampler - optimizer = QMCSampler( - param_space=param_space, - n_trials=32, # Power of 2 often works well for QMC - random_state=42, - experiment=experiment, - qmc_type="sobol", # Sobol or Halton sequences - scramble=True, # Randomized QMC (Owen scrambling) - ) - - # QMCSampler Configuration: - # n_trials: 32 (power of 2 for better QMC properties) - # qmc_type: 'sobol' sequence - # scramble: True (randomized QMC) - # Deterministic low-discrepancy sampling - - # QMC Sequence Types: - # Sobol: Excellent for moderate dimensions, binary-based - # Halton: Good for low dimensions, prime-based - # Scrambling: Adds randomization while preserving uniformity - - # Run optimization - # Running QMC sampling optimization... - best_params = optimizer.solve() - - # Results - print("\n=== Results ===") - print(f"Best parameters: {best_params}") - print(f"Best score: {optimizer.best_score_:.4f}") - print() - - # QMC behavior analysis: - # - # QMC Sampling Analysis: - # - # Sequence Properties: - # Deterministic generation (reproducible with same seed) - # Low-discrepancy (uniform distribution approximation) - # Space-filling (systematic coverage of parameter space) - # Stratification (even coverage of all regions) - # No clustering or large gaps - # - # Sobol Sequence Characteristics: - # Binary-based construction - # Good equidistribution properties - # Effective for dimensions up to ~40 - # Popular choice for QMC sampling - # - # Scrambling Benefits (when enabled): - # Breaks regularity patterns - # Provides Monte Carlo error estimates - # Maintains low-discrepancy properties - # Reduces correlation artifacts - # - # QMC vs Other Sampling Methods: - # - # vs Pure Random Sampling: - # + Better space coverage with fewer samples - # + More consistent performance - # + Faster convergence for integration-like problems - # - No true randomness (if needed for some applications) - # - # vs Grid Search: - # + Works well in higher dimensions - # + No exponential scaling - # + Covers continuous spaces naturally - # - No systematic guarantee of finding grid optimum - # - # vs Adaptive Methods (TPE, GP): - # + No assumptions about objective function - # + Embarrassingly parallel - # + Consistent performance regardless of function type - # - No learning from previous evaluations - # - May waste evaluations in clearly suboptimal regions - # - # Best Use Cases: - # Design of experiments (DoE) - # Initial exploration phase - # Baseline for comparing adaptive methods - # Integration and sampling problems - # When function evaluations are parallelizable - # Robustness testing across parameter space - # - # Implementation Considerations: - # Use powers of 2 for n_trials with Sobol sequences - # Consider scrambling for better statistical properties - # Choose sequence type based on dimensionality: - # - Sobol: Good general choice - # - Halton: Better for low dimensions (< 6) - # QMC works best with transformed uniform parameters - # - # Practical Recommendations: - # 1. Use QMC for initial exploration (first 20-50 evaluations) - # 2. Switch to adaptive methods (TPE/GP) for focused search - # 3. Use for sensitivity analysis across full parameter space - # 4. Good choice when unsure about objective function properties - # 5. Ideal for parallel evaluation scenarios - - return best_params, optimizer.best_score_ - - -if __name__ == "__main__": - best_params, best_score = main() +from hyperactive.opt.optuna import QMCOptimizer + + +# Quasi-Monte Carlo (QMC) Theory: +# +# 1. Low-Discrepancy Sequences: +# - Deterministic sequences that fill space uniformly +# - Better distribution than random sampling +# - Minimize gaps and clusters in parameter space +# - Convergence rate O(log^d(N)/N) vs O(1/√N) for random +# +# 2. Common QMC Sequences: +# - Sobol: Based on binary representations +# - Halton: Based on prime number bases +# - Faure: Generalization of Halton sequences +# - Each has different strengths for different dimensions +# +# 3. Space-Filling Properties: +# - Stratification: Even coverage of parameter regions +# - Low discrepancy: Uniform distribution approximation +# - Correlation breaking: Reduces clustering +# +# 4. Advantages over Random Sampling: +# - Better convergence for integration +# - More uniform exploration +# - Reproducible sequences +# - No unlucky clustering + + +# === QMCOptimizer Example === +# Quasi-Monte Carlo Low-Discrepancy Sampling + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = LogisticRegression(random_state=42, max_iter=1000) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=5) + +# Define search space +param_space = { + "C": (0.001, 100), # Regularization strength + "l1_ratio": (0.0, 1.0), # Elastic net mixing parameter + "solver": ["liblinear", "saga"], # Solver algorithm + "penalty": ["l1", "l2"], # Regularization type +} + +# Search Space: +# C: (0.001, 100) - Regularization strength +# l1_ratio: (0.0, 1.0) - Elastic net mixing parameter +# solver: ['liblinear', 'saga'] - Solver algorithm +# penalty: ['l1', 'l2', 'elasticnet'] - Regularization type + +# Configure QMCOptimizer +optimizer = QMCOptimizer( + param_space=param_space, + n_trials=8, # Power of 2 often works well for QMC + random_state=42, + experiment=experiment, + qmc_type="sobol", # Sobol or Halton sequences + scramble=True, # Randomized QMC (Owen scrambling) +) + +# QMCOptimizer Configuration: +# n_trials: 32 (power of 2 for better QMC properties) +# qmc_type: 'sobol' sequence +# scramble: True (randomized QMC) +# Deterministic low-discrepancy sampling + +# QMC Sequence Types: +# Sobol: Excellent for moderate dimensions, binary-based +# Halton: Good for low dimensions, prime-based +# Scrambling: Adds randomization while preserving uniformity + +# Run optimization +# Running QMC sampling optimization... +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print(f"Best score: {optimizer.best_score_:.4f}") +print() + +# QMC behavior analysis: +# +# QMC Sampling Analysis: +# +# Sequence Properties: +# Deterministic generation (reproducible with same seed) +# Low-discrepancy (uniform distribution approximation) +# Space-filling (systematic coverage of parameter space) +# Stratification (even coverage of all regions) +# No clustering or large gaps +# +# Sobol Sequence Characteristics: +# Binary-based construction +# Good equidistribution properties +# Effective for dimensions up to ~40 +# Popular choice for QMC sampling +# +# Scrambling Benefits (when enabled): +# Breaks regularity patterns +# Provides Monte Carlo error estimates +# Maintains low-discrepancy properties +# Reduces correlation artifacts +# +# QMC vs Other Sampling Methods: +# +# vs Pure Random Sampling: +# + Better space coverage with fewer samples +# + More consistent performance +# + Faster convergence for integration-like problems +# - No true randomness (if needed for some applications) +# +# vs Grid Search: +# + Works well in higher dimensions +# + No exponential scaling +# + Covers continuous spaces naturally +# - No systematic guarantee of finding grid optimum +# +# vs Adaptive Methods (TPE, GP): +# + No assumptions about objective function +# + Embarrassingly parallel +# + Consistent performance regardless of function type +# - No learning from previous evaluations +# - May waste evaluations in clearly suboptimal regions +# +# Best Use Cases: +# Design of experiments (DoE) +# Initial exploration phase +# Baseline for comparing adaptive methods +# Integration and sampling problems +# When function evaluations are parallelizable +# Robustness testing across parameter space +# +# Implementation Considerations: +# Use powers of 2 for n_trials with Sobol sequences +# Consider scrambling for better statistical properties +# Choose sequence type based on dimensionality: +# - Sobol: Good general choice +# - Halton: Better for low dimensions (< 6) +# QMC works best with transformed uniform parameters +# +# Practical Recommendations: +# 1. Use QMC for initial exploration (first 20-50 evaluations) +# 2. Switch to adaptive methods (TPE/GP) for focused search +# 3. Use for sensitivity analysis across full parameter space +# 4. Good choice when unsure about objective function properties +# 5. Ideal for parallel evaluation scenarios diff --git a/examples/optuna/random_sampler_example.py b/examples/optuna/random_sampler_example.py index 4fb99492..14733188 100644 --- a/examples/optuna/random_sampler_example.py +++ b/examples/optuna/random_sampler_example.py @@ -1,7 +1,7 @@ """ -RandomSampler Example - Random Search +RandomOptimizer Example - Random Search -The RandomSampler performs pure random sampling from the parameter space. +The RandomOptimizer performs pure random sampling from the parameter space. It serves as a baseline and is surprisingly effective for many problems, especially when the parameter space is high-dimensional or when you have limited computational budget. @@ -21,99 +21,89 @@ from sklearn.model_selection import cross_val_score from hyperactive.experiment.integrations import SklearnCvExperiment -from hyperactive.opt.optuna import RandomSampler - - -def objective_function_analysis(): - """Demonstrate when random sampling is effective.""" - # When Random Sampling Works Well: - # 1. High-dimensional parameter spaces (curse of dimensionality) - # 2. Noisy objective functions - # 3. Limited computational budget - # 4. As a baseline for comparison - # 5. When parallel evaluation is important - # 6. Uniform exploration is desired - - -def main(): - # === RandomSampler Example === - # Pure Random Search - Uniform Parameter Space Exploration - - objective_function_analysis() - - # Load dataset - using digits for a more challenging problem - X, y = load_digits(return_X_y=True) - print(f"Dataset: Handwritten digits ({X.shape[0]} samples, {X.shape[1]} features)") - - # Create experiment - estimator = SVC(random_state=42) - experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) - - # Define search space - SVM hyperparameters - param_space = { - "C": (0.001, 1000), # Regularization - log scale would be better - "gamma": (1e-6, 1e2), # RBF kernel parameter - "kernel": ["rbf", "poly", "sigmoid"], # Kernel type - "degree": (2, 5), # Polynomial degree (only for poly kernel) - "coef0": (0.0, 10.0), # Independent term (poly/sigmoid) - } - - # Search Space: - # for param, space in param_space.items(): - # print(f" {param}: {space}") - - # Configure RandomSampler - optimizer = RandomSampler( - param_space=param_space, - n_trials=30, # More trials to show random behavior - random_state=42, # For reproducible random sampling - experiment=experiment, - ) - - # RandomSampler Configuration: - # n_trials: configured above - # random_state: set for reproducibility - # No learning parameters - pure random sampling - - # Run optimization - # Running random search... - best_params = optimizer.solve() - - # Results - print("\n=== Results ===") - print(f"Best parameters: {best_params}") - print(f"Best score: {optimizer.best_score_:.4f}") - print() - - # Analysis of Random Sampling behavior: - # Each trial is independent - no learning from history - # Uniform coverage of parameter space - # No convergence issues or local optima concerns - # Embarrassingly parallel - can run trials simultaneously - # Works equally well for continuous, discrete, and categorical parameters - - # Comparison with Other Methods: - # vs Grid Search: - # + Better coverage in high dimensions - # + More efficient for continuous parameters - # - No systematic coverage guarantee - # - # vs Bayesian Optimization (TPE, GP): - # + No assumptions about objective function smoothness - # + Works well with noisy objectives - # + No risk of model misspecification - # - No exploitation of promising regions - # - May waste trials on clearly bad regions - - # Practical Usage: - # Use as baseline to validate more sophisticated methods - # Good first choice when objective is very noisy - # Ideal for parallel optimization setups - # Consider for high-dimensional problems (>10 parameters) - # Use with log-uniform distributions for scale-sensitive parameters - - return best_params, optimizer.best_score_ - - -if __name__ == "__main__": - best_params, best_score = main() +from hyperactive.opt.optuna import RandomOptimizer + + +# When Random Sampling Works Well: +# 1. High-dimensional parameter spaces (curse of dimensionality) +# 2. Noisy objective functions +# 3. Limited computational budget +# 4. As a baseline for comparison +# 5. When parallel evaluation is important +# 6. Uniform exploration is desired + + +# === RandomOptimizer Example === +# Pure Random Search - Uniform Parameter Space Exploration + + +# Load dataset - using digits for a more challenging problem +X, y = load_digits(return_X_y=True) +print(f"Dataset: Handwritten digits ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = SVC(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define search space - SVM hyperparameters +param_space = { + "C": (0.001, 1000), # Regularization - log scale would be better + "gamma": (1e-6, 1e2), # RBF kernel parameter + "kernel": ["rbf", "poly", "sigmoid"], # Kernel type + "degree": (2, 5), # Polynomial degree (only for poly kernel) + "coef0": (0.0, 10.0), # Independent term (poly/sigmoid) +} + +# Search Space: +# for param, space in param_space.items(): +# print(f" {param}: {space}") + +# Configure RandomOptimizer +optimizer = RandomOptimizer( + param_space=param_space, + n_trials=20, + random_state=42, # For reproducible random sampling + experiment=experiment, +) + +# RandomOptimizer Configuration: +# n_trials: configured above +# random_state: set for reproducibility +# No learning parameters - pure random sampling + +# Run optimization +# Running random search... +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print(f"Best score: {optimizer.best_score_:.4f}") +print() + +# Analysis of Random Sampling behavior: +# Each trial is independent - no learning from history +# Uniform coverage of parameter space +# No convergence issues or local optima concerns +# Embarrassingly parallel - can run trials simultaneously +# Works equally well for continuous, discrete, and categorical parameters + +# Comparison with Other Methods: +# vs Grid Search: +# + Better coverage in high dimensions +# + More efficient for continuous parameters +# - No systematic coverage guarantee +# +# vs Bayesian Optimization (TPE, GP): +# + No assumptions about objective function smoothness +# + Works well with noisy objectives +# + No risk of model misspecification +# - No exploitation of promising regions +# - May waste trials on clearly bad regions + +# Practical Usage: +# Use as baseline to validate more sophisticated methods +# Good first choice when objective is very noisy +# Ideal for parallel optimization setups +# Consider for high-dimensional problems (>10 parameters) +# Use with log-uniform distributions for scale-sensitive parameters diff --git a/examples/optuna/tpe_sampler_example.py b/examples/optuna/tpe_sampler_example.py index f62462f9..ba4736dd 100644 --- a/examples/optuna/tpe_sampler_example.py +++ b/examples/optuna/tpe_sampler_example.py @@ -19,7 +19,7 @@ from sklearn.model_selection import cross_val_score from hyperactive.experiment.integrations import SklearnCvExperiment -from hyperactive.opt.optuna import TPESampler +from hyperactive.opt.optuna import TPEOptimizer def main(): @@ -54,7 +54,7 @@ def main(): "min_samples_leaf": 1, "max_features": "sqrt", "bootstrap": True} ] - optimizer = TPESampler( + optimizer = TPEOptimizer( param_space=param_space, n_trials=50, random_state=42, diff --git a/examples/sklearn/README.md b/examples/sklearn/README.md new file mode 100644 index 00000000..84b49a3a --- /dev/null +++ b/examples/sklearn/README.md @@ -0,0 +1,221 @@ +# Sklearn Backend Examples + +This directory contains examples demonstrating Hyperactive's sklearn backend integration. This backend provides direct access to scikit-learn's mature and optimized hyperparameter search implementations (GridSearchCV and RandomizedSearchCV) through Hyperactive's unified interface. + +## Quick Start + +Run any example directly: +```bash +python grid_search_example.py +python random_search_example.py +``` + +## Available Algorithms + +| Algorithm | Sklearn Equivalent | Best For | Characteristics | +|-----------|-------------------|----------|-----------------| +| [GridSearchSk](grid_search_example.py) | GridSearchCV | Small discrete spaces | Exhaustive, deterministic | +| [RandomSearchSk](random_search_example.py) | RandomizedSearchCV | General purpose | Efficient sampling | + +## When to Use Sklearn Backend + +The sklearn backend is ideal when you: +- Want to leverage sklearn's mature, well-tested implementations +- Need compatibility with existing sklearn pipelines and workflows +- Require sklearn's specific cross-validation and scoring features +- Prefer sklearn's parameter specification format +- Want to benefit from sklearn's optimized parallel execution + +## Sklearn vs Other Backends + +### Advantages of Sklearn Backend: +- **Mature implementation**: Battle-tested in production environments +- **Sklearn integration**: Natural fit for sklearn-based workflows +- **Efficient execution**: Optimized for sklearn estimators +- **Familiar interface**: Uses sklearn's parameter specification style +- **Built-in features**: Cross-validation, scoring, and parallelization + +### When to Consider Other Backends: +- **Advanced algorithms**: GFO backend offers more sophisticated optimization +- **Optuna features**: Optuna backend provides state-of-the-art Bayesian optimization +- **Custom objectives**: Non-sklearn experiments may benefit from other backends + +## Parameter Space Specification + +### Grid Search - Discrete Values Only +```python +param_space = { + "n_estimators": [10, 50, 100, 200], # Explicit discrete values + "max_depth": [5, 10, 15, None], # Include special values + "criterion": ["gini", "entropy"], # Categorical choices + "bootstrap": [True, False], # Boolean options +} +``` + +### Random Search - Ranges and Distributions +```python +param_space = { + "n_estimators": (10, 200), # Continuous range (sampled as int) + "max_depth": (1, 20), # Integer range + "min_samples_split": (2, 20), # Integer range + "max_features": ["sqrt", "log2", None], # Discrete choices +} +``` + +## Integration with Sklearn Pipelines + +The sklearn backend works seamlessly with sklearn pipelines: + +```python +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler +from sklearn.ensemble import RandomForestClassifier + +# Create pipeline +pipeline = Pipeline([ + ('scaler', StandardScaler()), + ('classifier', RandomForestClassifier(random_state=42)) +]) + +# Define parameter space with pipeline syntax +param_space = { + 'classifier__n_estimators': [50, 100, 200], + 'classifier__max_depth': [5, 10, None], + 'scaler__with_mean': [True, False], +} + +# Use with Hyperactive +experiment = SklearnCvExperiment(estimator=pipeline, X=X, y=y, cv=5) +optimizer = GridSearchSk(param_space=param_space, experiment=experiment) +``` + +## Cross-Validation Strategies + +Leverage sklearn's flexible cross-validation: + +```python +from sklearn.model_selection import StratifiedKFold, TimeSeriesSplit + +# Stratified K-Fold (maintains class distribution) +cv_strat = StratifiedKFold(n_splits=5, shuffle=True, random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=cv_strat) + +# Time Series Split (for temporal data) +cv_time = TimeSeriesSplit(n_splits=5) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=cv_time) +``` + +## Scoring Options + +Use sklearn's extensive scoring options: + +```python +# Single metric +experiment = SklearnCvExperiment( + estimator=estimator, + X=X, y=y, + cv=3, + scoring='f1_macro' # Use F1 score with macro averaging +) + +# Multiple metrics (requires custom handling) +experiment = SklearnCvExperiment( + estimator=estimator, + X=X, y=y, + cv=3, + scoring='accuracy' # Primary scoring metric +) +``` + +## Performance Optimization + +### Parallel Execution +Both algorithms support parallel execution through sklearn's `n_jobs` parameter: + +```python +# Note: n_jobs is handled internally by sklearn backend +# The degree of parallelization depends on sklearn's implementation +``` + +### Memory Efficiency +For large datasets, consider: +- Reducing cross-validation folds for faster iteration +- Using smaller parameter grids for GridSearchSk +- Limiting n_trials for RandomSearchSk + +## Comparison: Grid vs Random Search + +### GridSearchSk +**Pros:** +- Exhaustive coverage of parameter space +- Deterministic and reproducible results +- Guarantees finding optimal solution within the grid +- Good for small, well-defined parameter spaces + +**Cons:** +- Exponential growth in computation time +- Requires discrete parameter values +- Not suitable for high-dimensional spaces + +### RandomSearchSk +**Pros:** +- Scales well to high-dimensional spaces +- Can sample from continuous distributions +- Often finds good solutions with fewer evaluations +- More flexible parameter specification + +**Cons:** +- Stochastic - results may vary between runs +- No guarantee of finding optimal solution +- Requires choosing appropriate number of trials + +## Best Practices + +1. **Start with RandomSearchSk**: Good default choice for most problems +2. **Use GridSearchSk for small spaces**: When you can afford exhaustive search +3. **Leverage sklearn features**: Use appropriate CV strategies and scoring +4. **Set random_state**: For reproducible results +5. **Monitor convergence**: Check if more trials would help +6. **Consider other backends**: For more advanced optimization needs + +## Integration Examples + +### With Preprocessing Pipelines +```python +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler, PCA + +pipeline = Pipeline([ + ('scaling', StandardScaler()), + ('pca', PCA()), + ('classifier', RandomForestClassifier()) +]) + +param_space = { + 'pca__n_components': [5, 10, 15, 20], + 'classifier__n_estimators': (50, 300), + 'classifier__max_depth': [5, 10, None] +} +``` + +### With Feature Selection +```python +from sklearn.feature_selection import SelectKBest, f_classif + +pipeline = Pipeline([ + ('feature_selection', SelectKBest(f_classif)), + ('classifier', RandomForestClassifier()) +]) + +param_space = { + 'feature_selection__k': [5, 10, 15, 'all'], + 'classifier__n_estimators': [50, 100, 200] +} +``` + +## Further Reading + +- [Sklearn GridSearchCV Documentation](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html) +- [Sklearn RandomizedSearchCV Documentation](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.RandomizedSearchCV.html) +- [Sklearn Model Selection Guide](https://scikit-learn.org/stable/model_selection.html) +- [Cross-validation Strategies](https://scikit-learn.org/stable/modules/cross_validation.html) diff --git a/examples/sklearn/grid_search_example.py b/examples/sklearn/grid_search_example.py new file mode 100644 index 00000000..2c6606b5 --- /dev/null +++ b/examples/sklearn/grid_search_example.py @@ -0,0 +1,65 @@ +""" +Sklearn Grid Search Example - Native Sklearn GridSearchCV Integration + +This example demonstrates using Hyperactive's sklearn backend that provides +a direct interface to scikit-learn's GridSearchCV. This approach leverages +sklearn's mature and optimized grid search implementation while maintaining +compatibility with Hyperactive's unified interface. + +Characteristics: +- Direct integration with sklearn's GridSearchCV +- Optimized for sklearn estimators and pipelines +- Supports sklearn's built-in cross-validation strategies +- Familiar sklearn-style parameter specification +- Efficient parallel execution when n_jobs > 1 +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt import GridSearchSk + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define parameter grid in sklearn style +# Using lists of discrete values for exhaustive search +param_grid = { + "n_estimators": [10, 50, 100, 150], # Discrete values for grid + "max_depth": [5, 10, 15, None], # Include None for unlimited + "min_samples_split": [2, 5, 10], # Small discrete set + "max_features": ["sqrt", "log2", None], # Feature selection strategies + "bootstrap": [True, False], # Bootstrap sampling options +} + +# Calculate total parameter combinations +total_combinations = 1 +for param_values in param_grid.values(): + total_combinations *= len(param_values) + +print(f"Total parameter combinations to evaluate: {total_combinations}") + +# Configure Sklearn Grid Search +optimizer = GridSearchSk( + param_grid=param_grid, + experiment=experiment +) + +# Run optimization +# The sklearn backend uses GridSearchCV internally, providing +# optimized grid search with sklearn's efficient implementation +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print(f"Best score: {optimizer.best_score_:.4f}") +print(f"Exhaustively evaluated {total_combinations} combinations") diff --git a/examples/sklearn/random_search_example.py b/examples/sklearn/random_search_example.py new file mode 100644 index 00000000..415ccdc6 --- /dev/null +++ b/examples/sklearn/random_search_example.py @@ -0,0 +1,62 @@ +""" +Sklearn Random Search Example - Native Sklearn RandomizedSearchCV Integration + +This example demonstrates using Hyperactive's sklearn backend with +RandomizedSearchCV. This provides access to sklearn's mature random search +implementation with support for probability distributions and efficient +sampling strategies. + +Characteristics: +- Direct integration with sklearn's RandomizedSearchCV +- Support for probability distributions (not just discrete lists) +- Efficient random sampling from continuous and discrete spaces +- Sklearn-native cross-validation and scoring +- Good baseline performance with minimal configuration +""" + +import numpy as np +from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score + +from hyperactive.experiment.integrations import SklearnCvExperiment +from hyperactive.opt import RandomSearchSk + +# Load dataset +X, y = load_wine(return_X_y=True) +print(f"Dataset: Wine classification ({X.shape[0]} samples, {X.shape[1]} features)") + +# Create experiment +estimator = RandomForestClassifier(random_state=42) +experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=3) + +# Define parameter distributions for random sampling +# RandomizedSearchCV can sample from lists (uniform) or distributions +param_distributions = { + "n_estimators": list(range(10, 201)), # Discrete list (uniform sampling) + "max_depth": list(range(1, 21)), # Discrete integer range + "min_samples_split": list(range(2, 21)), # Discrete integer range + "min_samples_leaf": list(range(1, 11)), # Discrete integer range + "max_features": ["sqrt", "log2", None], # Discrete categorical choices + "bootstrap": [True, False], # Binary choice +} + +# Configure Sklearn Random Search +optimizer = RandomSearchSk( + param_distributions=param_distributions, + n_iter=30, + random_state=42, + experiment=experiment +) + +# Run optimization +# The sklearn backend uses RandomizedSearchCV internally, which efficiently +# samples from the defined parameter distributions without requiring +# explicit enumeration of all possible values +best_params = optimizer.solve() + +# Results +print("\n=== Results ===") +print(f"Best parameters: {best_params}") +print(f"Best score: {optimizer.best_score_:.4f}") +print(f"Randomly sampled 30 parameter combinations") diff --git a/examples/tensorflow_example.py b/examples/tensorflow_example.py deleted file mode 100644 index 292bd5bf..00000000 --- a/examples/tensorflow_example.py +++ /dev/null @@ -1,102 +0,0 @@ -from __future__ import division, print_function, absolute_import -import tensorflow as tf -from tensorflow.examples.tutorials.mnist import input_data - -from hyperactive import Hyperactive - -mnist = input_data.read_data_sets("/tmp/data/", one_hot=False) - -learning_rate = 0.001 -num_steps = 500 -batch_size = 128 - -num_input = 784 -num_classes = 10 -dropout = 0.25 - -X_train = mnist.train.images -y_train = mnist.train.labels - -X_test = mnist.test.images -y_test = mnist.test.labels - - -def cnn(para, X_train, y_train): - def conv_net(x_dict, n_classes, dropout, reuse, is_training): - with tf.variable_scope("ConvNet", reuse=reuse): - x = x_dict["images"] - x = tf.reshape(x, shape=[-1, 28, 28, 1]) - conv1 = tf.layers.conv2d(x, para["filters_0"], 5, activation=tf.nn.relu) - conv1 = tf.layers.max_pooling2d(conv1, 2, 2) - conv2 = tf.layers.conv2d(conv1, para["filters_1"], 3, activation=tf.nn.relu) - conv2 = tf.layers.max_pooling2d(conv2, 2, 2) - fc1 = tf.contrib.layers.flatten(conv2) - fc1 = tf.layers.dense(fc1, para["dense_0"]) - fc1 = tf.layers.dropout(fc1, rate=dropout, training=is_training) - out = tf.layers.dense(fc1, n_classes) - - return out - - def model_fn(features, labels, mode): - logits_train = conv_net( - features, num_classes, dropout, reuse=False, is_training=True - ) - logits_test = conv_net( - features, num_classes, dropout, reuse=True, is_training=False - ) - - pred_classes = tf.argmax(logits_test, axis=1) - # pred_probas = tf.nn.softmax(logits_test) - - if mode == tf.estimator.ModeKeys.PREDICT: - return tf.estimator.EstimatorSpec(mode, predictions=pred_classes) - - loss_op = tf.reduce_mean( - tf.nn.sparse_softmax_cross_entropy_with_logits( - logits=logits_train, labels=tf.cast(labels, dtype=tf.int32) - ) - ) - optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) - train_op = optimizer.minimize(loss_op, global_step=tf.train.get_global_step()) - - acc_op = tf.metrics.accuracy(labels=labels, predictions=pred_classes) - - estim_specs = tf.estimator.EstimatorSpec( - mode=mode, - predictions=pred_classes, - loss=loss_op, - train_op=train_op, - eval_metric_ops={"accuracy": acc_op}, - ) - - return estim_specs - - model = tf.estimator.Estimator(model_fn) - - input_fn = tf.estimator.inputs.numpy_input_fn( - x={"images": X_train}, - y=y_train, - batch_size=batch_size, - num_epochs=None, - shuffle=True, - ) - model.train(input_fn, steps=num_steps) - - input_fn = tf.estimator.inputs.numpy_input_fn( - x={"images": X_test}, y=y_test, batch_size=batch_size, shuffle=False - ) - e = model.evaluate(input_fn) - - return float(e["accuracy"]) - - -search_space = { - "filters_0": [16, 32, 64], - "filters_1": [16, 32, 64], - "dense_0": list(range(100, 2000, 100)), -} - - -hyper = Hyperactive(X_train, y_train) -hyper.add_search(cnn, search_space, n_iter=100) -hyper.run() diff --git a/examples/test_examples.py b/examples/test_examples.py new file mode 100644 index 00000000..12e147df --- /dev/null +++ b/examples/test_examples.py @@ -0,0 +1,212 @@ +""" +Test script for all Hyperactive v5 examples. + +This test script runs all examples in the examples directory and verifies +that they execute without errors. It's designed for continuous integration +and regression testing to ensure all examples remain functional. + +The test captures stdout/stderr but focuses on successful execution rather +than output validation, making it suitable for verifying that API changes +don't break the examples. +""" + +import os +import sys +import subprocess +import tempfile +from pathlib import Path +import pytest + + +def get_hyperactive_root(): + """Get the root directory of the Hyperactive project.""" + # Since test file is now in examples/, go up one level to find the root + current = Path(__file__).parent.parent + return current + + +def find_all_python_examples(): + """Find all Python example files in the examples directory.""" + root = get_hyperactive_root() + examples_dir = root / "examples" + + if not examples_dir.exists(): + return [] + + python_files = [] + + # Find all .py files recursively in examples directory + for py_file in examples_dir.rglob("*.py"): + # Skip __pycache__, test files, and other non-example files + if ("__pycache__" not in str(py_file) and + ".pytest_cache" not in str(py_file) and + not py_file.name.startswith("test_")): + python_files.append(py_file) + + return sorted(python_files) + + +# Generate the list of examples at module level for pytest parametrize +ALL_EXAMPLES = find_all_python_examples() + + +def run_example(example_path, timeout=120): + """ + Run a single example file and return success status and output. + + Parameters + ---------- + example_path : Path + Path to the example Python file + timeout : int, default=120 + Maximum time to wait for example to complete (seconds) + + Returns + ------- + success : bool + True if example ran without errors + stdout : str + Standard output from the example + stderr : str + Standard error from the example + """ + try: + # Run the example with timeout + result = subprocess.run( + [sys.executable, str(example_path)], + cwd=example_path.parent, + capture_output=True, + text=True, + timeout=timeout + ) + + success = result.returncode == 0 + return success, result.stdout, result.stderr + + except subprocess.TimeoutExpired: + return False, "", f"Example timed out after {timeout} seconds" + except Exception as e: + return False, "", f"Error running example: {str(e)}" + + +class TestExamples: + """Test class for all Hyperactive examples.""" + + @pytest.mark.parametrize("example_path", ALL_EXAMPLES, ids=lambda x: str(x.relative_to(get_hyperactive_root())) if ALL_EXAMPLES else "no_examples") + def test_example_runs_successfully(self, example_path): + """Test that an example runs without errors.""" + if not ALL_EXAMPLES: + pytest.skip("No examples found") + + # Get relative path for cleaner test names + root = get_hyperactive_root() + relative_path = example_path.relative_to(root) + + print(f"\\nRunning example: {relative_path}") + + # Run the example + success, stdout, stderr = run_example(example_path) + + # Print output for debugging if needed + if stdout and len(stdout.strip()) > 0: + print(f"STDOUT:\\n{stdout[:500]}{'...' if len(stdout) > 500 else ''}") + + if stderr and len(stderr.strip()) > 0: + print(f"STDERR:\\n{stderr[:500]}{'...' if len(stderr) > 500 else ''}") + + # Assert that the example ran successfully + assert success, f"Example failed: {relative_path}\\nSTDERR: {stderr}" + + def test_examples_directory_exists(self): + """Test that the examples directory exists and contains files.""" + root = get_hyperactive_root() + examples_dir = root / "examples" + + assert examples_dir.exists(), f"Examples directory not found: {examples_dir}" + assert examples_dir.is_dir(), f"Examples path is not a directory: {examples_dir}" + + # Check that we have some examples + python_files = list(examples_dir.rglob("*.py")) + assert len(python_files) > 0, "No Python example files found" + + print(f"Found {len(python_files)} Python example files") + + def test_backend_directories_exist(self): + """Test that all expected backend directories exist.""" + root = get_hyperactive_root() + examples_dir = root / "examples" + + expected_backends = ["gfo", "sklearn", "optuna"] + + for backend in expected_backends: + backend_dir = examples_dir / backend + assert backend_dir.exists(), f"Backend directory not found: {backend_dir}" + assert backend_dir.is_dir(), f"Backend path is not a directory: {backend_dir}" + + # Check that each backend has Python files + py_files = list(backend_dir.glob("*.py")) + assert len(py_files) > 0, f"No Python files found in {backend} backend" + + def test_readme_files_exist(self): + """Test that README files exist for each backend.""" + root = get_hyperactive_root() + examples_dir = root / "examples" + + expected_backends = ["gfo", "sklearn", "optuna"] + + for backend in expected_backends: + readme_path = examples_dir / backend / "README.md" + assert readme_path.exists(), f"README.md not found for {backend} backend" + assert readme_path.is_file(), f"README.md is not a file for {backend} backend" + + # Check that README is not empty + content = readme_path.read_text() + assert len(content.strip()) > 0, f"README.md is empty for {backend} backend" + + +def main(): + """Main function to run tests directly.""" + # Find all examples + examples = find_all_python_examples() + + if not examples: + print("No example files found!") + return 1 + + print(f"Found {len(examples)} example files to test") + + failed_examples = [] + + # Test each example + for i, example_path in enumerate(examples, 1): + root = get_hyperactive_root() + relative_path = example_path.relative_to(root) + + print(f"[{i}/{len(examples)}] Testing: {relative_path}") + + success, stdout, stderr = run_example(example_path) + + if success: + print(f"✓ PASSED: {relative_path}") + else: + print(f"✗ FAILED: {relative_path}") + if stderr: + print(f" Error: {stderr[:200]}{'...' if len(stderr) > 200 else ''}") + failed_examples.append(relative_path) + + # Print summary + print(f"\\n{'='*50}") + print(f"SUMMARY: {len(examples) - len(failed_examples)}/{len(examples)} examples passed") + + if failed_examples: + print(f"\\nFailed examples:") + for failed in failed_examples: + print(f" - {failed}") + return 1 + else: + print("\\n🎉 All examples passed!") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/tested_and_supported_packages/catboost_example.py b/examples/tested_and_supported_packages/catboost_example.py deleted file mode 100644 index 370be4e4..00000000 --- a/examples/tested_and_supported_packages/catboost_example.py +++ /dev/null @@ -1,27 +0,0 @@ -from sklearn.model_selection import cross_val_score -from catboost import CatBoostClassifier -from sklearn.datasets import load_breast_cancer -from hyperactive import Hyperactive - -data = load_breast_cancer() -X, y = data.data, data.target - - -def model(opt): - cbc = CatBoostClassifier( - iterations=10, depth=opt["depth"], learning_rate=opt["learning_rate"] - ) - scores = cross_val_score(cbc, X, y, cv=3) - - return scores.mean() - - -search_space = { - "depth": list(range(2, 12)), - "learning_rate": [1e-3, 1e-2, 1e-1, 0.5, 1.0], -} - - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=10) -hyper.run() diff --git a/examples/tested_and_supported_packages/joblib_example.py b/examples/tested_and_supported_packages/joblib_example.py deleted file mode 100644 index 577fb25d..00000000 --- a/examples/tested_and_supported_packages/joblib_example.py +++ /dev/null @@ -1,92 +0,0 @@ -import numpy as np -from sklearn.model_selection import cross_val_score -from sklearn.ensemble import GradientBoostingClassifier -from sklearn.ensemble import RandomForestClassifier -from sklearn.ensemble import ExtraTreesClassifier -from xgboost import XGBClassifier -from sklearn.datasets import load_breast_cancer -from hyperactive import Hyperactive - -data = load_breast_cancer() -X, y = data.data, data.target - - -def model_etc(opt): - etc = ExtraTreesClassifier( - n_estimators=opt["n_estimators"], - criterion=opt["criterion"], - max_features=opt["max_features"], - min_samples_split=opt["min_samples_split"], - min_samples_leaf=opt["min_samples_leaf"], - bootstrap=opt["bootstrap"], - ) - scores = cross_val_score(etc, X, y, cv=3) - - return scores.mean() - - -def model_rfc(opt): - rfc = RandomForestClassifier( - n_estimators=opt["n_estimators"], - criterion=opt["criterion"], - max_features=opt["max_features"], - min_samples_split=opt["min_samples_split"], - min_samples_leaf=opt["min_samples_leaf"], - bootstrap=opt["bootstrap"], - ) - scores = cross_val_score(rfc, X, y, cv=3) - - return scores.mean() - - -def model_gbc(opt): - gbc = GradientBoostingClassifier( - n_estimators=opt["n_estimators"], - learning_rate=opt["learning_rate"], - max_depth=opt["max_depth"], - min_samples_split=opt["min_samples_split"], - min_samples_leaf=opt["min_samples_leaf"], - subsample=opt["subsample"], - max_features=opt["max_features"], - ) - scores = cross_val_score(gbc, X, y, cv=3) - - return scores.mean() - - -search_space_etc = { - "n_estimators": list(range(10, 200, 10)), - "criterion": ["gini", "entropy"], - "max_features": list(np.arange(0.05, 1.01, 0.05)), - "min_samples_split": list(range(2, 21)), - "min_samples_leaf": list(range(1, 21)), - "bootstrap": [True, False], -} - - -search_space_rfc = { - "n_estimators": list(range(10, 200, 10)), - "criterion": ["gini", "entropy"], - "max_features": list(np.arange(0.05, 1.01, 0.05)), - "min_samples_split": list(range(2, 21)), - "min_samples_leaf": list(range(1, 21)), - "bootstrap": [True, False], -} - - -search_space_gbc = { - "n_estimators": list(range(10, 200, 10)), - "learning_rate": [1e-3, 1e-2, 1e-1, 0.5, 1.0], - "max_depth": list(range(1, 11)), - "min_samples_split": list(range(2, 21)), - "min_samples_leaf": list(range(1, 21)), - "subsample": list(np.arange(0.05, 1.01, 0.05)), - "max_features": list(np.arange(0.05, 1.01, 0.05)), -} - - -hyper = Hyperactive(distribution="joblib") -hyper.add_search(model_etc, search_space_etc, n_iter=50) -hyper.add_search(model_rfc, search_space_rfc, n_iter=50) -hyper.add_search(model_gbc, search_space_gbc, n_iter=50) -hyper.run(max_time=5) diff --git a/examples/tested_and_supported_packages/keras_example.py b/examples/tested_and_supported_packages/keras_example.py deleted file mode 100644 index c2cd3929..00000000 --- a/examples/tested_and_supported_packages/keras_example.py +++ /dev/null @@ -1,86 +0,0 @@ -import tensorflow as tf -from keras.models import Sequential -from keras.layers import ( - Dense, - Conv2D, - MaxPooling2D, - Flatten, - Dropout, - Activation, -) -from keras.datasets import cifar10 -from keras.utils import to_categorical - -from hyperactive import Hyperactive - - -config = tf.compat.v1.ConfigProto() -config.gpu_options.allow_growth = True -config.log_device_placement = True - -sess = tf.compat.v1.Session(config=config) -tf.compat.v1.keras.backend.set_session(sess) - - -(X_train, y_train), (X_test, y_test) = cifar10.load_data() - -y_train = to_categorical(y_train, 10) -y_test = to_categorical(y_test, 10) - - -# to make the example quick -X_train = X_train[0:1000] -y_train = y_train[0:1000] - - -X_test = X_test[0:1000] -y_test = y_test[0:1000] - - -def cnn(opt): - nn = Sequential() - nn.add( - Conv2D( - opt["filter.0"], - (3, 3), - padding="same", - input_shape=X_train.shape[1:], - ) - ) - nn.add(Activation("relu")) - nn.add(Conv2D(opt["filter.0"], (3, 3))) - nn.add(Activation("relu")) - nn.add(MaxPooling2D(pool_size=(2, 2))) - nn.add(Dropout(0.25)) - - nn.add(Conv2D(opt["filter.0"], (3, 3), padding="same")) - nn.add(Activation("relu")) - nn.add(Conv2D(opt["filter.0"], (3, 3))) - nn.add(Activation("relu")) - nn.add(MaxPooling2D(pool_size=(2, 2))) - nn.add(Dropout(0.25)) - - nn.add(Flatten()) - nn.add(Dense(opt["layer.0"])) - nn.add(Activation("relu")) - nn.add(Dropout(0.5)) - nn.add(Dense(10)) - nn.add(Activation("softmax")) - - nn.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"]) - nn.fit(X_train, y_train, epochs=20, batch_size=512) - - _, score = nn.evaluate(x=X_test, y=y_test) - - return score - - -search_space = { - "filter.0": [16, 32, 64, 128], - "layer.0": list(range(100, 1000, 100)), -} - - -hyper = Hyperactive() -hyper.add_search(cnn, search_space, n_iter=5) -hyper.run() diff --git a/examples/tested_and_supported_packages/lightgbm_example.py b/examples/tested_and_supported_packages/lightgbm_example.py deleted file mode 100644 index 0055d1e7..00000000 --- a/examples/tested_and_supported_packages/lightgbm_example.py +++ /dev/null @@ -1,30 +0,0 @@ -from sklearn.model_selection import cross_val_score -from lightgbm import LGBMRegressor -from sklearn.datasets import load_diabetes -from hyperactive import Hyperactive - -data = load_diabetes() -X, y = data.data, data.target - - -def model(opt): - lgbm = LGBMRegressor( - num_leaves=opt["num_leaves"], - bagging_freq=opt["bagging_freq"], - learning_rate=opt["learning_rate"], - ) - scores = cross_val_score(lgbm, X, y, cv=3) - - return scores.mean() - - -search_space = { - "num_leaves": list(range(2, 50)), - "bagging_freq": list(range(2, 12)), - "learning_rate": [1e-3, 1e-2, 1e-1, 0.5, 1.0], -} - - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=20) -hyper.run() diff --git a/examples/tested_and_supported_packages/mlxtend_example.py b/examples/tested_and_supported_packages/mlxtend_example.py deleted file mode 100644 index 4f6091a5..00000000 --- a/examples/tested_and_supported_packages/mlxtend_example.py +++ /dev/null @@ -1,43 +0,0 @@ -from sklearn.datasets import load_breast_cancer -from sklearn.model_selection import cross_val_score -from mlxtend.classifier import EnsembleVoteClassifier -from sklearn.tree import DecisionTreeClassifier -from sklearn.neural_network import MLPClassifier -from sklearn.svm import SVC -from hyperactive import Hyperactive - - -data = load_breast_cancer() -X, y = data.data, data.target - - -def model(opt): - dtc = DecisionTreeClassifier( - min_samples_split=opt["min_samples_split"], - min_samples_leaf=opt["min_samples_leaf"], - ) - mlp = MLPClassifier(hidden_layer_sizes=opt["hidden_layer_sizes"]) - svc = SVC(C=opt["C"], degree=opt["degree"], gamma="auto", probability=True) - - eclf = EnsembleVoteClassifier( - clfs=[dtc, mlp, svc], weights=opt["weights"], voting="soft", - ) - - scores = cross_val_score(eclf, X, y, cv=3) - - return scores.mean() - - -search_space = { - "min_samples_split": list(range(2, 15)), - "min_samples_leaf": list(range(1, 15)), - "hidden_layer_sizes": list(range(5, 50, 5)), - "weights": [[1, 1, 1], [2, 1, 1], [1, 2, 1], [1, 1, 2]], - "C": list(range(1, 1000)), - "degree": list(range(0, 8)), -} - - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=25) -hyper.run() diff --git a/examples/tested_and_supported_packages/multiprocessing_example.py b/examples/tested_and_supported_packages/multiprocessing_example.py deleted file mode 100644 index 0294ea8b..00000000 --- a/examples/tested_and_supported_packages/multiprocessing_example.py +++ /dev/null @@ -1,113 +0,0 @@ -""" -Hyperactive can perform optimizations of multiple different objective functions -in parallel. This can be done via multiprocessing, joblib or a custom wrapper-function. -The processes won't communicate with each other. - -You can add as many searches to the optimization run (.add_search(...)) and -run each of those searches n-times (n_jobs). - -In the example below we are performing 4 searches in parallel: - - model_etc one time - - model_rfc one time - - model_gbc two times - -""" -import numpy as np -from sklearn.model_selection import cross_val_score -from sklearn.ensemble import GradientBoostingClassifier -from sklearn.ensemble import RandomForestClassifier -from sklearn.ensemble import ExtraTreesClassifier -from xgboost import XGBClassifier -from sklearn.datasets import load_breast_cancer -from hyperactive import Hyperactive - -data = load_breast_cancer() -X, y = data.data, data.target - - -def model_etc(opt): - etc = ExtraTreesClassifier( - n_estimators=opt["n_estimators"], - criterion=opt["criterion"], - max_features=opt["max_features"], - min_samples_split=opt["min_samples_split"], - min_samples_leaf=opt["min_samples_leaf"], - bootstrap=opt["bootstrap"], - ) - scores = cross_val_score(etc, X, y, cv=3) - - return scores.mean() - - -def model_rfc(opt): - rfc = RandomForestClassifier( - n_estimators=opt["n_estimators"], - criterion=opt["criterion"], - max_features=opt["max_features"], - min_samples_split=opt["min_samples_split"], - min_samples_leaf=opt["min_samples_leaf"], - bootstrap=opt["bootstrap"], - ) - scores = cross_val_score(rfc, X, y, cv=3) - - return scores.mean() - - -def model_gbc(opt): - gbc = GradientBoostingClassifier( - n_estimators=opt["n_estimators"], - learning_rate=opt["learning_rate"], - max_depth=opt["max_depth"], - min_samples_split=opt["min_samples_split"], - min_samples_leaf=opt["min_samples_leaf"], - subsample=opt["subsample"], - max_features=opt["max_features"], - ) - scores = cross_val_score(gbc, X, y, cv=3) - - return scores.mean() - - -search_space_etc = { - "n_estimators": list(range(10, 200, 10)), - "criterion": ["gini", "entropy"], - "max_features": list(np.arange(0.05, 1.01, 0.05)), - "min_samples_split": list(range(2, 21)), - "min_samples_leaf": list(range(1, 21)), - "bootstrap": [True, False], -} - - -search_space_rfc = { - "n_estimators": list(range(10, 200, 10)), - "criterion": ["gini", "entropy"], - "max_features": list(np.arange(0.05, 1.01, 0.05)), - "min_samples_split": list(range(2, 21)), - "min_samples_leaf": list(range(1, 21)), - "bootstrap": [True, False], -} - - -search_space_gbc = { - "n_estimators": list(range(10, 200, 10)), - "learning_rate": [1e-3, 1e-2, 1e-1, 0.5, 1.0], - "max_depth": list(range(1, 11)), - "min_samples_split": list(range(2, 21)), - "min_samples_leaf": list(range(1, 21)), - "subsample": list(np.arange(0.05, 1.01, 0.05)), - "max_features": list(np.arange(0.05, 1.01, 0.05)), -} - - -hyper = Hyperactive() -hyper.add_search(model_etc, search_space_etc, n_iter=50) -hyper.add_search(model_rfc, search_space_rfc, n_iter=50) -hyper.add_search(model_gbc, search_space_gbc, n_iter=50, n_jobs=2) -hyper.run(max_time=5) - -search_data_etc = hyper.search_data(model_etc) -search_data_rfc = hyper.search_data(model_rfc) -search_data_gbc = hyper.search_data(model_gbc) - -print("\n ExtraTreesClassifier search data \n", search_data_etc) -print("\n GradientBoostingClassifier search data \n", search_data_gbc) diff --git a/examples/tested_and_supported_packages/pytorch_example.py b/examples/tested_and_supported_packages/pytorch_example.py deleted file mode 100644 index 81773cc8..00000000 --- a/examples/tested_and_supported_packages/pytorch_example.py +++ /dev/null @@ -1,108 +0,0 @@ -import os - -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim -import torch.utils.data -from torchvision import datasets -from torchvision import transforms - -from hyperactive import Hyperactive - - -""" -derived from optuna example: -https://github.com/optuna/optuna/blob/master/examples/pytorch_simple.py -""" -DEVICE = torch.device("cpu") -BATCHSIZE = 256 -CLASSES = 10 -DIR = os.getcwd() -EPOCHS = 10 -LOG_INTERVAL = 10 -N_TRAIN_EXAMPLES = BATCHSIZE * 30 -N_VALID_EXAMPLES = BATCHSIZE * 10 - - -# Get the MNIST dataset. -train_loader = torch.utils.data.DataLoader( - datasets.MNIST(DIR, train=True, download=True, transform=transforms.ToTensor()), - batch_size=BATCHSIZE, - shuffle=True, -) -valid_loader = torch.utils.data.DataLoader( - datasets.MNIST(DIR, train=False, transform=transforms.ToTensor()), - batch_size=BATCHSIZE, - shuffle=True, -) - - -def pytorch_cnn(params): - linear0 = params["linear.0"] - linear1 = params["linear.1"] - - layers = [] - - in_features = 28 * 28 - - layers.append(nn.Linear(in_features, linear0)) - layers.append(nn.ReLU()) - layers.append(nn.Dropout(0.2)) - - layers.append(nn.Linear(linear0, linear1)) - layers.append(nn.ReLU()) - layers.append(nn.Dropout(0.2)) - - layers.append(nn.Linear(linear1, CLASSES)) - layers.append(nn.LogSoftmax(dim=1)) - - model = nn.Sequential(*layers) - - # model = create_model(params).to(DEVICE) - optimizer = getattr(optim, "Adam")(model.parameters(), lr=0.01) - - # Training of the model. - for epoch in range(EPOCHS): - model.train() - for batch_idx, (data, target) in enumerate(train_loader): - # Limiting training data for faster epochs. - if batch_idx * BATCHSIZE >= N_TRAIN_EXAMPLES: - break - - data, target = data.view(data.size(0), -1).to(DEVICE), target.to(DEVICE) - - optimizer.zero_grad() - output = model(data) - loss = F.nll_loss(output, target) - loss.backward() - optimizer.step() - - # Validation of the model. - model.eval() - correct = 0 - with torch.no_grad(): - for batch_idx, (data, target) in enumerate(valid_loader): - # Limiting validation data. - if batch_idx * BATCHSIZE >= N_VALID_EXAMPLES: - break - data, target = data.view(data.size(0), -1).to(DEVICE), target.to(DEVICE) - output = model(data) - # Get the index of the max log-probability. - pred = output.argmax(dim=1, keepdim=True) - correct += pred.eq(target.view_as(pred)).sum().item() - - accuracy = correct / min(len(valid_loader.dataset), N_VALID_EXAMPLES) - - return accuracy - - -search_space = { - "linear.0": list(range(10, 200, 10)), - "linear.1": list(range(10, 200, 10)), -} - - -hyper = Hyperactive() -hyper.add_search(pytorch_cnn, search_space, n_iter=5) -hyper.run() diff --git a/examples/tested_and_supported_packages/read_hdf5.py b/examples/tested_and_supported_packages/read_hdf5.py deleted file mode 100644 index ad0ce8c4..00000000 --- a/examples/tested_and_supported_packages/read_hdf5.py +++ /dev/null @@ -1,24 +0,0 @@ -import h5py - -filename = "my_model_weights_noTraining.h5" - -with h5py.File(filename, "r") as f: - # Print all root level object names (aka keys) - # these can be group or dataset names - print("Keys: %s" % f.keys()) - # get first object name/key; may or may NOT be a group - a_group_key = list(f.keys())[0] - - # get the object type for a_group_key: usually group or dataset - print("\n type a_group_key \n", type(f[a_group_key]), "\n") - - # If a_group_key is a group name, - # this gets the object names in the group and returns as a list - data = list(f[a_group_key]) - - # If a_group_key is a dataset name, - # this gets the dataset values and returns as a list - data = list(f[a_group_key]) - print("\n data \n", data, "\n") - # preferred methods to get dataset values: - ds_obj = f[a_group_key] # returns as a h5py dataset object diff --git a/examples/tested_and_supported_packages/rgf_example.py b/examples/tested_and_supported_packages/rgf_example.py deleted file mode 100644 index c61da256..00000000 --- a/examples/tested_and_supported_packages/rgf_example.py +++ /dev/null @@ -1,33 +0,0 @@ -from sklearn.datasets import load_breast_cancer -from sklearn.model_selection import cross_val_score -from rgf.sklearn import RGFClassifier - -from hyperactive import Hyperactive - -data = load_breast_cancer() -X, y = data.data, data.target - - -def model(opt): - rgf = RGFClassifier( - max_leaf=opt["max_leaf"], - reg_depth=opt["reg_depth"], - min_samples_leaf=opt["min_samples_leaf"], - algorithm="RGF_Sib", - test_interval=100, - verbose=False, - ) - scores = cross_val_score(rgf, X, y, cv=3) - - return scores.mean() - - -search_space = { - "max_leaf": list(range(10, 2000, 10)), - "reg_depth": list(range(1, 21)), - "min_samples_leaf": list(range(1, 21)), -} - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=10) -hyper.run() diff --git a/examples/tested_and_supported_packages/sklearn_example.py b/examples/tested_and_supported_packages/sklearn_example.py deleted file mode 100644 index 20df6cf0..00000000 --- a/examples/tested_and_supported_packages/sklearn_example.py +++ /dev/null @@ -1,30 +0,0 @@ -from sklearn.model_selection import cross_val_score -from sklearn.ensemble import GradientBoostingRegressor -from sklearn.datasets import load_diabetes -from hyperactive import Hyperactive - -data = load_diabetes() -X, y = data.data, data.target - - -def model(opt): - gbr = GradientBoostingRegressor( - n_estimators=opt["n_estimators"], - max_depth=opt["max_depth"], - min_samples_split=opt["min_samples_split"], - ) - scores = cross_val_score(gbr, X, y, cv=3) - - return scores.mean() - - -search_space = { - "n_estimators": list(range(10, 150, 5)), - "max_depth": list(range(2, 12)), - "min_samples_split": list(range(2, 22)), -} - - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=20) -hyper.run() diff --git a/examples/tested_and_supported_packages/tensorflow_example.py b/examples/tested_and_supported_packages/tensorflow_example.py deleted file mode 100644 index c2e1608f..00000000 --- a/examples/tested_and_supported_packages/tensorflow_example.py +++ /dev/null @@ -1,37 +0,0 @@ -import tensorflow as tf - -from hyperactive import Hyperactive - -mnist = tf.keras.datasets.mnist - -(x_train, y_train), (x_test, y_test) = mnist.load_data() -x_train, x_test = x_train / 255.0, x_test / 255.0 - - -def cnn(params): - nn = tf.keras.models.Sequential( - [ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128, activation="relu"), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10), - ] - ) - loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - - nn.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"]) - nn.fit(x_train, y_train, epochs=5) - _, score = nn.evaluate(x=x_test, y=y_test) - - return score - - -search_space = { - "filters_0": [16, 32, 64], - "filters_1": [16, 32, 64], - "dense_0": list(range(100, 2000, 100)), -} - -hyper = Hyperactive() -hyper.add_search(cnn, search_space, n_iter=5) -hyper.run() diff --git a/examples/tested_and_supported_packages/xgboost_example.py b/examples/tested_and_supported_packages/xgboost_example.py deleted file mode 100644 index a42a9dcc..00000000 --- a/examples/tested_and_supported_packages/xgboost_example.py +++ /dev/null @@ -1,30 +0,0 @@ -from sklearn.model_selection import cross_val_score -from xgboost import XGBClassifier -from sklearn.datasets import load_breast_cancer -from hyperactive import Hyperactive - -data = load_breast_cancer() -X, y = data.data, data.target - - -def model(opt): - xgb = XGBClassifier( - n_estimators=opt["n_estimators"], - max_depth=opt["max_depth"], - learning_rate=opt["learning_rate"], - ) - scores = cross_val_score(xgb, X, y, cv=3) - - return scores.mean() - - -search_space = { - "n_estimators": list(range(10, 200, 10)), - "max_depth": list(range(2, 12)), - "learning_rate": [1e-3, 1e-2, 1e-1, 0.5, 1.0], -} - - -hyper = Hyperactive() -hyper.add_search(model, search_space, n_iter=30) -hyper.run() diff --git a/examples/v5_API_example/_optimizer_example.py b/examples/v5_API_example/_optimizer_example.py deleted file mode 100644 index 5fcc200c..00000000 --- a/examples/v5_API_example/_optimizer_example.py +++ /dev/null @@ -1,37 +0,0 @@ -import numpy as np -from sklearn.datasets import load_diabetes -from sklearn.tree import DecisionTreeRegressor - - -from hyperactive.search_config import SearchConfig -from hyperactive.optimization.gradient_free_optimizers import ( - HillClimbingOptimizer, - RandomRestartHillClimbingOptimizer, - RandomSearchOptimizer, -) -from hyperactive.optimization.talos import TalosOptimizer - -from experiments.sklearn import SklearnExperiment -from experiments.test_function import AckleyFunction - - -data = load_diabetes() -X, y = data.data, data.target - - -search_config1 = SearchConfig( - max_depth=list(np.arange(2, 15, 1)), - min_samples_split=list(np.arange(2, 25, 2)), -) - - -TalosOptimizer() - -experiment1 = SklearnExperiment() -experiment1.setup(DecisionTreeRegressor, X, y, cv=4) - - -optimizer = HillClimbingOptimizer() -optimizer.add_search(experiment1, search_config1, n_iter=100) -hyper = optimizer -hyper.run(max_time=5) diff --git a/examples/v5_API_example/test.py b/examples/v5_API_example/test.py deleted file mode 100644 index 17a6fe6c..00000000 --- a/examples/v5_API_example/test.py +++ /dev/null @@ -1,35 +0,0 @@ -finetuned_opt = OptPipe([("first", FooOpt())("second", BarOpt(params))], more_params) - -column_transformer = BetterColumnTransformer( - [ - {"name": "num", "transformer": StandardScaler(), "columns": ["age", "income"]}, - {"name": "cat", "transformer": OneHotEncoder(), "columns": ["gender", "city"]}, - ] -) - - -class MyPipeline(Pipeline): - def transform(self, data): - numeric = self.columns(["age", "income"]).apply(StandardScaler()) - categorical = self.columns(["gender", "city"]).apply(OneHotEncoder()) - combined = self.concat(numeric, categorical) - return combined.then(SVC()) - - -finetuned_opt = OptPipe(more_params) -finetuned_opt.add_step(RandomOpt(), fraction=0.3) -finetuned_opt.add_step(fraction=0.4) -finetuned_opt.add_step(fraction=0.3) - -finetuned_opt - - -class OptPipe: - def __init__(self, a, b, c): - self.a = a - self.b = b - self.c = c - - def set_params(self): - # einzige Möglichkeit um a b c zu ändern - pass diff --git a/src/hyperactive/experiment/integrations/sklearn_cv.py b/src/hyperactive/experiment/integrations/sklearn_cv.py index 77485e90..3a649801 100644 --- a/src/hyperactive/experiment/integrations/sklearn_cv.py +++ b/src/hyperactive/experiment/integrations/sklearn_cv.py @@ -1,4 +1,5 @@ """Experiment adapter for sklearn cross-validation experiments.""" + # copyright: hyperactive developers, MIT License (see LICENSE file) from sklearn import clone @@ -109,6 +110,9 @@ def __init__(self, estimator, X, y, scoring=None, cv=None): from sklearn.metrics import make_scorer self._scoring = make_scorer(scoring) + else: + # scoring is a string (scorer name) + self._scoring = check_scoring(self.estimator, scoring=scoring) self.scorer_ = self._scoring # Set the sign of the scoring function @@ -199,6 +203,7 @@ def get_test_params(cls, parameter_set="default"): from sklearn.metrics import accuracy_score, mean_absolute_error from sklearn.model_selection import KFold from sklearn.svm import SVC, SVR + from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor X, y = load_iris(return_X_y=True) params_classif = { @@ -218,6 +223,24 @@ def get_test_params(cls, parameter_set="default"): "y": y, } + X, y = load_iris(return_X_y=True) + params_classif_f1_str = { + "estimator": DecisionTreeClassifier(), + "scoring": "f1", + "cv": 2, + "X": X, + "y": y, + } + + X, y = load_diabetes(return_X_y=True) + params_regress_r2_str = { + "estimator": DecisionTreeRegressor(), + "scoring": "r2", + "cv": 2, + "X": X, + "y": y, + } + X, y = load_diabetes(return_X_y=True) params_all_default = { "estimator": SVR(), @@ -225,7 +248,13 @@ def get_test_params(cls, parameter_set="default"): "y": y, } - return [params_classif, params_regress, params_all_default] + return [ + params_classif, + params_regress, + params_classif_f1_str, + params_regress_r2_str, + params_all_default, + ] @classmethod def _get_score_params(self): @@ -241,9 +270,17 @@ def _get_score_params(self): The parameters to be used for scoring. """ score_params_classif = {"C": 1.0, "kernel": "linear"} + score_params_trees = {"max_depth": 3, "min_samples_split": 2} score_params_regress = {"C": 1.0, "kernel": "linear"} score_params_defaults = {"C": 1.0, "kernel": "linear"} - return [score_params_classif, score_params_regress, score_params_defaults] + params = [ + score_params_classif, + score_params_regress, + score_params_trees, + score_params_trees, + score_params_defaults, + ] + return params def _guess_sign_of_sklmetric(scorer):