diff --git a/eng/Versions.props b/eng/Versions.props index 558c580693..da5f9229e5 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -90,6 +90,7 @@ 4.8.3 5.3.0.1 2.3.0 + 6.0.0 13.0.1 6.0.1 4.4.0 diff --git a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs index 3b822b647f..6c0ba66972 100644 --- a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs +++ b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs @@ -294,6 +294,11 @@ internal SweepableEstimator CreateSweepableEstimator(Func factory(context, param.AsType()), ss); } + internal AutoMLExperiment CreateExperiment() + { + return new AutoMLExperiment(this._context, new AutoMLExperiment.AutoMLExperimentSettings()); + } + internal SweepableEstimator[] BinaryClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) @@ -306,7 +311,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa fastTreeOption.LabelColumnName = labelColumnName; fastTreeOption.FeatureColumnName = featureColumnName; fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateFastTreeBinary(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastTreeBinary(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace(fastTreeOption))); } if (useFastForest) @@ -315,7 +320,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa fastForestOption.LabelColumnName = labelColumnName; fastForestOption.FeatureColumnName = featureColumnName; fastForestOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateFastForestBinary(fastForestOption, fastForestSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastForestBinary(fastForestOption, fastForestSearchSpace ?? new SearchSpace(fastForestOption))); } if (useLgbm) @@ -324,7 +329,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa lgbmOption.LabelColumnName = labelColumnName; lgbmOption.FeatureColumnName = featureColumnName; lgbmOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateLightGbmBinary(lgbmOption, lgbmSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLightGbmBinary(lgbmOption, lgbmSearchSpace ?? new SearchSpace(lgbmOption))); } if (useLbfgs) @@ -333,7 +338,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa lbfgsOption.LabelColumnName = labelColumnName; lbfgsOption.FeatureColumnName = featureColumnName; lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionBinary(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionBinary(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace(lbfgsOption))); } if (useSdca) @@ -342,7 +347,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa sdcaOption.LabelColumnName = labelColumnName; sdcaOption.FeatureColumnName = featureColumnName; sdcaOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionBinary(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionBinary(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); } return res.ToArray(); @@ -360,7 +365,7 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau fastTreeOption.LabelColumnName = labelColumnName; fastTreeOption.FeatureColumnName = featureColumnName; fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateFastTreeOva(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastTreeOva(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace(fastTreeOption))); } if (useFastForest) @@ -369,7 +374,7 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau fastForestOption.LabelColumnName = labelColumnName; fastForestOption.FeatureColumnName = featureColumnName; fastForestOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateFastForestOva(fastForestOption, fastForestSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastForestOva(fastForestOption, fastForestSearchSpace ?? new SearchSpace(fastForestOption))); } if (useLgbm) @@ -378,7 +383,7 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau lgbmOption.LabelColumnName = labelColumnName; lgbmOption.FeatureColumnName = featureColumnName; lgbmOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateLightGbmMulti(lgbmOption, lgbmSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLightGbmMulti(lgbmOption, lgbmSearchSpace ?? new SearchSpace(lgbmOption))); } if (useLbfgs) @@ -387,8 +392,8 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau lbfgsOption.LabelColumnName = labelColumnName; lbfgsOption.FeatureColumnName = featureColumnName; lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionOva(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); - res.Add(SweepableEstimatorFactory.CreateLbfgsMaximumEntropyMulti(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionOva(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace(lbfgsOption))); + res.Add(SweepableEstimatorFactory.CreateLbfgsMaximumEntropyMulti(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace(lbfgsOption))); } if (useSdca) @@ -397,8 +402,8 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau sdcaOption.LabelColumnName = labelColumnName; sdcaOption.FeatureColumnName = featureColumnName; sdcaOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateSdcaMaximumEntropyMulti(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); - res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionOva(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateSdcaMaximumEntropyMulti(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); + res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionOva(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); } return res.ToArray(); @@ -416,8 +421,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN fastTreeOption.LabelColumnName = labelColumnName; fastTreeOption.FeatureColumnName = featureColumnName; fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateFastTreeRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); - res.Add(SweepableEstimatorFactory.CreateFastTreeTweedieRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastTreeRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace(fastTreeOption))); } if (useFastForest) @@ -426,7 +430,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN fastForestOption.LabelColumnName = labelColumnName; fastForestOption.FeatureColumnName = featureColumnName; fastForestOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateFastForestRegression(fastForestOption, fastForestSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastForestRegression(fastForestOption, fastForestSearchSpace ?? new SearchSpace(fastForestOption))); } if (useLgbm) @@ -435,7 +439,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN lgbmOption.LabelColumnName = labelColumnName; lgbmOption.FeatureColumnName = featureColumnName; lgbmOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateLightGbmRegression(lgbmOption, lgbmSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLightGbmRegression(lgbmOption, lgbmSearchSpace ?? new SearchSpace(lgbmOption))); } if (useLbfgs) @@ -444,7 +448,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN lbfgsOption.LabelColumnName = labelColumnName; lbfgsOption.FeatureColumnName = featureColumnName; lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateLbfgsPoissonRegressionRegression(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLbfgsPoissonRegressionRegression(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace(lbfgsOption))); } if (useSdca) @@ -453,7 +457,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN sdcaOption.LabelColumnName = labelColumnName; sdcaOption.FeatureColumnName = featureColumnName; sdcaOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateSdcaRegression(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateSdcaRegression(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); } return res.ToArray(); diff --git a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs index 1848d8c128..20193642ba 100644 --- a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs +++ b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs @@ -39,6 +39,14 @@ public static MultiModelPipeline Append(this IEstimator estimator, return multiModelPipeline; } + public static MultiModelPipeline Append(this MultiModelPipeline pipeline, IEstimator estimator) + { + var sweepableEstimator = new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace()); + var multiModelPipeline = pipeline.Append(sweepableEstimator); + + return multiModelPipeline; + } + public static MultiModelPipeline Append(this SweepableEstimatorPipeline pipeline, params SweepableEstimator[] estimators) { var multiModelPipeline = new MultiModelPipeline(); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs new file mode 100644 index 0000000000..5de91cedfd --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -0,0 +1,284 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.ML.Runtime; +using static Microsoft.ML.DataOperationsCatalog; + +namespace Microsoft.ML.AutoML +{ + internal class AutoMLExperiment + { + private readonly AutoMLExperimentSettings _settings; + private readonly MLContext _context; + private double _bestError = double.MaxValue; + private TrialResult _bestTrialResult = null; + private readonly IServiceCollection _serviceCollection; + + public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) + { + this._context = context; + this._settings = settings; + this._serviceCollection = new ServiceCollection(); + } + + private void InitializeServiceCollection() + { + this._serviceCollection.TryAddSingleton(this._context); + this._serviceCollection.TryAddSingleton(this._settings); + this._serviceCollection.TryAddSingleton(); + this._serviceCollection.TryAddSingleton(); + this._serviceCollection.TryAddSingleton(); + this._serviceCollection.TryAddTransient(); + this._serviceCollection.TryAddTransient(); + this._serviceCollection.TryAddTransient(); + this._serviceCollection.TryAddTransient(); + this._serviceCollection.TryAddTransient(); + this._serviceCollection.TryAddTransient(); + this._serviceCollection.TryAddScoped(); + this._serviceCollection.TryAddScoped(); + } + + public AutoMLExperiment SetTrainingTimeInSeconds(uint trainingTimeInSeconds) + { + this._settings.MaxExperimentTimeInSeconds = trainingTimeInSeconds; + return this; + } + + public AutoMLExperiment SetDataset(IDataView train, IDataView test) + { + this._settings.DatasetSettings = new TrainTestDatasetSettings() + { + TrainDataset = train, + TestDataset = test + }; + + return this; + } + + public AutoMLExperiment SetDataset(TrainTestData trainTestSplit) + { + this.SetDataset(trainTestSplit.TrainSet, trainTestSplit.TestSet); + + return this; + } + + public AutoMLExperiment SetDataset(IDataView dataset, int fold = 10) + { + this._settings.DatasetSettings = new CrossValidateDatasetSettings() + { + Dataset = dataset, + Fold = fold, + }; + + return this; + } + + public AutoMLExperiment SetTunerFactory() + where TTunerFactory : ITunerFactory + { + var descriptor = new ServiceDescriptor(typeof(ITunerFactory), typeof(TTunerFactory), ServiceLifetime.Singleton); + if (this._serviceCollection.Contains(descriptor)) + { + this._serviceCollection.Replace(descriptor); + } + else + { + this._serviceCollection.Add(descriptor); + } + + return this; + } + + public AutoMLExperiment SetMonitor(IMonitor monitor) + { + var descriptor = new ServiceDescriptor(typeof(IMonitor), monitor); + if (this._serviceCollection.Contains(descriptor)) + { + this._serviceCollection.Replace(descriptor); + } + else + { + this._serviceCollection.Add(descriptor); + } + + return this; + } + + public AutoMLExperiment SetPipeline(MultiModelPipeline pipeline) + { + this._settings.Pipeline = pipeline; + return this; + } + + public AutoMLExperiment SetTrialRunnerFactory(ITrialRunnerFactory factory) + { + var descriptor = new ServiceDescriptor(typeof(ITrialRunnerFactory), factory); + if (this._serviceCollection.Contains(descriptor)) + { + this._serviceCollection.Replace(descriptor); + } + else + { + this._serviceCollection.Add(descriptor); + } + + return this; + } + + public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) + { + var res = new MultiModelPipeline(); + foreach (var e in pipeline.Estimators) + { + res = res.Append(e); + } + + this.SetPipeline(res); + + return this; + } + + public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "Predicted") + { + this._settings.EvaluateMetric = new BinaryMetricSettings() + { + Metric = metric, + PredictedColumn = predictedColumn, + LabelColumn = labelColumn, + }; + + return this; + } + + public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, string labelColumn = "label", string predictedColumn = "Predicted") + { + this._settings.EvaluateMetric = new MultiClassMetricSettings() + { + Metric = metric, + PredictedColumn = predictedColumn, + LabelColumn = labelColumn, + }; + + return this; + } + + public AutoMLExperiment SetEvaluateMetric(RegressionMetric metric, string labelColumn = "label", string scoreColumn = "Score") + { + this._settings.EvaluateMetric = new RegressionMetricSettings() + { + Metric = metric, + ScoreColumn = scoreColumn, + LabelColumn = labelColumn, + }; + + return this; + } + + /// + /// Run experiment and return the best trial result. + /// + /// + public Task Run() + { + this.ValidateSettings(); + var cts = new CancellationTokenSource(); + cts.CancelAfter((int)this._settings.MaxExperimentTimeInSeconds * 1000); + this._settings.CancellationToken.Register(() => cts.Cancel()); + var ct = cts.Token; + ct.Register(() => this._context.CancelExecution()); + + return this.RunAsync(ct); + } + + private async Task RunAsync(CancellationToken ct) + { + this.InitializeServiceCollection(); + var serviceProvider = this._serviceCollection.BuildServiceProvider(); + var monitor = serviceProvider.GetService(); + var trialNum = 0; + var pipelineProposer = serviceProvider.GetService(); + var hyperParameterProposer = serviceProvider.GetService(); + var runnerFactory = serviceProvider.GetService(); + + while (true) + { + try + { + if (ct.IsCancellationRequested) + { + break; + } + + var setting = new TrialSettings() + { + ExperimentSettings = this._settings, + TrialId = trialNum++, + }; + + setting = pipelineProposer.Propose(setting); + setting = hyperParameterProposer.Propose(setting); + monitor.ReportRunningTrial(setting); + var runner = runnerFactory.CreateTrialRunner(setting); + var trialResult = runner.Run(setting); + monitor.ReportCompletedTrial(trialResult); + hyperParameterProposer.Update(setting, trialResult); + pipelineProposer.Update(setting, trialResult); + + var error = this._settings.EvaluateMetric.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; + if (error < this._bestError) + { + this._bestTrialResult = trialResult; + this._bestError = error; + monitor.ReportBestTrial(trialResult); + } + } + catch (Exception) + { + if (ct.IsCancellationRequested) + { + break; + } + else + { + throw; + } + } + } + + if (this._bestTrialResult == null) + { + throw new TimeoutException("Training time finished without completing a trial run"); + } + else + { + return await Task.FromResult(this._bestTrialResult); + } + } + + private void ValidateSettings() + { + Contracts.Assert(this._settings.MaxExperimentTimeInSeconds > 0, $"{nameof(ExperimentSettings.MaxExperimentTimeInSeconds)} must be larger than 0"); + Contracts.Assert(this._settings.DatasetSettings != null, $"{nameof(this._settings.DatasetSettings)} must be not null"); + Contracts.Assert(this._settings.EvaluateMetric != null, $"{nameof(this._settings.EvaluateMetric)} must be not null"); + } + + + public class AutoMLExperimentSettings : ExperimentSettings + { + public IDatasetSettings DatasetSettings { get; set; } + + public IMetricSettings EvaluateMetric { get; set; } + + public MultiModelPipeline Pipeline { get; set; } + + public int? Seed { get; set; } + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetSettings.cs new file mode 100644 index 0000000000..023a35ecc6 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetSettings.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.AutoML +{ + internal interface IDatasetSettings + { + } + + internal class TrainTestDatasetSettings : IDatasetSettings + { + public IDataView TrainDataset { get; set; } + + public IDataView TestDataset { get; set; } + } + + internal class CrossValidateDatasetSettings : IDatasetSettings + { + public IDataView Dataset { get; set; } + + public int? Fold { get; set; } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs new file mode 100644 index 0000000000..50c2ef6225 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs @@ -0,0 +1,75 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.ML.Data; + +namespace Microsoft.ML.AutoML +{ + internal interface IMetricSettings + { + bool IsMaximize { get; } + } + + internal class BinaryMetricSettings : IMetricSettings + { + public BinaryClassificationMetric Metric { get; set; } + + public string PredictedColumn { get; set; } + + public string LabelColumn { get; set; } + + public bool IsMaximize => this.Metric switch + { + BinaryClassificationMetric.Accuracy => true, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => true, + BinaryClassificationMetric.AreaUnderRocCurve => true, + BinaryClassificationMetric.PositivePrecision => true, + BinaryClassificationMetric.NegativePrecision => true, + BinaryClassificationMetric.NegativeRecall => true, + BinaryClassificationMetric.PositiveRecall => true, + BinaryClassificationMetric.F1Score => throw new NotImplementedException(), + _ => throw new NotImplementedException(), + }; + } + + internal class MultiClassMetricSettings : IMetricSettings + { + public MulticlassClassificationMetric Metric { get; set; } + + public string PredictedColumn { get; set; } + + public string LabelColumn { get; set; } + + public bool IsMaximize => this.Metric switch + { + MulticlassClassificationMetric.MacroAccuracy => true, + MulticlassClassificationMetric.MicroAccuracy => true, + MulticlassClassificationMetric.LogLoss => false, + MulticlassClassificationMetric.LogLossReduction => false, + MulticlassClassificationMetric.TopKAccuracy => true, + _ => throw new NotImplementedException(), + }; + } + + internal class RegressionMetricSettings : IMetricSettings + { + public RegressionMetric Metric { get; set; } + + public string ScoreColumn { get; set; } + + public string LabelColumn { get; set; } + + public bool IsMaximize => this.Metric switch + { + RegressionMetric.RSquared => true, + RegressionMetric.RootMeanSquaredError => false, + RegressionMetric.MeanSquaredError => false, + RegressionMetric.MeanAbsoluteError => false, + _ => throw new NotImplementedException(), + }; + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs new file mode 100644 index 0000000000..5d25f7645f --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.ML.Runtime; + +namespace Microsoft.ML.AutoML +{ + internal interface IMonitor + { + void ReportCompletedTrial(TrialResult result); + + void ReportBestTrial(TrialResult result); + + void ReportFailTrial(TrialResult result); + + void ReportRunningTrial(TrialSettings setting); + } + + // this monitor redirects output result to context.log + internal class MLContextMonitor : IMonitor + { + private readonly MLContext _context; + private readonly IServiceProvider _serviceProvider; + private readonly IChannel _logger; + private readonly List _completedTrials; + + public MLContextMonitor(MLContext context, IServiceProvider provider) + { + this._context = context; + this._serviceProvider = provider; + this._logger = ((IChannelProvider)context).Start(nameof(AutoMLExperiment)); + this._completedTrials = new List(); + } + + public void ReportBestTrial(TrialResult result) + { + this._logger.Info($"Update Best Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline}"); + } + + public void ReportCompletedTrial(TrialResult result) + { + this._logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline} - Duration: {result.DurationInMilliseconds}"); + this._completedTrials.Add(result); + } + + public void ReportFailTrial(TrialResult result) + { + this._logger.Info($"Update Failed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline}"); + } + + public void ReportRunningTrial(TrialSettings setting) + { + this._logger.Info($"Update Running Trial - Id: {setting.TrialId} - Pipeline: {setting.Pipeline}"); + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs new file mode 100644 index 0000000000..4b0afab04d --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.AutoML +{ + internal class TrialResult + { + public TrialSettings TrialSettings { get; set; } + + public ITransformer Model { get; set; } + + public double Metric { get; set; } + + public double DurationInMilliseconds { get; set; } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs new file mode 100644 index 0000000000..99a29361df --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs @@ -0,0 +1,299 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Diagnostics; + +namespace Microsoft.ML.AutoML +{ + internal interface ITrialRunner + { + TrialResult Run(TrialSettings settings); + } + + internal class BinaryClassificationCVRunner : ITrialRunner + { + private readonly MLContext _context; + public BinaryClassificationCVRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings) + { + var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); + if (settings.ExperimentSettings.DatasetSettings is CrossValidateDatasetSettings datasetSettings + && settings.ExperimentSettings.EvaluateMetric is BinaryMetricSettings metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var fold = datasetSettings.Fold ?? 5; + + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); + var metrics = this._context.BinaryClassification.CrossValidateNonCalibrated(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var res = metrics[rnd.Next(fold)]; + var model = res.Model; + var metric = metricSettings.Metric switch + { + BinaryClassificationMetric.PositivePrecision => res.Metrics.PositivePrecision, + BinaryClassificationMetric.Accuracy => res.Metrics.Accuracy, + BinaryClassificationMetric.AreaUnderRocCurve => res.Metrics.AreaUnderRocCurve, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => res.Metrics.AreaUnderPrecisionRecallCurve, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + } + + throw new ArgumentException(); + } + } + + internal class BinaryClassificationTrainTestRunner : ITrialRunner + { + private readonly MLContext _context; + public BinaryClassificationTrainTestRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings) + { + var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); + if (settings.ExperimentSettings.DatasetSettings is TrainTestDatasetSettings datasetSettings + && settings.ExperimentSettings.EvaluateMetric is BinaryMetricSettings metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); + var model = pipeline.Fit(datasetSettings.TrainDataset); + var eval = model.Transform(datasetSettings.TestDataset); + var metrics = this._context.BinaryClassification.EvaluateNonCalibrated(eval, metricSettings.LabelColumn, predictedLabelColumnName: metricSettings.PredictedColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var metric = metricSettings.Metric switch + { + BinaryClassificationMetric.PositivePrecision => metrics.PositivePrecision, + BinaryClassificationMetric.Accuracy => metrics.Accuracy, + BinaryClassificationMetric.AreaUnderRocCurve => metrics.AreaUnderRocCurve, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => metrics.AreaUnderPrecisionRecallCurve, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + } + + throw new ArgumentException(); + } + } + + internal class MultiClassificationTrainTestRunner : ITrialRunner + { + private readonly MLContext _context; + public MultiClassificationTrainTestRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings) + { + if (settings.ExperimentSettings.DatasetSettings is TrainTestDatasetSettings datasetSettings + && settings.ExperimentSettings.EvaluateMetric is MultiClassMetricSettings metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); + var model = pipeline.Fit(datasetSettings.TrainDataset); + var eval = model.Transform(datasetSettings.TestDataset); + var metrics = this._context.MulticlassClassification.Evaluate(eval, metricSettings.LabelColumn, predictedLabelColumnName: metricSettings.PredictedColumn); + + var metric = metricSettings.Metric switch + { + MulticlassClassificationMetric.MicroAccuracy => metrics.MicroAccuracy, + MulticlassClassificationMetric.MacroAccuracy => metrics.MacroAccuracy, + MulticlassClassificationMetric.TopKAccuracy => metrics.TopKAccuracy, + MulticlassClassificationMetric.LogLoss => metrics.LogLoss, + MulticlassClassificationMetric.LogLossReduction => metrics.LogLossReduction, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + } + + throw new ArgumentException(); + } + } + + internal class MultiClassificationCVRunner : ITrialRunner + { + private readonly MLContext _context; + public MultiClassificationCVRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings) + { + var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); + if (settings.ExperimentSettings.DatasetSettings is CrossValidateDatasetSettings datasetSettings + && settings.ExperimentSettings.EvaluateMetric is MultiClassMetricSettings metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var fold = datasetSettings.Fold ?? 5; + + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); + var metrics = this._context.MulticlassClassification.CrossValidate(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn, seed: settings.ExperimentSettings?.Seed); + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var res = metrics[rnd.Next(fold)]; + var model = res.Model; + var metric = metricSettings.Metric switch + { + MulticlassClassificationMetric.MicroAccuracy => res.Metrics.MicroAccuracy, + MulticlassClassificationMetric.MacroAccuracy => res.Metrics.MacroAccuracy, + MulticlassClassificationMetric.TopKAccuracy => res.Metrics.TopKAccuracy, + MulticlassClassificationMetric.LogLoss => res.Metrics.LogLoss, + MulticlassClassificationMetric.LogLossReduction => res.Metrics.LogLossReduction, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + } + + throw new ArgumentException(); + } + } + + internal class RegressionTrainTestRunner : ITrialRunner + { + private readonly MLContext _context; + public RegressionTrainTestRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings) + { + if (settings.ExperimentSettings.DatasetSettings is TrainTestDatasetSettings datasetSettings + && settings.ExperimentSettings.EvaluateMetric is RegressionMetricSettings metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); + var model = pipeline.Fit(datasetSettings.TrainDataset); + var eval = model.Transform(datasetSettings.TestDataset); + var metrics = this._context.Regression.Evaluate(eval, metricSettings.LabelColumn, scoreColumnName: metricSettings.ScoreColumn); + + var metric = metricSettings.Metric switch + { + RegressionMetric.RootMeanSquaredError => metrics.RootMeanSquaredError, + RegressionMetric.RSquared => metrics.RSquared, + RegressionMetric.MeanSquaredError => metrics.MeanSquaredError, + RegressionMetric.MeanAbsoluteError => metrics.MeanAbsoluteError, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + } + + throw new ArgumentException(); + } + } + + internal class RegressionCVRunner : ITrialRunner + { + private readonly MLContext _context; + public RegressionCVRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings) + { + var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); + if (settings.ExperimentSettings.DatasetSettings is CrossValidateDatasetSettings datasetSettings + && settings.ExperimentSettings.EvaluateMetric is RegressionMetricSettings metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var fold = datasetSettings.Fold ?? 5; + + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); + var metrics = this._context.Regression.CrossValidate(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn, seed: settings.ExperimentSettings?.Seed); + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var res = metrics[rnd.Next(fold)]; + var model = res.Model; + var metric = metricSettings.Metric switch + { + RegressionMetric.RootMeanSquaredError => res.Metrics.RootMeanSquaredError, + RegressionMetric.RSquared => res.Metrics.RSquared, + RegressionMetric.MeanSquaredError => res.Metrics.MeanSquaredError, + RegressionMetric.MeanAbsoluteError => res.Metrics.MeanAbsoluteError, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + } + + throw new ArgumentException(); + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs new file mode 100644 index 0000000000..e4ed5f293c --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Extensions.DependencyInjection; + +#nullable enable +namespace Microsoft.ML.AutoML +{ + internal interface ITrialRunnerFactory + { + ITrialRunner? CreateTrialRunner(TrialSettings settings); + } + + internal class TrialRunnerFactory : ITrialRunnerFactory + { + private readonly IServiceProvider _provider; + + public TrialRunnerFactory(IServiceProvider provider) + { + this._provider = provider; + } + + public ITrialRunner? CreateTrialRunner(TrialSettings settings) + { + ITrialRunner? runner = (settings.ExperimentSettings.DatasetSettings, settings.ExperimentSettings.EvaluateMetric) switch + { + (CrossValidateDatasetSettings, BinaryMetricSettings) => this._provider.GetService(), + (TrainTestDatasetSettings, BinaryMetricSettings) => this._provider.GetService(), + (CrossValidateDatasetSettings, MultiClassMetricSettings) => this._provider.GetService(), + (TrainTestDatasetSettings, MultiClassMetricSettings) => this._provider.GetService(), + (CrossValidateDatasetSettings, RegressionMetricSettings) => this._provider.GetService(), + (TrainTestDatasetSettings, RegressionMetricSettings) => this._provider.GetService(), + _ => throw new NotImplementedException(), + }; + + return runner; + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs new file mode 100644 index 0000000000..2b7b6ac4d0 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + internal class TrialSettings + { + public int TrialId { get; set; } + + public SweepableEstimatorPipeline Pipeline { get; set; } + + public string Schema { get; set; } + + public Parameter Parameter { get; set; } + + public AutoMLExperiment.AutoMLExperimentSettings ExperimentSettings { get; set; } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs new file mode 100644 index 0000000000..3e98895f54 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs @@ -0,0 +1,48 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + internal class HyperParameterProposer : ITrialSettingsProposer + { + private readonly Dictionary _tuners; + private readonly IServiceProvider _provider; + + public HyperParameterProposer(IServiceProvider provider) + { + this._tuners = new Dictionary(); + this._provider = provider; + } + + public TrialSettings Propose(TrialSettings settings) + { + var tunerFactory = this._provider.GetService(); + if (!this._tuners.ContainsKey(settings.Schema)) + { + var t = tunerFactory.CreateTuner(settings); + this._tuners.Add(settings.Schema, t); + } + + var tuner = this._tuners[settings.Schema]; + var parameter = tuner.Propose(settings); + settings.Parameter = parameter; + + return settings; + } + + public void Update(TrialSettings settings, TrialResult result) + { + var schema = settings.Schema; + if (this._tuners.TryGetValue(schema, out var tuner)) + { + tuner.Update(result); + } + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs new file mode 100644 index 0000000000..ad73331fab --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML +{ + internal interface ITrialSettingsProposer + { + TrialSettings Propose(TrialSettings settings); + + void Update(TrialSettings parameter, TrialResult result); + } + + internal interface ISavableProposer : ITrialSettingsProposer + { + void SaveStatusToFile(string fileName); + + void LoadStatusFromFile(string fileName); + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs new file mode 100644 index 0000000000..6f619d0e93 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs @@ -0,0 +1,270 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.ML.AutoML.CodeGen; +using Newtonsoft.Json; +using static Microsoft.ML.AutoML.AutoMLExperiment; + +namespace Microsoft.ML.AutoML +{ + /// + /// propose sweepable estimator pipeline from a group of candidates using eci in flaml (https://arxiv.org/abs/1911.04706) + /// + internal class PipelineProposer : ISavableProposer + { + private readonly Dictionary _estimatorCost; + private Dictionary _learnerInitialCost; + + // total time spent on last best error for each learner. + private Dictionary _k1; + + // total time spent on second last best error for each learner. + private Dictionary _k2; + + // last best error for each learner. + private Dictionary _e1; + + // second last best error for each learner. + private Dictionary _e2; + + // flaml ECI + private Dictionary _eci; + private double _globalBestError; + + private readonly Random _rand; + private MultiModelPipeline _multiModelPipeline; + + public PipelineProposer(AutoMLExperimentSettings settings) + { + // this cost is used to initialize eci when started, the smaller the number, the less cost this trainer will use at start, and more likely it will be + // picked. + this._estimatorCost = new Dictionary() + { + { EstimatorType.LightGbmRegression, 0.788 }, + { EstimatorType.FastTreeRegression, 0.382 }, + { EstimatorType.FastForestRegression, 0.374 }, + { EstimatorType.SdcaRegression, 0.566 }, + { EstimatorType.FastTreeTweedieRegression, 0.401 }, + { EstimatorType.LbfgsPoissonRegressionRegression, 4.73 }, + { EstimatorType.FastForestOva, 4.283 }, + { EstimatorType.FastTreeOva, 3.701 }, + { EstimatorType.LightGbmMulti, 14.765 }, + { EstimatorType.SdcaMaximumEntropyMulti, 1.129 }, + { EstimatorType.SdcaLogisticRegressionOva, 3.16 }, + { EstimatorType.LbfgsMaximumEntropyMulti, 7.980 }, + { EstimatorType.LbfgsLogisticRegressionOva, 11.513 }, + { EstimatorType.LightGbmBinary, 14.765 }, + { EstimatorType.FastTreeBinary, 3.701 }, + { EstimatorType.FastForestBinary, 4.283 }, + { EstimatorType.SdcaLogisticRegressionBinary, 3.16 }, + { EstimatorType.LbfgsLogisticRegressionBinary, 11.513 }, + { EstimatorType.ForecastBySsa, 1 }, + { EstimatorType.ImageClassificationMulti, 1 }, + { EstimatorType.MatrixFactorization, 1 }, + }; + this._rand = new Random(settings.Seed ?? 0); + + this._multiModelPipeline = null; + } + + public TrialSettings Propose(TrialSettings settings) + { + this._multiModelPipeline = settings.ExperimentSettings.Pipeline; + this._learnerInitialCost = _multiModelPipeline.PipelineIds.ToDictionary(kv => kv, kv => this.GetEstimatedCostForPipeline(kv, this._multiModelPipeline)); + var pipelineIds = this._multiModelPipeline.PipelineIds; + + if (this._eci == null) + { + // initialize eci with the estimated cost and always start from pipeline which has lowest cost. + this._eci = pipelineIds.ToDictionary(kv => kv, kv => this.GetEstimatedCostForPipeline(kv, this._multiModelPipeline)); + settings.Schema = this._eci.OrderBy(kv => kv.Value).First().Key; + } + else + { + var probabilities = pipelineIds.Select(id => this._eci[id]).ToArray(); + probabilities = ArrayMath.Inverse(probabilities); + probabilities = ArrayMath.Normalize(probabilities); + + // sample + var randdouble = this._rand.NextDouble(); + var sum = 0.0; + // selected pipeline id index + int i; + + for (i = 0; i != pipelineIds.Length; ++i) + { + sum += ((double[])probabilities)[i]; + if (sum > randdouble) + { + break; + } + } + + settings.Schema = pipelineIds[i]; + } + + settings.Pipeline = this._multiModelPipeline.BuildSweepableEstimatorPipeline(settings.Schema); + return settings; + } + + public void SaveStatusToFile(string fileName) + { + using (var writer = new FileStream(fileName, FileMode.Create)) + { + this.SaveStatusToStream(writer); + } + } + + public void SaveStatusToStream(Stream stream) + { + var status = new Status() + { + K1 = this._k1, + K2 = this._k2, + E1 = this._e1, + E2 = this._e2, + Eci = this._eci, + GlobalBestError = this._globalBestError, + }; + + using (var fileWriter = new StreamWriter(stream)) + { + var json = JsonConvert.SerializeObject(status); + fileWriter.Write(json); + } + } + + public void LoadStatusFromFile(string fileName) + { + if (File.Exists(fileName)) + { + var json = File.ReadAllText(fileName); + var status = JsonConvert.DeserializeObject(json); + this._k1 = status.K1; + this._k2 = status.K2; + this._e1 = status.E1; + this._e2 = status.E2; + this._eci = status.Eci; + this._globalBestError = status.GlobalBestError; + } + } + + public void Update(TrialSettings parameter, TrialResult result) + { + var schema = parameter.Schema; + var error = this.CaculateError(result.Metric, result.TrialSettings.ExperimentSettings.EvaluateMetric.IsMaximize); + var duration = result.DurationInMilliseconds / 1000; + var pipelineIds = this._multiModelPipeline.PipelineIds; + var isSuccess = duration != 0; + + // if k1 is null, it means this is the first completed trial. + // in that case, initialize k1, k2, e1, e2 in the following way: + // k1: for every learner, k1[l] = c * duration where c is a ratio defined in learnerInitialCost + // k2: k2 = k1, which indicates the hypothesis that it costs the same time for learners to reach the next break through. + // e1: current error + // e2: 1.001*e1 + + if (isSuccess) + { + if (this._k1 == null) + { + this._k1 = pipelineIds.ToDictionary(id => id, id => duration * this._learnerInitialCost[id] / this._learnerInitialCost[schema]); + this._k2 = this._k1.ToDictionary(kv => kv.Key, kv => kv.Value); + this._e1 = pipelineIds.ToDictionary(id => id, id => error); + this._e2 = pipelineIds.ToDictionary(id => id, id => 1.05 * error); + this._globalBestError = error; + } + else if (error >= this._e1[schema]) + { + // if error is larger than current best error, which means there's no improvements for + // the last trial with the current learner. + // In that case, simply increase the total spent time since the last best error for that learner. + this._k1[schema] += duration; + } + else + { + // there's an improvement. + // k2 <= k1 && e2 <= e1, and update k1, e2. + this._k2[schema] = this._k1[schema]; + this._k1[schema] = duration; + this._e2[schema] = this._e1[schema]; + this._e1[schema] = error; + + // update global best error as well + if (error < this._globalBestError) + { + this._globalBestError = error; + } + } + + // update eci + var eci1 = Math.Max(this._k1[schema], this._k2[schema]); + var estimatorCostForBreakThrough = 2 * (error - this._globalBestError) / ((this._e2[schema] - this._e1[schema]) / (this._k2[schema] + this._k1[schema])); + this._eci[schema] = Math.Max(eci1, estimatorCostForBreakThrough); + } + else + { + // double eci of current trial twice of maxium ecis. + this._eci[schema] = this._eci.Select(kv => kv.Value).Max() * 2; + } + + // normalize eci + var sum = this._eci.Select(x => x.Value).Sum(); + this._eci = this._eci.Select(x => (x.Key, x.Value / sum)).ToDictionary(x => x.Key, x => x.Item2); + + // TODO + // save k1,k2,e1,e2,eci,bestError to training configuration + return; + } + + private double CaculateError(double loss, bool isMaximize) + { + return isMaximize ? 1 - loss : loss; + } + + private double GetEstimatedCostForPipeline(string kv, MultiModelPipeline multiModelPipeline) + { + var entity = Entity.FromExpression(kv); + + var estimatorTypes = entity.ValueEntities().Where(v => v is StringEntity s && s.Value != "Nil") + .Select(v => + { + var s = v as StringEntity; + var estimator = multiModelPipeline.Estimators[s.Value]; + return estimator.EstimatorType; + }); + + var res = 1; + + foreach (var estimatorType in estimatorTypes) + { + if (this._estimatorCost.ContainsKey(estimatorType)) + { + return this._estimatorCost[estimatorType]; + } + } + + return res; + } + + public class Status + { + public Dictionary K1 { get; set; } + + public Dictionary K2 { get; set; } + + public Dictionary E1 { get; set; } + + public Dictionary E2 { get; set; } + + public Dictionary Eci { get; set; } + + public double GlobalBestError { get; set; } + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs new file mode 100644 index 0000000000..1f7201d694 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.SearchSpace.Tuner; + +namespace Microsoft.ML.AutoML +{ + internal interface ITunerFactory + { + ITuner CreateTuner(TrialSettings settings); + } + + internal class CostFrugalTunerFactory : ITunerFactory + { + private readonly IServiceProvider _provider; + + public CostFrugalTunerFactory(IServiceProvider provider) + { + this._provider = provider; + } + + public ITuner CreateTuner(TrialSettings settings) + { + var experimentSetting = this._provider.GetService(); + var searchSpace = settings.Pipeline.SearchSpace; + var initParameter = settings.Pipeline.Parameter; + var isMaximize = experimentSetting.EvaluateMetric.IsMaximize; + + return new CostFrugalTuner(searchSpace, initParameter, !isMaximize); + } + } + + internal class RandomTunerFactory : ITunerFactory + { + private readonly IServiceProvider _provider; + + public RandomTunerFactory(IServiceProvider provider) + { + this._provider = provider; + } + + public ITuner CreateTuner(TrialSettings settings) + { + var searchSpace = settings.Pipeline.SearchSpace; + + return new RandomSearchTuner(searchSpace); + } + } + + internal class GridSearchTunerFactory : ITunerFactory + { + private readonly IServiceProvider _provider; + + public GridSearchTunerFactory(IServiceProvider provider) + { + this._provider = provider; + } + + public ITuner CreateTuner(TrialSettings settings) + { + var searchSpace = settings.Pipeline.SearchSpace; + + return new GridSearchTuner(searchSpace); + } + } +} diff --git a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj index 6d79838dff..36a14ec5ce 100644 --- a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj +++ b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj @@ -20,6 +20,7 @@ true + diff --git a/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs b/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs new file mode 100644 index 0000000000..437039abb8 --- /dev/null +++ b/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs @@ -0,0 +1,123 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using Microsoft.ML.AutoMLService; +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + // an implemetation of "Frugal Optimization for Cost-related Hyperparameters" : https://www.aaai.org/AAAI21Papers/AAAI-10128.WuQ.pdf + internal class CostFrugalTuner : ITuner + { + private readonly RandomNumberGenerator _rng = new RandomNumberGenerator(); + private readonly SearchSpace.SearchSpace _searchSpace; + private readonly bool _minimize; + private readonly Flow2 _localSearch; + private readonly Dictionary _searchThreadPool = new Dictionary(); + private int _currentThreadId; + private readonly Dictionary _trialProposedBy = new Dictionary(); + + private readonly Dictionary _configs = new Dictionary(); + private readonly double[] _lsBoundMax; + private readonly double[] _lsBoundMin; + private bool _initUsed = false; + private double _bestMetric; + + public CostFrugalTuner(SearchSpace.SearchSpace searchSpace, Parameter initValue = null, bool minimizeMode = true) + { + this._searchSpace = searchSpace; + this._minimize = minimizeMode; + + this._localSearch = new Flow2(searchSpace, initValue, true); + this._currentThreadId = 0; + this._lsBoundMin = this._searchSpace.MappingToFeatureSpace(initValue); + this._lsBoundMax = this._searchSpace.MappingToFeatureSpace(initValue); + this._initUsed = false; + this._bestMetric = double.MaxValue; + } + + public Parameter Propose(TrialSettings settings) + { + var trialId = settings.TrialId; + if (this._initUsed) + { + var searchThread = this._searchThreadPool[this._currentThreadId]; + this._configs[trialId] = this._searchSpace.MappingToFeatureSpace(searchThread.Suggest(trialId)); + this._trialProposedBy[trialId] = this._currentThreadId; + } + else + { + this._configs[trialId] = this.CreateInitConfigFromAdmissibleRegion(); + this._trialProposedBy[trialId] = this._currentThreadId; + } + + var param = this._configs[trialId]; + return this._searchSpace.SampleFromFeatureSpace(param); + } + + public Parameter BestConfig { get; set; } + + public void Update(TrialResult result) + { + var trialId = result.TrialSettings.TrialId; + var metric = result.Metric; + metric = this._minimize ? metric : -metric; + if (metric < this._bestMetric) + { + this.BestConfig = this._searchSpace.SampleFromFeatureSpace(this._configs[trialId]); + this._bestMetric = metric; + } + + var cost = result.DurationInMilliseconds; + int threadId = this._trialProposedBy[trialId]; + if (this._searchThreadPool.Count == 0) + { + var initParameter = this._searchSpace.SampleFromFeatureSpace(this._configs[trialId]); + this._searchThreadPool[this._currentThreadId] = this._localSearch.CreateSearchThread(initParameter, metric, cost); + this._initUsed = true; + this.UpdateAdmissibleRegion(this._configs[trialId]); + } + else + { + this._searchThreadPool[threadId].OnTrialComplete(trialId, metric, cost); + if (this._searchThreadPool[threadId].IsConverged) + { + this._searchThreadPool.Remove(threadId); + this._currentThreadId += 1; + this._initUsed = false; + } + } + } + + private void UpdateAdmissibleRegion(double[] config) + { + for (int i = 0; i != config.Length; ++i) + { + if (config[i] < this._lsBoundMin[i]) + { + this._lsBoundMin[i] = config[i]; + continue; + } + + if (config[i] > this._lsBoundMax[i]) + { + this._lsBoundMax[i] = config[i]; + continue; + } + } + } + + private double[] CreateInitConfigFromAdmissibleRegion() + { + var res = new double[this._lsBoundMax.Length]; + for (int i = 0; i != this._lsBoundMax.Length; ++i) + { + res[i] = this._rng.Uniform(this._lsBoundMin[i], this._lsBoundMax[i]); + } + + return res; + } + } +} diff --git a/src/Microsoft.ML.AutoML/Tuner/Flow2.cs b/src/Microsoft.ML.AutoML/Tuner/Flow2.cs new file mode 100644 index 0000000000..2ae9c098d7 --- /dev/null +++ b/src/Microsoft.ML.AutoML/Tuner/Flow2.cs @@ -0,0 +1,160 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Microsoft.ML.AutoMLService; +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + /// + /// An implementation of Flow2 from https://www.aaai.org/AAAI21Papers/AAAI-10128.WuQ.pdf + /// + internal class Flow2 + { + private const double _stepSize = 0.1; + private const double _stepLowerBound = 0.0001; + + private readonly RandomNumberGenerator _rng = new RandomNumberGenerator(); + + public double? BestObj = null; + public double? CostIncumbent = null; + private readonly Parameter _initConfig; + private double _step; + + private readonly SearchSpace.SearchSpace _searchSpace; + private readonly bool _minimize; + private readonly double _convergeSpeed = 2; + private Parameter _bestConfig; + private readonly Dictionary _configs = new Dictionary(); + private double _costComplete4Incumbent = 0; + private readonly int _dim; + private double[] _directionTried = null; + private double[] _incumbent; + private int _numAllowed4Incumbent = 0; + private readonly Dictionary _proposedBy = new Dictionary(); + private readonly double _stepUpperBound; + private int _trialCount = 1; + + public Flow2(SearchSpace.SearchSpace searchSpace, Parameter initValue = null, bool minimizeMode = true, double convergeSpeed = 1.5) + { + this._searchSpace = searchSpace; + this._minimize = minimizeMode; + + this._initConfig = initValue; + this._bestConfig = this._initConfig; + this._incumbent = this._searchSpace.MappingToFeatureSpace(this._bestConfig); + this._dim = this._searchSpace.Count; + this._numAllowed4Incumbent = 2 * this._dim; + this._step = _stepSize * Math.Sqrt(this._dim); + this._stepUpperBound = Math.Sqrt(this._dim); + this._convergeSpeed = convergeSpeed; + if (this._step > this._stepUpperBound) + { + this._step = this._stepUpperBound; + } + } + + public bool IsConverged + { + get => this._step < _stepLowerBound; + } + + public Parameter BestConfig + { + get => this._bestConfig; + } + + public SearchThread CreateSearchThread(Parameter config, double metric, double cost) + { + var flow2 = new Flow2(this._searchSpace, config, this._minimize, convergeSpeed: this._convergeSpeed); + flow2.BestObj = metric; + flow2.CostIncumbent = cost; + return new SearchThread(flow2); + } + + public Parameter Suggest(int trialId) + { + this._numAllowed4Incumbent -= 1; + double[] move; + if (this._directionTried != null) + { + move = ArrayMath.Sub(this._incumbent, this._directionTried); + + this._directionTried = null; + } + else + { + this._directionTried = this.RandVectorSphere(); + move = ArrayMath.Add(this._incumbent, this._directionTried); + } + + move = this.Project(move); + var config = this._searchSpace.SampleFromFeatureSpace(move); + this._proposedBy[trialId] = this._incumbent; + this._configs[trialId] = config; + return config; + } + + public void ReceiveTrialResult(int trialId, double metric, double cost) + { + this._trialCount += 1; + if (this.BestObj == null || metric < this.BestObj) + { + this.BestObj = metric; + this._bestConfig = this._configs[trialId]; + this._incumbent = this._searchSpace.MappingToFeatureSpace(this._bestConfig); + this.CostIncumbent = cost; + this._costComplete4Incumbent = 0; + this._numAllowed4Incumbent = 2 * this._dim; + this._proposedBy.Clear(); + this._step *= this._convergeSpeed; + this._step = Math.Min(this._step, this._stepUpperBound); + this._directionTried = null; + return; + } + else + { + this._costComplete4Incumbent += cost; + if (this._numAllowed4Incumbent == 0) + { + this._numAllowed4Incumbent = 2; + if (!this.IsConverged) + { + this._step /= this._convergeSpeed; + } + } + } + } + + private double[] RandVectorSphere() + { + double[] vec = this._rng.Normal(0, 1, this._searchSpace.FeatureSpaceDim); + double mag = ArrayMath.Norm(vec); + vec = ArrayMath.Mul(vec, this._step / mag); + + return vec; + } + + private double[] Project(double[] move) + { + return move.Select(x => + { + if (x < 0) + { + x = 0; + } + else if (x > 1) + { + x = 0.99999999; + } + + return x; + }).ToArray(); + } + } +} diff --git a/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs b/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs new file mode 100644 index 0000000000..e22450144c --- /dev/null +++ b/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.ML.SearchSpace; +using Microsoft.ML.SearchSpace.Tuner; + +namespace Microsoft.ML.AutoML +{ + internal class GridSearchTuner : ITuner + { + private readonly SearchSpace.Tuner.GridSearchTuner _tuner; + private IEnumerator _enumerator; + + public GridSearchTuner(SearchSpace.SearchSpace searchSpace) + { + this._tuner = new SearchSpace.Tuner.GridSearchTuner(searchSpace); + this._enumerator = this._tuner.Propose().GetEnumerator(); + } + public Parameter Propose(TrialSettings settings) + { + if (!this._enumerator.MoveNext()) + { + this._enumerator = this._tuner.Propose().GetEnumerator(); + return this.Propose(settings); + } + else + { + var res = this._enumerator.Current; + return res; + } + } + + public void Update(TrialResult result) + { + return; + } + } +} diff --git a/src/Microsoft.ML.AutoML/Tuner/ITuner.cs b/src/Microsoft.ML.AutoML/Tuner/ITuner.cs new file mode 100644 index 0000000000..5844232c33 --- /dev/null +++ b/src/Microsoft.ML.AutoML/Tuner/ITuner.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + internal interface ITuner + { + Parameter Propose(TrialSettings settings); + + void Update(TrialResult result); + } +} diff --git a/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs b/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs new file mode 100644 index 0000000000..dfde0ade86 --- /dev/null +++ b/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.ML.SearchSpace; +using Microsoft.ML.SearchSpace.Tuner; + +namespace Microsoft.ML.AutoML +{ + internal class RandomSearchTuner : ITuner + { + private readonly RandomTuner _tuner; + private readonly SearchSpace.SearchSpace _searchSpace; + + public RandomSearchTuner(SearchSpace.SearchSpace searchSpace) + { + this._tuner = new RandomTuner(); + this._searchSpace = searchSpace; + } + public Parameter Propose(TrialSettings settings) + { + return this._tuner.Propose(this._searchSpace); + } + + public void Update(TrialResult result) + { + return; + } + } +} diff --git a/src/Microsoft.ML.AutoML/Tuner/SearchThread.cs b/src/Microsoft.ML.AutoML/Tuner/SearchThread.cs new file mode 100644 index 0000000000..7889549ded --- /dev/null +++ b/src/Microsoft.ML.AutoML/Tuner/SearchThread.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Microsoft.ML.SearchSpace; +using FlamlParameters = System.Collections.Generic.Dictionary; + +namespace Microsoft.ML.AutoML +{ + internal class SearchThread + { + private const double _eps = 1e-10; + + private readonly Flow2 _searchAlg; + + private double _costBest; + private double _costBest1; + private double _costBest2; + private double _costLast; + private double _costTotal; + private double _objBest1; + private double _objBest2; + private double _speed = 0; + + public SearchThread(Flow2 searchAlgorithm) + { + this._searchAlg = searchAlgorithm; + this._costLast = searchAlgorithm.CostIncumbent == null ? 0 : (double)searchAlgorithm.CostIncumbent; + this._costTotal = this._costLast; + this._costBest = this._costLast; + this._costBest1 = this._costLast; + this._costBest2 = 0; + this._objBest1 = searchAlgorithm.BestObj == null ? double.PositiveInfinity : (double)searchAlgorithm.BestObj; + this._objBest2 = this._objBest1; + } + + public bool IsConverged { get => this._searchAlg.IsConverged; } + + public Flow2 SearchArg { get => this._searchAlg; } + + public void OnTrialComplete(int trialId, double metric, double cost) + { + this._searchAlg.ReceiveTrialResult(trialId, metric, cost); + this._costLast = cost; + this._costTotal += cost; + if (metric < this._objBest1) + { + this._costBest2 = this._costBest1; + this._costBest1 = this._costTotal; + this._objBest2 = double.IsInfinity(this._objBest1) ? metric : this._objBest1; + this._objBest1 = metric; + this._costBest = this._costLast; + } + + if (this._objBest2 > this._objBest1) + { + this._speed = (this._objBest2 - this._objBest1) / (this._costTotal - this._costBest2 + _eps); + } + else + { + this._speed = 0; + } + } + + public Parameter Suggest(int trialId) + { + return this._searchAlg.Suggest(trialId); + } + } +} diff --git a/src/Microsoft.ML.AutoML/Utils/ArrayMath.cs b/src/Microsoft.ML.AutoML/Utils/ArrayMath.cs new file mode 100644 index 0000000000..bb881a4cf4 --- /dev/null +++ b/src/Microsoft.ML.AutoML/Utils/ArrayMath.cs @@ -0,0 +1,168 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Linq; + +namespace Microsoft.ML.AutoML +{ + public class ArrayMath + { + /* x + y */ + public static double[] Add(double[] xArray, double y) + { + return xArray.Select(x => x + y).ToArray(); + } + + /* np.argmax */ + public static int ArgMax(double[] array) + { + int index = 0; + for (int i = 1; i < array.Length; i++) + { + if (array[i] > array[index]) + { + index = i; + } + } + + return index; + } + + /* np.argsort */ + public static int[] ArgSort(double[] array) + { + return Enumerable.Range(0, array.Length).OrderBy(index => array[index]).ToArray(); + } + + /* np.clip */ + public static double[] Clip(double[] xArray, double min, double max) + { + return xArray.Select(x => Math.Min(Math.Max(x, min), max)).ToArray(); + } + + /* x / y */ + public static double[] Div(double[] xArray, double y) + { + return xArray.Select(x => x / y).ToArray(); + } + + /* x[y] */ + public static double[] Index(double[] array, int[] indices) + { + return indices.Select(index => array[index]).ToArray(); + } + + /* List.Insert */ + public static double[] Insert(double[] array, int index, double item) + { + double[] ret = new double[array.Length + 1]; + Array.Copy(array, 0, ret, 0, index); + ret[index] = item; + Array.Copy(array, index, ret, index + 1, array.Length - index); + return ret; + } + + /* np.linalg.norm */ + public static double Norm(double[] array) + { + double s = 0; + foreach (double x in array) + { + s += x * x; + } + + return Math.Sqrt(s); + } + + /* x * y */ + public static double[] Mul(double[] xArray, double y) + { + return xArray.Select(x => x * y).ToArray(); + } + + /* np.log */ + public static double[] Log(double[] xArray) + { + return xArray.Select(x => Math.Log(x)).ToArray(); + } + + /* x * y */ + public static double[] Mul(double[] xArray, double[] yArray) + { + return Enumerable.Zip(xArray, yArray, (x, y) => x * y).ToArray(); + } + + /* np.searchsorted */ + public static int SearchSorted(double[] array, double item) + { + int index = Array.BinarySearch(array, item); + return index >= 0 ? index : ~index; + } + + /* x - y */ + public static double[] Add(double[] xArray, double[] yArray) + { + return Enumerable.Zip(xArray, yArray, (x, y) => x + y).ToArray(); + } + + public static double[] Sub(double[] xArray, double[] yArray) + { + return Add(xArray, Mul(yArray, -1)); + } + + public static double[] Inverse(double[] array) + { + return array.Select(v => 1 / v).ToArray(); + } + + public static double[] Normalize(double[] array) + { + var sum = array.Sum(); + return array.Select(v => v / sum).ToArray(); + } + + public static double Rmse(double[] truth, double[] pred) + { + if (truth.Length != pred.Length) + { + throw new ArgumentException($"length doesn't match, {truth.Length} != {pred.Length}"); + } + + var diff = Enumerable.Range(0, truth.Length).Select(i => truth[i] - pred[i]).ToArray(); + var sqaure = diff.Select(x => x * x); + var mean = sqaure.Average(); + var rmse = Math.Sqrt(mean); + + return rmse; + } + + public static double Mape(double[] truth, double[] pred) + { + if (truth.Length != pred.Length) + { + throw new ArgumentException($"length doesn't match, {truth.Length} != {pred.Length}"); + } + + var diff = Enumerable.Range(0, truth.Length).Select(i => truth[i] - pred[i]).ToArray(); + var ape = diff.Select((x, i) => Math.Abs(x) / truth[i]); + var mape = ape.Average(); + + return mape; + } + + public static double Mae(double[] truth, double[] pred) + { + if (truth.Length != pred.Length) + { + throw new ArgumentException($"length doesn't match, {truth.Length} != {pred.Length}"); + } + + var diff = Enumerable.Range(0, truth.Length).Select(i => Math.Abs(truth[i] - pred[i])).ToArray(); + var mae = diff.Average(); + + return mae; + } + } +} diff --git a/src/Microsoft.ML.AutoML/Utils/RandomNumberGenerator.cs b/src/Microsoft.ML.AutoML/Utils/RandomNumberGenerator.cs new file mode 100644 index 0000000000..1528432231 --- /dev/null +++ b/src/Microsoft.ML.AutoML/Utils/RandomNumberGenerator.cs @@ -0,0 +1,79 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.ML.AutoMLService +{ + internal class RandomNumberGenerator + { + private readonly Random _random; + + public RandomNumberGenerator() + { + this._random = new Random(); + } + + public RandomNumberGenerator(int seed) + { + this._random = new Random(seed); + } + + public int Integer(int high) + { + return this._random.Next(high); + } + + public double Uniform(double low, double high) + { + return (this._random.NextDouble() * (high - low)) + low; + } + + public double Normal(double location, double scale) + { + double u = 1 - this.Uniform(0, 1); + double v = 1 - this.Uniform(0, 1); + double std = Math.Sqrt(-2.0 * Math.Log(u)) * Math.Sin(2.0 * Math.PI * v); + return location + (std * scale); + } + + public double[] Normal(double location, double scale, int size) + { + double[] ret = new double[size]; + for (int i = 0; i < size; i++) + { + ret[i] = this.Normal(location, scale); + } + + return ret; + } + + public int Categorical(double[] possibility) + { + double x = this.Uniform(0, 1); + for (int i = 0; i < possibility.Length; i++) + { + x -= possibility[i]; + if (x < 0) { return i; } + } + + return possibility.Length - 1; + } + + public int[] Categorical(double[] possibility, int size) + { + int[] ret = new int[size]; + for (int i = 0; i < ret.Length; i++) + { + ret[i] = this.Categorical(possibility); + } + + return ret; + } + } +} diff --git a/src/Microsoft.ML.SearchSpace/Tuner/RandomTuner.cs b/src/Microsoft.ML.SearchSpace/Tuner/RandomTuner.cs index 7c22e735df..51cbf35c60 100644 --- a/src/Microsoft.ML.SearchSpace/Tuner/RandomTuner.cs +++ b/src/Microsoft.ML.SearchSpace/Tuner/RandomTuner.cs @@ -9,26 +9,23 @@ namespace Microsoft.ML.SearchSpace.Tuner { internal sealed class RandomTuner { - private readonly SearchSpace _searchSpace; private readonly Random _rnd; - public RandomTuner(SearchSpace searchSpace) + public RandomTuner() { - this._searchSpace = searchSpace; this._rnd = new Random(); } - public RandomTuner(SearchSpace searchSpace, int seed) + public Parameter Propose(SearchSpace searchSpace) { - this._searchSpace = searchSpace; - this._rnd = new Random(seed); + var d = searchSpace.FeatureSpaceDim; + var featureVec = Enumerable.Repeat(0, d).Select(i => this._rnd.NextDouble()).ToArray(); + return searchSpace.SampleFromFeatureSpace(featureVec); } - public Parameter Propose() + public void Update(Parameter param, double metric, bool isMaximize) { - var d = this._searchSpace.FeatureSpaceDim; - var featureVec = Enumerable.Repeat(0, d).Select(i => this._rnd.NextDouble()).ToArray(); - return this._searchSpace.SampleFromFeatureSpace(featureVec); + // do nothing } } } diff --git a/test/Microsoft.ML.AutoML.Tests/TunerTests.cs b/test/Microsoft.ML.AutoML.Tests/TunerTests.cs new file mode 100644 index 0000000000..fa92a91e83 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/TunerTests.cs @@ -0,0 +1,249 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Threading; +using FluentAssertions; +using Microsoft.ML.SearchSpace; +using Microsoft.ML.TestFramework; +using Xunit; +using Xunit.Abstractions; +using Microsoft.ML.AutoML.CodeGen; + +namespace Microsoft.ML.AutoML.Test +{ + public class TunerTests : BaseTestClass + { + public TunerTests(ITestOutputHelper output) + : base(output) + { + } + + [Fact] + public void CFO_e2e_test() + { + var searchSpace = new SearchSpace(); + var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); + var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues)); + for (int i = 0; i != 1000; ++i) + { + var trialSettings = new TrialSettings() + { + TrialId = i, + }; + + var param = cfo.Propose(trialSettings); + var option = param.AsType(); + + option.L1Regularization.Should().BeInRange(0.03125f, 32768.0f); + option.L2Regularization.Should().BeInRange(0.03125f, 32768.0f); + + cfo.Update(new TrialResult() + { + DurationInMilliseconds = i * 1000, + Metric = i, + TrialSettings = trialSettings, + }); + } + } + + [Fact] + public void CFO_should_start_from_init_point_if_provided() + { + var trialSettings = new TrialSettings() + { + TrialId = 0, + }; + var searchSpace = new SearchSpace(); + var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); + var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true); + var param = cfo.Propose(trialSettings).AsType(); + var x = param.X; + var y = param.Y; + var z = param.Z; + + (x * x + y * y + z * z).Should().Be(0); + } + + [Fact] + public void CFO_should_find_maximum_value_when_function_is_convex() + { + var searchSpace = new SearchSpace(); + var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); + var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), false); + double bestMetric = 0; + for (int i = 0; i != 100; ++i) + { + var trialSettings = new TrialSettings() + { + TrialId = 0, + }; + + var param = cfo.Propose(trialSettings).AsType(); + var x = param.X; + var y = param.Y; + var z = param.Z; + var metric = this.LSE3D(x, y, z); + bestMetric = Math.Max(bestMetric, metric); + this.Output.WriteLine($"{i} x: {x} y: {y} z: {z}"); + if (x == 10 && y == 10 && z == 10) + { + break; + } + cfo.Update(new TrialResult() + { + DurationInMilliseconds = 1 * 1000, + Metric = metric, + TrialSettings = trialSettings, + }); + } + + bestMetric.Should().BeGreaterThan(this.LSE3D(10, 10, 10) - 2); + } + + [Fact] + public void CFO_should_find_minimum_value_when_function_is_convex() + { + var searchSpace = new SearchSpace(); + var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); + var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true); + double loss = 0; + for (int i = 0; i != 100; ++i) + { + var trialSettings = new TrialSettings() + { + TrialId = i, + }; + + var param = cfo.Propose(trialSettings).AsType(); + var x = param.X; + var y = param.Y; + var z = param.Z; + loss = this.LSE3D(x, y, z); + this.Output.WriteLine(loss.ToString()); + this.Output.WriteLine($"{i} x: {x} y: {y} z: {z}"); + + if (x == -10 && y == -10 && z == -10) + { + break; + } + + cfo.Update(new TrialResult() + { + DurationInMilliseconds = 1000, + Metric = loss, + TrialSettings = trialSettings, + }); + } + + loss.Should().BeLessThan(this.LSE3D(-10, -10, -10) + 2); + } + + [Fact] + public void CFO_should_find_minimum_value_when_function_is_F1() + { + var searchSpace = new SearchSpace(); + var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); + var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true); + double bestMetric = 0; + for (int i = 0; i != 1000; ++i) + { + var trialSettings = new TrialSettings() + { + TrialId = i, + }; + var param = cfo.Propose(trialSettings).AsType(); + var x = param.X; + var y = param.Y; + var z = param.Z; + var metric = this.F1(x, y, z); + bestMetric = Math.Min(bestMetric, metric); + this.Output.WriteLine($"{i} x: {x} y: {y} z: {z}"); + + if (x == -1 && y == 1 && z == 0) + { + break; + } + + cfo.Update(new TrialResult() + { + DurationInMilliseconds = 1, + Metric = metric, + TrialSettings = trialSettings, + }); + } + + bestMetric.Should().BeLessThan(this.F1(-1, 1, 0) + 2); + } + + [Fact] + public void Hyper_parameters_from_CFO_should_be_culture_invariant_string() + { + var searchSpace = new SearchSpace(); + var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); + var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true); + var originalCuture = Thread.CurrentThread.CurrentCulture; + var usCulture = new CultureInfo("en-US", false); + Thread.CurrentThread.CurrentCulture = usCulture; + + Thread.CurrentThread.CurrentCulture.NumberFormat.NumberDecimalSeparator.Should().Be("."); + for (int i = 0; i != 100; ++i) + { + var trialSettings = new TrialSettings() + { + TrialId = i, + }; + var param = cfo.Propose(trialSettings).AsType(); + param.X.Should().BeInRange(-10, 10); + } + + var frCulture = new CultureInfo("fr-FR", false); + Thread.CurrentThread.CurrentCulture = frCulture; + Thread.CurrentThread.CurrentCulture.NumberFormat.NumberDecimalSeparator.Should().Be(","); + for (int i = 0; i != 100; ++i) + { + var trialSettings = new TrialSettings() + { + TrialId = i, + }; + var param = cfo.Propose(trialSettings).AsType(); + param.X.Should().BeInRange(-10, 10); + } + + Thread.CurrentThread.CurrentCulture = originalCuture; + } + + /// + /// LSE is strictly convex, use this as test object function. + /// + /// + /// + /// + /// + private double LSE3D(double x, double y, double z) + { + return Math.Log(Math.Exp(x) + Math.Exp(y) + Math.Exp(z)); + } + + private double F1(double x, double y, double z) + { + return x * x + 2 * x + 1 + y * y - 2 * y + 1 + z * z; + } + + private class LSE3DSearchSpace + { + [Range(-10.0, 10.0, 0.0, false)] + public double X { get; set; } + + [Range(-10.0, 10.0, 0.0, false)] + public double Y { get; set; } + + [Range(-10.0, 10.0, 0.0, false)] + public double Z { get; set; } + } + } +}