From 68bf3e935c169f6649298358d60137572a4ff951 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 25 Apr 2022 12:02:15 -0700 Subject: [PATCH 01/10] make public --- src/Microsoft.ML.AutoML/API/AutoCatalog.cs | 80 +++++++++++++++++-- .../AutoMLExperiment/AutoMLExperiment.cs | 13 +-- ...IDatasetSettings.cs => IDatasetManager.cs} | 6 +- .../AutoMLExperiment/IMetricSettings.cs | 2 +- .../AutoMLExperiment/IMonitor.cs | 2 +- .../AutoMLExperiment/TrialResult.cs | 2 +- .../AutoMLExperiment/TrialRunner.cs | 45 +++++++---- .../AutoMLExperiment/TrialRunnerFactory.cs | 17 ++-- .../AutoMLExperiment/TrialSettings.cs | 2 +- .../AutoMLExperiment/TunerFactory.cs | 2 +- .../SweepableEstimator/Estimator.cs | 2 +- .../SweepableEstimator/MultiModelPipeline.cs | 2 +- .../SweepableEstimator/SweepableEstimator.cs | 2 +- .../SweepableEstimatorPipeline.cs | 2 +- src/Microsoft.ML.AutoML/Tuner/ITuner.cs | 2 +- 15 files changed, 135 insertions(+), 46 deletions(-) rename src/Microsoft.ML.AutoML/AutoMLExperiment/{IDatasetSettings.cs => IDatasetManager.cs} (73%) diff --git a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs index 8c4642ba70..859247e6c4 100644 --- a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs +++ b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs @@ -7,6 +7,7 @@ using Microsoft.ML.AutoML.CodeGen; using Microsoft.ML.Data; using Microsoft.ML.SearchSpace; +using Microsoft.ML.Trainers.FastTree; namespace Microsoft.ML.AutoML { @@ -286,18 +287,43 @@ public ColumnInferenceResults InferColumns(string path, uint labelColumnIndex, b /// /// Create a sweepable estimator with a custom factory and search space. /// - internal SweepableEstimator CreateSweepableEstimator(Func> factory, SearchSpace ss = null) + public SweepableEstimator CreateSweepableEstimator(Func> factory, SearchSpace ss = null) where T : class, new() { return new SweepableEstimator((MLContext context, Parameter param) => factory(context, param.AsType()), ss); } - internal AutoMLExperiment CreateExperiment() + /// + /// Create an . + /// + public AutoMLExperiment CreateExperiment() { return new AutoMLExperiment(_context, new AutoMLExperiment.AutoMLExperimentSettings()); } - internal SweepableEstimator[] BinaryClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + /// + /// Create a list of for binary classification. + /// + /// label column name. + /// feature column name. + /// example weight column name. + /// true if use fast forest as available trainer. + /// true if use lgbm as available trainer. + /// true if use fast tree as available trainer. + /// true if use lbfgs as available trainer. + /// true if use sdca as available trainer. + /// if provided, use it as initial option for fast tree, otherwise the default option will be used. + /// if provided, use it as initial option for lgbm, otherwise the default option will be used. + /// if provided, use it as initial option for fast forest, otherwise the default option will be used. + /// if provided, use it as initial option for lbfgs, otherwise the default option will be used. + /// if provided, use it as initial option for sdca, otherwise the default option will be used. + /// if provided, use it as search space for fast tree, otherwise the default search space will be used. + /// if provided, use it as search space for lgbm, otherwise the default search space will be used. + /// if provided, use it as search space for fast forest, otherwise the default search space will be used. + /// if provided, use it as search space for lbfgs, otherwise the default search space will be used. + /// if provided, use it as search space for sdca, otherwise the default search space will be used. + /// + public SweepableEstimator[] BinaryClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) { @@ -351,7 +377,29 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa return res.ToArray(); } - internal SweepableEstimator[] MultiClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + /// + /// Create a list of for multiclass classification. + /// + /// label column name. + /// feature column name. + /// example weight column name. + /// true if use fast forest as available trainer. + /// true if use lgbm as available trainer. + /// true if use fast tree as available trainer. + /// true if use lbfgs as available trainer. + /// true if use sdca as available trainer. + /// if provided, use it as initial option for fast tree, otherwise the default option will be used. + /// if provided, use it as initial option for lgbm, otherwise the default option will be used. + /// if provided, use it as initial option for fast forest, otherwise the default option will be used. + /// if provided, use it as initial option for lbfgs, otherwise the default option will be used. + /// if provided, use it as initial option for sdca, otherwise the default option will be used. + /// if provided, use it as search space for fast tree, otherwise the default search space will be used. + /// if provided, use it as search space for lgbm, otherwise the default search space will be used. + /// if provided, use it as search space for fast forest, otherwise the default search space will be used. + /// if provided, use it as search space for lbfgs, otherwise the default search space will be used. + /// if provided, use it as search space for sdca, otherwise the default search space will be used. + /// + public SweepableEstimator[] MultiClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) { @@ -407,7 +455,29 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau return res.ToArray(); } - internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + /// + /// Create a list of for regression. + /// + /// label column name. + /// feature column name. + /// example weight column name. + /// true if use fast forest as available trainer. + /// true if use lgbm as available trainer. + /// true if use fast tree as available trainer. + /// true if use lbfgs as available trainer. + /// true if use sdca as available trainer. + /// if provided, use it as initial option for fast tree, otherwise the default option will be used. + /// if provided, use it as initial option for lgbm, otherwise the default option will be used. + /// if provided, use it as initial option for fast forest, otherwise the default option will be used. + /// if provided, use it as initial option for lbfgs, otherwise the default option will be used. + /// if provided, use it as initial option for sdca, otherwise the default option will be used. + /// if provided, use it as search space for fast tree, otherwise the default search space will be used. + /// if provided, use it as search space for lgbm, otherwise the default search space will be used. + /// if provided, use it as search space for fast forest, otherwise the default search space will be used. + /// if provided, use it as search space for lbfgs, otherwise the default search space will be used. + /// if provided, use it as search space for sdca, otherwise the default search space will be used. + /// + public SweepableEstimator[] Regression(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) { diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index dfa34dcf33..c956673389 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -12,7 +12,7 @@ namespace Microsoft.ML.AutoML { - internal class AutoMLExperiment + public class AutoMLExperiment { private readonly AutoMLExperimentSettings _settings; private readonly MLContext _context; @@ -52,12 +52,14 @@ public AutoMLExperiment SetTrainingTimeInSeconds(uint trainingTimeInSeconds) public AutoMLExperiment SetDataset(IDataView train, IDataView test) { - _settings.DatasetSettings = new TrainTestDatasetSettings() + var datasetManager = new TrainTestDatasetManager() { TrainDataset = train, TestDataset = test }; + _serviceCollection.AddSingleton(datasetManager); + return this; } @@ -70,12 +72,14 @@ public AutoMLExperiment SetDataset(TrainTestData trainTestSplit) public AutoMLExperiment SetDataset(IDataView dataset, int fold = 10) { - _settings.DatasetSettings = new CrossValidateDatasetSettings() + var datasetManager = new CrossValidateDatasetManager() { Dataset = dataset, Fold = fold, }; + _serviceCollection.AddSingleton(datasetManager); + return this; } @@ -264,15 +268,12 @@ private async Task RunAsync(CancellationToken ct) private void ValidateSettings() { Contracts.Assert(_settings.MaxExperimentTimeInSeconds > 0, $"{nameof(ExperimentSettings.MaxExperimentTimeInSeconds)} must be larger than 0"); - Contracts.Assert(_settings.DatasetSettings != null, $"{nameof(_settings.DatasetSettings)} must be not null"); Contracts.Assert(_settings.EvaluateMetric != null, $"{nameof(_settings.EvaluateMetric)} must be not null"); } public class AutoMLExperimentSettings : ExperimentSettings { - public IDatasetSettings DatasetSettings { get; set; } - public IMetricSettings EvaluateMetric { get; set; } public MultiModelPipeline Pipeline { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs similarity index 73% rename from src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetSettings.cs rename to src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs index c551139928..6180807ec1 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetSettings.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs @@ -4,18 +4,18 @@ namespace Microsoft.ML.AutoML { - internal interface IDatasetSettings + public interface IDatasetManager { } - internal class TrainTestDatasetSettings : IDatasetSettings + public class TrainTestDatasetManager : IDatasetManager { public IDataView TrainDataset { get; set; } public IDataView TestDataset { get; set; } } - internal class CrossValidateDatasetSettings : IDatasetSettings + public class CrossValidateDatasetManager : IDatasetManager { public IDataView Dataset { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs index 2374ba75f4..c05eeac33f 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs @@ -6,7 +6,7 @@ namespace Microsoft.ML.AutoML { - internal interface IMetricSettings + public interface IMetricSettings { bool IsMaximize { get; } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index 64a10e1a4a..14a89b29cd 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -8,7 +8,7 @@ namespace Microsoft.ML.AutoML { - internal interface IMonitor + public interface IMonitor { void ReportCompletedTrial(TrialResult result); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs index fa60ec5d9c..bceaa08cde 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs @@ -4,7 +4,7 @@ namespace Microsoft.ML.AutoML { - internal class TrialResult + public class TrialResult { public TrialSettings TrialSettings { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs index 899c6ad3c9..66a822e354 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs @@ -7,7 +7,7 @@ namespace Microsoft.ML.AutoML { - internal interface ITrialRunner + public interface ITrialRunner { TrialResult Run(TrialSettings settings); } @@ -15,15 +15,18 @@ internal interface ITrialRunner internal class BinaryClassificationCVRunner : ITrialRunner { private readonly MLContext _context; - public BinaryClassificationCVRunner(MLContext context) + private readonly IDatasetManager _datasetManager; + + public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetManager) { _context = context; + _datasetManager = datasetManager; } public TrialResult Run(TrialSettings settings) { var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); - if (settings.ExperimentSettings.DatasetSettings is CrossValidateDatasetSettings datasetSettings + if (_datasetManager is CrossValidateDatasetManager datasetSettings && settings.ExperimentSettings.EvaluateMetric is BinaryMetricSettings metricSettings) { var stopWatch = new Stopwatch(); @@ -64,15 +67,17 @@ public TrialResult Run(TrialSettings settings) internal class BinaryClassificationTrainTestRunner : ITrialRunner { private readonly MLContext _context; - public BinaryClassificationTrainTestRunner(MLContext context) + private readonly IDatasetManager _datasetManager; + + public BinaryClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager) { _context = context; + _datasetManager = datasetManager; } public TrialResult Run(TrialSettings settings) { - var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); - if (settings.ExperimentSettings.DatasetSettings is TrainTestDatasetSettings datasetSettings + if (_datasetManager is TrainTestDatasetManager datasetSettings && settings.ExperimentSettings.EvaluateMetric is BinaryMetricSettings metricSettings) { var stopWatch = new Stopwatch(); @@ -112,14 +117,17 @@ public TrialResult Run(TrialSettings settings) internal class MultiClassificationTrainTestRunner : ITrialRunner { private readonly MLContext _context; - public MultiClassificationTrainTestRunner(MLContext context) + private readonly IDatasetManager _datasetManager; + + public MultiClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager) { _context = context; + _datasetManager = datasetManager; } public TrialResult Run(TrialSettings settings) { - if (settings.ExperimentSettings.DatasetSettings is TrainTestDatasetSettings datasetSettings + if (_datasetManager is TrainTestDatasetManager datasetSettings && settings.ExperimentSettings.EvaluateMetric is MultiClassMetricSettings metricSettings) { var stopWatch = new Stopwatch(); @@ -159,15 +167,18 @@ public TrialResult Run(TrialSettings settings) internal class MultiClassificationCVRunner : ITrialRunner { private readonly MLContext _context; - public MultiClassificationCVRunner(MLContext context) + private readonly IDatasetManager _datasetManager; + + public MultiClassificationCVRunner(MLContext context, IDatasetManager datasetManager) { _context = context; + _datasetManager = datasetManager; } public TrialResult Run(TrialSettings settings) { var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); - if (settings.ExperimentSettings.DatasetSettings is CrossValidateDatasetSettings datasetSettings + if (_datasetManager is CrossValidateDatasetManager datasetSettings && settings.ExperimentSettings.EvaluateMetric is MultiClassMetricSettings metricSettings) { var stopWatch = new Stopwatch(); @@ -207,14 +218,17 @@ public TrialResult Run(TrialSettings settings) internal class RegressionTrainTestRunner : ITrialRunner { private readonly MLContext _context; - public RegressionTrainTestRunner(MLContext context) + private readonly IDatasetManager _datasetManager; + + public RegressionTrainTestRunner(MLContext context, IDatasetManager datasetManager) { _context = context; + _datasetManager = datasetManager; } public TrialResult Run(TrialSettings settings) { - if (settings.ExperimentSettings.DatasetSettings is TrainTestDatasetSettings datasetSettings + if (_datasetManager is TrainTestDatasetManager datasetSettings && settings.ExperimentSettings.EvaluateMetric is RegressionMetricSettings metricSettings) { var stopWatch = new Stopwatch(); @@ -253,15 +267,18 @@ public TrialResult Run(TrialSettings settings) internal class RegressionCVRunner : ITrialRunner { private readonly MLContext _context; - public RegressionCVRunner(MLContext context) + private readonly IDatasetManager _datasetManager; + + public RegressionCVRunner(MLContext context, IDatasetManager datasetManager) { _context = context; + _datasetManager = datasetManager; } public TrialResult Run(TrialSettings settings) { var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); - if (settings.ExperimentSettings.DatasetSettings is CrossValidateDatasetSettings datasetSettings + if (_datasetManager is CrossValidateDatasetManager datasetSettings && settings.ExperimentSettings.EvaluateMetric is RegressionMetricSettings metricSettings) { var stopWatch = new Stopwatch(); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs index 18a3dfc4d1..e688668410 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs @@ -8,7 +8,7 @@ #nullable enable namespace Microsoft.ML.AutoML { - internal interface ITrialRunnerFactory + public interface ITrialRunnerFactory { ITrialRunner? CreateTrialRunner(TrialSettings settings); } @@ -24,14 +24,15 @@ public TrialRunnerFactory(IServiceProvider provider) public ITrialRunner? CreateTrialRunner(TrialSettings settings) { - ITrialRunner? runner = (settings.ExperimentSettings.DatasetSettings, settings.ExperimentSettings.EvaluateMetric) switch + var datasetManager = this._provider.GetService(); + ITrialRunner? runner = (datasetManager, settings.ExperimentSettings.EvaluateMetric) switch { - (CrossValidateDatasetSettings, BinaryMetricSettings) => _provider.GetService(), - (TrainTestDatasetSettings, BinaryMetricSettings) => _provider.GetService(), - (CrossValidateDatasetSettings, MultiClassMetricSettings) => _provider.GetService(), - (TrainTestDatasetSettings, MultiClassMetricSettings) => _provider.GetService(), - (CrossValidateDatasetSettings, RegressionMetricSettings) => _provider.GetService(), - (TrainTestDatasetSettings, RegressionMetricSettings) => _provider.GetService(), + (CrossValidateDatasetManager, BinaryMetricSettings) => _provider.GetService(), + (TrainTestDatasetManager, BinaryMetricSettings) => _provider.GetService(), + (CrossValidateDatasetManager, MultiClassMetricSettings) => _provider.GetService(), + (TrainTestDatasetManager, MultiClassMetricSettings) => _provider.GetService(), + (CrossValidateDatasetManager, RegressionMetricSettings) => _provider.GetService(), + (TrainTestDatasetManager, RegressionMetricSettings) => _provider.GetService(), _ => throw new NotImplementedException(), }; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs index 8cd8a2e2ab..19294ffde9 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs @@ -6,7 +6,7 @@ namespace Microsoft.ML.AutoML { - internal class TrialSettings + public class TrialSettings { public int TrialId { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs index 0e89da262a..c844299bfd 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs @@ -7,7 +7,7 @@ namespace Microsoft.ML.AutoML { - internal interface ITunerFactory + public interface ITunerFactory { ITuner CreateTuner(TrialSettings settings); } diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimator.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimator.cs index f7938bfe5b..bcb5da9b26 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimator.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimator.cs @@ -7,7 +7,7 @@ namespace Microsoft.ML.AutoML { - internal class Estimator + public class Estimator { protected Estimator() { diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs index e11202ed98..eb1b58d1a1 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs @@ -10,7 +10,7 @@ namespace Microsoft.ML.AutoML { [JsonConverter(typeof(MultiModelPipelineConverter))] - internal class MultiModelPipeline + public class MultiModelPipeline { private static readonly StringEntity _nilStringEntity = new StringEntity("Nil"); private static readonly EstimatorEntity _nilSweepableEntity = new EstimatorEntity(null); diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs index 26f3e8ef3b..35a6e0f3a8 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs @@ -14,7 +14,7 @@ namespace Microsoft.ML.AutoML /// Estimator with search space. /// [JsonConverter(typeof(SweepableEstimatorConverter))] - internal class SweepableEstimator : Estimator + public class SweepableEstimator : Estimator { private readonly Func> _factory; diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs index 76e5682ba5..f04b7ae63b 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs @@ -11,7 +11,7 @@ namespace Microsoft.ML.AutoML { [JsonConverter(typeof(SweepableEstimatorPipelineConverter))] - internal class SweepableEstimatorPipeline + public class SweepableEstimatorPipeline { private readonly List _estimators; diff --git a/src/Microsoft.ML.AutoML/Tuner/ITuner.cs b/src/Microsoft.ML.AutoML/Tuner/ITuner.cs index 5844232c33..db522c192f 100644 --- a/src/Microsoft.ML.AutoML/Tuner/ITuner.cs +++ b/src/Microsoft.ML.AutoML/Tuner/ITuner.cs @@ -6,7 +6,7 @@ namespace Microsoft.ML.AutoML { - internal interface ITuner + public interface ITuner { Parameter Propose(TrialSettings settings); From d481bfe00ca1c2eda8648b82faa34a37159fff5b Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 25 Apr 2022 12:10:54 -0700 Subject: [PATCH 02/10] rename metricSetting to MetricManage --- .../AutoMLExperiment/AutoMLExperiment.cs | 14 +++---- .../AutoMLExperiment/IDatasetManager.cs | 6 +-- .../{IMetricSettings.cs => IMetricManager.cs} | 8 ++-- .../AutoMLExperiment/TrialResult.cs | 2 + .../AutoMLExperiment/TrialRunner.cs | 42 +++++++++++++------ .../AutoMLExperiment/TrialRunnerFactory.cs | 12 +++--- 6 files changed, 52 insertions(+), 32 deletions(-) rename src/Microsoft.ML.AutoML/AutoMLExperiment/{IMetricSettings.cs => IMetricManager.cs} (91%) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index c956673389..36546b702c 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -150,36 +150,39 @@ public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "Predicted") { - _settings.EvaluateMetric = new BinaryMetricSettings() + var metricManager = new BinaryMetricManager() { Metric = metric, PredictedColumn = predictedColumn, LabelColumn = labelColumn, }; + _serviceCollection.AddSingleton(metricManager); return this; } public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, string labelColumn = "label", string predictedColumn = "Predicted") { - _settings.EvaluateMetric = new MultiClassMetricSettings() + var metricManager = new MultiClassMetricManager() { Metric = metric, PredictedColumn = predictedColumn, LabelColumn = labelColumn, }; + _serviceCollection.AddSingleton(metricManager); return this; } public AutoMLExperiment SetEvaluateMetric(RegressionMetric metric, string labelColumn = "label", string scoreColumn = "Score") { - _settings.EvaluateMetric = new RegressionMetricSettings() + var metricManager = new RegressionMetricManager() { Metric = metric, ScoreColumn = scoreColumn, LabelColumn = labelColumn, }; + _serviceCollection.AddSingleton(metricManager); return this; } @@ -234,7 +237,7 @@ private async Task RunAsync(CancellationToken ct) hyperParameterProposer.Update(setting, trialResult); pipelineProposer.Update(setting, trialResult); - var error = _settings.EvaluateMetric.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; + var error = trialResult.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; if (error < _bestError) { _bestTrialResult = trialResult; @@ -268,14 +271,11 @@ private async Task RunAsync(CancellationToken ct) private void ValidateSettings() { Contracts.Assert(_settings.MaxExperimentTimeInSeconds > 0, $"{nameof(ExperimentSettings.MaxExperimentTimeInSeconds)} must be larger than 0"); - Contracts.Assert(_settings.EvaluateMetric != null, $"{nameof(_settings.EvaluateMetric)} must be not null"); } public class AutoMLExperimentSettings : ExperimentSettings { - public IMetricSettings EvaluateMetric { get; set; } - public MultiModelPipeline Pipeline { get; set; } public int? Seed { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs index 6180807ec1..5e1e83538d 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs @@ -4,18 +4,18 @@ namespace Microsoft.ML.AutoML { - public interface IDatasetManager + internal interface IDatasetManager { } - public class TrainTestDatasetManager : IDatasetManager + internal class TrainTestDatasetManager : IDatasetManager { public IDataView TrainDataset { get; set; } public IDataView TestDataset { get; set; } } - public class CrossValidateDatasetManager : IDatasetManager + internal class CrossValidateDatasetManager : IDatasetManager { public IDataView Dataset { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs similarity index 91% rename from src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs rename to src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs index c05eeac33f..9e060b22bc 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs @@ -6,12 +6,12 @@ namespace Microsoft.ML.AutoML { - public interface IMetricSettings + internal interface IMetricManager { bool IsMaximize { get; } } - internal class BinaryMetricSettings : IMetricSettings + internal class BinaryMetricManager : IMetricManager { public BinaryClassificationMetric Metric { get; set; } @@ -33,7 +33,7 @@ internal class BinaryMetricSettings : IMetricSettings }; } - internal class MultiClassMetricSettings : IMetricSettings + internal class MultiClassMetricManager : IMetricManager { public MulticlassClassificationMetric Metric { get; set; } @@ -52,7 +52,7 @@ internal class MultiClassMetricSettings : IMetricSettings }; } - internal class RegressionMetricSettings : IMetricSettings + internal class RegressionMetricManager : IMetricManager { public RegressionMetric Metric { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs index bceaa08cde..bd3f19d47b 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs @@ -12,6 +12,8 @@ public class TrialResult public double Metric { get; set; } + public bool IsMaximize { get; set; } + public double DurationInMilliseconds { get; set; } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs index 66a822e354..fe054fbb64 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs @@ -16,18 +16,20 @@ internal class BinaryClassificationCVRunner : ITrialRunner { private readonly MLContext _context; private readonly IDatasetManager _datasetManager; + private readonly IMetricManager _metricManager; - public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetManager) + public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) { _context = context; _datasetManager = datasetManager; + _metricManager = metricManager; } public TrialResult Run(TrialSettings settings) { var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); if (_datasetManager is CrossValidateDatasetManager datasetSettings - && settings.ExperimentSettings.EvaluateMetric is BinaryMetricSettings metricSettings) + && _metricManager is BinaryMetricManager metricSettings) { var stopWatch = new Stopwatch(); stopWatch.Start(); @@ -57,6 +59,7 @@ public TrialResult Run(TrialSettings settings) Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + IsMaximize = _metricManager.IsMaximize, }; } @@ -68,17 +71,19 @@ internal class BinaryClassificationTrainTestRunner : ITrialRunner { private readonly MLContext _context; private readonly IDatasetManager _datasetManager; + private readonly IMetricManager _metricManager; - public BinaryClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager) + public BinaryClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) { _context = context; + _metricManager = metricManager; _datasetManager = datasetManager; } public TrialResult Run(TrialSettings settings) { if (_datasetManager is TrainTestDatasetManager datasetSettings - && settings.ExperimentSettings.EvaluateMetric is BinaryMetricSettings metricSettings) + && _metricManager is BinaryMetricManager metricSettings) { var stopWatch = new Stopwatch(); stopWatch.Start(); @@ -107,6 +112,7 @@ public TrialResult Run(TrialSettings settings) Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + IsMaximize = _metricManager.IsMaximize, }; } @@ -118,17 +124,19 @@ internal class MultiClassificationTrainTestRunner : ITrialRunner { private readonly MLContext _context; private readonly IDatasetManager _datasetManager; + private readonly IMetricManager _metricManager; - public MultiClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager) + public MultiClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) { _context = context; + _metricManager = metricManager; _datasetManager = datasetManager; } public TrialResult Run(TrialSettings settings) { if (_datasetManager is TrainTestDatasetManager datasetSettings - && settings.ExperimentSettings.EvaluateMetric is MultiClassMetricSettings metricSettings) + && _metricManager is MultiClassMetricManager metricSettings) { var stopWatch = new Stopwatch(); stopWatch.Start(); @@ -157,6 +165,7 @@ public TrialResult Run(TrialSettings settings) Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + IsMaximize = _metricManager.IsMaximize, }; } @@ -168,10 +177,12 @@ internal class MultiClassificationCVRunner : ITrialRunner { private readonly MLContext _context; private readonly IDatasetManager _datasetManager; + private readonly IMetricManager _metricManager; - public MultiClassificationCVRunner(MLContext context, IDatasetManager datasetManager) + public MultiClassificationCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) { _context = context; + _metricManager = metricManager; _datasetManager = datasetManager; } @@ -179,7 +190,7 @@ public TrialResult Run(TrialSettings settings) { var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); if (_datasetManager is CrossValidateDatasetManager datasetSettings - && settings.ExperimentSettings.EvaluateMetric is MultiClassMetricSettings metricSettings) + && _metricManager is MultiClassMetricManager metricSettings) { var stopWatch = new Stopwatch(); stopWatch.Start(); @@ -208,6 +219,7 @@ public TrialResult Run(TrialSettings settings) Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + IsMaximize = _metricManager.IsMaximize, }; } @@ -219,17 +231,19 @@ internal class RegressionTrainTestRunner : ITrialRunner { private readonly MLContext _context; private readonly IDatasetManager _datasetManager; + private readonly IMetricManager _metricManager; - public RegressionTrainTestRunner(MLContext context, IDatasetManager datasetManager) + public RegressionTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) { _context = context; + _metricManager = metricManager; _datasetManager = datasetManager; } public TrialResult Run(TrialSettings settings) { if (_datasetManager is TrainTestDatasetManager datasetSettings - && settings.ExperimentSettings.EvaluateMetric is RegressionMetricSettings metricSettings) + && _metricManager is RegressionMetricManager metricSettings) { var stopWatch = new Stopwatch(); stopWatch.Start(); @@ -257,6 +271,7 @@ public TrialResult Run(TrialSettings settings) Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + IsMaximize = _metricManager.IsMaximize, }; } @@ -268,10 +283,12 @@ internal class RegressionCVRunner : ITrialRunner { private readonly MLContext _context; private readonly IDatasetManager _datasetManager; + private readonly IMetricManager _metricManager; - public RegressionCVRunner(MLContext context, IDatasetManager datasetManager) + public RegressionCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) { _context = context; + _metricManager = metricManager; _datasetManager = datasetManager; } @@ -279,7 +296,7 @@ public TrialResult Run(TrialSettings settings) { var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); if (_datasetManager is CrossValidateDatasetManager datasetSettings - && settings.ExperimentSettings.EvaluateMetric is RegressionMetricSettings metricSettings) + && _metricManager is RegressionMetricManager metricSettings) { var stopWatch = new Stopwatch(); stopWatch.Start(); @@ -307,6 +324,7 @@ public TrialResult Run(TrialSettings settings) Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + IsMaximize = _metricManager.IsMaximize, }; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs index e688668410..a89f42cfb3 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs @@ -27,12 +27,12 @@ public TrialRunnerFactory(IServiceProvider provider) var datasetManager = this._provider.GetService(); ITrialRunner? runner = (datasetManager, settings.ExperimentSettings.EvaluateMetric) switch { - (CrossValidateDatasetManager, BinaryMetricSettings) => _provider.GetService(), - (TrainTestDatasetManager, BinaryMetricSettings) => _provider.GetService(), - (CrossValidateDatasetManager, MultiClassMetricSettings) => _provider.GetService(), - (TrainTestDatasetManager, MultiClassMetricSettings) => _provider.GetService(), - (CrossValidateDatasetManager, RegressionMetricSettings) => _provider.GetService(), - (TrainTestDatasetManager, RegressionMetricSettings) => _provider.GetService(), + (CrossValidateDatasetManager, BinaryMetricManager) => _provider.GetService(), + (TrainTestDatasetManager, BinaryMetricManager) => _provider.GetService(), + (CrossValidateDatasetManager, MultiClassMetricManager) => _provider.GetService(), + (TrainTestDatasetManager, MultiClassMetricManager) => _provider.GetService(), + (CrossValidateDatasetManager, RegressionMetricManager) => _provider.GetService(), + (TrainTestDatasetManager, RegressionMetricManager) => _provider.GetService(), _ => throw new NotImplementedException(), }; From a07e0365d53d48ecb52ea117183c189d6f564dc8 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 25 Apr 2022 12:17:07 -0700 Subject: [PATCH 03/10] add custom runner factory --- .../AutoMLExperiment/AutoMLExperiment.cs | 3 ++- .../AutoMLExperiment/TrialRunnerFactory.cs | 23 ++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 36546b702c..ecae9375b2 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -120,8 +120,9 @@ public AutoMLExperiment SetPipeline(MultiModelPipeline pipeline) return this; } - public AutoMLExperiment SetTrialRunnerFactory(ITrialRunnerFactory factory) + public AutoMLExperiment SetTrialRunner(ITrialRunner runner) { + var factory = new CustomRunnerFactory(runner); var descriptor = new ServiceDescriptor(typeof(ITrialRunnerFactory), factory); if (_serviceCollection.Contains(descriptor)) { diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs index a89f42cfb3..ca9401d10c 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs @@ -10,7 +10,22 @@ namespace Microsoft.ML.AutoML { public interface ITrialRunnerFactory { - ITrialRunner? CreateTrialRunner(TrialSettings settings); + ITrialRunner? CreateTrialRunner(); + } + + internal class CustomRunnerFactory : ITrialRunnerFactory + { + private readonly ITrialRunner _instance; + + public CustomRunnerFactory(ITrialRunner runner) + { + _instance = runner; + } + + public ITrialRunner? CreateTrialRunner() + { + return _instance; + } } internal class TrialRunnerFactory : ITrialRunnerFactory @@ -22,10 +37,12 @@ public TrialRunnerFactory(IServiceProvider provider) _provider = provider; } - public ITrialRunner? CreateTrialRunner(TrialSettings settings) + public ITrialRunner? CreateTrialRunner() { var datasetManager = this._provider.GetService(); - ITrialRunner? runner = (datasetManager, settings.ExperimentSettings.EvaluateMetric) switch + var metricManager = this._provider.GetService(); + + ITrialRunner? runner = (datasetManager, metricManager) switch { (CrossValidateDatasetManager, BinaryMetricManager) => _provider.GetService(), (TrainTestDatasetManager, BinaryMetricManager) => _provider.GetService(), From ca5337020efb56e102bf8575f70c84e351811a2e Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 25 Apr 2022 12:33:14 -0700 Subject: [PATCH 04/10] add soner example --- .../Microsoft.ML.AutoML.Samples.csproj | 1 + .../Microsoft.ML.AutoML.Samples/Program.cs | 1 + .../Microsoft.ML.AutoML.Samples/Sonar.cs | 198 ++++++++++++++++++ .../API/SweepableExtension.cs | 2 +- .../AutoMLExperiment/AutoMLExperiment.cs | 4 +- .../AutoMLExperiment/IDatasetManager.cs | 6 +- .../AutoMLExperiment/TrialRunner.cs | 14 +- .../TrialSettingsProposer/PipelineProposer.cs | 2 +- .../AutoMLExperiment/TunerFactory.cs | 3 +- 9 files changed, 216 insertions(+), 15 deletions(-) create mode 100644 docs/samples/Microsoft.ML.AutoML.Samples/Sonar.cs diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj index 9d511df6a5..1c4da82682 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj @@ -10,6 +10,7 @@ + diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs b/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs index 4342873c64..3560899e18 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs @@ -8,6 +8,7 @@ public static void Main(string[] args) { try { + Sonar.Run(); RecommendationExperiment.Run(); Console.Clear(); diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Sonar.cs b/docs/samples/Microsoft.ML.AutoML.Samples/Sonar.cs new file mode 100644 index 0000000000..dc209256b4 --- /dev/null +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Sonar.cs @@ -0,0 +1,198 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Microsoft.Data.Analysis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.Data; +using Microsoft.ML.SearchSpace; +using Microsoft.ML.Transforms.TimeSeries; + +namespace Microsoft.ML.AutoML.Samples +{ + /// + /// Time series forecasting using Sonar dataset. + /// + public static class Sonar + { + public static void Run() + { + // Load file + var trainDataPath = @"C:\Users\xiaoyuz\Desktop\forecasting\-0401_load.csv"; + var evaluatePath = @"C:\Users\xiaoyuz\Desktop\forecasting\0401_0415_load.csv"; + var predictedPath = @"C:\Users\xiaoyuz\Desktop\forecasting\predicted.csv"; + var trainDf = DataFrame.LoadCsv(trainDataPath); + var evaluateDf = DataFrame.LoadCsv(evaluatePath); + + var mlContext = new MLContext(); + var searchSpace = new SearchSpace(); + var runner = new CustomRunner(mlContext); + + Console.WriteLine($"train data Length: {trainDf.Rows.Count}"); + var pipeline = mlContext.Transforms.CopyColumns("newLoad", "load") + .Append(mlContext.Auto().CreateSweepableEstimator((context, ss) => + { + return mlContext.Forecasting.ForecastBySsa("predict", "load", ss.WindowSize, ss.SeriesLength, Convert.ToInt32(trainDf.Rows.Count), ss.Horizon, rank: ss.Rank, variableHorizon: true); + }, searchSpace)); + + var autoMLExperiment = mlContext.Auto().CreateExperiment(); + + autoMLExperiment.SetPipeline(pipeline) + .SetTrialRunner(runner) + .SetTrainingTimeInSeconds(600) + .SetEvaluateMetric(RegressionMetric.RootMeanSquaredError) + .SetDataset(trainDf, evaluateDf); + + mlContext.Log += (e, o) => + { + if (o.Source.StartsWith("AutoMLExperiment")) + { + Console.WriteLine(o.RawMessage); + } + }; + + var res = autoMLExperiment.Run().Result; + var bestModel = res.Model; + Console.WriteLine($"best model rmse: {res.Metric}"); + + // evaluate + var predictEngine = bestModel.CreateTimeSeriesEngine(mlContext); + + var predictLoads1H = new List(); + var predictLoads2H = new List(); + predictLoads2H.Add(0); + foreach (var load in evaluateDf.GetColumn("load")) + { + // firstly, get next n predict where n is horizon + var predict = predictEngine.Predict(); + + predictLoads1H.Add(predict.Predict[0]); + predictLoads2H.Add(predict.Predict[1]); + + // update model with truth value + predictEngine.Predict(new ForecastInput() + { + Load = load, + }); + } + + evaluateDf["predict_load_1h"] = DataFrameColumn.Create("predict_load_1h", predictLoads1H); + evaluateDf["predict_load_2h"] = DataFrameColumn.Create("predict_load_2h", predictLoads2H.SkipLast(1)); + DataFrame.WriteCsv(evaluateDf, predictedPath); + } + } + + public class ForecastInput + { + [ColumnName("load")] + public float Load { get; set; } + } + + public class ForecastOutnput + { + [ColumnName("predict")] + public float[] Predict { get; set; } + } + + public class CustomRunner : ITrialRunner + { + private MLContext _context; + + public CustomRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings, IServiceProvider provider) + { + var datasetManager = provider.GetService(); + if (datasetManager is TrainTestDatasetManager trainTestDatasetManager) + { + try + { + var trainDataset = trainTestDatasetManager.TrainDataset; + var testDataset = trainTestDatasetManager.TestDataset; + + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); + var model = pipeline.Fit(trainDataset); + + var predictEngine = model.CreateTimeSeriesEngine(this._context); + + // check point + predictEngine.CheckPoint(this._context, "origin"); + + var predictedLoad1H = new List(); + var predictedLoad2H = new List(); + var N = testDataset.GetRowCount(); + + // evaluate + foreach (var load in testDataset.GetColumn("load")) + { + // firstly, get next n predict where n is horizon + var predict = predictEngine.Predict(); + + predictedLoad1H.Add(predict.Predict[0]); + predictedLoad2H.Add(predict.Predict[1]); + + // update model with truth value + predictEngine.Predict(new ForecastInput() + { + Load = load, + }); + } + + var rmse1H = Enumerable.Zip(testDataset.GetColumn("load"), predictedLoad1H) + .Select(x => Math.Pow(x.First - x.Second, 2)) + .Average(); + rmse1H = Math.Sqrt(rmse1H); + + var rmse2H = Enumerable.Zip(testDataset.GetColumn("load").Skip(1), predictedLoad2H) + .Select(x => Math.Pow(x.First - x.Second, 2)) + .Average(); + rmse2H = Math.Sqrt(rmse2H); + + stopWatch.Stop(); + var rmse = (rmse1H + rmse2H) / 2; + + return new TrialResult() + { + Metric = rmse, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + + } + catch (Exception) + { + return new TrialResult() + { + Metric = double.MaxValue, + Model = null, + TrialSettings = settings, + DurationInMilliseconds = 0, + }; + } + } + + throw new ArgumentException(); + } + } + public class ForecastBySsaSearchSpace + { + [Range(2, 24 * 7 * 30)] + public int WindowSize { get; set; } = 2; + + [Range(2, 24 * 7 * 30)] + public int SeriesLength { get; set; } = 2; + + [Range(1, 24 * 7 * 30)] + public int Rank { get; set; } = 1; + + [Range(2, 50)] + public int Horizon { get; set; } = 2; + } +} diff --git a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs index 6922a9597d..006e35ee6e 100644 --- a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs +++ b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs @@ -4,7 +4,7 @@ namespace Microsoft.ML.AutoML { - internal static class SweepableExtension + public static class SweepableExtension { public static SweepableEstimatorPipeline Append(this IEstimator estimator, SweepableEstimator estimator1) { diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index ecae9375b2..119fd960b6 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -232,8 +232,8 @@ private async Task RunAsync(CancellationToken ct) setting = pipelineProposer.Propose(setting); setting = hyperParameterProposer.Propose(setting); monitor.ReportRunningTrial(setting); - var runner = runnerFactory.CreateTrialRunner(setting); - var trialResult = runner.Run(setting); + var runner = runnerFactory.CreateTrialRunner(); + var trialResult = runner.Run(setting, serviceProvider); monitor.ReportCompletedTrial(trialResult); hyperParameterProposer.Update(setting, trialResult); pipelineProposer.Update(setting, trialResult); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs index 5e1e83538d..6180807ec1 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs @@ -4,18 +4,18 @@ namespace Microsoft.ML.AutoML { - internal interface IDatasetManager + public interface IDatasetManager { } - internal class TrainTestDatasetManager : IDatasetManager + public class TrainTestDatasetManager : IDatasetManager { public IDataView TrainDataset { get; set; } public IDataView TestDataset { get; set; } } - internal class CrossValidateDatasetManager : IDatasetManager + public class CrossValidateDatasetManager : IDatasetManager { public IDataView Dataset { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs index fe054fbb64..b1d3816055 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs @@ -9,7 +9,7 @@ namespace Microsoft.ML.AutoML { public interface ITrialRunner { - TrialResult Run(TrialSettings settings); + TrialResult Run(TrialSettings settings, IServiceProvider provider = null); } internal class BinaryClassificationCVRunner : ITrialRunner @@ -25,7 +25,7 @@ public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetMa _metricManager = metricManager; } - public TrialResult Run(TrialSettings settings) + public TrialResult Run(TrialSettings settings, IServiceProvider provider) { var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); if (_datasetManager is CrossValidateDatasetManager datasetSettings @@ -80,7 +80,7 @@ public BinaryClassificationTrainTestRunner(MLContext context, IDatasetManager da _datasetManager = datasetManager; } - public TrialResult Run(TrialSettings settings) + public TrialResult Run(TrialSettings settings, IServiceProvider provider) { if (_datasetManager is TrainTestDatasetManager datasetSettings && _metricManager is BinaryMetricManager metricSettings) @@ -133,7 +133,7 @@ public MultiClassificationTrainTestRunner(MLContext context, IDatasetManager dat _datasetManager = datasetManager; } - public TrialResult Run(TrialSettings settings) + public TrialResult Run(TrialSettings settings, IServiceProvider provider) { if (_datasetManager is TrainTestDatasetManager datasetSettings && _metricManager is MultiClassMetricManager metricSettings) @@ -186,7 +186,7 @@ public MultiClassificationCVRunner(MLContext context, IDatasetManager datasetMan _datasetManager = datasetManager; } - public TrialResult Run(TrialSettings settings) + public TrialResult Run(TrialSettings settings, IServiceProvider provider) { var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); if (_datasetManager is CrossValidateDatasetManager datasetSettings @@ -240,7 +240,7 @@ public RegressionTrainTestRunner(MLContext context, IDatasetManager datasetManag _datasetManager = datasetManager; } - public TrialResult Run(TrialSettings settings) + public TrialResult Run(TrialSettings settings, IServiceProvider provider) { if (_datasetManager is TrainTestDatasetManager datasetSettings && _metricManager is RegressionMetricManager metricSettings) @@ -292,7 +292,7 @@ public RegressionCVRunner(MLContext context, IDatasetManager datasetManager, IMe _datasetManager = datasetManager; } - public TrialResult Run(TrialSettings settings) + public TrialResult Run(TrialSettings settings, IServiceProvider provider) { var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); if (_datasetManager is CrossValidateDatasetManager datasetSettings diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs index b157117054..72c175dbdd 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs @@ -157,7 +157,7 @@ public void LoadStatusFromFile(string fileName) public void Update(TrialSettings parameter, TrialResult result) { var schema = parameter.Schema; - var error = CaculateError(result.Metric, result.TrialSettings.ExperimentSettings.EvaluateMetric.IsMaximize); + var error = CaculateError(result.Metric, result.IsMaximize); var duration = result.DurationInMilliseconds / 1000; var pipelineIds = _multiModelPipeline.PipelineIds; var isSuccess = duration != 0; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs index c844299bfd..b7c320b449 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs @@ -24,9 +24,10 @@ public CostFrugalTunerFactory(IServiceProvider provider) public ITuner CreateTuner(TrialSettings settings) { var experimentSetting = _provider.GetService(); + var metricManager = _provider.GetService(); var searchSpace = settings.Pipeline.SearchSpace; var initParameter = settings.Pipeline.Parameter; - var isMaximize = experimentSetting.EvaluateMetric.IsMaximize; + var isMaximize = metricManager.IsMaximize; return new CostFrugalTuner(searchSpace, initParameter, !isMaximize); } From 5907a72b614ac70a80049199937475f46c91c5fe Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 25 Apr 2022 12:39:58 -0700 Subject: [PATCH 05/10] add taxi fare example --- .../Microsoft.ML.AutoML.Samples.csproj | 11 ++-- .../Microsoft.ML.AutoML.Samples/Program.cs | 1 + .../Microsoft.ML.AutoML.Samples/Sonar.cs | 2 +- .../Microsoft.ML.AutoML.Samples/TaxiFare.cs | 53 +++++++++++++++++++ 4 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 docs/samples/Microsoft.ML.AutoML.Samples/TaxiFare.cs diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj index 1c4da82682..1c8f0bb04f 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj @@ -11,12 +11,17 @@ + - + + + + + + + - - diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs b/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs index 3560899e18..a5b5470666 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs @@ -8,6 +8,7 @@ public static void Main(string[] args) { try { + TaxiFare.Run(); Sonar.Run(); RecommendationExperiment.Run(); Console.Clear(); diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Sonar.cs b/docs/samples/Microsoft.ML.AutoML.Samples/Sonar.cs index dc209256b4..b8488f2691 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Sonar.cs +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Sonar.cs @@ -12,7 +12,7 @@ namespace Microsoft.ML.AutoML.Samples { /// - /// Time series forecasting using Sonar dataset. + /// Time series automl forecasting using Sonar dataset. /// public static class Sonar { diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/TaxiFare.cs b/docs/samples/Microsoft.ML.AutoML.Samples/TaxiFare.cs new file mode 100644 index 0000000000..43597b721c --- /dev/null +++ b/docs/samples/Microsoft.ML.AutoML.Samples/TaxiFare.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Data.Analysis; +using System.IO; +using Microsoft.ML; +using Microsoft.ML.AutoML; +using Microsoft.ML.Data; +using static Microsoft.ML.Transforms.OneHotEncodingEstimator.OutputKind; + +namespace Microsoft.ML.AutoML.Samples +{ + /// + /// Regression automl experiment using taxi fare dataset. + /// + public static class TaxiFare + { + public static void Run() + { + //Load File + var trainDataPath = @"C:\Users\xiaoyuz\Desktop\taxi-fare-train.csv"; + var df = DataFrame.LoadCsv(trainDataPath); + var mlContext = new MLContext(); + + // Append the trainer to the data processing pipeline + var pipeline = mlContext.Transforms.Categorical.OneHotEncoding(new[] { new InputOutputColumnPair(@"vendor_id", @"vendor_id"), new InputOutputColumnPair(@"payment_type", @"payment_type") }) + .Append(mlContext.Transforms.ReplaceMissingValues(new[] { new InputOutputColumnPair(@"rate_code", @"rate_code"), new InputOutputColumnPair(@"passenger_count", @"passenger_count"), new InputOutputColumnPair(@"trip_time_in_secs", @"trip_time_in_secs"), new InputOutputColumnPair(@"trip_distance", @"trip_distance") })) + .Append(mlContext.Transforms.Concatenate(@"Features", new[] { @"vendor_id", @"payment_type", @"rate_code", @"passenger_count", @"trip_time_in_secs", @"trip_distance" })) + .Append(mlContext.Auto().Regression(labelColumnName: "fare_amount")); + + // Configure AutoML + var trainTestSplit = mlContext.Data.TrainTestSplit(df, 0.1); + + var experiment = mlContext.Auto().CreateExperiment() + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(50) + .SetDataset(trainTestSplit.TrainSet, trainTestSplit.TestSet) + .SetEvaluateMetric(RegressionMetric.RSquared, "fare_amount", "Score"); + + mlContext.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + Console.WriteLine(e.RawMessage); + } + }; + + // Start Experiment + var res = experiment.Run().Result; + + } + } +} From c8b830a90f721c309aac17c58e0f79b2e217d37b Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 25 Apr 2022 14:01:40 -0700 Subject: [PATCH 06/10] rm sonar.cs --- .../Microsoft.ML.AutoML.Samples/Program.cs | 2 - .../Microsoft.ML.AutoML.Samples/Sonar.cs | 198 ------------------ .../AutoMLExperiment/AutoMLExperiment.cs | 13 +- .../AutoMLExperiment/TunerFactory.cs | 3 +- 4 files changed, 13 insertions(+), 203 deletions(-) delete mode 100644 docs/samples/Microsoft.ML.AutoML.Samples/Sonar.cs diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs b/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs index a5b5470666..4342873c64 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs @@ -8,8 +8,6 @@ public static void Main(string[] args) { try { - TaxiFare.Run(); - Sonar.Run(); RecommendationExperiment.Run(); Console.Clear(); diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Sonar.cs b/docs/samples/Microsoft.ML.AutoML.Samples/Sonar.cs deleted file mode 100644 index b8488f2691..0000000000 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Sonar.cs +++ /dev/null @@ -1,198 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text; -using Microsoft.Data.Analysis; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.ML.Data; -using Microsoft.ML.SearchSpace; -using Microsoft.ML.Transforms.TimeSeries; - -namespace Microsoft.ML.AutoML.Samples -{ - /// - /// Time series automl forecasting using Sonar dataset. - /// - public static class Sonar - { - public static void Run() - { - // Load file - var trainDataPath = @"C:\Users\xiaoyuz\Desktop\forecasting\-0401_load.csv"; - var evaluatePath = @"C:\Users\xiaoyuz\Desktop\forecasting\0401_0415_load.csv"; - var predictedPath = @"C:\Users\xiaoyuz\Desktop\forecasting\predicted.csv"; - var trainDf = DataFrame.LoadCsv(trainDataPath); - var evaluateDf = DataFrame.LoadCsv(evaluatePath); - - var mlContext = new MLContext(); - var searchSpace = new SearchSpace(); - var runner = new CustomRunner(mlContext); - - Console.WriteLine($"train data Length: {trainDf.Rows.Count}"); - var pipeline = mlContext.Transforms.CopyColumns("newLoad", "load") - .Append(mlContext.Auto().CreateSweepableEstimator((context, ss) => - { - return mlContext.Forecasting.ForecastBySsa("predict", "load", ss.WindowSize, ss.SeriesLength, Convert.ToInt32(trainDf.Rows.Count), ss.Horizon, rank: ss.Rank, variableHorizon: true); - }, searchSpace)); - - var autoMLExperiment = mlContext.Auto().CreateExperiment(); - - autoMLExperiment.SetPipeline(pipeline) - .SetTrialRunner(runner) - .SetTrainingTimeInSeconds(600) - .SetEvaluateMetric(RegressionMetric.RootMeanSquaredError) - .SetDataset(trainDf, evaluateDf); - - mlContext.Log += (e, o) => - { - if (o.Source.StartsWith("AutoMLExperiment")) - { - Console.WriteLine(o.RawMessage); - } - }; - - var res = autoMLExperiment.Run().Result; - var bestModel = res.Model; - Console.WriteLine($"best model rmse: {res.Metric}"); - - // evaluate - var predictEngine = bestModel.CreateTimeSeriesEngine(mlContext); - - var predictLoads1H = new List(); - var predictLoads2H = new List(); - predictLoads2H.Add(0); - foreach (var load in evaluateDf.GetColumn("load")) - { - // firstly, get next n predict where n is horizon - var predict = predictEngine.Predict(); - - predictLoads1H.Add(predict.Predict[0]); - predictLoads2H.Add(predict.Predict[1]); - - // update model with truth value - predictEngine.Predict(new ForecastInput() - { - Load = load, - }); - } - - evaluateDf["predict_load_1h"] = DataFrameColumn.Create("predict_load_1h", predictLoads1H); - evaluateDf["predict_load_2h"] = DataFrameColumn.Create("predict_load_2h", predictLoads2H.SkipLast(1)); - DataFrame.WriteCsv(evaluateDf, predictedPath); - } - } - - public class ForecastInput - { - [ColumnName("load")] - public float Load { get; set; } - } - - public class ForecastOutnput - { - [ColumnName("predict")] - public float[] Predict { get; set; } - } - - public class CustomRunner : ITrialRunner - { - private MLContext _context; - - public CustomRunner(MLContext context) - { - this._context = context; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - var datasetManager = provider.GetService(); - if (datasetManager is TrainTestDatasetManager trainTestDatasetManager) - { - try - { - var trainDataset = trainTestDatasetManager.TrainDataset; - var testDataset = trainTestDatasetManager.TestDataset; - - var stopWatch = new Stopwatch(); - stopWatch.Start(); - var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); - var model = pipeline.Fit(trainDataset); - - var predictEngine = model.CreateTimeSeriesEngine(this._context); - - // check point - predictEngine.CheckPoint(this._context, "origin"); - - var predictedLoad1H = new List(); - var predictedLoad2H = new List(); - var N = testDataset.GetRowCount(); - - // evaluate - foreach (var load in testDataset.GetColumn("load")) - { - // firstly, get next n predict where n is horizon - var predict = predictEngine.Predict(); - - predictedLoad1H.Add(predict.Predict[0]); - predictedLoad2H.Add(predict.Predict[1]); - - // update model with truth value - predictEngine.Predict(new ForecastInput() - { - Load = load, - }); - } - - var rmse1H = Enumerable.Zip(testDataset.GetColumn("load"), predictedLoad1H) - .Select(x => Math.Pow(x.First - x.Second, 2)) - .Average(); - rmse1H = Math.Sqrt(rmse1H); - - var rmse2H = Enumerable.Zip(testDataset.GetColumn("load").Skip(1), predictedLoad2H) - .Select(x => Math.Pow(x.First - x.Second, 2)) - .Average(); - rmse2H = Math.Sqrt(rmse2H); - - stopWatch.Stop(); - var rmse = (rmse1H + rmse2H) / 2; - - return new TrialResult() - { - Metric = rmse, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - }; - - } - catch (Exception) - { - return new TrialResult() - { - Metric = double.MaxValue, - Model = null, - TrialSettings = settings, - DurationInMilliseconds = 0, - }; - } - } - - throw new ArgumentException(); - } - } - public class ForecastBySsaSearchSpace - { - [Range(2, 24 * 7 * 30)] - public int WindowSize { get; set; } = 2; - - [Range(2, 24 * 7 * 30)] - public int SeriesLength { get; set; } = 2; - - [Range(1, 24 * 7 * 30)] - public int Rank { get; set; } = 1; - - [Range(2, 50)] - public int Horizon { get; set; } = 2; - } -} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 119fd960b6..014942d0d6 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -120,6 +120,12 @@ public AutoMLExperiment SetPipeline(MultiModelPipeline pipeline) return this; } + public AutoMLExperiment SetIsMaximizeMetric(bool isMaximize) + { + _settings.IsMaximizeMetric = isMaximize; + return this; + } + public AutoMLExperiment SetTrialRunner(ITrialRunner runner) { var factory = new CustomRunnerFactory(runner); @@ -158,6 +164,7 @@ public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, str LabelColumn = labelColumn, }; _serviceCollection.AddSingleton(metricManager); + this.SetIsMaximizeMetric(metricManager.IsMaximize); return this; } @@ -171,6 +178,7 @@ public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, LabelColumn = labelColumn, }; _serviceCollection.AddSingleton(metricManager); + this.SetIsMaximizeMetric(metricManager.IsMaximize); return this; } @@ -184,6 +192,7 @@ public AutoMLExperiment SetEvaluateMetric(RegressionMetric metric, string labelC LabelColumn = labelColumn, }; _serviceCollection.AddSingleton(metricManager); + this.SetIsMaximizeMetric(metricManager.IsMaximize); return this; } @@ -238,7 +247,7 @@ private async Task RunAsync(CancellationToken ct) hyperParameterProposer.Update(setting, trialResult); pipelineProposer.Update(setting, trialResult); - var error = trialResult.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; + var error = _settings.IsMaximizeMetric ? 1 - trialResult.Metric : trialResult.Metric; if (error < _bestError) { _bestTrialResult = trialResult; @@ -280,6 +289,8 @@ public class AutoMLExperimentSettings : ExperimentSettings public MultiModelPipeline Pipeline { get; set; } public int? Seed { get; set; } + + public bool IsMaximizeMetric { get; set; } } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs index b7c320b449..ec58c6e144 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs @@ -24,10 +24,9 @@ public CostFrugalTunerFactory(IServiceProvider provider) public ITuner CreateTuner(TrialSettings settings) { var experimentSetting = _provider.GetService(); - var metricManager = _provider.GetService(); var searchSpace = settings.Pipeline.SearchSpace; var initParameter = settings.Pipeline.Parameter; - var isMaximize = metricManager.IsMaximize; + var isMaximize = experimentSetting.IsMaximizeMetric; return new CostFrugalTuner(searchSpace, initParameter, !isMaximize); } From d99a7dd92ee83c29e4988e4adadab93886b33c7b Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 25 Apr 2022 14:01:58 -0700 Subject: [PATCH 07/10] rm taxi fare --- .../Microsoft.ML.AutoML.Samples/TaxiFare.cs | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 docs/samples/Microsoft.ML.AutoML.Samples/TaxiFare.cs diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/TaxiFare.cs b/docs/samples/Microsoft.ML.AutoML.Samples/TaxiFare.cs deleted file mode 100644 index 43597b721c..0000000000 --- a/docs/samples/Microsoft.ML.AutoML.Samples/TaxiFare.cs +++ /dev/null @@ -1,53 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; -using Microsoft.Data.Analysis; -using System.IO; -using Microsoft.ML; -using Microsoft.ML.AutoML; -using Microsoft.ML.Data; -using static Microsoft.ML.Transforms.OneHotEncodingEstimator.OutputKind; - -namespace Microsoft.ML.AutoML.Samples -{ - /// - /// Regression automl experiment using taxi fare dataset. - /// - public static class TaxiFare - { - public static void Run() - { - //Load File - var trainDataPath = @"C:\Users\xiaoyuz\Desktop\taxi-fare-train.csv"; - var df = DataFrame.LoadCsv(trainDataPath); - var mlContext = new MLContext(); - - // Append the trainer to the data processing pipeline - var pipeline = mlContext.Transforms.Categorical.OneHotEncoding(new[] { new InputOutputColumnPair(@"vendor_id", @"vendor_id"), new InputOutputColumnPair(@"payment_type", @"payment_type") }) - .Append(mlContext.Transforms.ReplaceMissingValues(new[] { new InputOutputColumnPair(@"rate_code", @"rate_code"), new InputOutputColumnPair(@"passenger_count", @"passenger_count"), new InputOutputColumnPair(@"trip_time_in_secs", @"trip_time_in_secs"), new InputOutputColumnPair(@"trip_distance", @"trip_distance") })) - .Append(mlContext.Transforms.Concatenate(@"Features", new[] { @"vendor_id", @"payment_type", @"rate_code", @"passenger_count", @"trip_time_in_secs", @"trip_distance" })) - .Append(mlContext.Auto().Regression(labelColumnName: "fare_amount")); - - // Configure AutoML - var trainTestSplit = mlContext.Data.TrainTestSplit(df, 0.1); - - var experiment = mlContext.Auto().CreateExperiment() - .SetPipeline(pipeline) - .SetTrainingTimeInSeconds(50) - .SetDataset(trainTestSplit.TrainSet, trainTestSplit.TestSet) - .SetEvaluateMetric(RegressionMetric.RSquared, "fare_amount", "Score"); - - mlContext.Log += (o, e) => - { - if (e.Source.StartsWith("AutoMLExperiment")) - { - Console.WriteLine(e.RawMessage); - } - }; - - // Start Experiment - var res = experiment.Run().Result; - - } - } -} From 03cde7875d2419d13d724077d6f497e5e4ea14e2 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 25 Apr 2022 14:06:09 -0700 Subject: [PATCH 08/10] checkout samples.csproj --- .../Microsoft.ML.AutoML.Samples.csproj | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj index 1c8f0bb04f..9d511df6a5 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj @@ -10,18 +10,12 @@ - - - - - - - - - + + + From 603d490b01b636d17a5e56b2ee51260c28fadf93 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 26 Apr 2022 13:09:53 -0700 Subject: [PATCH 09/10] remove .this --- .../AutoMLExperiment/AutoMLExperiment.cs | 6 +++--- .../AutoMLExperiment/TrialRunnerFactory.cs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 014942d0d6..2120c66572 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -164,7 +164,7 @@ public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, str LabelColumn = labelColumn, }; _serviceCollection.AddSingleton(metricManager); - this.SetIsMaximizeMetric(metricManager.IsMaximize); + SetIsMaximizeMetric(metricManager.IsMaximize); return this; } @@ -178,7 +178,7 @@ public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, LabelColumn = labelColumn, }; _serviceCollection.AddSingleton(metricManager); - this.SetIsMaximizeMetric(metricManager.IsMaximize); + SetIsMaximizeMetric(metricManager.IsMaximize); return this; } @@ -192,7 +192,7 @@ public AutoMLExperiment SetEvaluateMetric(RegressionMetric metric, string labelC LabelColumn = labelColumn, }; _serviceCollection.AddSingleton(metricManager); - this.SetIsMaximizeMetric(metricManager.IsMaximize); + SetIsMaximizeMetric(metricManager.IsMaximize); return this; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs index ca9401d10c..5d8a72fc3c 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs @@ -39,8 +39,8 @@ public TrialRunnerFactory(IServiceProvider provider) public ITrialRunner? CreateTrialRunner() { - var datasetManager = this._provider.GetService(); - var metricManager = this._provider.GetService(); + var datasetManager = _provider.GetService(); + var metricManager = _provider.GetService(); ITrialRunner? runner = (datasetManager, metricManager) switch { From d4c38cd0a1a29761abf89dc92493f11385f4ded8 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 26 Apr 2022 13:20:43 -0700 Subject: [PATCH 10/10] add comments to public interface --- src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs | 4 ++++ src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs | 3 +++ src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs | 3 +++ src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs | 3 +++ .../AutoMLExperiment/TrialRunnerFactory.cs | 3 +++ src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs | 3 +++ 6 files changed, 19 insertions(+) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs index 6180807ec1..ac057be17d 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs @@ -4,6 +4,10 @@ namespace Microsoft.ML.AutoML { + /// + /// Interface for dataset manager. This interface doesn't include any method or property definition and is used by and other components to retrieve the instance of the actual + /// dataset manager from containers. + /// public interface IDatasetManager { } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs index 9e060b22bc..cee384f7b9 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs @@ -6,6 +6,9 @@ namespace Microsoft.ML.AutoML { + /// + /// Interface for metric manager. + /// internal interface IMetricManager { bool IsMaximize { get; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index 14a89b29cd..1d17908370 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -8,6 +8,9 @@ namespace Microsoft.ML.AutoML { + /// + /// instance for monitor, which is used by to report training progress. + /// public interface IMonitor { void ReportCompletedTrial(TrialResult result); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs index b1d3816055..3bac432dc7 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs @@ -7,6 +7,9 @@ namespace Microsoft.ML.AutoML { + /// + /// interface for all trial runners. + /// public interface ITrialRunner { TrialResult Run(TrialSettings settings, IServiceProvider provider = null); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs index 5d8a72fc3c..8bba47d321 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs @@ -8,6 +8,9 @@ #nullable enable namespace Microsoft.ML.AutoML { + /// + /// interface for trial runner factory. + /// public interface ITrialRunnerFactory { ITrialRunner? CreateTrialRunner(); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs index ec58c6e144..6acc5f37e6 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs @@ -7,6 +7,9 @@ namespace Microsoft.ML.AutoML { + /// + /// interface for all tuner factories. + /// public interface ITunerFactory { ITuner CreateTuner(TrialSettings settings);